diff --git a/.devops/cpu.Dockerfile b/.devops/cpu.Dockerfile index 0eba8f008e46..0809c8b7ec90 100644 --- a/.devops/cpu.Dockerfile +++ b/.devops/cpu.Dockerfile @@ -1,4 +1,4 @@ -ARG UBUNTU_VERSION=24.04 +ARG UBUNTU_VERSION=26.04 ARG BUILD_DATE=N/A ARG APP_VERSION=N/A ARG APP_REVISION=N/A @@ -22,9 +22,7 @@ FROM docker.io/ubuntu:$UBUNTU_VERSION AS build ARG TARGETARCH RUN apt-get update && \ - apt-get install -y gcc-14 g++-14 build-essential git cmake libssl-dev - -ENV CC=gcc-14 CXX=g++-14 + apt-get install -y build-essential git cmake libssl-dev WORKDIR /app diff --git a/.devops/cuda.Dockerfile b/.devops/cuda.Dockerfile index 9a2d88bc02b3..2c872ab4adeb 100644 --- a/.devops/cuda.Dockerfile +++ b/.devops/cuda.Dockerfile @@ -84,6 +84,9 @@ RUN apt-get update \ && find /var/cache/apt/archives /var/lib/apt/lists -not -name lock -type f -delete \ && find /var/cache -type f -delete +# Some hosts (e.g. GKE) only mount the driver here, and CUDA 12.8.1-12.9.1 base images don't set these paths +ENV LD_LIBRARY_PATH=/usr/local/nvidia/lib64:/usr/local/nvidia/lib:${LD_LIBRARY_PATH} + COPY --from=build /app/lib/ /app ### Full diff --git a/.devops/intel.Dockerfile b/.devops/intel.Dockerfile index 09daa5fe5d1a..9f4a8fc992e5 100644 --- a/.devops/intel.Dockerfile +++ b/.devops/intel.Dockerfile @@ -1,4 +1,4 @@ -ARG ONEAPI_VERSION=2026.1.1-devel-ubuntu24.04 +ARG ONEAPI_VERSION=2026.2.0-devel-ubuntu24.04 ARG BUILD_DATE=N/A ARG APP_VERSION=N/A ARG APP_REVISION=N/A @@ -22,20 +22,20 @@ RUN LLAMA_BUILD_NUMBER="$APP_VERSION" npm run build FROM docker.io/intel/deep-learning-essentials:$ONEAPI_VERSION AS build RUN echo "Download oneDNN installation package" && \ - wget https://registrationcenter-download.intel.com/akdlm/IRC_NAS/94c3dbac-0852-45be-a57d-21c204cada3e/intel-onednn-2026.0.2.46_offline.sh -O intel-onednn_offline.sh && \ + wget https://registrationcenter-download.intel.com/akdlm/IRC_NAS/46452a4a-7b7f-49f2-be4c-602a900180bb/intel-onednn-2026.1.0.625_offline.sh -O intel-onednn_offline.sh && \ echo "Install oneDNN components" && \ bash intel-onednn_offline.sh -s -a --silent --eula accept ARG GGML_SYCL_F16=ON -ARG LEVEL_ZERO_VERSION=1.28.2 +ARG LEVEL_ZERO_VERSION=1.34.0 ARG LEVEL_ZERO_UBUNTU_VERSION=u24.04 RUN apt-get update && \ apt-get install -y git libssl-dev wget ca-certificates && \ cd /tmp && \ - wget -q "https://github.com/oneapi-src/level-zero/releases/download/v${LEVEL_ZERO_VERSION}/level-zero_${LEVEL_ZERO_VERSION}%2B${LEVEL_ZERO_UBUNTU_VERSION}_amd64.deb" -O level-zero.deb && \ - wget -q "https://github.com/oneapi-src/level-zero/releases/download/v${LEVEL_ZERO_VERSION}/level-zero-devel_${LEVEL_ZERO_VERSION}%2B${LEVEL_ZERO_UBUNTU_VERSION}_amd64.deb" -O level-zero-devel.deb && \ - apt-get -o Dpkg::Options::="--force-overwrite" install -y ./level-zero.deb ./level-zero-devel.deb && \ - rm -f /tmp/level-zero.deb /tmp/level-zero-devel.deb + wget -q "https://github.com/oneapi-src/level-zero/releases/download/v${LEVEL_ZERO_VERSION}/libze1_${LEVEL_ZERO_VERSION}%2B${LEVEL_ZERO_UBUNTU_VERSION}_amd64.deb" -O libze1.deb && \ + wget -q "https://github.com/oneapi-src/level-zero/releases/download/v${LEVEL_ZERO_VERSION}/libze-dev_${LEVEL_ZERO_VERSION}%2B${LEVEL_ZERO_UBUNTU_VERSION}_amd64.deb" -O libze-dev.deb && \ + apt-get -o Dpkg::Options::="--force-overwrite" install -y ./libze1.deb ./libze-dev.deb && \ + rm -f /tmp/libze1.deb /tmp/libze-dev.deb WORKDIR /app @@ -64,7 +64,7 @@ RUN mkdir -p /app/full \ && cp requirements.txt /app/full \ && cp .devops/tools.sh /app/full/tools.sh -FROM docker.io/intel/oneapi-toolkit:$ONEAPI_VERSION AS base +FROM docker.io/intel/deep-learning-essentials:$ONEAPI_VERSION AS base ARG BUILD_DATE=N/A ARG APP_VERSION=N/A @@ -79,6 +79,8 @@ LABEL org.opencontainers.image.created=$BUILD_DATE \ org.opencontainers.image.url=$IMAGE_URL \ org.opencontainers.image.source=$IMAGE_SOURCE +COPY --from=build /opt/intel/oneapi/dnnl /opt/intel/oneapi/dnnl + #Following versions are for multiple GPUs, since 26.x has known issue: # https://github.com/ggml-org/llama.cpp/issues/21747, # https://github.com/intel/compute-runtime/issues/921. @@ -89,11 +91,11 @@ LABEL org.opencontainers.image.created=$BUILD_DATE \ #ARG IGDGMM_VERSION=22.8.2 -ARG IGC_VERSION=v2.34.4 -ARG IGC_VERSION_FULL=2_2.34.4+21428 -ARG COMPUTE_RUNTIME_VERSION=26.18.38308.1 -ARG COMPUTE_RUNTIME_VERSION_FULL=26.18.38308.1-0 -ARG IGDGMM_VERSION=22.10.0 +ARG IGC_VERSION=v2.42.6 +ARG IGC_VERSION_FULL=2_2.42.6+23070 +ARG COMPUTE_RUNTIME_VERSION=26.40.40196.8 +ARG COMPUTE_RUNTIME_VERSION_FULL=26.40.40196.8-0 +ARG IGDGMM_VERSION=22.10.2 RUN mkdir /tmp/neo/ && cd /tmp/neo/ \ && wget https://github.com/intel/intel-graphics-compiler/releases/download/$IGC_VERSION/intel-igc-core-${IGC_VERSION_FULL}_amd64.deb \ && wget https://github.com/intel/intel-graphics-compiler/releases/download/$IGC_VERSION/intel-igc-opencl-${IGC_VERSION_FULL}_amd64.deb \ diff --git a/.devops/vulkan.Dockerfile b/.devops/vulkan.Dockerfile index 90e8693cf04c..96b7e66ec0e0 100644 --- a/.devops/vulkan.Dockerfile +++ b/.devops/vulkan.Dockerfile @@ -82,20 +82,14 @@ COPY --from=build /app/full /app WORKDIR /app -ENV PATH="/root/.venv/bin:/root/.local/bin:${PATH}" - -# Flag for compatibility with pip -ARG UV_INDEX_STRATEGY="unsafe-best-match" RUN apt-get update \ && apt-get install -y \ - build-essential \ - curl \ git \ - ca-certificates \ - && curl -LsSf https://astral.sh/uv/install.sh | sh \ - && uv python install 3.13 \ - && uv venv --python 3.13 /root/.venv \ - && uv pip install --python /root/.venv/bin/python -r requirements.txt \ + python3 \ + python3-pip \ + python3-wheel \ + && pip install --break-system-packages --upgrade setuptools \ + && pip install --break-system-packages -r requirements.txt \ && apt autoremove -y \ && apt clean -y \ && rm -rf /tmp/* /var/tmp/* \ diff --git a/.github/actions/windows-setup-cuda/action.yml b/.github/actions/windows-setup-cuda/action.yml index e67b6321e95a..51f1147e18b8 100644 --- a/.github/actions/windows-setup-cuda/action.yml +++ b/.github/actions/windows-setup-cuda/action.yml @@ -100,56 +100,56 @@ runs: echo "CUDA_PATH=C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.1" | Out-File -FilePath $env:GITHUB_ENV -Append -Encoding utf8 echo "CUDA_PATH_V13_1=C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.1" | Out-File -FilePath $env:GITHUB_ENV -Append -Encoding utf8 - - name: Install Cuda Toolkit 13.4 for x64 - if: ${{ inputs.cuda_version == '13.4' && inputs.cuda_arch == 'x64' }} + - name: Install Cuda Toolkit 13.5 for x64 + if: ${{ inputs.cuda_version == '13.5' && inputs.cuda_arch == 'x64' }} shell: pwsh run: | - mkdir -p "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" + mkdir -p "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5" choco install unzip -y - curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_crt/windows-x86_64/cuda_crt-windows-x86_64-13.4.59-archive.zip" - curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_cudart/windows-x86_64/cuda_cudart-windows-x86_64-13.4.49-archive.zip" - curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_nvcc/windows-x86_64/cuda_nvcc-windows-x86_64-13.4.59-archive.zip" - curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_nvrtc/windows-x86_64/cuda_nvrtc-windows-x86_64-13.4.59-archive.zip" - curl -O "https://developer.download.nvidia.com/compute/cuda/redist/libcublas/windows-x86_64/libcublas-windows-x86_64-13.7.0.27-archive.zip" - curl -O "https://developer.download.nvidia.com/compute/cuda/redist/libnvvm/windows-x86_64/libnvvm-windows-x86_64-13.4.59-archive.zip" - curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_nvtx/windows-x86_64/cuda_nvtx-windows-x86_64-13.4.49-archive.zip" - curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_profiler_api/windows-x86_64/cuda_profiler_api-windows-x86_64-13.4.49-archive.zip" - curl -O "https://developer.download.nvidia.com/compute/cuda/redist/visual_studio_integration/windows-x86_64/visual_studio_integration-windows-x86_64-13.4.49-archive.zip" - curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cccl/windows-x86_64/cccl-windows-x86_64-13.3.4.2.1-archive.zip" - unzip '*.zip' -d "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" - xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4\cuda_crt-windows-x86_64-13.4.59-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" /E /I /H /Y - xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4\cuda_cudart-windows-x86_64-13.4.49-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" /E /I /H /Y - xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4\cuda_nvcc-windows-x86_64-13.4.59-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" /E /I /H /Y - xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4\cuda_nvrtc-windows-x86_64-13.4.59-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" /E /I /H /Y - xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4\libcublas-windows-x86_64-13.7.0.27-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" /E /I /H /Y - xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4\libnvvm-windows-x86_64-13.4.59-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" /E /I /H /Y - xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4\cuda_nvtx-windows-x86_64-13.4.49-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" /E /I /H /Y - xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4\cuda_profiler_api-windows-x86_64-13.4.49-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" /E /I /H /Y - xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4\visual_studio_integration-windows-x86_64-13.4.49-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" /E /I /H /Y - xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4\cccl-windows-x86_64-13.3.4.2.1-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" /E /I /H /Y - echo "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4\bin" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append - echo "CUDA_PATH=C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" | Out-File -FilePath $env:GITHUB_ENV -Append -Encoding utf8 - echo "CUDA_PATH_V13_4=C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" | Out-File -FilePath $env:GITHUB_ENV -Append -Encoding utf8 + curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_crt/windows-x86_64/cuda_crt-windows-x86_64-13.5.37-archive.zip" + curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_cudart/windows-x86_64/cuda_cudart-windows-x86_64-13.5.37-archive.zip" + curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_nvcc/windows-x86_64/cuda_nvcc-windows-x86_64-13.5.37-archive.zip" + curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_nvrtc/windows-x86_64/cuda_nvrtc-windows-x86_64-13.5.37-archive.zip" + curl -O "https://developer.download.nvidia.com/compute/cuda/redist/libcublas/windows-x86_64/libcublas-windows-x86_64-13.10.0.20-archive.zip" + curl -O "https://developer.download.nvidia.com/compute/cuda/redist/libnvvm/windows-x86_64/libnvvm-windows-x86_64-13.5.37-archive.zip" + curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_nvtx/windows-x86_64/cuda_nvtx-windows-x86_64-13.5.25-archive.zip" + curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_profiler_api/windows-x86_64/cuda_profiler_api-windows-x86_64-13.5.25-archive.zip" + curl -O "https://developer.download.nvidia.com/compute/cuda/redist/visual_studio_integration/windows-x86_64/visual_studio_integration-windows-x86_64-13.5.25-archive.zip" + curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cccl/windows-x86_64/cccl-windows-x86_64-13.3.5.0.6-archive.zip" + unzip '*.zip' -d "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5" + xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5\cuda_crt-windows-x86_64-13.5.37-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5" /E /I /H /Y + xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5\cuda_cudart-windows-x86_64-13.5.37-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5" /E /I /H /Y + xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5\cuda_nvcc-windows-x86_64-13.5.37-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5" /E /I /H /Y + xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5\cuda_nvrtc-windows-x86_64-13.5.37-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5" /E /I /H /Y + xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5\libcublas-windows-x86_64-13.10.0.20-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5" /E /I /H /Y + xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5\libnvvm-windows-x86_64-13.5.37-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5" /E /I /H /Y + xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5\cuda_nvtx-windows-x86_64-13.5.25-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5" /E /I /H /Y + xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5\cuda_profiler_api-windows-x86_64-13.5.25-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5" /E /I /H /Y + xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5\visual_studio_integration-windows-x86_64-13.5.25-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5" /E /I /H /Y + xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5\cccl-windows-x86_64-13.3.5.0.6-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5" /E /I /H /Y + echo "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5\bin" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append + echo "CUDA_PATH=C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5" | Out-File -FilePath $env:GITHUB_ENV -Append -Encoding utf8 + echo "CUDA_PATH_V13_5=C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5" | Out-File -FilePath $env:GITHUB_ENV -Append -Encoding utf8 - - name: Install Cuda Toolkit 13.4 for ARM64 - if: ${{ inputs.cuda_version == '13.4' && inputs.cuda_arch == 'arm64' }} + - name: Install Cuda Toolkit 13.5 for ARM64 + if: ${{ inputs.cuda_version == '13.5' && inputs.cuda_arch == 'arm64' }} shell: pwsh run: | - mkdir -p "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" + mkdir -p "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5" choco install unzip -y - curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cccl/windows-x86_64/cccl-windows-x86_64-13.3.4.2.1-archive.zip" - curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_crt/windows-x86_64/cuda_crt-windows-x86_64-13.4.59-archive.zip" - curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_nvcc/windows-x86_64/cuda_nvcc-windows-x86_64-13.4.59-archive.zip" - curl -O "https://developer.download.nvidia.com/compute/cuda/redist/libnvvm/windows-x86_64/libnvvm-windows-x86_64-13.4.59-archive.zip" - curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_cudart/windows-arm64/cuda_cudart-windows-arm64-13.4.49-archive.zip" - curl -O "https://developer.download.nvidia.com/compute/cuda/redist/libcublas/windows-arm64/libcublas-windows-arm64-13.7.0.27-archive.zip" - unzip '*.zip' -d "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" - xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4\cccl-windows-x86_64-13.3.4.2.1-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" /E /I /H /Y - xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4\cuda_crt-windows-x86_64-13.4.59-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" /E /I /H /Y - xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4\cuda_nvcc-windows-x86_64-13.4.59-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" /E /I /H /Y - xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4\libnvvm-windows-x86_64-13.4.59-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" /E /I /H /Y - xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4\cuda_cudart-windows-arm64-13.4.49-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" /E /I /H /Y - xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4\libcublas-windows-arm64-13.7.0.27-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" /E /I /H /Y - echo "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4\bin" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append - echo "CUDA_PATH=C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" | Out-File -FilePath $env:GITHUB_ENV -Append -Encoding utf8 - echo "CUDA_PATH_V13_4=C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" | Out-File -FilePath $env:GITHUB_ENV -Append -Encoding utf8 + curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cccl/windows-x86_64/cccl-windows-x86_64-13.3.5.0.6-archive.zip" + curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_crt/windows-x86_64/cuda_crt-windows-x86_64-13.5.37-archive.zip" + curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_nvcc/windows-x86_64/cuda_nvcc-windows-x86_64-13.5.37-archive.zip" + curl -O "https://developer.download.nvidia.com/compute/cuda/redist/libnvvm/windows-x86_64/libnvvm-windows-x86_64-13.5.37-archive.zip" + curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_cudart/windows-arm64/cuda_cudart-windows-arm64-13.5.37-archive.zip" + curl -O "https://developer.download.nvidia.com/compute/cuda/redist/libcublas/windows-arm64/libcublas-windows-arm64-13.10.0.20-archive.zip" + unzip '*.zip' -d "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5" + xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5\cccl-windows-x86_64-13.3.5.0.6-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5" /E /I /H /Y + xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5\cuda_crt-windows-x86_64-13.5.37-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5" /E /I /H /Y + xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5\cuda_nvcc-windows-x86_64-13.5.37-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5" /E /I /H /Y + xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5\libnvvm-windows-x86_64-13.5.37-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5" /E /I /H /Y + xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5\cuda_cudart-windows-arm64-13.5.37-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5" /E /I /H /Y + xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5\libcublas-windows-arm64-13.10.0.20-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5" /E /I /H /Y + echo "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5\bin" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append + echo "CUDA_PATH=C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5" | Out-File -FilePath $env:GITHUB_ENV -Append -Encoding utf8 + echo "CUDA_PATH_V13_5=C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5" | Out-File -FilePath $env:GITHUB_ENV -Append -Encoding utf8 diff --git a/.github/workflows/build-3rd-party.yml b/.github/workflows/build-3rd-party.yml index fe25f3909957..0b9e52647add 100644 --- a/.github/workflows/build-3rd-party.yml +++ b/.github/workflows/build-3rd-party.yml @@ -31,8 +31,8 @@ env: LLAMA_ARG_LOG_TIMESTAMPS: 1 jobs: - ubuntu-24-llguidance: - runs-on: ${{ 'ubuntu-24.04-arm' || 'ubuntu-24.04' }} + ubuntu-26-llguidance: + runs-on: ${{ 'ubuntu-26.04-arm' || 'ubuntu-26.04' }} steps: - name: Clone diff --git a/.github/workflows/build-and-test-snapdragon.yml b/.github/workflows/build-and-test-snapdragon.yml index f7f94d7ee996..a95f64acdb22 100644 --- a/.github/workflows/build-and-test-snapdragon.yml +++ b/.github/workflows/build-and-test-snapdragon.yml @@ -33,7 +33,7 @@ concurrency: jobs: android-ndk-snapdragon: - runs-on: ubuntu-24.04 # previously ubuntu-latest + runs-on: ubuntu-26.04 container: image: 'ghcr.io/snapdragon-toolchain/arm64-android:v0.8' defaults: @@ -67,7 +67,7 @@ jobs: path: pkg-snapdragon/llama.cpp linux-iot-snapdragon: - runs-on: ubuntu-24.04 # previously ubuntu-latest + runs-on: ubuntu-26.04 container: image: 'ghcr.io/snapdragon-toolchain/arm64-linux:v0.8' defaults: @@ -103,7 +103,7 @@ jobs: test-snapdragon-qdc: name: Test on QDC Device (${{ matrix.device }}) needs: [android-ndk-snapdragon, linux-iot-snapdragon] - runs-on: ubuntu-24.04-arm + runs-on: ubuntu-26.04-arm timeout-minutes: 180 strategy: fail-fast: false diff --git a/.github/workflows/build-android.yml b/.github/workflows/build-android.yml index 4db7c2269044..bbe23cf3f497 100644 --- a/.github/workflows/build-android.yml +++ b/.github/workflows/build-android.yml @@ -37,7 +37,7 @@ env: jobs: default: - runs-on: ubuntu-24.04 # previously ubuntu-latest + runs-on: ubuntu-26.04 steps: - name: Clone @@ -56,6 +56,7 @@ jobs: uses: android-actions/setup-android@be39fa834029ff78f1a44aa3bb0819b8fc2bd8fd # v4.0.4 with: log-accepted-android-sdk-licenses: false + packages: platform-tools cmake;3.31.6 - name: Build run: | @@ -63,7 +64,7 @@ jobs: ./gradlew build --no-daemon ndk: - runs-on: ubuntu-24.04 # previously ubuntu-latest + runs-on: ubuntu-26.04 container: image: 'ghcr.io/snapdragon-toolchain/arm64-android:v0.3' defaults: @@ -101,7 +102,7 @@ jobs: path: pkg-adb/llama.cpp arm64: - runs-on: ubuntu-24.04 # previously ubuntu-latest + runs-on: ubuntu-26.04 env: NDK_VERSION: "29.0.14206865" diff --git a/.github/workflows/build-cache.yml b/.github/workflows/build-cache.yml index 72442ae95863..6de8138b2c82 100644 --- a/.github/workflows/build-cache.yml +++ b/.github/workflows/build-cache.yml @@ -40,8 +40,8 @@ jobs: # path: ./spacemit_toolchain # version: ${{ env.SPACEMIT_IME_TOOLCHAIN_VERSION }} - ubuntu-24-openvino-cache: - runs-on: ubuntu-24.04 + ubuntu-26-openvino-cache: + runs-on: ubuntu-26.04 env: # Sync versions in build-openvino.yml, ci-self-hosted-openvino.yml, release.yml, release-publish.yml, build-cache.yml, .devops/openvino.Dockerfile, docs/backend/OPENVINO.md diff --git a/.github/workflows/build-cmake-pkg.yml b/.github/workflows/build-cmake-pkg.yml deleted file mode 100644 index 4e351ecb77db..000000000000 --- a/.github/workflows/build-cmake-pkg.yml +++ /dev/null @@ -1,57 +0,0 @@ -name: Build relocatable cmake package -on: - workflow_dispatch: - workflow_call: - -cache-mode: none -permissions: - contents: read - -jobs: - linux: - runs-on: [self-hosted, Linux, fast] - steps: - - uses: actions/checkout@v6 - with: - fetch-depth: 0 - - - name: Build - run: | - PREFIX="$(pwd)"/inst - cmake -S . -B build \ - -DCMAKE_PREFIX_PATH="$PREFIX" \ - -DLLAMA_OPENSSL=OFF \ - -DLLAMA_BUILD_TESTS=OFF \ - -DLLAMA_BUILD_TOOLS=OFF \ - -DLLAMA_BUILD_EXAMPLES=OFF \ - -DLLAMA_BUILD_APP=OFF \ - -DLLAMA_BUILD_IS_DEV=OFF \ - -DCMAKE_BUILD_TYPE=Release - cmake --build build --config Release -j $(nproc) - cmake --install build --prefix "$PREFIX" --config Release - - export LLAMA_CONFIG="$PREFIX"/lib/cmake/llama/llama-config.cmake - build_commit=$(git rev-parse --short HEAD | xargs) - build_number=$(git rev-list --count HEAD | xargs) - - major=$(grep -oE "set\(LLAMA_VERSION_MAJOR[[:space:]]+[0-9]+" CMakeLists.txt | grep -oE "[0-9]+$") - minor=$(grep -oE "set\(LLAMA_VERSION_MINOR[[:space:]]+[0-9]+" CMakeLists.txt | grep -oE "[0-9]+$") - patch=$(grep -oE "set\(LLAMA_VERSION_PATCH[[:space:]]+[0-9]+" CMakeLists.txt | grep -oE "[0-9]+$") - build_version="$major.$minor.$patch" - - checks=("set\(LLAMA_VERSION[[:space:]]+$build_version\)" - "set\(LLAMA_BUILD_COMMIT[[:space:]]+$build_commit\)" - "set\(LLAMA_BUILD_NUMBER[[:space:]]+$build_number\)") - - for check in "${checks[@]}"; do - if ! grep -qE "$check" "$LLAMA_CONFIG"; then - echo "Checking llama-config.cmake version... \"$check\" failed!" - exit 1 - fi - done - - echo "Checking llama-config.cmake version... success." - - cd examples/simple-cmake-pkg - cmake -S . -B build -DCMAKE_PREFIX_PATH="$PREFIX"/lib/cmake - cmake --build build -j $(nproc) diff --git a/.github/workflows/build-cpu.yml b/.github/workflows/build-cpu.yml index 76a030e67d35..35d610e1e9fc 100644 --- a/.github/workflows/build-cpu.yml +++ b/.github/workflows/build-cpu.yml @@ -7,7 +7,6 @@ on: - master paths: [ '.github/workflows/build-cpu.yml', - '.github/workflows/build-cmake-pkg.yml', '**/CMakeLists.txt', '**/.cmake', '**/*.h', @@ -20,7 +19,6 @@ on: types: [opened, synchronize, reopened] paths: [ '.github/workflows/build-cpu.yml', - '.github/workflows/build-cmake-pkg.yml', 'ggml/src/ggml-rpc/**', '**/CMakeLists.txt', '**/.cmake', @@ -47,17 +45,14 @@ env: LLAMA_ARG_LOG_TIMESTAMPS: 1 jobs: - build-cmake-pkg: - uses: ./.github/workflows/build-cmake-pkg.yml - ubuntu: strategy: matrix: include: - build: 'x64' - os: ubuntu-22.04 + os: ubuntu-26.04 - build: 'arm64' - os: ubuntu-24.04-arm + os: ubuntu-26.04-arm runs-on: ${{ matrix.os }} @@ -81,13 +76,6 @@ jobs: libjpeg-dev build-essential libssl-dev \ git-lfs - - name: Toolchain workaround (GCC 14) - if: ${{ contains(matrix.os, 'ubuntu-24.04') }} - run: | - sudo apt-get install -y gcc-14 g++-14 - echo "CC=gcc-14" >> "$GITHUB_ENV" - echo "CXX=g++-14" >> "$GITHUB_ENV" - - name: Python Dependencies id: python_depends run: | @@ -242,3 +230,79 @@ jobs: older: 5m min: 1 dry-run: ${{ github.event_name != 'push' || github.ref != 'refs/heads/master' }} + + build-cmake-pkg: + runs-on: ubuntu-24.04 + steps: + - name: Clone + uses: actions/checkout@v6 + with: + fetch-depth: 0 + + - name: ccache + uses: ggml-org/ccache-action@v1.2.24 + with: + restore: false + save: false + + - name: ccache-buckets-restore + uses: ./.github/actions/ccache-buckets + env: + HF_TOKEN: ${{ secrets.HF_TOKEN_CI }} + with: + key: build-cmake-pkg + folder: llama.cpp + hf_bucket: ggml-org/cache + + - name: Build + run: | + PREFIX="$(pwd)"/inst + cmake -S . -B build \ + -DCMAKE_PREFIX_PATH="$PREFIX" \ + -DLLAMA_OPENSSL=OFF \ + -DLLAMA_BUILD_TESTS=OFF \ + -DLLAMA_BUILD_TOOLS=OFF \ + -DLLAMA_BUILD_EXAMPLES=OFF \ + -DLLAMA_BUILD_APP=OFF \ + -DLLAMA_BUILD_IS_DEV=OFF \ + -DCMAKE_BUILD_TYPE=Release + cmake --build build --config Release -j $(nproc) + cmake --install build --prefix "$PREFIX" --config Release + + export LLAMA_CONFIG="$PREFIX"/lib/cmake/llama/llama-config.cmake + build_commit=$(git rev-parse --short HEAD | xargs) + build_number=$(git rev-list --count HEAD | xargs) + + major=$(grep -oE "set\(LLAMA_VERSION_MAJOR[[:space:]]+[0-9]+" CMakeLists.txt | grep -oE "[0-9]+$") + minor=$(grep -oE "set\(LLAMA_VERSION_MINOR[[:space:]]+[0-9]+" CMakeLists.txt | grep -oE "[0-9]+$") + patch=$(grep -oE "set\(LLAMA_VERSION_PATCH[[:space:]]+[0-9]+" CMakeLists.txt | grep -oE "[0-9]+$") + build_version="$major.$minor.$patch" + + checks=("set\(LLAMA_VERSION[[:space:]]+$build_version\)" + "set\(LLAMA_BUILD_COMMIT[[:space:]]+$build_commit\)" + "set\(LLAMA_BUILD_NUMBER[[:space:]]+$build_number\)") + + for check in "${checks[@]}"; do + if ! grep -qE "$check" "$LLAMA_CONFIG"; then + echo "Checking llama-config.cmake version... \"$check\" failed!" + exit 1 + fi + done + + echo "Checking llama-config.cmake version... success." + + cd examples/simple-cmake-pkg + cmake -S . -B build -DCMAKE_PREFIX_PATH="$PREFIX"/lib/cmake + cmake --build build -j $(nproc) + + - name: ccache-buckets-save + if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }} + uses: ./.github/actions/ccache-buckets + env: + HF_TOKEN: ${{ secrets.HF_TOKEN_CACHE_OUTPUT }} + with: + key: build-cmake-pkg + folder: llama.cpp + evict-old-files: 1d + hf_bucket: ggml-org/cache + save: true diff --git a/.github/workflows/build-cross.yml b/.github/workflows/build-cross.yml index cc432bd4f419..00cd5dc5ceca 100644 --- a/.github/workflows/build-cross.yml +++ b/.github/workflows/build-cross.yml @@ -163,7 +163,7 @@ jobs: # cmake --build build --config Release -j $(nproc) debian-13-loongarch64-cpu-cross: - runs-on: ${{ 'ubuntu-24.04-arm' || 'ubuntu-24.04' }} + runs-on: ${{ 'ubuntu-26.04-arm' || 'ubuntu-26.04' }} container: debian@sha256:653dfb9f86c3782e8369d5f7d29bb8faba1f4bff9025db46e807fa4c22903671 steps: @@ -218,7 +218,7 @@ jobs: cmake --build build --config Release -j $(nproc) debian-13-loongarch64-vulkan-cross: - runs-on: ${{ 'ubuntu-24.04-arm' || 'ubuntu-24.04' }} + runs-on: ${{ 'ubuntu-26.04-arm' || 'ubuntu-26.04' }} container: debian@sha256:653dfb9f86c3782e8369d5f7d29bb8faba1f4bff9025db46e807fa4c22903671 steps: @@ -276,8 +276,8 @@ jobs: cmake --build build --config Release -j $(nproc) - ubuntu-24-riscv64-cpu-spacemit-ime-cross: - runs-on: ubuntu-24.04 + ubuntu-26-riscv64-cpu-spacemit-ime-cross: + runs-on: ubuntu-26.04 env: # Make sure this is in sync with build-cache.yml diff --git a/.github/workflows/build-cuda-windows.yml b/.github/workflows/build-cuda-windows.yml index 309f41789c40..636572f13b96 100644 --- a/.github/workflows/build-cuda-windows.yml +++ b/.github/workflows/build-cuda-windows.yml @@ -40,12 +40,12 @@ jobs: - cuda: '12.4' arch: x64 defines: '-DGGML_CUDA_CCCL_VERSION=v3.4.3' - - cuda: '13.4' + - cuda: '13.5' arch: x64 - defines: '-DGGML_CUDA_CCCL_VERSION=v3.4.3' - - cuda: '13.4' + defines: '' + - cuda: '13.5' arch: arm64 - defines: '-DGGML_CUDA_CCCL_VERSION=v3.4.3 -DCMAKE_TOOLCHAIN_FILE=cmake/arm64-windows-msvc-cuda.cmake' + defines: '-DCMAKE_TOOLCHAIN_FILE=cmake/arm64-windows-msvc-cuda.cmake' steps: - name: Clone diff --git a/.github/workflows/build-ibm.yml b/.github/workflows/build-ibm.yml index e896b87017ad..67ca8b683e40 100644 --- a/.github/workflows/build-ibm.yml +++ b/.github/workflows/build-ibm.yml @@ -100,7 +100,7 @@ jobs: run: | cd build echo "Fetch llama2c big-endian model" - wget https://huggingface.co/ggml-org/models/resolve/main/tinyllamas/stories260K-be.gguf + wget https://huggingface.co/ggml-org-dev/stories260K/resolve/main/stories260K-be.gguf ./bin/llama-completion -m stories260K-be.gguf -p "One day, Lily met a Shoggoth" -n 500 -c 256 - name: Test quantize (s390x) diff --git a/.github/workflows/build-openvino.yml b/.github/workflows/build-openvino.yml index f4c13b611d4d..009bde4e48fb 100644 --- a/.github/workflows/build-openvino.yml +++ b/.github/workflows/build-openvino.yml @@ -37,7 +37,7 @@ env: LLAMA_ARG_LOG_PREFIX: 1 LLAMA_ARG_LOG_TIMESTAMPS: 1 # TODO: fix failing tests on OpenVINO backend - CTEST_EXCLUDE: "test-llama-archs|test-llama-context" + CTEST_EXCLUDE: "test-llama-archs|test-llama-context|test-fp8-kv" jobs: ubuntu-24-openvino: diff --git a/.github/workflows/build-sanitize.yml b/.github/workflows/build-sanitize.yml index ed75a2dc1cee..33662423ece8 100644 --- a/.github/workflows/build-sanitize.yml +++ b/.github/workflows/build-sanitize.yml @@ -43,13 +43,12 @@ jobs: strategy: matrix: include: - # thread and address doesn't run properly on some self hosted machines, so run it on Github instead - sanitizer: ADDRESS - machine: ubuntu-24.04 + machine: ubuntu-26.04 - sanitizer: THREAD - machine: ubuntu-24.04 + machine: ubuntu-26.04 - sanitizer: UNDEFINED - machine: [self-hosted, X64, Linux, CPU] + machine: ubuntu-26.04 runs-on: ${{ matrix.machine }} @@ -58,31 +57,24 @@ jobs: id: checkout uses: actions/checkout@v6 - # - name: ccache - # uses: ggml-org/ccache-action@v1.2.24 - # if: ${{ matrix.sanitizer != 'UNDEFINED' }} - # with: - # key: ctest-${{ matrix.sanitizer }}-ubuntu-24.04 - # variant: ccache - # evict-old-files: 1d - # save: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }} - - # with UNDEFINED sanitizer, we have to build in Debug to avoid GCC 13 false-positive warnings - - name: Build (undefined) - id: cmake_build_undefined - if: ${{ matrix.sanitizer == 'UNDEFINED' }} - run: | - cmake -B build \ - -DCMAKE_BUILD_TYPE=Debug \ - -DLLAMA_FATAL_WARNINGS=ON \ - -DLLAMA_SANITIZE_${{ matrix.sanitizer }}=ON \ - -DGGML_SANITIZE_${{ matrix.sanitizer }}=ON - - cmake --build build --config Debug -j $(nproc) + - name: ccache + uses: ggml-org/ccache-action@v1.2.24 + with: + restore: false + save: false + + - name: ccache-buckets-restore + uses: ./.github/actions/ccache-buckets + env: + HF_TOKEN: ${{ secrets.HF_TOKEN_CI }} + with: + key: ctest-sanitizer-${{ matrix.sanitizer }} + folder: llama.cpp + hf_bucket: ggml-org/cache - name: Build id: cmake_build - if: ${{ matrix.sanitizer == 'ADDRESS' }} + if: ${{ matrix.sanitizer == 'ADDRESS' || matrix.sanitizer == 'UNDEFINED' }} run: | cmake -B build \ -DCMAKE_BUILD_TYPE=RelWithDebInfo \ @@ -103,6 +95,18 @@ jobs: cmake --build build --config RelWithDebInfo -j $(nproc) + - name: ccache-buckets-save + if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }} + uses: ./.github/actions/ccache-buckets + env: + HF_TOKEN: ${{ secrets.HF_TOKEN_CACHE_OUTPUT }} + with: + key: ctest-sanitizer-${{ matrix.sanitizer }} + folder: llama.cpp + evict-old-files: 1d + hf_bucket: ggml-org/cache + save: true + - name: Test id: cmake_test run: | diff --git a/.github/workflows/build-sycl.yml b/.github/workflows/build-sycl.yml index ecfab4e52403..23ac7e6ddf33 100644 --- a/.github/workflows/build-sycl.yml +++ b/.github/workflows/build-sycl.yml @@ -38,7 +38,7 @@ env: LLAMA_ARG_LOG_TIMESTAMPS: 1 jobs: - ubuntu-24-sycl: + ubuntu-26-sycl: strategy: matrix: build: [fp32, fp16] @@ -48,12 +48,12 @@ jobs: - build: fp16 fp16: ON - runs-on: ubuntu-24.04 + runs-on: ubuntu-26.04 env: ONEAPI_ROOT: /opt/intel/oneapi/ - ONEAPI_INSTALLER_VERSION: "2026.1" - LEVEL_ZERO_VERSION: "1.33.1" + ONEAPI_INSTALLER_VERSION: "2026.2" + LEVEL_ZERO_VERSION: "1.34.0" LEVEL_ZERO_UBUNTU_VERSION: "u24.04" continue-on-error: true @@ -68,12 +68,12 @@ jobs: run: | cd /tmp echo "Download oneAPI Deep Learning Essentials installation package" - wget https://registrationcenter-download.intel.com/akdlm/IRC_NAS/b5c100a7-8052-4e88-90e2-63d9916a998b/intel-deep-learning-essentials-2026.1.4.17_offline.sh -O intel-deep-learning-essentials_offline.sh + wget https://registrationcenter-download.intel.com/akdlm/IRC_NAS/aef232d2-aac5-4cd6-bceb-911019bbb6a8/intel-deep-learning-essentials-2026.2.0.640_offline.sh -O intel-deep-learning-essentials_offline.sh echo "Install oneAPI Deep Learning Essentials components" sudo bash intel-deep-learning-essentials_offline.sh -s -a --silent --eula accept echo "Download oneDNN installation package" - wget https://registrationcenter-download.intel.com/akdlm/IRC_NAS/94c3dbac-0852-45be-a57d-21c204cada3e/intel-onednn-2026.0.2.46_offline.sh -O intel-onednn_offline.sh + wget https://registrationcenter-download.intel.com/akdlm/IRC_NAS/46452a4a-7b7f-49f2-be4c-602a900180bb/intel-onednn-2026.1.0.625_offline.sh -O intel-onednn_offline.sh echo "Install oneDNN components" sudo bash intel-onednn_offline.sh -s -a --silent --eula accept @@ -103,7 +103,7 @@ jobs: env: HF_TOKEN: ${{ secrets.HF_TOKEN_CI }} with: - key: sycl-ubuntu-24-${{ matrix.build }} + key: sycl-ubuntu-26-${{ matrix.build }} folder: llama.cpp hf_bucket: ggml-org/cache @@ -131,7 +131,7 @@ jobs: env: HF_TOKEN: ${{ secrets.HF_TOKEN_CACHE_OUTPUT }} with: - key: sycl-ubuntu-24-${{ matrix.build }} + key: sycl-ubuntu-26-${{ matrix.build }} folder: llama.cpp evict-old-files: 1d hf_bucket: ggml-org/cache @@ -150,13 +150,13 @@ jobs: shell: bash env: - WINDOWS_DLE_URL: https://registrationcenter-download.intel.com/akdlm/IRC_NAS/a51104eb-6f4a-4a0e-9dae-1b81b0c81ae1/intel-deep-learning-essentials-2026.1.4.16_offline.exe - WINDOWS_ONEDNN_URL: https://registrationcenter-download.intel.com/akdlm/IRC_NAS/34e28083-d04b-46a3-826c-8c1d00bf1e1c/intel-onednn-2026.0.2.47_offline.exe + WINDOWS_DLE_URL: https://registrationcenter-download.intel.com/akdlm/IRC_NAS/b177a006-5cd9-4cb2-807f-8199cc73d53c/intel-deep-learning-essentials-2026.2.0.638_offline.exe + WINDOWS_ONEDNN_URL: https://registrationcenter-download.intel.com/akdlm/IRC_NAS/8f905666-c85c-43fb-9bd0-2618c7ecd33b/intel-onednn-2026.1.0.622_offline.exe WINDOWS_DPCPP_MKL: intel.oneapi.win.cpp-dpcpp-common:intel.oneapi.win.mkl.devel:intel.oneapi.win.tbb.devel WINDOWS_DNNL: intel.oneapi.win.dnnl - LEVEL_ZERO_SDK_URL: https://github.com/oneapi-src/level-zero/releases/download/v1.33.1/level-zero-win-sdk-1.33.1.zip + LEVEL_ZERO_SDK_URL: https://github.com/oneapi-src/level-zero/releases/download/v1.34.0/level-zero-win-sdk-1.34.0.zip ONEAPI_ROOT: "C:/Program Files (x86)/Intel/oneAPI" - ONEAPI_INSTALLER_VERSION: "2026.1" + ONEAPI_INSTALLER_VERSION: "2026.2" steps: - name: Clone id: checkout diff --git a/.github/workflows/build-virtgpu.yml b/.github/workflows/build-virtgpu.yml index 57686578304f..6e7bf342ebf3 100644 --- a/.github/workflows/build-virtgpu.yml +++ b/.github/workflows/build-virtgpu.yml @@ -31,8 +31,8 @@ concurrency: cancel-in-progress: true jobs: - ubuntu-24-virtgpu: - runs-on: ${{ 'ubuntu-24.04-arm' || 'ubuntu-24.04' }} + ubuntu-26-virtgpu: + runs-on: ${{ 'ubuntu-26.04-arm' || 'ubuntu-26.04' }} steps: - name: Clone diff --git a/.github/workflows/build-vulkan.yml b/.github/workflows/build-vulkan.yml index c4c08621f958..d2375b0141fb 100644 --- a/.github/workflows/build-vulkan.yml +++ b/.github/workflows/build-vulkan.yml @@ -42,7 +42,7 @@ env: jobs: ubuntu-arm64: - runs-on: ubuntu-24.04-arm + runs-on: ubuntu-26.04-arm steps: - name: Clone @@ -53,9 +53,7 @@ jobs: id: depends run: | sudo apt-get update - sudo apt-get install -y gcc-14 g++-14 build-essential glslc libvulkan-dev spirv-headers libssl-dev ninja-build - echo "CC=gcc-14" >> "$GITHUB_ENV" - echo "CXX=g++-14" >> "$GITHUB_ENV" + sudo apt-get install -y build-essential glslc libvulkan-dev spirv-headers libssl-dev ninja-build - name: ccache uses: ggml-org/ccache-action@v1.2.24 @@ -69,7 +67,7 @@ jobs: env: HF_TOKEN: ${{ secrets.HF_TOKEN_CI }} with: - key: vulkan-ubuntu-24.04-arm + key: vulkan-ubuntu-26.04-arm folder: llama.cpp hf_bucket: ggml-org/cache @@ -92,14 +90,14 @@ jobs: env: HF_TOKEN: ${{ secrets.HF_TOKEN_CACHE_OUTPUT }} with: - key: vulkan-ubuntu-24.04-arm + key: vulkan-ubuntu-26.04-arm folder: llama.cpp evict-old-files: 1d hf_bucket: ggml-org/cache save: true ubuntu-llvmpipe: - runs-on: ubuntu-24.04 + runs-on: ubuntu-26.04 steps: - name: Clone @@ -137,7 +135,7 @@ jobs: env: HF_TOKEN: ${{ secrets.HF_TOKEN_CI }} with: - key: vulkan-ubuntu-24.04-llvmpipe + key: vulkan-ubuntu-26.04-llvmpipe folder: llama.cpp hf_bucket: ggml-org/cache @@ -156,7 +154,7 @@ jobs: env: HF_TOKEN: ${{ secrets.HF_TOKEN_CACHE_OUTPUT }} with: - key: vulkan-ubuntu-24.04-llvmpipe + key: vulkan-ubuntu-26.04-llvmpipe folder: llama.cpp evict-old-files: 1d hf_bucket: ggml-org/cache diff --git a/.github/workflows/build-wasi.yml b/.github/workflows/build-wasi.yml index c6dbd66665bc..2ea854c0160f 100644 --- a/.github/workflows/build-wasi.yml +++ b/.github/workflows/build-wasi.yml @@ -49,7 +49,7 @@ env: jobs: ubuntu-webgpu: - runs-on: ubuntu-24.04-arm + runs-on: ubuntu-26.04-arm steps: - name: Clone @@ -97,7 +97,7 @@ jobs: env: HF_TOKEN: ${{ secrets.HF_TOKEN_CI }} with: - key: webgpu-ubuntu-24.04-arm-wasi + key: webgpu-ubuntu-26.04-arm-wasi folder: llama.cpp hf_bucket: ggml-org/cache @@ -135,7 +135,7 @@ jobs: env: HF_TOKEN: ${{ secrets.HF_TOKEN_CACHE_OUTPUT }} with: - key: webgpu-ubuntu-24.04-arm-wasi + key: webgpu-ubuntu-26.04-arm-wasi folder: llama.cpp evict-old-files: 1d hf_bucket: ggml-org/cache diff --git a/.github/workflows/build-wasm.yml b/.github/workflows/build-wasm.yml index f96fd85a914a..bfe57033e899 100644 --- a/.github/workflows/build-wasm.yml +++ b/.github/workflows/build-wasm.yml @@ -50,7 +50,7 @@ env: jobs: ubuntu-webgpu: - runs-on: ubuntu-24.04-arm + runs-on: ubuntu-26.04-arm steps: - name: Clone @@ -84,7 +84,7 @@ jobs: env: HF_TOKEN: ${{ secrets.HF_TOKEN_CI }} with: - key: webgpu-ubuntu-24.04-arm-wasm + key: webgpu-ubuntu-26.04-arm-wasm folder: llama.cpp hf_bucket: ggml-org/cache @@ -107,7 +107,7 @@ jobs: env: HF_TOKEN: ${{ secrets.HF_TOKEN_CACHE_OUTPUT }} with: - key: webgpu-ubuntu-24.04-arm-wasm + key: webgpu-ubuntu-26.04-arm-wasm folder: llama.cpp evict-old-files: 1d hf_bucket: ggml-org/cache diff --git a/.github/workflows/build-webgpu.yml b/.github/workflows/build-webgpu.yml index 88f2c6f5dfa2..ec515adcc61e 100644 --- a/.github/workflows/build-webgpu.yml +++ b/.github/workflows/build-webgpu.yml @@ -34,7 +34,7 @@ env: jobs: format: - runs-on: ubuntu-24.04 + runs-on: ubuntu-26.04 steps: - name: Clone @@ -45,7 +45,7 @@ jobs: wget -qO- https://apt.llvm.org/llvm-snapshot.gpg.key | sudo tee /etc/apt/trusted.gpg.d/apt.llvm.org.asc > /dev/null sudo add-apt-repository -y \ - "deb http://apt.llvm.org/noble/ llvm-toolchain-noble-22 main" + "deb http://apt.llvm.org/resolute/ llvm-toolchain-resolute-22 main" sudo apt-get update sudo apt-get install -y clang-format-22 diff --git a/.github/workflows/ci-self-hosted-cpu.yml b/.github/workflows/ci-self-hosted-cpu.yml index 7152c4de8725..c03acdaa009c 100644 --- a/.github/workflows/ci-self-hosted-cpu.yml +++ b/.github/workflows/ci-self-hosted-cpu.yml @@ -46,18 +46,55 @@ env: jobs: cpu-x64-high-perf: - runs-on: [self-hosted, Linux, X64, CPU] + runs-on: hf-jobs-cpu-upgrade:ubuntu26_04 + + # note: this runner has 8 CPUs, but std::thread::hardware_concurrency reports the cores of the + # host (https://github.com/ggml-org/llama.cpp/pull/31439#issuecomment-6683386059) + env: + TEST_ARG_HARDWARE_CONCURRENCY: 8 + TEST_ARG_PARALLEL_WORKERS: 4 steps: - name: Clone id: checkout uses: actions/checkout@v6 + - name: Dependencies + id: depends + run: | + sudo apt update + sudo apt install -y build-essential cmake libssl-dev time unzip wget python3 python3-venv python3-dev + + - name: ccache + uses: ggml-org/ccache-action@v1.2.24 + with: + restore: false + save: false + + - name: ccache-buckets-restore + uses: ./.github/actions/ccache-buckets + with: + key: self-hosted-cpu-x64-high-perf + folder: llama.cpp + hf_bucket: ggml-org/cache + - name: Test id: ggml-ci run: | LLAMA_ARG_THREADS=$(nproc) GG_BUILD_HIGH_PERF=1 GG_BUILD_EXTRA_TESTS_0=1 bash ./ci/run.sh ~/results/llama.cpp/$RUNNER_NAME ~/mnt/llama.cpp + - name: ccache-buckets-save + if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }} + uses: ./.github/actions/ccache-buckets + env: + HF_TOKEN: ${{ secrets.HF_TOKEN_CACHE_OUTPUT }} + with: + key: self-hosted-cpu-x64-high-perf + folder: llama.cpp + evict-old-files: 1d + hf_bucket: ggml-org/cache + save: true + cpu-arm64-high-perf-graviton4: runs-on: ah-ubuntu_24_04-c8g_8x diff --git a/.github/workflows/ci-self-hosted-cuda.yml b/.github/workflows/ci-self-hosted-cuda.yml index 89605f4dd7ac..d433b95269c1 100644 --- a/.github/workflows/ci-self-hosted-cuda.yml +++ b/.github/workflows/ci-self-hosted-cuda.yml @@ -93,13 +93,72 @@ jobs: save: true gpu-rocm: - runs-on: [self-hosted, Linux, AMD] + strategy: + fail-fast: false + matrix: + include: + # Strix Halo (iGPU, RDNA3.5) + - gpu: gfx1151 + runner: [self-hosted, Linux, strix-halo, devlab-dispatch] + # async-execution correctness workaround, gfx1151 only (see Test step) + hip_launch_blocking: "1" + # Radeon AI PRO R9700 (dGPU, RDNA4) + - gpu: gfx1201 + runner: [self-hosted, Linux, radeon-r9700, devlab-dispatch] + hip_launch_blocking: "0" + # HIP-graph capture fails on gfx1201 (hipMemset -> hipErrorStreamCaptureImplicit) + disable_hip_graphs: true + + name: gpu-rocm (${{ matrix.gpu }}) + runs-on: ${{ matrix.runner }} + timeout-minutes: 120 + + env: + ROCM_VERSION: "10.0.0" steps: - name: Clone id: checkout uses: actions/checkout@v6 + - name: Verify GPU device + run: | + ls /dev/kfd /dev/dri/renderD* || { echo "ERROR: no amdgpu compute device"; exit 1; } + + # same TheRock wheels as the ubuntu-24-rocm release build, so the runner does not need a ROCm install + - name: Setup TheRock with Wheels + run: | + python3 -m venv "$RUNNER_TEMP/rocm" + "$RUNNER_TEMP/rocm/bin/python" -m pip install --index-url https://stable.repo.amd.com/rocm/whl-next/ "rocm[libraries,devel,device-${{ matrix.gpu }}]==${ROCM_VERSION}" + ROCM_PATH=$("$RUNNER_TEMP/rocm/bin/rocm-sdk" path --root) + { + echo "ROCM_PATH=$ROCM_PATH" + echo "HIP_PATH=$ROCM_PATH" + echo "CMAKE_PREFIX_PATH=$("$RUNNER_TEMP/rocm/bin/rocm-sdk" path --cmake)" + echo "LD_LIBRARY_PATH=$ROCM_PATH/lib:${LD_LIBRARY_PATH:-}" + } >> "$GITHUB_ENV" + "$RUNNER_TEMP/rocm/bin/rocm-sdk" path --bin >> "$GITHUB_PATH" + + - name: ccache + uses: ggml-org/ccache-action@v1.2.24 + with: + restore: false + save: false + + # ROCm is installed again on each run, so hash the compiler content, not its mtime + - name: Tune ccache for reinstalled ROCm toolchain + run: | + ccache --set-config=compiler_check=content + ccache --set-config=sloppiness=time_macros,include_file_mtime,include_file_ctime + ccache --set-config=depend_mode=true + + - name: ccache-buckets-restore + uses: ./.github/actions/ccache-buckets + with: + key: self-hosted-gpu-rocm-${{ matrix.gpu }} + folder: llama.cpp + hf_bucket: ggml-org/cache + - name: Test id: ggml-ci # HIP_LAUNCH_BLOCKING=1: workaround for an async-execution correctness @@ -107,10 +166,25 @@ jobs: # incorrect output (perplexity ~88 vs ~9.4). Serializing kernel launches # restores correctness. Remove once the underlying ROCm/HIP issue is fixed. env: - HIP_LAUNCH_BLOCKING: "1" + HIP_LAUNCH_BLOCKING: ${{ matrix.hip_launch_blocking }} run: | rocminfo - GG_BUILD_ROCM=1 GG_BUILD_AMDGPU_TARGETS=gfx1151 bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp + # Exported rather than set in env:, because ggml only checks that the + # variable is defined: an empty value would disable graphs on gfx1151 too. + if [ "${{ matrix.disable_hip_graphs }}" = "true" ]; then export GGML_CUDA_DISABLE_GRAPHS=1; fi + GG_BUILD_ROCM=1 GG_BUILD_AMDGPU_TARGETS=${{ matrix.gpu }} bash ./ci/run.sh "$RUNNER_TEMP/results/llama.cpp" "$RUNNER_TEMP/mnt/llama.cpp" + + - name: ccache-buckets-save + if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }} + uses: ./.github/actions/ccache-buckets + env: + HF_TOKEN: ${{ secrets.HF_TOKEN_CACHE_OUTPUT }} + with: + key: self-hosted-gpu-rocm-${{ matrix.gpu }} + folder: llama.cpp + evict-old-files: 1d + hf_bucket: ggml-org/cache + save: true # TODO: provision AMD GPU machine # amd-rocm: diff --git a/.github/workflows/ci-self-hosted-webgpu.yml b/.github/workflows/ci-self-hosted-webgpu.yml index e2ef40cd25d1..f94689006dbe 100644 --- a/.github/workflows/ci-self-hosted-webgpu.yml +++ b/.github/workflows/ci-self-hosted-webgpu.yml @@ -58,12 +58,7 @@ jobs: - name: Install dependencies run: | sudo apt update - sudo apt install -y software-properties-common - sudo add-apt-repository ppa:deadsnakes/ppa - sudo apt update - sudo apt install -y build-essential cmake libxcb-xinput0 libxcb-xinerama0 libxcb-cursor-dev libvulkan1 mesa-vulkan-drivers libglvnd0 libgl1 libglx0 libegl1 libgles2 libssl-dev time unzip wget python3.12 python3.12-venv - sudo update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.12 1 - sudo update-alternatives --config python3 + sudo apt install -y build-essential cmake libxcb-xinput0 libxcb-xinerama0 libxcb-cursor-dev libvulkan1 mesa-vulkan-drivers libglvnd0 libgl1 libglx0 libegl1 libgles2 libssl-dev time unzip wget python3 python3-venv python3-pip - name: ccache uses: ggml-org/ccache-action@v1.2.24 diff --git a/.github/workflows/copilot-setup-steps.yml b/.github/workflows/copilot-setup-steps.yml index 5527e73b6f3d..9eb1113abf82 100644 --- a/.github/workflows/copilot-setup-steps.yml +++ b/.github/workflows/copilot-setup-steps.yml @@ -16,7 +16,7 @@ cache-mode: none jobs: # The job MUST be called `copilot-setup-steps` or it will not be picked up by Copilot. copilot-setup-steps: - runs-on: ubuntu-24.04 + runs-on: ubuntu-26.04 # Set the permissions to the lowest permissions possible needed for your steps. # Copilot will be given its own token for its operations. diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 75b3343095a9..ad1a9de9a05f 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -70,7 +70,7 @@ jobs: prepare_matrices: name: Prepare Docker matrices - runs-on: ubuntu-24.04 + runs-on: ubuntu-26.04 outputs: build_matrix: ${{ steps.matrices.outputs.build_matrix }} merge_matrix: ${{ steps.matrices.outputs.merge_matrix }} @@ -87,19 +87,19 @@ jobs: # Keep all build targets in one place and derive merge targets from it. cat > build-matrix.json <<'JSON' [ - { "tag": "cpu", "dockerfile": ".devops/cpu.Dockerfile", "platforms": "linux/amd64", "full": true, "light": true, "server": true, "free_disk_space": false, "runs_on": "ubuntu-24.04" }, - { "tag": "cpu", "dockerfile": ".devops/cpu.Dockerfile", "platforms": "linux/arm64", "full": true, "light": true, "server": true, "free_disk_space": false, "runs_on": "ubuntu-24.04-arm" }, - { "tag": "cpu", "dockerfile": ".devops/s390x.Dockerfile", "platforms": "linux/s390x", "full": true, "light": true, "server": true, "free_disk_space": false, "runs_on": "ubuntu-24.04-s390x", "prebuilt_ui": true }, - { "tag": "cuda cuda12", "dockerfile": ".devops/cuda.Dockerfile", "cuda_version": "12.8.1", "platforms": "linux/amd64", "full": true, "light": true, "server": true, "free_disk_space": true, "runs_on": "ubuntu-24.04" }, - { "tag": "cuda cuda12", "dockerfile": ".devops/cuda.Dockerfile", "cuda_version": "12.8.1", "platforms": "linux/arm64", "full": true, "light": true, "server": true, "free_disk_space": true, "runs_on": "ubuntu-24.04-arm" }, - { "tag": "cuda13", "dockerfile": ".devops/cuda.Dockerfile", "cuda_version": "13.4.1", "platforms": "linux/amd64", "full": true, "light": true, "server": true, "free_disk_space": true, "runs_on": "ubuntu-24.04" }, - { "tag": "cuda13", "dockerfile": ".devops/cuda.Dockerfile", "cuda_version": "13.4.1", "platforms": "linux/arm64", "full": true, "light": true, "server": true, "free_disk_space": true, "runs_on": "ubuntu-24.04-arm" }, - { "tag": "musa", "dockerfile": ".devops/musa.Dockerfile", "platforms": "linux/amd64", "full": true, "light": true, "server": true, "free_disk_space": true, "runs_on": "ubuntu-24.04" }, - { "tag": "intel", "dockerfile": ".devops/intel.Dockerfile", "platforms": "linux/amd64", "full": true, "light": true, "server": true, "free_disk_space": true, "runs_on": "ubuntu-24.04" }, - { "tag": "vulkan", "dockerfile": ".devops/vulkan.Dockerfile", "platforms": "linux/amd64", "full": true, "light": true, "server": true, "free_disk_space": false, "runs_on": "ubuntu-24.04" }, - { "tag": "vulkan", "dockerfile": ".devops/vulkan.Dockerfile", "platforms": "linux/arm64", "full": true, "light": true, "server": true, "free_disk_space": false, "runs_on": "ubuntu-24.04-arm" }, - { "tag": "rocm", "dockerfile": ".devops/rocm.Dockerfile", "platforms": "linux/amd64", "full": true, "light": true, "server": true, "free_disk_space": true, "runs_on": "ubuntu-24.04" }, - { "tag": "openvino", "dockerfile": ".devops/openvino.Dockerfile", "platforms": "linux/amd64", "full": true, "light": true, "server": true, "free_disk_space": false, "runs_on": "ubuntu-24.04" } + { "tag": "cpu", "dockerfile": ".devops/cpu.Dockerfile", "platforms": "linux/amd64", "full": true, "light": true, "server": true, "free_disk_space": false, "runs_on": "ubuntu-26.04" }, + { "tag": "cpu", "dockerfile": ".devops/cpu.Dockerfile", "platforms": "linux/arm64", "full": true, "light": true, "server": true, "free_disk_space": false, "runs_on": "ubuntu-26.04-arm" }, + { "tag": "cpu", "dockerfile": ".devops/s390x.Dockerfile", "platforms": "linux/s390x", "full": true, "light": true, "server": true, "free_disk_space": false, "runs_on": "ubuntu-26.04-s390x", "prebuilt_ui": true }, + { "tag": "cuda cuda12", "dockerfile": ".devops/cuda.Dockerfile", "cuda_version": "12.8.1", "platforms": "linux/amd64", "full": true, "light": true, "server": true, "free_disk_space": true, "runs_on": "ubuntu-26.04" }, + { "tag": "cuda cuda12", "dockerfile": ".devops/cuda.Dockerfile", "cuda_version": "12.8.1", "platforms": "linux/arm64", "full": true, "light": true, "server": true, "free_disk_space": true, "runs_on": "ubuntu-26.04-arm" }, + { "tag": "cuda13", "dockerfile": ".devops/cuda.Dockerfile", "cuda_version": "13.5.0", "platforms": "linux/amd64", "full": true, "light": true, "server": true, "free_disk_space": true, "runs_on": "ubuntu-26.04" }, + { "tag": "cuda13", "dockerfile": ".devops/cuda.Dockerfile", "cuda_version": "13.5.0", "platforms": "linux/arm64", "full": true, "light": true, "server": true, "free_disk_space": true, "runs_on": "ubuntu-26.04-arm" }, + { "tag": "musa", "dockerfile": ".devops/musa.Dockerfile", "platforms": "linux/amd64", "full": true, "light": true, "server": true, "free_disk_space": true, "runs_on": "ubuntu-26.04" }, + { "tag": "intel", "dockerfile": ".devops/intel.Dockerfile", "platforms": "linux/amd64", "full": true, "light": true, "server": true, "free_disk_space": true, "runs_on": "ubuntu-26.04" }, + { "tag": "vulkan", "dockerfile": ".devops/vulkan.Dockerfile", "platforms": "linux/amd64", "full": true, "light": true, "server": true, "free_disk_space": false, "runs_on": "ubuntu-26.04" }, + { "tag": "vulkan", "dockerfile": ".devops/vulkan.Dockerfile", "platforms": "linux/arm64", "full": true, "light": true, "server": true, "free_disk_space": false, "runs_on": "ubuntu-26.04-arm" }, + { "tag": "rocm", "dockerfile": ".devops/rocm.Dockerfile", "platforms": "linux/amd64", "full": true, "light": true, "server": true, "free_disk_space": true, "runs_on": "ubuntu-26.04" }, + { "tag": "openvino", "dockerfile": ".devops/openvino.Dockerfile", "platforms": "linux/amd64", "full": true, "light": true, "server": true, "free_disk_space": false, "runs_on": "ubuntu-26.04" } ] JSON @@ -400,7 +400,7 @@ jobs: merge_arch_tags: name: Create shared tags from digests needs: [prepare_matrices, push_to_registry, create_tag] - runs-on: ubuntu-24.04 + runs-on: ubuntu-26.04 permissions: contents: read packages: write diff --git a/.github/workflows/gguf-publish.yml b/.github/workflows/gguf-publish.yml index 367b29c60f94..94a49146681e 100644 --- a/.github/workflows/gguf-publish.yml +++ b/.github/workflows/gguf-publish.yml @@ -24,7 +24,7 @@ permissions: jobs: deploy: - runs-on: ubuntu-24.04 # previously ubuntu-latest + runs-on: ubuntu-26.04 steps: - uses: actions/checkout@v6 diff --git a/.github/workflows/make-release.yml b/.github/workflows/make-release.yml index 2e5e44caaab8..1f5d43755e78 100644 --- a/.github/workflows/make-release.yml +++ b/.github/workflows/make-release.yml @@ -39,7 +39,7 @@ permissions: jobs: make-release: - runs-on: ubuntu-24.04 # previously ubuntu-latest + runs-on: ubuntu-26.04 steps: - name: Checkout diff --git a/.github/workflows/models-check.yml b/.github/workflows/models-check.yml index 66a150aff3aa..2774cefd3791 100644 --- a/.github/workflows/models-check.yml +++ b/.github/workflows/models-check.yml @@ -11,7 +11,8 @@ on: 'tests/fusion/**', 'tests/test-fusion.cpp', 'tests/test-llama-archs.cpp', - 'src/llama-graph.cpp', + 'src/llama-graph*', + 'src/llama-kv-cache*', 'src/llama-model*', 'src/models/**' ] @@ -24,7 +25,8 @@ on: 'tests/fusion/**', 'tests/test-fusion.cpp', 'tests/test-llama-archs.cpp', - 'src/llama-graph.cpp', + 'src/llama-graph*', + 'src/llama-kv-cache*', 'src/llama-model*', 'src/models/**' ] @@ -145,6 +147,11 @@ jobs: rm -rf build-ci-models && mkdir -p build-ci-models ./build/bin/test-llama-archs -o build-ci-models + - name: Test KV shift + id: test_kv_shift + run: | + ./build/bin/test-llama-archs --kv-shift + - name: Test fusion id: test_fusion run: | @@ -293,6 +300,11 @@ jobs: # run: | # ./build/bin/test-fusion --models build-ci-models --device Vulkan0 --check tests/fusion/Vulkan.csv + - name: Test KV shift + id: test_kv_shift + run: | + ./build/bin/test-llama-archs --kv-shift + - name: Test archs id: test_archs run: | diff --git a/.github/workflows/pr-video-demo-label.yml b/.github/workflows/pr-video-demo-label.yml new file mode 100644 index 000000000000..9b3165fa22ab --- /dev/null +++ b/.github/workflows/pr-video-demo-label.yml @@ -0,0 +1,37 @@ +name: "PR video demo label" + +on: + pull_request_target: + types: [opened, edited, reopened, synchronize] + +cache-mode: none +permissions: + pull-requests: write + issues: write + +jobs: + video-demo-label: + runs-on: ubuntu-slim + steps: + - name: Add or remove the video demo label + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + PR_URL: ${{ github.event.pull_request.html_url }} + PR_NUMBER: ${{ github.event.pull_request.number }} + # passed through env, so the body is only matched, never run, and is safe to read from a fork + BODY: ${{ github.event.pull_request.body }} + HAS_LABEL: ${{ contains(github.event.pull_request.labels.*.name, 'video demo') }} + run: | + # sections written by demo:pr (see tools/ui/demo/cli.ts): the managed comment + # markers, or a preview linking to a demo video uploaded for this PR + has_demo=false + if { grep -qF '' <<< "$BODY" && grep -qF '' <<< "$BODY"; } || + grep -qE "/resolve/videos/pr/$PR_NUMBER/[0-9a-f]{10}(-[0-9]+)?\.mp4" <<< "$BODY"; then + has_demo=true + fi + + if [ "$has_demo" = true ] && [ "$HAS_LABEL" = false ]; then + gh pr edit "$PR_URL" --add-label 'video demo' + elif [ "$has_demo" = false ] && [ "$HAS_LABEL" = true ]; then + gh pr edit "$PR_URL" --remove-label 'video demo' + fi diff --git a/.github/workflows/python-check-requirements.yml b/.github/workflows/python-check-requirements.yml index 9738ff778920..464cc94e1c84 100644 --- a/.github/workflows/python-check-requirements.yml +++ b/.github/workflows/python-check-requirements.yml @@ -24,7 +24,7 @@ concurrency: jobs: python-check-requirements: - runs-on: ${{ 'ubuntu-24.04-arm' || 'ubuntu-24.04' }} + runs-on: ${{ 'ubuntu-26.04-arm' || 'ubuntu-26.04' }} name: check-requirements steps: - name: Check out source repository diff --git a/.github/workflows/python-type-check.yml b/.github/workflows/python-type-check.yml index 8fb4ffd47a7c..f4ba40fc2736 100644 --- a/.github/workflows/python-type-check.yml +++ b/.github/workflows/python-type-check.yml @@ -35,7 +35,7 @@ jobs: uses: actions/setup-python@v6 with: python-version: "3.11" - pip-install: -r requirements/requirements-all.txt ty==0.0.88 + pip-install: -r requirements/requirements-all.txt ty==0.0.100 # - name: Type-check with Pyright # uses: jakebailey/pyright-action@v2 # with: diff --git a/.github/workflows/release-publish.yml b/.github/workflows/release-publish.yml index 243b870e6664..3efa82e4453a 100644 --- a/.github/workflows/release-publish.yml +++ b/.github/workflows/release-publish.yml @@ -171,14 +171,14 @@ jobs: **Linux:** - [Ubuntu x64 (CPU)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-x64.tar.gz) - [Ubuntu arm64 (CPU)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-arm64.tar.gz) - - [Ubuntu s390x (CPU)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-s390x.tar.gz) + - [Ubuntu 24.04 s390x (CPU)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-s390x.tar.gz) - [Ubuntu 26.04 s390x (CPU)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-26.04-s390x.tar.gz) - [Ubuntu 26.04 s390x (zDNN)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-zdnn-s390x.tar.gz) - [Ubuntu x64 (Vulkan)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-vulkan-x64.tar.gz) - [Ubuntu arm64 (Vulkan)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-vulkan-arm64.tar.gz) - [Ubuntu x64 (CUDA 12)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-cuda-12.8-x64.tar.gz) - [CUDA 12.8 libraries](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/cudart-llama-${{ steps.tag.outputs.name }}-bin-ubuntu-cuda-12.8-x64.tar.gz) - - [Ubuntu x64 (CUDA 13)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-cuda-13.4-x64.tar.gz) - [CUDA 13.4 libraries](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/cudart-llama-${{ steps.tag.outputs.name }}-bin-ubuntu-cuda-13.4-x64.tar.gz) - - [Ubuntu arm64 (CUDA 13)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-cuda-13.4-arm64.tar.gz) - [CUDA 13.4 libraries](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/cudart-llama-${{ steps.tag.outputs.name }}-bin-ubuntu-cuda-13.4-arm64.tar.gz) + - [Ubuntu x64 (CUDA 13)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-cuda-13.5-x64.tar.gz) - [CUDA 13.5 libraries](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/cudart-llama-${{ steps.tag.outputs.name }}-bin-ubuntu-cuda-13.5-x64.tar.gz) + - [Ubuntu arm64 (CUDA 13)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-cuda-13.5-arm64.tar.gz) - [CUDA 13.5 libraries](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/cudart-llama-${{ steps.tag.outputs.name }}-bin-ubuntu-cuda-13.5-arm64.tar.gz) - [Ubuntu x64 (ROCm 10.0)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-rocm-10.0-x64.tar.gz) - [Ubuntu x64 (OpenVINO)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-openvino-2026.4.1-x64.tar.gz) - [Ubuntu x64 (SYCL FP32)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-sycl-fp32-x64.tar.gz) @@ -194,8 +194,8 @@ jobs: - [Windows arm64 (CPU)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-win-cpu-arm64.zip) - [Windows arm64 (OpenCL Adreno)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-win-opencl-adreno-arm64.zip) - [Windows x64 (CUDA 12)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-win-cuda-12.4-x64.zip) - [CUDA 12.4 DLLs](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/cudart-llama-bin-win-cuda-12.4-x64.zip) - - [Windows x64 (CUDA 13)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-win-cuda-13.4-x64.zip) - [CUDA 13.4 DLLs](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/cudart-llama-bin-win-cuda-13.4-x64.zip) - - [Windows arm64 (CUDA 13)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-win-cuda-13.4-arm64.zip) - [CUDA 13.4 DLLs](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/cudart-llama-bin-win-cuda-13.4-arm64.zip) + - [Windows x64 (CUDA 13)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-win-cuda-13.5-x64.zip) - [CUDA 13.5 DLLs](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/cudart-llama-bin-win-cuda-13.5-x64.zip) + - [Windows arm64 (CUDA 13)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-win-cuda-13.5-arm64.zip) - [CUDA 13.5 DLLs](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/cudart-llama-bin-win-cuda-13.5-arm64.zip) - [Windows x64 (Vulkan)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-win-vulkan-x64.zip) - [Windows arm64 (Vulkan)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-win-vulkan-arm64.zip) - [Windows x64 (OpenVINO)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-win-openvino-2026.4.1-x64.zip) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index acda37fb9cfc..867dcaf49420 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -345,14 +345,14 @@ jobs: defines: '-DGGML_CUDA_CCCL_VERSION=v3.4.3' - build: 'x64' os: ubuntu-24.04 - cuda: '13.4.1' - label: '13.4' - defines: '-DGGML_CUDA_CCCL_VERSION=v3.4.3' + cuda: '13.5.0' + label: '13.5' + defines: '' - build: 'arm64' os: ubuntu-24.04-arm - cuda: '13.4.1' - label: '13.4' - defines: '-DGGML_CUDA_CCCL_VERSION=v3.4.3' + cuda: '13.5.0' + label: '13.5' + defines: '' runs-on: ${{ matrix.os }} container: nvidia/cuda:${{ matrix.cuda }}-devel-ubuntu24.04 @@ -463,7 +463,7 @@ jobs: needs: [check-release, ui-build] if: ${{ needs.check-release.outputs.should_release == 'true' }} - runs-on: ubuntu-24.04 # previously ubuntu-latest + runs-on: ubuntu-24.04 env: NDK_VERSION: "29.0.14206865" @@ -548,7 +548,7 @@ jobs: needs: [check-release, ui-build] if: ${{ needs.check-release.outputs.should_release == 'true' }} - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 container: 'ghcr.io/snapdragon-toolchain/arm64-android:v0.8' defaults: @@ -601,7 +601,7 @@ jobs: needs: [check-release, ui-build] if: ${{ needs.check-release.outputs.should_release == 'true' }} - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 container: 'ghcr.io/snapdragon-toolchain/arm64-linux:v0.8' defaults: @@ -1211,12 +1211,12 @@ jobs: - cuda: '12.4' arch: x64 defines: '-DGGML_CUDA_CCCL_VERSION=v3.4.3' - - cuda: '13.4' + - cuda: '13.5' arch: x64 - defines: '-DGGML_CUDA_CCCL_VERSION=v3.4.3' - - cuda: '13.4' + defines: '' + - cuda: '13.5' arch: arm64 - defines: '-DGGML_CUDA_CCCL_VERSION=v3.4.3 -DCMAKE_TOOLCHAIN_FILE=cmake/arm64-windows-msvc-cuda.cmake' + defines: '-DCMAKE_TOOLCHAIN_FILE=cmake/arm64-windows-msvc-cuda.cmake' steps: - name: Clone @@ -1307,14 +1307,14 @@ jobs: shell: bash env: - WINDOWS_DLE_URL: https://registrationcenter-download.intel.com/akdlm/IRC_NAS/a51104eb-6f4a-4a0e-9dae-1b81b0c81ae1/intel-deep-learning-essentials-2026.1.4.16_offline.exe - WINDOWS_ONEDNN_URL: https://registrationcenter-download.intel.com/akdlm/IRC_NAS/34e28083-d04b-46a3-826c-8c1d00bf1e1c/intel-onednn-2026.0.2.47_offline.exe + WINDOWS_DLE_URL: https://registrationcenter-download.intel.com/akdlm/IRC_NAS/b177a006-5cd9-4cb2-807f-8199cc73d53c/intel-deep-learning-essentials-2026.2.0.638_offline.exe + WINDOWS_ONEDNN_URL: https://registrationcenter-download.intel.com/akdlm/IRC_NAS/8f905666-c85c-43fb-9bd0-2618c7ecd33b/intel-onednn-2026.1.0.622_offline.exe WINDOWS_DPCPP_MKL: intel.oneapi.win.cpp-dpcpp-common:intel.oneapi.win.mkl.devel:intel.oneapi.win.tbb.devel WINDOWS_DNNL: intel.oneapi.win.dnnl - LEVEL_ZERO_SDK_URL: https://github.com/oneapi-src/level-zero/releases/download/v1.33.1/level-zero-win-sdk-1.33.1.zip + LEVEL_ZERO_SDK_URL: https://github.com/oneapi-src/level-zero/releases/download/v1.34.0/level-zero-win-sdk-1.34.0.zip ONEAPI_ROOT: "C:/Program Files (x86)/Intel/oneAPI" - ONEAPI_INSTALLER_VERSION: "2026.1" + ONEAPI_INSTALLER_VERSION: "2026.2" steps: - name: Clone @@ -1363,11 +1363,12 @@ jobs: cp "${{ env.ONEAPI_ROOT }}/mkl/latest/bin/mkl_def.3.dll" ./build/bin cp "${{ env.ONEAPI_ROOT }}/mkl/latest/bin/mkl_avx2.3.dll" ./build/bin cp "${{ env.ONEAPI_ROOT }}/mkl/latest/bin/mkl_tbb_thread.3.dll" ./build/bin + cp "${{ env.ONEAPI_ROOT }}/mkl/latest/bin/mkl_intel_thread.3.dll" ./build/bin + cp "${{ env.ONEAPI_ROOT }}/mkl/latest/bin/libimalloc.dll" ./build/bin cp "${{ env.ONEAPI_ROOT }}/compiler/latest/bin/ur_adapter_level_zero.dll" ./build/bin cp "${{ env.ONEAPI_ROOT }}/compiler/latest/bin/ur_adapter_level_zero_v2.dll" ./build/bin - cp "${{ env.ONEAPI_ROOT }}/compiler/latest/bin/ur_adapter_opencl.dll" ./build/bin - cp "${{ env.ONEAPI_ROOT }}/compiler/latest/bin/ur_loader.dll" ./build/bin + cp "${{ env.ONEAPI_ROOT }}/compiler/latest/bin/ur_loader_shared.dll" ./build/bin cp "${{ env.ONEAPI_ROOT }}/compiler/latest/bin/ur_win_proxy_loader.dll" ./build/bin ZE_LOADER_DLL=$(find "${{ env.ONEAPI_ROOT }}" "$LEVEL_ZERO_V1_SDK_PATH" -iname ze_loader.dll -print -quit 2>/dev/null || true) if [ -n "$ZE_LOADER_DLL" ]; then @@ -1421,8 +1422,8 @@ jobs: env: ONEAPI_ROOT: /opt/intel/oneapi/ - ONEAPI_INSTALLER_VERSION: "2026.1" - LEVEL_ZERO_VERSION: "1.33.1" + ONEAPI_INSTALLER_VERSION: "2026.2" + LEVEL_ZERO_VERSION: "1.34.0" LEVEL_ZERO_UBUNTU_VERSION: "u24.04" steps: @@ -1437,12 +1438,12 @@ jobs: run: | cd /tmp echo "Download oneAPI Deep Learning Essentials installation package" - wget https://registrationcenter-download.intel.com/akdlm/IRC_NAS/b5c100a7-8052-4e88-90e2-63d9916a998b/intel-deep-learning-essentials-2026.1.4.17_offline.sh -O intel-deep-learning-essentials_offline.sh + wget https://registrationcenter-download.intel.com/akdlm/IRC_NAS/aef232d2-aac5-4cd6-bceb-911019bbb6a8/intel-deep-learning-essentials-2026.2.0.640_offline.sh -O intel-deep-learning-essentials_offline.sh echo "Install oneAPI Deep Learning Essentials components" sudo bash intel-deep-learning-essentials_offline.sh -s -a --silent --eula accept echo "Download oneDNN installation package" - wget https://registrationcenter-download.intel.com/akdlm/IRC_NAS/94c3dbac-0852-45be-a57d-21c204cada3e/intel-onednn-2026.0.2.46_offline.sh -O intel-onednn_offline.sh + wget https://registrationcenter-download.intel.com/akdlm/IRC_NAS/46452a4a-7b7f-49f2-be4c-602a900180bb/intel-onednn-2026.1.0.625_offline.sh -O intel-onednn_offline.sh echo "Install oneDNN components" sudo bash intel-onednn_offline.sh -s -a --silent --eula accept diff --git a/.github/workflows/server-speculative.yml b/.github/workflows/server-speculative.yml new file mode 100644 index 000000000000..de247afb6fb7 --- /dev/null +++ b/.github/workflows/server-speculative.yml @@ -0,0 +1,65 @@ +name: Server (speculative decoding) + +on: + workflow_dispatch: + inputs: + sha: + description: 'Commit SHA1 to build' + required: false + type: string + +cache-mode: none +permissions: + contents: read + +env: + LLAMA_ARG_LOG_COLORS: 1 + LLAMA_ARG_LOG_PREFIX: 1 + LLAMA_ARG_LOG_TIMESTAMPS: 1 + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +jobs: + server-speculative: + runs-on: [self-hosted, Linux, gb10, perf] + container: "nvidia/cuda:13.4.3-devel-ubuntu26.04" + timeout-minutes: 60 + defaults: + run: + shell: bash + + steps: + - name: Clone + uses: actions/checkout@v6 + with: + fetch-depth: 0 + ref: ${{ github.event.inputs.sha || github.event.pull_request.head.sha || github.sha || github.head_ref || github.ref_name }} + + - name: Install dependencies + run: | + apt update + apt install -y cmake libssl-dev python3 python3-venv python3-pip + + - name: Build + run: | + cmake -B build -DGGML_CUDA=ON -DCMAKE_CUDA_COMPILER=/usr/local/cuda/bin/nvcc + cmake --build build --config Release -j $(nproc) --target llama-server + + - name: Python setup + run: | + cd tools/server/tests + python3 -m venv venv + source venv/bin/activate + pip install -r requirements.txt + + - name: Test speculative decoding + env: + SPEC_METHODS: compact + SPEC_MODEL_DIR: ${{ vars.SPEC_METHOD_MODEL_DIR || format('{0}/spec-models', runner.temp) }} + N_GPU_LAYERS: 99 + run: | + cd tools/server/tests + source venv/bin/activate + PYTEST_WORKERS=1 ./tests.sh -s -v unit/test_spec_methods.py diff --git a/.github/workflows/server.yml b/.github/workflows/server.yml index 05dd0ac4595a..f3c577ec6d28 100644 --- a/.github/workflows/server.yml +++ b/.github/workflows/server.yml @@ -59,7 +59,7 @@ concurrency: jobs: ubuntu: - runs-on: ubuntu-24.04-arm + runs-on: ubuntu-26.04-arm steps: - name: Dependencies @@ -94,7 +94,7 @@ jobs: env: HF_TOKEN: ${{ secrets.HF_TOKEN_CI }} with: - key: server-ubuntu-24.04-arm + key: server-ubuntu-26.04-arm folder: llama.cpp hf_bucket: ggml-org/cache @@ -111,7 +111,7 @@ jobs: env: HF_TOKEN: ${{ secrets.HF_TOKEN_CACHE_OUTPUT }} with: - key: server-ubuntu-24.04-arm + key: server-ubuntu-26.04-arm folder: llama.cpp evict-old-files: 1d hf_bucket: ggml-org/cache diff --git a/.github/workflows/ui-build.yml b/.github/workflows/ui-build.yml index e0d8dd065e98..f229fcd04480 100644 --- a/.github/workflows/ui-build.yml +++ b/.github/workflows/ui-build.yml @@ -15,7 +15,7 @@ permissions: jobs: build: - runs-on: ubuntu-slim + runs-on: ubuntu-26.04-arm steps: - name: Checkout code diff --git a/.github/workflows/winget.yml b/.github/workflows/winget.yml index 8b5bd355c16e..1f0f24905d06 100644 --- a/.github/workflows/winget.yml +++ b/.github/workflows/winget.yml @@ -12,7 +12,7 @@ permissions: jobs: update: name: Update Winget Package - runs-on: ubuntu-24.04 # previously ubuntu-latest + runs-on: ubuntu-26.04 if: github.repository_owner == 'ggml-org' steps: diff --git a/AUTHORS b/AUTHORS index 14921cde5499..96ca4c1d9bff 100644 --- a/AUTHORS +++ b/AUTHORS @@ -1,4 +1,4 @@ -# date: Wed Oct 28 12:38:26 EET 2026 +# date: Wed Nov 4 12:03:31 EET 2026 # this file is auto-generated by scripts/gen-authors.sh Нияз Гарифзянов <112617865+garrnizon@users.noreply.github.com> @@ -48,6 +48,7 @@ Abheek Gulati abhijain1204fujitsu <139222713+abhijain1204fujitsu@users.noreply.github.com> Abhijit Ramesh abhijitb11 <113058133+abhijitb11@users.noreply.github.com> +Abhijith Jajam <129675815+jajamabhijith@users.noreply.github.com> Abhilash Majumder <30946547+abhilash1910@users.noreply.github.com> Abhinay Krishna Abhiram <78226909+geckguy@users.noreply.github.com> @@ -203,14 +204,17 @@ Ankit Khandelwal Ankur Verma <31362771+ankurvdev@users.noreply.github.com> anon998 <131767832+anon998@users.noreply.github.com> Anri Lombard +Anthony Maltsev <77708730+anthonymaltsev@users.noreply.github.com> Anthony Umfer Anthony Van de Gejuchte Anthony Wang antichristHater <142441588+antichristHater@users.noreply.github.com> Antoine Viallon +Anton I Anton Mitkov Anton Mitkov Antonis Makropoulos +anubhavfujitsu Anudit Nagar Anuj Attri anujj @@ -293,6 +297,7 @@ Bernat Vadell Bernhard M. Wiedemann Bert Wagner Bertay Eren <39909689+bertaye@users.noreply.github.com> +Bhargav Krish Bhavik Sharda <10757940+BLSharda@users.noreply.github.com> bhubbb <79117352+bhubbb@users.noreply.github.com> Bill Sideris @@ -533,6 +538,7 @@ Don Mahurin Dong Won Kim <63934649+ddwkim@users.noreply.github.com> Donghyeon Jeong <54725479+djeong20@users.noreply.github.com> Dongliang Wei <121270393+wdl339@users.noreply.github.com> +Dongsheng Luo Doomsdayrs <38189170+Doomsdayrs@users.noreply.github.com> DooWoong Lee (David) DorianRudolph @@ -612,6 +618,7 @@ Faez Shakil fairydreaming <166155368+fairydreaming@users.noreply.github.com> Faisal Zaghloul Faisal Zaghloul +FamousMonsterr Fan Shupei FantasyGmm <16450052+FantasyGmm@users.noreply.github.com> fanyang @@ -654,6 +661,7 @@ Frederik Vogel Fredrik Hultin fredzillman frob +frogscooper Frost-54 <91168148+Frost-54@users.noreply.github.com> Frosty40 Funtowicz Morgan @@ -694,6 +702,7 @@ Gilad S. <7817232+giladgd@users.noreply.github.com> Giovanni Rivera github-actions[bot] GittyBurstein +Giulianno Giuseppe Scrivano Giuseppe Scrivano GiviMAD @@ -762,6 +771,7 @@ Henry147147 <44851451+Henry147147@users.noreply.github.com> Herman Semenoff Herman Semenov Hesen Peng +hesiru hey-gm HighDoping HimariO @@ -919,6 +929,7 @@ Jesse Ikonen Jesse Jojo Johnson Jesse LaRose Jesse Posner +Jesse Symons <155509533+Torchit1@users.noreply.github.com> Jesus Gulfo Jesus Talavera <145992175+jesus-talavera-ibm@users.noreply.github.com> Jetson Tan @@ -1071,6 +1082,7 @@ Kevin Liu <4396kevinliu@gmail.com> Kevin Lo Kevin Pouget Kevin Wang +Keyu Zhu Khashayar Ghafouri <43180261+khashayarghafouri@users.noreply.github.com> khimaros Kilian Hu <90606809+kilian-hu@users.noreply.github.com> @@ -1078,6 +1090,7 @@ Kilian Krampf kiltyj Kim S. kimminsu <80271594+kimminsu38oo@users.noreply.github.com> +Kirill A. Korinsky KITAITI Makoto kiwi <122582483+kiwi142857@users.noreply.github.com> klosax <131523366+klosax@users.noreply.github.com> @@ -1136,6 +1149,7 @@ leejet Leng Yue Lennart Austenfeld <53152202+l-austenfeld@users.noreply.github.com> leo-pony +Leon <118652405+LeON-Nie-code@users.noreply.github.com> Leon Knauer Leonard Mosescu leonardHONG <2695316095@qq.com> @@ -1146,8 +1160,10 @@ lexasub lgai-exaone lhez lhez +Li Chuyang <1418587903@qq.com> Li Pengzhan <151381994+Lpzhan931@users.noreply.github.com> Li Tan +li-lizhe <82501526+li-lizhe@users.noreply.github.com> liangliangchang liminfei-amd <91481003+liminfei-amd@users.noreply.github.com> limitedAtonement @@ -1166,6 +1182,7 @@ LoganDark Loïc Carrère lon <114724657+longregen@users.noreply.github.com> loonerin <132926317+loonerin@users.noreply.github.com> +Lorenzo Bona losewayy <1134285612@qq.com> LostRuins <39025047+LostRuins@users.noreply.github.com> LostRuins Concedo <39025047+LostRuins@users.noreply.github.com> @@ -1253,6 +1270,7 @@ Masaya, Kato <62578291+msy-kato@users.noreply.github.com> mashdragon <122402293+mashdragon@users.noreply.github.com> Mason Milburn MasterYi1024 <39848311+MasterYi1024@users.noreply.github.com> +Maternion <98753158+maternion@users.noreply.github.com> Mateusz Charytoniuk Matheus C. França Matheus Gabriel Alves Silva @@ -1350,6 +1368,7 @@ Miwa / Ensan <63481257+ensan-hcl@users.noreply.github.com> miyan <1138989048@qq.com> mj-shifu <77107165+mj-shifu@users.noreply.github.com> mkoker <132301062+mkoker@users.noreply.github.com> +mmalpe-nv mmyjona mnehete32 <33429707+mnehete32@users.noreply.github.com> Mohamed Elashri @@ -1368,6 +1387,7 @@ mtmcp <141645996+mtmcp@users.noreply.github.com> muggle-stack Muhammad Saad Muhammad Salem +MuoDoo <36472129+MuoDoo@users.noreply.github.com> Murilo Santana Musab Gultekin musoles <135031143+musoles@users.noreply.github.com> @@ -1416,10 +1436,12 @@ Nicolas Budyn <16596345+nbud@users.noreply.github.com> Nicolas Mowen Nicolás Pérez Nicolò Scipione +Nidhi Nigel Bosch Nigel Jones Nik Bogatyrev <50734723+notforu777@users.noreply.github.com> Nikhil Jain +Nikhil Sharma <150424968+nikhilfujitsu@users.noreply.github.com> Nikita Gordeev Nikita Sarychev <42014488+sARY77@users.noreply.github.com> Niklas Korz @@ -1592,6 +1614,7 @@ Raman Shinde Rand Xie Randall Fitzgerald Random Fly +randomized <286843573@qq.com> rankaiyx rankaiyx RapidMark <32768622+RapidMark@users.noreply.github.com> @@ -2009,6 +2032,7 @@ Tungsten842 Tushar tv1wnd <55383215+tv1wnd@users.noreply.github.com> tyronecai +Uber <20715993+YagUber@users.noreply.github.com> ubergarm ubik2 UEXTM.com <84163508+uextm@users.noreply.github.com> @@ -2242,4 +2266,5 @@ Zijun Yu zql <37731799+zqlcode@users.noreply.github.com> zrm Zsapi +zyang-dev <267119621+zyang-dev@users.noreply.github.com> zzzzwc diff --git a/CMakeLists.txt b/CMakeLists.txt index e37c8aa4cbe7..b8b40e75f68f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -4,7 +4,7 @@ include(CheckIncludeFileCXX) ### llama.cpp version set(LLAMA_VERSION_MAJOR 0) -set(LLAMA_VERSION_MINOR 8) +set(LLAMA_VERSION_MINOR 9) set(LLAMA_VERSION_PATCH 0) set(LLAMA_VERSION_BASE "${LLAMA_VERSION_MAJOR}.${LLAMA_VERSION_MINOR}.${LLAMA_VERSION_PATCH}") diff --git a/CODEOWNERS b/CODEOWNERS index a1e27dec368f..f34a07f02255 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -87,6 +87,7 @@ /src/llama-graph.* @CISC /src/llama-model.* @CISC /src/llama-vocab.* @CISC +/src/llama-moe-cache.* @am17an /src/models/ @ggml-org/llama-models /tests/ @ggerganov /tests/test-chat.* @pwilkin diff --git a/ci/run.sh b/ci/run.sh index 30a6bf329147..1beb54b0759d 100755 --- a/ci/run.sh +++ b/ci/run.sh @@ -180,7 +180,7 @@ if [ ! -z ${GG_BUILD_OPENVINO} ]; then CMAKE_EXTRA="${CMAKE_EXTRA} -DGGML_OPENVINO=ON" # TODO: fix failing tests on OpenVINO backend - CTEST_EXTRA="-E test-llama-archs|test-llama-context" + CTEST_EXTRA="-E test-llama-archs|test-llama-context|test-fp8-kv" fi ## helpers @@ -296,25 +296,6 @@ function gg_run_test_llama_archs_tensor_split { set +e } -# test_llama_archs_models - -function gg_run_test_llama_archs_models { - cd ${SRC} - - set -e - - # TODO: fix and re-enable `test-llama-archs` on OpenVINO - # TODO: the `test-llama-archs` currently does not build on Windows, so we check if the binary exists - if [ -z ${GG_BUILD_OPENVINO} ] && [ -f ./build-ci-release/bin/test-llama-archs ]; then - rm -rf build-ci-models && mkdir -p build-ci-models - - # generate the dummy models used by the model-dependent tests - ./build-ci-release/bin/test-llama-archs -o build-ci-models 2>&1 - fi - - set +e -} - # test_scripts function gg_run_test_scripts { @@ -456,12 +437,8 @@ function gg_run_qwen3_0_6b { (time ./bin/llama-imatrix --model ${model_f16} -f ${wiki_test} -ngl 99 -c 1024 -b 512 --chunks 2 ) 2>&1 | tee -a $OUT/${ci}-imatrix.log - # TODO: these tests are extremely slow on WebGPU - re-enable when fixed - # https://github.com/ggml-org/llama.cpp/actions/runs/39639985732 - if [ -z ${GG_BUILD_WEBGPU} ]; then - (time ./bin/test-llama-context --model ${model_q4_0} -ngl 10 -c 1024 -fa off --no-op-offload) 2>&1 | tee -a $OUT/${ci}-test-llama-context.log - (time ./bin/test-llama-context --model ${model_q4_0} -ngl 99 -c 1024 -fa off ) 2>&1 | tee -a $OUT/${ci}-test-llama-context.log - fi + (time ./bin/test-llama-context --model ${model_q4_0} -ngl 10 -c 1024 -fa off --no-op-offload) 2>&1 | tee -a $OUT/${ci}-test-llama-context.log + (time ./bin/test-llama-context --model ${model_q4_0} -ngl 99 -c 1024 -fa off ) 2>&1 | tee -a $OUT/${ci}-test-llama-context.log (time ./bin/test-llama-context --model ${model_q4_0} -ngl 10 -c 1024 -fa on --no-op-offload) 2>&1 | tee -a $OUT/${ci}-test-llama-context.log (time ./bin/test-llama-context --model ${model_q4_0} -ngl 99 -c 1024 -fa on ) 2>&1 | tee -a $OUT/${ci}-test-llama-context.log @@ -701,7 +678,6 @@ test $ret -eq 0 && gg_run ctest_release test $ret -eq 0 && gg_run test_backend_ops -test $ret -eq 0 && gg_run test_llama_archs_models test $ret -eq 0 && gg_run test_llama_archs_tensor_split if [ -z ${GG_BUILD_LOW_PERF} ]; then diff --git a/cmake/download-models.cmake b/cmake/download-models.cmake index de252906a01f..57c141265be7 100644 --- a/cmake/download-models.cmake +++ b/cmake/download-models.cmake @@ -2,11 +2,11 @@ get_filename_component(DEST_DIR "${DEST}" DIRECTORY) file(MAKE_DIRECTORY "${DEST_DIR}") if(NOT EXISTS "${DEST}") - message(STATUS "Downloading ${NAME} from ggml-org/models...") + message(STATUS "Downloading ${NAME} from ${REPO}...") endif() file(DOWNLOAD - "https://huggingface.co/ggml-org/models/resolve/main/${NAME}?download=true" + "https://huggingface.co/${REPO}/resolve/main/${NAME}?download=true" "${DEST}" TLS_VERIFY ON EXPECTED_HASH ${HASH} diff --git a/cmake/win-app.cmake b/cmake/win-app.cmake index 5456e48fd2c6..cf03089ba202 100644 --- a/cmake/win-app.cmake +++ b/cmake/win-app.cmake @@ -13,7 +13,7 @@ if(WIN32) endif() enable_language(RC) set(LLAMA_WIN_APP_MANIFEST_RC "${CMAKE_CURRENT_BINARY_DIR}/win-app-manifest.rc") - configure_file("${CMAKE_CURRENT_LIST_DIR}/win-app.rc.in" "${LLAMA_WIN_APP_MANIFEST_RC}" @ONLY) + file(CONFIGURE OUTPUT "${LLAMA_WIN_APP_MANIFEST_RC}" CONTENT "1 24 \"@LLAMA_WIN_APP_MANIFEST@\"\n" @ONLY) set_property(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS "${LLAMA_WIN_APP_MANIFEST}") endif() endif() diff --git a/cmake/win-app.rc.in b/cmake/win-app.rc.in deleted file mode 100644 index 38288a5f848d..000000000000 --- a/cmake/win-app.rc.in +++ /dev/null @@ -1,3 +0,0 @@ -// generated from cmake/win-app.rc.in - do not edit -// RT_MANIFEST (24), ID 1: the application manifest shared by all llama.cpp executables -1 24 "@LLAMA_WIN_APP_MANIFEST@" diff --git a/common/CMakeLists.txt b/common/CMakeLists.txt index 8e3c88dac9dc..e38485ca017c 100644 --- a/common/CMakeLists.txt +++ b/common/CMakeLists.txt @@ -66,6 +66,8 @@ add_library(${TARGET} chat-diff-analyzer.cpp chat-peg-parser.cpp chat-peg-parser.h + chat-tts-norm.cpp + chat-tts-norm.h chat.cpp chat.h common.cpp diff --git a/common/arg.cpp b/common/arg.cpp index e3ab084c67f9..6bf9355fb0ff 100644 --- a/common/arg.cpp +++ b/common/arg.cpp @@ -4,6 +4,7 @@ #include "chat.h" #include "common.h" #include "download.h" +#include "http.h" #include "json-schema-to-grammar.h" #include "json.h" #include "llama.h" @@ -281,6 +282,7 @@ const std::vector kv_cache_types = { GGML_TYPE_F32, GGML_TYPE_F16, GGML_TYPE_BF16, + GGML_TYPE_F8_E4M3, GGML_TYPE_Q8_0, GGML_TYPE_Q4_0, GGML_TYPE_Q4_1, @@ -399,7 +401,26 @@ bool common_models_handler_is_preset_repo(const common_models_handler & handler) return !handler.plan.preset.url.empty(); } +static common_download_opts download_opts_for_url(const common_download_opts & opts, const std::string & url) { + common_download_opts result = opts; + try { + const auto hf = common_http_parse_url(common_get_model_endpoint()); + const auto parts = common_http_parse_url(url); + + if (parts.scheme == hf.scheme && + parts.host == hf.host && + parts.port == hf.port && + string_starts_with(result.bearer_token, "hf_")) { + return result; + } + } catch (const std::exception &) { + } + result.bearer_token.clear(); + return result; +} + static std::vector build_url_tasks(const common_params_model & model, common_download_opts opts) { + opts = download_opts_for_url(opts, model.url); auto parts = common_download_get_all_parts(model.url); std::vector tasks; @@ -488,7 +509,7 @@ void common_models_handler_apply(common_models_handler & handler, common_params common_download_task task; task.url = params.mmproj.url; task.local_path = params.mmproj.path; - task.opts = opts; + task.opts = download_opts_for_url(opts, task.url); tasks.push_back(task); } bool had_spec_url = false; @@ -496,7 +517,7 @@ void common_models_handler_apply(common_models_handler & handler, common_params common_download_task task; task.url = params.speculative.draft.mparams.url; task.local_path = params.speculative.draft.mparams.path; - task.opts = opts; + task.opts = download_opts_for_url(opts, task.url); tasks.push_back(task); had_spec_url = true; } @@ -710,7 +731,7 @@ static void common_params_apply_system_config(common_params & params, llama_exam std::vector paths; #if defined(_WIN32) - const std::filesystem::path program_data = common_get_path_from_env("PROGRAMDATA"); + const std::filesystem::path program_data = common_get_env("PROGRAMDATA"); if (!program_data.empty()) { paths.push_back(program_data / "llama.cpp" / "config.ini"); } @@ -922,8 +943,8 @@ static bool common_params_parse_ex(int argc, char ** argv, common_params_context } const bool mcp_enabled = !params.mcp_servers_config.empty() || !params.mcp_servers_json.empty(); - if ((!params.server_tools.empty() || mcp_enabled) && !params.cors_origins_explicit) { - LOG_WRN("server tools or MCP servers are enabled, using localhost as default CORS origin (change via --cors-origins)\n"); + if ((!params.server_tools.empty() || mcp_enabled || params.ui_mcp_proxy) && !params.cors_origins_explicit) { + LOG_WRN("server tools, MCP servers or MCP proxy are enabled, using localhost as default CORS origin (change via --cors-origins)\n"); params.cors_origins = "localhost"; } @@ -2480,15 +2501,6 @@ common_params_context common_params_parser_init(common_params & params, llama_ex params.ppl_output_type = value; } ).set_examples({LLAMA_EXAMPLE_PERPLEXITY})); - add_opt(common_arg( - {"-dt", "--defrag-thold"}, "N", - string_format("KV cache defragmentation threshold (DEPRECATED)"), - [](common_params & params, const std::string & value) { - GGML_UNUSED(params); - GGML_UNUSED(value); - LOG_WRN("DEPRECATED: --defrag-thold is deprecated and no longer necessary to specify\n"); - } - ).set_env("LLAMA_ARG_DEFRAG_THOLD")); if (ex == LLAMA_EXAMPLE_SERVER) { // this is to make sure this option appears in the server-specific section of the help message add_opt(common_arg( @@ -2734,6 +2746,17 @@ common_params_context common_params_parser_init(common_params & params, llama_ex params.moe_cache_size = (size_t) value*1024*1024; } ).set_env("LLAMA_ARG_MOE_CACHE_MIB")); + add_opt(common_arg( + {"--moe-cache-cpu-share"}, "F", + string_format("share of the MoE cache misses that the CPU computes from the host experts instead of uploading them to the GPU once the cache is full, from 0 to 1 (default: %.2f)", (double) params.moe_cache_cpu_share), + [](common_params & params, const std::string & value) { + const float share = std::stof(value); + if (share < 0.0f || share > 1.0f) { + throw std::invalid_argument("invalid value"); + } + params.moe_cache_cpu_share = share; + } + ).set_env("LLAMA_ARG_MOE_CACHE_CPU_SHARE")); add_opt(common_arg( {"-ncffn", "--n-cpu-ffn"}, "N", "keep the dense FFN weights of the first N layers in the CPU\n" @@ -3001,6 +3024,16 @@ common_params_context common_params_parser_init(common_params & params, llama_ex params.model.path = value; } ).set_examples({LLAMA_EXAMPLE_COMMON, LLAMA_EXAMPLE_EXPORT_LORA, LLAMA_EXAMPLE_DOWNLOAD, LLAMA_EXAMPLE_TOKENIZE}).set_env("LLAMA_ARG_MODEL")); + add_opt(common_arg( + {"--ple", "--shard"}, "FNAME,...", + "extra GGUF shards for the model, comma separated; may be given more than once\n" + "paths may live anywhere, including on different drives; --ple is an alias for --shard", + [](common_params & params, const std::string & value) { + for (const auto & item : parse_csv_row(value)) { + params.model.shards.push_back(item); + } + } + ).set_examples({LLAMA_EXAMPLE_COMMON}).set_env("LLAMA_ARG_SHARD")); add_opt(common_arg( {"-mu", "--model-url"}, "MODEL_URL", "model download url (default: unused)", @@ -3319,7 +3352,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex {"--cors-origins"}, "ORIGINS", string_format( "comma-separated list of allowed origins for CORS (default: %s)\n" - "if set to special value 'localhost', reflect the Origin header only if it is localhost", + "if set to special value 'localhost', only allow same-origin requests (e.g. the built-in UI)", params.cors_origins.c_str()), [](common_params & params, const std::string & value) { params.cors_origins = value; @@ -3375,7 +3408,8 @@ common_params_context common_params_parser_init(common_params & params, llama_ex add_opt(common_arg( {"--ui-mcp-proxy", "--webui-mcp-proxy"}, {"--no-ui-mcp-proxy", "--no-webui-mcp-proxy"}, - "experimental: whether to enable MCP CORS proxy - do not enable in untrusted environments (default: disabled)", + "experimental: whether to enable MCP CORS proxy, for the Web UI to reach public MCP servers without CORS headers; local and private addresses are refused - do not enable in untrusted environments (default: disabled)\n" + "note: for security reasons, this will limit --cors-origins to localhost by default", [](common_params & params, bool value) { params.ui_mcp_proxy = value; } @@ -3401,6 +3435,13 @@ common_params_context common_params_parser_init(common_params & params, llama_ex params.server_tools_runtime = value; } ).set_examples({LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_ARG_TOOLS_RUNTIME")); + add_opt(common_arg( + {"--allow-unauthenticated-tools"}, + "allow server tools, MCP servers or MCP proxy on a non-loopback address without an API key - only for trusted networks (default: disabled)", + [](common_params & params) { + params.server_tools_allow_unauth = true; + } + ).set_examples({LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_ARG_ALLOW_UNAUTHENTICATED_TOOLS")); add_opt(common_arg( {"--mcp-servers-config"}, "PATH", "experimental: path to JSON file with MCP server definitions (Cursor-compatible format) - do not enable in untrusted environments (default: none)\n" @@ -3626,6 +3667,13 @@ common_params_context common_params_parser_init(common_params & params, llama_ex } } ).set_examples({LLAMA_EXAMPLE_SERVER})); + add_opt(common_arg( + {"--media-allow-private-urls"}, + "allow http(s) media URLs (e.g. image_url) that point to local or private addresses (default: disabled, only public addresses)", + [](common_params & params) { + params.media_allow_private_urls = true; + } + ).set_examples({LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_ARG_MEDIA_ALLOW_PRIVATE_URLS")); add_opt(common_arg( {"--models-dir"}, "PATH", "directory containing models for the router server (default: disabled)", @@ -4450,7 +4498,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex [](common_params & params, const std::string & value) { params.tts_lang = value; } - ).set_examples({LLAMA_EXAMPLE_TTS})); + ).set_examples({LLAMA_EXAMPLE_TTS, LLAMA_EXAMPLE_SERVER})); add_opt(common_arg( {"--tts-voice"}, "VOICE", "voice for audio generation (model-dependent)\n" @@ -4458,7 +4506,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex [](common_params & params, const std::string & value) { params.tts_voice = value; } - ).set_examples({LLAMA_EXAMPLE_TTS})); + ).set_examples({LLAMA_EXAMPLE_TTS, LLAMA_EXAMPLE_SERVER})); add_opt(common_arg( {"--tts-speaker-file"}, "FNAME", "speaker file path for audio generation", diff --git a/common/chat-peg-parser.cpp b/common/chat-peg-parser.cpp index 74b81dcddb27..968cbfa98f6c 100644 --- a/common/chat-peg-parser.cpp +++ b/common/chat-peg-parser.cpp @@ -131,6 +131,12 @@ static std::string normalize_quotes_to_json(const std::string & input) { continue; } + // Models can emit raw control characters inside string literals; escape them to keep the JSON valid. + if ((in_single_quoted || in_double_quoted) && static_cast(c) < 0x20) { + result += escape_json_string_inner(std::string(1, c)); + continue; + } + // Handle quote characters if (c == '"') { if (in_single_quoted) { @@ -348,7 +354,8 @@ void common_chat_peg_mapper::map(const common_peg_ast_node & node) { } if (is_tool_id && current_tool) { - auto text = trim_trailing_space(node.text); + auto sanitized = node.sanitized_text(); + auto text = trim_trailing_space(sanitized); if (text.size() >= 2 && text.front() == '"' && text.back() == '"') { text = text.substr(1, text.size() - 2); } @@ -401,7 +408,7 @@ void common_chat_peg_mapper::map(const common_peg_ast_node & node) { } if ((is_arg_value || is_arg_string_value) && current_tool) { - std::string value_content = std::string(node.text); + std::string value_content = node.sanitized_text(); std::string value_to_add; if (value_content.empty() && is_arg_string_value) { @@ -966,6 +973,21 @@ void common_chat_peg_gemma4_mapper::from_ast(const common_peg_ast_arena & arena, } } +using gemma4 = common_chat_peg_gemma4_mapper; + +// Collects the nearest descendants carrying tag, looking through the rules the parser +// wraps around the tagged nodes. +static void gemma4_collect(const common_peg_ast_arena & arena, const common_peg_ast_node & node, const std::string & tag, std::vector & out) { + for (auto child_id : node.children) { + const auto & child = arena.get(child_id); + if (child.tag == tag) { + out.push_back(child_id); + } else { + gemma4_collect(arena, child, tag, out); + } + } +} + static std::string gemma4_to_json(const common_peg_ast_arena & arena, common_peg_ast_id id) { const auto & node = arena.get(id); @@ -973,51 +995,45 @@ static std::string gemma4_to_json(const common_peg_ast_arena & arena, common_peg return ""; } - if (node.rule == "gemma4-number" || node.rule == "gemma4-bool" || node.rule == "gemma4-null") { + if (node.tag == gemma4::NUMBER || node.tag == gemma4::BOOL || node.tag == gemma4::NULL_) { return std::string(node.text); } - if (node.rule == "gemma4-string-content") { - return escape_json_string_inner(std::string(node.text)); + if (node.tag == gemma4::STRING_CONTENT) { + return escape_json_string_inner(node.sanitized_text()); } - if (node.rule == "gemma4-string") { + if (node.tag == gemma4::STRING) { std::string result = "\""; - if (!node.children.empty()) { - result += gemma4_to_json(arena, node.children[0]); - if (!node.is_partial) { - result += "\""; - } + for (auto child_id : node.children) { + result += gemma4_to_json(arena, child_id); + } + if (!node.is_partial) { + result += "\""; } return result; } - if (node.rule == "gemma4-array") { + if (node.tag == gemma4::ARRAY) { std::string result = "["; - - bool add_comma = false; - for (auto child_id : node.children) { - if (add_comma) { + for (size_t i = 0; i < node.children.size(); i++) { + if (i > 0) { result += ','; } - add_comma = true; - result += gemma4_to_json(arena, child_id); + result += gemma4_to_json(arena, node.children[i]); } - if (!node.is_partial) { result += ']'; } return result; } - if (node.rule == "gemma4-dict-key-name") { - return std::string(node.text); - } + if (node.tag == gemma4::DICT_KEY) { + auto name_id = arena.find_by_tag(node, gemma4::DICT_KEY_NAME); - if (node.rule == "gemma4-dict-key") { std::string result = "\""; - if (!node.children.empty()) { - result += escape_json_string_inner(gemma4_to_json(arena, node.children[0])); + if (name_id != COMMON_PEG_INVALID_AST_ID) { + result += escape_json_string_inner(std::string(arena.get(name_id).text)); } if (!node.is_partial) { result += "\":"; @@ -1025,40 +1041,29 @@ static std::string gemma4_to_json(const common_peg_ast_arena & arena, common_peg return result; } - if (node.rule == "gemma4-dict-kv") { - std::string result; - for (auto child_id : node.children) { - result += gemma4_to_json(arena, child_id); - } - return result; - } + if (node.tag == gemma4::DICT) { + std::vector members; + gemma4_collect(arena, node, gemma4::DICT_KV, members); - if (node.rule == "gemma4-dict") { std::string result = "{"; - - bool add_comma = false; - for (auto child_id : node.children) { - if (add_comma) { + for (size_t i = 0; i < members.size(); i++) { + if (i > 0) { result += ','; } - add_comma = true; - result += gemma4_to_json(arena, child_id); + result += gemma4_to_json(arena, members[i]); } - if (!node.is_partial) { result += '}'; } return result; } - if (node.rule == "gemma4-value") { - if (!node.children.empty()) { - return gemma4_to_json(arena, node.children[0]); - } - return ""; + // members, rules and other wrappers + std::string result; + for (auto child_id : node.children) { + result += gemma4_to_json(arena, child_id); } - - return ""; + return result; } void common_chat_peg_gemma4_mapper::visit(const common_peg_ast_arena & arena, common_peg_ast_id id) { @@ -1195,7 +1200,7 @@ static std::string minimax_m3_value_to_json(const common_peg_ast_arena & arena, } if (node.tag == common_chat_peg_builder::TOOL_ARG_STRING_VALUE) { - return "\"" + escape_json_string_inner(std::string(node.text)) + (closed ? "\"" : ""); + return "\"" + escape_json_string_inner(node.sanitized_text()) + (closed ? "\"" : ""); } // Numbers and booleans are written verbatim by the template diff --git a/common/chat-peg-parser.h b/common/chat-peg-parser.h index 114fa049fa74..b827b60b059c 100644 --- a/common/chat-peg-parser.h +++ b/common/chat-peg-parser.h @@ -34,6 +34,18 @@ class common_chat_peg_mapper { class common_chat_peg_gemma4_mapper : public common_chat_peg_mapper { public: + // Tags of the Gemma 4 argument syntax. The parser may wrap them in any rules. + static constexpr const char * STRING = "gemma4-string"; + static constexpr const char * STRING_CONTENT = "gemma4-string-content"; + static constexpr const char * NUMBER = "gemma4-number"; + static constexpr const char * BOOL = "gemma4-bool"; + static constexpr const char * NULL_ = "gemma4-null"; + static constexpr const char * ARRAY = "gemma4-array"; + static constexpr const char * DICT = "gemma4-dict"; + static constexpr const char * DICT_KV = "gemma4-dict-kv"; + static constexpr const char * DICT_KEY = "gemma4-dict-key"; + static constexpr const char * DICT_KEY_NAME = "gemma4-dict-key-name"; + common_chat_peg_gemma4_mapper(common_chat_msg & msg) : common_chat_peg_mapper(msg) {} virtual void from_ast(const common_peg_ast_arena & arena, const common_peg_parse_result & result); private: diff --git a/common/chat-tts-norm.cpp b/common/chat-tts-norm.cpp new file mode 100644 index 000000000000..f84540ff96e0 --- /dev/null +++ b/common/chat-tts-norm.cpp @@ -0,0 +1,864 @@ +#include "chat-tts-norm.h" + +#include "jinja/runtime.h" +#include "jinja/value.h" + +#include +#include +#include +#include +#include + +// TTS normalization patterns, see docs/tts-norm-patterns.md for the specification +// keep this file in sync with the python reference: scripts/chat_tts_norm.py + +using namespace jinja; + +// +// character classes +// they are defined with explicit codepoint ranges, independent of any unicode database version, +// so that the C++ and python implementations agree on every codepoint +// + +struct tts_norm_range { + uint32_t lo; + uint32_t hi; +}; + +static const tts_norm_range TTS_NORM_SPACE[] = { + {0x0009, 0x000D}, {0x0020, 0x0020}, {0x0085, 0x0085}, {0x00A0, 0x00A0}, {0x1680, 0x1680}, + {0x2000, 0x200A}, {0x2028, 0x2029}, {0x202F, 0x202F}, {0x205F, 0x205F}, {0x3000, 0x3000}, +}; + +static const tts_norm_range TTS_NORM_PUNCT[] = { + {0x0021, 0x002F}, {0x003A, 0x0040}, {0x005B, 0x0060}, {0x007B, 0x007E}, + {0x00A1, 0x00A9}, {0x00AB, 0x00B4}, {0x00B6, 0x00B9}, {0x00BB, 0x00BF}, {0x00D7, 0x00D7}, {0x00F7, 0x00F7}, + {0x2010, 0x2027}, {0x2030, 0x205E}, {0x20A0, 0x20CF}, {0x2100, 0x218F}, {0x2190, 0x2BFF}, + {0x3001, 0x3003}, {0x3008, 0x3020}, {0x3030, 0x3030}, {0x303D, 0x303D}, {0x30FB, 0x30FB}, + {0xFE10, 0xFE19}, {0xFE30, 0xFE4F}, {0xFE50, 0xFE6B}, + {0xFF01, 0xFF0F}, {0xFF1A, 0xFF20}, {0xFF3B, 0xFF40}, {0xFF5B, 0xFF65}, + {0x1F000, 0x1FAFF}, +}; + +static const tts_norm_range TTS_NORM_CONTROL[] = { + {0x0000, 0x001F}, {0x007F, 0x009F}, {0x200B, 0x200F}, {0x2060, 0x206F}, {0xFEFF, 0xFEFF}, +}; + +static const tts_norm_range TTS_NORM_HAN[] = { + {0x2E80, 0x2FDF}, {0x3005, 0x3005}, {0x3007, 0x3007}, {0x3021, 0x3029}, {0x3038, 0x303B}, + {0x3400, 0x4DBF}, {0x4E00, 0x9FFF}, {0xF900, 0xFAFF}, {0x20000, 0x323AF}, +}; + +template +static bool tts_norm_in(const tts_norm_range (&ranges)[N], uint32_t c) { + for (const auto & r : ranges) { + if (c >= r.lo && c <= r.hi) { + return true; + } + } + return false; +} + +static bool tts_norm_is_alpha(uint32_t c) { + if ((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z')) { + return true; + } + if (c == 0xAA || c == 0xB5 || c == 0xBA) { + return true; + } + return c >= 0xC0 && !tts_norm_in(TTS_NORM_SPACE, c) && !tts_norm_in(TTS_NORM_PUNCT, c) && !tts_norm_in(TTS_NORM_CONTROL, c); +} + +// kind is one of "dspawulhc" +static bool tts_norm_class_match(char kind, uint32_t c) { + switch (kind) { + case 'd': return c >= '0' && c <= '9'; + case 's': return tts_norm_in(TTS_NORM_SPACE, c); + case 'p': return tts_norm_in(TTS_NORM_PUNCT, c); + case 'a': return tts_norm_is_alpha(c); + case 'w': return tts_norm_is_alpha(c) || (c >= '0' && c <= '9'); + case 'u': return c >= 'A' && c <= 'Z'; + case 'l': return c >= 'a' && c <= 'z'; + case 'h': return tts_norm_in(TTS_NORM_HAN, c); + case 'c': return tts_norm_in(TTS_NORM_CONTROL, c); + } + return false; +} + +static bool tts_norm_is_class_letter(uint32_t c) { + if (c >= 'A' && c <= 'Z') { + c += 'a' - 'A'; + } + return c < 0x80 && std::string("dspawulhc").find(static_cast(c)) != std::string::npos; +} + +static uint32_t tts_norm_ascii_lower(uint32_t c) { + return (c >= 'A' && c <= 'Z') ? c + ('a' - 'A') : c; +} + +static uint32_t tts_norm_ascii_swapcase(uint32_t c) { + if (c >= 'A' && c <= 'Z') return c + ('a' - 'A'); + if (c >= 'a' && c <= 'z') return c - ('a' - 'A'); + return c; +} + +static bool tts_norm_is_ascii_alnum(uint32_t c) { + return (c >= '0' && c <= '9') || (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z'); +} + +// invalid bytes are decoded as U+FFFD, one per byte; offs[i] is the byte offset of codepoint i, offs[n] = text size +static void tts_norm_decode_utf8(const std::string & s, std::vector & cpts, std::vector & offs) { + cpts.clear(); + offs.clear(); + size_t i = 0; + const size_t n = s.size(); + while (i < n) { + offs.push_back(i); + const unsigned char c = static_cast(s[i]); + uint32_t cp = 0; + size_t len = 0; + if (c < 0x80) { + cp = c; len = 1; + } else if ((c & 0xE0) == 0xC0) { + cp = c & 0x1F; len = 2; + } else if ((c & 0xF0) == 0xE0) { + cp = c & 0x0F; len = 3; + } else if ((c & 0xF8) == 0xF0) { + cp = c & 0x07; len = 4; + } + bool ok = len > 0 && i + len <= n; + for (size_t k = 1; ok && k < len; k++) { + const unsigned char cc = static_cast(s[i + k]); + if ((cc & 0xC0) != 0x80) { + ok = false; + } else { + cp = (cp << 6) | (cc & 0x3F); + } + } + if (!ok) { + cpts.push_back(0xFFFD); + i += 1; + } else { + cpts.push_back(cp); + i += len; + } + } + offs.push_back(n); +} + +// +// AST +// + +enum class tts_norm_op { + LITERAL, // one codepoint + SET, // one codepoint from a set; class escapes like %d are sets with a single class + ANY, // any codepoint + START, // ^ + END, // $ + BOUNDARY, // %b + GROUP, // (...), (?:...), (?...) + LOOKAHEAD, // (?=...), (?!...) + LOOKBEHIND, // (?<=c), (?> ranges; + std::vector classes; +}; + +struct tts_norm_item; +using tts_norm_seq = std::vector; + +struct tts_norm_pattern_node { + tts_norm_op op = tts_norm_op::GROUP; + uint32_t cp = 0; // LITERAL + tts_norm_set set; // SET + std::vector alts; // GROUP, LOOKAHEAD; LOOKBEHIND holds a single item + int capture = -1; // GROUP: capture index, -1 for non-capturing + bool neg = false; // LOOKAHEAD, LOOKBEHIND +}; + +static constexpr int TTS_NORM_INF = INT_MAX; +static constexpr int TTS_NORM_MAX_REPEAT = 1000; +static constexpr size_t TTS_NORM_MAX_PATTERN_LEN = 4096; + +struct tts_norm_item { + tts_norm_pattern_node atom; + int min = 1; + int max = 1; +}; + +static bool tts_norm_is_zero_width(tts_norm_op op) { + return op == tts_norm_op::START || op == tts_norm_op::END || op == tts_norm_op::BOUNDARY + || op == tts_norm_op::LOOKAHEAD || op == tts_norm_op::LOOKBEHIND; +} + +// +// parser +// + +struct tts_norm_parser { + std::vector s; + size_t p = 0; + size_t & n_caps; + + tts_norm_parser(std::vector src, size_t & n_caps) : s(std::move(src)), n_caps(n_caps) {} + + [[noreturn]] void fail(const std::string & msg) const { + throw std::runtime_error("invalid TTS norm pattern: " + msg + " at position " + std::to_string(p)); + } + + bool eof() const { return p >= s.size(); } + uint32_t peek() const { return s[p]; } + bool peek_is(uint32_t c) const { return !eof() && s[p] == c; } + + void expect(uint32_t c, const char * what) { + if (!peek_is(c)) { + fail(std::string("expected ") + what); + } + p++; + } + + std::vector parse_alts() { + std::vector alts; + alts.push_back(parse_seq()); + while (peek_is('|')) { + p++; + alts.push_back(parse_seq()); + } + return alts; + } + + tts_norm_seq parse_seq() { + tts_norm_seq seq; + while (!eof() && peek() != '|' && peek() != ')') { + tts_norm_item item; + item.atom = parse_atom(); + parse_quantifier(item); + seq.push_back(std::move(item)); + } + return seq; + } + + int parse_int() { + if (eof() || peek() < '0' || peek() > '9') { + fail("expected number in quantifier"); + } + int v = 0; + while (!eof() && peek() >= '0' && peek() <= '9') { + v = v * 10 + static_cast(peek() - '0'); + if (v > TTS_NORM_MAX_REPEAT) { + fail("repetition count exceeds " + std::to_string(TTS_NORM_MAX_REPEAT)); + } + p++; + } + return v; + } + + void parse_quantifier(tts_norm_item & item) { + if (eof()) { + return; + } + const uint32_t c = peek(); + if (c != '?' && c != '*' && c != '+' && c != '{') { + return; + } + if (tts_norm_is_zero_width(item.atom.op)) { + fail("zero-width assertion cannot be repeated"); + } + p++; + if (c == '?') { + item.min = 0; item.max = 1; + } else if (c == '*') { + item.min = 0; item.max = TTS_NORM_INF; + } else if (c == '+') { + item.min = 1; item.max = TTS_NORM_INF; + } else { + item.min = parse_int(); + item.max = item.min; + if (peek_is(',')) { + p++; + item.max = peek_is('}') ? TTS_NORM_INF : parse_int(); + } + expect('}', "'}' to close quantifier"); + if (item.max < item.min) { + fail("quantifier max is smaller than min"); + } + } + if (!eof() && (peek() == '?' || peek() == '*' || peek() == '+' || peek() == '{')) { + fail("multiple quantifiers"); + } + } + + tts_norm_pattern_node parse_group() { + // '(' already consumed + tts_norm_pattern_node node; + node.op = tts_norm_op::GROUP; + if (peek_is('?')) { + p++; + if (eof()) { + fail("unterminated group"); + } + const uint32_t c = peek(); + if (c == ':') { + p++; + } else if (c == '=' || c == '!') { + p++; + node.op = tts_norm_op::LOOKAHEAD; + node.neg = c == '!'; + } else if (c == '<' && p + 1 < s.size() && (s[p + 1] == '=' || s[p + 1] == '!')) { + node.op = tts_norm_op::LOOKBEHIND; + node.neg = s[p + 1] == '!'; + p += 2; + tts_norm_item item; + if (eof() || peek() == ')') { + fail("lookbehind must contain exactly one codepoint matcher"); + } + item.atom = parse_atom(); + if (item.atom.op != tts_norm_op::LITERAL && item.atom.op != tts_norm_op::SET && item.atom.op != tts_norm_op::ANY) { + fail("lookbehind must contain exactly one codepoint matcher"); + } + if (!peek_is(')')) { + fail("lookbehind must contain exactly one codepoint matcher"); + } + p++; + node.alts.push_back(tts_norm_seq{}); + node.alts[0].push_back(std::move(item)); + return node; + } else if (c == 'i') { + fail("(?i) is only allowed at the start of the pattern"); + } else { + fail("unknown group type"); + } + } else { + node.capture = static_cast(n_caps++); + } + node.alts = parse_alts(); + expect(')', "')' to close group"); + return node; + } + + // after '%' inside or outside of a set + // returns true if it is a class (stored in cls), false if it is a literal (stored in lit) + bool parse_escape(bool in_set, tts_norm_class & cls, uint32_t & lit, bool & boundary) { + boundary = false; + if (eof()) { + fail("dangling '%'"); + } + const uint32_t e = s[p++]; + if (tts_norm_is_class_letter(e)) { + cls.kind = static_cast(tts_norm_ascii_lower(e)); + cls.neg = e >= 'A' && e <= 'Z'; + return true; + } + if (e == 'b' && !in_set) { + boundary = true; + return false; + } + if (tts_norm_is_ascii_alnum(e)) { + p--; + fail(std::string("unknown escape '%") + static_cast(e) + "'"); + } + lit = e; + return false; + } + + tts_norm_pattern_node parse_set() { + // '[' already consumed + tts_norm_pattern_node node; + node.op = tts_norm_op::SET; + if (peek_is('^')) { + node.set.neg = true; + p++; + } + bool empty = true; + while (true) { + if (eof()) { + fail("unterminated set"); + } + if (peek() == ']') { + p++; + break; + } + empty = false; + uint32_t lo = 0; + if (peek() == '%') { + p++; + tts_norm_class cls{}; + bool boundary = false; + if (parse_escape(true, cls, lo, boundary)) { + node.set.classes.push_back(cls); + continue; + } + } else { + lo = s[p++]; + } + uint32_t hi = lo; + if (peek_is('-') && p + 1 < s.size() && s[p + 1] != ']') { + p++; + if (peek() == '%') { + p++; + tts_norm_class cls{}; + bool boundary = false; + if (parse_escape(true, cls, hi, boundary)) { + fail("character class cannot be used in a range"); + } + } else { + hi = s[p++]; + } + if (hi < lo) { + fail("invalid range in set"); + } + } + node.set.ranges.emplace_back(lo, hi); + } + if (empty) { + fail("empty set"); + } + return node; + } + + tts_norm_pattern_node parse_atom() { + const uint32_t c = s[p++]; + tts_norm_pattern_node node; + switch (c) { + case '(': + return parse_group(); + case '[': + return parse_set(); + case '.': + node.op = tts_norm_op::ANY; + return node; + case '^': + node.op = tts_norm_op::START; + return node; + case '$': + node.op = tts_norm_op::END; + return node; + case '%': { + tts_norm_class cls{}; + uint32_t lit = 0; + bool boundary = false; + if (parse_escape(false, cls, lit, boundary)) { + node.op = tts_norm_op::SET; + node.set.classes.push_back(cls); + } else if (boundary) { + node.op = tts_norm_op::BOUNDARY; + } else { + node.op = tts_norm_op::LITERAL; + node.cp = lit; + } + return node; + } + case '?': + case '*': + case '+': + case '{': + p--; + fail("nothing to repeat"); + case ']': + case '}': + p--; + fail(std::string("unescaped '") + static_cast(c) + "', use '%" + static_cast(c) + "'"); + default: + node.op = tts_norm_op::LITERAL; + node.cp = c; + return node; + } + } +}; + +// +// matcher +// PEG-style: alternatives are ordered (the first one that matches wins) and quantifiers are greedy and possessive +// (they never give back what they consumed), so there is no backtracking +// + +struct tts_norm_matcher { + const std::vector & t; + bool icase; + std::vector> caps; + + bool is_w(size_t i) const { + return i < t.size() && tts_norm_class_match('w', t[i]); + } + + bool set_match(const tts_norm_set & st, uint32_t c) const { + auto test = [&](uint32_t x) { + for (const auto & r : st.ranges) { + if (x >= r.first && x <= r.second) { + return true; + } + } + for (const auto & k : st.classes) { + if (tts_norm_class_match(k.kind, x) != k.neg) { + return true; + } + } + return false; + }; + bool m = test(c); + if (!m && icase) { + const uint32_t o = tts_norm_ascii_swapcase(c); + if (o != c) { + m = test(o); + } + } + return m != st.neg; + } + + bool match_alts(const std::vector & alts, size_t i, size_t & end) { + for (const auto & seq : alts) { + auto saved = caps; + if (match_seq(seq, i, end)) { + return true; + } + caps = std::move(saved); + } + return false; + } + + bool match_seq(const tts_norm_seq & seq, size_t i, size_t & end) { + size_t pos = i; + for (const auto & it : seq) { + int count = 0; + while (count < it.max) { + const bool is_group = it.atom.op == tts_norm_op::GROUP; + std::vector> saved; + if (is_group) { + saved = caps; + } + size_t e = pos; + if (!match_once(it.atom, pos, e)) { + if (is_group) { + caps = std::move(saved); + } + break; + } + if (e == pos) { + // an empty iteration would repeat forever, it satisfies any remaining minimum + count = std::max(count + 1, it.min); + break; + } + pos = e; + count++; + } + if (count < it.min) { + return false; + } + } + end = pos; + return true; + } + + bool match_once(const tts_norm_pattern_node & node, size_t i, size_t & end) { + const size_t n = t.size(); + switch (node.op) { + case tts_norm_op::LITERAL: + if (i < n && (t[i] == node.cp || (icase && tts_norm_ascii_lower(t[i]) == tts_norm_ascii_lower(node.cp)))) { + end = i + 1; + return true; + } + return false; + case tts_norm_op::SET: + if (i < n && set_match(node.set, t[i])) { + end = i + 1; + return true; + } + return false; + case tts_norm_op::ANY: + if (i < n) { + end = i + 1; + return true; + } + return false; + case tts_norm_op::START: + end = i; + return i == 0; + case tts_norm_op::END: + end = i; + return i == n; + case tts_norm_op::BOUNDARY: + end = i; + return (i > 0 && is_w(i - 1)) != is_w(i); + case tts_norm_op::GROUP: { + size_t e = i; + if (!match_alts(node.alts, i, e)) { + return false; + } + if (node.capture >= 0) { + caps[node.capture] = {static_cast(i), static_cast(e)}; + } + end = e; + return true; + } + case tts_norm_op::LOOKAHEAD: { + // captures made inside a lookahead are discarded + auto saved = caps; + size_t e = i; + const bool ok = match_alts(node.alts, i, e); + caps = std::move(saved); + end = i; + return ok != node.neg; + } + case tts_norm_op::LOOKBEHIND: { + size_t e = 0; + const bool ok = i > 0 && match_once(node.alts[0][0].atom, i - 1, e) && e == i; + end = i; + return ok != node.neg; + } + } + return false; + } +}; + +// +// common_tts_norm_pattern +// + +common_tts_norm_pattern::common_tts_norm_pattern(const std::string & src) { + if (src.size() > TTS_NORM_MAX_PATTERN_LEN) { + throw std::runtime_error("invalid TTS norm pattern: pattern is longer than " + std::to_string(TTS_NORM_MAX_PATTERN_LEN) + " bytes"); + } + std::vector cpts; + std::vector offs; + tts_norm_decode_utf8(src, cpts, offs); + + tts_norm_parser parser(std::move(cpts), n_caps); + static const std::vector icase_prefix = {'(', '?', 'i', ')'}; + if (parser.s.size() >= 4 && std::equal(icase_prefix.begin(), icase_prefix.end(), parser.s.begin())) { + icase = true; + parser.p = 4; + } + + root = std::make_unique(); + root->op = tts_norm_op::GROUP; + root->alts = parser.parse_alts(); + if (!parser.eof()) { + parser.fail("unbalanced ')'"); + } +} + +common_tts_norm_pattern::~common_tts_norm_pattern() = default; +common_tts_norm_pattern::common_tts_norm_pattern(common_tts_norm_pattern &&) noexcept = default; +common_tts_norm_pattern & common_tts_norm_pattern::operator=(common_tts_norm_pattern &&) noexcept = default; + +bool common_tts_norm_pattern::match_at(const std::vector & cpts, size_t pos, size_t & end, std::vector> & groups) const { + tts_norm_matcher m{cpts, icase, std::vector>(n_caps, {-1, -1})}; + size_t e = pos; + if (!m.match_alts(root->alts, pos, e)) { + return false; + } + end = e; + groups = std::move(m.caps); + return true; +} + +size_t common_tts_norm_pattern::n_groups() const { + return n_caps; +} + +// +// scan / sub +// + +std::vector common_tts_norm_scan(const std::string & text, const std::vector & rules) { + std::vector cpts; + std::vector offs; + tts_norm_decode_utf8(text, cpts, offs); + + std::vector out; + const size_t n = cpts.size(); + size_t i = 0; + size_t pending = 0; // start of the current unmatched run + std::vector> groups; + + while (i < n) { + bool matched = false; + for (size_t r = 0; r < rules.size(); r++) { + size_t end = i; + if (rules[r]->match_at(cpts, i, end, groups) && end > i) { + if (pending < i) { + out.push_back({-1, offs[pending], offs[i], {}}); + } + common_tts_norm_segment seg; + seg.rule = static_cast(r); + seg.start = offs[i]; + seg.end = offs[end]; + for (const auto & g : groups) { + if (g.first < 0) { + seg.groups.emplace_back(std::string::npos, std::string::npos); + } else { + seg.groups.emplace_back(offs[g.first], offs[g.second]); + } + } + out.push_back(std::move(seg)); + i = end; + pending = i; + matched = true; + break; + } + } + if (!matched) { + i++; + } + } + if (pending < n) { + out.push_back({-1, offs[pending], offs[n], {}}); + } + return out; +} + +struct tts_norm_repl_piece { + std::string literal; + int group = -1; // 0 = whole match, 1.. = groups +}; + +static std::vector tts_norm_parse_repl(const std::string & repl, const common_tts_norm_pattern & pattern) { + std::vector pieces; + std::string lit; + auto flush = [&]() { + if (!lit.empty()) { + pieces.push_back({lit, -1}); + lit.clear(); + } + }; + for (size_t i = 0; i < repl.size(); i++) { + if (repl[i] != '%') { + lit += repl[i]; + continue; + } + if (i + 1 >= repl.size()) { + throw std::runtime_error("invalid TTS norm replacement: dangling '%'"); + } + const char c = repl[++i]; + if (c == '%') { + lit += '%'; + } else if (c >= '0' && c <= '9') { + const int g = c - '0'; + if (static_cast(g) > pattern.n_groups()) { + throw std::runtime_error("invalid TTS norm replacement: group %" + std::string(1, c) + " does not exist"); + } + flush(); + pieces.push_back({"", g}); + } else { + throw std::runtime_error("invalid TTS norm replacement: unknown escape '%" + std::string(1, c) + "'"); + } + } + flush(); + return pieces; +} + +std::string common_tts_norm_sub(const std::string & text, const common_tts_norm_pattern & pattern, const std::string & repl) { + const auto pieces = tts_norm_parse_repl(repl, pattern); + std::string out; + for (const auto & seg : common_tts_norm_scan(text, {&pattern})) { + if (seg.rule < 0) { + out.append(text, seg.start, seg.end - seg.start); + continue; + } + for (const auto & piece : pieces) { + if (piece.group < 0) { + out += piece.literal; + } else if (piece.group == 0) { + out.append(text, seg.start, seg.end - seg.start); + } else { + const auto & g = seg.groups[piece.group - 1]; + if (g.first != std::string::npos) { + out.append(text, g.first, g.second - g.first); + } + } + } + } + return out; +} + +// +// template builtins +// + +// rules can be: a pattern string (rule name "match"), a dict {name: pattern}, or a list of [name, pattern] pairs +static std::vector> tts_norm_rules_from_value(const value & v) { + std::vector> rules; + if (is_val(v)) { + rules.emplace_back("match", v->as_string().str()); + } else if (is_val(v)) { + for (const auto & kv : v->as_ordered_object()) { + if (!is_val(kv.second)) { + throw std::runtime_error("tts_norm_scan: rule pattern must be a string"); + } + rules.emplace_back(kv.first->as_string().str(), kv.second->as_string().str()); + } + } else if (is_val(v)) { + for (const auto & item : v->as_array()) { + if (!is_val(item) || item->as_array().size() != 2 + || !is_val(item->as_array()[0]) || !is_val(item->as_array()[1])) { + throw std::runtime_error("tts_norm_scan: each rule must be a [name, pattern] pair of strings"); + } + rules.emplace_back(item->as_array()[0]->as_string().str(), item->as_array()[1]->as_string().str()); + } + } else { + throw std::runtime_error("tts_norm_scan: rules must be a string, a dict or a list of [name, pattern] pairs, got " + v->type()); + } + return rules; +} + +static value tts_norm_builtin_scan(const func_args & args) { + args.ensure_count(2, 2); + args.ensure_val(args.get_pos(0)); + const string text_val = args.get_pos(0)->as_string(); + const std::string text = text_val.str(); + const bool is_input = text_val.all_parts_are_input(); + + const auto rule_srcs = tts_norm_rules_from_value(args.get_pos(1)); + std::vector patterns; + patterns.reserve(rule_srcs.size()); + std::vector ptrs; + for (const auto & r : rule_srcs) { + patterns.emplace_back(r.second); + } + for (const auto & p : patterns) { + ptrs.push_back(&p); + } + + auto mk_str = [&](size_t start, size_t end) -> value { + return mk_val(string(text.substr(start, end - start), is_input)); + }; + + auto out = mk_val(); + for (const auto & seg : common_tts_norm_scan(text, ptrs)) { + auto groups = mk_val(); + for (const auto & g : seg.groups) { + // groups that did not participate are empty strings, so they render as nothing and are falsy + groups->push_back(g.first == std::string::npos ? mk_str(0, 0) : mk_str(g.first, g.second)); + } + auto obj = mk_val(); + obj->insert("rule", seg.rule < 0 ? value(mk_val()) : value(mk_val(rule_srcs[seg.rule].first))); + obj->insert("text", mk_str(seg.start, seg.end)); + obj->insert("groups", groups); + out->push_back(obj); + } + return out; +} + +static value tts_norm_builtin_sub(const func_args & args) { + args.ensure_count(3, 3); + args.ensure_vals(); + const string text_val = args.get_pos(0)->as_string(); + const common_tts_norm_pattern pattern(args.get_pos(1)->as_string().str()); + const std::string res = common_tts_norm_sub(text_val.str(), pattern, args.get_pos(2)->as_string().str()); + return mk_val(string(res, text_val.all_parts_are_input())); +} + +void common_chat_tts_norm_add_builtins(context & ctx) { + ctx.set_val("tts_norm_scan", mk_val("tts_norm_scan", tts_norm_builtin_scan)); + ctx.set_val("tts_norm_sub", mk_val("tts_norm_sub", tts_norm_builtin_sub)); +} + diff --git a/common/chat-tts-norm.h b/common/chat-tts-norm.h new file mode 100644 index 000000000000..14695bb52d99 --- /dev/null +++ b/common/chat-tts-norm.h @@ -0,0 +1,62 @@ +#pragma once + +#include +#include +#include +#include +#include + +// TTS normalization patterns: a small, fully specified matching language for TTS text normalization templates +// +// it is NOT a regex engine: the semantics are defined by llama.cpp (see docs/tts-norm-patterns.md), so that this +// implementation and the python reference (scripts/chat_tts_norm.py) give the same result for every input +// +// two functions are exposed to task templates (e.g. "tts") via common_chat_tts_norm_add_builtins(): +// tts_norm_scan(text, rules) -> list of segments {rule, text, groups} +// tts_norm_sub(text, pattern, repl) -> string + +namespace jinja { +struct context; +} + +struct tts_norm_pattern_node; + +class common_tts_norm_pattern { +public: + // throws std::runtime_error on invalid pattern + explicit common_tts_norm_pattern(const std::string & src); + ~common_tts_norm_pattern(); + common_tts_norm_pattern(common_tts_norm_pattern &&) noexcept; + common_tts_norm_pattern & operator=(common_tts_norm_pattern &&) noexcept; + + // match anchored at codepoint index pos; on success, end is the codepoint index after the match + // groups[i] holds [start, end) codepoint indices of capture i+1, or {-1, -1} if the group did not participate + bool match_at(const std::vector & cpts, size_t pos, size_t & end, std::vector> & groups) const; + + size_t n_groups() const; + +private: + std::unique_ptr root; + size_t n_caps = 0; + bool icase = false; +}; + +// segment produced by common_tts_norm_scan(); byte offsets into the input text +struct common_tts_norm_segment { + int rule = -1; // index of the matched rule, -1 for unmatched text + size_t start = 0; + size_t end = 0; + std::vector> groups; // byte offsets; {npos, npos} for groups that did not participate +}; + +// split text into matched and unmatched segments +// at each position, rules are tried in order and the first non-empty match wins; otherwise one codepoint is skipped +std::vector common_tts_norm_scan(const std::string & text, const std::vector & rules); + +// replace every non-overlapping match of pattern with repl +// repl references: %0 (whole match), %1..%9 (groups), %% (literal percent) +std::string common_tts_norm_sub(const std::string & text, const common_tts_norm_pattern & pattern, const std::string & repl); + +// add tts_norm_scan() and tts_norm_sub() to the context of a task template +// they are not jinja global builtins on purpose: chat templates are not affected +void common_chat_tts_norm_add_builtins(jinja::context & ctx); diff --git a/common/chat.cpp b/common/chat.cpp index 5f956e64e6f3..da29cd671d4c 100644 --- a/common/chat.cpp +++ b/common/chat.cpp @@ -3,6 +3,7 @@ #include "chat-auto-parser-helpers.h" #include "chat-auto-parser.h" #include "chat-peg-parser.h" +#include "chat-tts-norm.h" #include "common.h" #include "ggml.h" #include "json-schema-to-grammar.h" @@ -807,6 +808,8 @@ bool common_chat_templates_was_explicit(const struct common_chat_templates * tmp // renders a task template (e.g. "tts") with the given variables static std::string common_chat_template_render(const common_chat_template & tmpl, const json & vars) { jinja::context ctx(tmpl.source()); + // text normalization helpers: tts_norm_scan() and tts_norm_sub(), see docs/tts-norm-patterns.md + common_chat_tts_norm_add_builtins(ctx); jinja::global_from_json(ctx, vars, false); ctx.set_val("bos_token", jinja::mk_val(tmpl.bos_token())); ctx.set_val("eos_token", jinja::mk_val(tmpl.eos_token())); @@ -819,21 +822,29 @@ static bool is_lfm3_template(const std::string & src) { return src.find("") != std::string::npos; } -// LFM3-Audio: a known language code maps to its name, no language lets the model detect it -static std::string lfm3_asr_system_prompt(const std::string & language) { +static std::string string_lower(std::string s) { + std::transform(s.begin(), s.end(), s.begin(), [](unsigned char c) { return (char) std::tolower(c); }); + return s; +} + +// LFM3-Audio: language code -> name +static std::string lfm3_language_name(const std::string & language) { static const std::map names = { { "ar", "Arabic" }, { "da", "Danish" }, { "de", "German" }, { "el", "Greek" }, { "en", "English" }, { "es", "Spanish" }, { "fi", "Finnish" }, { "fr", "French" }, { "he", "Hebrew" }, { "it", "Italian" }, { "ja", "Japanese" }, { "ko", "Korean" }, { "nl", "Dutch" }, { "pl", "Polish" }, { "pt", "Portuguese" }, { "sv", "Swedish" }, { "tr", "Turkish" }, { "zh", "Chinese" }, }; + const auto it = names.find(string_lower(language)); + return it != names.end() ? it->second : language; +} + +// LFM3-Audio: no language = auto detect +static std::string lfm3_asr_system_prompt(const std::string & language) { if (language.empty()) { return "Perform ASR."; } - std::string code = language; - std::transform(code.begin(), code.end(), code.begin(), [](unsigned char c) { return (char) std::tolower(c); }); - const auto it = names.find(code); - return "Perform ASR in " + (it != names.end() ? it->second : language) + "."; + return "Perform ASR in " + lfm3_language_name(language) + "."; } common_chat_prompt_preset common_chat_get_asr_prompt(const common_chat_templates * chat_templates, const std::string & language) { @@ -868,6 +879,27 @@ std::string common_chat_get_tts_prompt(const common_chat_templates * chat_templa }); } +std::string common_chat_get_speech_system_prompt(const common_chat_templates * chat_templates, + const std::string & language, const std::string & voice) { + if (!chat_templates || !chat_templates->template_default || !is_lfm3_template(chat_templates->template_default->source())) { + return ""; + } + // same voices as the "tts" template + const std::string v = string_lower(voice); + const bool has_voice = v == "female" || v == "male"; + if (language.empty() && !has_voice) { + return ""; + } + std::string out = "Speak"; + if (!language.empty()) { + out += " in " + lfm3_language_name(language); + } + if (has_voice) { + out += " using a " + v + " voice"; + } + return out + "."; +} + std::string common_chat_templates_source(const struct common_chat_templates * tmpls, const std::string & variant) { if (!variant.empty()) { if (variant == "tool_use") { @@ -1245,13 +1277,6 @@ static common_chat_params common_chat_params_init_lfm2_5(const common_chat_templ return common_chat_params_init_lfm2(tmpl, inputs, /* tool_list_tokens = */ false); } -// Older gemma4 templates need their tool responses rewritten before rendering -static common_chat_params common_chat_params_init_gemma4_legacy(const common_chat_template & tmpl, const autoparser::generation_params & inputs) { - auto adjusted = inputs; - workaround::convert_tool_responses_gemma4(adjusted.messages); - return common_chat_params_init_gemma4(tmpl, adjusted); -} - // Pick the dedicated handler for a template from its source, or null for the autoparser. // Order matters: the first match wins, and later checks assume the earlier ones did not match. static common_chat_params_init_fn common_chat_template_detect_params_init(const std::string & src) { @@ -1386,11 +1411,6 @@ static common_chat_params_init_fn common_chat_template_detect_params_init(const // Gemma4 format detection if (src.find("'<|tool_call>call:'") != std::string::npos) { LOG_DBG("Using specialized template: Gemma4\n"); - if (src.find("{#- OpenAI Chat Completions:") == std::string::npos) { - LOG_WRN("%s: detected an outdated gemma4 chat template, applying compatibility workarounds. " - "Consider updating to the official template.\n", __func__); - return common_chat_params_init_gemma4_legacy; - } return common_chat_params_init_gemma4; } diff --git a/common/chat.h b/common/chat.h index ebfc80f51258..304ccf9eca38 100644 --- a/common/chat.h +++ b/common/chat.h @@ -500,3 +500,7 @@ common_chat_prompt_preset common_chat_get_asr_prompt(const common_chat_templates // language and voice are optional, the template defines their defaults std::string common_chat_get_tts_prompt(const common_chat_templates * chat_templates, const std::string & text, const std::string & language = "", const std::string & voice = ""); + +// system prompt line for the speech language and voice (e.g. LFM3-Audio), empty if not used +std::string common_chat_get_speech_system_prompt(const common_chat_templates * chat_templates, + const std::string & language, const std::string & voice); diff --git a/common/common.cpp b/common/common.cpp index 04e348805ba3..657dcdbf530c 100644 --- a/common/common.cpp +++ b/common/common.cpp @@ -895,27 +895,7 @@ bool fs_validate_filename(const std::string & filename, bool allow_subdirs) { return true; } -#include - - #ifdef _WIN32 -std::wstring utf8_to_wstring(const std::string & str) { - if (str.empty()) { - return std::wstring(); - } - - int size = MultiByteToWideChar(CP_UTF8, 0, str.c_str(), (int)str.size(), NULL, 0); - - if (size <= 0) { - return std::wstring(); - } - - std::wstring wstr(size, 0); - MultiByteToWideChar(CP_UTF8, 0, str.c_str(), (int)str.size(), &wstr[0], size); - - return wstr; -} - std::string wstring_to_utf8(const std::wstring & str) { if (str.empty()) { return std::string(); @@ -983,20 +963,9 @@ void common_set_env(const std::string & name, const std::string & value) { #endif } -std::filesystem::path common_get_path_from_env(const std::string & name) { -#if defined(_WIN32) - const std::wstring wname = utf8_to_wstring(name); - const wchar_t * wvalue = _wgetenv(wname.c_str()); - return wvalue ? std::filesystem::path(wvalue) : std::filesystem::path(); -#else - const char * value = std::getenv(name.c_str()); - return value ? std::filesystem::path(value) : std::filesystem::path(); -#endif -} - #if !defined(_WIN32) static std::filesystem::path get_home_directory() { - std::filesystem::path home = common_get_path_from_env("HOME"); + std::filesystem::path home = common_get_env("HOME"); if (!home.empty()) { return home; } @@ -1009,19 +978,19 @@ static std::filesystem::path get_home_directory() { #endif std::filesystem::path fs_get_cache_directory() { - std::filesystem::path cache_directory = common_get_path_from_env("LLAMA_CACHE"); + std::filesystem::path cache_directory = common_get_env("LLAMA_CACHE"); if (!cache_directory.empty()) { return cache_directory; } #if defined(_WIN32) - cache_directory = common_get_path_from_env("LOCALAPPDATA"); + cache_directory = common_get_env("LOCALAPPDATA"); if (cache_directory.empty()) { throw std::runtime_error("Failed to find %LOCALAPPDATA% directory"); } #elif defined(__APPLE__) cache_directory = get_home_directory() / "Library/Caches"; #else - cache_directory = common_get_path_from_env("XDG_CACHE_HOME"); + cache_directory = common_get_env("XDG_CACHE_HOME"); if (cache_directory.empty()) { cache_directory = get_home_directory() / ".cache"; } @@ -1032,12 +1001,12 @@ std::filesystem::path fs_get_cache_directory() { std::filesystem::path fs_get_config_directory() { std::filesystem::path config_directory; #if defined(_WIN32) - config_directory = common_get_path_from_env("APPDATA"); + config_directory = common_get_env("APPDATA"); if (config_directory.empty()) { throw std::runtime_error("Failed to find %APPDATA% directory"); } #else - config_directory = common_get_path_from_env("XDG_CONFIG_HOME"); + config_directory = common_get_env("XDG_CONFIG_HOME"); if (config_directory.empty()) { config_directory = get_home_directory() / ".config"; } @@ -1053,7 +1022,7 @@ std::filesystem::path fs_get_cache_file(const std::string & filename) { if (ec) { throw std::runtime_error("failed to create cache directory: " + cache_directory.string()); } - return cache_directory / std::filesystem::u8path(filename); + return cache_directory / filename; } // @@ -1289,13 +1258,13 @@ void common_fit_apply(common_params & params, llama_model_params & mparams, llam cparams_dft.n_rs_seq = 0; const common_fit_extra_model extra = { - /*.path_model =*/ params_dft.model.path.c_str(), + /*.paths_model =*/ params_dft.model.get_paths(), /*.mparams =*/ &mparams_dft, /*.cparams =*/ &cparams_dft, /*.shares_model =*/ !has_draft, // an MTP context runs on the weights of the main model }; - common_fit_params(params.model.path.c_str(), &mparams, &cparams, + common_fit_params(params.model.get_paths(), &mparams, &cparams, params.tensor_split, params.tensor_buft_overrides.data(), params.fit_params_target.data(), @@ -1311,7 +1280,7 @@ common_init_result::common_init_result(common_params & params, bool model_only) common_fit_apply(params, mparams, cparams); - llama_model * model = llama_model_load_from_file(params.model.path.c_str(), mparams); + llama_model * model = common_model_load(params.model.get_paths(), mparams); if (model == NULL) { return; } @@ -1344,6 +1313,12 @@ common_init_result::common_init_result(common_params & params, bool model_only) LOG_INF("%s", "decision model reads the embeddings output, enabling embedding mode\n"); } + if (llama_model_classifier_free_guidance(model)) { + params.kv_unified = true; + cparams.kv_unified = true; + cparams.n_seq_max *= 2; + } + // embeddings need the whole batch in one ubatch, so n_batch must not be larger than n_ubatch // (server.cpp does this check for --embedding, but before the model is loaded) if (cparams.embeddings && cparams.n_batch > cparams.n_ubatch) { @@ -1747,6 +1722,19 @@ struct llama_model_params common_model_params_to_llama(common_params & params) { return mparams; } +struct llama_model * common_model_load(const std::vector & paths, const struct llama_model_params & mparams) { + if (paths.size() == 1) { + return llama_model_load_from_file(paths[0].c_str(), mparams); + } + + std::vector c_paths; + c_paths.reserve(paths.size()); + for (const auto & path : paths) { + c_paths.push_back(path.c_str()); + } + return llama_model_load_from_splits(c_paths.data(), c_paths.size(), mparams); +} + struct llama_context_params common_context_params_to_llama(const common_params & params) { auto cparams = llama_context_default_params(); @@ -1783,7 +1771,8 @@ struct llama_context_params common_context_params_to_llama(const common_params & cparams.type_k = params.cache_type_k; cparams.type_v = params.cache_type_v; - cparams.moe_cache_size = params.moe_cache_size; + cparams.moe_cache_size = params.moe_cache_size; + cparams.moe_cache_cpu_share = params.moe_cache_cpu_share; return cparams; } @@ -2266,6 +2255,14 @@ bool common_batch::set_embd_state(int32_t idx, llama_embd state) { return true; } +bool common_batch::set_embd_state_row(int32_t idx, int32_t row) { + if (idx < 0 || idx >= size() || row < 0 || tokens[idx].state.data != nullptr || tokens[idx].state_row >= 0) { + return false; + } + tokens[idx].state_row = row; + return true; +} + int32_t common_batch::add_embd(llama_embd embd, const llama_pos * pos, llama_seq_id seq_id, bool output) { token t = { LLAMA_TOKEN_NULL, { 0, 0, 0, 0 }, seq_id, output, embd, { nullptr, 0, 0 }, {} }; for (int32_t j = 0; j < n_pos; ++j) { @@ -2275,6 +2272,16 @@ int32_t common_batch::add_embd(llama_embd embd, const llama_pos * pos, llama_seq return size() - 1; } +int32_t common_batch::add_embd_row(int32_t row, const llama_pos * pos, llama_seq_id seq_id, bool output) { + token t = { LLAMA_TOKEN_NULL, { 0, 0, 0, 0 }, seq_id, output, { nullptr, 0, 0 }, { nullptr, 0, 0 }, {} }; + t.embd_row = row; + for (int32_t j = 0; j < n_pos; ++j) { + t.pos[j] = pos[j]; + } + tokens.push_back(t); + return size() - 1; +} + llama_batch_ext * common_batch::get_sub_batch(int32_t off, int32_t n) { GGML_ASSERT(batch && "common_batch was not initialized with a context"); GGML_ASSERT(off >= 0 && n >= 0 && off + n <= size()); @@ -2295,6 +2302,12 @@ llama_batch_ext * common_batch::get_sub_batch(int32_t off, int32_t n) { if (t.embd.data && !llama_batch_ext_set_embd_token(res, idx, t.embd)) { GGML_ABORT("%s: failed to set the embedding of token %d at index %d\n", __func__, t.id, i); } + } else if (t.embd_row >= 0) { + idx = llama_batch_ext_add_embd_row(res, t.seq_id, t.embd_row); + if (idx < 0) { + GGML_ABORT("%s: failed to add embd row %d at index %d (error %d, n = %d)\n", __func__, t.embd_row, i, idx, n); + } + llama_batch_ext_set_pos(res, idx, t.pos.data()); } else { idx = llama_batch_ext_add_embd(res, t.seq_id, t.embd); if (idx < 0) { @@ -2315,6 +2328,9 @@ llama_batch_ext * common_batch::get_sub_batch(int32_t off, int32_t n) { if (t.state.data) { llama_batch_ext_set_embd_state(res, idx, t.state); // contexts without a state input ignore it } + if (t.state_row >= 0 && !llama_batch_ext_set_embd_state_row(res, idx, t.state_row)) { + GGML_ABORT("%s: failed to set the state row of the entry at index %d\n", __func__, i); + } if (t.decision_order != 0) { llama_batch_ext_set_decision_order(res, idx, (llama_decision_order) t.decision_order); } diff --git a/common/common.h b/common/common.h index d0e33fb7500d..6043ab34239f 100644 --- a/common/common.h +++ b/common/common.h @@ -311,6 +311,18 @@ struct common_params_model { std::string hf_file = ""; // HF file std::string docker_repo = ""; // Docker repo + std::vector shards; + + std::vector get_paths() const { + std::vector paths = { path }; + for (const auto & shard : shards) { + if (std::find(paths.begin(), paths.end(), shard) == paths.end()) { + paths.push_back(shard); + } + } + return paths; + } + std::string get_name() const { if (!hf_repo.empty()) { return hf_repo; @@ -597,7 +609,8 @@ struct common_params { ggml_type cache_type_k = GGML_TYPE_F16; // KV cache data type for the K ggml_type cache_type_v = GGML_TYPE_F16; // KV cache data type for the V - size_t moe_cache_size = 0; // GPU cache size in bytes for the MoE experts kept in the CPU, split among the GPUs like the layers + size_t moe_cache_size = 0; // GPU cache size in bytes for the MoE experts kept in the CPU, split among the GPUs like the layers + float moe_cache_cpu_share = 0.0f; // share of the MoE cache misses computed on the CPU instead of uploaded common_conversation_mode conversation_mode = COMMON_CONVERSATION_MODE_AUTO; @@ -692,6 +705,7 @@ struct common_params { // enable built-in tools std::vector server_tools; std::string server_tools_runtime; + bool server_tools_allow_unauth = false; // allow tools/MCP on a non-loopback address without API key // MCP server configs (Cursor-compatible JSON) std::string mcp_servers_config; // path to JSON file with MCP server definitions @@ -709,6 +723,7 @@ struct common_params { std::string slot_save_path; std::string media_path; // path to directory for loading media files + bool media_allow_private_urls = false; // allow http(s) media URLs on local or private addresses float slot_prompt_similarity = 0.1f; @@ -906,8 +921,7 @@ bool glob_match(const std::string & pattern, const std::string & str); // #ifdef _WIN32 -std::wstring utf8_to_wstring(const std::string & str); -std::string wstring_to_utf8(const std::wstring & str); +std::string wstring_to_utf8(const std::wstring & str); #endif // @@ -919,9 +933,6 @@ std::string wstring_to_utf8(const std::wstring & str); std::string common_get_env(const std::string & name); void common_set_env(const std::string & name, const std::string & value); -// reads a path from the environment, an unset variable gives an empty path -std::filesystem::path common_get_path_from_env(const std::string & name); - // // Filesystem utils // @@ -1022,6 +1033,8 @@ struct llama_context_params common_context_params_to_llama(const common_params & // apply --fit to mparams and cparams, no-op if params.fit_params is false void common_fit_apply(common_params & params, struct llama_model_params & mparams, struct llama_context_params & cparams); +struct llama_model * common_model_load(const std::vector & paths, const struct llama_model_params & mparams); + // clear LoRA adapters from context, then apply new list of adapters void common_set_adapter_lora(struct llama_context * ctx, std::vector & lora); @@ -1096,6 +1109,8 @@ struct common_batch { llama_embd state; // non-owning view of the data passed to set_embd_state(), data == NULL if none std::vector seq_ids_extra; // see add_seq() int32_t decision_order = 0; // see llama_batch_ext_set_decision_order() + int32_t embd_row = -1; // see add_embd_row(), -1 if none + int32_t state_row = -1; // see set_embd_state_row(), -1 if none }; std::vector tokens; // mirror of the entries, tokens[i] describes batch index i @@ -1134,10 +1149,17 @@ struct common_batch { // attach a state embedding (e.g. the target hidden state for MTP) to the entry at idx, can only be set once per entry bool set_embd_state(int32_t idx, llama_embd state); + // the state of entry idx is row `row` of the h_nextn device buffer of the state source context, see llama_set_ctx_state_src() + bool set_embd_state_row(int32_t idx, int32_t row); + // add an embedding-only entry (no token id) // pos points to n_pos positions int32_t add_embd(llama_embd embd, const llama_pos * pos, llama_seq_id seq_id, bool output); + // add an entry whose embedding is row `row` of the layer input device buffer of the context's ctx_other + // pos points to n_pos positions + int32_t add_embd_row(int32_t row, const llama_pos * pos, llama_seq_id seq_id, bool output); + int32_t size() const { return (int32_t) tokens.size(); } }; @@ -1288,6 +1310,20 @@ enum ggml_opt_optimizer_type common_opt_get_optimizer(const char *); // prompt utils // +// leave default-inserted elements uninitialized; callers must fill them before use +template +struct common_no_init_allocator : std::allocator { + template struct rebind { using other = common_no_init_allocator; }; + + common_no_init_allocator() = default; + template common_no_init_allocator(const common_no_init_allocator &) {} + + template void construct(U * p) { ::new ((void *) p) U; } +}; + +// sequence state data, filled by llama_state_seq_get_data_ext() +using common_state_data = std::vector>; + struct common_prompt_checkpoint { int64_t n_tokens; @@ -1297,8 +1333,8 @@ struct common_prompt_checkpoint { llama_pos pos_min; llama_pos pos_max; - std::vector data_tgt; - std::vector data_dft; + common_state_data data_tgt; + common_state_data data_dft; // (optional) speculative-decoding implementation state stashed with the checkpoint // (e.g. eagle3's deferred-boundary g_embd row) diff --git a/common/download.cpp b/common/download.cpp index 08531a2d09d4..682fd6e3ef7a 100644 --- a/common/download.cpp +++ b/common/download.cpp @@ -45,7 +45,7 @@ static const std::string PARTIAL_SUFFIX = ".downloadInProgress"; static void write_etag(const std::string & path, const std::string & etag) { const std::string etag_path = path + ".etag"; - fs_write_atomic(std::filesystem::u8path(etag_path), etag); + fs_write_atomic(etag_path, etag); LOG_DBG("%s: file etag saved: %s\n", __func__, etag_path.c_str()); } @@ -293,8 +293,7 @@ static int common_download_file_single_online(const std::string & url, // alt_path is the HF snapshot: the blob may have been moved there when symlinks are unavailable auto cache_hit = [&] { - return std::filesystem::exists(std::filesystem::u8path(path)) || - (!alt_path.empty() && std::filesystem::exists(std::filesystem::u8path(alt_path))); + return std::filesystem::exists(path) || (!alt_path.empty() && std::filesystem::exists(alt_path)); }; if (skip_etag && cache_hit()) { @@ -462,38 +461,55 @@ static int common_download_file_single_online(const std::string & url, return finish(success ? head->status : -1); } -std::pair> common_remote_get_content(const std::string & url, +std::pair> common_remote_get_content(const std::string & url_in, const common_remote_params & params) { - auto [cli, parts] = common_http_client(url); + static constexpr int max_redirects = 5; + std::string url = url_in; + for (int n_redirects = 0; ; n_redirects++) { + auto [cli, parts] = common_http_client(url); + + httplib::Headers headers; + for (const auto & h : params.headers) { + headers.emplace(h.first, h.second); + } + if (headers.find("User-Agent") == headers.end()) { + headers.emplace("User-Agent", "llama-cpp/" + std::string(llama_build_info())); + } - httplib::Headers headers; - for (const auto & h : params.headers) { - headers.emplace(h.first, h.second); - } - if (headers.find("User-Agent") == headers.end()) { - headers.emplace("User-Agent", "llama-cpp/" + std::string(llama_build_info())); - } + if (params.timeout > 0) { + cli.set_read_timeout(params.timeout, 0); + cli.set_write_timeout(params.timeout, 0); + } + if (params.resolve_host) { + // pin checked address, follow redirects manually to check each hop + cli.set_direct_hostname_addr_map({{parts.host, params.resolve_host(parts.host)}}); + cli.set_follow_location(false); + } - if (params.timeout > 0) { - cli.set_read_timeout(params.timeout, 0); - cli.set_write_timeout(params.timeout, 0); - } + std::vector buf; + auto res = cli.Get(parts.path, headers, + [&](const char *data, size_t len) { + buf.insert(buf.end(), data, data + len); + return params.max_size == 0 || + buf.size() <= static_cast(params.max_size); + }, + nullptr + ); + + if (!res) { + throw std::runtime_error("error: cannot make GET request"); + } - std::vector buf; - auto res = cli.Get(parts.path, headers, - [&](const char *data, size_t len) { - buf.insert(buf.end(), data, data + len); - return params.max_size == 0 || - buf.size() <= static_cast(params.max_size); - }, - nullptr - ); + if (params.resolve_host && res->status >= 300 && res->status < 400 && res->has_header("Location")) { + if (n_redirects >= max_redirects) { + throw std::runtime_error("error: too many redirects"); + } + url = common_http_resolve_location(url, res->get_header_value("Location")); + continue; + } - if (!res) { - throw std::runtime_error("error: cannot make GET request"); + return { res->status, std::move(buf) }; } - - return { res->status, std::move(buf) }; } int common_download_file_single(const std::string & url, @@ -511,7 +527,7 @@ int common_download_file_single(const std::string & url, return common_download_file_single_online(url, path, online_opts, skip_etag, lock_path, alt_path); } - if (!std::filesystem::exists(std::filesystem::u8path(path))) { + if (!std::filesystem::exists(path)) { LOG_ERR("%s: required file is not available in cache (offline mode): %s\n", __func__, path.c_str()); return -1; } @@ -748,7 +764,8 @@ static hf_cache::hf_file find_best_model(const hf_cache::hf_files & files, } for (const auto & t : tags) { - std::regex pattern(t + "[.-]", std::regex::icase); + // match the tag as a full token, so ":F16" does not match a BF16 file + std::regex pattern("(^|[-./])" + t + "[.-]", std::regex::icase); for (const auto & f : files) { if (gguf_filename_is_model(f.path) && std::regex_search(f.path, pattern)) { @@ -1211,7 +1228,7 @@ bool common_download_remove_sidecars(const std::string & hf_repo_with_tag, const for (const auto & s : sidecars) { const auto & f = plan.*s.file; if (!f.local_path.empty()) { - to_remove.emplace_back(std::filesystem::u8path(f.local_path)); + to_remove.emplace_back(f.local_path); } } if (to_remove.empty()) { diff --git a/common/download.h b/common/download.h index dd09d68eeff9..c0ff5da2f687 100644 --- a/common/download.h +++ b/common/download.h @@ -31,6 +31,8 @@ struct common_remote_params { common_header_list headers; long timeout = 0; // in seconds, 0 means no timeout long max_size = 0; // unlimited if 0 + // if set: called per hop (redirects too), returns the address to connect to, throws to refuse + std::function resolve_host; }; // get remote file content, returns diff --git a/common/fit.cpp b/common/fit.cpp index 55b457e1060d..7ab89cc092e5 100644 --- a/common/fit.cpp +++ b/common/fit.cpp @@ -56,11 +56,12 @@ struct common_fit_log_scope { }; static std::vector common_get_device_memory_data_impl( - const char * path_model, + const std::vector & paths_model, const llama_model_params * mparams, const llama_context_params * cparams, std::vector & devs, uint32_t & hp_ngl, + uint32_t & hp_ngl_skip, uint32_t & hp_n_ctx_train, uint32_t & hp_n_expert, ggml_log_level log_level) { @@ -70,7 +71,7 @@ static std::vector common_get_device_memory_data_impl( mparams_copy.no_alloc = true; mparams_copy.load_mode = LLAMA_LOAD_MODE_NONE; - llama_model * model = llama_model_load_from_file(path_model, mparams_copy); + llama_model * model = common_model_load(paths_model, mparams_copy); if (model == nullptr) { throw std::runtime_error("failed to load model"); } @@ -152,6 +153,8 @@ static std::vector common_get_device_memory_data_impl( if (mparams->load_mtp) { hp_ngl += llama_model_n_layer_nextn(model); } + // without MTP the NextN layers hold no weights but the model loader still counts them for n_gpu_layers + hp_ngl_skip = mparams->load_mtp ? 0 : llama_model_n_layer_nextn(model); hp_n_ctx_train = llama_model_n_ctx_train(model); hp_n_expert = llama_model_n_expert(model); @@ -164,7 +167,7 @@ static std::vector common_get_device_memory_data_impl( } std::vector common_fit_measure( - const char * path_model, + const std::vector & paths_model, const llama_model_params * mparams, const llama_context_params * cparams, uint32_t * n_ctx_train, @@ -176,7 +179,7 @@ std::vector common_fit_measure( mparams_copy.no_alloc = true; mparams_copy.load_mode = LLAMA_LOAD_MODE_NONE; - llama_model_ptr model(llama_model_load_from_file(path_model, mparams_copy)); + llama_model_ptr model(common_model_load(paths_model, mparams_copy)); if (!model) { throw std::runtime_error("failed to load model"); } @@ -198,7 +201,7 @@ std::vector common_fit_measure( common_memory_reports common_measure_memory(common_params & params) { const ggml_log_level log_level = params.verbosity >= LOG_LEVEL_DEBUG ? GGML_LOG_LEVEL_DEBUG : GGML_LOG_LEVEL_ERROR; - const char * path_model = params.model.path.c_str(); + const std::vector paths_model = params.model.get_paths(); common_memory_reports res; @@ -210,7 +213,7 @@ common_memory_reports common_measure_memory(common_params & params) { auto mparams_fit = mparams; auto cparams_fit = cparams; common_fit_apply(params, mparams_fit, cparams_fit); - res.info_max = common_fit_measure(path_model, &mparams_fit, &cparams_fit, &res.n_ctx_train, &res.n_ctx_max, log_level); + res.info_max = common_fit_measure(paths_model, &mparams_fit, &cparams_fit, &res.n_ctx_train, &res.n_ctx_max, log_level); } // min: the smallest context --fit may pick (--fit only changes n_ctx when it is 0) @@ -228,23 +231,24 @@ common_memory_reports common_measure_memory(common_params & params) { res.info_min = res.info_max; } else { cparams.n_ctx = n_ctx_min; - res.info_min = common_fit_measure(path_model, &mparams, &cparams, nullptr, &res.n_ctx_min, log_level); + res.info_min = common_fit_measure(paths_model, &mparams, &cparams, nullptr, &res.n_ctx_min, log_level); } return res; } common_device_memory_data_vec common_get_device_memory_data( - const char * path_model, + const std::vector & paths_model, const llama_model_params * mparams, const llama_context_params * cparams, std::vector & devs, uint32_t & hp_ngl, + uint32_t & hp_ngl_skip, uint32_t & hp_n_ctx_train, uint32_t & hp_n_expert, ggml_log_level log_level) { std::vector impl = common_get_device_memory_data_impl( - path_model, mparams, cparams, devs, hp_ngl, hp_n_ctx_train, hp_n_expert, log_level); + paths_model, mparams, cparams, devs, hp_ngl, hp_ngl_skip, hp_n_ctx_train, hp_n_expert, log_level); common_device_memory_data_vec ret(impl.size()); for (size_t i = 0; i < impl.size(); i++) { @@ -258,7 +262,7 @@ common_device_memory_data_vec common_get_device_memory_data( } static void common_params_fit_impl( - const char * path_model, struct llama_model_params * mparams, struct llama_context_params * cparams, + const std::vector & paths_model, struct llama_model_params * mparams, struct llama_context_params * cparams, float * tensor_split, struct llama_model_tensor_buft_override * tensor_buft_overrides, size_t * margins_s, uint32_t n_ctx_min, const common_fit_extra_model * extra, enum ggml_log_level log_level) { if (mparams->split_mode == LLAMA_SPLIT_MODE_TENSOR) { @@ -270,6 +274,7 @@ static void common_params_fit_impl( std::vector devs; uint32_t hp_ngl = 0; // hparams.n_gpu_layers + uint32_t hp_nsk = 0; // number of skipped NextN layers, counted by the model loader but not by hp_ngl uint32_t hp_nct = 0; // hparams.n_ctx_train uint32_t hp_nex = 0; // hparams.n_expert @@ -291,6 +296,7 @@ static void common_params_fit_impl( if (dmds_extra.empty() || n_ctx_extra != cparams->n_ctx) { std::vector devs_extra; uint32_t ngl_extra = 0; + uint32_t nsk_extra = 0; uint32_t nct_extra = 0; uint32_t nex_extra = 0; @@ -302,7 +308,7 @@ static void common_params_fit_impl( dmds_t measured; try { measured = common_get_device_memory_data_impl( - extra->path_model, extra->mparams, extra->cparams, devs_extra, ngl_extra, nct_extra, nex_extra, log_level); + extra->paths_model, extra->mparams, extra->cparams, devs_extra, ngl_extra, nsk_extra, nct_extra, nex_extra, log_level); } catch (const std::runtime_error & e) { // the extra model is optional, fit the main model alone rather than giving up LOG_WRN("%s: failed to measure the memory of the extra model, fitting without it: %s\n", __func__, e.what()); @@ -342,7 +348,7 @@ static void common_params_fit_impl( // step 1: get data for default parameters and check whether any changes are necessary in the first place LOG_TRC("%s: getting device memory data for initial parameters:\n", __func__); - dmds_t dmds_full = common_get_device_memory_data_impl(path_model, mparams, cparams, devs, hp_ngl, hp_nct, hp_nex, log_level); + dmds_t dmds_full = common_get_device_memory_data_impl(paths_model, mparams, cparams, devs, hp_ngl, hp_nsk, hp_nct, hp_nex, log_level); // saturate instead of overflowing, this also preserves the UINT32_MAX sentinel of n_ctx_min: const uint32_t n_ctx_max = (uint32_t) std::min(uint64_t(hp_nct) * n_streams, UINT32_MAX); @@ -354,7 +360,7 @@ static void common_params_fit_impl( if (n_streams > 1) { LOG_TRC("%s: context size unset and KV cache not unified -> using %" PRIu32 " for %" PRIu32 " sequences:\n", __func__, n_ctx_max, n_streams); - dmds_full = common_get_device_memory_data_impl(path_model, mparams, cparams, devs, hp_ngl, hp_nct, hp_nex, log_level); + dmds_full = common_get_device_memory_data_impl(paths_model, mparams, cparams, devs, hp_ngl, hp_nsk, hp_nct, hp_nex, log_level); } } add_extra_memory(dmds_full); @@ -493,7 +499,7 @@ static void common_params_fit_impl( int64_t sum_projected_used_min_ctx = 0; cparams->n_ctx = n_ctx_min_total; - dmds_t dmds_min_ctx = common_get_device_memory_data_impl(path_model, mparams, cparams, devs, hp_ngl, hp_nct, hp_nex, log_level); + dmds_t dmds_min_ctx = common_get_device_memory_data_impl(paths_model, mparams, cparams, devs, hp_ngl, hp_nsk, hp_nct, hp_nex, log_level); add_extra_memory(dmds_min_ctx); if (nd == 0) { sum_projected_used_min_ctx = dmds_min_ctx.back().mb.total(); @@ -640,6 +646,20 @@ static void common_params_fit_impl( assert(uint32_t(mparams.n_gpu_layers) <= hp_ngl + 1); uint32_t il0 = hp_ngl + 1 - mparams.n_gpu_layers; // start index for tensor buft overrides + // the skipped NextN layers come right before the output layer, put them on the same device as the output layer + // so that the layers assigned by the model loader match the layers above + if (hp_nsk > 0 && mparams.n_gpu_layers > 0) { + mparams.n_gpu_layers += hp_nsk; + if (nd > 1) { + // the last devices have no layers if not even the output layer fits + size_t id_last = nd - 1; + while (ngl_per_device[id_last].n_layer == 0) { + id_last--; + } + tensor_split[id_last] += hp_nsk; + } + } + mparams.tensor_split = tensor_split; size_t itbo = 0; @@ -675,7 +695,7 @@ static void common_params_fit_impl( set_ngl_tensor_split_tbo(ngl_per_device, overflow_bufts, mparams_copy); dmds_t dmd_nl = common_get_device_memory_data_impl( - path_model, &mparams_copy, cparams, devs, hp_ngl, hp_nct, hp_nex, log_level); + paths_model, &mparams_copy, cparams, devs, hp_ngl, hp_nsk, hp_nct, hp_nex, log_level); add_extra_memory(dmd_nl); LOG_TRC("%s: memory for test allocation by device:\n", func_name); @@ -704,7 +724,7 @@ static void common_params_fit_impl( LOG_TRC("%s: getting device memory data with all MoE tensors moved to system memory:\n", __func__); dmds_t dmds_cpu_moe = common_get_device_memory_data_impl( - path_model, mparams, cparams, devs, hp_ngl, hp_nct, hp_nex, log_level); + paths_model, mparams, cparams, devs, hp_ngl, hp_nsk, hp_nct, hp_nex, log_level); add_extra_memory(dmds_cpu_moe); for (size_t id = 0; id < nd; id++) { @@ -958,7 +978,7 @@ static void common_params_fit_impl( } enum common_params_fit_status common_fit_params( - const char * path_model, + const std::vector & paths_model, llama_model_params * mparams, llama_context_params * cparams, float * tensor_split, @@ -970,7 +990,7 @@ enum common_params_fit_status common_fit_params( const int64_t t0_us = llama_time_us(); common_params_fit_status status = COMMON_PARAMS_FIT_STATUS_SUCCESS; try { - common_params_fit_impl(path_model, mparams, cparams, tensor_split, tensor_buft_overrides, margins, n_ctx_min, extra, log_level); + common_params_fit_impl(paths_model, mparams, cparams, tensor_split, tensor_buft_overrides, margins, n_ctx_min, extra, log_level); LOG_TRC("%s: successfully fit params to free device memory\n", __func__); } catch (const common_params_fit_exception & e) { LOG_WRN("%s: failed to fit params to free device memory: %s\n", __func__, e.what()); @@ -1180,15 +1200,16 @@ void common_memory_breakdown_print(const struct llama_context * ctx) { } void common_fit_print( - const char * path_model, + const std::vector & paths_model, llama_model_params * mparams, llama_context_params * cparams) { std::vector devs; uint32_t hp_ngl = 0; // hparams.n_gpu_layers + uint32_t hp_nsk = 0; // number of skipped NextN layers, counted by the model loader but not by hp_ngl uint32_t hp_nct = 0; // hparams.n_ctx_train uint32_t hp_nex = 0; // hparams.n_expert - auto dmd = common_get_device_memory_data_impl(path_model, mparams, cparams, devs, hp_ngl, hp_nct, hp_nex, GGML_LOG_LEVEL_ERROR); + auto dmd = common_get_device_memory_data_impl(paths_model, mparams, cparams, devs, hp_ngl, hp_nsk, hp_nct, hp_nex, GGML_LOG_LEVEL_ERROR); GGML_ASSERT(dmd.size() == devs.size() + 1); for (size_t id = 0; id < devs.size(); id++) { diff --git a/common/fit.h b/common/fit.h index 47478bba12e0..431c418366da 100644 --- a/common/fit.h +++ b/common/fit.h @@ -16,7 +16,7 @@ enum common_params_fit_status { // - its context follows the context of the main model, so its memory is measured again whenever that context changes // - shares_model tells the fit that the weights are already counted in the main model, as for an MTP context struct common_fit_extra_model { - const char * path_model; + std::vector paths_model; llama_model_params * mparams; llama_context_params * cparams; bool shares_model; @@ -28,19 +28,19 @@ struct common_fit_extra_model { // - only parameters that have the same value as in llama_default_model_params are modified // with the exception of the context size which is modified if and only if equal to 0 common_params_fit_status common_fit_params( - const char * path_model, + const std::vector & paths_model, llama_model_params * mparams, llama_context_params * cparams, float * tensor_split, // writable buffer for tensor split, needs at least llama_max_devices elements llama_model_tensor_buft_override * tensor_buft_overrides, // writable buffer for overrides, needs at least llama_max_tensor_buft_overrides elements size_t * margins, // margins of memory to leave per device in bytes uint32_t n_ctx_min, // minimum context size to set when trying to reduce memory use - const common_fit_extra_model * extra, // model to fit alongside the main one, nullptr if there is none + const common_fit_extra_model * extra, // model to fit alongside the main one, nullptr if there is none ggml_log_level log_level); // minimum log level to print during fitting, lower levels go to debug log // print estimated memory to stdout void common_fit_print( - const char * path_model, + const std::vector & paths_model, llama_model_params * mparams, llama_context_params * cparams); @@ -75,7 +75,7 @@ std::vector common_memory_breakdown_rows(const llam // load model + context with no_alloc and return the memory breakdown, params are used as-is // n_ctx_train and n_ctx are optional outputs; throws on failure std::vector common_fit_measure( - const char * path_model, + const std::vector & paths_model, const llama_model_params * mparams, const llama_context_params * cparams, uint32_t * n_ctx_train, @@ -99,11 +99,12 @@ using common_device_memory_data_vec = std::vector; // Load a model + context with no_alloc and return the per-device memory breakdown. common_device_memory_data_vec common_get_device_memory_data( - const char * path_model, + const std::vector & paths_model, const llama_model_params * mparams, const llama_context_params * cparams, std::vector & devs, uint32_t & hp_ngl, + uint32_t & hp_ngl_skip, uint32_t & hp_n_ctx_train, uint32_t & hp_n_expert, ggml_log_level log_level); diff --git a/common/hf-cache.cpp b/common/hf-cache.cpp index aafc940c5789..4c65eff92e9b 100644 --- a/common/hf-cache.cpp +++ b/common/hf-cache.cpp @@ -45,7 +45,7 @@ static fs::path get_cache_directory() { {HOME_DIR, fs::path(".cache") / "huggingface" / "hub"} }; for (const auto & entry : entries) { - if (fs::path base = common_get_path_from_env(entry.var); !base.empty()) { + if (fs::path base = common_get_env(entry.var); !base.empty()) { return entry.path.empty() ? base : base / entry.path; } } @@ -242,7 +242,7 @@ static std::string get_repo_commit(const std::string & repo_id, LOG_WRN("%s: skip invalid commit: %s\n", __func__, _commit.c_str()); continue; } - const fs::path candidate = fs::u8path(_name); + const fs::path candidate = _name; if (!is_valid_subpath(refs_path, candidate)) { LOG_WRN("%s: skip invalid branch: %s\n", __func__, _name.c_str()); @@ -317,7 +317,7 @@ hf_files get_repo_files(const std::string & repo_id, file.repo_id = repo_id; file.path = item["path"].get(); - const fs::path subpath = fs::u8path(file.path); + const fs::path subpath = file.path; if (!is_valid_subpath(commit_path, subpath)) { LOG_WRN("%s: skip invalid path: %s\n", __func__, file.path.c_str()); @@ -434,8 +434,7 @@ hf_files get_cached_files(const std::string & repo_id) { if (!path.empty()) { hf_file file; file.repo_id = _repo_id; - const auto generic_path = path.generic_u8string(); - file.path = std::string(generic_path.begin(), generic_path.end()); + file.path = path.generic_string(); file.local_path = entry.path().string(); file.final_path = file.local_path; files.push_back(std::move(file)); @@ -450,8 +449,8 @@ std::string finalize_file(const hf_file & file, const std::function & ke static std::atomic symlinks_disabled{false}; std::error_code ec; - fs::path local_path = fs::u8path(file.local_path); - fs::path final_path = fs::u8path(file.final_path); + fs::path local_path = file.local_path; + fs::path final_path = file.final_path; if (local_path == final_path || fs::exists(final_path, ec)) { return file.final_path; diff --git a/common/http.h b/common/http.h index 10d017f17ed5..d8cfde262019 100644 --- a/common/http.h +++ b/common/http.h @@ -99,6 +99,26 @@ static common_http_url common_http_parse_url(const std::string & url) { return parts; } +// resolve a redirect Location against the URL it came from +static std::string common_http_resolve_location(const std::string & base_url, const std::string & location) { + if (location.find("://") != std::string::npos) { + return location; + } + const common_http_url base = common_http_parse_url(base_url); + if (location.compare(0, 2, "//") == 0) { + return base.scheme + ":" + location; + } + const std::string origin = base.scheme + "://" + common_http_format_host(base.host) + ":" + std::to_string(base.port); + const std::string path = base.path.substr(0, base.path.find('?')); + if (!location.empty() && location[0] == '/') { + return origin + location; + } + if (!location.empty() && location[0] == '?') { + return origin + path + location; + } + return origin + path.substr(0, path.rfind('/') + 1) + location; +} + // match a loopback destination: a remote proxy can never reach the client machine's own // loopback, so these hosts are always contacted directly, before NO_PROXY is consulted // (similar to Go's net/http) diff --git a/common/jinja/lexer.h b/common/jinja/lexer.h index 439c85764c24..3f4ebb62a8d5 100644 --- a/common/jinja/lexer.h +++ b/common/jinja/lexer.h @@ -37,6 +37,7 @@ struct token { call_operator, // () additive_binary_operator, // + - ~ multiplicative_binary_operator, // * / % + power_operator, // ** comparison_binary_operator, // < > <= >= == != unary_operator, // ! - + comment, // {# ... #} @@ -71,6 +72,7 @@ static std::string type_to_string(token::type t) { case token::call_operator: return "call_operator"; case token::additive_binary_operator: return "additive_binary_operator"; case token::multiplicative_binary_operator: return "multiplicative_binary_operator"; + case token::power_operator: return "power_operator"; case token::comparison_binary_operator: return "comparison_binary_operator"; case token::unary_operator: return "unary_operator"; case token::comment: return "comment"; @@ -137,6 +139,7 @@ struct lexer { {"+", token::additive_binary_operator}, {"-", token::additive_binary_operator}, {"~", token::additive_binary_operator}, + {"**", token::power_operator}, {"*", token::multiplicative_binary_operator}, {"/", token::multiplicative_binary_operator}, {"%", token::multiplicative_binary_operator}, diff --git a/common/jinja/parser.cpp b/common/jinja/parser.cpp index 7e99a1ac8fd0..0841828c454a 100644 --- a/common/jinja/parser.cpp +++ b/common/jinja/parser.cpp @@ -178,7 +178,7 @@ class parser { // bool has_caller_args = false; if (is(token::open_paren)) { // Optional caller arguments, e.g. {% call(user) dump_users(...) %} - caller_args = parse_args(); + caller_args = parse_args(false); // has_caller_args = true; } auto callee = parse_primary_expression(); @@ -283,7 +283,7 @@ class parser { statement_ptr parse_macro_statement(size_t start_pos) { auto name = parse_primary_expression(); - auto args = parse_args(); + auto args = parse_args(false); expect(token::close_statement, "Expected %}"); statements body; // Keep going until we hit {% endmacro @@ -397,11 +397,13 @@ class parser { statement_ptr parse_comparison_expression() { // NOTE: membership has same precedence as comparison - // e.g., ('a' in 'apple' == 'b' in 'banana') evaluates as ('a' in ('apple' == ('b' in 'banana'))) - auto left = parse_additive_expression(); + // e.g., ('a' in 'apple' == 'b' in 'banana') evaluates as ('a' in 'apple' and 'apple' == 'b' and 'b' in 'banana') + statements operands; + std::vector ops; + operands.push_back(parse_additive_expression()); + size_t pos = current; while (true) { token op; - size_t start_pos = current; if (is_identifier("not") && peek(1).t == token::identifier && peek(1).value == "in") { op = {token::identifier, "not in", tokens[current].pos}; ++current; // consume 'not' @@ -411,9 +413,17 @@ class parser { } else if (is(token::comparison_binary_operator)) { op = next(); } else break; - left = mk_stmt(start_pos, op, std::move(left), parse_additive_expression()); + ops.push_back(std::move(op)); + operands.push_back(parse_additive_expression()); } - return left; + if (ops.empty()) { + return std::move(operands[0]); + } + if (ops.size() == 1) { + return mk_stmt(pos, ops[0], std::move(operands[0]), std::move(operands[1])); + } + // chained comparison, e.g. 1 <= x <= 100 + return mk_stmt(pos, std::move(operands), std::move(ops)); } statement_ptr parse_additive_expression() { @@ -500,17 +510,22 @@ class parser { return parse_call_expression(std::move(member)); // foo.x()() } - statements parse_args() { + statements parse_args(bool allow_spread = true) { // comma-separated arguments list expect(token::open_paren, "Expected ("); statements args; while (!is(token::close_paren)) { statement_ptr arg; - // unpacking: *expr - if (peek().t == token::multiplicative_binary_operator && peek().value == "*") { + // unpacking: *expr or **expr + const auto & t = peek(); + if (t.t == token::power_operator || (t.t == token::multiplicative_binary_operator && t.value == "*")) { + if (!allow_spread) { + throw std::runtime_error("Unexpected token: " + t.value + " of type " + std::to_string(t.t)); + } size_t start_pos = current; - ++current; // consume * - arg = mk_stmt(start_pos, parse_expression()); + ++current; // consume * or ** + bool is_kwargs = t.t == token::power_operator; + arg = mk_stmt(start_pos, parse_expression(), is_kwargs); } else { arg = parse_expression(); if (is(token::equals)) { diff --git a/common/jinja/runtime.cpp b/common/jinja/runtime.cpp index d32c79fa5cb5..d60a99d2591e 100644 --- a/common/jinja/runtime.cpp +++ b/common/jinja/runtime.cpp @@ -117,20 +117,8 @@ value object_literal::execute_impl(context & ctx) const { return obj; } -value binary_expression::execute_impl(context & ctx) const { - value left_val = left->execute(ctx); - - // Logical operators - if (op.value == "and") { - JJ_DEBUG("Executing logical test: %s AND %s", left->type().c_str(), right->type().c_str()); - return left_val->as_bool() ? right->execute(ctx) : std::move(left_val); - } else if (op.value == "or") { - JJ_DEBUG("Executing logical test: %s OR %s", left->type().c_str(), right->type().c_str()); - return left_val->as_bool() ? std::move(left_val) : right->execute(ctx); - } - +static value exec_binary_op(context & ctx, const token & op, value left_val, value right_val) { // Equality operators - value right_val = right->execute(ctx); JJ_DEBUG("Executing binary expression %s '%s' %s", left_val->type().c_str(), op.value.c_str(), right_val->type().c_str()); if (op.value == "==") { return mk_val(*left_val == *right_val); @@ -307,6 +295,67 @@ value binary_expression::execute_impl(context & ctx) const { throw std::runtime_error("Unknown operator \"" + op.value + "\" between " + left_val->type() + " and " + right_val->type()); } +value binary_expression::execute_impl(context & ctx) const { + value left_val = left->execute(ctx); + + // Logical operators + if (op.value == "and") { + JJ_DEBUG("Executing logical test: %s AND %s", left->type().c_str(), right->type().c_str()); + return left_val->as_bool() ? right->execute(ctx) : std::move(left_val); + } else if (op.value == "or") { + JJ_DEBUG("Executing logical test: %s OR %s", left->type().c_str(), right->type().c_str()); + return left_val->as_bool() ? std::move(left_val) : right->execute(ctx); + } + + return exec_binary_op(ctx, op, std::move(left_val), right->execute(ctx)); +} + +value comparison_chain_expression::execute_impl(context & ctx) const { + value left_val = operands[0]->execute(ctx); + value res; + for (size_t i = 0; i < ops.size(); ++i) { + value right_val = operands[i + 1]->execute(ctx); + res = exec_binary_op(ctx, ops[i], left_val, right_val); + if (!res->as_bool()) { + break; // same as "and", the next operands are not executed + } + left_val = std::move(right_val); + } + return res; +} + +static void exec_call_args(context & ctx, const statements & arg_stmts, func_args & args) { + for (const auto & arg_stmt : arg_stmts) { + if (!is_stmt(arg_stmt)) { + auto arg_val = arg_stmt->execute(ctx); + JJ_DEBUG(" Argument type: %s", arg_val->type().c_str()); + args.push_back(arg_val); + continue; + } + auto spread = cast_stmt(arg_stmt); + value val = spread->argument->execute(ctx); + JJ_DEBUG(" Unpacked argument type: %s", val->type().c_str()); + if (spread->is_kwargs) { + if (!is_val(val)) { + throw std::runtime_error("Argument after ** must be an object, got " + val->type()); + } + for (const auto & pair : val->as_ordered_object()) { + if (!is_val(pair.first)) { + throw std::runtime_error("Keyword argument key must be a string, got " + pair.first->type()); + } + args.push_back(mk_val(pair.first->as_string().str(), pair.second)); + } + } else { + if (!is_val(val)) { + throw std::runtime_error("Argument after * must be an array, got " + val->type()); + } + for (const auto & item : val->as_array()) { + args.push_back(item); + } + } + } +} + static value apply_filter(context & ctx, const statement_ptr & filter, value input) { JJ_DEBUG("Applying filter to %s", input->type().c_str()); @@ -324,9 +373,7 @@ static value apply_filter(context & ctx, const statement_ptr & filter, value inp JJ_DEBUG("Applying filter '%s' with arguments to %s", filter_id.c_str(), input->type().c_str()); func_args args(ctx); - for (const auto & arg_expr : call->args) { - args.push_back(arg_expr->execute(ctx)); - } + exec_call_args(ctx, call->args, args); return apply_filter_by_name(ctx, filter_id, input, args); @@ -369,9 +416,7 @@ value test_expression::execute_impl(context & ctx) const { test_id = cast_stmt(call->callee)->val; JJ_DEBUG("Applying test '%s' with arguments to %s", test_id.c_str(), input->type().c_str()); - for (const auto & arg_expr : call->args) { - args.push_back(arg_expr->execute(ctx)); - } + exec_call_args(ctx, call->args, args); } else { throw std::runtime_error("Invalid test expression"); @@ -762,12 +807,7 @@ value call_statement::execute_impl(context & ctx) const { call_ctx.set_val("caller", mk_val("caller", func)); func_args args(call_ctx); - - for (const auto & arg_expr : call_expr->args) { - auto arg_val = arg_expr->execute(ctx); - JJ_DEBUG(" Argument type: %s", arg_val->type().c_str()); - args.push_back(arg_val); - } + exec_call_args(ctx, call_expr->args, args); JJ_DEBUG("Calling macro '%s' with %zu arguments", callee_func->name.c_str(), args.count()); return callee_func->invoke(args); @@ -907,11 +947,7 @@ value member_expression::execute_impl(context & ctx) const { value call_expression::execute_impl(context & ctx) const { // gather arguments func_args args(ctx); - for (auto & arg_stmt : this->args) { - auto arg_val = arg_stmt->execute(ctx); - JJ_DEBUG(" Argument type: %s", arg_val->type().c_str()); - args.push_back(arg_val); - } + exec_call_args(ctx, this->args, args); // execute callee value callee_val = callee->execute(ctx); if (!is_val(callee_val)) { diff --git a/common/jinja/runtime.h b/common/jinja/runtime.h index 101e927c4b7f..43e1085516da 100644 --- a/common/jinja/runtime.h +++ b/common/jinja/runtime.h @@ -471,6 +471,28 @@ struct binary_expression : public expression { } }; +/** + * Two or more comparisons in a row, e.g. `a < b <= c` + * It is the same as `a < b and b <= c`, but each operand is executed only one time. + */ +struct comparison_chain_expression : public expression { + statements operands; + std::vector ops; + + comparison_chain_expression(statements && operands, std::vector && ops) + : operands(std::move(operands)), ops(std::move(ops)) { + assert(this->operands.size() == this->ops.size() + 1); + for (const auto & operand : this->operands) chk_type(operand); + } + std::string type() const override { return "ComparisonChainExpression"; } + value execute_impl(context & ctx) const override; + void visit(context & ctx) const override { + ctx.visitor(false, this, { + {"operands", stmts_to_ptr(operands)} + }); + } +}; + /** * An operation with two sides, separated by the | operator. * Operator precedence: https://github.com/pallets/jinja/issues/379#issuecomment-168076202 @@ -633,7 +655,9 @@ struct keyword_argument_expression : public expression { struct spread_expression : public expression { statement_ptr argument; - explicit spread_expression(statement_ptr && argument) : argument(std::move(argument)) { + bool is_kwargs; + spread_expression(statement_ptr && argument, bool is_kwargs) + : argument(std::move(argument)), is_kwargs(is_kwargs) { chk_type(this->argument); } std::string type() const override { return "SpreadExpression"; } diff --git a/common/json-schema-to-grammar.cpp b/common/json-schema-to-grammar.cpp index 6f3ecc992113..b6ea61fc3870 100644 --- a/common/json-schema-to-grammar.cpp +++ b/common/json-schema-to-grammar.cpp @@ -860,6 +860,7 @@ class common_chat_schema_converter { std::unordered_set required; std::vector> properties; std::map enum_values; + std::vector leaves; std::function add_component = [&](const common_chat_schema & comp, bool is_required, int depth) { if (depth >= COMMON_RECURSIVE_LIMIT) { throw std::invalid_argument("JSON schema is nested more than " + std::to_string(COMMON_RECURSIVE_LIMIT) + " levels deep"); @@ -879,6 +880,10 @@ class common_chat_schema_converter { for (const auto & v : as(comp).values) { enum_values[_generate_constant_rule(v)] += 1; } + } else if (is_required && comp.kind() != common_chat_schema::KIND_ANY) { + // Direct non-structural children are used for merging. anyOf alternatives are not merged. + // Empty schemas are skipped. + leaves.push_back(&comp); } }; for (const auto & child : schema.children) { @@ -901,6 +906,38 @@ class common_chat_schema_converter { return _add_rule(rule_name, "(" + string_join(enum_intersection, " | ") + ")"); } } + + if (properties.empty() && !leaves.empty()) { + if (leaves.size() == 1) { + return visit(*leaves[0], name); + } + bool all_numeric = true; + bool any_integer = false; + int64_t lo = std::numeric_limits::min(); + int64_t hi = std::numeric_limits::max(); + for (const auto * leaf : leaves) { + if (leaf->kind() == common_chat_schema::KIND_INTEGER) { + const auto & i = as(*leaf); + any_integer = true; + lo = std::max(lo, i.minimum); + hi = std::min(hi, i.maximum); + } else if (leaf->kind() != common_chat_schema::KIND_NUMBER) { + all_numeric = false; + } + } + if (all_numeric) { + if (lo > hi) { + throw std::invalid_argument("allOf has an empty numeric range"); + } + if (any_integer) { + common_chat_schema_integer merged; + merged.minimum = lo; + merged.maximum = hi; + return visit(merged, name); + } + return visit(*leaves[0], name); + } + } return _add_rule(rule_name, _build_object_rule(properties, required, name, nullptr)); } diff --git a/common/json-schema.h b/common/json-schema.h index 084208c96201..b203e7777aa8 100644 --- a/common/json-schema.h +++ b/common/json-schema.h @@ -85,92 +85,124 @@ struct common_chat_schema { static const char * kind_name(node_kind kind); static const char * type_name(value_type type); + + template + const T & as() const { + GGML_ASSERT(kind() == T::KIND); + return static_cast(*this); + } }; using common_chat_schema_ptr = std::unique_ptr; struct common_chat_schema_any : common_chat_schema { - node_kind kind() const override { return KIND_ANY; } + static constexpr node_kind KIND = KIND_ANY; + + node_kind kind() const override { return KIND; } }; // {"$ref": "#/..."}, only references into the same document are supported struct common_chat_schema_ref : common_chat_schema { + static constexpr node_kind KIND = KIND_REF; + std::string ref; const common_chat_schema * target = nullptr; // owned by common_chat_schema_document::refs explicit common_chat_schema_ref(std::string ref) : ref(std::move(ref)) {} - node_kind kind() const override { return KIND_REF; } + node_kind kind() const override { return KIND; } }; // oneOf / anyOf, or a "type" array expanded to one alternative per type struct common_chat_schema_any_of : common_chat_schema { + static constexpr node_kind KIND = KIND_ANY_OF; + std::vector children; - node_kind kind() const override { return KIND_ANY_OF; } + node_kind kind() const override { return KIND; } }; struct common_chat_schema_all_of : common_chat_schema { + static constexpr node_kind KIND = KIND_ALL_OF; + std::vector children; - node_kind kind() const override { return KIND_ALL_OF; } + node_kind kind() const override { return KIND; } }; struct common_chat_schema_const : common_chat_schema { + static constexpr node_kind KIND = KIND_CONST; + common_json value; explicit common_chat_schema_const(common_json value) : value(std::move(value)) {} - node_kind kind() const override { return KIND_CONST; } + node_kind kind() const override { return KIND; } }; struct common_chat_schema_enum : common_chat_schema { + static constexpr node_kind KIND = KIND_ENUM; + std::vector values; - node_kind kind() const override { return KIND_ENUM; } + node_kind kind() const override { return KIND; } }; struct common_chat_schema_null : common_chat_schema { - node_kind kind() const override { return KIND_NULL; } + static constexpr node_kind KIND = KIND_NULL; + + node_kind kind() const override { return KIND; } }; struct common_chat_schema_boolean : common_chat_schema { - node_kind kind() const override { return KIND_BOOLEAN; } + static constexpr node_kind KIND = KIND_BOOLEAN; + + node_kind kind() const override { return KIND; } }; struct common_chat_schema_number : common_chat_schema { - node_kind kind() const override { return KIND_NUMBER; } + static constexpr node_kind KIND = KIND_NUMBER; + + node_kind kind() const override { return KIND; } }; // bounds are inclusive, exclusiveMinimum / exclusiveMaximum are folded in struct common_chat_schema_integer : common_chat_schema { + static constexpr node_kind KIND = KIND_INTEGER; + int64_t minimum = INT64_MIN; // INT64_MIN for unbounded int64_t maximum = INT64_MAX; // INT64_MAX for unbounded - node_kind kind() const override { return KIND_INTEGER; } + node_kind kind() const override { return KIND; } }; struct common_chat_schema_string : common_chat_schema { + static constexpr node_kind KIND = KIND_STRING; + std::string pattern; // empty when absent string_format format = FORMAT_NONE; int min_length = 0; int max_length = -1; // -1 for unbounded - node_kind kind() const override { return KIND_STRING; } + node_kind kind() const override { return KIND; } }; struct common_chat_schema_array : common_chat_schema { + static constexpr node_kind KIND = KIND_ARRAY; + common_chat_schema_ptr items; // a common_chat_schema_any when "items" is absent int min_items = 0; int max_items = -1; // -1 for unbounded - node_kind kind() const override { return KIND_ARRAY; } + node_kind kind() const override { return KIND; } }; struct common_chat_schema_tuple : common_chat_schema { + static constexpr node_kind KIND = KIND_TUPLE; + std::vector items; - node_kind kind() const override { return KIND_TUPLE; } + node_kind kind() const override { return KIND; } }; struct common_chat_schema_property { @@ -180,10 +212,12 @@ struct common_chat_schema_property { }; struct common_chat_schema_object : common_chat_schema { + static constexpr node_kind KIND = KIND_OBJECT; + std::vector properties; // in schema order common_chat_schema_ptr additional_properties; // null when not allowed - node_kind kind() const override { return KIND_OBJECT; } + node_kind kind() const override { return KIND; } }; struct common_chat_schema_document { diff --git a/common/parsers/deepseek.cpp b/common/parsers/deepseek.cpp index dac180715a52..e5f10347a6eb 100644 --- a/common/parsers/deepseek.cpp +++ b/common/parsers/deepseek.cpp @@ -300,7 +300,7 @@ common_chat_params common_chat_params_init_deepseek_v3_2(const common_chat_templ }); if (include_grammar) { - data.grammar_lazy = has_tools && !require_tools; + data.grammar_lazy = !has_response_format && has_tools && !require_tools; data.grammar = build_grammar([&](const common_grammar_builder & builder) { data.parser.build_grammar(builder, data.grammar_lazy); }); diff --git a/common/parsers/gemma4.cpp b/common/parsers/gemma4.cpp index 1a63fa927c40..5902e290bbc6 100644 --- a/common/parsers/gemma4.cpp +++ b/common/parsers/gemma4.cpp @@ -1,152 +1,284 @@ #include "parsers.h" -namespace workaround { - -// Gemma4 uses a custom tool_responses field instead of role:tool messages. -// -// This will transform a sequence of messages: -// assistant(tool_call+) -> tool+ -> assistant(content) -// -// Into a single assistant message containing a tool_responses field: -// assistant(content + tool_call + tool_responses) -// -// This is necessary for the Gemma4 chat template to properly format the prompt. -// See https://ai.google.dev/gemma/docs/core/prompt-formatting-gemma4 -struct gemma4_model_turn_builder { - json & messages; - size_t pos; - json tool_calls = json::array(); - json tool_responses = json::array(); - json content; - json reasoning_content; - - gemma4_model_turn_builder(json & msgs, size_t pos) : messages(msgs), pos(pos) {} - - void collect() { - // Collect the first assistant message - auto & msg = messages[pos]; - if (msg.contains("reasoning_content") && msg.at("reasoning_content").is_string()) { - // According to the prompt formatting guide, we need to preserve reasoning_content - // between function calls. The current chat templates do not support this, but we will do it anyway. - reasoning_content = msg.at("reasoning_content"); - } - for (auto & tc : msg.at("tool_calls")) { - tool_calls.push_back(tc); - } - pos++; +#include "log.h" + +#include +#include +#include +#include + +namespace { + +using gemma4 = common_chat_peg_gemma4_mapper; - // Collect tool call results - while (pos < messages.size() && messages[pos].value("role", "") == "tool") { - collect_result(messages[pos]); - pos++; +struct depth_guard { + int & depth; + depth_guard(int & depth) : depth(depth) { + if (++depth > COMMON_RECURSIVE_LIMIT) { + throw std::invalid_argument("JSON schema is nested more than " + std::to_string(COMMON_RECURSIVE_LIMIT) + " levels deep"); } + } + ~depth_guard() { depth--; } +}; - // Check if the next assistant message is the final message - if (pos < messages.size() && messages[pos].value("role", "") == "assistant") { - auto & next = messages[pos]; - if (!has_tool_calls(next) && has_content(next)) { - content = next.at("content"); - pos++; - } +// Gemma 4 value parsers +void gemma4_value_rules(common_chat_peg_builder & p) { + // The tags are what gemma4_to_json() in the mapper reads, the rules may wrap them freely. + p.rule("gemma4-string", p.tag(gemma4::STRING, p.literal("<|\"|>") + p.tag(gemma4::STRING_CONTENT, p.until("<|\"|>")) + p.literal("<|\"|>"))); + + p.rule("gemma4-bool", p.tag(gemma4::BOOL, p.json_bool())); + + p.rule("gemma4-null", p.tag(gemma4::NULL_, p.json_null())); + + p.rule("gemma4-number", p.tag(gemma4::NUMBER, p.json_number())); + + // Keys are bare in the template, but Gemma 4 occasionally wraps them in quotes + p.rule("gemma4-dict-key", p.tag(gemma4::DICT_KEY, p.choice({ + p.literal("<|\"|>") + p.tag(gemma4::DICT_KEY_NAME, p.until("<|\"|>")) + p.literal("<|\"|>"), + p.literal("\"") + p.tag(gemma4::DICT_KEY_NAME, p.chars("[^\"]", 1, -1)) + p.literal("\""), + p.literal("'") + p.tag(gemma4::DICT_KEY_NAME, p.chars("[^']", 1, -1)) + p.literal("'"), + p.tag(gemma4::DICT_KEY_NAME, p.chars("[^:}]", 1, -1)), + }) + p.literal(":"))); + + p.rule("gemma4-dict-kv", p.tag(gemma4::DICT_KV, p.ref("gemma4-dict-key") + p.space() + p.ref("gemma4-value"))); + + p.rule("gemma4-dict", [&]() { + auto ws = p.space(); + auto member = p.ref("gemma4-dict-kv"); + auto members = p.sequence({member, p.zero_or_more(p.sequence({p.literal(","), ws, member}))}); + return p.tag(gemma4::DICT, p.sequence({ + p.literal("{"), ws, + p.choice({p.literal("}"), p.sequence({members, ws, p.literal("}")})}) + })); + }); + + p.rule("gemma4-array", [&]() { + auto ws = p.space(); + auto value = p.ref("gemma4-value"); + auto elements = p.sequence({value, p.zero_or_more(p.sequence({p.literal(","), ws, value}))}); + return p.tag(gemma4::ARRAY, p.sequence({ + p.literal("["), ws, + p.choice({p.literal("]"), p.sequence({elements, ws, p.literal("]")})}) + })); + }); + + p.rule("gemma4-value", [&]() { + return p.choice({ + p.ref("gemma4-string"), p.ref("gemma4-dict"), p.ref("gemma4-array"), + p.ref("gemma4-number"), p.ref("gemma4-bool"), p.ref("gemma4-null") + }); + }); +} + +// Gemma 4 uses different delimiters and cannot leverage the json-schema-to-grammar implementation. +// This implements a subset to enforce properties. +class gemma4_schema_parser { + common_chat_peg_builder & p_; + std::string root_; + int depth_ = 0; + + std::map refs_; + + public: + gemma4_schema_parser(common_chat_peg_builder & p, std::string root) : p_(p), root_(std::move(root)) {} + + common_peg_parser visit(const common_chat_schema & node) { return visit(node, root_); } + + private: + common_peg_parser visit(const common_chat_schema & node, const std::string & name) { + depth_guard guard(depth_); + switch (node.kind()) { + case common_chat_schema::KIND_REF: return visit_ref(node.as()); + case common_chat_schema::KIND_ANY_OF: return visit_any_of(node.as(), name); + case common_chat_schema::KIND_ALL_OF: return visit_all_of(node.as(), name); + case common_chat_schema::KIND_ARRAY: return visit_array(node.as(), name); + case common_chat_schema::KIND_TUPLE: return visit_tuple(node.as(), name); + case common_chat_schema::KIND_OBJECT: return visit_object(node.as(), name); + default: return p_.ref("gemma4-value"); } } - void collect_result(const json & curr) { - json response; - if (curr.contains("content")) { - const auto & content = curr.at("content"); - if (content.is_string()) { - // Try to parse the content as JSON; fall back to raw string - try { - response = json::parse(content.get()); - } catch (...) { - response = content; + // Whether a schema contributes a shape of its own, as opposed to accepting any value. + static bool shaped(const common_chat_schema & node) { + switch (node.kind()) { + case common_chat_schema::KIND_REF: + return node.as().target != nullptr; + case common_chat_schema::KIND_ANY_OF: + for (const auto & child : node.as().children) { + if (shaped(*child)) { + return true; + } } - } else { - response = content; + return false; + case common_chat_schema::KIND_ALL_OF: { + const auto & children = node.as().children; + return children.size() == 1 && shaped(*children[0]); } + case common_chat_schema::KIND_ARRAY: + case common_chat_schema::KIND_TUPLE: + case common_chat_schema::KIND_OBJECT: + return true; + default: + return false; } + } - std::string name; + // The template writes keys bare. Models keep that for identifiers and quote anything else, + // with single or double quotes or the string delimiter, so those are accepted as well. + common_peg_parser key(const std::string & key) { + auto name = [&]() { return p_.tag(gemma4::DICT_KEY_NAME, p_.literal(key)); }; - // Match name with corresponding tool call - size_t idx = tool_responses.size(); - if (idx < tool_calls.size()) { - auto & tc = tool_calls[idx]; - if (tc.contains("function")) { - name = tc.at("function").value("name", ""); + bool identifier = !key.empty() && !std::isdigit(static_cast(key[0])); + for (char c : key) { + if (!std::isalnum(static_cast(c)) && c != '_') { + identifier = false; } } - - // Fallback to the tool call id - if (name.empty()) { - name = curr.value("tool_call_id", ""); + if (identifier) { + return p_.tag(gemma4::DICT_KEY, name() + p_.literal(":")); } - tool_responses.push_back({{"name", name}, {"response", response}}); + auto alts = p_.choice(); + if (!key.empty() && key.find_first_of(":}") == std::string::npos && std::string("'\"<").find(key[0]) == std::string::npos) { + // Check if key is unambiguous + alts |= name(); + } + // Gemma 4 may emit keys in its string delimiters or quotes + alts |= p_.literal("<|\"|>") + name() + p_.literal("<|\"|>"); + if (key.find('\'') == std::string::npos) { + alts |= p_.literal("'") + name() + p_.literal("'"); + } + if (key.find('"') == std::string::npos) { + alts |= p_.literal("\"") + name() + p_.literal("\""); + } + return p_.tag(gemma4::DICT_KEY, alts + p_.literal(":")); } - json build() { - collect(); - - json msg = { - {"role", "assistant"}, - {"tool_calls", tool_calls}, - }; - if (!tool_responses.empty()) { - msg["tool_responses"] = tool_responses; + common_peg_parser visit_ref(const common_chat_schema_ref & ref) { + if (!ref.target) { + return p_.ref("gemma4-value"); } - if (!content.is_null()) { - msg["content"] = content; + auto it = refs_.find(ref.target); + if (it != refs_.end()) { + return p_.ref(it->second); } - if (!reasoning_content.is_null()) { - msg["reasoning_content"] = reasoning_content; - } - return msg; + // A target reached from several places gets one rule, named by the order it was first seen. + auto rule = root_ + "-ref-" + std::to_string(refs_.size()); + refs_[ref.target] = rule; + return p_.rule(rule, [&]() { return visit(*ref.target, rule + "-value"); }); } - static bool has_content(const json & msg) { - if (!msg.contains("content") || msg.at("content").is_null()) { - return false; + // Shaped alternatives are tried first, then anything else collapses into one generic value. + common_peg_parser visit_any_of(const common_chat_schema_any_of & node, const std::string & name) { + if (!shaped(node)) { + return p_.ref("gemma4-value"); + } + if (node.children.size() == 1) { + return visit(*node.children[0], name); } - const auto & content = msg.at("content"); - if (content.is_string() && !content.get().empty()) { - return true; + auto alts = p_.choice(); + bool loose = false; + for (size_t i = 0; i < node.children.size(); i++) { + if (shaped(*node.children[i])) { + alts |= visit(*node.children[i], name + "-" + std::to_string(i)); + } else { + loose = true; + } } - if (content.is_array() && !content.empty()) { - return true; + if (loose) { + alts |= p_.ref("gemma4-value"); } - return false; + return p_.rule(name, alts); } - static bool has_tool_calls(const json & msg) { - return msg.contains("tool_calls") && msg.at("tool_calls").is_array() && !msg.at("tool_calls").empty(); + common_peg_parser visit_all_of(const common_chat_schema_all_of & node, const std::string & name) { + if (node.children.size() == 1) { + return visit(*node.children[0], name); + } + return p_.ref("gemma4-value"); } -}; -void convert_tool_responses_gemma4(json & messages) { - json result = json::array(); - size_t i = 0; + common_peg_parser visit_array(const common_chat_schema_array & node, const std::string & name) { + auto ws = p_.space(); + auto item = node.items && shaped(*node.items) ? visit(*node.items, name + "-item") : p_.ref("gemma4-value"); + auto elements = item + p_.zero_or_more(p_.literal(",") + ws + item); + return p_.rule(name, p_.tag(gemma4::ARRAY, + p_.literal("[") + ws + (p_.literal("]") | (elements + ws + p_.literal("]"))))); + } - while (i < messages.size()) { - auto & msg = messages[i]; + common_peg_parser visit_tuple(const common_chat_schema_tuple & node, const std::string & name) { + auto ws = p_.space(); + auto items = p_.sequence(); + for (size_t i = 0; i < node.items.size(); i++) { + if (i > 0) { + items = items + p_.literal(",") + ws; + } + items = items + visit(*node.items[i], name + "-" + std::to_string(i)); + } + return p_.rule(name, p_.tag(gemma4::ARRAY, p_.literal("[") + ws + items + ws + p_.literal("]"))); + } + + common_peg_parser visit_object(const common_chat_schema_object & node, const std::string & name) { + auto ws = p_.space(); - if (msg.value("role", "") != "assistant" || !msg.contains("tool_calls") || - !msg.at("tool_calls").is_array() || msg.at("tool_calls").empty()) { - result.push_back(msg); - i++; - continue; + std::vector props; + for (const auto & prop : node.properties) { + props.push_back(&prop); } - gemma4_model_turn_builder builder(messages, i); - result.push_back(builder.build()); - i = builder.pos; - } + // The Gemma 4 jinja template sorts properties, so our rules need to match. + auto lower = [](const std::string & s) { + std::string out = s; + for (auto & c : out) { + c = static_cast(std::tolower(static_cast(c))); + } + return out; + }; + std::stable_sort(props.begin(), props.end(), [&](const auto * a, const auto * b) { + return lower(a->name) < lower(b->name); + }); - messages = result; -} + if (props.empty()) { + if (node.additional_properties) { + return p_.ref("gemma4-dict"); + } + return p_.rule(name, p_.tag(gemma4::DICT, p_.literal("{") + ws + p_.literal("}"))); + } -} + std::optional rest; + std::optional first; + if (node.additional_properties) { + auto kv = p_.ref("gemma4-dict-kv"); + rest = p_.zero_or_more(p_.literal(",") + ws + kv); + first = p_.optional(kv + *rest); + } + + auto then_rest = [&](const common_peg_parser & parser) { + return rest ? parser + *rest : parser; + }; + + for (size_t i = props.size(); i-- > 0;) { + const auto & prop = *props[i]; + auto kv = p_.tag(gemma4::DICT_KV, key(prop.name) + ws + visit(*prop.schema, name + "-" + std::to_string(i))); + + common_peg_parser rest_i = then_rest(p_.literal(",") + ws + kv); + common_peg_parser first_i = then_rest(kv); + if (!prop.required) { + rest_i = then_rest(p_.optional(p_.literal(",") + ws + kv)); + first_i = first ? (first_i | *first) : p_.optional(first_i); + } + + if (i > 0) { + rest = p_.rule(name + "-rest-" + std::to_string(i), rest_i); + } + first = first_i; + } + + return p_.rule(name, p_.tag(gemma4::DICT, p_.literal("{") + ws + *first + ws + p_.literal("}"))); + } +}; + +} // namespace common_chat_params common_chat_params_init_gemma4(const common_chat_template & tmpl, const autoparser::generation_params & inputs) { @@ -218,51 +350,29 @@ common_chat_params common_chat_params_init_gemma4(const common_chat_template & } if (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE) { - // Gemma4 tool calling syntax - // Rules should match traversal logic in gemma4_to_json() - p.rule("gemma4-string-content", p.until("<|\"|>")); - p.rule("gemma4-string", p.literal("<|\"|>") + p.ref("gemma4-string-content") + p.literal("<|\"|>")); - p.rule("gemma4-bool", p.json_bool()); - p.rule("gemma4-null", p.json_null()); - p.rule("gemma4-number", p.json_number()); - p.rule("gemma4-dict-key", p.rule("gemma4-dict-key-name", p.chars("[^:}]", 1, -1)) + p.literal(":")); - p.rule("gemma4-dict-kv", p.ref("gemma4-dict-key") + p.space() + p.ref("gemma4-value")); - p.rule("gemma4-dict", [&]() { - auto ws = p.space(); - auto member = p.ref("gemma4-dict-kv"); - auto members = p.sequence({member, p.zero_or_more(p.sequence({p.literal(","), ws, member}))}); - return p.sequence({ - p.literal("{"), ws, - p.choice({p.literal("}"), p.sequence({members, ws, p.literal("}")})}) - }); - }); - p.rule("gemma4-array", [&]() { - auto ws = p.space(); - auto value = p.ref("gemma4-value"); - auto elements = p.sequence({value, p.zero_or_more(p.sequence({p.literal(","), ws, value}))}); - return p.sequence({ - p.literal("["), ws, - p.choice({p.literal("]"), p.sequence({elements, ws, p.literal("]")})}) - }); - }); - p.rule("gemma4-value", [&]() { - return p.choice({ - p.ref("gemma4-string"), p.ref("gemma4-dict"), p.ref("gemma4-array"), - p.ref("gemma4-number"), p.ref("gemma4-bool"), p.ref("gemma4-null") - }); - }); + gemma4_value_rules(p); auto tool_choice = p.choice(); foreach_function(inputs.tools, [&](size_t tool_index, const json & tool) { const auto & function = tool.at("function"); std::string name = function.at("name"); - // TODO @aldehir : need to extend json-schema-to-grammar to produce more than JSON rules - // const auto & params = function.at("parameters"); + + // Arguments follow the shape of the parameter schema when it is an object, + // see gemma4_schema_parser, and are a generic dict otherwise. + auto args = p.ref("gemma4-dict"); + try { + auto doc = common_chat_schema_from_json(common_chat_tool_parameters(function)); + if (doc.root->kind() == common_chat_schema::KIND_OBJECT) { + args = gemma4_schema_parser(p, "tool-" + std::to_string(tool_index) + "-args").visit(*doc.root); + } + } catch (const std::exception & e) { + LOG_WRN("%s: tool %s uses the generic argument parser: %s\n", __func__, name.c_str(), e.what()); + } tool_choice |= p.rule("tool-" + std::to_string(tool_index), p.tool(p.sequence({ p.tool_open(p.tool_name(p.literal(name)) + p.peek(p.literal("{"))), - p.tool_args(p.ref("gemma4-dict")), + p.tool_args(args), }))); }); diff --git a/common/parsers/gpt-oss.cpp b/common/parsers/gpt-oss.cpp index 5ccdff57de5a..4cbf929f9bfd 100644 --- a/common/parsers/gpt-oss.cpp +++ b/common/parsers/gpt-oss.cpp @@ -71,6 +71,7 @@ common_chat_params common_chat_params_init_gpt_oss(const common_chat_template & auto content = p.rule("message-content", p.until("<|end|>")); auto channel = p.literal("<|channel|>") + (p.literal("commentary") | p.literal("analysis")); auto constrain_type = p.chars("[A-Za-z0-9_-]", 1, -1); + auto constraint = p.optional(p.space() + p.optional(p.literal("<|constrain|>")) + constrain_type); // Occasionally, gpt-oss-20b will prefix channels with this commentary auto stray_commentary = p.optional(p.literal("<|channel|>commentary") + p.optional(p.literal(" to=assistant"))); @@ -84,7 +85,7 @@ common_chat_params common_chat_params_init_gpt_oss(const common_chat_template & auto analysis = p.ref("analysis"); auto preamble = p.rule("preamble", p.literal("<|channel|>commentary<|message|>") + p.content(content) + end); - auto final_msg = p.rule("final", stray_commentary + p.literal("<|channel|>final<|message|>") + p.content(content)); + auto final_msg = p.rule("final", stray_commentary + p.literal("<|channel|>final") + constraint + p.literal("<|message|>") + p.content(content)); // Consume any unsolicited tool calls, e.g. builtin functions auto unsolicited = p.rule("unsolicited", p.atomic(p.optional(channel) + p.literal(" to=") + content + end)); @@ -92,7 +93,6 @@ common_chat_params common_chat_params_init_gpt_oss(const common_chat_template & auto any = p.rule("any", preamble | analysis); if (has_response_format) { - auto constraint = p.optional(p.space() + p.optional(p.literal("<|constrain|>")) + constrain_type); auto response_format = p.rule("response-format", p.literal("<|channel|>final") + constraint + p.literal("<|message|>") + p.content(p.schema(p.json(), "response-format-schema", inputs.json_schema))); @@ -109,7 +109,6 @@ common_chat_params common_chat_params_init_gpt_oss(const common_chat_template & const auto params = common_chat_tool_parameters(function); auto func_name = p.literal(" to=functions.") + p.tool_name(p.literal(name)); - auto constraint = p.optional(p.space() + p.optional(p.literal("<|constrain|>")) + constrain_type); auto args = p.tool_args(p.schema(p.json(), "tool-" + std::to_string(tool_index) + "-schema", params)); // recipient in role header diff --git a/common/parsers/ministral3.cpp b/common/parsers/ministral3.cpp index 634163959307..c865a3cae40a 100644 --- a/common/parsers/ministral3.cpp +++ b/common/parsers/ministral3.cpp @@ -109,7 +109,7 @@ common_chat_params common_chat_params_init_ministral_3(const common_chat_templat }); if (include_grammar) { - data.grammar_lazy = has_tools && inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_AUTO; + data.grammar_lazy = !has_response_format && has_tools && inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_AUTO; data.grammar = build_grammar([&](const common_grammar_builder & builder) { data.parser.build_grammar(builder, data.grammar_lazy); diff --git a/common/parsers/muse-spark.cpp b/common/parsers/muse-spark.cpp index 577090aa5cc9..945ee2aaf7ba 100644 --- a/common/parsers/muse-spark.cpp +++ b/common/parsers/muse-spark.cpp @@ -46,9 +46,10 @@ common_chat_params common_chat_params_init_muse_spark(const common_chat_template auto extract_reasoning = inputs.reasoning_format != COMMON_REASONING_FORMAT_NONE; - auto has_tools = inputs.tools.is_array() && !inputs.tools.empty(); - // Constrained grammar whenever tools are offered. - auto include_grammar = has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE; + auto has_tools = inputs.tools.is_array() && !inputs.tools.empty(); + auto has_response_format = !inputs.json_schema.is_null() && inputs.json_schema.is_object(); + // Constrained grammar whenever tools are offered or a response format is requested. + auto include_grammar = has_response_format || (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE); data.parser = build_chat_peg_parser([&](common_chat_peg_builder & p) { auto start = p.rule("start", p.literal("<|start|>assistant")); @@ -75,6 +76,15 @@ common_chat_params common_chat_params_init_muse_spark(const common_chat_template auto final_msg = p.rule("final", recipient + p.literal("<|message|>") + p.content(p.until_one_of({ "<|eot|>", "<|eom|>" }))); + if (has_response_format) { + auto response_json = p.content(p.schema(p.json(), "response-format-schema", inputs.json_schema)); + auto response_format = p.rule("response-format", + recipient + p.literal("<|message|>") + + ((p.literal("```json") + p.space() + response_json + p.space() + p.literal("```")) | response_json)); + + return p.zero_or_more(start + analysis) + start + response_format; + } + if (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE) { auto string_value = p.ac( p.tool_arg_string_value(p.until("")) + p.tool_arg_close(p.literal("")), @@ -132,7 +142,7 @@ common_chat_params common_chat_params_init_muse_spark(const common_chat_template }); if (include_grammar) { - data.grammar_lazy = inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_REQUIRED; + data.grammar_lazy = !(has_response_format || (has_tools && inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED)); data.grammar = build_grammar([&](const common_grammar_builder & builder) { data.parser.build_grammar(builder, data.grammar_lazy); }); diff --git a/common/parsers/parsers.h b/common/parsers/parsers.h index 532a44f014fe..b1b6d166a124 100644 --- a/common/parsers/parsers.h +++ b/common/parsers/parsers.h @@ -41,12 +41,6 @@ std::string common_chat_template_generation_prompt_impl( bool is_lfm2_template(const std::string & src); -namespace workaround { - -void convert_tool_responses_gemma4(json & messages); - -} - common_chat_params common_chat_params_init_cohere2moe(const common_chat_template & tmpl, const autoparser::generation_params & inputs); common_chat_params common_chat_params_init_deepseek_v3_2(const common_chat_template & tmpl, const autoparser::generation_params & inputs); diff --git a/common/parsers/qwen3-coder.cpp b/common/parsers/qwen3-coder.cpp index a68a6835e94b..b8dc0226722b 100644 --- a/common/parsers/qwen3-coder.cpp +++ b/common/parsers/qwen3-coder.cpp @@ -191,7 +191,7 @@ common_chat_params common_chat_params_init_qwen3_coder(const common_chat_templat }); if (include_grammar) { - data.grammar_lazy = has_tools && inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_AUTO; + data.grammar_lazy = !has_response_format && has_tools && inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_AUTO; data.grammar = build_grammar([&](const common_grammar_builder & builder) { data.parser.build_grammar(builder, data.grammar_lazy); diff --git a/common/preset.cpp b/common/preset.cpp index 31a8bd5131e4..3d90c242a741 100644 --- a/common/preset.cpp +++ b/common/preset.cpp @@ -385,7 +385,7 @@ static bool is_draft_file(const std::string & fname) { } common_presets common_preset_context::load_from_models_dir(const std::string & models_dir) const { - const std::filesystem::path dir = std::filesystem::u8path(models_dir); + const std::filesystem::path dir = models_dir; if (!std::filesystem::exists(dir) || !std::filesystem::is_directory(dir)) { throw std::runtime_error(string_format("error: '%s' does not exist or is not a directory\n", models_dir.c_str())); } diff --git a/common/speculative.cpp b/common/speculative.cpp index 4305ac6acb41..a7bd221127a3 100644 --- a/common/speculative.cpp +++ b/common/speculative.cpp @@ -10,8 +10,6 @@ #include "ngram-mod.h" #include "sampling.h" -#include "../src/llama-ext.h" // staging API: llama_set_embeddings_nextn / llama_get_embeddings_nextn_ith (used by MTP) - #include #include #include @@ -1027,6 +1025,9 @@ struct common_speculative_impl_draft_dflash : public common_speculative_impl { const int32_t * target_layer_ids = nullptr; // model_dft's extract layer indices uint32_t target_layer_ids_n = 0; + // inject target features from the target's device buffer instead of host copies (env LLAMA_DFLASH_DEV=0 disables) + bool use_dev_features = false; + common_speculative_impl_draft_dflash(const common_params_speculative & params, uint32_t n_seq, common_speculative_type type = COMMON_SPECULATIVE_TYPE_DRAFT_DFLASH) : common_speculative_impl(type, n_seq, params.draft.n_max) @@ -1141,6 +1142,23 @@ struct common_speculative_impl_draft_dflash : public common_speculative_impl { llama_set_embeddings_layer_inp(ctx_tgt, (uint32_t) target_layer_ids[k], true); } + // the device buffer keeps layers in ascending id order, the fc input expects target_layer_ids order + { + const char * env = getenv("LLAMA_DFLASH_DEV"); + const bool ascending = std::adjacent_find(target_layer_ids, target_layer_ids + target_layer_ids_n, + [](int32_t a, int32_t b) { return a >= b; }) == target_layer_ids + target_layer_ids_n; + + use_dev_features = !is_lilicorr && ascending && (env == nullptr || atoi(env) != 0); + if (use_dev_features) { + llama_set_embeddings_layer_inp_dev(ctx_tgt, true); + use_dev_features = llama_supports_embd_row(ctx_dft); + if (!use_dev_features) { + llama_set_embeddings_layer_inp_dev(ctx_tgt, false); + } + } + LOG_INF("%s: DFlash target features via %s\n", __func__, use_dev_features ? "device buffer" : "host copies"); + } + // DFlash2 reads its selector lattice from h_nextn and never consumes raw logits. llama_set_embeddings_nextn(ctx_dft, true, /*masked*/ !is_dflash2 && !is_lilicorr); llama_set_causal_attn(ctx_dft, causal_attn); // DFlash needs non-causal attention unless the model says otherwise @@ -1233,6 +1251,25 @@ struct common_speculative_impl_draft_dflash : public common_speculative_impl { for (int32_t offset = 0; offset < n_rows; offset += n_ubatch) { const int32_t n_chunk = std::min(n_ubatch, n_rows - offset); + if (use_dev_features) { + // the features are the target batch rows in the target's device buffer + batch_inject.clear(); + for (int32_t i = 0; i < n_chunk; ++i) { + const int32_t row = i_batch_beg[seq_id] + offset + i; + const llama_pos p = batch_in.tokens[row].pos[0]; + const llama_pos pos_arr[4] = { p, p, p, 0 }; + batch_inject.add_embd_row(row, pos_arr, seq_id, false); + } + + const int32_t rc = llama_process(ctx_dft, LLAMA_PROCESS_TYPE_DECODE, batch_inject.get()); + if (rc != 0) { + LOG_ERR("%s: llama_process(ctx_dft) failed rc=%d (n_tokens=%d, offset=%d)\n", + __func__, rc, (int) n_chunk, (int) offset); + return false; + } + continue; + } + // gather target features per extract layer; the fused decode encodes and // injects them into the K/V cache at the target positions features_buf.resize((size_t) n_chunk * n_embd_enc); @@ -1524,6 +1561,13 @@ struct common_speculative_impl_draft_mtp : public common_speculative_impl { std::vector i_last; std::vector> chain_h; + // target h rows on the device (llama_set_embeddings_nextn_dev): the catch-up in process() reads row k - 1 of the last + // target decode in place, and row n_batch_tgt + seq_id for the first token of seq_id, uploaded from pending_h + // pending_h stays on the host: process() and accept() fetch it before the next target decode overwrites the rows + bool dev_rows = false; + int32_t n_batch_tgt = 0; + std::vector verify_row0; // [n_seq] device row of the first verify row, see verify_h_rows + common_speculative_impl_draft_mtp(const common_params_speculative & params, uint32_t n_seq) : common_speculative_impl(COMMON_SPECULATIVE_TYPE_DRAFT_MTP, n_seq, params.draft.n_max) , params(params.draft) @@ -1574,12 +1618,12 @@ struct common_speculative_impl_draft_mtp : public common_speculative_impl { } } - llama_set_embeddings_nextn(ctx_tgt, true, /*masked*/ false); - llama_set_embeddings_nextn(ctx_dft, true, /*masked*/ true); - is_mem_shared = llama_get_ctx_other(ctx_dft) == ctx_tgt; chain_heads = n_mtp_layers > 1 && !is_mem_shared; + llama_set_embeddings_nextn(ctx_tgt, true, /*masked*/ is_mem_shared); + llama_set_embeddings_nextn(ctx_dft, true, /*masked*/ true); + if (chain_heads) { this->params.n_max = std::min(this->params.n_max, n_mtp_layers); @@ -1598,10 +1642,27 @@ struct common_speculative_impl_draft_mtp : public common_speculative_impl { verify_h.assign(n_seq, {}); verify_h_rows.assign(n_seq, 0); + + n_batch_tgt = (int32_t) llama_n_batch(ctx_tgt); + verify_row0.assign(n_seq, -1); + + // shared memory and chained heads keep the host rows + if (!is_mem_shared && !chain_heads) { + dev_rows = llama_set_embeddings_nextn_dev(ctx_tgt, true) && llama_set_ctx_state_src(ctx_dft, ctx_tgt); + if (!dev_rows) { + llama_set_ctx_state_src(ctx_dft, nullptr); + llama_set_embeddings_nextn_dev(ctx_tgt, false); + } + } + SPC_TRC("- target h rows on the device: %s\n", dev_rows ? "yes" : "no"); } ~common_speculative_impl_draft_mtp() override { auto * ctx_dft = this->params.ctx_dft; + if (dev_rows) { + llama_set_ctx_state_src(ctx_dft, nullptr); + llama_set_embeddings_nextn_dev(this->params.ctx_tgt, false); + } for (llama_seq_id seq_id = 0; seq_id < (llama_seq_id) backend_chains.size(); ++seq_id) { if (backend_chains[seq_id] == nullptr) { continue; @@ -1676,7 +1737,16 @@ struct common_speculative_impl_draft_mtp : public common_speculative_impl { // i.e. we cannot have seq_id like this: [0, 0, 0, 1, 1, 0, 1, 1] // ^--- this is a problem // TODO:this is generally true, but would be nice to assert it - const float * h_tgt = llama_get_embeddings_nextn(ctx_tgt); + if (dev_rows) { + GGML_ASSERT(n_tokens <= n_batch_tgt && "the target batch exceeds the h_nextn device buffer"); + for (llama_seq_id seq_id = 0; seq_id < (llama_seq_id) n_seq; ++seq_id) { + if (i_batch_beg[seq_id] >= 0) { + llama_set_embeddings_nextn_rows(ctx_tgt, n_batch_tgt + seq_id, 1, pending_h[seq_id].data()); + } + } + } + + const float * h_tgt = dev_rows ? nullptr : llama_get_embeddings_nextn(ctx_tgt); for (int k = 0; k < n_tokens; ++k) { const auto & t = batch_in.tokens[k]; @@ -1688,6 +1758,11 @@ struct common_speculative_impl_draft_mtp : public common_speculative_impl { ? batch.add(t.id, t.pos[0], seq_id, false) : batch.add_embd(t.embd, t.pos.data(), seq_id, false); + if (dev_rows) { + batch.set_embd_state_row(idx, k == i_batch_beg[seq_id] ? n_batch_tgt + seq_id : k - 1); + continue; + } + const float * h_row = k == i_batch_beg[seq_id] ? pending_h[seq_id].data() : h_tgt + (size_t) (k - 1) * n_embd; @@ -1733,16 +1808,36 @@ struct common_speculative_impl_draft_mtp : public common_speculative_impl { } const int32_t n_rows = i_batch_end[seq_id] - i_batch_beg[seq_id] + 1; - verify_h_rows[seq_id] = n_rows; - verify_h[seq_id].resize((size_t) n_rows * n_embd); + if (is_mem_shared && !batch_in.tokens[i_batch_end[seq_id]].output) { + verify_h_rows[seq_id] = 0; + continue; + } + + const bool all_rows_output = std::all_of( + batch_in.tokens.begin() + i_batch_beg[seq_id], + batch_in.tokens.begin() + i_batch_end[seq_id] + 1, + [](const common_batch::token & token) { return token.output; }); + const int32_t h_rows = is_mem_shared && !all_rows_output ? 1 : n_rows; + const int32_t h_offset = is_mem_shared && !all_rows_output ? i_batch_end[seq_id] : i_batch_beg[seq_id]; + + verify_h_rows[seq_id] = h_rows; + + if (dev_rows) { + // the rows stay on the device until the next target decode, only the pending row comes to the host + verify_row0[seq_id] = h_offset; + llama_get_embeddings_nextn_rows(ctx_tgt, h_offset + h_rows - 1, 1, pending_h[seq_id].data()); + continue; + } - for (int32_t i = 0; i < n_rows; ++i) { - const float * h = llama_get_embeddings_nextn_ith(ctx_tgt, i_batch_beg[seq_id] + i); + verify_h[seq_id].resize((size_t) h_rows * n_embd); + + for (int32_t i = 0; i < h_rows; ++i) { + const float * h = llama_get_embeddings_nextn_ith(ctx_tgt, h_offset + i); std::memcpy(verify_h[seq_id].data() + (size_t) i * n_embd, h, row_bytes); } std::memcpy(pending_h[seq_id].data(), - verify_h[seq_id].data() + (size_t) (n_rows - 1) * n_embd, row_bytes); + verify_h[seq_id].data() + (size_t) (h_rows - 1) * n_embd, row_bytes); } return true; @@ -1925,9 +2020,39 @@ struct common_speculative_impl_draft_mtp : public common_speculative_impl { } const int32_t i_h = std::min(n_accepted, n_rows - 1); + if (dev_rows) { + // process() already fetched the last row + if (i_h != n_rows - 1) { + llama_get_embeddings_nextn_rows(params.ctx_tgt, verify_row0[seq_id] + i_h, 1, pending_h[seq_id].data()); + } + return; + } const size_t row_bytes = (size_t) n_embd * sizeof(float); std::memcpy(pending_h[seq_id].data(), verify_h[seq_id].data() + (size_t) i_h * n_embd, row_bytes); } + + bool get_state(llama_seq_id seq_id, std::vector & data) const override { + if (seq_id < 0 || seq_id >= (llama_seq_id) n_seq) { + return false; + } + + const size_t row_bytes = (size_t) n_embd * sizeof(float); + data.resize(row_bytes); + std::memcpy(data.data(), pending_h[seq_id].data(), row_bytes); + return true; + } + + void set_state(llama_seq_id seq_id, const std::vector & data) override { + if (seq_id < 0 || seq_id >= (llama_seq_id) n_seq) { + return; + } + + const size_t row_bytes = (size_t) n_embd * sizeof(float); + if (data.size() != row_bytes) { + return; + } + std::memcpy(pending_h[seq_id].data(), data.data(), row_bytes); + } }; // state of self-speculation (simple implementation, not ngram-map) @@ -2731,7 +2856,7 @@ common_speculative_init_result::common_speculative_init_result( model_path = params.speculative.draft.mparams.path; LOG_INF("%s: loading draft model '%s'\n", __func__, model_path.c_str()); - llama_model * model_dft = llama_model_load_from_file(model_path.c_str(), mparams); + llama_model * model_dft = common_model_load(params.speculative.draft.mparams.get_paths(), mparams); if (model_dft == NULL) { LOG_ERR("%s: failed to load draft model, '%s'\n", __func__, model_path.c_str()); return; diff --git a/conversion/__init__.py b/conversion/__init__.py index 6a34da41fe4a..6800093ae4be 100644 --- a/conversion/__init__.py +++ b/conversion/__init__.py @@ -20,6 +20,7 @@ "NEOChatModel": "sensenova_u1", "AfmoeForCausalLM": "afmoe", "LagunaForCausalLM": "laguna", + "RicursiveForCausalLM": "ricursive", "FlintForCausalLM": "flint", "ApertusForCausalLM": "llama", "ArceeForCausalLM": "llama", @@ -45,6 +46,7 @@ "ChameleonForConditionalGeneration": "chameleon", "ChatGLMForConditionalGeneration": "chatglm", "ChatGLMModel": "chatglm", + "ChatterboxModel": "chatterbox", "ClefModel": "clef", "CloverMoeModelLM": "clover", "CodeShellForCausalLM": "codeshell", @@ -61,6 +63,7 @@ "DeepseekV2ForCausalLM": "deepseek", "DeepseekV3ForCausalLM": "deepseek", "DeepseekV32ForCausalLM": "deepseek", + "Xing4_0ForCausalLM": "xing", "DFlashDraftModel": "qwen", "DFlash2DraftModel": "qwen", "LiLiCorrDraftModel": "nemotron", @@ -94,6 +97,7 @@ "FalconForCausalLM": "falcon", "FalconH1ForCausalLM": "falcon_h1", "FalconMambaForCausalLM": "mamba", + "FishOmniModel": "drama3", "GPT2LMHeadModel": "gpt2", "GPTBigCodeForCausalLM": "starcoder", "GPTNeoXForCausalLM": "gptneox", @@ -154,6 +158,7 @@ "KORMoForCausalLM": "qwen", "KimiK25ForConditionalGeneration": "deepseek", "KimiK3ForConditionalGeneration": "kimi_k3", + "KimiK31ForConditionalGeneration": "kimi_k3", "KimiLinearForCausalLM": "kimi_linear", "KimiLinearModel": "kimi_linear", "KimiVLForConditionalGeneration": "deepseek", @@ -188,6 +193,7 @@ "Eagle3LlamaForCausalLM": "llama", "LlamaForCausalLMEagle3": "llama", "LlavaForConditionalGeneration": "llama", + "LlavaOnevision2ForConditionalGeneration": "qwen", "LlavaStableLMEpochForCausalLM": "stablelm", "MPTForCausalLM": "mpt", "MT5ForConditionalGeneration": "t5", @@ -197,6 +203,7 @@ "MambaLMHeadModel": "mamba", "MapleForCausalLM": "maple", "MellumForCausalLM": "mellum", + "Metis16ForCausalLM": "metis", "MiMoV2FlashForCausalLM": "mimo", "MiMoV2ForCausalLM": "mimo", "MiniCPM3ForCausalLM": "minicpm", @@ -223,6 +230,7 @@ "ModernBertForSequenceClassification": "bert", "ModernBertModel": "bert", "NanbeigeForCausalLM": "nanbeige", + "Nemotron3_5TranscribeForConditionalGeneration": "nemotron", "NemotronForCausalLM": "nemotron", "NemotronHForCausalLM": "nemotron", "NemotronHPuzzleForCausalLM": "nemotron", @@ -328,6 +336,7 @@ MMPROJ_MODEL_MAP: dict[str, str] = { "AudioFlamingo3ForConditionalGeneration": "ultravox", + "ChatterboxModel": "chatterbox", "ClefModel": "clef", "CogVLMForCausalLM": "cogvlm", "Cohere2VisionForConditionalGeneration": "command_r", @@ -342,6 +351,7 @@ "DotsOCRForCausalLM": "dotsocr", "EmbeddingGemma2Model": "gemma", "Exaone4_5_ForConditionalGeneration": "exaone", + "FishOmniModel": "drama3", "Gemma3ForConditionalGeneration": "gemma", "Gemma3nForConditionalGeneration": "gemma", "Gemma4ForConditionalGeneration": "gemma", @@ -361,6 +371,7 @@ "InternVisionModel": "internvl", "JanusForConditionalGeneration": "januspro", "KimiK25ForConditionalGeneration": "kimivl", + "KimiK31ForConditionalGeneration": "kimivl", "KimiVLForConditionalGeneration": "kimivl", "Lfm2AudioForConditionalGeneration": "lfm2", "D1OmniModel": "lfm2", @@ -371,6 +382,7 @@ "LightOnOCRForConditionalGeneration": "lighton_ocr", "Llama4ForConditionalGeneration": "llama4", "LlavaForConditionalGeneration": "llava", + "LlavaOnevision2ForConditionalGeneration": "magevl", "MERaLiON2ForConditionalGeneration": "ultravox", "MiMoV2ForCausalLM": "mimo", "MiniMaxM3SparseForConditionalGeneration": "minimax", @@ -380,6 +392,7 @@ "MiniCPMOcrForConditionalGeneration": "minicpm", "Mistral3ForConditionalGeneration": "llava", "Mistral3Model": "llava", + "Nemotron3_5TranscribeForConditionalGeneration": "nemotron", "NemotronH_Omni_Reasoning_V3": "nemotron", "NemotronH_Nano_VL_V2": "nemotron", "MuseGlimmerForConditionalGeneration": "muse_glimmer", diff --git a/conversion/bailingmoe.py b/conversion/bailingmoe.py index 351be1df1758..18fe8a2b7006 100644 --- a/conversion/bailingmoe.py +++ b/conversion/bailingmoe.py @@ -1,5 +1,6 @@ from __future__ import annotations +import re from typing import Callable, Iterable, TYPE_CHECKING import torch @@ -198,6 +199,18 @@ class SarvamMoEModel(BailingMoeV2Model): # - full rotary (no partial_rotary_factor) # - expert bias is zero-mean normalized at load time + def set_vocab(self): + # BPE with byte fallback: mark <0xXX> tokens as BYTE so they decode to raw bytes + tokens, toktypes, tokpre = self.get_vocab_base() + toktypes = [gguf.TokenType.BYTE if re.fullmatch(r"<0x[0-9A-F]{2}>", tok) else typ for tok, typ in zip(tokens, toktypes)] + self.gguf_writer.add_tokenizer_model("gpt2") + self.gguf_writer.add_tokenizer_pre(tokpre) + self.gguf_writer.add_token_list(tokens) + self.gguf_writer.add_token_types(toktypes) + + special_vocab = gguf.SpecialVocab(self.dir_model, load_merges=True) + special_vocab.add_to_gguf(self.gguf_writer) + def set_gguf_parameters(self): super().set_gguf_parameters() hparams = self.hparams diff --git a/conversion/base.py b/conversion/base.py index 6347d7fd32a5..58ddb5defe55 100644 --- a/conversion/base.py +++ b/conversion/base.py @@ -7,6 +7,7 @@ import logging import contextlib import json +import math import os import re import sys @@ -31,10 +32,10 @@ from gguf.vocab import MistralTokenizerType, MistralVocab try: - from mistral_common.tokens.tokenizers.base import TokenizerVersion # type: ignore[import-not-found, ty:unresolved-import] - from mistral_common.tokens.tokenizers.multimodal import DATASET_MEAN as _MISTRAL_COMMON_DATASET_MEAN, DATASET_STD as _MISTRAL_COMMON_DATASET_STD # type: ignore[import-not-found, ty:unresolved-import] - from mistral_common.tokens.tokenizers.tekken import Tekkenizer # type: ignore[import-not-found, ty:unresolved-import] - from mistral_common.tokens.tokenizers.sentencepiece import ( # type: ignore[import-not-found, ty:unresolved-import] + from mistral_common.tokens.tokenizers.base import TokenizerVersion # ty: ignore[import-not-found] + from mistral_common.tokens.tokenizers.multimodal import DATASET_MEAN as _MISTRAL_COMMON_DATASET_MEAN, DATASET_STD as _MISTRAL_COMMON_DATASET_STD # ty: ignore[import-not-found] + from mistral_common.tokens.tokenizers.tekken import Tekkenizer # ty: ignore[import-not-found] + from mistral_common.tokens.tokenizers.sentencepiece import ( # ty: ignore[import-not-found] SentencePieceTokenizer, ) @@ -133,7 +134,8 @@ def __init__(self, dir_model: Path, ftype: gguf.LlamaFileType, fname_out: Path, target_model_dir: Path | None = None, fuse_gate_up_exps: bool = False, fp8_as_q8: bool = False, - fuse_qkv: bool = False): + fuse_qkv: bool = False, + ple_only: bool = False, no_ple: bool = False): if type(self) is ModelBase or \ type(self) is TextModel or \ type(self) is MmprojModel: @@ -142,6 +144,11 @@ def __init__(self, dir_model: Path, ftype: gguf.LlamaFileType, fname_out: Path, if self.is_mistral_format and not _mistral_common_installed: raise ImportError(_mistral_import_error_msg) + if ple_only and no_ple: + raise ValueError("ple_only and no_ple are mutually exclusive") + self.ple_only = ple_only + self.no_ple = no_ple + self._has_ple = False self.dir_model = dir_model self.ftype = ftype self.fname_out = fname_out @@ -167,6 +174,8 @@ def __init__(self, dir_model: Path, ftype: gguf.LlamaFileType, fname_out: Path, self._fusable_qkv_weight_layers: set[int] = set() self._fusable_qkv_bias_layers: set[int] = set() self.hparams = ModelBase.load_hparams(self.dir_model, self.is_mistral_format) if hparams is None else hparams + # source tensor name to filtered tensor name mapping. + self._source_tensor_names: dict[str, str | None] = {} self.model_tensors = self.index_tensors(remote_hf_model_id=remote_hf_model_id) self.metadata_override = metadata_override self.model_name = model_name @@ -202,7 +211,8 @@ def __init__(self, dir_model: Path, ftype: gguf.LlamaFileType, fname_out: Path, # Configure GGUF Writer self.gguf_writer = gguf.GGUFWriter(path=None, arch=gguf.MODEL_ARCH_NAMES[self.model_arch], endianess=self.endianess, use_temp_file=self.use_temp_file, - split_max_tensors=split_max_tensors, split_max_size=split_max_size, dry_run=dry_run, small_first_shard=small_first_shard) + split_max_tensors=split_max_tensors, split_max_size=split_max_size, dry_run=dry_run, small_first_shard=small_first_shard, + tensor_filter=self._include_ple_tensor if ple_only or no_ple else None) # Mistral specific self.disable_mistral_community_chat_template = disable_mistral_community_chat_template @@ -225,21 +235,25 @@ def index_tensors(self, remote_hf_model_id: str | None = None) -> dict[str, Call tensors: dict[str, Callable[[], Tensor]] = {} if remote_hf_model_id is not None: - is_safetensors = True - logger.info(f"Using remote model with HuggingFace id: {remote_hf_model_id}") remote_tensors = gguf.utility.SafetensorRemote.get_list_tensors_hf_model(remote_hf_model_id) for name, remote_tensor in remote_tensors.items(): data_gen = lambda r=remote_tensor: LazyTorchTensor.from_remote_tensor(r) # noqa: E731 - if titem := self.filter_tensors((name, data_gen)): + titem = self.filter_tensors((name, data_gen)) + self._source_tensor_names[name] = titem[0] if titem else None + if titem: tname, tgen = titem tensors[tname] = tgen return tensors - prefix = "model" if not self.is_mistral_format else "consolidated" + return self.index_local_tensors("model" if not self.is_mistral_format else "consolidated") + + # index the local "*.safetensors" files (or pytorch_model*.bin), through filter_tensors + def index_local_tensors(self, prefix: str) -> dict[str, Callable[[], Tensor]]: + tensors: dict[str, Callable[[], Tensor]] = {} part_names: list[str] = ModelBase.get_model_part_names(self.dir_model, prefix, ".safetensors") - is_safetensors: bool = len(part_names) > 0 or (not self.is_mistral_format and (self.dir_model / "model.safetensors.index.json").is_file()) + is_safetensors: bool = len(part_names) > 0 or (not self.is_mistral_format and (self.dir_model / f"{prefix}.safetensors.index.json").is_file()) if not is_safetensors: part_names = ModelBase.get_model_part_names(self.dir_model, "pytorch_model", ".bin") @@ -247,7 +261,7 @@ def index_tensors(self, remote_hf_model_id: str | None = None) -> dict[str, Call tensor_names_from_parts: set[str] = set() if not self.is_mistral_format: - index_name = "model.safetensors" if is_safetensors else "pytorch_model.bin" + index_name = f"{prefix}.safetensors" if is_safetensors else "pytorch_model.bin" index_name += ".index.json" index_file = self.dir_model / index_name @@ -259,7 +273,7 @@ def index_tensors(self, remote_hf_model_id: str | None = None) -> dict[str, Call if weight_map is None or not isinstance(weight_map, dict): raise ValueError(f"Can't load 'weight_map' from {index_name!r}") tensor_names_from_index.update(weight_map.keys()) - part_dict: dict[str, None] = dict.fromkeys(weight_map.values(), None) # ty: ignore[invalid-assignment] + part_dict: dict[str, None] = dict.fromkeys(weight_map.values(), None) part_names = sorted(part_dict.keys()) else: weight_map = {} @@ -292,7 +306,9 @@ def index_tensors(self, remote_hf_model_id: str | None = None) -> dict[str, Call data_gen = lambda data=data_torch: LazyTorchTensor.from_eager(data) # noqa: E731 else: data_gen = lambda data=data_torch: data # noqa: E731 - if titem := self.filter_tensors((name, data_gen)): + titem = self.filter_tensors((name, data_gen)) + self._source_tensor_names[name] = titem[0] if titem else None + if titem: tname, tgen = titem tensors[tname] = tgen @@ -328,6 +344,36 @@ def _write_scales_tensor(self, scale_name: str, scales: list[float]): logger.info(f" + {scale_name} (per-expert scale, shape [{len(scales)}])") self.gguf_writer.add_tensor(scale_name, scale_vals) + def _prepare_kv_cache_scales(self): + for name in list(self.model_tensors.keys()): + if not name.endswith((".k_scale", ".v_scale")): + continue + + mapped = self.tensor_map.get_type_and_name(key=name, try_suffixes=(".k_scale", ".v_scale")) + if mapped is None: + continue + + tensor_type, new_name = mapped + if tensor_type not in (gguf.MODEL_TENSOR.ATTN_K, gguf.MODEL_TENSOR.ATTN_V, gguf.MODEL_TENSOR.ATTN_QKV): + continue + + bid = int(new_name.split(".")[1]) + scale_type = gguf.MODEL_TENSOR.ATTN_K if name.endswith(".k_scale") else gguf.MODEL_TENSOR.ATTN_V + suffix = ".k_scale" if name.endswith(".k_scale") else ".v_scale" + new_name = gguf.TENSOR_NAMES[scale_type].format(bid=bid) + suffix + + scale = LazyTorchTensor.to_eager(self.model_tensors.pop(name)()) + if scale.dtype not in (torch.bfloat16, torch.float32) or scale.numel() != 1: + raise ValueError(f"KV cache scale {name!r} must be a scalar BF16 or FP32 tensor") + + scale = scale.float() + value = float(scale.item()) + if not math.isfinite(value) or value == 0.0: + raise ValueError(f"KV cache scale {name!r} must be finite and nonzero") + + logger.info(f" + {new_name} (KV cache scale, shape [1])") + self.gguf_writer.add_tensor(new_name, scale.flatten().numpy()) + def dequant_model(self): # If all quantized tensors were already handled (e.g. pure NVFP4), skip if self._is_nvfp4 and not any(k.endswith((".weight_scale", ".weight_scale_inv", ".input_scale", ".activation_scale", "_activation_scale", ".k_scale", ".v_scale")) for k in self.model_tensors): @@ -492,7 +538,7 @@ def dequant_fp8(fp8_configs: list[dict[str, Any]] | None = None) -> None: s = self.model_tensors[name] self.model_tensors[weight_name] = lambda w=w, s=s: dequant_bitnet(w(), s()) tensors_to_remove.append(name) - elif quant_method == "fp8": + elif quant_method in ("fp8", "bsgemm-moe"): # bsgemm-moe: drama-3, same block layout block_size = quant_config.get("weight_block_size") for name in self.model_tensors.keys(): if name.endswith("_scale_inv"): @@ -1277,6 +1323,7 @@ def _generate_nvfp4_tensors(self): expert_scales: dict[tuple[int, str], list[tuple[int, float]]] = {} expert_input_scales: dict[tuple[int, str], list[tuple[int, float]]] = {} expert_shapes: dict[tuple[int, str], list[int]] = {} + expert_names: dict[tuple[int, str], str] = {} n_experts = self.find_hparam(["num_local_experts", "num_experts", "n_routed_experts"], optional=True) or 0 consumed: list[str] = [] @@ -1325,6 +1372,7 @@ def _generate_nvfp4_tensors(self): expert_scales[key] = [] expert_input_scales[key] = [] expert_shapes[key] = shape + expert_names[key] = name expert_blocks[key].append((expert_id, raw.copy())) # Collect per-expert scale2 (scalar per expert) expert_scales[key].append((expert_id, float(scale2.float().sum()))) @@ -1333,13 +1381,13 @@ def _generate_nvfp4_tensors(self): # Flush when all experts for this (layer, proj) are collected if n_experts > 0 and len(expert_blocks[key]) >= n_experts: - self._flush_nvfp4_experts(key, expert_blocks, expert_scales, expert_input_scales, expert_shapes, bid, proj_type) + self._flush_nvfp4_experts(key, expert_blocks, expert_scales, expert_input_scales, expert_shapes, expert_names, bid, proj_type) else: self._repack_nvfp4(name, weight, scale, scale2, input_scale) # Flush any remaining experts (fallback if n_experts was unknown) for bid, proj_type in list(expert_blocks.keys()): - self._flush_nvfp4_experts((bid, proj_type), expert_blocks, expert_scales, expert_input_scales, expert_shapes, bid, proj_type) + self._flush_nvfp4_experts((bid, proj_type), expert_blocks, expert_scales, expert_input_scales, expert_shapes, expert_names, bid, proj_type) # Remove consumed tensors so get_tensors/modify_tensors won't see them for name in consumed: @@ -1350,7 +1398,7 @@ def _generate_nvfp4_tensors(self): if name.endswith((".k_scale", ".v_scale")): del self.model_tensors[name] - def _flush_nvfp4_experts(self, key, expert_blocks, expert_scales, expert_input_scales, expert_shapes, bid, proj_type): + def _flush_nvfp4_experts(self, key, expert_blocks, expert_scales, expert_input_scales, expert_shapes, expert_names, bid, proj_type): experts = expert_blocks.pop(key) scales = expert_scales.pop(key) input_scales = expert_input_scales.pop(key) @@ -1362,7 +1410,7 @@ def _flush_nvfp4_experts(self, key, expert_blocks, expert_scales, expert_input_s new_name = self.map_tensor_name(merged_name) logger.info(f"Repacked {new_name} with shape [{len(experts)}, {shape[0]}, {shape[1]}] and quantization NVFP4") self.gguf_writer.add_tensor(new_name, merged, raw_dtype=gguf.GGMLQuantizationType.NVFP4) - self._tag_prec_a4(merged_name, new_name) + self._tag_prec_a4(expert_names.pop(key), new_name) scales.sort(key=lambda x: x[0]) self._write_scales_tensor(new_name.replace(".weight", ".scale"), [s[1] for s in scales]) @@ -1372,6 +1420,11 @@ def _flush_nvfp4_experts(self, key, expert_blocks, expert_scales, expert_input_s del experts, merged + def _include_ple_tensor(self, name: str) -> bool: + is_ple = name == gguf.TENSOR_NAMES[gguf.MODEL_TENSOR.PER_LAYER_TOKEN_EMBD] + ".weight" + self._has_ple |= is_ple + return is_ple if self.ple_only else not is_ple + def prepare_tensors(self): # detect NVFP4 quantization (ModelOpt and Compressed-tensors formats) quantization_config = self.hparams.get("quantization_config") or {} @@ -1413,18 +1466,27 @@ def prepare_tensors(self): # Per-tensor NVFP4 precision. self._nvfp4_layer_algo = {} if quant_layers: - # store all possible module paths and assert if a quantized layer is not in the model - modules: set[str] = set() - for name in self.model_tensors: - while name := name.rpartition(".")[0]: - modules.add(name) - - for layer_name, entry in quant_layers.items(): - if not isinstance(entry, dict): + # quantized_layers keys are module paths in the source checkpoint, not tensor names + # resolve each source tensor to the closest quantized_layers parent and record the algo under its filtered name for _tag_prec_a4 + layer_algo = {k: v.get("quant_algo") for k, v in quant_layers.items() if isinstance(v, dict)} + matched: set[str] = set() + for src_name, filtered_name in self._source_tensor_names.items(): + module = src_name + while module := module.rpartition(".")[0]: + if module in layer_algo: + break + if not module: continue - if titem := self.filter_tensors((layer_name, lambda: torch.empty(0))): - assert titem[0] in modules, f"quantized_layers entry {layer_name!r} is not in the model tensors" - self._nvfp4_layer_algo[titem[0]] = entry.get("quant_algo") + + matched.add(module) + # filter_tensors may drop a tensors so we need to check for that here. + if filtered_name is not None: + self._nvfp4_layer_algo[filtered_name.removesuffix(".weight").removesuffix(".bias")] = layer_algo[module] + + unmatched = layer_algo.keys() - matched + assert not unmatched, f"quantized_layers entries not in the model tensors: {sorted(unmatched)}" + + self._prepare_kv_cache_scales() # NVFP4 weights are repacked and written directly to gguf_writer. # This must run before dequant_model so NVFP4 tensors are removed @@ -1496,6 +1558,8 @@ def load(): break for new_name, data_torch in (self.modify_tensors(data_torch, name, bid)): + if (self.ple_only or self.no_ple) and not self._include_ple_tensor(new_name): + continue n_dims = len(data_torch.shape) data_qtype: gguf.GGMLQuantizationType | bool = self.tensor_force_quant(name, new_name, bid, n_dims) @@ -1628,6 +1692,8 @@ def write_vocab(self): def write(self): self.prepare_tensors() + if (self.ple_only or self.no_ple) and not self._has_ple: + raise ValueError("model has no per-layer embedding table to export separately") self.prepare_metadata(vocab_only=False) self.gguf_writer.write_header_to_file(path=self.fname_out) self.gguf_writer.write_kv_data_to_file() @@ -1820,7 +1886,8 @@ def prepare_metadata(self, vocab_only: bool): fname_default: str = gguf.naming_convention(self.metadata.name, self.metadata.basename, self.metadata.finetune, self.metadata.version, size_label=None, output_type=None, model_type="vocab") # Use the default filename - self.fname_out = self.fname_out / f"{fname_default}.gguf" + prefix = "ple-" if self.ple_only else "no-ple-" if self.no_ple else "" + self.fname_out = self.fname_out / f"{prefix}{fname_default}.gguf" else: # Output path is a custom defined templated filename # Note: `not is_dir()` is used because `.is_file()` will not detect @@ -1829,8 +1896,9 @@ def prepare_metadata(self, vocab_only: bool): # Process templated file name with the output ftype, useful with the "auto" ftype self.fname_out = self.fname_out.parent / gguf.fill_templated_filename(self.fname_out.name, output_type) - logger.info("Set model tokenizer") - self.set_vocab() + if not self.ple_only: + logger.info("Set model tokenizer") + self.set_vocab() def set_gguf_parameters(self): self.gguf_writer.add_block_count(self.block_count) @@ -1987,15 +2055,16 @@ def get_vocab_base(self) -> tuple[list[str], list[int], str]: from transformers import AutoTokenizer tokenizer = AutoTokenizer.from_pretrained(self.dir_model) - vocab_size = self.hparams.get("vocab_size", len(tokenizer.vocab)) # ty: ignore[unresolved-attribute] - assert max(tokenizer.vocab.values()) < vocab_size # ty: ignore[unresolved-attribute] + assert tokenizer is not None + vocab_size = self.hparams.get("vocab_size", len(tokenizer.vocab)) + assert max(tokenizer.vocab.values()) < vocab_size tokpre = self.get_vocab_base_pre(tokenizer) - reverse_vocab = {id_: encoded_tok for encoded_tok, id_ in tokenizer.vocab.items()} # ty: ignore[unresolved-attribute] - added_vocab = tokenizer.get_added_vocab() # ty: ignore[unresolved-attribute] + reverse_vocab = {id_: encoded_tok for encoded_tok, id_ in tokenizer.vocab.items()} + added_vocab = tokenizer.get_added_vocab() - added_tokens_decoder = tokenizer.added_tokens_decoder # ty: ignore[unresolved-attribute] + added_tokens_decoder = tokenizer.added_tokens_decoder for i in range(vocab_size): if i not in reverse_vocab: @@ -2008,7 +2077,7 @@ def get_vocab_base(self) -> tuple[list[str], list[int], str]: # To avoid unexpected issues - we make sure to normalize non-normalized tokens if not added_tokens_decoder[i].normalized: previous_token = token - token = tokenizer.decode(tokenizer.encode(token, add_special_tokens=False)) # ty: ignore[unresolved-attribute, invalid-assignment] + token = tokenizer.decode(tokenizer.encode(token, add_special_tokens=False)) # ty: ignore[invalid-assignment] if previous_token != token: logger.info(f"{repr(previous_token)} is encoded and decoded back to {repr(token)} using AutoTokenizer") @@ -2114,6 +2183,9 @@ def get_vocab_base_pre(self, tokenizer) -> str: if chkhsh == "0fe1cf6eda062318a1af7270f3331a85c539a01778ff948e24388e949c5282f4": # ref: https://huggingface.co/evilfreelancer/ruGPT3XL res = "gpt-2" + if chkhsh == "df1bc34a6366466bcb36a9a1b1cc0afdb08cfe21c9250602c930f159085390a9": + # ref: https://huggingface.co/Lernex/Metis-1.6-Logos + res = "gpt-2" if chkhsh == "9e454714343b69b99b71795c1d27a68c2a1d15dab111f4d353109f966af29da7": # ref: https://huggingface.co/LiquidAI/LFM2.5-8B-A1B res = "lfm2" @@ -2415,6 +2487,7 @@ def _is_hybriddna(self) -> bool: def _set_vocab_hybriddna(self): from transformers import AutoTokenizer tokenizer = AutoTokenizer.from_pretrained(self.dir_model, trust_remote_code=True) + assert tokenizer is not None vocab = tokenizer.get_vocab() vocab_size = self.hparams.get("vocab_size", len(vocab)) assert max(vocab.values()) < vocab_size @@ -2428,11 +2501,11 @@ def _set_vocab_hybriddna(self): if tokenizer.k != 6 or tail not in ("pad_a", "pad_n"): raise NotImplementedError(f"HybridDNATokenizer with k={tokenizer.k}, tail={tail!r}") for kmer in [*tokenizer.kmers, *getattr(tokenizer, "npad_tokens", [])]: - reverse_vocab[tokenizer.dna_token_to_id[kmer]] = kmer + "\ue000" # ty: ignore[unresolved-attribute] + reverse_vocab[tokenizer.dna_token_to_id[kmer]] = kmer + "\ue000" # older HybridDNATokenizer versions (Carbon-500M/3B/8B) do not register the base tokenizer's added tokens (<|im_start|>, , ...) as their own # but they are still encoded as special tokens: take their types from the base tokenizer base_tokenizer = getattr(tokenizer, "_base_tokenizer", tokenizer) - added_tokens_decoder = {**base_tokenizer.added_tokens_decoder, **tokenizer.added_tokens_decoder} # ty: ignore[unresolved-attribute] + added_tokens_decoder = {**base_tokenizer.added_tokens_decoder, **tokenizer.added_tokens_decoder} tokens: list[str] = [] toktypes: list[int] = [] @@ -2479,24 +2552,25 @@ def _set_vocab_qwen(self): from transformers import AutoTokenizer tokenizer = AutoTokenizer.from_pretrained(dir_model, trust_remote_code=True) + assert tokenizer is not None vocab_size = hparams["vocab_size"] - assert max(tokenizer.get_vocab().values()) < vocab_size # ty: ignore[unresolved-attribute] + assert max(tokenizer.get_vocab().values()) < vocab_size tokpre = self.get_vocab_base_pre(tokenizer) merges = [] vocab = {} - mergeable_ranks = tokenizer.mergeable_ranks # ty: ignore[unresolved-attribute] + mergeable_ranks = tokenizer.mergeable_ranks for token, rank in mergeable_ranks.items(): vocab[QwenModel.token_bytes_to_string(token)] = rank if len(token) == 1: continue merged = QwenModel.bpe(mergeable_ranks, token, max_rank=rank) assert len(merged) == 2 - merges.append(' '.join(map(QwenModel.token_bytes_to_string, merged))) + merges.append(str(' '.join(map(QwenModel.token_bytes_to_string, merged)))) # for this kind of tokenizer, added_vocab is not a subset of vocab, so they need to be combined - added_vocab = tokenizer.special_tokens # ty: ignore[unresolved-attribute] + added_vocab = tokenizer.special_tokens reverse_vocab = {id_ : encoded_tok for encoded_tok, id_ in {**vocab, **added_vocab}.items()} for i in range(vocab_size): @@ -2519,10 +2593,10 @@ def _set_vocab_qwen(self): special_vocab.merges = merges # only add special tokens when they were not already loaded from config.json if len(special_vocab.special_token_ids) == 0: - special_vocab._set_special_token("bos", tokenizer.special_tokens["<|endoftext|>"]) # ty: ignore[unresolved-attribute] - special_vocab._set_special_token("eos", tokenizer.special_tokens["<|endoftext|>"]) # ty: ignore[unresolved-attribute] + special_vocab._set_special_token("bos", tokenizer.special_tokens["<|endoftext|>"]) + special_vocab._set_special_token("eos", tokenizer.special_tokens["<|endoftext|>"]) # this one is usually not in config.json anyway - special_vocab._set_special_token("unk", tokenizer.special_tokens["<|endoftext|>"]) # ty: ignore[unresolved-attribute] + special_vocab._set_special_token("unk", tokenizer.special_tokens["<|endoftext|>"]) special_vocab.add_to_gguf(self.gguf_writer) def _set_vocab_sentencepiece(self, add_to_gguf=True): @@ -2796,21 +2870,23 @@ def _try_set_pooling_type(self) -> None: def _set_vocab_glmedge(self): from transformers import AutoTokenizer tokenizer = AutoTokenizer.from_pretrained(self.dir_model) + assert tokenizer is not None special_vocab = gguf.SpecialVocab(self.dir_model, load_merges=True) tokens, toktypes, tokpre = self.get_vocab_base() self.gguf_writer.add_tokenizer_model("gpt2") self.gguf_writer.add_tokenizer_pre(tokpre) self.gguf_writer.add_token_list(tokens) self.gguf_writer.add_token_types(toktypes) - special_vocab._set_special_token("eos", tokenizer.get_added_vocab()["<|endoftext|>"]) # ty: ignore[unresolved-attribute] - special_vocab._set_special_token("eot", tokenizer.get_added_vocab()["<|user|>"]) # ty: ignore[unresolved-attribute] - special_vocab._set_special_token("unk", tokenizer.get_added_vocab()["<|endoftext|>"]) # ty: ignore[unresolved-attribute] - special_vocab._set_special_token("bos", tokenizer.get_added_vocab()["<|endoftext|>"]) # ty: ignore[unresolved-attribute] + special_vocab._set_special_token("eos", tokenizer.get_added_vocab()["<|endoftext|>"]) + special_vocab._set_special_token("eot", tokenizer.get_added_vocab()["<|user|>"]) + special_vocab._set_special_token("unk", tokenizer.get_added_vocab()["<|endoftext|>"]) + special_vocab._set_special_token("bos", tokenizer.get_added_vocab()["<|endoftext|>"]) special_vocab.add_to_gguf(self.gguf_writer) def _set_vocab_glm(self): from transformers import AutoTokenizer tokenizer = AutoTokenizer.from_pretrained(self.dir_model) + assert tokenizer is not None special_vocab = gguf.SpecialVocab(self.dir_model, load_merges=True) tokens, toktypes, tokpre = self.get_vocab_base() self.gguf_writer.add_tokenizer_model("gpt2") @@ -2819,10 +2895,10 @@ def _set_vocab_glm(self): self.gguf_writer.add_token_types(toktypes) # Special tokens # Note: Using <|endoftext|> (151329) for eot causes endless generation - special_vocab._set_special_token("bos", tokenizer.get_added_vocab()["[gMASK]"]) # ty: ignore[unresolved-attribute] # 151331 - special_vocab._set_special_token("eot", tokenizer.get_added_vocab()["<|user|>"]) # ty: ignore[unresolved-attribute] # 151336 - special_vocab._set_special_token("unk", tokenizer.get_added_vocab()["<|endoftext|>"]) # ty: ignore[unresolved-attribute] # 151329 - special_vocab._set_special_token("eom", tokenizer.get_added_vocab()["<|observation|>"]) # ty: ignore[unresolved-attribute] # 151338 + special_vocab._set_special_token("bos", tokenizer.get_added_vocab()["[gMASK]"]) # 151331 + special_vocab._set_special_token("eot", tokenizer.get_added_vocab()["<|user|>"]) # 151336 + special_vocab._set_special_token("unk", tokenizer.get_added_vocab()["<|endoftext|>"]) # 151329 + special_vocab._set_special_token("eom", tokenizer.get_added_vocab()["<|observation|>"]) # 151338 special_vocab.add_to_gguf(self.gguf_writer) def _set_vocab_interns1(self): @@ -2831,16 +2907,17 @@ def _set_vocab_interns1(self): from transformers import AutoTokenizer tokenizer = AutoTokenizer.from_pretrained(self.dir_model, trust_remote_code=True) - vocab = getattr(tokenizer, 'vocab', tokenizer.get_vocab()) # ty: ignore[unresolved-attribute] + assert tokenizer is not None + vocab = getattr(tokenizer, 'vocab', tokenizer.get_vocab()) vocab_size = self.hparams.get("vocab_size", len(vocab)) assert max(vocab.values()) < vocab_size tokpre = self.get_vocab_base_pre(tokenizer) reverse_vocab = {id_: encoded_tok for encoded_tok, id_ in vocab.items()} - added_vocab = tokenizer.get_added_vocab() # ty: ignore[unresolved-attribute] + added_vocab = tokenizer.get_added_vocab() - added_tokens_decoder = tokenizer.added_tokens_decoder # ty: ignore[unresolved-attribute] + added_tokens_decoder = tokenizer.added_tokens_decoder for i in range(vocab_size): if i not in reverse_vocab: @@ -2853,7 +2930,7 @@ def _set_vocab_interns1(self): # To avoid unexpected issues - we make sure to normalize non-normalized tokens if not added_tokens_decoder[i].normalized: previous_token = token - token = tokenizer.decode(tokenizer.encode(token, add_special_tokens=False)) # ty: ignore[unresolved-attribute, invalid-assignment] + token = tokenizer.decode(tokenizer.encode(token, add_special_tokens=False)) # ty: ignore[invalid-assignment] if previous_token != token: logger.info(f"{repr(previous_token)} is encoded and decoded back to {repr(token)} using AutoTokenizer") @@ -3384,7 +3461,7 @@ def get_model_architecture(hparams: dict[str, Any], model_type: ModelType) -> st # Kimi-K3's text_config reports "KimiLinearForCausalLM", which is the older # Kimi-Linear-48B architecture and cannot load K3 (no attention residuals, # latent MoE, situ, ...). Route on the top-level architecture instead. - if model_type == ModelType.TEXT and arch in ("NEOChatModel", "StepVLForConditionalGeneration", "Sarashina2VisionForCausalLM", "Exaone4_5_ForConditionalGeneration", "Step3p7ForConditionalGeneration", "KimiK3ForConditionalGeneration"): + if model_type == ModelType.TEXT and arch in ("NEOChatModel", "StepVLForConditionalGeneration", "Sarashina2VisionForCausalLM", "Exaone4_5_ForConditionalGeneration", "Step3p7ForConditionalGeneration", "KimiK3ForConditionalGeneration", "KimiK31ForConditionalGeneration", "Nemotron3_5TranscribeForConditionalGeneration"): return arch # if "architectures" is found in the sub-config, use that instead @@ -3392,6 +3469,9 @@ def get_model_architecture(hparams: dict[str, Any], model_type: ModelType) -> st arch = text_config["architectures"][0] elif model_type == ModelType.MMPROJ and vision_config.get("architectures") is not None: arch = vision_config["architectures"][0] + # some models (llava_onevision2) only declare the architecture inside text_config + if arch is None and text_config.get("architectures") is not None: + arch = text_config["architectures"][0] if arch is None: raise ValueError("Failed to detect model architecture") return arch diff --git a/conversion/beam.py b/conversion/beam.py index 67d0cc93ca63..66b28d8f677c 100644 --- a/conversion/beam.py +++ b/conversion/beam.py @@ -60,7 +60,8 @@ def set_vocab(self): # otherwise EOT is guessed from a hash-ordered scan from transformers import AutoTokenizer tok = AutoTokenizer.from_pretrained(self.dir_model) - ret_id = tok.convert_tokens_to_ids("<|return|>") # ty: ignore[unresolved-attribute] + assert tok is not None + ret_id = tok.convert_tokens_to_ids("<|return|>") if isinstance(ret_id, int) and ret_id >= 0: self.gguf_writer.add_eot_token_id(ret_id) diff --git a/conversion/bert.py b/conversion/bert.py index f247154269de..150b2667cc66 100644 --- a/conversion/bert.py +++ b/conversion/bert.py @@ -132,6 +132,7 @@ def _xlmroberta_set_vocab(self) -> None: from base64 import b64decode from transformers import AutoTokenizer tokenizer = AutoTokenizer.from_pretrained(self.dir_model) + assert tokenizer is not None with open(tokenizer_path, "r", encoding="utf-8") as fp: tokenizer_json = json.load(fp) @@ -140,13 +141,13 @@ def _xlmroberta_set_vocab(self) -> None: with open(tokenizer_config_path, "r", encoding="utf-8") as fp: tokenizer_config_json = json.load(fp) - add_prefix = tokenizer.add_prefix_space # ty: ignore[unresolved-attribute] - remove_whitespaces = tokenizer.clean_up_tokenization_spaces # ty: ignore[unresolved-attribute] + add_prefix = tokenizer.add_prefix_space + remove_whitespaces = tokenizer.clean_up_tokenization_spaces precompiled_charsmap = b64decode(tokenizer_json["normalizer"]["precompiled_charsmap"]) - vocab_size = max(self.hparams.get("vocab_size", 0), tokenizer.vocab_size) # ty: ignore[unresolved-attribute] + vocab_size = max(self.hparams.get("vocab_size", 0), tokenizer.vocab_size) else: - sentencepiece_model = model.ModelProto() # pyright: ignore[reportAttributeAccessIssue] # ty: ignore[unresolved-attribute] + sentencepiece_model = model.ModelProto() # pyright: ignore[reportAttributeAccessIssue] # ty: ignore[missing-attribute] sentencepiece_model.ParseFromString(open(tokenizer_path, "rb").read()) assert sentencepiece_model.trainer_spec.model_type == 1 # UNIGRAM @@ -183,20 +184,20 @@ def _xlmroberta_set_vocab(self) -> None: scores[token_id] = score toktypes[token_id] = toktype else: - added_vocab = tokenizer.get_added_vocab() # ty: ignore[unresolved-attribute] + added_vocab = tokenizer.get_added_vocab() unk_token = tokenizer_config_json.get("unk_token") - unk_token_id = added_vocab.get(unk_token, tokenizer_json["model"].get("unk_id", 3)) # ty: ignore[no-matching-overload] + unk_token_id = added_vocab.get(unk_token, tokenizer_json["model"].get("unk_id", 3)) - for token_id in range(tokenizer.vocab_size): # ty: ignore[unresolved-attribute] - piece = tokenizer._convert_id_to_token(token_id) # ty: ignore[unresolved-attribute] - if (piece := tokenizer._convert_id_to_token(token_id)) is not None: # ty: ignore[unresolved-attribute] + for token_id in range(tokenizer.vocab_size): + piece = tokenizer._convert_id_to_token(token_id) + if (piece := tokenizer._convert_id_to_token(token_id)) is not None: text = piece.encode("utf-8") score = tokenizer_json["model"]["vocab"][token_id][1] toktype = SentencePieceTokenTypes.NORMAL if token_id == unk_token_id: toktype = SentencePieceTokenTypes.UNKNOWN - elif token_id in tokenizer.all_special_ids: # ty: ignore[unresolved-attribute] + elif token_id in tokenizer.all_special_ids: toktype = SentencePieceTokenTypes.CONTROL elif token_id in added_vocab.values(): toktype = SentencePieceTokenTypes.USER_DEFINED diff --git a/conversion/chatglm.py b/conversion/chatglm.py index 9b902dae30c4..fb10393cbdcd 100644 --- a/conversion/chatglm.py +++ b/conversion/chatglm.py @@ -22,12 +22,13 @@ def set_vocab_chatglm3(self): from transformers import AutoTokenizer tokenizer = AutoTokenizer.from_pretrained(dir_model, trust_remote_code=True) - vocab_size = hparams.get("padded_vocab_size", len(tokenizer.get_vocab())) # ty: ignore[unresolved-attribute] - assert max(tokenizer.get_vocab().values()) < vocab_size # ty: ignore[unresolved-attribute] + assert tokenizer is not None + vocab_size = hparams.get("padded_vocab_size", len(tokenizer.get_vocab())) + assert max(tokenizer.get_vocab().values()) < vocab_size role_special_tokens = ["<|system|>", "<|user|>", "<|assistant|>", "<|observation|>"] special_tokens = ["[MASK]", "[gMASK]", "[sMASK]", "sop", "eop"] + role_special_tokens for token_id in range(vocab_size): - piece = tokenizer._convert_id_to_token(token_id) # ty: ignore[unresolved-attribute] + piece = tokenizer._convert_id_to_token(token_id) if token_id == 0: piece = "" elif token_id == 1: @@ -35,17 +36,18 @@ def set_vocab_chatglm3(self): elif token_id == 2: piece = "" - text = piece.encode("utf-8") # ty: ignore[unresolved-attribute] + assert isinstance(piece, str) + text = piece.encode("utf-8") score = 0.0 # Referencing the tokenizer Python implementation(https://huggingface.co/THUDM/chatglm3-6b/blob/main/tokenization_chatglm.py), # it is only valid if it is less than tokenizer.tokenizer.sp_model.vocab_size() - if len(piece) != 0 and token_id < tokenizer.tokenizer.sp_model.vocab_size(): # ty: ignore[unresolved-attribute, invalid-argument-type] - score = tokenizer.tokenizer.sp_model.get_score(token_id) # ty: ignore[unresolved-attribute] + if len(piece) != 0 and token_id < tokenizer.tokenizer.sp_model.vocab_size(): + score = tokenizer.tokenizer.sp_model.get_score(token_id) - if token_id >= tokenizer.tokenizer.sp_model.vocab_size(): # ty: ignore[unresolved-attribute] + if token_id >= tokenizer.tokenizer.sp_model.vocab_size(): if piece in special_tokens: toktype = SentencePieceTokenTypes.CONTROL - elif len(piece) == 0: # ty: ignore[invalid-argument-type] + elif len(piece) == 0: text = f"[PAD{token_id}]".encode("utf-8") toktype = SentencePieceTokenTypes.UNUSED else: @@ -56,13 +58,13 @@ def set_vocab_chatglm3(self): continue toktype = SentencePieceTokenTypes.NORMAL - if tokenizer.tokenizer.sp_model.is_unknown(token_id): # ty: ignore[unresolved-attribute] + if tokenizer.tokenizer.sp_model.is_unknown(token_id): toktype = SentencePieceTokenTypes.UNKNOWN - elif tokenizer.tokenizer.sp_model.is_control(token_id): # ty: ignore[unresolved-attribute] + elif tokenizer.tokenizer.sp_model.is_control(token_id): toktype = SentencePieceTokenTypes.CONTROL - elif tokenizer.tokenizer.sp_model.is_unused(token_id): # ty: ignore[unresolved-attribute] + elif tokenizer.tokenizer.sp_model.is_unused(token_id): toktype = SentencePieceTokenTypes.UNUSED - elif tokenizer.tokenizer.sp_model.is_byte(token_id): # ty: ignore[unresolved-attribute] + elif tokenizer.tokenizer.sp_model.is_byte(token_id): toktype = SentencePieceTokenTypes.BYTE tokens.append(text) @@ -115,8 +117,9 @@ def set_vocab(self): from transformers import AutoTokenizer tokenizer = AutoTokenizer.from_pretrained(dir_model, trust_remote_code=True) + assert tokenizer is not None vocab_size = hparams.get("padded_vocab_size",hparams["vocab_size"]) - assert max(tokenizer.get_vocab().values()) < vocab_size # ty: ignore[unresolved-attribute] + assert max(tokenizer.get_vocab().values()) < vocab_size tokens, toktypes, tokpre = self.get_vocab_base() self.gguf_writer.add_tokenizer_model("gpt2") @@ -125,10 +128,10 @@ def set_vocab(self): self.gguf_writer.add_token_types(toktypes) special_vocab = gguf.SpecialVocab(self.dir_model, load_merges=True) # only add special tokens when they were not already loaded from config.json - special_vocab._set_special_token("eos", tokenizer.get_added_vocab()["<|endoftext|>"]) # ty: ignore[unresolved-attribute] - special_vocab._set_special_token("eot", tokenizer.get_added_vocab()["<|user|>"]) # ty: ignore[unresolved-attribute] + special_vocab._set_special_token("eos", tokenizer.get_added_vocab()["<|endoftext|>"]) + special_vocab._set_special_token("eot", tokenizer.get_added_vocab()["<|user|>"]) # this one is usually not in config.json anyway - special_vocab._set_special_token("unk", tokenizer.get_added_vocab()["<|endoftext|>"]) # ty: ignore[unresolved-attribute] + special_vocab._set_special_token("unk", tokenizer.get_added_vocab()["<|endoftext|>"]) special_vocab.add_to_gguf(self.gguf_writer) def set_gguf_parameters(self): diff --git a/conversion/chatterbox.py b/conversion/chatterbox.py new file mode 100644 index 000000000000..d438228e5acb --- /dev/null +++ b/conversion/chatterbox.py @@ -0,0 +1,473 @@ +# Chatterbox (ResembleAI) turbo and multilingual v3, from the raw checkpoints of the official repos +# the checkpoints ship no config.json, the variant is detected from the talker file and the hparams come from the reference code +# talker: text_emb and speech_emb are fused into one [text | speech] vocab, speech token i is <|speech_i|> +# mmproj: the s3gen sidecar, the voice and conditioning encoders, the position and speech tables, and the conds.pt voice + +from __future__ import annotations + +import json +import math +from pathlib import Path +from typing import TYPE_CHECKING, Any, Callable, Iterable + +import torch +import torch.nn.functional as F + +if TYPE_CHECKING: + from torch import Tensor + +from .base import ModelBase, TextModel, MmprojModel, LazyTorchTensor, gguf, logger +from .llama import LlamaModel + +# checkpoint file prefixes, the files are ".safetensors" +TURBO_TALKER = "t3_turbo_v1" +MTL_TALKER = "t3_mtl23ls_v3" +TURBO_S3GEN = "s3gen_meanflow" +MTL_S3GEN = "s3gen_v3" +VE = "ve" + +# relative speech ids shared by both variants (start/stop_speech_token in the reference) +SPEECH_BOS = 6561 +SPEECH_EOS = 6562 + +# sampling defaults of the reference, samplers it does not use are disabled so the common defaults do not apply +TURBO_TOP_K = 1000 +MTL_TOP_K = 0 +TURBO_SAMPLING = {"min_p": 0.0, "top_p": 0.95, "temp": 0.8, "penalty_repeat": 1.2} +MTL_SAMPLING = {"min_p": 0.05, "top_p": 1.0, "temp": 0.8, "penalty_repeat": 1.2} +# the reference repetition penalty covers the whole generated history +PENALTY_LAST_N = -1 + + +def _has_file(dir_model: Path, prefix: str) -> bool: + return (dir_model / f"{prefix}.safetensors").is_file() + + +def _is_turbo(dir_model: Path) -> bool: + if _has_file(dir_model, TURBO_TALKER): + return True + if _has_file(dir_model, MTL_TALKER): + return False + raise FileNotFoundError(f"no chatterbox talker checkpoint in {dir_model}") + + +# talker hparams of the reference (GPT2_MEDIUM_CONFIG / LLAMA_520M_CONFIG_DICT, T3Config) +TURBO_TEXT_CONFIG = { + "n_layer": 24, "n_embd": 1024, "n_inner": 4096, "n_head": 16, "n_ctx": 8196, "layer_norm_epsilon": 1e-05, + "vocab_size": 50276, "speech_vocab_size": 6563, +} +MTL_TEXT_CONFIG = { + "num_hidden_layers": 30, "hidden_size": 1024, "intermediate_size": 4096, "num_attention_heads": 16, + "num_key_value_heads": 16, "head_dim": 64, "max_position_embeddings": 131072, "rms_norm_eps": 1e-05, + "rope_theta": 500000.0, + "rope_scaling": {"factor": 8.0, "high_freq_factor": 4.0, "low_freq_factor": 1.0, "original_max_position_embeddings": 8192}, + "vocab_size": 2454, "speech_vocab_size": 8194, +} + + +def _is_chatterbox_checkpoint(dir_model: Path) -> bool: + return _has_file(dir_model, VE) and (_has_file(dir_model, TURBO_TALKER) or _has_file(dir_model, MTL_TALKER)) + + +@ModelBase.register_hparams_loader(_is_chatterbox_checkpoint) +def _load_chatterbox_hparams(dir_model: Path) -> dict[str, Any]: + turbo = _is_turbo(dir_model) + logger.info(f"gguf: detected Chatterbox {'turbo' if turbo else 'multilingual v3'} checkpoint") + text_config = dict(TURBO_TEXT_CONFIG if turbo else MTL_TEXT_CONFIG) + # the mmproj tensors are named explicitly, they do not go through the block tensor map + return {"architectures": ["ChatterboxModel"], "text_config": text_config, "audio_config": {"num_hidden_layers": 0}} + + +@ModelBase.register("ChatterboxModel") +class ChatterboxTalkerModel(TextModel): + model_arch = gguf.MODEL_ARCH.LLAMA # multilingual; the turbo constructor switches to GPT2 + + def __init__(self, dir_model: Path, *args, **kwargs): + self.is_turbo = _is_turbo(dir_model) + if self.is_turbo: + self.model_arch = gguf.MODEL_ARCH.GPT2 + super().__init__(dir_model, *args, **kwargs) + self._text_embd: Tensor | None = None + self._speech_embd: Tensor | None = None + self._text_head: Tensor | None = None + self._speech_head: Tensor | None = None + + def index_tensors(self, remote_hf_model_id: str | None = None) -> dict[str, Callable[[], Tensor]]: + return self.index_local_tensors(TURBO_TALKER if self.is_turbo else MTL_TALKER) + + @classmethod + def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None: + name, _ = item + # the reference feeds inputs_embeds, the transformer input embeddings are unused + if name in ("tfmr.wte.weight", "tfmr.embed_tokens.weight"): + return None + if name.startswith(("cond_enc.", "text_pos_emb.", "speech_pos_emb.")): + return None + return super().filter_tensors(item) + + def set_vocab(self): + if self.is_turbo: + self._set_vocab_turbo() + else: + self._set_vocab_mtl() + + def _speech_token_names(self, n_speech: int) -> list[str]: + return [f"<|speech_{i}|>" for i in range(n_speech)] + + def _set_vocab_turbo(self): + # stock GPT-2 BPE from the checkpoint dir, extended with the speech tokens + tokens, toktypes, tokpre = self.get_vocab_base() + n_text = len(tokens) + n_speech = self.hparams["speech_vocab_size"] + speech = self._speech_token_names(n_speech) + tokens += speech + toktypes += [gguf.TokenType.CONTROL] * n_speech + + with open(self.dir_model / "merges.txt", "r", encoding="utf-8") as f: + merges = [line.rstrip("\n") for line in f if line.strip() and not line.startswith("#version")] + + self._write_vocab(tokens, toktypes, merges, tokpre, n_text) + + def _write_vocab(self, tokens: list[str], toktypes: list[int], merges: list[str], tokpre: str, n_text: int): + self.gguf_writer.add_tokenizer_model("gpt2") + self.gguf_writer.add_tokenizer_pre(tokpre) + self.gguf_writer.add_token_list(tokens) + self.gguf_writer.add_token_types(toktypes) + self.gguf_writer.add_token_merges(merges) + + self.gguf_writer.add_bos_token_id(n_text + SPEECH_BOS) + self.gguf_writer.add_eos_token_id(n_text + SPEECH_EOS) + self.gguf_writer.add_add_bos_token(False) + self.gguf_writer.add_add_eos_token(False) + + # the reference samples the speech head only + self.gguf_writer.add_suppress_tokens(list(range(n_text))) + + def _set_vocab_mtl(self): + # the reference BPE is char-level, it is re-encoded to byte-level with extra merges that rebuild each multi-byte char + # its lowercasing is folded into the embedding table, so the prompt needs no text preprocessing + with open(self.dir_model / "mtl_tokenizer.json", "r", encoding="utf-8") as f: + tok = json.load(f) + + byte_map = gguf.vocab.bytes_to_unicode() + + def enc(s: str) -> str: + return "".join(byte_map[b] for b in s.encode("utf-8")) + + n_text = self.hparams["vocab_size"] + tokens: list[str] = [f"[unused_{i}]" for i in range(n_text)] + toktypes = [int(gguf.TokenType.UNUSED)] * n_text + char_merges: list[str] = [] + for t, i in tok["model"]["vocab"].items(): + if t == " ": + # the reference substitutes [SPACE] before encoding, the raw space token is never used + continue + tokens[i] = enc(t) + toktypes[i] = int(gguf.TokenType.NORMAL) + if len(t) == 1 and len(t.encode("utf-8")) > 1: + parts = [byte_map[b] for b in t.encode("utf-8")] + for k in range(1, len(parts)): + char_merges.append("".join(parts[:k]) + " " + parts[k]) + for entry in tok.get("added_tokens", []): + tokens[entry["id"]] = entry["content"] + toktypes[entry["id"]] = int(gguf.TokenType.CONTROL) + + # raw spaces tokenize to [SPACE] directly + space_id = tok["model"]["vocab"]["[SPACE]"] + tokens[space_id] = enc(" ") + toktypes[space_id] = int(gguf.TokenType.NORMAL) + + n_speech = self.hparams["speech_vocab_size"] + tokens += self._speech_token_names(n_speech) + toktypes += [int(gguf.TokenType.CONTROL)] * n_speech + + # chars are atomic in the reference, they must form before any of its merges + merges = char_merges + for m in tok["model"].get("merges", []): + a, b = m if isinstance(m, list) else m.split(" ") + merges.append(enc(a) + " " + enc(b)) + + self._write_vocab(tokens, toktypes, merges, "default", n_text) + + def _mtl_case_fold_pairs(self) -> list[tuple[int, int]]: + # (upper_id, lower_id) of the single-char vocab entries, for every script in the vocab + with open(self.dir_model / "mtl_tokenizer.json", "r", encoding="utf-8") as f: + vocab = json.load(f)["model"]["vocab"] + pairs: list[tuple[int, int]] = [] + for t, i in vocab.items(): + if len(t) != 1: + continue + low = t.lower() + if low != t and len(low) == 1 and low in vocab: + pairs.append((i, vocab[low])) + return pairs + + def set_gguf_parameters(self): + super().set_gguf_parameters() + + sampling = TURBO_SAMPLING if self.is_turbo else MTL_SAMPLING + self.gguf_writer.add_sampling_top_k(TURBO_TOP_K if self.is_turbo else MTL_TOP_K) + self.gguf_writer.add_sampling_min_p(sampling["min_p"]) + self.gguf_writer.add_sampling_top_p(sampling["top_p"]) + self.gguf_writer.add_sampling_temp(sampling["temp"]) + self.gguf_writer.add_sampling_penalty_repeat(sampling["penalty_repeat"]) + self.gguf_writer.add_sampling_penalty_last_n(PENALTY_LAST_N) + if not self.is_turbo: + self.gguf_writer.add_classifier_free_guidance(True) + + def generate_extra_tensors(self) -> Iterable[tuple[str, Tensor]]: + if self.is_turbo: + return + # llama3 rope scaling baked into the rope_freqs factors tensor + rp = self.hparams["rope_scaling"] + dim = self.hparams["head_dim"] + base = self.hparams["rope_theta"] + freqs = 1.0 / (base ** (torch.arange(0, dim, 2, dtype=torch.float32) / dim)) + factor = rp["factor"] + low_freq_wavelen = rp["original_max_position_embeddings"] / rp["low_freq_factor"] + high_freq_wavelen = rp["original_max_position_embeddings"] / rp["high_freq_factor"] + rope_factors = [] + for freq in freqs: + wavelen = 2 * math.pi / freq + if wavelen < high_freq_wavelen: + rope_factors.append(1) + elif wavelen > low_freq_wavelen: + rope_factors.append(factor) + else: + smooth = (rp["original_max_position_embeddings"] / wavelen - rp["low_freq_factor"]) / (rp["high_freq_factor"] - rp["low_freq_factor"]) + rope_factors.append(1 / ((1 - smooth) / factor + smooth)) + yield (self.format_tensor_name(gguf.MODEL_TENSOR.ROPE_FREQS), torch.tensor(rope_factors, dtype=torch.float32)) + + def _maybe_emit_fused(self) -> Iterable[tuple[str, Tensor]]: + if self._text_embd is not None and self._speech_embd is not None: + yield (self.format_tensor_name(gguf.MODEL_TENSOR.TOKEN_EMBD), + torch.cat([self._text_embd, self._speech_embd], dim=0)) + self._text_embd = None + self._speech_embd = None + if self._text_head is not None and self._speech_head is not None: + yield (self.format_tensor_name(gguf.MODEL_TENSOR.OUTPUT), + torch.cat([self._text_head, self._speech_head], dim=0)) + self._text_head = None + self._speech_head = None + + def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]: + # generate_extra_tensors output comes back through here with its final name + if name.startswith("rope_freqs"): + yield (name, data_torch) + return + # fused [text | speech] vocab: embeddings and output head + if name == "text_emb.weight": + if not self.is_turbo: + # the reference lowercases its input, the uppercase rows take the lowercase ones + fold = list(range(data_torch.shape[0])) + for upper, lower in self._mtl_case_fold_pairs(): + fold[upper] = lower + data_torch = data_torch[torch.tensor(fold)] + self._text_embd = data_torch + yield from self._maybe_emit_fused() + return + if name == "speech_emb.weight": + self._speech_embd = data_torch + yield from self._maybe_emit_fused() + return + if name == "text_head.weight": + self._text_head = data_torch + yield from self._maybe_emit_fused() + return + if name == "speech_head.weight": + self._speech_head = data_torch + yield from self._maybe_emit_fused() + return + if name == "speech_head.bias": + # the text head has no bias, its zone is suppressed at sampling anyway + yield (self.format_tensor_name(gguf.MODEL_TENSOR.OUTPUT, suffix=".bias"), F.pad(data_torch, (self.hparams["vocab_size"], 0))) + return + + assert name.startswith("tfmr.") + name = name[len("tfmr."):] + + if self.is_turbo: + # HF GPT-2 layout: Conv1D style weights are stored transposed + if name.endswith((".c_attn.weight", ".c_proj.weight", ".c_fc.weight")): + data_torch = data_torch.transpose(1, 0) + yield (self.map_tensor_name(name), data_torch) + return + + if name.endswith("q_proj.weight"): + data_torch = LlamaModel.permute(data_torch, self.hparams["num_attention_heads"], None) + if name.endswith("k_proj.weight"): + data_torch = LlamaModel.permute(data_torch, self.hparams["num_attention_heads"], self.hparams["num_key_value_heads"]) + yield (self.map_tensor_name("model." + name), data_torch) + + +@ModelBase.register("ChatterboxModel") +class ChatterboxMmprojModel(MmprojModel): + has_vision_encoder = False + has_audio_encoder = True + + def __init__(self, dir_model: Path, *args, **kwargs): + self.is_turbo = _is_turbo(dir_model) + super().__init__(dir_model, *args, **kwargs) + self._wnorm_g: dict[str, Tensor] = {} + self._wnorm_v: dict[str, Tensor] = {} + + def get_audio_config(self) -> dict[str, Any] | None: + return self.global_config.get("audio_config") + + def index_tensors(self, remote_hf_model_id: str | None = None) -> dict[str, Callable[[], Tensor]]: + tensors: dict[str, Callable[[], Tensor]] = {} + for prefix in (TURBO_S3GEN if self.is_turbo else MTL_S3GEN, VE, TURBO_TALKER if self.is_turbo else MTL_TALKER): + tensors.update(self.index_local_tensors(prefix)) + return tensors + + @classmethod + def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None: + # the s3gen, ve and talker checkpoints have disjoint top-level names + name, gen = item + new_name: str | None = None + # batchnorm bookkeeping (over the gguf name length cap) and the tokenizer dsp buffers are unused + if name.endswith("num_batches_tracked") or name in ("tokenizer.window", "tokenizer._mel_filters"): + return None + for src, dst in ( + ("flow.encoder.", "a.gen.fenc."), + ("flow.decoder.estimator.", "a.gen.est."), + ("mel2wav.", "a.gen.hift."), + ("speaker_encoder.", "a.spk."), + ("tokenizer.", "a.s3tok."), + ("flow.", "a.gen.flow."), # input_embedding, encoder_proj, spk_embed_affine_layer + ("lstm.", "a.ve.lstm."), + ("proj.", "a.ve.proj."), + ("cond_enc.", "a.cenc."), + ): + if name.startswith(src): + new_name = dst + name[len(src):] + break + if name == "text_pos_emb.emb.weight": + new_name = "a.gen.t3.text_pos_emb" + elif name == "speech_pos_emb.emb.weight": + new_name = "a.gen.t3.speech_pos_emb" + elif name == "speech_emb.weight": + new_name = gguf.TENSOR_NAMES[gguf.MODEL_TENSOR.A_GEN_CODE_OUT_EMBD] + ".weight" + if new_name is None: + return None + return super().filter_tensors((new_name, gen)) + + def set_gguf_parameters(self): + self.gguf_writer.add_file_type(self.ftype) + + # speaker encoder, the front-end hparams are hard-coded in tools/mtmd + self.gguf_writer.add_clip_has_audio_encoder(True) + self.gguf_writer.add_clip_audio_projector_type(gguf.VisionProjectorType.CHATTERBOX_SPKENC) + self.gguf_writer.add_audio_projection_dim(80) + self.gguf_writer.add_audio_num_mel_bins(80) + self.gguf_writer.add_audio_block_count(0) + self.gguf_writer.add_audio_embedding_length(192) + self.gguf_writer.add_audio_head_count(1) + self.gguf_writer.add_audio_feed_forward_length(192) + self.gguf_writer.add_audio_attention_layernorm_eps(1e-5) + + # audio generator (s3gen, chatterbox projector); the flow encoder shape + self.gguf_writer.add_clip_has_gen_audio_encoder(True) + self.gguf_writer.add_clip_gen_audio_projector_type(gguf.VisionProjectorType.CHATTERBOX) + self.gguf_writer.add_gen_audio_projection_dim(80) + self.gguf_writer.add_gen_audio_embedding_length(512) + self.gguf_writer.add_gen_audio_feed_forward_length(2048) + self.gguf_writer.add_gen_audio_block_count(6) + self.gguf_writer.add_gen_audio_head_count(8) + self.gguf_writer.add_gen_audio_attention_layernorm_eps(1e-5) + + def _fuse_weight_norm(self, name: str, data_torch: Tensor) -> tuple[str, Tensor] | None: + # torch weight_norm parametrization: weight = g * v / |v| over dims 1..n + base = name.split(".parametrizations.weight.original")[0] + if name.endswith("original0"): + self._wnorm_g[base] = data_torch + else: + self._wnorm_v[base] = data_torch + if base in self._wnorm_g and base in self._wnorm_v: + g = self._wnorm_g.pop(base) + v = self._wnorm_v.pop(base) + norm = v.float().norm(dim=tuple(range(1, v.dim())), keepdim=True) + return (base + ".weight", g.float() * v.float() / norm) + return None + + def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]: + if ".parametrizations.weight.original" in name: + fused = self._fuse_weight_norm(name, data_torch) + if fused is not None: + yield fused + return + yield (name, data_torch) + + def tensor_force_quant(self, name, new_name, bid, n_dims): + del name, bid + # tensors read on the host stay F32 + if new_name.startswith(("a.ve.", "a.cenc.", "voice.", "a.gen.t3.", "a.gen.hift.m_source.")): + return gguf.GGMLQuantizationType.F32 + # the conv ops have no BF16 kernels + if n_dims >= 2 and new_name.endswith((".weight", ".weight_v")): + return gguf.GGMLQuantizationType.F16 + return gguf.GGMLQuantizationType.F32 + + def generate_extra_tensors(self) -> Iterable[tuple[str, Tensor]]: + conds = torch.load(self.dir_model / "conds.pt", map_location="cpu", weights_only=True) + t3c = conds["t3"] if isinstance(conds, dict) else conds.t3 + genc = conds["gen"] if isinstance(conds, dict) else conds.gen + t3c = vars(t3c) if not isinstance(t3c, dict) else t3c + genc = vars(genc) if not isinstance(genc, dict) else genc + + def talker_tensor(name: str) -> Tensor: + return LazyTorchTensor.to_eager(self.model_tensors[name]()) + + # the conds.pt voice: voice.default holds the talker conditioning rows, voice.default.ref_* the flow decoder reference + # 1D tensors are written as F32, the speech ids stay exact + yield ("voice.default.ref_tokens", genc["prompt_token"][0].float()) + yield ("voice.default.ref_feat", genc["prompt_feat"][0].float()) + yield ("voice.default.ref_spk", genc["embedding"][0].float()) + + spkr_w = talker_tensor("a.cenc.spkr_enc.weight").float() + spkr_b = talker_tensor("a.cenc.spkr_enc.bias").float() + spkr_row = spkr_w @ t3c["speaker_emb"][0].float() + spkr_b + speech_emb = talker_tensor(self.format_tensor_name(gguf.MODEL_TENSOR.A_GEN_CODE_OUT_EMBD)).float() + cond_tokens = t3c["cond_prompt_speech_tokens"][0] + + if self.is_turbo: + # turbo talker conditioning: [spkr, speech rows of the cond tokens] + yield ("voice.default", torch.cat([spkr_row[None], speech_emb[cond_tokens]], dim=0)) + return + + # multilingual talker conditioning [spkr, perceiver x32, emotion], from the reference perceiver over the cond speech tokens + with torch.no_grad(): + speech_pos = talker_tensor("a.gen.t3.speech_pos_emb").float() + pse = speech_emb[cond_tokens] + speech_pos[: cond_tokens.shape[0]] + + ln_w = talker_tensor("a.cenc.perceiver.attn.norm.weight").float() + ln_b = talker_tensor("a.cenc.perceiver.attn.norm.bias").float() + wq = talker_tensor("a.cenc.perceiver.attn.to_q.weight").float() + bq = talker_tensor("a.cenc.perceiver.attn.to_q.bias").float() + wk = talker_tensor("a.cenc.perceiver.attn.to_k.weight").float() + bk = talker_tensor("a.cenc.perceiver.attn.to_k.bias").float() + wv = talker_tensor("a.cenc.perceiver.attn.to_v.weight").float() + bv = talker_tensor("a.cenc.perceiver.attn.to_v.bias").float() + wo = talker_tensor("a.cenc.perceiver.attn.proj_out.weight").float() + bo = talker_tensor("a.cenc.perceiver.attn.proj_out.bias").float() + query = talker_tensor("a.cenc.perceiver.pre_attention_query")[0].float() + + n_head = 4 + n_e = query.shape[1] + + def attn_block(x1: Tensor, x2: Tensor) -> Tensor: + nx1 = F.layer_norm(x1, (n_e,), ln_w, ln_b) + nx2 = F.layer_norm(x2, (n_e,), ln_w, ln_b) + q = (nx1 @ wq.T + bq).view(-1, n_head, n_e // n_head).transpose(0, 1) + k = (nx2 @ wk.T + bk).view(-1, n_head, n_e // n_head).transpose(0, 1) + v = (nx2 @ wv.T + bv).view(-1, n_head, n_e // n_head).transpose(0, 1) + ctx = F.scaled_dot_product_attention(q, k, v) + ctx = ctx.transpose(0, 1).reshape(-1, n_e) + return ctx @ wo.T + bo + x1 + + pre = attn_block(query, pse) + p32 = attn_block(pre, pre) + + emo = talker_tensor("a.cenc.emotion_adv_fc.weight").float() + emo_row = emo[:, 0] * t3c["emotion_adv"].reshape(-1)[0] + yield ("voice.default", torch.cat([spkr_row[None], p32, emo_row[None]], dim=0)) diff --git a/conversion/deepseek.py b/conversion/deepseek.py index 581111da594d..733a5e5f9a2c 100644 --- a/conversion/deepseek.py +++ b/conversion/deepseek.py @@ -277,13 +277,14 @@ def set_vocab(self): from transformers import AutoTokenizer tokenizer = AutoTokenizer.from_pretrained(self.dir_model, trust_remote_code=True) + assert tokenizer is not None tokpre = self.get_vocab_base_pre(tokenizer) if tokpre == "kimi-k2": # Build merges list using the approach similar to HunYuanMoE merges = [] vocab = {} - mergeable_ranks = tokenizer.model._mergeable_ranks # ty: ignore[unresolved-attribute] + mergeable_ranks = tokenizer.model._mergeable_ranks # ty: ignore[missing-union-attribute] for token, rank in mergeable_ranks.items(): vocab[QwenModel.token_bytes_to_string(token)] = rank if len(token) == 1: @@ -294,7 +295,7 @@ def set_vocab(self): # Build token list vocab_size = self.hparams["vocab_size"] - special_tokens = tokenizer.special_tokens # ty: ignore[unresolved-attribute] + special_tokens = tokenizer.special_tokens reverse_vocab = {id_ : encoded_tok for encoded_tok, id_ in {**vocab, **special_tokens}.items()} tokens: list[str] = [] toktypes: list[int] = [] @@ -1078,7 +1079,7 @@ def set_gguf_parameters(self): self.gguf_writer.add_vision_use_silu(True) # SwiGLU MLP self.gguf_writer.add_vision_projector_scale_factor(self.hparams_vision["downsample_ratio"]) self.gguf_writer.add_vision_min_pixels(self.hparams_vision["min_pixels"]) - # hardcoded on the C++ side (see PROJECTOR_TYPE_DEEPSEEK4V in clip.cpp) + # hardcoded on the C++ side (see PROJECTOR_TYPE_DEEPSEEK4V in tools/mtmd) # if future models use different values, add GGUF keys for those assert self.global_config["vision_max_n_token"] == 384 assert self.global_config["vision_max_wh_ratio"] == 8 diff --git a/conversion/deepseek41.py b/conversion/deepseek41.py index 53980c1d041f..9a03b9912460 100644 --- a/conversion/deepseek41.py +++ b/conversion/deepseek41.py @@ -62,6 +62,7 @@ def engram_compressed_token_map(tokenizer_dir: Path) -> tuple[list[int], int]: from transformers import AutoTokenizer tokenizer = AutoTokenizer.from_pretrained(tokenizer_dir) + assert tokenizer is not None sentinel = "\ue000" normalizer = normalizers.Sequence([ normalizers.NFKC(), @@ -80,6 +81,7 @@ def engram_compressed_token_map(tokenizer_dir: Path) -> tuple[list[int], int]: text = backend.decode([token_id], skip_special_tokens=False) if "\ufffd" in text: key = backend.id_to_token(token_id) + assert key is not None else: normalized = normalizer.normalize_str(text) key = normalized if normalized else text @@ -458,7 +460,7 @@ def set_gguf_parameters(self): self.gguf_writer.add_vision_use_silu(True) self.gguf_writer.add_vision_projector_scale_factor(self.hparams_vision["downsample_ratio"]) self.gguf_writer.add_vision_min_pixels(self.hparams_vision["min_pixels"]) - # hardcoded in clip.cpp (PROJECTOR_TYPE_DEEPSEEK41V); add GGUF keys if a later model differs + # hardcoded in tools/mtmd (PROJECTOR_TYPE_DEEPSEEK41V); add GGUF keys if a later model differs assert self.hparams_vision["max_image_tokens"] == 1024, self.hparams_vision["max_image_tokens"] assert self.hparams_vision["max_wh_ratio"] is None, self.hparams_vision["max_wh_ratio"] assert self.hparams_vision["rope_theta"] == 10000, self.hparams_vision["rope_theta"] diff --git a/conversion/dots3.py b/conversion/dots3.py index c7ac2319e243..e34793e58baf 100644 --- a/conversion/dots3.py +++ b/conversion/dots3.py @@ -75,13 +75,14 @@ def _is_swa_layer(self, bid: int) -> bool: def set_vocab(self): from transformers import AutoTokenizer tokenizer = AutoTokenizer.from_pretrained(self.dir_model) + assert tokenizer is not None special_vocab = gguf.SpecialVocab(self.dir_model, load_merges=True) tokens, toktypes, tokpre = self.get_vocab_base() self.gguf_writer.add_tokenizer_model("gpt2") self.gguf_writer.add_tokenizer_pre(tokpre) self.gguf_writer.add_token_list(tokens) self.gguf_writer.add_token_types(toktypes) - special_vocab._set_special_token("eot", tokenizer.get_added_vocab()["<|endofassistant|>"]) # ty: ignore[unresolved-attribute] + special_vocab._set_special_token("eot", tokenizer.get_added_vocab()["<|endofassistant|>"]) special_vocab.add_to_gguf(self.gguf_writer) @classmethod diff --git a/conversion/drama3.py b/conversion/drama3.py new file mode 100644 index 000000000000..ac01f481c98a --- /dev/null +++ b/conversion/drama3.py @@ -0,0 +1,251 @@ +from __future__ import annotations + +from typing import Any, Callable, Iterable, TYPE_CHECKING + +import torch + +if TYPE_CHECKING: + from torch import Tensor + +from .base import ModelBase, MmprojModel, gguf +from .qwen import Qwen3_5MoeTextModel + +# Fish Audio Drama 3 (FishOmniModel) +# text model: Qwen3.5-MoE backbone, samples the main token of each frame +# mmproj: codebook tower and Fish DAC decoder (codec.pth) + +_TOWER_SHARED = "audio_decoder_mtp_shared.layers." +_TOWER_RESIDUAL = "audio_decoder_mtp.layers." + + +@ModelBase.register("FishOmniModel") +@ModelBase.example("fishaudio/drama-3") +class Drama3TextModel(Qwen3_5MoeTextModel): + model_arch = gguf.MODEL_ARCH.QWEN35MOE + no_mtp = True # the checkpoint has no MTP head + + @classmethod + def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None: + if item[0].startswith("audio_decoder"): + return None + return super().filter_tensors(item) + + +@ModelBase.register("FishOmniModel") +@ModelBase.example("fishaudio/drama-3") +class Drama3MmprojModel(MmprojModel): + has_vision_encoder = False + has_audio_encoder = False + + _LAYER_MAP = { + "enorm.weight": (gguf.MODEL_TENSOR.A_GEN_CODE_ENORM, True), + "hnorm.weight": (gguf.MODEL_TENSOR.A_GEN_CODE_HNORM, True), + "eh_proj.weight": (gguf.MODEL_TENSOR.A_GEN_CODE_EH_PROJ, False), + "semantic_embedding.weight": (gguf.MODEL_TENSOR.A_GEN_CODE_BLK_EMBD, False), + "codebook_embedding.weight": (gguf.MODEL_TENSOR.A_GEN_CODE_BLK_EMBD, False), + "final_layernorm.weight": (gguf.MODEL_TENSOR.A_GEN_CODE_BLK_OUT_NORM, True), + "transformer_layer.attention_norm.weight": (gguf.MODEL_TENSOR.A_GEN_CODE_ATTN_NORM, True), + "transformer_layer.ffn_norm.weight": (gguf.MODEL_TENSOR.A_GEN_CODE_FFN_NORM, True), + "transformer_layer.attention.q_norm.weight": (gguf.MODEL_TENSOR.A_GEN_CODE_ATTN_Q_NORM, True), + "transformer_layer.attention.k_norm.weight": (gguf.MODEL_TENSOR.A_GEN_CODE_ATTN_K_NORM, True), + "transformer_layer.attention.wo.weight": (gguf.MODEL_TENSOR.A_GEN_CODE_ATTN_OUT, False), + "transformer_layer.feed_forward.w1.weight": (gguf.MODEL_TENSOR.A_GEN_CODE_FFN_GATE, False), + "transformer_layer.feed_forward.w3.weight": (gguf.MODEL_TENSOR.A_GEN_CODE_FFN_UP, False), + "transformer_layer.feed_forward.w2.weight": (gguf.MODEL_TENSOR.A_GEN_CODE_FFN_DOWN, False), + } + + _heads: dict[int, Tensor] + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self._heads = {} + + def get_audio_config(self) -> dict[str, Any] | None: + cfg = dict(self.global_config["audio_decoder_config"]) + cfg["num_hidden_layers"] = cfg["shared_enrichment_layers"] + cfg["num_codebooks"] - 1 + return cfg + + def set_gguf_parameters(self): + cfg = self.hparams + self.gguf_writer.add_file_type(self.ftype) + self.gguf_writer.add_clip_has_gen_audio_encoder(True) + self.gguf_writer.add_clip_gen_audio_projector_type(gguf.VisionProjectorType.DRAMA3_GEN) + self.gguf_writer.add_gen_audio_projection_dim(self.n_embd_text) + self.gguf_writer.add_gen_audio_embedding_length(cfg["dim"]) + self.gguf_writer.add_gen_audio_feed_forward_length(cfg["intermediate_size"]) + self.gguf_writer.add_gen_audio_block_count(cfg["num_hidden_layers"]) + self.gguf_writer.add_gen_audio_head_count(cfg["n_head"]) + self.gguf_writer.add_gen_audio_head_count_kv(cfg["n_local_heads"]) + self.gguf_writer.add_gen_audio_attention_layernorm_eps(cfg["norm_eps"]) + # other tower and codec hparams are hardcoded in tools/mtmd + + @classmethod + def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None: + name = item[0] + if name.startswith("audio_decoder") or name == "model.language_model.embed_tokens.weight": + return item + return None + + def tensor_force_quant(self, name, new_name, bid, n_dims): + # conv1d/conv1d_dw kernels must be F16, ggml_conv_1d(_dw) has no BF16 path + if new_name.endswith(".weight") and ( + new_name in ("a.gen.wav.dac.entry.weight", "a.gen.wav.dac.post_conv.weight") + or (".up.blk." in new_name and new_name.endswith(".dwconv.weight")) + or (".dac.blk." in new_name and (new_name.endswith(".conv1.weight") or new_name.endswith(".conv2.weight"))) + ): + return gguf.GGMLQuantizationType.F16 + # keep ConvTranspose1d kernels and folded codebook tables in F32 + if new_name.endswith(".conv.weight") and (".up.blk." in new_name or ".dac.blk." in new_name): + return gguf.GGMLQuantizationType.F32 + if new_name.startswith("a.gen.wav.quant."): + return gguf.GGMLQuantizationType.F32 + return super().tensor_force_quant(name, new_name, bid, n_dims) + + def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]: + T = gguf.MODEL_TENSOR + cfg = self.hparams + + if name.startswith("a.gen.wav."): + yield (name, data_torch) + return + + if name == "model.language_model.embed_tokens.weight": + # semantic band of the backbone embeddings + begin = self.global_config["semantic_start_token_id"] + yield (self.format_tensor_name(T.A_GEN_CODE_OUT_EMBD), data_torch[begin:begin + cfg["vocab_size"]]) + return + + if name == "audio_decoder.codebook_embeddings.weight": + n_cb = cfg["num_codebooks"] + yield (self.format_tensor_name(T.A_GEN_CODE_EMBD), data_torch.reshape(n_cb, -1, data_torch.shape[-1])) + return + + if name.startswith("audio_decoder_mtp.heads."): + n_heads = cfg["num_codebooks"] - 1 + self._heads[int(name.split(".")[2])] = data_torch + if len(self._heads) < n_heads: + return + yield (self.format_tensor_name(T.A_GEN_CODE_HEAD), torch.stack([self._heads.pop(i) for i in range(n_heads)])) + return + + for prefix, offset in ((_TOWER_SHARED, 0), (_TOWER_RESIDUAL, cfg["shared_enrichment_layers"])): + if not name.startswith(prefix): + continue + idx, key = name[len(prefix):].split(".", 1) + il = int(idx) + offset + if key == "transformer_layer.attention.wqkv.weight": + # rows are [query | gate | key | value] + n_q = cfg["n_head"] * cfg["head_dim"] + n_kv = cfg["n_local_heads"] * cfg["head_dim"] + q, gate, k, v = data_torch.split([n_q, n_q, n_kv, n_kv], dim=0) + yield (self.format_tensor_name(T.A_GEN_CODE_ATTN_Q, il), q) + yield (self.format_tensor_name(T.A_GEN_CODE_ATTN_GATE, il), gate) + yield (self.format_tensor_name(T.A_GEN_CODE_ATTN_K, il), k) + yield (self.format_tensor_name(T.A_GEN_CODE_ATTN_V, il), v) + return + tensor, zero_centered = self._LAYER_MAP[key] + if zero_centered: + data_torch = data_torch.float() + 1 + yield (self.format_tensor_name(tensor, il), data_torch) + return + + raise ValueError(f"unexpected tensor {name}") + + def generate_extra_tensors(self) -> Iterable[tuple[str, Tensor]]: + # only the DAC decoder, the encoder is for voice cloning (not supported yet) + sd = torch.load(self.dir_model / "codec.pth", map_location="cpu", mmap=True, weights_only=True) + sd = sd.get("state_dict", sd) + T = gguf.MODEL_TENSOR + + def get(name: str) -> Tensor: + return sd[name].float() + + def conv(prefix: str) -> Tensor: + # fold weight norm + if prefix + ".conv.weight" in sd: + return get(prefix + ".conv.weight") + g = get(prefix + ".conv.parametrizations.weight.original0") + v = get(prefix + ".conv.parametrizations.weight.original1") + return g * v / v.norm(dim=(1, 2), keepdim=True) + + def quant_table(prefix: str) -> Tensor: + # fold out_proj into the codebook, so decoding is a row lookup + g = get(prefix + ".out_proj.weight_g") + v = get(prefix + ".out_proj.weight_v") + w = (g * v / v.norm(dim=(1, 2), keepdim=True)).squeeze(-1) + return get(prefix + ".codebook.weight") @ w.T + get(prefix + ".out_proj.bias") + + def snake(alpha: Tensor) -> tuple[Tensor, Tensor]: + alpha = alpha.flatten() + return alpha, 1.0 / (alpha + 1e-9) + + q = "quantizer." + yield (self.format_tensor_name(T.A_GEN_WAV_QUANT_FIRST_CB), quant_table(q + "semantic_quantizer.quantizers.0")) + n_rest = sum(1 for k in sd if k.startswith(q + "quantizer.quantizers.") and k.endswith(".codebook.weight")) + yield (self.format_tensor_name(T.A_GEN_WAV_QUANT_REST_CB), + torch.stack([quant_table(f"{q}quantizer.quantizers.{i}") for i in range(n_rest)])) + + # post_module uses interleaved RoPE, permute q/k rows so NEOX RoPE gives the same result + tfm = q + "post_module." + n_head, d_head = 16, 64 + order = torch.cat([torch.arange(0, d_head, 2), torch.arange(1, d_head, 2)]) + + def neox_rows(w: Tensor) -> Tensor: + return w.reshape(n_head, d_head, -1)[:, order].reshape(w.shape) + + n_tfm = sum(1 for k in sd if k.startswith(tfm + "layers.") and k.endswith(".ffn_norm.weight")) + for il in range(n_tfm): + p = f"{tfm}layers.{il}." + wq, wk, wv = get(p + "attention.wqkv.weight").chunk(3, dim=0) + yield (self.format_tensor_name(T.A_GEN_WAV_TFM_ATTN_Q, il), neox_rows(wq)) + yield (self.format_tensor_name(T.A_GEN_WAV_TFM_ATTN_K, il), neox_rows(wk)) + yield (self.format_tensor_name(T.A_GEN_WAV_TFM_ATTN_V, il), wv) + yield (self.format_tensor_name(T.A_GEN_WAV_TFM_ATTN_OUT, il), get(p + "attention.wo.weight")) + yield (self.format_tensor_name(T.A_GEN_WAV_TFM_ATTN_NORM, il), get(p + "attention_norm.weight")) + yield (self.format_tensor_name(T.A_GEN_WAV_TFM_ATTN_SCALE, il), get(p + "attention_layer_scale.gamma")) + yield (self.format_tensor_name(T.A_GEN_WAV_TFM_FFN_NORM, il), get(p + "ffn_norm.weight")) + yield (self.format_tensor_name(T.A_GEN_WAV_TFM_FFN_GATE, il), get(p + "feed_forward.w1.weight")) + yield (self.format_tensor_name(T.A_GEN_WAV_TFM_FFN_UP, il), get(p + "feed_forward.w3.weight")) + yield (self.format_tensor_name(T.A_GEN_WAV_TFM_FFN_DOWN, il), get(p + "feed_forward.w2.weight")) + yield (self.format_tensor_name(T.A_GEN_WAV_TFM_FFN_SCALE, il), get(p + "ffn_layer_scale.gamma")) + yield (self.format_tensor_name(T.A_GEN_WAV_TFM_OUTPUT_NORM), get(tfm + "norm.weight")) + + # upsample: 2x (ConvTranspose1d + ConvNeXt) + for il in range(2): + p = f"{q}upsample.{il}." + yield (self.format_tensor_name(T.A_GEN_WAV_UP_CONV, il, suffix=".weight"), conv(p + "0")) + yield (self.format_tensor_name(T.A_GEN_WAV_UP_CONV, il, suffix=".bias"), get(p + "0.conv.bias")) + yield (self.format_tensor_name(T.A_GEN_WAV_UP_DWCONV, il, suffix=".weight"), conv(p + "1.dwconv")) + yield (self.format_tensor_name(T.A_GEN_WAV_UP_DWCONV, il, suffix=".bias"), get(p + "1.dwconv.conv.bias")) + for key, tensor in (("norm", T.A_GEN_WAV_UP_NORM), ("pwconv1", T.A_GEN_WAV_UP_PW1), ("pwconv2", T.A_GEN_WAV_UP_PW2)): + yield (self.format_tensor_name(tensor, il, suffix=".weight"), get(f"{p}1.{key}.weight")) + yield (self.format_tensor_name(tensor, il, suffix=".bias"), get(f"{p}1.{key}.bias")) + yield (self.format_tensor_name(T.A_GEN_WAV_UP_GAMMA, il, suffix=""), get(p + "1.gamma")) + + # DAC: entry conv, 4x (Snake, ConvTranspose1d, 3 residual units), Snake, post conv + d = "decoder.model." + yield (self.format_tensor_name(T.A_GEN_WAV_DAC_ENTRY, suffix=".weight"), conv(d + "0")) + yield (self.format_tensor_name(T.A_GEN_WAV_DAC_ENTRY, suffix=".bias"), get(d + "0.conv.bias")) + for bid in range(4): + p = f"{d}{bid + 1}.block." + a, b = snake(get(p + "0.alpha")) + yield (self.format_tensor_name(T.A_GEN_WAV_DAC_UP_SNAKE, bid, suffix=".alpha"), a) + yield (self.format_tensor_name(T.A_GEN_WAV_DAC_UP_SNAKE, bid, suffix=".beta"), b) + yield (self.format_tensor_name(T.A_GEN_WAV_DAC_UP_CONV, bid, suffix=".weight"), conv(p + "1")) + yield (self.format_tensor_name(T.A_GEN_WAV_DAC_UP_CONV, bid, suffix=".bias"), get(p + "1.conv.bias")) + for xid in range(3): + r = f"{p}{xid + 2}.block." + for act, tensor in (("0", T.A_GEN_WAV_DAC_RES_ACT1), ("2", T.A_GEN_WAV_DAC_RES_ACT2)): + a, b = snake(get(r + act + ".alpha")) + name = gguf.TENSOR_NAMES[tensor].format(bid=bid, xid=xid) + yield (name + ".alpha", a) + yield (name + ".beta", b) + for cv, tensor in (("1", T.A_GEN_WAV_DAC_RES_CONV1), ("3", T.A_GEN_WAV_DAC_RES_CONV2)): + name = gguf.TENSOR_NAMES[tensor].format(bid=bid, xid=xid) + yield (name + ".weight", conv(r + cv)) + yield (name + ".bias", get(r + cv + ".conv.bias")) + a, b = snake(get(d + "5.alpha")) + yield (self.format_tensor_name(T.A_GEN_WAV_DAC_POST_SNAKE, suffix=".alpha"), a) + yield (self.format_tensor_name(T.A_GEN_WAV_DAC_POST_SNAKE, suffix=".beta"), b) + yield (self.format_tensor_name(T.A_GEN_WAV_DAC_POST_CONV, suffix=".weight"), conv(d + "6")) + yield (self.format_tensor_name(T.A_GEN_WAV_DAC_POST_CONV, suffix=".bias"), get(d + "6.conv.bias")) diff --git a/conversion/dream.py b/conversion/dream.py index 14f25404d67f..46c245eeea99 100644 --- a/conversion/dream.py +++ b/conversion/dream.py @@ -19,15 +19,16 @@ def get_vocab_base(self) -> tuple[list[str], list[int], str]: from transformers import AutoTokenizer tokenizer = AutoTokenizer.from_pretrained(self.dir_model, trust_remote_code=True) + assert tokenizer is not None - vocab_dict = tokenizer.get_vocab() # ty: ignore[unresolved-attribute] + vocab_dict = tokenizer.get_vocab() vocab_size = self.hparams.get("vocab_size", len(vocab_dict)) assert max(vocab_dict.values()) < vocab_size tokpre = self.get_vocab_base_pre(tokenizer) reverse_vocab = {id_: encoded_tok for encoded_tok, id_ in vocab_dict.items()} - added_vocab = tokenizer.get_added_vocab() # ty: ignore[unresolved-attribute] + added_vocab = tokenizer.get_added_vocab() for i in range(vocab_size): if i not in reverse_vocab: diff --git a/conversion/glm.py b/conversion/glm.py index 10f3ef113c4f..4aa2288dea73 100644 --- a/conversion/glm.py +++ b/conversion/glm.py @@ -31,6 +31,7 @@ def __init__(self, *args, **kwargs): def set_vocab(self): from transformers import AutoTokenizer tokenizer = AutoTokenizer.from_pretrained(self.dir_model, trust_remote_code=True) + assert tokenizer is not None special_vocab = gguf.SpecialVocab(self.dir_model, load_merges=True) tokens, toktypes, tokpre = self.get_vocab_base() self.gguf_writer.add_tokenizer_model("gpt2") @@ -38,10 +39,10 @@ def set_vocab(self): self.gguf_writer.add_token_list(tokens) self.gguf_writer.add_token_types(toktypes) special_vocab = gguf.SpecialVocab(self.dir_model, load_merges=True) - special_vocab._set_special_token("eos", tokenizer.get_added_vocab()["<|endoftext|>"]) # ty: ignore[unresolved-attribute] - special_vocab._set_special_token("eot", tokenizer.get_added_vocab()["<|user|>"]) # ty: ignore[unresolved-attribute] - special_vocab._set_special_token("unk", tokenizer.get_added_vocab()["<|endoftext|>"]) # ty: ignore[unresolved-attribute] - special_vocab._set_special_token("bos", tokenizer.get_added_vocab()["<|endoftext|>"]) # ty: ignore[unresolved-attribute] + special_vocab._set_special_token("eos", tokenizer.get_added_vocab()["<|endoftext|>"]) + special_vocab._set_special_token("eot", tokenizer.get_added_vocab()["<|user|>"]) + special_vocab._set_special_token("unk", tokenizer.get_added_vocab()["<|endoftext|>"]) + special_vocab._set_special_token("bos", tokenizer.get_added_vocab()["<|endoftext|>"]) special_vocab.add_to_gguf(self.gguf_writer) def set_gguf_parameters(self): @@ -391,16 +392,17 @@ class SolarOpenModel(Glm4MoeModel): def set_vocab(self): from transformers import AutoTokenizer tokenizer = AutoTokenizer.from_pretrained(self.dir_model) + assert tokenizer is not None special_vocab = gguf.SpecialVocab(self.dir_model, load_merges=True) tokens, toktypes, tokpre = self.get_vocab_base() self.gguf_writer.add_tokenizer_model("gpt2") self.gguf_writer.add_tokenizer_pre(tokpre) self.gguf_writer.add_token_list(tokens) self.gguf_writer.add_token_types(toktypes) - special_vocab._set_special_token("eos", tokenizer.get_added_vocab()["<|endoftext|>"]) # ty: ignore[unresolved-attribute] - special_vocab._set_special_token("eot", tokenizer.get_added_vocab()["<|endoftext|>"]) # ty: ignore[unresolved-attribute] - special_vocab._set_special_token("unk", tokenizer.get_added_vocab()[""]) # ty: ignore[unresolved-attribute] - special_vocab._set_special_token("bos", tokenizer.get_added_vocab()["<|startoftext|>"]) # ty: ignore[unresolved-attribute] + special_vocab._set_special_token("eos", tokenizer.get_added_vocab()["<|endoftext|>"]) + special_vocab._set_special_token("eot", tokenizer.get_added_vocab()["<|endoftext|>"]) + special_vocab._set_special_token("unk", tokenizer.get_added_vocab()[""]) + special_vocab._set_special_token("bos", tokenizer.get_added_vocab()["<|startoftext|>"]) special_vocab.add_to_gguf(self.gguf_writer) diff --git a/conversion/gpt_oss.py b/conversion/gpt_oss.py index 7542ec0ea8df..c4cae9611399 100644 --- a/conversion/gpt_oss.py +++ b/conversion/gpt_oss.py @@ -85,6 +85,16 @@ def generate_extra_tensors(self) -> Iterable[tuple[str, Tensor]]: def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None: name, gen = item + # Normalize erroneous GPT-OSS MXFP4 tensor naming. + # https://github.com/p-e-w/heretic/pull/211#issuecomment-5094370390 + if "mlp.experts." in name: + for proj in ("down_proj", "gate_up_proj"): + name = name.replace( + f"{proj}_blocks_blocks", f"{proj}_blocks" + ).replace( + f"{proj}_blocks_scales", f"{proj}_scales" + ) + if "sinks" in name: name += ".weight" diff --git a/conversion/hunyuan.py b/conversion/hunyuan.py index 58dee2ec8bc2..2088953c1aa1 100644 --- a/conversion/hunyuan.py +++ b/conversion/hunyuan.py @@ -24,6 +24,7 @@ class HunYuanMoEModel(TextModel): def set_vocab(self): from transformers import AutoTokenizer tokenizer = AutoTokenizer.from_pretrained(self.dir_model, trust_remote_code=True) + assert tokenizer is not None # 1. Get the pre-tokenizer identifier hash tokpre = self.get_vocab_base_pre(tokenizer) @@ -31,7 +32,7 @@ def set_vocab(self): # 2. Reverse-engineer the merges list from mergeable_ranks merges = [] vocab = {} - mergeable_ranks = tokenizer.mergeable_ranks # ty: ignore[unresolved-attribute] + mergeable_ranks = tokenizer.mergeable_ranks for token, rank in mergeable_ranks.items(): vocab[QwenModel.token_bytes_to_string(token)] = rank if len(token) == 1: @@ -42,8 +43,8 @@ def set_vocab(self): # 3. Generate the tokens and toktypes lists vocab_size = self.hparams["vocab_size"] - assert tokenizer.vocab_size == vocab_size # ty: ignore[unresolved-attribute] - special_tokens = tokenizer.special_tokens # ty: ignore[unresolved-attribute] + assert tokenizer.vocab_size == vocab_size + special_tokens = tokenizer.special_tokens reverse_vocab = {id_ : encoded_tok for encoded_tok, id_ in {**vocab, **special_tokens}.items()} tokens: list[str] = [] toktypes: list[int] = [] @@ -184,6 +185,7 @@ def set_vocab(self): else: from transformers import AutoTokenizer tokenizer = AutoTokenizer.from_pretrained(self.dir_model, trust_remote_code=True) + assert tokenizer is not None # 1. Get the pre-tokenizer identifier hash tokpre = self.get_vocab_base_pre(tokenizer) @@ -191,7 +193,7 @@ def set_vocab(self): # 2. Reverse-engineer the merges list from mergeable_ranks merges = [] vocab = {} - mergeable_ranks = tokenizer.mergeable_ranks # ty: ignore[unresolved-attribute] + mergeable_ranks = tokenizer.mergeable_ranks for token, rank in mergeable_ranks.items(): vocab[QwenModel.token_bytes_to_string(token)] = rank if len(token) == 1: @@ -202,8 +204,8 @@ def set_vocab(self): # 3. Generate the tokens and toktypes lists vocab_size = self.hparams["vocab_size"] - assert tokenizer.vocab_size == vocab_size # ty: ignore[unresolved-attribute] - special_tokens = tokenizer.special_tokens # ty: ignore[unresolved-attribute] + assert tokenizer.vocab_size == vocab_size + special_tokens = tokenizer.special_tokens reverse_vocab = {id_ : encoded_tok for encoded_tok, id_ in {**vocab, **special_tokens}.items()} tokens: list[str] = [] toktypes: list[int] = [] diff --git a/conversion/internlm.py b/conversion/internlm.py index df2668474fe6..b1c7bddd39d1 100644 --- a/conversion/internlm.py +++ b/conversion/internlm.py @@ -36,7 +36,7 @@ def set_vocab(self): logger.error(f'Error: Missing {tokenizer_path}') sys.exit(1) - sentencepiece_model = model.ModelProto() # pyright: ignore[reportAttributeAccessIssue] # ty: ignore[unresolved-attribute] + sentencepiece_model = model.ModelProto() # pyright: ignore[reportAttributeAccessIssue] # ty: ignore[missing-attribute] sentencepiece_model.ParseFromString(open(tokenizer_path, "rb").read()) add_prefix = sentencepiece_model.normalizer_spec.add_dummy_prefix diff --git a/conversion/kimi_k3.py b/conversion/kimi_k3.py index 70aabb70513b..6ea151f72d8f 100644 --- a/conversion/kimi_k3.py +++ b/conversion/kimi_k3.py @@ -343,3 +343,33 @@ def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iter return yield from super().modify_tensors(data_torch, name, bid) + + +@ModelBase.register("KimiK31ForConditionalGeneration") +@ModelBase.example("moonshotai/Kimi-K3.1-Pro") +class KimiK31Model(KimiK3Model): + """ + Kimi-K3.1: K3 with the full-attention MLA layers replaced by gated DSA. + + The tensor set is K3's plus a lightning indexer per full-attention layer, + and `dsa_config` adds the attention.indexer.* metadata. + """ + + model_arch = gguf.MODEL_ARCH.KIMI_K31 + + def set_gguf_parameters(self): + super().set_gguf_parameters() + + # --- DSA indexer --- + dsa = self.hparams["dsa_config"] + dsa_layers = dsa["dsa_layers"] + # the layer lists are 1-indexed, like linear_attn_config's + assert sorted(dsa_layers) == sorted(self.hparams["linear_attn_config"]["full_attn_layers"]), \ + "every full-attention layer is expected to own a DSA indexer" + self.gguf_writer.add_indexer_head_count(dsa["index_num_heads"]) + self.gguf_writer.add_indexer_key_length(dsa["index_head_dim"]) + self.gguf_writer.add_indexer_top_k(dsa["index_topk"]) + self.gguf_writer.add_indexer_types([(il + 1) in dsa_layers + for il in range(self.hparams["num_hidden_layers"])]) + # eps of the indexer key LayerNorm; the config declares no separate value + self.gguf_writer.add_layer_norm_eps(dsa.get("index_norm_eps", self.hparams["rms_norm_eps"])) diff --git a/conversion/kimi_linear.py b/conversion/kimi_linear.py index 697ab1b4a9ed..c205003a5958 100644 --- a/conversion/kimi_linear.py +++ b/conversion/kimi_linear.py @@ -29,13 +29,14 @@ def set_vocab(self): from transformers import AutoTokenizer tokenizer = AutoTokenizer.from_pretrained(self.dir_model, trust_remote_code=True) + assert tokenizer is not None tokpre = self.get_vocab_base_pre(tokenizer) if tokpre == "kimi-k2": # Build merges list using the approach similar to HunYuanMoE merges = [] vocab = {} - mergeable_ranks = tokenizer.model._mergeable_ranks # ty: ignore[unresolved-attribute] + mergeable_ranks = tokenizer.model._mergeable_ranks # ty: ignore[missing-union-attribute] for token, rank in mergeable_ranks.items(): vocab[QwenModel.token_bytes_to_string(token)] = rank if len(token) == 1: @@ -45,7 +46,7 @@ def set_vocab(self): merges.append(' '.join(map(QwenModel.token_bytes_to_string, merged))) # Build token list vocab_size = self.hparams["vocab_size"] - special_tokens = tokenizer.special_tokens # ty: ignore[unresolved-attribute] + special_tokens = tokenizer.special_tokens reverse_vocab = {id_ : encoded_tok for encoded_tok, id_ in {**vocab, **special_tokens}.items()} tokens: list[str] = [] toktypes: list[int] = [] @@ -71,7 +72,7 @@ def set_vocab(self): special_vocab = gguf.SpecialVocab(self.dir_model, load_merges=False) special_vocab.add_to_gguf(self.gguf_writer) # override eos id in config.json with tiktoken eos id - self.gguf_writer.add_eos_token_id(tokenizer.eos_id) # ty: ignore[unresolved-attribute] + self.gguf_writer.add_eos_token_id(tokenizer.eos_id) else: raise NotImplementedError(f"Deepseek pre-tokenizer {tokpre!r} is not supported yet!") diff --git a/conversion/kimivl.py b/conversion/kimivl.py index ae60abf30983..8234ac0271d4 100644 --- a/conversion/kimivl.py +++ b/conversion/kimivl.py @@ -57,6 +57,8 @@ def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iter class KimiK25Model(MmprojModel): """Kimi-K2.5 with MoonViT3d vision encoder""" + projector_type = gguf.VisionProjectorType.KIMIK25 + def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) @@ -78,7 +80,7 @@ def set_gguf_parameters(self): super().set_gguf_parameters() assert self.hparams_vision is not None - self.gguf_writer.add_clip_projector_type(gguf.VisionProjectorType.KIMIK25) + self.gguf_writer.add_clip_projector_type(self.projector_type) # Position embedding parameters (for interpolation) self.gguf_writer.add_uint32("vision.pos_emb_height", self.hparams_vision.get("init_pos_emb_height", 64)) @@ -87,7 +89,11 @@ def set_gguf_parameters(self): # Projector parameters self.gguf_writer.add_vision_use_gelu(self.hparams_vision.get("projector_hidden_act", "gelu") == "gelu") - self.gguf_writer.add_vision_attention_layernorm_eps(self.hparams_vision.get("projector_ln_eps", 1e-5)) + if self.hparams_vision.get("norm_type", "layernorm") == "rmsnorm": + # nn.RMSNorm without eps uses finfo(float32).eps, also for bf16 inputs + self.gguf_writer.add_vision_attention_layernorm_eps(torch.finfo(torch.float32).eps) + else: + self.gguf_writer.add_vision_attention_layernorm_eps(self.hparams_vision.get("projector_ln_eps", 1e-5)) self.gguf_writer.add_vision_projector_scale_factor(self.merge_kernel_size[0]) # Image size limits @@ -120,7 +126,7 @@ def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Ca def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]: assert self.hparams_vision is not None - n_head = self.hparams_vision.get("num_attention_heads", 16) + n_head = self.find_vparam(["num_attention_heads", "vt_num_attention_heads"], optional=True) or 16 # Permute Q/K weights/biases from interleaved to split RoPE format # This allows using build_rope_2d at runtime without post-permutation. @@ -171,3 +177,19 @@ def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iter name = name.replace("mm_projector.linear_", "mm_projector.proj.linear_", 1) yield from super().modify_tensors(data_torch, name, bid) + + +@ModelBase.register("KimiK31ForConditionalGeneration") +@ModelBase.example("moonshotai/Kimi-K3.1-Pro") +class KimiK31VisionModel(KimiK25Model): + """Kimi-K2.5 MoonViT3d with RMSNorm, a wider QKV and the patchmergerv2 projector""" + + projector_type = gguf.VisionProjectorType.KIMIK31 + + def set_gguf_parameters(self): + super().set_gguf_parameters() + assert self.hparams_vision is not None + assert self.hparams_vision["norm_type"] == "rmsnorm" + assert self.hparams_vision["mm_projector_type"] == "patchmergerv2" + n_head = self.find_vparam(["vt_num_attention_heads"]) + self.gguf_writer.add_vision_head_dim(self.hparams_vision["qkv_hidden_size"] // n_head) diff --git a/conversion/lev.py b/conversion/lev.py index 72b12445d8ee..5c319ec25fb3 100644 --- a/conversion/lev.py +++ b/conversion/lev.py @@ -77,10 +77,10 @@ def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iter delta = self.lora_scale * (lora["B"] @ lora["A"]) data_torch = data_torch.float() + LazyTorchTensor.from_eager(delta) self.lora_merged.add(name[name.index("layers."):]) - yield from super().modify_tensors(data_torch, name, bid) # ty: ignore[unresolved-attribute] + yield from super().modify_tensors(data_torch, name, bid) # ty: ignore[missing-attribute] def prepare_tensors(self): - super().prepare_tensors() # ty: ignore[unresolved-attribute] + super().prepare_tensors() # ty: ignore[missing-attribute] if len(self.lora_merged) != len(self.lora): raise ValueError(f"only {len(self.lora_merged)} of {len(self.lora)} LoRA tensors were merged into the base model") diff --git a/conversion/lfm3.py b/conversion/lfm3.py index a0287d4bbfe0..38fbb5a31a9e 100644 --- a/conversion/lfm3.py +++ b/conversion/lfm3.py @@ -181,11 +181,11 @@ class LFM3VLVisionMixin(MmprojModel): def _set_lfm3vl_gguf_parameters(self): assert self.hparams_vision is not None - # clip.cpp hardcodes the GELU projector and its input norm eps + # mtmd hardcodes the GELU projector and its input norm eps if (self.global_config["projector_hidden_act"] != "gelu" or not self.global_config["projector_bias"] or self.global_config["projector_layer_norm_eps"] != 1e-6): raise ValueError("unsupported projector_hidden_act / projector_bias / projector_layer_norm_eps") - # clip.cpp hardcodes the vision rope base, the RGB patchifier and the 3x3 spatial conv + # mtmd hardcodes the vision rope base, the RGB patchifier and the 3x3 spatial conv if (self.hparams_vision["rope_parameters"]["rope_theta"] != 1e6 or self.hparams_vision["in_channels"] != 3 or self.hparams_vision["conv_kernel_size"] != 3): raise ValueError("unsupported vision rope_theta / in_channels / conv_kernel_size") @@ -347,7 +347,7 @@ class LFM3AudioMmprojModel(MmprojModel): "feed_forward.w3": gguf.MODEL_TENSOR.A_GEN_WAV_TFM_FFN_UP, "feed_forward.w2": gguf.MODEL_TENSOR.A_GEN_WAV_TFM_FFN_DOWN, } - # the codec graph in clip.cpp hardcodes the liquid-codec FSQ front end and decoder shape, conversion fails on a mismatch + # the codec graph in mtmd hardcodes the liquid-codec FSQ front end and decoder shape, conversion fails on a mismatch _codec = { "model_type": "liquid_codec", "fsq_levels": [8, 7, 6, 6], "n_q": 12, "quantizer_dim": 256, "encoder_dim": 512, "stft_n_fft": 1280, "stft_hop_length": 320, "stft_win_length": 1280, "stft_frame_stack": 6, @@ -405,7 +405,7 @@ def set_gguf_parameters(self): if fe[k] != v: raise ValueError(f"unsupported feature_extractor.{k}={fe[k]!r}, expected {v!r}") - # clip.cpp hardcodes the encoder rope base + # mtmd hardcodes the encoder rope base if self.hparams_audio["rope_parameters"]["rope_theta"] != 1e6: raise ValueError("unsupported audio encoder rope_theta") @@ -420,7 +420,7 @@ def set_gguf_parameters(self): if not self._audio_output: return - # audio generation: depthformer hparams, the codec ones are constant and live in clip.cpp + # audio generation: depthformer hparams, the codec ones are constant and live in mtmd gen = self.get_generation_config() for k, v in self._generation.items(): if gen[k] != v: diff --git a/conversion/llada.py b/conversion/llada.py index c03607191a76..7f91536e13e0 100644 --- a/conversion/llada.py +++ b/conversion/llada.py @@ -22,15 +22,16 @@ def get_vocab_base(self) -> tuple[list[str], list[int], str]: from transformers import AutoTokenizer tokenizer = AutoTokenizer.from_pretrained(self.dir_model, trust_remote_code=True) + assert tokenizer is not None - vocab_dict = tokenizer.get_vocab() # ty: ignore[unresolved-attribute] + vocab_dict = tokenizer.get_vocab() vocab_size = self.hparams.get("vocab_size", len(vocab_dict)) assert max(vocab_dict.values()) < vocab_size tokpre = self.get_vocab_base_pre(tokenizer) reverse_vocab = {id_: encoded_tok for encoded_tok, id_ in vocab_dict.items()} - added_vocab = tokenizer.get_added_vocab() # ty: ignore[unresolved-attribute] + added_vocab = tokenizer.get_added_vocab() for i in range(vocab_size): if i not in reverse_vocab: diff --git a/conversion/magevl.py b/conversion/magevl.py new file mode 100644 index 000000000000..910aeecf160d --- /dev/null +++ b/conversion/magevl.py @@ -0,0 +1,77 @@ +from __future__ import annotations + +from typing import Callable, Iterable, TYPE_CHECKING + +import torch + +if TYPE_CHECKING: + from torch import Tensor + +from .base import MmprojModel, ModelBase, gguf + + +@ModelBase.register("LlavaOnevision2ForConditionalGeneration") +@ModelBase.example("microsoft/Mage-VL-Cowork") +class MageVLVisionModel(MmprojModel): + # a rope pair uses 2 different angles: the graph runs 2 ropes on heads permuted to [evens | odds] + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + assert self.hparams_vision is not None + + self.n_embd = self.find_vparam(["hidden_size"]) + self.n_head = self.find_vparam(["num_attention_heads"]) + self.d_head = self.n_embd // self.n_head + + d_half = self.d_head // 2 + assert d_half % 16 == 0, "the 4:6:6 split of the rope requires d_head to be a multiple of 32" + + # 4:6:6 split for t:h:w + n_t = 4 * (d_half // 16) + n_s = 6 * (d_half // 16) + theta = float(self.find_vparam(["rope_theta"])) + freq_t = 1.0 / theta ** (torch.arange(n_t, dtype=torch.float64) / n_t) + freq_s = 1.0 / theta ** (torch.arange(n_s, dtype=torch.float64) / n_s) + freq = torch.cat([freq_t, freq_s, freq_s]) + self.rope_freq = torch.cat([freq, freq]) + + def pair_axis(i: int) -> int: + dim = (2 * i) % d_half + return 0 if dim < n_t else 1 if dim < n_t + n_s else 2 + + self.rope_pairs = sorted(range(d_half), key=pair_axis) + + def set_gguf_parameters(self): + super().set_gguf_parameters() + self.gguf_writer.add_clip_projector_type(gguf.VisionProjectorType.MAGEVL) + self.gguf_writer.add_vision_attention_layernorm_eps(self.find_vparam(["layer_norm_eps"])) + self.gguf_writer.add_vision_spatial_merge_size(self.find_vparam(["spatial_merge_size"])) + self.gguf_writer.add_vision_min_pixels(int(self.preprocessor_config["min_pixels"])) + self.gguf_writer.add_vision_max_pixels(int(self.preprocessor_config["max_pixels"])) + + @classmethod + def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None: + name, _ = item + + if not name.startswith("visual."): + return None + + return super().filter_tensors(item) + + def generate_extra_tensors(self) -> Iterable[tuple[str, Tensor]]: + # ggml uses theta = pos / rope_freqs + pairs = torch.tensor(self.rope_pairs) + freq_a = self.rope_freq[2 * pairs] + freq_b = self.rope_freq[2 * pairs + 1] + yield ("visual.rope_freqs.weight", (1.0 / torch.cat([freq_a, freq_b])).to(torch.float32)) + + def permute_qk(self, data_torch: Tensor) -> Tensor: + rows = [2 * i for i in self.rope_pairs] + [2 * i + 1 for i in self.rope_pairs] + return data_torch.reshape(self.n_head, self.d_head, -1)[:, rows].reshape(data_torch.shape) + + def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]: + if ".self_attn.qkv." in name: + q, k, v = data_torch.split(self.n_embd, dim=0) + data_torch = torch.cat([self.permute_qk(q), self.permute_qk(k), v], dim=0) + + yield from super().modify_tensors(data_torch, name, bid) diff --git a/conversion/metis.py b/conversion/metis.py new file mode 100644 index 000000000000..a073ed4df0a9 --- /dev/null +++ b/conversion/metis.py @@ -0,0 +1,204 @@ +from __future__ import annotations + +import math +from typing import Callable, Iterable, TYPE_CHECKING + +import torch + +if TYPE_CHECKING: + from torch import Tensor + +from .base import LazyTorchTensor, ModelBase, TextModel, gguf + + +@ModelBase.register("Metis16ForCausalLM") +class Metis16Model(TextModel): + model_arch = gguf.MODEL_ARCH.METIS16 + + # read by generate_extra_tensors() or not used at inference: dropped in modify_tensors() + _folded = { + "pass_embeddings", "lm_head.weight", "depth_budget.multiplier", + "depth_memory.pass_embeddings", "depth_memory.anchor_embeddings", + "depth_memory.state_write.bias", "depth_memory.metadata_write.bias", + } + _folded_layer = { + "mixer_connection.controller.weight", "mixer_connection.controller.bias", "mixer_connection.mix_logits", + "mixer_connection.read_logits", "mixer_connection.write_logits", + "moe_connection.controller.weight", "moe_connection.controller.bias", "moe_connection.mix_logits", + "moe_connection.read_logits", "moe_connection.write_logits", + "moe.k_router.weight", "moe.k_router.bias", "moe.k_budget.multiplier", + } + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + hp = self.hparams + self.n_pass = hp["num_passes"] + self.attn_layers = list(hp["attention_layer_indices"]) + self._generated: set[str] = set() + # one cache slot per pass and layer, while the tensors exist once per physical layer + self.block_count = self.n_pass * hp["num_hidden_layers"] + + def set_vocab(self): + self._set_vocab_gpt2() + # turns end on the first eos id, so llama.cpp must stop on it too + eos = self.hparams.get("eos_token_id") + if isinstance(eos, list) and eos: + self.gguf_writer.add_eot_token_id(eos[0]) + + def set_gguf_parameters(self): + hp = self.hparams + super().set_gguf_parameters() + + n_layer = hp["num_hidden_layers"] + self.gguf_writer.add_head_count_kv([hp["num_key_value_heads"] if il % n_layer in self.attn_layers else 0 for il in range(self.block_count)]) + + self.gguf_writer.add_expert_feed_forward_length(hp["expert_intermediate_size"]) + self.gguf_writer.add_expert_shared_feed_forward_length(hp["shared_expert_intermediate_size"]) + self.gguf_writer.add_expert_shared_count(1) + self.gguf_writer.add_moe_latent_size(hp["latent_dim"]) + + d_inner = hp["mamba_expand"] * hp["hidden_size"] + self.gguf_writer.add_ssm_conv_kernel(hp["mamba_d_conv"]) + self.gguf_writer.add_ssm_inner_size(d_inner) + self.gguf_writer.add_ssm_state_size(hp["mamba_d_state"]) + self.gguf_writer.add_ssm_time_step_rank(d_inner // hp["mamba_head_dim"]) + self.gguf_writer.add_ssm_group_count(hp["mamba_n_groups"]) + self.gguf_writer.add_ssm_norm_epsilon(hp["mamba_norm_eps"]) + + self.gguf_writer.add_hyper_connection_count(hp["n_streams"]) + self.gguf_writer.add_metis_pass_count(self.n_pass) + self.gguf_writer.add_metis_memory_length(hp["memory_dim"]) + self.gguf_writer.add_metis_route_length(hp["route_feature_dim"]) + self.gguf_writer.add_metis_continuation_logit_limit(hp["continuation_logit_limit"]) + self.gguf_writer.add_attention_pass_lora_rank(hp["attention_pass_lora_rank"]) + + @classmethod + def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None: + name, gen = item + # the checkpoint leaves these names suffix-less; add the suffix of the GGUF tensor so the mapper keeps it + if name.endswith((".mixer.impl.mixer.dt_bias", ".moe.selection_bias")): + name += ".bias" + elif (name.endswith((".moe.expert_embeddings", ".mixer.pass_gates")) + or name in ("depth_memory.stream_gate", "stream_embeddings")): + name += ".weight" + elif name == "depth_memory.stream_gate_bias": + name = "depth_memory.stream_gate.bias" + return super().filter_tensors((name, gen)) + + def _load(self, name: str) -> Tensor: + return LazyTorchTensor.to_eager(self.model_tensors[name]()) + + def _emit(self, name: str, data: Tensor) -> tuple[str, Tensor]: + self._generated.add(name) + return name, data + + def generate_extra_tensors(self) -> Iterable[tuple[str, Tensor]]: + hp = self.hparams + r = hp["route_feature_dim"] + n_stream = hp["n_streams"] + T = gguf.MODEL_TENSOR + passes = self._load("pass_embeddings")[: self.n_pass] + + # the mixes depend only on the weights and the pass, so Sinkhorn runs once here + # the controller runs in the checkpoint dtype, as the reference does + for il in range(hp["num_hidden_layers"]): + for src, (mix, read, write) in ( + ("mixer_connection", (T.HC_ATTN_MIX, T.HC_ATTN_READ, T.HC_ATTN_WRITE)), + ("moe_connection", (T.HC_FFN_MIX, T.HC_FFN_READ, T.HC_FFN_WRITE)), + ): + prefix = f"layers.{il}.{src}." + ctrl_w, ctrl_b = self._load(prefix + "controller.weight"), self._load(prefix + "controller.bias") + mixes, reads, writes = [], [], [] + for p in range(self.n_pass): + delta = torch.nn.functional.linear(passes[p].to(ctrl_w.dtype), ctrl_w, ctrl_b).float() + m_delta, r_delta, w_delta = torch.split(delta, [n_stream * n_stream, n_stream, n_stream]) + log_m = self._load(prefix + "mix_logits").float() + m_delta.view(n_stream, n_stream) + for _ in range(hp["mhc_sinkhorn_iterations"]): + log_m = log_m - torch.logsumexp(log_m, dim=-1, keepdim=True) + log_m = log_m - torch.logsumexp(log_m, dim=-2, keepdim=True) + mixes.append(log_m.exp()) + reads.append(torch.softmax(self._load(prefix + "read_logits").float() + r_delta, dim=-1)) + writes.append(torch.softmax(self._load(prefix + "write_logits").float() + w_delta, dim=-1)) + act = ctrl_w.dtype + yield self._emit(self.format_tensor_name(mix, il), torch.stack(mixes).to(act).float()) + yield self._emit(self.format_tensor_name(read, il), torch.stack(reads).to(act).float()) + yield self._emit(self.format_tensor_name(write, il), torch.stack(writes).to(act).float()) + + # one bias per memory entry: write biases, pass and anchor embeddings, and the constant routed width + meta_w = self._load("depth_memory.metadata_write.weight").float() + mean_k = hp["num_experts_per_tok"] / hp["max_routed_k"] + base = (self._load("depth_memory.state_write.bias").float() + self._load("depth_memory.metadata_write.bias").float() + + mean_k * meta_w[:, r]) + pass_e = self._load("depth_memory.pass_embeddings").float() + anchor_e = self._load("depth_memory.anchor_embeddings").float() + n_anchor = len(self.attn_layers) + 1 + entry_bias = torch.stack([base + pass_e[p] + anchor_e[a] for p in range(self.n_pass) for a in range(n_anchor)]) + yield self._emit(self.format_tensor_name(T.DEPTH_MEM_ENTRY_BIAS), entry_bias) + + def _split_features(self, data_torch: Tensor, template: str, parts: tuple[str, ...], d: int, bid: int | None) -> Iterable[tuple[str, Tensor]]: + # the graph reads the concatenated features one block at a time; each block is its own tensor + for i, part in enumerate(parts): + piece = data_torch[:, i * d:(i + 1) * d] if i < 3 else data_torch[:, 3 * d:] + yield from super().modify_tensors(piece, template.format(part), bid) + + def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]: + if name in self._generated: + yield name, data_torch + return + if name in self._folded: + return + hp = self.hparams + d = hp["hidden_size"] + + if bid is None: + if name == "stream_embeddings.weight": + # the graph keeps the streams innermost: [n_stream, n_embd, n_tokens] + yield from super().modify_tensors(data_torch.T.contiguous(), name, bid) + return + if name == "depth_memory.metadata_write.weight": + # metadata input becomes [summary, entropy, confidence, continuation]; entropy arrives unnormalized + r = hp["route_feature_dim"] + meta_w = data_torch.float() + entropy_col = meta_w[:, r + 1:r + 2] / math.log(hp["n_routed_experts"]) + data_torch = torch.cat((meta_w[:, :r], entropy_col, meta_w[:, r + 2:]), dim=1) + elif name == "depth_memory.stream_gate.weight": + data_torch = data_torch.float() / math.sqrt(d) + elif name == "depth_memory.route_projection.weight": + yield from self._split_features(data_torch, "depth_memory.route_{}.weight", ("state", "retr", "diff", "hist"), d, bid) + return + elif name == "continuation.hidden.weight": + yield from self._split_features(data_torch, "continuation.hidden_{}.weight", ("state", "retr", "diff", "route"), d, bid) + return + yield from super().modify_tensors(data_torch, name, bid) + return + + local = name.removeprefix(f"layers.{bid}.") + if local in self._folded_layer: + return + + if local == "mixer.impl.mixer.A_log": + data_torch = -torch.exp(data_torch).unsqueeze(-1) + elif local == "mixer.impl.mixer.D": + data_torch = data_torch.unsqueeze(-1) + elif local == "mixer.impl.mixer.conv1d.weight": + data_torch = data_torch.squeeze(1) + elif local == "mixer.impl.mixer.norm.weight": + data_torch = data_torch.reshape(hp["mamba_n_groups"], -1) + elif local == "mixer.pass_gates.weight": + # the reference applies tanh(gate) in the checkpoint dtype + data_torch = torch.tanh(data_torch[: self.n_pass]).to(torch.bfloat16).float() + elif local == "moe.shared_expert.gate_up.weight": + gate, up = data_torch.chunk(2, dim=0) + yield from super().modify_tensors(gate, name.replace("gate_up", "gate"), bid) + yield from super().modify_tensors(up, name.replace("gate_up", "up"), bid) + return + + yield from super().modify_tensors(data_torch, name, bid) + + def tensor_force_quant(self, name: str, new_name: str, bid: int | None, n_dims: int) -> gguf.GGMLQuantizationType | bool: + # FP32 in the checkpoint, tiny and read as rows, or computed here at full precision: keep FP32 + if (new_name.startswith(("cont_", "depth_mem_entry_bias", "depth_mem_stream_gate")) + or new_name.endswith(("ffn_exp_embd.weight", "ffn_gate_inp.weight", "hc_attn_mix.weight", "hc_attn_read.weight", + "hc_attn_write.weight", "hc_ffn_mix.weight", "hc_ffn_read.weight", "hc_ffn_write.weight"))): + return gguf.GGMLQuantizationType.F32 + return super().tensor_force_quant(name, new_name, bid, n_dims) diff --git a/conversion/minicpm.py b/conversion/minicpm.py index d8fceb0c08e0..2b239437cd8e 100644 --- a/conversion/minicpm.py +++ b/conversion/minicpm.py @@ -161,7 +161,7 @@ def __init__(self, *args, **kwargs): # In MiniCPM-V 4.6 `vision_config.image_size` (980) describes the SigLIP # positional embedding bucket grid (70 x 70), while the per-slice processing # resolution is the preprocessor's `scale_resolution` (typically 448). - # The CLIP loader in tools/mtmd/clip.cpp consumes `clip.vision.image_size` + # The mmproj loader in tools/mtmd/mtmd-model.cpp consumes `clip.vision.image_size` # as the slice size and warmup resolution, so report `scale_resolution` there # to match the upstream MiniCPMV4_6ImageProcessorPil slicing rules. scale_resolution = self.preprocessor_config.get( @@ -173,7 +173,7 @@ def set_gguf_parameters(self): super().set_gguf_parameters() assert self.hparams_vision is not None - # projector type string is consumed by clip_projector_type_from_string() in clip.cpp + # projector type string is consumed by projector_type_from_string() in tools/mtmd/mtmd-impl.h self.gguf_writer.add_clip_projector_type(self.projector_type) self.gguf_writer.add_vision_projector_scale_factor( @@ -243,7 +243,7 @@ class MiniCPMV4_7VisionModel(MiniCPMV4_6VisionModel): projector_type = gguf.VisionProjectorType.MINICPMV4_7 # MiniCPMV4_7ImageProcessorPil default default_scale_resolution = 448 - # rows of v.tok_embd_sep, the order must match clip_suffix_rows() in clip-impl.h + # rows of v.tok_embd_sep, the order must match mtmd_suffix_rows() in tools/mtmd/mtmd-impl.h tok_embd_sep = ["", "", "", "\n"] def get_downsample_mode(self) -> str: @@ -263,6 +263,7 @@ def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iter # the tile separators are text tokens; clip appends their embeddings so that one chunk holds the whole image from transformers import AutoTokenizer tokenizer = AutoTokenizer.from_pretrained(self.dir_model) + assert tokenizer is not None ids = [] for text in self.tok_embd_sep: tok = tokenizer.encode(text, add_special_tokens=False) diff --git a/conversion/minimax.py b/conversion/minimax.py index aac340c61414..4d856fab855f 100644 --- a/conversion/minimax.py +++ b/conversion/minimax.py @@ -25,7 +25,8 @@ def _get_suppress_tokens(self) -> Sequence[int] | None: # they get in the way of the token sampling process and must be suppressed tokenizer = AutoTokenizer.from_pretrained(self.dir_model, trust_remote_code=True) - tokenizer_vocab_size = tokenizer.vocab_size # ty: ignore[unresolved-attribute] + assert tokenizer is not None + tokenizer_vocab_size = tokenizer.vocab_size with open(self.dir_model / "model.safetensors.index.json", "r", encoding="utf-8") as f: weight_map = json.load(f)["weight_map"] diff --git a/conversion/mistral.py b/conversion/mistral.py index aec22ca3877c..b2b5b255786a 100644 --- a/conversion/mistral.py +++ b/conversion/mistral.py @@ -12,9 +12,9 @@ from .llama import LlamaModel if _mistral_common_installed: - from mistral_common.tokens.tokenizers.base import TokenizerVersion # type: ignore[import-not-found, ty:unresolved-import] - from mistral_common.tokens.tokenizers.tekken import Tekkenizer # type: ignore[import-not-found, ty:unresolved-import] - from mistral_common.tokens.tokenizers.sentencepiece import SentencePieceTokenizer # type: ignore[import-not-found, ty:unresolved-import] + from mistral_common.tokens.tokenizers.base import TokenizerVersion # ty: ignore[import-not-found] + from mistral_common.tokens.tokenizers.tekken import Tekkenizer # ty: ignore[import-not-found] + from mistral_common.tokens.tokenizers.sentencepiece import SentencePieceTokenizer # ty: ignore[import-not-found] else: TokenizerVersion = None # type: ignore[assignment] Tekkenizer = None # type: ignore[assignment] diff --git a/conversion/muse_glimmer.py b/conversion/muse_glimmer.py index 9b63d3dba64a..f00b3e6b5823 100644 --- a/conversion/muse_glimmer.py +++ b/conversion/muse_glimmer.py @@ -37,7 +37,8 @@ def set_vocab(self): from transformers import AutoTokenizer tok = AutoTokenizer.from_pretrained(self.dir_model) - eot_id = tok.convert_tokens_to_ids("<|eot|>") # ty: ignore[unresolved-attribute] + assert tok is not None + eot_id = tok.convert_tokens_to_ids("<|eot|>") if isinstance(eot_id, int) and eot_id >= 0: self.gguf_writer.add_eot_token_id(eot_id) @@ -152,7 +153,7 @@ def set_vocab(self): target_arch = json.load(f)["architectures"][0] target_cls = get_model_class(target_arch) if target_cls is not type(self): - target_cls.set_vocab(self) # ty: ignore[unresolved-attribute] + target_cls.set_vocab(self) # ty: ignore[missing-attribute] else: super().set_vocab() diff --git a/conversion/muse_spark.py b/conversion/muse_spark.py index 150f22f95940..5378b43aca21 100644 --- a/conversion/muse_spark.py +++ b/conversion/muse_spark.py @@ -44,7 +44,8 @@ def set_vocab(self): from transformers import AutoTokenizer tok = AutoTokenizer.from_pretrained(self.dir_model) - eot_id = tok.convert_tokens_to_ids("<|eot|>") # ty: ignore[unresolved-attribute] + assert tok is not None + eot_id = tok.convert_tokens_to_ids("<|eot|>") if isinstance(eot_id, int) and eot_id >= 0: self.gguf_writer.add_eot_token_id(eot_id) diff --git a/conversion/nemotron.py b/conversion/nemotron.py index 371f5b616f64..76aa357ab952 100644 --- a/conversion/nemotron.py +++ b/conversion/nemotron.py @@ -73,7 +73,7 @@ def set_vocab(self): try: if target_arch == "NemotronHForCausalLM": self.is_moe = "num_experts_per_tok" in target_text_config - target_cls.set_vocab(self) # ty: ignore[unresolved-attribute] + target_cls.set_vocab(self) # ty: ignore[missing-attribute] finally: self.dir_model = original_dir self.gguf_writer.add_mask_token_id(self.hparams["dflash_config"]["mask_token_id"]) @@ -208,7 +208,7 @@ def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Ca return name, gen def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]: - # RADIO's pos_embed doesn't have .weight suffix, but clip.cpp expects it + # RADIO's pos_embed doesn't have .weight suffix, but mtmd expects it if "patch_generator.pos_embed" in name: # Downsample position embeddings for fixed 512x512 image size import torch.nn.functional as F @@ -428,6 +428,9 @@ class NemotronHModel(GraniteHybridModel): is_moe: bool = False supports_mtp_export = True _experts: list[dict[str, Tensor]] | None = None + # tokens past the embedding rows (left out of the vocab), and special tokens that are printed as text + _vocab_skip: tuple[str, ...] = () + _user_defined_tokens: re.Pattern[str] | None = None _SSM_LAYER_TYPES = {"mamba", "linear_attention"} _ATTN_LAYER_TYPES = {"attention", "full_attention"} @@ -606,6 +609,7 @@ def set_vocab(self): from transformers import AutoTokenizer tokenizer = AutoTokenizer.from_pretrained(self.dir_model, trust_remote_code=True) + assert tokenizer is not None # Pad vocab size (from Mamba2Model/GraniteHybridModel) self.hparams["pad_vocab_size_multiple"] = 8 # Setting this here since GraniteHybridModel.set_vocab() isn't being invoked now. @@ -616,14 +620,18 @@ def set_vocab(self): vocab_size = -(vocab_size // -pad_vocab) * pad_vocab self.hparams["vocab_size"] = vocab_size - assert max(tokenizer.vocab.values()) < vocab_size # ty: ignore[unresolved-attribute] + # draft-model converters (DFlash, LiLiCorr) call this with their own instance, which has neither hook + vocab_skip = getattr(self, "_vocab_skip", ()) + user_defined_tokens = getattr(self, "_user_defined_tokens", None) + + assert max(id_ for tok, id_ in tokenizer.vocab.items() if tok not in vocab_skip) < vocab_size tokpre = self.get_vocab_base_pre(tokenizer) - reverse_vocab = {id_: encoded_tok for encoded_tok, id_ in tokenizer.vocab.items()} # ty: ignore[unresolved-attribute] - added_vocab = tokenizer.get_added_vocab() # ty: ignore[unresolved-attribute] + reverse_vocab = {id_: encoded_tok for encoded_tok, id_ in tokenizer.vocab.items()} + added_vocab = tokenizer.get_added_vocab() - added_tokens_decoder = tokenizer.added_tokens_decoder # ty: ignore[unresolved-attribute] + added_tokens_decoder = tokenizer.added_tokens_decoder for i in range(vocab_size): if i not in reverse_vocab: @@ -634,11 +642,12 @@ def set_vocab(self): if token in added_vocab: if not added_tokens_decoder[i].normalized: previous_token = token - token = tokenizer.decode(tokenizer.encode(token, add_special_tokens=False)) # ty: ignore[unresolved-attribute, invalid-assignment] + token = tokenizer.decode(tokenizer.encode(token, add_special_tokens=False)) # ty: ignore[invalid-assignment] if previous_token != token: logger.info(f"{repr(previous_token)} is encoded and decoded back to {repr(token)} using AutoTokenizer") - if added_tokens_decoder[i].special or self.does_token_look_special(token): + printed = user_defined_tokens is not None and user_defined_tokens.fullmatch(token) + if (added_tokens_decoder[i].special or self.does_token_look_special(token)) and not printed: toktypes.append(gguf.TokenType.CONTROL) else: token = token.replace(b"\xe2\x96\x81".decode("utf-8"), " ") # pre-normalize user-defined spaces @@ -834,3 +843,140 @@ def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Ca if item[0].startswith("mtp."): return None return super().filter_tensors(item) + + +# NeMo speechlm2 SALM checkpoint: a Nemotron-H decoder under llm.*, the speech encoder under perception.* +@ModelBase.register("Nemotron3_5TranscribeForConditionalGeneration") +@ModelBase.example("nvidia/NVIDIA-Nemotron-3.5-Transcribe-30B-A4B") +class NemotronTranscribeModel(NemotronHModel): + model_arch = gguf.MODEL_ARCH.NEMOTRON_H + # the audio placeholder (id vocab_size) has no embedding row; speaker, event and style tags print with the transcript + _vocab_skip = ("<|audio|>",) + _user_defined_tokens = re.compile(r"||") + _renames = { + "llm.model.embed_tokens.weight": "backbone.embeddings.weight", + "llm.model.norm.weight": "backbone.norm_f.weight", + "llm.lm_head.weight": "lm_head.weight", + } + # grouped experts: [n_expert, n_in, n_out] -> [n_expert, n_out, n_in] + _grouped_experts = {"gate_and_up_projs": "up_proj", "down_projs": "down_proj"} + + @classmethod + def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None: + name, gen = item + if name.startswith(("perception.", "ctc_timestamp.")) or "_extra_state" in name: + return None + if not name.startswith("llm."): + # the FP8 and NVFP4 releases store the decoder as backbone.* + raise ValueError(f"unexpected tensor {name}: only the BF16 checkpoint layout (llm.*) is supported") + if name in cls._renames: + name = cls._renames[name] + elif name.startswith("llm.model.layers."): + name = "backbone.layers." + name[len("llm.model.layers."):] + elif name.startswith("llm.mtp.layers."): + name = "mtp.layers." + name[len("llm.mtp.layers."):] + else: + raise ValueError(f"unexpected decoder tensor {name}") + return super().filter_tensors((name.replace("._fp32_params.", "."), gen)) + + def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]: + leaf = name.rsplit(".", 1)[-1] + if ".mixer.experts." in name and leaf in self._grouped_experts: + if name.startswith("mtp."): + assert self._mtp_bid is not None, f"{name}: MTP head is not exported" + bid = self._mtp_bid + assert bid is not None + data_torch = data_torch.permute(0, 2, 1).contiguous() + name = f"model.layers.{bid}.mlp.experts.{self._grouped_experts[leaf]}.weight" + yield from ModelBase.modify_tensors(self, data_torch, name, bid) + return + yield from super().modify_tensors(data_torch, name, bid) + + +@ModelBase.register("Nemotron3_5TranscribeForConditionalGeneration") +class NemotronTranscribeAudioModel(MmprojModel): + # NeMo ParallelExpertEncoder: ASR and speaker transformers, speaker activity fused into the ASR states + has_vision_encoder = False + has_audio_encoder = True + + def get_audio_config(self) -> dict[str, Any] | None: + asr = self.global_config["pe_encoder_config"]["asr_encoder_cfg"] + return { + "hidden_size": asr["d_model"], + "intermediate_size": int(float(asr.get("ff_expansion", 4)) * asr["d_model"]), + "num_hidden_layers": asr["n_layers"], + "num_attention_heads": asr["n_heads"], + } + + @staticmethod + def _rope_dims(cfg: dict[str, Any]) -> int: + assert cfg.get("self_attention_model") == "rope" and cfg.get("subsampling", "feature_stacking") == "feature_stacking" + assert cfg.get("attn_mode", "full") == "full" and not cfg.get("xscaling", False) and cfg.get("pre_block_norm", True) + assert float(cfg.get("rope_base", 10000.0)) == 10000.0 + n_rot = int(cfg["d_model"] // cfg["n_heads"] * float(cfg.get("rotary_fraction", 1.0))) + assert n_rot >= 2 and n_rot % 2 == 0 + return n_rot + + def set_gguf_parameters(self): + super().set_gguf_parameters() + pe = self.global_config["pe_encoder_config"] + asr = pe["asr_encoder_cfg"] + diar = pe["diarization_model_cfg"] + spk = diar["encoder"] + assert pe.get("asr_encoder_type") == "transformer" and pe.get("speaker_feature_mode", "thresholded") == "thresholded" + assert pe.get("asr_normalize_type") == "per_feature" and pe.get("diar_normalize_type") in (None, "NA") + assert asr["subsampling_factor"] == spk["subsampling_factor"] == 8 and asr["feat_in"] == spk["feat_in"] + assert self._rope_dims(spk) == spk["d_model"] // spk["n_heads"] and not spk.get("qkv_bias", False) + assert diar.get("high_resolution", False) + # constants in clip: the ParallelExpertEncoder online settings and the Sortformer streaming config + assert (pe.get("online_inference_length", 500), pe.get("chunk_left_context", 50), pe.get("chunk_right_context", 50), + pe.get("diar_spkcache_len", 188), pe.get("diar_fifo_len", 40), pe.get("diar_spkcache_update_period", 300), + pe.get("speaker_activity_threshold", 0.5)) == (500, 50, 50, 188, 40, 300, 0.5), \ + "streaming settings differ from the constants in tools/mtmd/models/nemotron-transcribe.cpp" + sm = diar["sortformer_modules"] + assert (sm["spkcache_sil_frames_per_spk"], sm["pred_score_threshold"], sm["scores_boost_latest"], sm["strong_boost_rate"], + sm["weak_boost_rate"], sm["min_pos_scores_rate"], sm["use_learnable_sil_emb"]) == (1, 0.25, 0.05, 0.75, 1.5, 0.5, True), \ + "speaker cache settings differ from the constants in tools/mtmd/models/nemotron-transcribe.cpp" + + self.gguf_writer.add_clip_projector_type(gguf.VisionProjectorType.NEMOTRON_TRANSCRIBE) + self.gguf_writer.add_audio_num_mel_bins(asr["feat_in"]) + self.gguf_writer.add_audio_attention_layernorm_eps(1e-5) + self.gguf_writer.add_audio_subsampling_factor(asr["subsampling_factor"]) + self.gguf_writer.add_audio_rope_dims(self._rope_dims(asr)) + self.gguf_writer.add_audio_spk_block_count(spk["n_layers"]) + self.gguf_writer.add_audio_spk_head_count(spk["n_heads"]) + + def generate_extra_tensors(self) -> Iterable[tuple[str, Tensor]]: + # ParallelExpertEncoder._build_sinusoid_position_encoding + pe = self.global_config["pe_encoder_config"] + n_spk = int(pe["diarization_model_cfg"]["sortformer_modules"]["num_spks"]) + d = int(pe["asr_encoder_cfg"]["d_model"]) + scale = float(pe.get("spk_kernel_scale", 1.0)) + pos = torch.arange(n_spk, dtype=torch.float32)[:, None] + div = torch.exp(torch.arange(0, d, 2, dtype=torch.float32) * -(np.log(10000.0) / d)) + kernel = torch.zeros(n_spk, d, dtype=torch.float32) + kernel[:, 0::2] = torch.sin(pos * div) + kernel[:, 1::2] = torch.cos(pos * div) + # stored as [d, n_spk]: the graph multiplies it with the speaker activity directly + yield self.format_tensor_name(gguf.MODEL_TENSOR.A_FUSION_SPK_KERNEL, suffix=""), (kernel * scale).T.contiguous() + + def tensor_force_quant(self, name, new_name, bid, n_dims): + if self.match_model_tensor_name(new_name, gguf.MODEL_TENSOR.A_SPK_HEAD_UPSAMPLE, None): + return gguf.GGMLQuantizationType.F16 # ggml_conv_1d kernel + return super().tensor_force_quant(name, new_name, bid, n_dims) + + @classmethod + def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None: + name, gen = item + if not name.startswith("perception.") or "_extra_state" in name: + return None + # training-only head; the speaker model's unused front end + if name.startswith(("perception.encoder.diarization_model.sortformer_modules.hidden_to_spks.", + "perception.encoder.diarization_model.preprocessor.")): + return None + return name, gen + + def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]: + if name == "perception.preprocessor.featurizer.fb": + data_torch = data_torch.squeeze(0) + yield from super().modify_tensors(data_torch, name, bid) diff --git a/conversion/pockettts.py b/conversion/pockettts.py index 1b5c841110cf..b0cddc685ca0 100644 --- a/conversion/pockettts.py +++ b/conversion/pockettts.py @@ -191,7 +191,7 @@ def set_vocab(self): # unigram segmentation, so use the UGM tokenizer instead from sentencepiece import sentencepiece_model_pb2 as model - proto = model.ModelProto() # pyright: ignore[reportAttributeAccessIssue] # ty: ignore[unresolved-attribute] + proto = model.ModelProto() # pyright: ignore[reportAttributeAccessIssue] # ty: ignore[missing-attribute] proto.ParseFromString(open(self.dir_model / "tokenizer.model", "rb").read()) assert proto.trainer_spec.model_type == 1, "expected a unigram tokenizer" @@ -302,7 +302,7 @@ def set_gguf_parameters(self): # voice-prompt encoder: mimi encoder + speaker_proj if self.has_audio_encoder: self.gguf_writer.add_clip_has_audio_encoder(True) - # note: the 24kHz sample rate is hardcoded on the clip.cpp side, like the other audio models + # note: the 24kHz sample rate is hardcoded on the mtmd side, like the other audio models self.gguf_writer.add_clip_audio_projector_type(gguf.VisionProjectorType.POCKETTTS_SPKENC) self.gguf_writer.add_audio_projection_dim(self.n_embd_text) self.gguf_writer.add_audio_block_count(self.hparams_audio["num_hidden_layers"]) @@ -314,7 +314,7 @@ def set_gguf_parameters(self): self.gguf_writer.add_audio_num_mel_bins(1) # generation: flow-matching decoder + mimi decoder - # the SEANet and flow net hparams are constant across the family, clip.cpp holds them + # the SEANet and flow net hparams are constant across the family, mtmd holds them self.gguf_writer.add_clip_has_gen_audio_encoder(True) self.gguf_writer.add_clip_gen_audio_projector_type(gguf.VisionProjectorType.POCKETTTS_GEN) self.gguf_writer.add_gen_audio_projection_dim(self.n_embd_text) diff --git a/conversion/qwen.py b/conversion/qwen.py index e6456e5a54de..376535460a2c 100644 --- a/conversion/qwen.py +++ b/conversion/qwen.py @@ -170,7 +170,7 @@ def prepare_tensors(self): raise ValueError(f"Unprocessed experts: {experts}") -@ModelBase.register("Qwen3ForCausalLM", "Qwen3Model") +@ModelBase.register("Qwen3ForCausalLM", "Qwen3Model", "LlavaOnevision2ForConditionalGeneration") @ModelBase.example("Qwen/Qwen3-8B") class Qwen3Model(Qwen2Model): model_arch = gguf.MODEL_ARCH.QWEN3 @@ -225,12 +225,13 @@ def set_vocab(self): def _find_rerank_config(self): from transformers import AutoTokenizer tokenizer = AutoTokenizer.from_pretrained(self.dir_model) + assert tokenizer is not None self.is_rerank = True self.is_tied_embeddings = self.hparams.get("tie_word_embeddings", False) - self.token_false_id = tokenizer.convert_tokens_to_ids("no") # ty: ignore[unresolved-attribute, invalid-assignment] - self.token_true_id = tokenizer.convert_tokens_to_ids("yes") # ty: ignore[unresolved-attribute, invalid-assignment] - self.sep_token_id = tokenizer.convert_tokens_to_ids("|") # ty: ignore[unresolved-attribute] + self.token_false_id = tokenizer.convert_tokens_to_ids("no") # ty: ignore[invalid-assignment] + self.token_true_id = tokenizer.convert_tokens_to_ids("yes") # ty: ignore[invalid-assignment] + self.sep_token_id = tokenizer.convert_tokens_to_ids("|") assert self.token_false_id is not None and self.token_true_id is not None @@ -329,7 +330,7 @@ def index_tensors(self, remote_hf_model_id: str | None = None) -> dict[str, Call key = next((k for k in ["n_layers", "num_hidden_layers", "n_layer", "num_layers"] if k in hparams), None) type(self)._original_block_count = hparams.get(key) type(self).opt_num_mtp_layers = 0 - return super().index_tensors(remote_hf_model_id=remote_hf_model_id) # ty: ignore[unresolved-attribute] + return super().index_tensors(remote_hf_model_id=remote_hf_model_id) # ty: ignore[missing-attribute] @classmethod def filter_tensors(cls, item): @@ -366,7 +367,7 @@ def filter_tensors(cls, item): return name, gen def set_gguf_parameters(self): - super().set_gguf_parameters() # ty: ignore[unresolved-attribute] + super().set_gguf_parameters() # ty: ignore[missing-attribute] if self.no_mtp: return if (n := self.block_count - self.hparams["num_hidden_layers"]) > 0: @@ -374,15 +375,15 @@ def set_gguf_parameters(self): def prepare_metadata(self, vocab_only: bool): from_dir = self.fname_out.is_dir() - super().prepare_metadata(vocab_only=vocab_only) # ty: ignore[unresolved-attribute] + super().prepare_metadata(vocab_only=vocab_only) # ty: ignore[missing-attribute] if not self.mtp_only or not from_dir: return - output_type: str = self.ftype.name.partition("_")[2] # pyright: ignore[reportAttributeAccessIssue] # ty: ignore[unresolved-attribute] + output_type: str = self.ftype.name.partition("_")[2] # pyright: ignore[reportAttributeAccessIssue] # ty: ignore[missing-attribute] fname_default: str = gguf.naming_convention( - self.metadata.name, self.metadata.basename, self.metadata.finetune, # pyright: ignore[reportAttributeAccessIssue] # ty: ignore[unresolved-attribute] - self.metadata.version, size_label=None, output_type=output_type, model_type=None) # pyright: ignore[reportAttributeAccessIssue] # ty: ignore[unresolved-attribute] + self.metadata.name, self.metadata.basename, self.metadata.finetune, # pyright: ignore[reportAttributeAccessIssue] # ty: ignore[missing-attribute] + self.metadata.version, size_label=None, output_type=output_type, model_type=None) # pyright: ignore[reportAttributeAccessIssue] # ty: ignore[missing-attribute] self.fname_out = self.fname_out.parent / f"mtp-{fname_default}.gguf" @@ -492,6 +493,45 @@ class _LinearAttentionVReorderBase(Qwen3NextModel): interleaved repeat: [G0_v0, G1_v0, ..., G0_v1, G1_v1, ...]. """ + _SHARED_INPUT_MERGES: tuple[tuple[gguf.MODEL_TENSOR, tuple[gguf.MODEL_TENSOR, gguf.MODEL_TENSOR], bool], ...] = () + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self._merge_parts: dict[tuple[int, gguf.MODEL_TENSOR], dict[int, Tensor]] = {} + + # caches one half of a merged weight until the other half arrives, then yields [a | b] + def _merge_shared_input(self, new_name: str, data: Tensor, bid: int | None) -> Iterable[tuple[str, Tensor]]: + # a pre-quantized checkpoint carries per-tensor scales, which one merged weight cannot hold + if not self._SHARED_INPUT_MERGES or bid is None or self.hparams.get("quantization_config") or self._fp8_e4m3_preserved or self.hadamard_folded_names() or hasattr(data, "get_lora_A_B"): + yield (new_name, data) + return + + layer_types = self.hparams.get("layer_types") + if layer_types is None: + is_linear = bid < self.hparams["num_hidden_layers"] and (bid + 1) % self.hparams.get("full_attention_interval", 4) != 0 + else: + is_linear = bid < len(layer_types) and layer_types[bid] == "linear_attention" + + match = next(((merged, parts.index(part)) for merged, parts, linear_only in self._SHARED_INPUT_MERGES + if merged in gguf.MODEL_TENSORS[self.model_arch] + for part in parts + if (is_linear or not linear_only) and self.match_model_tensor_name(new_name, part, bid)), None) + if match is None: + yield (new_name, data) + return + + merged, idx = match + halves = self._merge_parts.setdefault((bid, merged), {}) + halves[idx] = data + if len(halves) == 2: + del self._merge_parts[(bid, merged)] + yield (self.format_tensor_name(merged, bid), torch.cat([halves[0], halves[1]], dim=0)) + + def prepare_tensors(self): + super().prepare_tensors() + if self._merge_parts: + raise ValueError(f"merged weights missing a half: {sorted((bid, t.name) for bid, t in self._merge_parts)}") + @staticmethod def _reorder_v_heads(tensor: Tensor, dim: int, num_k_heads: int, num_v_per_k: int, head_dim: int) -> Tensor: """Reorder V heads from grouped (by K head) to tiled order along the given dimension.""" @@ -671,7 +711,8 @@ def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iter # Out projection weight: reorder columns (input dimension) data_torch = self._reorder_v_heads(data_torch, 1, num_k_heads, num_v_per_k, head_v_dim) - yield from super().modify_tensors(data_torch, name, bid) + for new_name, data in super().modify_tensors(data_torch, name, bid): + yield from self._merge_shared_input(new_name, data, bid) class _Qwen35MRopeMixin: @@ -685,7 +726,7 @@ class _Qwen35MRopeMixin: rope_parameters: dict def set_gguf_parameters(self): - super().set_gguf_parameters() # ty: ignore[unresolved-attribute] + super().set_gguf_parameters() # ty: ignore[missing-attribute] if "mrope_section" not in self.rope_parameters: self.gguf_writer.add_rope_dimension_sections(self._QWEN35_DEFAULT_MROPE_SECTION) @@ -695,6 +736,11 @@ def set_gguf_parameters(self): class Qwen3_5TextModel(_Qwen35MRopeMixin, _LinearAttentionVReorderBase): model_arch = gguf.MODEL_ARCH.QWEN35 + _SHARED_INPUT_MERGES = ( + (gguf.MODEL_TENSOR.ATTN_QKV_GATE, (gguf.MODEL_TENSOR.ATTN_QKV, gguf.MODEL_TENSOR.ATTN_GATE), True), + (gguf.MODEL_TENSOR.SSM_BETA_ALPHA_CAT, (gguf.MODEL_TENSOR.SSM_BETA, gguf.MODEL_TENSOR.SSM_ALPHA), True), + ) + def __init__(self, dir_model, *args, **kwargs): # Inner TextModel does not own mtp.*. Set no_mtp before mixin bumps block_count. hparams = kwargs.pop("hparams", None) @@ -843,7 +889,7 @@ def set_vocab(self): if target_cls is not type(self): if target_arch == "NemotronHForCausalLM": setattr(self, "is_moe", "num_experts_per_tok" in target_hparams) - target_cls.set_vocab(self) # ty: ignore[unresolved-attribute] + target_cls.set_vocab(self) # ty: ignore[missing-attribute] else: super().set_vocab() diff --git a/conversion/qwen3tts.py b/conversion/qwen3tts.py index cdb1e0c9b370..28bb6e095f56 100644 --- a/conversion/qwen3tts.py +++ b/conversion/qwen3tts.py @@ -235,11 +235,11 @@ def set_gguf_parameters(self): # handle speaker encoder config self.gguf_writer.add_audio_projection_dim(self.n_embd_text) - # mel_spectrogram() front-end: sr=24000, n_fft=1024, hop=256, n_mels=128, fmin=0, fmax=12000 (=sr/2, the clip.cpp default) + # mel_spectrogram() front-end: sr=24000, n_fft=1024, hop=256, n_mels=128, fmin=0, fmax=12000 (=sr/2, the mtmd default) self.gguf_writer.add_audio_num_mel_bins(128) # 3 SE-Res2Net stages; the stem conv, mfa, asp and fc are not counted here self.gguf_writer.add_audio_block_count(3) - # ECAPA-TDNN has no attention/FFN, these are dummy to allow clip.cpp to load it + # ECAPA-TDNN has no attention/FFN, these are dummy to allow mtmd to load it self.gguf_writer.add_audio_embedding_length(1536) self.gguf_writer.add_audio_head_count(1) self.gguf_writer.add_audio_feed_forward_length(1536) @@ -256,7 +256,7 @@ def set_gguf_parameters(self): self.gguf_writer.add_gen_audio_head_count(code_predictor_config["num_attention_heads"]) self.gguf_writer.add_gen_audio_head_count_kv(code_predictor_config["num_key_value_heads"]) self.gguf_writer.add_gen_audio_attention_layernorm_eps(code_predictor_config["rms_norm_eps"]) - # note: code2wav hparams are hardcoded on the mtmd/clip.cpp side for now, not written here + # note: code2wav hparams are hardcoded on the mtmd side for now, not written here def _wav_decoder_config(self) -> dict[str, Any] | None: # code2wav has its own config.json, inside the speech_tokenizer dir diff --git a/conversion/qwen4exp.py b/conversion/qwen4exp.py index 8a55d678d74a..9680d2937dfb 100644 --- a/conversion/qwen4exp.py +++ b/conversion/qwen4exp.py @@ -41,8 +41,6 @@ def __init__(self, *args, **kwargs): self._ple_shards: dict[int, str] = {} self._ple_row_dim: int | None = None self._mtp_fc: dict[str, Tensor] = {} - # halves of the merged weights, keyed by (bid, merged tensor), see _merge_shared_input - self._merge_parts: dict[tuple[int, gguf.MODEL_TENSOR], dict[int, Tensor]] = {} @classmethod def filter_tensors(cls, item): @@ -171,8 +169,7 @@ def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iter yield (self.map_tensor_name(name), data_torch.squeeze()) return - for new_name, data in super().modify_tensors(data_torch, name, bid): - yield from self._merge_shared_input(new_name, data, bid) + yield from super().modify_tensors(data_torch, name, bid) # matmuls that read the same input, written as one weight [a | b] so the runtime does one matmul: # (merged tensor, (a, b), linear-attention layers only) @@ -183,30 +180,6 @@ def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iter (gguf.MODEL_TENSOR.SSM_BETA_ALPHA_CAT, (gguf.MODEL_TENSOR.SSM_BETA, gguf.MODEL_TENSOR.SSM_ALPHA), True), ) - # caches one half of a merged weight until the other half arrives, then yields [a | b] - def _merge_shared_input(self, new_name: str, data: Tensor, bid: int | None) -> Iterable[tuple[str, Tensor]]: - # a pre-quantized checkpoint carries per-tensor scales, which one merged weight cannot hold - if bid is None or self.hparams.get("quantization_config"): - yield (new_name, data) - return - - layer_types = self.hparams.get("layer_types") or [] - is_linear = bid < len(layer_types) and layer_types[bid] == "linear_attention" - - match = next(((merged, parts.index(part)) for merged, parts, linear_only in self._SHARED_INPUT_MERGES - for part in parts - if (is_linear or not linear_only) and self.match_model_tensor_name(new_name, part, bid)), None) - if match is None: - yield (new_name, data) - return - - merged, idx = match - halves = self._merge_parts.setdefault((bid, merged), {}) - halves[idx] = data - if len(halves) == 2: - del self._merge_parts[(bid, merged)] - yield (self.format_tensor_name(merged, bid), torch.cat([halves[0], halves[1]], dim=0)) - # the shards concatenate into a tensor of well over 100 GB # use LazyChunkedTensor here, a single shard resident at a time def _place_ple_shard(self, data_torch: Tensor, name: str) -> Iterable[tuple[str, Tensor]]: @@ -251,8 +224,6 @@ def prepare_tensors(self): super().prepare_tensors() if self._mtp_fc: raise ValueError(f"MTP projection missing its other half: {sorted(self._mtp_fc)}") - if self._merge_parts: - raise ValueError(f"merged weights missing a half: {sorted((bid, t.name) for bid, t in self._merge_parts)}") n_parts = self.hparams.get("split_ngram_parts", 0) if self._ple_shards and len(self._ple_shards) != n_parts: raise ValueError( diff --git a/conversion/ricursive.py b/conversion/ricursive.py new file mode 100644 index 000000000000..13cb77367ed4 --- /dev/null +++ b/conversion/ricursive.py @@ -0,0 +1,92 @@ +from __future__ import annotations + +import math + +from typing import Iterable, TYPE_CHECKING + +if TYPE_CHECKING: + from torch import Tensor + +from .base import ModelBase, TextModel, gguf, logger + + +@ModelBase.register("RicursiveForCausalLM") +@ModelBase.example("ricursiveAI/Ripple-65B-Preview") +class RicursiveModel(TextModel): + model_arch = gguf.MODEL_ARCH.RICURSIVE + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + hparams = self.hparams + + if (act := hparams.get("hidden_act", "silu")) != "silu": + raise ValueError(f"Ricursive: unsupported hidden_act {act!r}") + + # RicursiveConfig derives the layer pattern from layer_types alone, by default full attention iff i % 5 == 4 + if (layer_types := hparams.get("layer_types")) is None: + layer_types = ["full_attention" if i % 5 == 4 else "sliding_attention" for i in range(self.block_count)] + if len(layer_types) != self.block_count: + raise ValueError(f"Ricursive: layer_types has {len(layer_types)} entries, expected {self.block_count}") + if unknown := set(layer_types) - {"sliding_attention", "full_attention"}: + raise ValueError(f"Ricursive: unsupported layer_types {sorted(unknown)}") + self.is_swa = [t == "sliding_attention" for t in layer_types] + # only sliding layers use RoPE, an all-full-attention model would have no positional encoding + if not any(self.is_swa): + raise ValueError("Ricursive: layer_types has no sliding_attention layer, all-full-attention models are not supported") + # RicursiveConfig pops a stored hybrid_layer_pattern and recomputes it from layer_types + if (pattern := hparams.get("hybrid_layer_pattern")) is not None and list(pattern) != [int(swa) for swa in self.is_swa]: + logger.warning("Ricursive: hybrid_layer_pattern does not match layer_types, ignoring it") + + if not hparams.get("gate_full_precision", True): + logger.warning("Ricursive: gate_full_precision is false, llama.cpp computes the attention gate in f32") + + # RicursiveConfig defaults; the reference ignores the tensors of a disabled feature + self.use_valres = hparams.get("use_valres", False) + self.use_gate = hparams.get("use_gated_attention", True) + names = list(self.model_tensors) + if bias := [name for name in names if name.endswith("bias")]: + raise ValueError(f"Ricursive: bias tensors are not supported, found {bias[0]!r}") + for enabled, suffix in ((self.use_valres, ".self_attn.value_res_lambda"), (self.use_gate, ".self_attn.g_proj.weight")): + if enabled and sum(name.endswith(suffix) for name in names) != self.block_count: + raise ValueError(f"Ricursive: expected one {suffix!r} tensor per layer") + + def set_vocab(self): + self._set_vocab_gpt2() + # eos_token_id is [<|im_end|>, <|endoftext|>]: register the turn end as EOT so generation stops on it + eos = self.hparams.get("eos_token_id") + if isinstance(eos, list): + extra = [e for e in eos if e != self.hparams.get("bos_token_id")] + if extra: + self.gguf_writer.add_eot_token_id(extra[0]) + logger.info(f"gguf: eot_token_id = {extra[0]}") + + def set_gguf_parameters(self): + super().set_gguf_parameters() + hparams = self.hparams + self.gguf_writer.add_vocab_size(hparams["vocab_size"]) + self.gguf_writer.add_rope_dimension_count(hparams["head_dim"]) + + self.gguf_writer.add_sliding_window(hparams["sliding_window"]) + self.gguf_writer.add_sliding_window_pattern(self.is_swa) + # sliding layers use RoPE, full attention layers use no positional encoding + self.gguf_writer.add_rope_pattern(self.is_swa) + + if (embed_scale := hparams.get("embed_scale")) is not None: + if embed_scale == 0: + raise ValueError(f"Ricursive: unsupported embed_scale {embed_scale!r}") + self.gguf_writer.add_embedding_scale(float(embed_scale)) + # absent and 0 both mean no clamp (the reference tests `if clamp_threshold:`) + if (clamp := hparams.get("ffn_clamp_threshold")) is not None: + if clamp < 0 or not math.isfinite(clamp): + raise ValueError(f"Ricursive: unsupported ffn_clamp_threshold {clamp!r}") + self.gguf_writer.add_swiglu_clamp([float(clamp)] * self.block_count) + + def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]: + # value_res_lambda has no .weight suffix in the checkpoint + if name.endswith(".value_res_lambda"): + if not self.use_valres: + return + name += ".weight" + elif name.endswith(".g_proj.weight") and not self.use_gate: + return + yield from super().modify_tensors(data_torch, name, bid) diff --git a/conversion/t5.py b/conversion/t5.py index 3466ce49da63..e42525794098 100644 --- a/conversion/t5.py +++ b/conversion/t5.py @@ -40,7 +40,7 @@ def set_vocab(self): if not tokenizer_path.is_file(): raise FileNotFoundError(f"File not found: {tokenizer_path}") - sentencepiece_model = model.ModelProto() # pyright: ignore[reportAttributeAccessIssue] # ty: ignore[unresolved-attribute] + sentencepiece_model = model.ModelProto() # pyright: ignore[reportAttributeAccessIssue] # ty: ignore[missing-attribute] sentencepiece_model.ParseFromString(open(tokenizer_path, "rb").read()) # some models like Pile-T5 family use BPE tokenizer instead of Unigram @@ -178,7 +178,7 @@ def set_vocab(self): if not tokenizer_path.is_file(): raise FileNotFoundError(f"File not found: {tokenizer_path}") - sentencepiece_model = model.ModelProto() # pyright: ignore[reportAttributeAccessIssue] # ty: ignore[unresolved-attribute] + sentencepiece_model = model.ModelProto() # pyright: ignore[reportAttributeAccessIssue] # ty: ignore[missing-attribute] sentencepiece_model.ParseFromString(open(tokenizer_path, "rb").read()) # some models like Pile-T5 family use BPE tokenizer instead of Unigram diff --git a/conversion/xing.py b/conversion/xing.py new file mode 100644 index 000000000000..5ee4670505ef --- /dev/null +++ b/conversion/xing.py @@ -0,0 +1,133 @@ +from __future__ import annotations + +import re +from typing import Callable, Iterable, TYPE_CHECKING + +if TYPE_CHECKING: + from torch import Tensor + +from .base import ModelBase, gguf +from .deepseek import DeepseekV2Model + + +@ModelBase.register("Xing4_0ForCausalLM") +class Xing4_0Model(DeepseekV2Model): + """Xing4_0 : DeepSeek-V2/V3 backbone (MLA + MoE) + mHC residual mixing.""" + + model_arch = gguf.MODEL_ARCH.XING4_0 + + skip_mtp = False + supports_mtp_export = True + _n_main_layers: int | None = None + + # map (prefix, kind) -> MODEL_TENSOR enum + _hc_tensor_map = { + ("hc_attn", "fn"): gguf.MODEL_TENSOR.HC_ATTN_FN, + ("hc_attn", "base"): gguf.MODEL_TENSOR.HC_ATTN_BASE, + ("hc_attn", "scale"): gguf.MODEL_TENSOR.HC_ATTN_SCALE, + ("hc_ffn", "fn"): gguf.MODEL_TENSOR.HC_FFN_FN, + ("hc_ffn", "base"): gguf.MODEL_TENSOR.HC_FFN_BASE, + ("hc_ffn", "scale"): gguf.MODEL_TENSOR.HC_FFN_SCALE, + } + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + self.block_count = self.hparams["num_hidden_layers"] + if not self.no_mtp: + self.block_count += self.hparams.get("num_nextn_predict_layers", 0) + self.tensor_map = gguf.get_tensor_name_map(self.model_arch, self.block_count) + + def index_tensors(self, remote_hf_model_id: str | None = None): + # needed by filter_tensors() before any tensor is filtered + type(self)._n_main_layers = self.hparams["num_hidden_layers"] + return super().index_tensors(remote_hf_model_id=remote_hf_model_id) + + @classmethod + def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None: + if (titem := super().filter_tensors(item)) is None: + return None + name, gen = titem + + # the NextN/MTP block lives past num_hidden_layers (model.layers.40 -> blk.40) + assert cls._n_main_layers is not None + is_mtp = (m := re.match(r"model\.layers\.(\d+)\.", name)) is not None and int(m.group(1)) >= cls._n_main_layers + + # --no-mtp: drop the appended NextN block entirely. + if is_mtp and cls.no_mtp: + return None + # --mtp: keep ONLY NextN-block tensors plus the shared embeddings/ + # norm/lm_head (so the resulting GGUF carries just the draft head). + if cls.mtp_only and not is_mtp and name not in ( + "model.embed_tokens.weight", "model.norm.weight", "lm_head.weight", + ): + return None + + return name, gen + + def set_gguf_parameters(self): + rope_type = self.rope_parameters.get("rope_type") or self.rope_parameters.get("type") + if rope_type == "rope": + self.rope_parameters["rope_type"] = "yarn" + if "type" in self.rope_parameters: + self.rope_parameters["type"] = "yarn" + # base.py's YARN branch accesses this with a direct key lookup + self.rope_parameters.setdefault("original_max_position_embeddings", + self.hparams.get("original_max_position_embeddings", 4096)) + + super().set_gguf_parameters() + hparams = self.hparams + + # NextN/MTP prediction layers + if not self.no_mtp and (num_nextn_predict_layers := hparams.get("num_nextn_predict_layers")) is not None: + self.gguf_writer.add_nextn_predict_layers(num_nextn_predict_layers) + + self.gguf_writer.add_hyper_connection_count( + hparams.get("hc_mult", 1)) + self.gguf_writer.add_hyper_connection_sinkhorn_iterations( + hparams.get("hc_sinkhorn_iters", 20)) + self.gguf_writer.add_hyper_connection_epsilon( + hparams.get("hc_eps", 1e-6)) + + def set_vocab(self): + # Xing4_0 uses a SentencePiece tokenizer (tokenizer.model + Xing4_0Tokenizer). + # V2's set_vocab tries GPT2/BPE first, which fails for SPM. Use the SPM path directly. + # Note: no add_tokenizer_pre override — the C++ SPM load path ignores + # tokenizer.ggml.pre entirely (pre-tokenizers only apply to BPE vocabs). + self._set_vocab_sentencepiece() + + def prepare_metadata(self, vocab_only: bool): + from_dir = self.fname_out.is_dir() + super().prepare_metadata(vocab_only=vocab_only) + + if not self.mtp_only or not from_dir: + return + + output_type: str = self.ftype.name.partition("_")[2] + fname_default: str = gguf.naming_convention( + self.metadata.name, self.metadata.basename, self.metadata.finetune, + self.metadata.version, size_label=None, output_type=output_type, model_type=None) + self.fname_out = self.fname_out.parent / f"mtp-{fname_default}.gguf" + + def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]: + # handle mHC tensors: model.layers.{N}.{attn_hc|ffn_hc}.{hc_fn|hc_base|hc_scale} + match = re.match(r"model\.layers\.(\d+)\.(attn_hc|ffn_hc)\.(.+)$", name) + if match: + layer_idx = int(match.group(1)) + hc_type = match.group(2) # "attn_hc" or "ffn_hc" + param = match.group(3) # "hc_fn", "hc_base", "hc_scale" + + if bid is None: + bid = layer_idx + + prefix = "hc_attn" if hc_type == "attn_hc" else "hc_ffn" + + kind = {"hc_fn": "fn", "hc_base": "base", "hc_scale": "scale"}.get(param) + if kind is None: + raise ValueError(f"unexpected mHC tensor: {name}") + tensor_enum = self._hc_tensor_map[(prefix, kind)] + gguf_name = self.format_tensor_name(tensor_enum, bid) + yield (gguf_name, data_torch) + return + + yield from super().modify_tensors(data_torch, name, bid) diff --git a/conversion/xverse.py b/conversion/xverse.py index aa3b338802e4..27fcff9d37a9 100644 --- a/conversion/xverse.py +++ b/conversion/xverse.py @@ -25,15 +25,16 @@ def set_vocab(self): from transformers import AutoTokenizer tokenizer = AutoTokenizer.from_pretrained(dir_model) - vocab_size = hparams.get("vocab_size", len(tokenizer.vocab)) # ty: ignore[unresolved-attribute] + assert tokenizer is not None + vocab_size = hparams.get("vocab_size", len(tokenizer.vocab)) # Since we are checking the maximum index, we need to ensure it's strictly less than vocab_size, # because vocab_size is the count of items, and indexes start at 0. - max_vocab_index = max(tokenizer.get_vocab().values()) # ty: ignore[unresolved-attribute] + max_vocab_index = max(tokenizer.get_vocab().values()) if max_vocab_index >= vocab_size: raise ValueError("Vocabulary size exceeds expected maximum size.") - reverse_vocab: dict[int, str] = {id_: encoded_tok for encoded_tok, id_ in tokenizer.vocab.items()} # ty: ignore[unresolved-attribute] - added_vocab = tokenizer.get_added_vocab() # ty: ignore[unresolved-attribute] + reverse_vocab: dict[int, str] = {id_: encoded_tok for encoded_tok, id_ in tokenizer.vocab.items()} + added_vocab = tokenizer.get_added_vocab() for token_id in range(vocab_size): token_text = reverse_vocab[token_id].encode('utf-8') @@ -44,7 +45,7 @@ def set_vocab(self): elif re.fullmatch(br"<0x[0-9A-Fa-f]{2}>", token_text): toktype = gguf.TokenType.BYTE # special elif reverse_vocab[token_id] in added_vocab: - if tokenizer.added_tokens_decoder[token_id].special: # ty: ignore[unresolved-attribute] + if tokenizer.added_tokens_decoder[token_id].special: toktype = gguf.TokenType.CONTROL else: toktype = gguf.TokenType.USER_DEFINED diff --git a/convert_hf_to_gguf.py b/convert_hf_to_gguf.py index b863bccc5a24..8cee1f8646e1 100755 --- a/convert_hf_to_gguf.py +++ b/convert_hf_to_gguf.py @@ -125,6 +125,15 @@ def parse_args() -> argparse.Namespace: "--no-nextn", "--no-mtp", dest="no_mtp", action="store_true", help="Exclude NextN speculative draft tensors from the converted GGUF. Pair with --mtp or --dspark on a second run to publish target and draft as two files.", ) + ple_group = parser.add_mutually_exclusive_group() + ple_group.add_argument( + "--ple-only", action="store_true", + help="Export only the per-layer embedding table as a separate GGUF for use with --shard (alias: --ple).", + ) + ple_group.add_argument( + "--no-ple", action="store_true", + help="Exclude the per-layer embedding table while retaining its metadata. Load the table separately with --shard (alias: --ple).", + ) parser.add_argument( "--dspark", action="store_true", help="Export only the DeepSeek-V4 or Mistral Large 4 DSpark draft tensors as a separate GGUF.", @@ -173,6 +182,10 @@ def parse_args() -> argparse.Namespace: args = parser.parse_args() if not args.print_supported_models and args.model is None: parser.error("the following arguments are required: model") + if (args.ple_only or args.no_ple) and (args.vocab_only or args.mmproj or args.mtp or args.dspark): + parser.error("--ple-only / --no-ple cannot be combined with --vocab-only, --mmproj, --mtp, or --dspark") + if args.ple_only and args.no_tensor_first_split: + parser.error("--ple-only cannot be combined with --no-tensor-first-split") return args @@ -308,6 +321,7 @@ def main() -> None: fuse_gate_up_exps=args.fuse_gate_up_exps, fp8_as_q8=args.fp8_as_q8, fuse_qkv=args.fuse_qkv, + ple_only=args.ple_only, no_ple=args.no_ple, ) if args.vocab_only: diff --git a/convert_hf_to_gguf_update.py b/convert_hf_to_gguf_update.py index 37df033ef3d6..4340a9feeaac 100755 --- a/convert_hf_to_gguf_update.py +++ b/convert_hf_to_gguf_update.py @@ -198,6 +198,7 @@ class TOKENIZER_TYPE(IntEnum): # jina-v2-de variants {"name": "jina-v2-de", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/aari1995/German_Semantic_V3", "chkhsh": "b3d1dd861f1d4c5c0d2569ce36baf3f90fe8a102db3de50dd71ff860d91be3df"}, {"name": "gpt-2", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/evilfreelancer/ruGPT3XL", "chkhsh": "0fe1cf6eda062318a1af7270f3331a85c539a01778ff948e24388e949c5282f4"}, + {"name": "gpt-2", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/Lernex/Metis-1.6-Logos", "chkhsh": "df1bc34a6366466bcb36a9a1b1cc0afdb08cfe21c9250602c930f159085390a9"}, # lfm2 variants {"name": "lfm2", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/LiquidAI/LFM2.5-8B-A1B", "chkhsh": "9e454714343b69b99b71795c1d27a68c2a1d15dab111f4d353109f966af29da7"}, # hrm-text (DFM Mimir) is SPM-style BPE: normalizer maps ' ' -> '▁', merges @@ -324,7 +325,8 @@ def get_existing_models(convert_py): except Exception as e: raise OSError(f"Error loading tokenizer for model {name}.") from e - chktok = tokenizer.encode(CHK_TXT) # ty: ignore[unresolved-attribute] + assert tokenizer is not None + chktok = tokenizer.encode(CHK_TXT) chkhsh = sha256(str(chktok).encode()).hexdigest() logger.info(f"model: {name}") @@ -485,6 +487,7 @@ def get_vocab_base_pre(self, tokenizer) -> str: logger.error(f"Failed to load tokenizer for model {name}. Error: {e}") continue # Skip this model and continue with the next one in the loop + assert tokenizer is not None if not os.path.exists(f"models/ggml-vocab-{name}.gguf"): logger.info(f"Skip vocab files for model {name}, no GGUF file found") continue @@ -496,7 +499,7 @@ def get_vocab_base_pre(self, tokenizer) -> str: with open(f"models/ggml-vocab-{name}.gguf.out", "w") as f: for text in tests: - res = tokenizer.encode(text, add_special_tokens=False) # ty: ignore[unresolved-attribute] + res = tokenizer.encode(text, add_special_tokens=False) for r in res: f.write(f" {r}") f.write("\n") diff --git a/convert_lora_to_gguf.py b/convert_lora_to_gguf.py index f9adecbb11d8..a6c9002aa6e7 100755 --- a/convert_lora_to_gguf.py +++ b/convert_lora_to_gguf.py @@ -443,7 +443,8 @@ def set_gguf_parameters(self): # the invocation string includes the "<|start_of_turn|>" # token, but the adapters themselves were trained to # activate _after_ that first token, so we drop it here. - alora_invocation_tokens = tokenizer(invocation_string)["input_ids"][1:] # ty: ignore[call-non-callable] + assert tokenizer is not None + alora_invocation_tokens = tokenizer(invocation_string)["input_ids"][1:] if alora_invocation_tokens: logger.debug("GGUF KV: %s = %s", gguf.Keys.Adapter.ALORA_INVOCATION_TOKENS, alora_invocation_tokens) self.gguf_writer.add_key_value( diff --git a/docs/backend/SYCL.md b/docs/backend/SYCL.md index fb800025d2d3..8649fca0f8a7 100644 --- a/docs/backend/SYCL.md +++ b/docs/backend/SYCL.md @@ -52,6 +52,9 @@ The packages for FP32 and FP16 would have different accuracy and performance on ## News +- 2026.11 + - Update the CI and release build for oneAPI 2026.2 and Level Zero SDK 1.34.0. oneDNN is updated to the 2026.1 stand-alone package. + - 2026.09 - Update the CI build environment for oneAPI 2026.1 (unified oneAPI Toolkit). oneDNN is removed from the Deep Learning Essentials package in 2026.0, so the CI now uses the oneAPI Toolkit installer which still includes oneDNN. - oneAPI 2026.1 improves the SYCL build performance: measured with the same code on Arc B570, prompt processing 1331 vs 434 t/s (3.1x) vs the 2025.3-based release build. @@ -285,6 +288,7 @@ Upon a successful installation, SYCL is enabled for the available Intel devices, |Verified release| |-| +|2026.2 | |2026.1 | |2025.3.3 | |2025.2.1| @@ -817,7 +821,7 @@ User can use the device management in [docs/multi-gpu.md](https://github.com/ggm | GGML_SYCL_ENABLE_VMM | 0 or 1 (default) | Enable the virtual-memory device pool. | | GGML_SYCL_ENABLE_MKL_FA | 1 (default) or 0 | Enable oneMKL GEMM flash attention for XMX-accelerated prompt processing with quantized KV cache. Automatically activates during prefill (prompt processing) when all conditions are met: (1) flash-attn enabled (`-fa` or `--flash-attn on`), (2) KV cache quantized (`--cache-type-k q8_0 --cache-type-v q8_0` or other `*_0/*_1` types), (3) batch size ≥ 1024 (`--batch-size 1024`), (4) prompt length ≥ 1024 tokens. Set to 0 to force the TILE kernel for A/B testing. Example minimum command: `llama-cli -m model.gguf -fa -ngl 99 --cache-type-k q8_0 --cache-type-v q8_0 --batch-size 1024 -p "your prompt"` | | GGML_SYCL_MKL_FA_DEBUG | 0 (default) or 1 | Enable per-call diagnostic logging for MKL flash attention: GEMM/softmax timings, interleaved-head detection, and buffer memory usage. | -| GGML_SYCL_MEMTRACE | 0 (default), 1, 2 | Enable record and output memory allocation diagnostics. Requires `-lv 4`.
0 - Disable
1 - Basic memory info, including current and peak allocations, as well allocations from other sources, around 50 lines per model load.
2 - More verbose, logging around 900 specific allocations and deallocations. | +| GGML_SYCL_MEMTRACE | 0 (default), 1, 2 | Enable record and output memory allocation diagnostics. Requires `-lv 4`.
0 - Disable
1 - Basic memory info, including current and peak allocations, as well allocations from other sources, and the device free memory before each model buffer allocation, around 50 lines per model load.
2 - More verbose, logging around 900 specific allocations and deallocations. | | GGML_SYCL_MEMTRACE_STEP | 64 (default) or positive integer | With GGML_SYCL_MEMTRACE=1, the minimum growth in memory usage to trigger another log record. | | GGML_SYCL_MKL_FA_DIAG | 0 (default) or 1 | Enable output fingerprinting for MKL flash attention. Dumps the first 64 float output values for the first 6 FA calls with n_kv ≥ 1024, labeled with kernel type (MKL/TILE/VEC) for cross-kernel comparison. | | GGML_SYCL_ENABLE_FUSION | 0 or 1 (default) | Enable fused-kernel dispatch in graph compute. Unsupported types and layouts fall back to the standalone op kernels. See `ggml_sycl_can_fuse()`. | @@ -825,7 +829,8 @@ User can use the device management in [docs/multi-gpu.md](https://github.com/ggm | GGML_SYCL_ENABLE_XMX_ESIMD | 0 or 1 (default) | Use the XMX (DPAS) multi-column mat-vec kernels for reordered Q2_K to Q6_K and Q8_0 weights on Xe2 GPUs. Speeds up speculative decoding and short prompt batches. | | GGML_SYCL_XMX_MAX_COLS | non-negative integer, 80 (default) | Most activation columns (batch size) the `GGML_SYCL_ENABLE_XMX_ESIMD` kernels take; wider batches use the other mat-mul kernels. A larger value keeps XMX for bigger batches (as repeated launches), `0` never uses it. | | GGML_SYCL_XMX_GLU_MAX_COLS | 0 to 16, 16 (default) | Most columns the fused gate and up XMX kernel takes; wider batches run gate and up as separate mat-vecs. 16 is the most the kernel is built for, larger values are clamped to 16 and `0` turns the fused kernel off. | -| GGML_SYCL_XMX_GATHER_TYPES | decimal bitmask, all bits set (default) | Weight formats that may use the XMX dequant-GEMM paths, which dequantize weights straight into the XMX tiles. This speeds up prompt processing of MoE models on GPUs with XMX units (Arc A- and B-series, Arc Pro, Data Center GPU Max), for example pp512 of Qwen3-30B-A3B UD-IQ3_XXS by about 50% on an Arc Pro B60. Bits:
* 1: IQ4_NL, 2: IQ3_S, 4: IQ4_XS, 8: IQ3_XXS, 16: IQ2_XXS, 32: IQ2_XS, 64: IQ2_S, 128: IQ1_S, 256: IQ1_M
* 512: Q8_0, 1024: Q4_K, 2048: Q5_K, 4096: Q6_K (MoE `MUL_MAT_ID` only)
Add values to combine them, for example `3` for IQ4_NL and IQ3_S; `0` disables the paths. A set bit does not force the path: batches of more than 64 tokens per expert or row lengths that are not a multiple of 256 (32 for IQ4_NL and Q8_0) use the library GEMM. | +| GGML_SYCL_XMX_GATHER_TYPES | decimal bitmask, all bits set (default) | Weight formats that may use the XMX dequant-GEMM paths, which dequantize weights straight into the XMX tiles. This speeds up prompt processing of MoE models on GPUs with XMX units (Arc A- and B-series, Arc Pro, Data Center GPU Max), for example pp512 of Qwen3-30B-A3B UD-IQ3_XXS by about 50% on an Arc Pro B60. Bits:
* 1: IQ4_NL, 2: IQ3_S, 4: IQ4_XS, 8: IQ3_XXS, 16: IQ2_XXS, 32: IQ2_XS, 64: IQ2_S, 128: IQ1_S, 256: IQ1_M
* 512: Q8_0, 1024: Q4_K, 2048: Q5_K, 4096: Q6_K, 8192: Q2_K, 16384: Q3_K (MoE `MUL_MAT_ID` only)
Add values to combine them, for example `3` for IQ4_NL and IQ3_S; `0` disables the paths. A set bit does not force the path: batches of more than 64 tokens per expert use the library GEMM unless the device schedule width heuristic is overridden with `GGML_SYCL_MMID_SCHED=3`. Row lengths must be a multiple of 256 (32 for IQ4_NL and Q8_0). | +| GGML_SYCL_MMID_SCHED | decimal bitmask, 1 (default) | Build the grouped MoE routing and tile schedule on the GPU, avoiding the expert-ID host readback and CPU sort.
* 1: enable the device schedule.
* 2: override the fallback for models with few routed experts by disabling the performance heuristic that rejects batches averaging more than 64 routed rows per expert. This may reduce performance; type, shape, precision and device capability checks still apply.
Use `3` to enable the device schedule and override this heuristic, or `0` to use the host schedule. Bit 2 alone does not enable the device schedule. Higher bits are ignored. | | GGML_SYCL_XMX_GATHER_SHAPES | decimal bitmask, 255 (default) | XMX `joint_matrix` combinations the paths of `GGML_SYCL_XMX_GATHER_TYPES` may use; the operand type comes from `GGML_SYCL_DYNAMIC_PRECISION` and the best supported combination is picked automatically (logged as `fg_pick_combo`). Bits:
* Xe2, Xe3, Xe-HPC: 1: f16 8x16x16, 2: f16 16x16x16, 4: f16 32x64x16, 8: f16 32x64x32, 32: tf32 8x16x8, 64: bf16 8x16x16
* Xe-HPG (Arc A770, ARL-H): 16: f16 8x8x16, 128: bf16 8x8x16
Clear a bit to exclude a combination, or set a single bit to force one for testing. | | GGML_SYCL_DYNAMIC_PRECISION | `F16` (default with `GGML_SYCL_F16=ON`), `BF16`, `TF32` or `F32` (default otherwise) | Operand type of the XMX dequant-GEMM paths (`GGML_SYCL_XMX_GATHER_TYPES`); accumulation is always f32. `F16` is the fastest, but activations above 65504 overflow. `BF16` keeps the f32 range at a 7-bit mantissa, `TF32` keeps the range and the f16 mantissa but is about 30% slower and needs Xe2, Xe3 or Xe-HPC, and `F32` turns the XMX paths off. Ops that request a higher src1 precision ([TAG_GGML_PREC]) get it regardless of this setting. | | GGML_SYCL_DYNAMIC_REQUIRED_PRECISION | `F32` (default), `TF32`, `BF16` or `F16` | Lowest type the XMX paths may use for an op that requests an F32 src1, such as Mistral 4 `ffn_down_exps`. The default runs such ops on the library f32 GEMM; `TF32` or `BF16` trade mantissa for speed while keeping the f32 range. `F16` ignores the request and can overflow; it is meant for testing only. | @@ -836,6 +841,7 @@ User can use the device management in [docs/multi-gpu.md](https://github.com/ggm | ZES_ENABLE_SYSMAN | 0 (default) or 1 | Support to get free memory of GPU by sycl::aspect::ext_intel_free_memory.
Recommended to use when --split-mode = layer | | UR_L0_ENABLE_RELAXED_ALLOCATION_LIMITS | 0 (default) or 1 | Allow SYCL/Unified Runtime Level Zero device allocations larger than 4 GiB. llama.cpp's direct Level Zero allocation path requests the relaxed maximum-size limit itself when GGML_SYCL_ENABLE_LEVEL_ZERO=1. | | GGML_SYCL_USM_SYSTEM | 0 (default) or 1 | Enable experimental support for [USM system allocations](https://github.khronos.org/SYCL_Reference/iface/usm_basic_concept.html#system-allocations) for large GPU buffers. This requires enough host memory for model weights and caches, an Intel Xe2+ GPU such as BMG or newer and supported on Linux only, with CONFIG_DRM_XE_GPUSVM enabled. | +| GGML_SYCL_BMG_VRAM_CHECK | 1 (default) or 0 | Workaround for BMG (Arc B-series) GPUs, where the Level Zero driver accepts device allocations beyond VRAM and moves the overflow to system RAM, which can exhaust host memory when a model does not fit (e.g. `-ngl 999`, see https://github.com/intel/compute-runtime/issues/1047). When enabled, a model or split buffer that does not fit in the device's free memory is refused and the model load fails with an error. Set to 0 to disable. | 1. SYCL Graph support (`GGML_SYCL_ENABLE_GRAPH=1`) is limited to single-GPU workloads and may also depend on the model architecture and kernels in use. Performance depends on how often a recorded graph can be reused, which will vary by workload. diff --git a/docs/backend/snapdragon/README.md b/docs/backend/snapdragon/README.md index ea4e3f386deb..bd970370591c 100644 --- a/docs/backend/snapdragon/README.md +++ b/docs/backend/snapdragon/README.md @@ -325,10 +325,14 @@ on 4 physical NPUs, or `--devices 'HTP0[0-1:0],HTP1[0-1:1]'` on 2 physical NPUs Controls the number of HVX hardware threads to use. The default is all (actual number varies depending on the hardware version). - `GGML_HEXAGON_VTCM_SIZE=0` - Overrides the VTCM size used by each session. Accepts `K`, `M` and `G` suffixes; a plain number is in MB. The default (0) uses the size the hardware reports. Values above the hardware size will fail to start the session. - -- `GGML_HEXAGON_HOSTBUF=1` (default: 0, disabled) - Enables allocating host buffers for debugging. By default, host buffers are disabled. + Overrides the VTCM size used by each session. Accepts `K`, `M` and `G` suffixes; a plain number is in MB. + The default (0) uses the size the hardware reports. Values above the hardware size will fail to start the session. + +- `GGML_HEXAGON_HOSTBUF=0` (default: 1, enabled) + Controls host buffer (`HTP*-HOST`) allocation in shared FastRPC memory to avoid host-device tensor copies. + CPU-assigned weights (input embeddings like `token_embd` and `per_layer_token_embd`, and tensors overridden with `-ot ...=CPU`) + are placed in host buffers as well. On older drivers/FW, large models can run out of IOVA space when these are mapped. + Pass `--no-host` to keep CPU-assigned weights in regular CPU memory (`-ot ...=CPU` alone is not sufficient). - `GGML_HEXAGON_DMA64=0` (default: enabled on v81+) Disables 64-bit DMA for model weights. Set to `1` to enable it explicitly on a supported architecture. diff --git a/docs/backend/snapdragon/developer.md b/docs/backend/snapdragon/developer.md index 363be41ba9fb..69a8d5f7d318 100644 --- a/docs/backend/snapdragon/developer.md +++ b/docs/backend/snapdragon/developer.md @@ -60,7 +60,7 @@ Implementation: [`htp-repack.cpp`](../../../ggml/src/ggml-hexagon/htp-repack.cpp ### Tile Dimensions and Grid Layout Each tile represents a submatrix of `N_rows = 32` rows and `K_cols` columns: -- Standard types (`Q4_0`, `Q4_1`, `Q8_0`, `IQ4_NL`, `MXFP4`, and K-quants): `K_cols = 32` (`HTP_TENSOR_TILE_N_COLS = 32`). +- Standard types (`Q4_0`, `Q4_1`, `Q5_0`, `Q5_1`, `Q8_0`, `IQ4_NL`, `MXFP4`, and K-quants): `K_cols = 32` (`HTP_TENSOR_TILE_N_COLS = 32`). - `Q2_0`: `K_cols = 64` (`HTP_TENSOR_TILE_K_Q2_0 = 64`). - `Q1_0`: `K_cols = 128` (`HTP_TENSOR_TILE_K_Q1_0 = 128`). @@ -123,6 +123,17 @@ Block-32 formats use a tile size of 32 rows x 32 columns (`K_cols = 32`): - **Scales** (bytes 512..639, 128 bytes): 32 pairs of FP16 values (`ggml_half`): `scale[2 * row + 0] = d` (scale) and `scale[2 * row + 1] = m` (min) for each row `row = 0..31`. +- **Q5_0** (704 bytes per tile, `HTP_TENSOR_TILE_SIZE_Q5_0`): + - **Quants** (bytes 0..511): same 4-bit nibble transposition and storage layout as `Q4_0`, holding the low 4 bits. + - **5th-bit plane** (bytes 512..639): byte `32*c + row` (`c = 0..3`) holds a flag for `k = 2*(4i + c)` at bit `2i` and for `k = 2*(4i + c) + 1` at bit `2i + 1`, for `i = 0..3`. The flag is set when the 5th bit of the quant is clear. + - **Scales** (bytes 640..703, 64 bytes): 32 FP16 values, storing scale `d` for each row `row = 0..31`. + - **Dequantization**: `0xF0` is OR-ed into the nibble where the flag is set, giving the signed value `q - 16` (-16..15), scaled by `d`. + +- **Q5_1** (768 bytes per tile, `HTP_TENSOR_TILE_SIZE_Q5_1`, same layout as the Q5_K tile): + - **Quants** (bytes 0..511) and **scales** (bytes 512..639): same as `Q4_1`, holding the low 4 bits and `(d, m)` pairs. + - **5th-bit plane** (bytes 640..767): same as `Q5_K`; a bit is set when the 5th bit of the quant is set. + - Uses the `Q5_K` HVX/HMX kernels (Q8_1 activations with block sums). + ### Q1_0 Format (1-Bit Quantization) `Q1_0` packs 128 columns per tile (`K_cols = 128`, `HTP_TENSOR_TILE_K_Q1_0 = 128`): @@ -255,7 +266,7 @@ Writing high-performance operators for Hexagon requires following specific guide ### Tracing Instrumentation - All kernels must include trace events for performance profiling and timeline visualization in Perfetto - ([`hex-profile.h`](../../../ggml/src/ggml-hexagon/htp/hex-profile.h)). + ([`htp-trace.h`](../../../ggml/src/ggml-hexagon/htp/htp-trace.h)). - Surround compute sections with `htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) info)` and `htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) info)`. - Use specific event types for major phases: diff --git a/docs/build.md b/docs/build.md index 4e3bfe651b39..e369aad01f49 100644 --- a/docs/build.md +++ b/docs/build.md @@ -608,7 +608,7 @@ You can test with: If the following info is output on screen, you are using `llama.cpp` with the CANN backend: ```bash llm_load_tensors: CANN model buffer size = 13313.00 MiB -llama_new_context_with_model: CANN compute buffer size = 1260.81 MiB +llama_init_from_model: CANN compute buffer size = 1260.81 MiB ``` For detailed info, such as model/device supports, CANN install, please refer to [llama.cpp for CANN](./backend/CANN.md). diff --git a/docs/development/HOWTO-add-model.md b/docs/development/HOWTO-add-model.md index abf6503f239b..558363f4ea68 100644 --- a/docs/development/HOWTO-add-model.md +++ b/docs/development/HOWTO-add-model.md @@ -128,7 +128,7 @@ Note: to debug the inference graph: you can use [llama-eval-callback](/examples/ If the new model supports multimodal inputs, you will need to add a new encoder definition in `libmtmd`. You can find more information about llama.cpp's multimodal support in [the docs](../multimodal.md) and in the `tools/mtmd` source directory. 1. In the conversion script, make sure you add a subclass that extends `MmprojModel` or another class that inherits from the same base class. -2. Add the encoder definition in `clip.cpp`. +2. Add the encoder definition: hparams and tensor loading in `mtmd-model.cpp`, graph inputs in `mtmd-subcontext.cpp`. 3. Implement the preprocessor in `mtmd.cpp`. In most cases, you can reuse an existing preprocessor. 4. Implement the encoder GGML graph, either in a dedicated file if the model is truly different from existing ones, or by reusing an existing implementation (for example: siglip, pixtral, or qwen) and adding a model-specific projector. diff --git a/docs/models.md b/docs/models.md index eee5952634be..f466d76e50f4 100644 --- a/docs/models.md +++ b/docs/models.md @@ -24,3 +24,23 @@ The Hugging Face platform provides a variety of online tools for converting, qua - Use the [GGUF-my-LoRA space](https://huggingface.co/spaces/ggml-org/gguf-my-lora) to convert LoRA adapters to GGUF format (more info: https://github.com/ggml-org/llama.cpp/discussions/10123) - Use the [GGUF-editor space](https://huggingface.co/spaces/CISCai/gguf-editor) to edit GGUF meta data in the browser (more info: https://github.com/ggml-org/llama.cpp/discussions/9268) - Use the [Inference Endpoints](https://ui.endpoints.huggingface.co/) to directly host `llama.cpp` in the cloud (more info: https://github.com/ggml-org/llama.cpp/discussions/9669) + +## Exporting per-layer embeddings separately + +For models with a per-layer embedding (PLE) table, export the backbone and table from the same checkpoint: + +```sh +python3 convert_hf_to_gguf.py /path/to/model --no-ple --outfile model-no-ple.gguf +python3 convert_hf_to_gguf.py /path/to/model --ple-only --outfile model-ple.gguf +llama-cli -m model-no-ple.gguf --shard model-ple.gguf +``` + +`--no-ple` excludes only `per_layer_token_embd.weight` and keeps the PLE metadata and projection weights. `--ple-only` exports only that table, without a tokenizer. Each export supports `--outtype`, so the table and backbone can use different precisions. Default output names start with `no-ple-` and `ple-` respectively. + +These modes are mutually exclusive and require a model with a PLE table, such as Qwen3.8 Flash Next or Gemma 4. They cannot be combined with vocabulary, multimodal projector, MTP-only, or DSpark-only export. `--no-ple` can be combined with `--no-mtp`. + +`--ple` is an alias for `--shard`. Extra shards can be comma separated or supplied with repeated options, for example `-m backbone.gguf --shard /fast/ple.gguf,/other/token-embd.gguf`. Paths may be on different drives. Tensor names must be unique across all files, except for the PLE case below. + +A numbered backbone is auto-discovered and checked as usual; explicitly supplied paths outside that numbered set are appended as extra shards. Their split numbering is ignored, and the loader uses the actual combined tensor count. Paths already in the discovered set are loaded once. If the main filename does not follow the numbered naming scheme, an explicit backbone list must still match its declared split count and order. + +If an extra shard holds only `per_layer_token_embd.weight`, it replaces the PLE table of the model. If the model already has the table, the extra shard overrides it. A backbone split that holds only that tensor is skipped, and it can be absent from disk. This lets one PLE file be shared across quants and releases, for example `-m model-Q4_K_M-00001-of-00003.gguf --shard /fast/ple-Q8_0.gguf`. diff --git a/docs/moe-cache.md b/docs/moe-cache.md new file mode 100644 index 000000000000..7491a2a31d0a --- /dev/null +++ b/docs/moe-cache.md @@ -0,0 +1,26 @@ +# MoE cache + +When the experts of MoE layers stay in host memory (`-cmoe`, `-ncmoe`, `-ot`), the MoE cache keeps the most used ones in device memory for small batches (decode, speculative decoding). Large batches still upload the experts they use. + +| Argument | | +|---|---| +| `--moe-cache-mib N` | cache size in MiB, split among the GPUs like the layers (default: 0, disabled) | +| `--moe-cache-cpu-share F` | share of the cache misses computed on the CPU from the host experts instead of uploaded, once the cache is full (default: 0) | + +The cache hits run on the device. With `--moe-cache-cpu-share F` the CPU computes a share `F` of the misses while the device computes the hits and uploads the other misses into the cache. This is faster when the CPU reads the host experts faster than the device can upload them, for example with fast RAM, a PCIe 4 link or experts that are not in pinned memory. The host experts are pinned with `-lm none`, `-lm mlock` and `-lm dio`, unless `--no-host`, but not when they are mmap'd (the default). The device uploads pinned experts at the full speed of the link, so the CPU share often does not help then. Until the cache is full every miss is uploaded, so the cache fills as fast as without the CPU share. The best share depends on the machine, the model and the cache size. + +## Picking the arguments + +Put the script in front of your server command: + +```bash +python3 scripts/moe-cache-tune.py -- build/bin/llama-server -m model.gguf -cmoe -t 8 -c 32768 +``` + +It starts the server 6 times, at two cache sizes × shares 0, 0.5 and 0.95, and times the generation of a few thousand tokens. That takes a few minutes, mostly model loading. + +From these runs it fits the upload and CPU speeds of the machine, then replays the routing of all the runs through a copy of the cache logic for other sizes and shares. It prints the measured speeds, the predicted t/s and the recommended arguments: the lowest share within 5% of the fastest. + +- **Server command:** keep your own arguments (`-t`, `-c`, expert placement, load mode); the speeds and the free device memory depend on them. +- **`--max-mib N`:** tunes a smaller cache than what fits on the device. The runs in `--out` are reused while the command stays the same, so trying another value is instant. +- **`--markdown`:** prints Markdown tables to paste in a PR. diff --git a/docs/ops.md b/docs/ops.md index 2885441c5a37..3c2bcd239bb1 100644 --- a/docs/ops.md +++ b/docs/ops.md @@ -19,7 +19,7 @@ Legend: | ADD | ❌ | ✅ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ | | ADD1 | ❌ | ✅ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | | ADD_ID | ❌ | ❌ | ✅ | ✅ | ❌ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | -| ARANGE | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ | ✅ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ | +| ARANGE | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ | | ARGMAX | ❌ | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | | ARGSORT | ❌ | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ | | CEIL | ❌ | ❌ | ✅ | 🟡 | 🟡 | 🟡 | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ | @@ -34,13 +34,12 @@ Legend: | CONV_TRANSPOSE_1D | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ | ✅ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ | | CONV_TRANSPOSE_2D | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ✅ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ | | COS | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ | -| COUNT_EQUAL | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ | ✅ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ | +| COUNT_EQUAL | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ | | CPY | ❌ | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | ❌ | ❌ | | CROSS_ENTROPY_LOSS | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ | | CROSS_ENTROPY_LOSS_BACK | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ | | CUMSUM | ❌ | ❌ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | | DIAG | ❌ | ❌ | ✅ | ✅ | 🟡 | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | -| DIAG_MASK_INF | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ | 🟡 | ✅ | ✅ | ❌ | ❌ | ❌ | | DIV | ❌ | ✅ | ✅ | ✅ | ❌ | 🟡 | 🟡 | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ | | DSV4_HC_COMB | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | | DSV4_HC_POST | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | @@ -62,7 +61,7 @@ Legend: | GELU_QUICK | ❌ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ | | GET_ROWS | ❌ | 🟡 | ✅ | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | ✅ | 🟡 | ❌ | ❌ | | GET_ROWS_BACK | ❌ | ❌ | 🟡 | 🟡 | ❌ | ❌ | ❌ | ❌ | 🟡 | 🟡 | ❌ | ❌ | ❌ | -| GROUP_NORM | ❌ | ✅ | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ | +| GROUP_NORM | ❌ | ✅ | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | | HARDSIGMOID | ❌ | ✅ | ✅ | 🟡 | 🟡 | ❌ | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ | | HARDSWISH | ❌ | ✅ | ✅ | 🟡 | 🟡 | ❌ | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ | | IM2COL | ❌ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | @@ -71,7 +70,7 @@ Legend: | LEAKY_RELU | ❌ | ✅ | ✅ | ✅ | ❌ | 🟡 | 🟡 | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ | | LIGHTNING_INDEXER | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | 🟡 | ✅ | ❌ | ❌ | ❌ | ❌ | | LOG | ❌ | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ | -| MEAN | ❌ | ✅ | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | 🟡 | ✅ | ❌ | ❌ | ❌ | +| MEAN | ❌ | ✅ | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | 🟡 | ✅ | ✅ | ❌ | ❌ | | MUL | ❌ | ✅ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ | | MUL_MAT | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | | MUL_MAT_HADAMARD | ❌ | ❌ | ✅ | ❌ | ✅ | ✅ | ❌ | ✅ | ✅ | ✅ | ✅ | ❌ | 🟡 | @@ -86,12 +85,12 @@ Legend: | OPT_STEP_SGD | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ✅ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ | | OUT_PROD | 🟡 | 🟡 | 🟡 | 🟡 | ❌ | ❌ | ❌ | ❌ | 🟡 | 🟡 | ❌ | ❌ | ❌ | | PAD | ❌ | 🟡 | ✅ | 🟡 | ❌ | ✅ | 🟡 | 🟡 | 🟡 | ✅ | ✅ | ❌ | ❌ | -| PAD_REFLECT_1D | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ | ✅ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | +| PAD_REFLECT_1D | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ | ✅ | ❌ | ✅ | ❌ | ✅ | ❌ | ❌ | | POOL_1D | ❌ | ❌ | ✅ | ❌ | ❌ | ✅ | ✅ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ | | POOL_2D | ❌ | 🟡 | ✅ | ✅ | ❌ | ✅ | ✅ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ | | REGLU | ❌ | ✅ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ | | RELU | ❌ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ | -| REPEAT | ❌ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | ✅ | 🟡 | ✅ | ✅ | 🟡 | ❌ | ❌ | +| REPEAT | ❌ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | ✅ | ✅ | ✅ | ✅ | 🟡 | ❌ | ❌ | | REPEAT_BACK | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ | | RMS_NORM | ❌ | ✅ | ✅ | ✅ | 🟡 | ✅ | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ | | RMS_NORM_BACK | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ | diff --git a/docs/ops/BLAS.csv b/docs/ops/BLAS.csv index 3af72452aadb..eabdd6f8118b 100644 --- a/docs/ops/BLAS.csv +++ b/docs/ops/BLAS.csv @@ -8741,9 +8741,6 @@ "BLAS","ROUND","type=f32,ne=[1024,1024,1,1]","support","0","no","BLAS" "BLAS","TRUNC","type=f32,ne=[7,1,5,3]","support","0","no","BLAS" "BLAS","TRUNC","type=f32,ne=[1024,1024,1,1]","support","0","no","BLAS" -"BLAS","DIAG_MASK_INF","type=f32,ne=[10,10,1,1],n_past=5","support","0","no","BLAS" -"BLAS","DIAG_MASK_INF","type=f32,ne=[10,10,3,1],n_past=5","support","0","no","BLAS" -"BLAS","DIAG_MASK_INF","type=f32,ne=[10,10,3,2],n_past=5","support","0","no","BLAS" "BLAS","SOFT_MAX","type=f32,ne=[16,16,1,1],mask=0,sinks=0,m_prec=f32,nr23=[1,1],scale=1.000000,max_bias=0.000000,inplace=0","support","0","no","BLAS" "BLAS","SOFT_MAX","type=f32,ne=[15,15,1,1],mask=0,sinks=0,m_prec=f32,nr23=[1,1],scale=1.000000,max_bias=0.000000,inplace=0","support","0","no","BLAS" "BLAS","SOFT_MAX","type=f32,ne=[16,1024,1,1],mask=0,sinks=0,m_prec=f32,nr23=[1,1],scale=1.000000,max_bias=0.000000,inplace=0","support","0","no","BLAS" diff --git a/docs/ops/CANN.csv b/docs/ops/CANN.csv index 8434461a5f7e..de317ad1588b 100644 --- a/docs/ops/CANN.csv +++ b/docs/ops/CANN.csv @@ -8764,9 +8764,6 @@ "CANN0","ROUND","type=f32,ne=[1024,1024,1,1]","support","0","no","CANN" "CANN0","TRUNC","type=f32,ne=[7,1,5,3]","support","0","no","CANN" "CANN0","TRUNC","type=f32,ne=[1024,1024,1,1]","support","0","no","CANN" -"CANN0","DIAG_MASK_INF","type=f32,ne=[10,10,1,1],n_past=5","support","1","yes","CANN" -"CANN0","DIAG_MASK_INF","type=f32,ne=[10,10,3,1],n_past=5","support","1","yes","CANN" -"CANN0","DIAG_MASK_INF","type=f32,ne=[10,10,3,2],n_past=5","support","1","yes","CANN" "CANN0","SOFT_MAX","type=f32,ne=[16,16,1,1],mask=0,sinks=0,m_prec=f32,nr23=[1,1],scale=1.000000,max_bias=0.000000,inplace=0","support","1","yes","CANN" "CANN0","SOFT_MAX","type=f32,ne=[15,15,1,1],mask=0,sinks=0,m_prec=f32,nr23=[1,1],scale=1.000000,max_bias=0.000000,inplace=0","support","1","yes","CANN" "CANN0","SOFT_MAX","type=f32,ne=[16,1024,1,1],mask=0,sinks=0,m_prec=f32,nr23=[1,1],scale=1.000000,max_bias=0.000000,inplace=0","support","1","yes","CANN" diff --git a/docs/ops/CPU.csv b/docs/ops/CPU.csv index 5d491fc4bf5d..49803f22700a 100644 --- a/docs/ops/CPU.csv +++ b/docs/ops/CPU.csv @@ -12962,9 +12962,6 @@ "CPU","ROUND","type=f32,ne=[1024,1024,1,1]","support","1","yes","CPU" "CPU","TRUNC","type=f32,ne=[7,1,5,3]","support","1","yes","CPU" "CPU","TRUNC","type=f32,ne=[1024,1024,1,1]","support","1","yes","CPU" -"CPU","DIAG_MASK_INF","type=f32,ne=[10,10,1,1],n_past=5","support","1","yes","CPU" -"CPU","DIAG_MASK_INF","type=f32,ne=[10,10,3,1],n_past=5","support","1","yes","CPU" -"CPU","DIAG_MASK_INF","type=f32,ne=[10,10,3,2],n_past=5","support","1","yes","CPU" "CPU","SOFT_MAX","type=f32,ne=[16,16,1,1],mask=0,sinks=0,m_prec=f32,nr23=[1,1],scale=1.000000,max_bias=0.000000,inplace=0","support","1","yes","CPU" "CPU","SOFT_MAX","type=f32,ne=[15,15,1,1],mask=0,sinks=0,m_prec=f32,nr23=[1,1],scale=1.000000,max_bias=0.000000,inplace=0","support","1","yes","CPU" "CPU","SOFT_MAX","type=f32,ne=[16,1024,1,1],mask=0,sinks=0,m_prec=f32,nr23=[1,1],scale=1.000000,max_bias=0.000000,inplace=0","support","1","yes","CPU" diff --git a/docs/ops/CUDA.csv b/docs/ops/CUDA.csv index 7499a492767b..2b8c8b7a8ef8 100644 --- a/docs/ops/CUDA.csv +++ b/docs/ops/CUDA.csv @@ -8725,9 +8725,6 @@ "CUDA0","ROUND","type=f32,ne=[1024,1024,1,1]","support","1","yes","CUDA" "CUDA0","TRUNC","type=f32,ne=[7,1,5,3]","support","1","yes","CUDA" "CUDA0","TRUNC","type=f32,ne=[1024,1024,1,1]","support","1","yes","CUDA" -"CUDA0","DIAG_MASK_INF","type=f32,ne=[10,10,1,1],n_past=5","support","1","yes","CUDA" -"CUDA0","DIAG_MASK_INF","type=f32,ne=[10,10,3,1],n_past=5","support","1","yes","CUDA" -"CUDA0","DIAG_MASK_INF","type=f32,ne=[10,10,3,2],n_past=5","support","1","yes","CUDA" "CUDA0","SOFT_MAX","type=f32,ne=[16,16,1,1],mask=0,sinks=0,m_prec=f32,nr23=[1,1],scale=1.000000,max_bias=0.000000,inplace=0","support","1","yes","CUDA" "CUDA0","SOFT_MAX","type=f32,ne=[15,15,1,1],mask=0,sinks=0,m_prec=f32,nr23=[1,1],scale=1.000000,max_bias=0.000000,inplace=0","support","1","yes","CUDA" "CUDA0","SOFT_MAX","type=f32,ne=[16,1024,1,1],mask=0,sinks=0,m_prec=f32,nr23=[1,1],scale=1.000000,max_bias=0.000000,inplace=0","support","1","yes","CUDA" diff --git a/docs/ops/ET.csv b/docs/ops/ET.csv index 91774c58e84b..56a8f970596a 100644 --- a/docs/ops/ET.csv +++ b/docs/ops/ET.csv @@ -9637,9 +9637,6 @@ "ET","ROUND","type=f32,ne=[1024,1024,1,1]","support","1","yes","ET" "ET","TRUNC","type=f32,ne=[7,1,5,3]","support","0","no","ET" "ET","TRUNC","type=f32,ne=[1024,1024,1,1]","support","1","yes","ET" -"ET","DIAG_MASK_INF","type=f32,ne=[10,10,1,1],n_past=5","support","0","no","ET" -"ET","DIAG_MASK_INF","type=f32,ne=[10,10,3,1],n_past=5","support","0","no","ET" -"ET","DIAG_MASK_INF","type=f32,ne=[10,10,3,2],n_past=5","support","0","no","ET" "ET","SOFT_MAX","type=f32,ne=[16,16,1,1],mask=0,sinks=0,m_prec=f32,nr23=[1,1],scale=1.000000,max_bias=0.000000,inplace=0","support","1","yes","ET" "ET","SOFT_MAX","type=f32,ne=[15,15,1,1],mask=0,sinks=0,m_prec=f32,nr23=[1,1],scale=1.000000,max_bias=0.000000,inplace=0","support","1","yes","ET" "ET","SOFT_MAX","type=f32,ne=[16,1024,1,1],mask=0,sinks=0,m_prec=f32,nr23=[1,1],scale=1.000000,max_bias=0.000000,inplace=0","support","1","yes","ET" diff --git a/docs/ops/Hexagon.csv b/docs/ops/Hexagon.csv index c9608b715dd7..160c414992d6 100644 --- a/docs/ops/Hexagon.csv +++ b/docs/ops/Hexagon.csv @@ -14564,9 +14564,6 @@ "HTP0","ROUND","type=f32,ne=[1024,1024,1,1]","support","0","no","HTP" "HTP0","TRUNC","type=f32,ne=[7,1,5,3]","support","0","no","HTP" "HTP0","TRUNC","type=f32,ne=[1024,1024,1,1]","support","0","no","HTP" -"HTP0","DIAG_MASK_INF","type=f32,ne=[10,10,1,1],n_past=5","support","0","no","HTP" -"HTP0","DIAG_MASK_INF","type=f32,ne=[10,10,3,1],n_past=5","support","0","no","HTP" -"HTP0","DIAG_MASK_INF","type=f32,ne=[10,10,3,2],n_past=5","support","0","no","HTP" "HTP0","SOFT_MAX","type=f32,ne=[16,16,1,1],mask=0,sinks=0,m_prec=f32,nr23=[1,1],scale=1.000000,max_bias=0.000000,inplace=0","support","1","yes","HTP" "HTP0","SOFT_MAX","type=f32,ne=[15,15,1,1],mask=0,sinks=0,m_prec=f32,nr23=[1,1],scale=1.000000,max_bias=0.000000,inplace=0","support","1","yes","HTP" "HTP0","SOFT_MAX","type=f32,ne=[16,1024,1,1],mask=0,sinks=0,m_prec=f32,nr23=[1,1],scale=1.000000,max_bias=0.000000,inplace=0","support","1","yes","HTP" diff --git a/docs/ops/Metal.csv b/docs/ops/Metal.csv index ec34bb66b899..8d38300a0ced 100644 --- a/docs/ops/Metal.csv +++ b/docs/ops/Metal.csv @@ -9628,9 +9628,6 @@ "MTL0","ROUND","type=f32,ne=[1024,1024,1,1]","support","1","yes","MTL" "MTL0","TRUNC","type=f32,ne=[7,1,5,3]","support","1","yes","MTL" "MTL0","TRUNC","type=f32,ne=[1024,1024,1,1]","support","1","yes","MTL" -"MTL0","DIAG_MASK_INF","type=f32,ne=[10,10,1,1],n_past=5","support","0","no","MTL" -"MTL0","DIAG_MASK_INF","type=f32,ne=[10,10,3,1],n_past=5","support","0","no","MTL" -"MTL0","DIAG_MASK_INF","type=f32,ne=[10,10,3,2],n_past=5","support","0","no","MTL" "MTL0","SOFT_MAX","type=f32,ne=[16,16,1,1],mask=0,sinks=0,m_prec=f32,nr23=[1,1],scale=1.000000,max_bias=0.000000,inplace=0","support","1","yes","MTL" "MTL0","SOFT_MAX","type=f32,ne=[15,15,1,1],mask=0,sinks=0,m_prec=f32,nr23=[1,1],scale=1.000000,max_bias=0.000000,inplace=0","support","1","yes","MTL" "MTL0","SOFT_MAX","type=f32,ne=[16,1024,1,1],mask=0,sinks=0,m_prec=f32,nr23=[1,1],scale=1.000000,max_bias=0.000000,inplace=0","support","1","yes","MTL" diff --git a/docs/ops/OpenCL.csv b/docs/ops/OpenCL.csv index db66c5655b46..8b595e4b9d89 100644 --- a/docs/ops/OpenCL.csv +++ b/docs/ops/OpenCL.csv @@ -7682,19 +7682,19 @@ "OpenCL","REPEAT","type=f32,ne=[10,5,4,1],nr=[1,2,1,1]","support","1","yes","OpenCL" "OpenCL","REPEAT","type=f32,ne=[10,5,4,1],nr=[1,1,2,1]","support","1","yes","OpenCL" "OpenCL","REPEAT","type=f32,ne=[10,5,4,1],nr=[1,1,1,2]","support","1","yes","OpenCL" -"OpenCL","REPEAT","type=f16,ne=[10,5,4,1],nr=[2,1,1,1]","support","0","no","OpenCL" -"OpenCL","REPEAT","type=i32,ne=[10,5,4,1],nr=[2,1,1,1]","support","0","no","OpenCL" -"OpenCL","REPEAT","type=i16,ne=[10,5,4,1],nr=[1,1,1,2]","support","0","no","OpenCL" -"OpenCL","REPEAT","type=bf16,ne=[10,5,4,1],nr=[2,1,1,1]","support","0","no","OpenCL" +"OpenCL","REPEAT","type=f16,ne=[10,5,4,1],nr=[2,1,1,1]","support","1","yes","OpenCL" +"OpenCL","REPEAT","type=i32,ne=[10,5,4,1],nr=[2,1,1,1]","support","1","yes","OpenCL" +"OpenCL","REPEAT","type=i16,ne=[10,5,4,1],nr=[1,1,1,2]","support","1","yes","OpenCL" +"OpenCL","REPEAT","type=bf16,ne=[10,5,4,1],nr=[2,1,1,1]","support","1","yes","OpenCL" "OpenCL","REPEAT","type=f32,ne=[10,5,4,3],nr=[1,1,1,1]","support","1","yes","OpenCL" "OpenCL","REPEAT","type=f32,ne=[10,5,4,3],nr=[2,1,1,1]","support","1","yes","OpenCL" "OpenCL","REPEAT","type=f32,ne=[10,5,4,3],nr=[1,2,1,1]","support","1","yes","OpenCL" "OpenCL","REPEAT","type=f32,ne=[10,5,4,3],nr=[1,1,2,1]","support","1","yes","OpenCL" "OpenCL","REPEAT","type=f32,ne=[10,5,4,3],nr=[1,1,1,2]","support","1","yes","OpenCL" -"OpenCL","REPEAT","type=f16,ne=[10,5,4,3],nr=[2,1,1,1]","support","0","no","OpenCL" -"OpenCL","REPEAT","type=i32,ne=[10,5,4,3],nr=[2,1,1,1]","support","0","no","OpenCL" -"OpenCL","REPEAT","type=i16,ne=[10,5,4,3],nr=[1,1,1,2]","support","0","no","OpenCL" -"OpenCL","REPEAT","type=bf16,ne=[10,5,4,3],nr=[2,1,1,1]","support","0","no","OpenCL" +"OpenCL","REPEAT","type=f16,ne=[10,5,4,3],nr=[2,1,1,1]","support","1","yes","OpenCL" +"OpenCL","REPEAT","type=i32,ne=[10,5,4,3],nr=[2,1,1,1]","support","1","yes","OpenCL" +"OpenCL","REPEAT","type=i16,ne=[10,5,4,3],nr=[1,1,1,2]","support","1","yes","OpenCL" +"OpenCL","REPEAT","type=bf16,ne=[10,5,4,3],nr=[2,1,1,1]","support","1","yes","OpenCL" "OpenCL","REPEAT_BACK","type=f32,ne=[8,6,4,2],nr=[1,1,1,1],v=0","support","0","no","OpenCL" "OpenCL","REPEAT_BACK","type=f32,ne=[8,6,4,2],nr=[2,1,1,1],v=0","support","0","no","OpenCL" "OpenCL","REPEAT_BACK","type=f32,ne=[8,6,4,2],nr=[1,2,1,1],v=0","support","0","no","OpenCL" @@ -14887,9 +14887,6 @@ "OpenCL","ROUND","type=f32,ne=[1024,1024,1,1]","support","1","yes","OpenCL" "OpenCL","TRUNC","type=f32,ne=[7,1,5,3]","support","1","yes","OpenCL" "OpenCL","TRUNC","type=f32,ne=[1024,1024,1,1]","support","1","yes","OpenCL" -"OpenCL","DIAG_MASK_INF","type=f32,ne=[10,10,1,1],n_past=5","support","1","yes","OpenCL" -"OpenCL","DIAG_MASK_INF","type=f32,ne=[10,10,3,1],n_past=5","support","1","yes","OpenCL" -"OpenCL","DIAG_MASK_INF","type=f32,ne=[10,10,3,2],n_past=5","support","0","no","OpenCL" "OpenCL","SOFT_MAX","type=f32,ne=[16,16,1,1],mask=0,sinks=0,m_prec=f32,nr23=[1,1],scale=1.000000,max_bias=0.000000,inplace=0","support","1","yes","OpenCL" "OpenCL","SOFT_MAX","type=f32,ne=[15,15,1,1],mask=0,sinks=0,m_prec=f32,nr23=[1,1],scale=1.000000,max_bias=0.000000,inplace=0","support","1","yes","OpenCL" "OpenCL","SOFT_MAX","type=f32,ne=[16,1024,1,1],mask=0,sinks=0,m_prec=f32,nr23=[1,1],scale=1.000000,max_bias=0.000000,inplace=0","support","1","yes","OpenCL" diff --git a/docs/ops/SYCL.csv b/docs/ops/SYCL.csv index 3ef1ca1d7718..e01a536a7284 100644 --- a/docs/ops/SYCL.csv +++ b/docs/ops/SYCL.csv @@ -14534,9 +14534,6 @@ "SYCL0","ROUND","type=f32,ne=[1024,1024,1,1]","support","1","yes","SYCL" "SYCL0","TRUNC","type=f32,ne=[7,1,5,3]","support","1","yes","SYCL" "SYCL0","TRUNC","type=f32,ne=[1024,1024,1,1]","support","1","yes","SYCL" -"SYCL0","DIAG_MASK_INF","type=f32,ne=[10,10,1,1],n_past=5","support","1","yes","SYCL" -"SYCL0","DIAG_MASK_INF","type=f32,ne=[10,10,3,1],n_past=5","support","1","yes","SYCL" -"SYCL0","DIAG_MASK_INF","type=f32,ne=[10,10,3,2],n_past=5","support","1","yes","SYCL" "SYCL0","SOFT_MAX","type=f32,ne=[16,16,1,1],mask=0,sinks=0,m_prec=f32,nr23=[1,1],scale=1.000000,max_bias=0.000000,inplace=0","support","1","yes","SYCL" "SYCL0","SOFT_MAX","type=f32,ne=[15,15,1,1],mask=0,sinks=0,m_prec=f32,nr23=[1,1],scale=1.000000,max_bias=0.000000,inplace=0","support","1","yes","SYCL" "SYCL0","SOFT_MAX","type=f32,ne=[16,1024,1,1],mask=0,sinks=0,m_prec=f32,nr23=[1,1],scale=1.000000,max_bias=0.000000,inplace=0","support","1","yes","SYCL" diff --git a/docs/ops/Vulkan.csv b/docs/ops/Vulkan.csv index 13e67eb797ff..65ac8bf33984 100644 --- a/docs/ops/Vulkan.csv +++ b/docs/ops/Vulkan.csv @@ -12313,9 +12313,6 @@ "Vulkan0","ROUND","type=f32,ne=[1024,1024,1,1]","support","1","yes","Vulkan" "Vulkan0","TRUNC","type=f32,ne=[7,1,5,3]","support","1","yes","Vulkan" "Vulkan0","TRUNC","type=f32,ne=[1024,1024,1,1]","support","1","yes","Vulkan" -"Vulkan0","DIAG_MASK_INF","type=f32,ne=[10,10,1,1],n_past=5","support","1","yes","Vulkan" -"Vulkan0","DIAG_MASK_INF","type=f32,ne=[10,10,3,1],n_past=5","support","1","yes","Vulkan" -"Vulkan0","DIAG_MASK_INF","type=f32,ne=[10,10,3,2],n_past=5","support","1","yes","Vulkan" "Vulkan0","SOFT_MAX","type=f32,ne=[16,16,1,1],mask=0,sinks=0,m_prec=f32,nr23=[1,1],scale=1.000000,max_bias=0.000000,inplace=0","support","1","yes","Vulkan" "Vulkan0","SOFT_MAX","type=f32,ne=[15,15,1,1],mask=0,sinks=0,m_prec=f32,nr23=[1,1],scale=1.000000,max_bias=0.000000,inplace=0","support","1","yes","Vulkan" "Vulkan0","SOFT_MAX","type=f32,ne=[16,1024,1,1],mask=0,sinks=0,m_prec=f32,nr23=[1,1],scale=1.000000,max_bias=0.000000,inplace=0","support","1","yes","Vulkan" diff --git a/docs/ops/WebGPU.csv b/docs/ops/WebGPU.csv index c19396c03e4d..3b874fd68e59 100644 --- a/docs/ops/WebGPU.csv +++ b/docs/ops/WebGPU.csv @@ -5465,8 +5465,10 @@ "WebGPU: WebGPU","CONV_TRANSPOSE_2D","kernel_type=f16,ne_input=[3,2,3,1],ne_kernel=[2,2,1,3],stride=1","support","0","no","WebGPU" "WebGPU: WebGPU","CONV_TRANSPOSE_2D","kernel_type=f16,ne_input=[10,10,9,1],ne_kernel=[3,3,1,9],stride=2","support","0","no","WebGPU" "WebGPU: WebGPU","CONV_TRANSPOSE_2D","kernel_type=f16,ne_input=[129,63,35,1],ne_kernel=[3,3,48,35],stride=1","support","0","no","WebGPU" -"WebGPU: WebGPU","COUNT_EQUAL","type=f32,ne=[4,500,1,1]","support","0","no","WebGPU" -"WebGPU: WebGPU","COUNT_EQUAL","type=f32,ne=[4,5000,1,1]","support","0","no","WebGPU" +"WebGPU: WebGPU","COUNT_EQUAL","type=f32,ne=[4,500,1,1],same_src=0","support","1","yes","WebGPU" +"WebGPU: WebGPU","COUNT_EQUAL","type=f32,ne=[4,5000,1,1],same_src=0","support","1","yes","WebGPU" +"WebGPU: WebGPU","COUNT_EQUAL","type=f32,ne=[4,500,1,1],same_src=1","support","1","yes","WebGPU" +"WebGPU: WebGPU","COUNT_EQUAL","type=f32,ne=[4,5000,1,1],same_src=1","support","1","yes","WebGPU" "WebGPU: WebGPU","ARGMAX","type=f32,ne=[32,1,1,1]","support","1","yes","WebGPU" "WebGPU: WebGPU","ARGMAX","type=f32,ne=[32,513,1,1]","support","1","yes","WebGPU" "WebGPU: WebGPU","ARGMAX","type=f32,ne=[100,10,1,1]","support","1","yes","WebGPU" @@ -10558,9 +10560,6 @@ "WebGPU: WebGPU","ROUND","type=f32,ne=[1024,1024,1,1]","support","1","yes","WebGPU" "WebGPU: WebGPU","TRUNC","type=f32,ne=[7,1,5,3]","support","1","yes","WebGPU" "WebGPU: WebGPU","TRUNC","type=f32,ne=[1024,1024,1,1]","support","1","yes","WebGPU" -"WebGPU: WebGPU","DIAG_MASK_INF","type=f32,ne=[10,10,1,1],n_past=5","support","0","no","WebGPU" -"WebGPU: WebGPU","DIAG_MASK_INF","type=f32,ne=[10,10,3,1],n_past=5","support","0","no","WebGPU" -"WebGPU: WebGPU","DIAG_MASK_INF","type=f32,ne=[10,10,3,2],n_past=5","support","0","no","WebGPU" "WebGPU: WebGPU","SOFT_MAX","type=f32,ne=[16,16,1,1],mask=0,sinks=0,m_prec=f32,nr23=[1,1],scale=1.000000,max_bias=0.000000,inplace=0","support","1","yes","WebGPU" "WebGPU: WebGPU","SOFT_MAX","type=f32,ne=[15,15,1,1],mask=0,sinks=0,m_prec=f32,nr23=[1,1],scale=1.000000,max_bias=0.000000,inplace=0","support","1","yes","WebGPU" "WebGPU: WebGPU","SOFT_MAX","type=f32,ne=[16,1024,1,1],mask=0,sinks=0,m_prec=f32,nr23=[1,1],scale=1.000000,max_bias=0.000000,inplace=0","support","1","yes","WebGPU" @@ -12278,13 +12277,23 @@ "WebGPU: WebGPU","SUM","type=f32,ne=[11,5,6,3],permute=[0,2,1,3]","support","1","yes","WebGPU" "WebGPU: WebGPU","SUM","type=f32,ne=[11,5,6,3],permute=[0,3,2,1]","support","1","yes","WebGPU" "WebGPU: WebGPU","SUM","type=f32,ne=[11,5,6,3],permute=[0,1,3,2]","support","1","yes","WebGPU" -"WebGPU: WebGPU","MEAN","type=f32,ne=[10,5,4,3]","support","0","no","WebGPU" -"WebGPU: WebGPU","MEAN","type=f32,ne=[33,1,1,1]","support","0","no","WebGPU" -"WebGPU: WebGPU","MEAN","type=f32,ne=[33,256,1,1]","support","0","no","WebGPU" -"WebGPU: WebGPU","MEAN","type=f32,ne=[32769,1,1,1]","support","0","no","WebGPU" -"WebGPU: WebGPU","MEAN","type=f32,ne=[32,1,1,1]","support","0","no","WebGPU" -"WebGPU: WebGPU","MEAN","type=f32,ne=[32,256,1,1]","support","0","no","WebGPU" -"WebGPU: WebGPU","MEAN","type=f32,ne=[32768,1,1,1]","support","0","no","WebGPU" +"WebGPU: WebGPU","MEAN","type=f32,ne=[10,5,4,3],permute=0,slice=0","support","1","yes","WebGPU" +"WebGPU: WebGPU","MEAN","type=f32,ne=[33,1,1,1],permute=0,slice=0","support","1","yes","WebGPU" +"WebGPU: WebGPU","MEAN","type=f32,ne=[33,256,1,1],permute=0,slice=0","support","1","yes","WebGPU" +"WebGPU: WebGPU","MEAN","type=f32,ne=[32769,1,1,1],permute=0,slice=0","support","1","yes","WebGPU" +"WebGPU: WebGPU","MEAN","type=f32,ne=[32,1,1,1],permute=0,slice=0","support","1","yes","WebGPU" +"WebGPU: WebGPU","MEAN","type=f32,ne=[32,256,1,1],permute=0,slice=0","support","1","yes","WebGPU" +"WebGPU: WebGPU","MEAN","type=f32,ne=[32768,1,1,1],permute=0,slice=0","support","1","yes","WebGPU" +"WebGPU: WebGPU","MEAN","type=f32,ne=[11,5,6,3],permute=1,slice=0","support","1","yes","WebGPU" +"WebGPU: WebGPU","MEAN","type=f32,ne=[11,5,6,3],permute=0,slice=1","support","1","yes","WebGPU" +"WebGPU: WebGPU","MEAN","type=f32,ne=[11,5,6,3],permute=1,slice=1","support","1","yes","WebGPU" +"WebGPU: WebGPU","MEAN","type=f32,ne=[1,7,3,2],permute=0,slice=0","support","1","yes","WebGPU" +"WebGPU: WebGPU","MEAN","type=f32,ne=[1023,5,1,1],permute=0,slice=0","support","1","yes","WebGPU" +"WebGPU: WebGPU","MEAN","type=f32,ne=[1024,5,1,1],permute=0,slice=0","support","1","yes","WebGPU" +"WebGPU: WebGPU","MEAN","type=f32,ne=[1025,5,1,1],permute=0,slice=0","support","1","yes","WebGPU" +"WebGPU: WebGPU","MEAN","type=f32,ne=[17,65537,1,1],permute=0,slice=0","support","1","yes","WebGPU" +"WebGPU: WebGPU","MEAN","type=f32,ne=[512,1,1,1],permute=0,slice=0","support","1","yes","WebGPU" +"WebGPU: WebGPU","MEAN","type=f32,ne=[281,1536,1,1],permute=0,slice=0","support","1","yes","WebGPU" "WebGPU: WebGPU","SUM","type=f32,ne=[33,1,1,1]","support","1","yes","WebGPU" "WebGPU: WebGPU","SUM","type=f32,ne=[33,1024,1,1]","support","1","yes","WebGPU" "WebGPU: WebGPU","SUM","type=f32,ne=[33,256,1,1]","support","1","yes","WebGPU" @@ -12299,8 +12308,9 @@ "WebGPU: WebGPU","SUM_ROWS","type=f32,ne=[33,1,1,1],permute=0,slice=0","support","1","yes","WebGPU" "WebGPU: WebGPU","SUM_ROWS","type=f32,ne=[33,1024,1,1],permute=0,slice=0","support","1","yes","WebGPU" "WebGPU: WebGPU","SUM_ROWS","type=f32,ne=[33,256,1,1],permute=0,slice=0","support","1","yes","WebGPU" -"WebGPU: WebGPU","GROUP_NORM","type=f32,ne=[64,64,320,1],num_groups=32,eps=0.000001","support","0","no","WebGPU" -"WebGPU: WebGPU","GROUP_NORM","type=f32,ne=[9,9,1280,1],num_groups=32,eps=0.000001","support","0","no","WebGPU" +"WebGPU: WebGPU","GROUP_NORM","type=f32,ne=[64,64,320,1],num_groups=32,eps=0.000001","support","1","yes","WebGPU" +"WebGPU: WebGPU","GROUP_NORM","type=f32,ne=[9,9,1280,1],num_groups=32,eps=0.000001","support","1","yes","WebGPU" +"WebGPU: WebGPU","GROUP_NORM","type=f32,ne=[3,1,1,65537],num_groups=1,eps=0.000001","support","1","yes","WebGPU" "WebGPU: WebGPU","ACC","type=f32,ne_a=[256,17,1,1],ne_b=[256,16,1,1],stride_dim=-1","support","0","no","WebGPU" "WebGPU: WebGPU","ACC","type=f32,ne_a=[256,17,2,3],ne_b=[256,16,2,3],stride_dim=-1","support","0","no","WebGPU" "WebGPU: WebGPU","ACC","type=f32,ne_a=[256,17,2,3],ne_b=[128,16,2,3],stride_dim=-1","support","0","no","WebGPU" @@ -12323,11 +12333,24 @@ "WebGPU: WebGPU","PAD","type=f32,ne_a=[100,1,1,1],pad_0=100,pad_1=0,circular=0","support","1","yes","WebGPU" "WebGPU: WebGPU","PAD","type=f32,ne_a=[100,1,1,1],pad_0=0,pad_1=100,circular=0","support","1","yes","WebGPU" "WebGPU: WebGPU","PAD","type=f32,ne_a=[100,100,1,1],pad_0=50,pad_1=50,circular=0","support","1","yes","WebGPU" -"WebGPU: WebGPU","PAD_REFLECT_1D","type=f32,ne_a=[512,34,2,1],pad_0=10,pad_1=9","support","0","no","WebGPU" -"WebGPU: WebGPU","PAD_REFLECT_1D","type=f32,ne_a=[3000,384,4,1],pad_0=10,pad_1=9","support","0","no","WebGPU" +"WebGPU: WebGPU","PAD_REFLECT_1D","type=f32,ne_a=[512,34,2,1],pad_0=10,pad_1=9,view=0","support","1","yes","WebGPU" +"WebGPU: WebGPU","PAD_REFLECT_1D","type=f32,ne_a=[1,3,2,2],pad_0=0,pad_1=0,view=0","support","1","yes","WebGPU" +"WebGPU: WebGPU","PAD_REFLECT_1D","type=f32,ne_a=[5,3,2,2],pad_0=4,pad_1=1,view=1","support","1","yes","WebGPU" +"WebGPU: WebGPU","PAD_REFLECT_1D","type=f32,ne_a=[281,128,1,1],pad_0=2,pad_1=2,view=0","support","1","yes","WebGPU" +"WebGPU: WebGPU","PAD_REFLECT_1D","type=f32,ne_a=[257,7,1,1],pad_0=0,pad_1=256,view=0","support","1","yes","WebGPU" +"WebGPU: WebGPU","PAD_REFLECT_1D","type=f32,ne_a=[2,16777217,1,1],pad_0=1,pad_1=1,view=0","support","1","yes","WebGPU" +"WebGPU: WebGPU","PAD_REFLECT_1D","type=f32,ne_a=[3000,384,4,1],pad_0=10,pad_1=9,view=0","support","1","yes","WebGPU" "WebGPU: WebGPU","ROLL","shift0=3,shift1=-2,shift3=1,shift4=-1","support","0","no","WebGPU" -"WebGPU: WebGPU","ARANGE","type=f32,start=0.000000,stop=10.000000,step=1.000000","support","0","no","WebGPU" -"WebGPU: WebGPU","ARANGE","type=f32,start=0.000000,stop=1048576.000000,step=1.000000","support","0","no","WebGPU" +"WebGPU: WebGPU","ARANGE","type=f32,start=0.000000,stop=10.000000,step=1.000000","support","1","yes","WebGPU" +"WebGPU: WebGPU","ARANGE","type=f32,start=3.000000,stop=3.500000,step=1.000000","support","1","yes","WebGPU" +"WebGPU: WebGPU","ARANGE","type=f32,start=-10.000000,stop=10.000000,step=0.300000","support","1","yes","WebGPU" +"WebGPU: WebGPU","ARANGE","type=f32,start=-1.000000,stop=62.750000,step=0.250000","support","1","yes","WebGPU" +"WebGPU: WebGPU","ARANGE","type=f32,start=-1.000000,stop=63.000000,step=0.250000","support","1","yes","WebGPU" +"WebGPU: WebGPU","ARANGE","type=f32,start=-1.000000,stop=63.250000,step=0.250000","support","1","yes","WebGPU" +"WebGPU: WebGPU","ARANGE","type=f32,start=-1.000000,stop=254.750000,step=0.250000","support","1","yes","WebGPU" +"WebGPU: WebGPU","ARANGE","type=f32,start=-1.000000,stop=255.000000,step=0.250000","support","1","yes","WebGPU" +"WebGPU: WebGPU","ARANGE","type=f32,start=-1.000000,stop=255.250000,step=0.250000","support","1","yes","WebGPU" +"WebGPU: WebGPU","ARANGE","type=f32,start=0.000000,stop=1048576.000000,step=1.000000","support","1","yes","WebGPU" "WebGPU: WebGPU","TIMESTEP_EMBEDDING","type=f32,ne_a=[2,1,1,1],dim=320,max_period=10000","support","0","no","WebGPU" "WebGPU: WebGPU","LEAKY_RELU","type=f32,ne_a=[10,5,4,3],negative_slope=0.100000","support","0","no","WebGPU" "WebGPU: WebGPU","CUMSUM","type=f32,ne=[10,5,4,3]","support","1","yes","WebGPU" diff --git a/docs/ops/ZenDNN.csv b/docs/ops/ZenDNN.csv index 19d579fb8c50..b309f1a0d880 100644 --- a/docs/ops/ZenDNN.csv +++ b/docs/ops/ZenDNN.csv @@ -9319,9 +9319,6 @@ "ZenDNN","ROUND","type=f32,ne=[1024,1024,1,1]","support","0","no","ZenDNN" "ZenDNN","TRUNC","type=f32,ne=[7,1,5,3]","support","0","no","ZenDNN" "ZenDNN","TRUNC","type=f32,ne=[1024,1024,1,1]","support","0","no","ZenDNN" -"ZenDNN","DIAG_MASK_INF","type=f32,ne=[10,10,1,1],n_past=5","support","0","no","ZenDNN" -"ZenDNN","DIAG_MASK_INF","type=f32,ne=[10,10,3,1],n_past=5","support","0","no","ZenDNN" -"ZenDNN","DIAG_MASK_INF","type=f32,ne=[10,10,3,2],n_past=5","support","0","no","ZenDNN" "ZenDNN","SOFT_MAX","type=f32,ne=[16,16,1,1],mask=0,sinks=0,m_prec=f32,nr23=[1,1],scale=1.000000,max_bias=0.000000,inplace=0","support","0","no","ZenDNN" "ZenDNN","SOFT_MAX","type=f32,ne=[15,15,1,1],mask=0,sinks=0,m_prec=f32,nr23=[1,1],scale=1.000000,max_bias=0.000000,inplace=0","support","0","no","ZenDNN" "ZenDNN","SOFT_MAX","type=f32,ne=[16,1024,1,1],mask=0,sinks=0,m_prec=f32,nr23=[1,1],scale=1.000000,max_bias=0.000000,inplace=0","support","0","no","ZenDNN" diff --git a/docs/ops/zDNN.csv b/docs/ops/zDNN.csv index 005a1cb4de99..a44ef3ebba83 100644 --- a/docs/ops/zDNN.csv +++ b/docs/ops/zDNN.csv @@ -14550,9 +14550,6 @@ "zDNN","ROUND","type=f32,ne=[1024,1024,1,1]","support","0","no","zDNN" "zDNN","TRUNC","type=f32,ne=[7,1,5,3]","support","0","no","zDNN" "zDNN","TRUNC","type=f32,ne=[1024,1024,1,1]","support","0","no","zDNN" -"zDNN","DIAG_MASK_INF","type=f32,ne=[10,10,1,1],n_past=5","support","0","no","zDNN" -"zDNN","DIAG_MASK_INF","type=f32,ne=[10,10,3,1],n_past=5","support","0","no","zDNN" -"zDNN","DIAG_MASK_INF","type=f32,ne=[10,10,3,2],n_past=5","support","0","no","zDNN" "zDNN","SOFT_MAX","type=f32,ne=[16,16,1,1],mask=0,sinks=0,m_prec=f32,nr23=[1,1],scale=1.000000,max_bias=0.000000,inplace=0","support","0","no","zDNN" "zDNN","SOFT_MAX","type=f32,ne=[15,15,1,1],mask=0,sinks=0,m_prec=f32,nr23=[1,1],scale=1.000000,max_bias=0.000000,inplace=0","support","0","no","zDNN" "zDNN","SOFT_MAX","type=f32,ne=[16,1024,1,1],mask=0,sinks=0,m_prec=f32,nr23=[1,1],scale=1.000000,max_bias=0.000000,inplace=0","support","0","no","zDNN" diff --git a/docs/tts-norm-patterns.md b/docs/tts-norm-patterns.md new file mode 100644 index 000000000000..802dde5ce7e0 --- /dev/null +++ b/docs/tts-norm-patterns.md @@ -0,0 +1,125 @@ +# TTS normalization patterns + +Text-to-speech models often need their input normalized before tokenization: numbers, currencies, dates and symbols spelled out, punctuation mapped, spacing fixed. The exact rules are model-specific (they must match what the model was trained on), so they live in the model's `tts` task template rather than in C++. + +Jinja has no regex support, and exposing `std::regex` / Python `re` would not help: the two differ in many details (`\d`, `\w`, `\b`, lookbehind, case folding, backtracking limits), so a template could give different results in llama.cpp and in Python. Instead, task templates get two functions with a small matching language whose semantics are defined here: + +- `tts_norm_scan(text, rules)`: split text into matched and unmatched segments +- `tts_norm_sub(text, pattern, repl)`: replace every match + +Implementations: +- C++: [common/chat-tts-norm.cpp](../common/chat-tts-norm.cpp), injected into task templates by `common_chat_template_render()` in [common/chat.cpp](../common/chat.cpp). They are not available to chat templates. +- Python reference: [scripts/chat_tts_norm.py](../scripts/chat_tts_norm.py), for use with `jinja2` (`env.globals["tts_norm_scan"] = chat_tts_norm.tts_norm_scan`) +- Both are checked against each other by `tests/test-jinja.cpp` (`test-jinja -py "tts norm patterns.*"`) + +## Example + +```jinja +{%- for s in tts_norm_scan(text, [ + ['money', '%$(%d+)(?:%.(%d%d))?'], + ['percent', '(%d+)%%'], + ['time', '(%d%d?):(%d%d)(?!%d)'], + ['neg', '(? 10: diff --git a/examples/model-conversion/scripts/embedding/run-original-model.py b/examples/model-conversion/scripts/embedding/run-original-model.py index 001d58896553..da6aec773a49 100755 --- a/examples/model-conversion/scripts/embedding/run-original-model.py +++ b/examples/model-conversion/scripts/embedding/run-original-model.py @@ -64,7 +64,7 @@ def load_model_and_tokenizer(model_path, use_sentence_transformers=False, device print("Using SentenceTransformer to apply all numbered layers") model = SentenceTransformer(model_path) tokenizer = model.tokenizer - config = model[0].auto_model.config # ty: ignore[unresolved-attribute] + config = model[0].auto_model.config # ty: ignore[missing-union-attribute] else: tokenizer = AutoTokenizer.from_pretrained(model_path) config = AutoConfig.from_pretrained(model_path, trust_remote_code=True) diff --git a/examples/model-conversion/scripts/utils/check-nmse.py b/examples/model-conversion/scripts/utils/check-nmse.py index 324e3858e368..363d803263b5 100755 --- a/examples/model-conversion/scripts/utils/check-nmse.py +++ b/examples/model-conversion/scripts/utils/check-nmse.py @@ -5,7 +5,7 @@ import os import argparse from pathlib import Path -from common import get_model_name_from_env_path # type: ignore[import-not-found, ty:unresolved-import] +from common import get_model_name_from_env_path # ty: ignore[import-not-found] def calculate_nmse(reference, test): mse = np.mean((test - reference) ** 2) diff --git a/examples/model-conversion/scripts/utils/compare_tokens.py b/examples/model-conversion/scripts/utils/compare_tokens.py index 55e3f26ab4d1..0d4612ef8153 100755 --- a/examples/model-conversion/scripts/utils/compare_tokens.py +++ b/examples/model-conversion/scripts/utils/compare_tokens.py @@ -2,7 +2,7 @@ import argparse import sys -from common import compare_tokens # type: ignore[import-not-found, ty:unresolved-import] +from common import compare_tokens # ty: ignore[import-not-found] def parse_arguments(): diff --git a/examples/model-conversion/scripts/utils/semantic_check.py b/examples/model-conversion/scripts/utils/semantic_check.py index 754ae733da2b..f600bcadeb02 100644 --- a/examples/model-conversion/scripts/utils/semantic_check.py +++ b/examples/model-conversion/scripts/utils/semantic_check.py @@ -7,7 +7,7 @@ from pathlib import Path from transformers import AutoTokenizer, AutoConfig, AutoModelForCausalLM, AutoModel -from common import compare_tokens, exit_with_warning # type: ignore[import-not-found, ty:unresolved-import] +from common import compare_tokens, exit_with_warning # ty: ignore[import-not-found] unreleased_model_name = os.getenv('UNRELEASED_MODEL_NAME') @@ -183,6 +183,7 @@ def main(): # Load the python model to get configuration information and also to load the tokenizer. print("Loading model and tokenizer using AutoTokenizer:", args.model_path) tokenizer = AutoTokenizer.from_pretrained(args.model_path) + assert tokenizer is not None config = AutoConfig.from_pretrained(args.model_path, trust_remote_code=True) if unreleased_model_name: @@ -207,8 +208,8 @@ def main(): else: model = AutoModel.from_pretrained(args.model_path, trust_remote_code=True) - encoded = tokenizer(prompt, return_tensors="pt") # ty: ignore[call-non-callable] - tokens = tokenizer.convert_ids_to_tokens(encoded['input_ids'][0]) # ty: ignore[unresolved-attribute] + encoded = tokenizer(prompt, return_tensors="pt") + tokens = tokenizer.convert_ids_to_tokens(encoded['input_ids'][0]) n_tokens = len(tokens) print(f"n_tokens: {n_tokens}"); print(f"hidden_size: {model.config.hidden_size}") diff --git a/examples/pydantic_models_to_grammar.py b/examples/pydantic_models_to_grammar.py index 736b2b7df104..ff6e120e45bb 100644 --- a/examples/pydantic_models_to_grammar.py +++ b/examples/pydantic_models_to_grammar.py @@ -1285,7 +1285,7 @@ def convert_dictionary_to_pydantic_model(dictionary: dict[str, Any], model_name: if items != {}: array = {"properties": items} array_type = convert_dictionary_to_pydantic_model(array, f"{model_name}_{field_name}_items") - fields[field_name] = (list[array_type], ...) # ty: ignore[invalid-type-form] + fields[field_name] = (list[array_type], ...) else: fields[field_name] = (list, ...) elif field_type == "object": diff --git a/examples/speculative/speculative.cpp b/examples/speculative/speculative.cpp index 8f347b0d4eee..f3c6bec6ff4c 100644 --- a/examples/speculative/speculative.cpp +++ b/examples/speculative/speculative.cpp @@ -103,6 +103,9 @@ int main(int argc, char ** argv) { params.cpuparams_batch.n_threads = params.speculative.draft.cpuparams_batch.n_threads; params.tensor_buft_overrides = params.speculative.draft.tensor_buft_overrides; + // the MoE cache is only used by the target context + params.moe_cache_size = 0; + auto llama_init_dft = common_init_from_params(params); model_dft = llama_init_dft->model(); diff --git a/examples/sycl/build.sh b/examples/sycl/build.sh index dc9e44a8ea28..c3017b6c5eb1 100755 --- a/examples/sycl/build.sh +++ b/examples/sycl/build.sh @@ -4,55 +4,87 @@ # SPDX-License-Identifier: MIT print_usage() { - echo "Usage: ./build.sh [fp32|fp16] [ssl|nossl] [--help]" + echo "Usage: ./build.sh [fp32|fp16] [ssl|no-ssl] [no-xmx] [--aot aot_flags] [--help]" echo "" echo "Options:" echo " fp32 Build with FP32 precision (default)" echo " fp16 Build with FP16 precision (faster for long-prompt inference)" echo " ssl Build with OpenSSL (default)" - echo " nossl Build without OpenSSL" + echo " no-ssl Build without OpenSSL" + echo " no-xmx Disable XMX (DPAS) support" + echo " --aot Set SYCL AOT device architecture, e.g. arl-h,acm-g10,bmg-g21 or bmg-g21" echo " --help Print this help message" } PRECISION=fp32 SSL=ON +AOT_FLAGS="" +SYCL_SUPPORT_XMX=ON -for arg in "$@"; do - case "$arg" in +while [[ $# -gt 0 ]]; do + case "$1" in --help) print_usage exit 0 ;; fp32|fp16) - PRECISION="$arg" + PRECISION="$1" + shift ;; ssl) SSL=ON + shift ;; - nossl) + no-ssl) SSL=OFF + shift + ;; + no-xmx) + SYCL_SUPPORT_XMX=OFF + shift + ;; + --aot) + if [[ $# -lt 2 || "$2" == --* ]]; then + echo "Error: --aot requires aot_flags" + print_usage + exit 1 + fi + AOT_FLAGS="$2" + shift 2 ;; *) - echo "Error: unknown option '$arg'" + echo "Error: unknown option '$1'" print_usage exit 1 ;; esac done +AOT_OPTION=() +if [[ -n "$AOT_FLAGS" ]]; then + AOT_OPTION=("-DGGML_SYCL_DEVICE_ARCH=$AOT_FLAGS") +fi + mkdir -p build cd build source /opt/intel/oneapi/setvars.sh if [ "$PRECISION" = "fp16" ]; then #for FP16 - echo "cmake .. -DGGML_SYCL=ON -DCMAKE_C_COMPILER=icx -DCMAKE_CXX_COMPILER=icpx -DGGML_SYCL_F16=ON -DLLAMA_OPENSSL=$SSL # faster for long-prompt inference -" - cmake .. -DGGML_SYCL=ON -DCMAKE_C_COMPILER=icx -DCMAKE_CXX_COMPILER=icpx -DGGML_SYCL_F16=ON -DLLAMA_OPENSSL="$SSL" # faster for long-prompt inference + printf 'cmake .. -DGGML_SYCL=ON -DCMAKE_C_COMPILER=icx -DCMAKE_CXX_COMPILER=icpx -DGGML_SYCL_F16=ON -DGGML_SYCL_SUPPORT_XMX=%s -DLLAMA_OPENSSL=%s' "$SYCL_SUPPORT_XMX" "$SSL" + if [[ ${#AOT_OPTION[@]} -gt 0 ]]; then + printf ' %s' "${AOT_OPTION[@]}" + fi + echo " # faster for long-prompt inference" + cmake .. -DGGML_SYCL=ON -DCMAKE_C_COMPILER=icx -DCMAKE_CXX_COMPILER=icpx -DGGML_SYCL_F16=ON -DGGML_SYCL_SUPPORT_XMX="$SYCL_SUPPORT_XMX" -DLLAMA_OPENSSL="$SSL" "${AOT_OPTION[@]}" # faster for long-prompt inference else #for FP32 - echo "cmake .. -DGGML_SYCL=ON -DCMAKE_C_COMPILER=icx -DCMAKE_CXX_COMPILER=icpx -DLLAMA_OPENSSL=$SSL" - cmake .. -DGGML_SYCL=ON -DCMAKE_C_COMPILER=icx -DCMAKE_CXX_COMPILER=icpx -DLLAMA_OPENSSL="$SSL" + printf 'cmake .. -DGGML_SYCL=ON -DCMAKE_C_COMPILER=icx -DCMAKE_CXX_COMPILER=icpx -DGGML_SYCL_SUPPORT_XMX=%s -DLLAMA_OPENSSL=%s' "$SYCL_SUPPORT_XMX" "$SSL" + if [[ ${#AOT_OPTION[@]} -gt 0 ]]; then + printf ' %s' "${AOT_OPTION[@]}" + fi + echo + cmake .. -DGGML_SYCL=ON -DCMAKE_C_COMPILER=icx -DCMAKE_CXX_COMPILER=icpx -DGGML_SYCL_SUPPORT_XMX="$SYCL_SUPPORT_XMX" -DLLAMA_OPENSSL="$SSL" "${AOT_OPTION[@]}" fi #build example/main @@ -62,5 +94,6 @@ fi #cmake --build . --config Release --target llama-bench #build all binary -echo "cmake --build . --config Release -j$((($(nproc)+1)/2)) -v" -cmake --build . --config Release -j$((($(nproc)+1)/2)) -v +BUILD_JOBS=$((($(nproc) + 1) / 2)) +echo "cmake --build . --config Release -j$BUILD_JOBS -v" +cmake --build . --config Release -j"$BUILD_JOBS" -v diff --git a/examples/sycl/win-build-sycl.bat b/examples/sycl/win-build-sycl.bat index 4b3906a6371f..eac95ece0516 100644 --- a/examples/sycl/win-build-sycl.bat +++ b/examples/sycl/win-build-sycl.bat @@ -6,6 +6,8 @@ SET "PRECISION=fp32" SET "SSL=ON" SET "OPENSSL_OPTION=" +SET "AOT_OPTION=" +SET "XMX_OPTION=" :PARSE_ARGS IF "%~1"=="" GOTO ARGS_DONE @@ -13,7 +15,9 @@ IF /I "%~1"=="--help" GOTO HELP IF /I "%~1"=="fp32" GOTO ARG_FP32 IF /I "%~1"=="fp16" GOTO ARG_FP16 IF /I "%~1"=="ssl" GOTO ARG_SSL -IF /I "%~1"=="nossl" GOTO ARG_NOSSL +IF /I "%~1"=="no-ssl" GOTO ARG_NO_SSL +IF /I "%~1"=="no-xmx" GOTO ARG_NO_XMX +IF /I "%~1"=="--aot" GOTO ARG_AOT GOTO INVALID_ARG :ARG_FP32 @@ -29,11 +33,23 @@ SET "SSL=ON" SET "OPENSSL_OPTION=" GOTO NEXT_ARG -:ARG_NOSSL +:ARG_NO_SSL SET "SSL=OFF" SET "OPENSSL_OPTION=-DLLAMA_OPENSSL=OFF" GOTO NEXT_ARG +:ARG_NO_XMX +SET "XMX_OPTION=-DGGML_SYCL_SUPPORT_XMX=OFF" +GOTO NEXT_ARG + +:ARG_AOT +IF "%~2"=="" GOTO MISSING_AOT +SET "AOT_VALUE=%~2" +IF "%AOT_VALUE:~0,2%"=="--" GOTO MISSING_AOT +SET "AOT_OPTION=-DGGML_SYCL_DEVICE_ARCH=%~2" +SHIFT +GOTO NEXT_ARG + :NEXT_ARG SHIFT GOTO PARSE_ARGS @@ -45,15 +61,22 @@ GOTO PRINT_USAGE :INVALID_ARG SET "ARG_ERROR=1" echo Error: unknown option '%~1' +GOTO PRINT_USAGE + +:MISSING_AOT +SET "ARG_ERROR=1" +echo Error: --aot requires aot_flags :PRINT_USAGE -echo Usage: win-build-sycl.bat [fp32^|fp16] [ssl^|nossl] [--help] +echo Usage: win-build-sycl.bat [fp32^|fp16] [ssl^|no-ssl] [no-xmx] [--aot aot_flags] [--help] echo. echo Options: echo fp32 Build with FP32 precision ^(default^) echo fp16 Build with FP16 precision ^(faster for long-prompt inference^) echo ssl Build with BoringSSL ^(default^) -echo nossl Build without BoringSSL +echo no-ssl Build without BoringSSL +echo no-xmx Disable XMX ^(DPAS^) support +echo --aot Set SYCL AOT device architecture, e.g. arl-h,acm-g10,bmg-g21 or bmg-g21 echo --help Print this help message IF "%ARG_ERROR%"=="1" exit /B 1 exit /B 0 @@ -70,10 +93,10 @@ if %errorlevel% neq 0 goto ERROR IF /I "%PRECISION%"=="fp16" ( :: for FP16 :: faster for long-prompt inference - cmake -G "MinGW Makefiles" .. -DLLAMA_BUILD_BORINGSSL=%SSL% %OPENSSL_OPTION% -DGGML_SYCL=ON -DCMAKE_CXX_COMPILER=icx -DBUILD_SHARED_LIBS=ON -DCMAKE_BUILD_TYPE=Release -DGGML_SYCL_F16=ON + cmake -G "MinGW Makefiles" .. -DLLAMA_BUILD_BORINGSSL=%SSL% %OPENSSL_OPTION% -DGGML_SYCL=ON %XMX_OPTION% %AOT_OPTION% -DCMAKE_CXX_COMPILER=icx -DBUILD_SHARED_LIBS=ON -DCMAKE_BUILD_TYPE=Release -DGGML_SYCL_F16=ON ) ELSE ( :: for FP32 - cmake -G "Ninja" .. -DLLAMA_BUILD_BORINGSSL=%SSL% %OPENSSL_OPTION% -DGGML_SYCL=ON -DCMAKE_C_COMPILER=cl -DCMAKE_CXX_COMPILER=icx -DBUILD_SHARED_LIBS=ON -DCMAKE_BUILD_TYPE=Release + cmake -G "Ninja" .. -DLLAMA_BUILD_BORINGSSL=%SSL% %OPENSSL_OPTION% -DGGML_SYCL=ON %XMX_OPTION% %AOT_OPTION% -DCMAKE_C_COMPILER=cl -DCMAKE_CXX_COMPILER=icx -DBUILD_SHARED_LIBS=ON -DCMAKE_BUILD_TYPE=Release ) if %errorlevel% neq 0 goto ERROR diff --git a/ggml/CMakeLists.txt b/ggml/CMakeLists.txt index dbc1a1f5df2d..55da43e57c4e 100644 --- a/ggml/CMakeLists.txt +++ b/ggml/CMakeLists.txt @@ -4,7 +4,7 @@ project("ggml" C CXX ASM) ### GGML Version set(GGML_VERSION_MAJOR 0) -set(GGML_VERSION_MINOR 28) +set(GGML_VERSION_MINOR 29) set(GGML_VERSION_PATCH 0) set(GGML_VERSION_BASE "${GGML_VERSION_MAJOR}.${GGML_VERSION_MINOR}.${GGML_VERSION_PATCH}") @@ -205,7 +205,7 @@ option(GGML_CUDA_NO_PEER_COPY "ggml: do not use peer to peer copie option(GGML_CUDA_NO_VMM "ggml: do not try to use CUDA VMM" OFF) option(GGML_CUDA_FA "ggml: compile ggml FlashAttention CUDA kernels" ON) option(GGML_CUDA_FA_ALL_QUANTS "ggml: compile all quants for FlashAttention" OFF) -set (GGML_CUDA_FA_QUANTS "q4_0-q4_0;q8_0-q8_0;f16-f16;bf16-bf16" CACHE STRING +set (GGML_CUDA_FA_QUANTS "q4_0-q4_0;q8_0-q8_0;f16-f16;bf16-bf16;f8_e4m3-f8_e4m3" CACHE STRING "ggml: FlashAttention K-V type combinations to compile, \"all\" or a list such as \"q8_0-q8_0;q8_0-q4_0\"") option(GGML_CUDA_GRAPHS "ggml: use CUDA graphs (llama.cpp only)" ${GGML_CUDA_GRAPHS_DEFAULT}) option(GGML_CUDA_NCCL "ggml: use NVIDIA Collective Comm. Library" ON) diff --git a/ggml/cmake/common.cmake b/ggml/cmake/common.cmake index 42b65a58051c..c8d232294740 100644 --- a/ggml/cmake/common.cmake +++ b/ggml/cmake/common.cmake @@ -58,7 +58,7 @@ endfunction() # Determines which FlashAttention vector kernel template instances to compile, returns them in OUT_SRCS. function(ggml_cuda_fattn_vec_instances DIR OUT_SRCS) - set(FA_TYPES q4_0 q4_1 q5_0 q5_1 q8_0 bf16 f16) + set(FA_TYPES q4_0 q4_1 q5_0 q5_1 q8_0 bf16 f16 f8_e4m3) string(TOLOWER "${GGML_CUDA_FA_QUANTS}" FA_QUANTS) string(STRIP "${FA_QUANTS}" FA_QUANTS) diff --git a/ggml/include/ggml-backend.h b/ggml/include/ggml-backend.h index 735ed2c2b017..afb86338aac8 100644 --- a/ggml/include/ggml-backend.h +++ b/ggml/include/ggml-backend.h @@ -211,6 +211,18 @@ extern "C" { // Context management and operations for faster communication between backends, used for tensor parallelism (meta backend) typedef void * (*ggml_backend_comm_init_t)(ggml_backend_t * backends, size_t n_backends); + + // Builds the communicator for the case where each rank is a separate process, so a process + // owns one backend of the communicator rather than all of them. + // + // The caller provides bootstrap_callback, which moves bytes between the ranks before the + // communicator exists. Each rank passes size bytes in send_buf, and recv_buf receives + // world_size * size bytes in rank order. The backend decides what to exchange and how many + // times to call it, and every rank must make the same calls in the same order. + // comm_init_rank blocks until all world_size ranks have joined. + typedef bool (*ggml_backend_comm_bootstrap_callback)(void * user_data, const void * send_buf, void * recv_buf, size_t size); + typedef void * (*ggml_backend_comm_init_rank_t)(ggml_backend_t backend, int rank, int world_size, ggml_backend_comm_bootstrap_callback bootstrap_callback, void * bootstrap_callback_data); + typedef void (*ggml_backend_comm_free_t)(void * comm_ctx); typedef bool (*ggml_backend_comm_allreduce_tensor_t)(void * comm_ctx, struct ggml_tensor ** tensors); diff --git a/ggml/include/ggml-rpc.h b/ggml/include/ggml-rpc.h index ecf0a3a6dc25..da1d4e1f167b 100644 --- a/ggml/include/ggml-rpc.h +++ b/ggml/include/ggml-rpc.h @@ -6,26 +6,31 @@ extern "C" { #endif -#define RPC_PROTO_MAJOR_VERSION 8 +#define RPC_PROTO_MAJOR_VERSION 10 #define RPC_PROTO_MINOR_VERSION 0 -#define RPC_PROTO_PATCH_VERSION 1 +#define RPC_PROTO_PATCH_VERSION 0 #ifdef __cplusplus -static_assert(GGML_OP_COUNT == 102, "GGML_OP_COUNT has changed - update RPC_PROTO_PATCH_VERSION"); +static_assert(GGML_OP_COUNT == 99, "GGML_OP_COUNT has changed - update RPC_PROTO_PATCH_VERSION"); #endif #define GGML_RPC_MAX_SERVERS 16 // backend API GGML_BACKEND_API ggml_backend_t ggml_backend_rpc_init(const char * endpoint, uint32_t device); + GGML_BACKEND_API bool ggml_backend_is_rpc(ggml_backend_t backend); GGML_BACKEND_API ggml_backend_buffer_type_t ggml_backend_rpc_buffer_type(const char * endpoint, uint32_t device); GGML_BACKEND_API void ggml_backend_rpc_get_device_memory(const char * endpoint, uint32_t device, size_t * free, size_t * total); -GGML_BACKEND_API void ggml_backend_rpc_start_server(const char * endpoint, const char * cache_dir, - size_t n_threads, size_t n_devices, ggml_backend_dev_t * devices); +GGML_BACKEND_API void ggml_backend_rpc_start_server( + const char * endpoint, + const char * cache_dir, + size_t n_threads, + size_t n_devices, + ggml_backend_dev_t * devices); GGML_BACKEND_API ggml_backend_reg_t ggml_backend_rpc_reg(void); GGML_BACKEND_API ggml_backend_reg_t ggml_backend_rpc_add_server(const char * endpoint); diff --git a/ggml/include/ggml.h b/ggml/include/ggml.h index 3ed4181408fb..2341fa6c107e 100644 --- a/ggml/include/ggml.h +++ b/ggml/include/ggml.h @@ -1,178 +1,5 @@ #pragma once -// -// GGML Tensor Library -// -// This documentation is still a work in progress. -// If you wish some specific topics to be covered, feel free to drop a comment: -// -// https://github.com/ggml-org/whisper.cpp/issues/40 -// -// ## Overview -// -// This library implements: -// -// - a set of tensor operations -// - automatic differentiation -// - basic optimization algorithms -// -// The aim of this library is to provide a minimalistic approach for various machine learning tasks. This includes, -// but is not limited to, the following: -// -// - linear regression -// - support vector machines -// - neural networks -// -// The library allows the user to define a certain function using the available tensor operations. This function -// definition is represented internally via a computation graph. Each tensor operation in the function definition -// corresponds to a node in the graph. Having the computation graph defined, the user can choose to compute the -// function's value and/or its gradient with respect to the input variables. Optionally, the function can be optimized -// using one of the available optimization algorithms. -// -// For example, here we define the function: f(x) = a*x^2 + b -// -// { -// struct ggml_init_params params = { -// .mem_size = 16*1024*1024, -// .mem_buffer = NULL, -// }; -// -// // memory allocation happens here -// struct ggml_context * ctx = ggml_init(params); -// -// struct ggml_tensor * x = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, 1); -// -// ggml_set_param(ctx, x); // x is an input variable -// -// struct ggml_tensor * a = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, 1); -// struct ggml_tensor * b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, 1); -// struct ggml_tensor * x2 = ggml_mul(ctx, x, x); -// struct ggml_tensor * f = ggml_add(ctx, ggml_mul(ctx, a, x2), b); -// -// ... -// } -// -// Notice that the function definition above does not involve any actual computation. The computation is performed only -// when the user explicitly requests it. For example, to compute the function's value at x = 2.0: -// -// { -// ... -// -// struct ggml_cgraph * gf = ggml_new_graph(ctx); -// ggml_build_forward_expand(gf, f); -// -// // set the input variable and parameter values -// ggml_set_f32(x, 2.0f); -// ggml_set_f32(a, 3.0f); -// ggml_set_f32(b, 4.0f); -// -// ggml_graph_compute_with_ctx(ctx, &gf, n_threads); -// -// printf("f = %f\n", ggml_get_f32_1d(f, 0)); -// -// ... -// } -// -// The actual computation is performed in the ggml_graph_compute() function. -// -// The ggml_new_tensor_...() functions create new tensors. They are allocated in the memory buffer provided to the -// ggml_init() function. You have to be careful not to exceed the memory buffer size. Therefore, you have to know -// in advance how much memory you need for your computation. Alternatively, you can allocate a large enough memory -// and after defining the computation graph, call the ggml_used_mem() function to find out how much memory was -// actually needed. -// -// The ggml_set_param() function marks a tensor as an input variable. This is used by the automatic -// differentiation and optimization algorithms. -// -// The described approach allows to define the function graph once and then compute its forward or backward graphs -// multiple times. All computations will use the same memory buffer allocated in the ggml_init() function. This way -// the user can avoid the memory allocation overhead at runtime. -// -// The library supports multi-dimensional tensors - up to 4 dimensions. The FP16 and FP32 data types are first class -// citizens, but in theory the library can be extended to support FP8 and integer data types. -// -// Each tensor operation produces a new tensor. Initially the library was envisioned to support only the use of unary -// and binary operations. Most of the available operations fall into one of these two categories. With time, it became -// clear that the library needs to support more complex operations. The way to support these operations is not clear -// yet, but a few examples are demonstrated in the following operations: -// -// - ggml_permute() -// - ggml_conv_1d_1s() -// - ggml_conv_1d_2s() -// -// For each tensor operator, the library implements a forward and backward computation function. The forward function -// computes the output tensor value given the input tensor values. The backward function computes the adjoint of the -// input tensors given the adjoint of the output tensor. For a detailed explanation of what this means, take a -// calculus class, or watch the following video: -// -// What is Automatic Differentiation? -// https://www.youtube.com/watch?v=wG_nF1awSSY -// -// -// ## Tensor data (struct ggml_tensor) -// -// The tensors are stored in memory via the ggml_tensor struct. The structure provides information about the size of -// the tensor, the data type, and the memory buffer where the tensor data is stored. Additionally, it contains -// pointers to the "source" tensors - i.e. the tensors that were used to compute the current tensor. For example: -// -// { -// struct ggml_tensor * c = ggml_add(ctx, a, b); -// -// assert(c->src[0] == a); -// assert(c->src[1] == b); -// } -// -// The multi-dimensional tensors are stored in row-major order. The ggml_tensor struct contains fields for the -// number of elements in each dimension ("ne") as well as the number of bytes ("nb", a.k.a. stride). This allows -// to store tensors that are not contiguous in memory, which is useful for operations such as transposition and -// permutation. All tensor operations have to take the stride into account and not assume that the tensor is -// contiguous in memory. -// -// The data of the tensor is accessed via the "data" pointer. For example: -// -// { -// const int nx = 2; -// const int ny = 3; -// -// struct ggml_tensor * a = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, nx, ny); -// -// for (int y = 0; y < ny; y++) { -// for (int x = 0; x < nx; x++) { -// *(float *) ((char *) a->data + y*a->nb[1] + x*a->nb[0]) = x + y; -// } -// } -// -// ... -// } -// -// Alternatively, there are helper functions, such as ggml_get_f32_1d() and ggml_set_f32_1d() that can be used. -// -// ## The matrix multiplication operator (ggml_mul_mat) -// -// TODO -// -// -// ## Multi-threading -// -// TODO -// -// -// ## Overview of ggml.c -// -// TODO -// -// -// ## SIMD optimizations -// -// TODO -// -// -// ## Debugging ggml -// -// TODO -// -// - #ifdef GGML_SHARED # if defined(_WIN32) && !defined(__MINGW32__) # ifdef GGML_BUILD @@ -445,7 +272,6 @@ extern "C" { // the precision parameters are stored as ggml_tensor.op_params to the respective ops enum ggml_prec { GGML_PREC_UNDEFINED = 0, - GGML_PREC_DEFAULT = 0, // note: deprecated, use GGML_PREC_UNDEFINED GGML_PREC_F32 = 10, GGML_PREC_BF16 = 15, GGML_PREC_F16 = 20, @@ -499,7 +325,6 @@ extern "C" { GGML_OP_DUP, GGML_OP_ADD, GGML_OP_ADD_ID, - GGML_OP_ADD1, GGML_OP_ACC, GGML_OP_SUB, GGML_OP_MUL, @@ -541,8 +366,6 @@ extern "C" { GGML_OP_GET_ROWS_BACK, GGML_OP_SET_ROWS, GGML_OP_DIAG, - GGML_OP_DIAG_MASK_INF, - GGML_OP_DIAG_MASK_ZERO, GGML_OP_SOFT_MAX, GGML_OP_SOFT_MAX_BACK, GGML_OP_ROPE, @@ -765,10 +588,6 @@ extern "C" { GGML_API size_t ggml_type_size(enum ggml_type type); // size in bytes for all elements in a block GGML_API size_t ggml_row_size (enum ggml_type type, int64_t ne); // size in bytes for all elements in a row - GGML_DEPRECATED( - GGML_API double ggml_type_sizef(enum ggml_type type), // ggml_type_size()/ggml_blck_size() as float - "use ggml_row_size() instead"); - GGML_API const char * ggml_type_name(enum ggml_type type); GGML_API const char * ggml_op_name (enum ggml_op op); GGML_API const char * ggml_op_symbol(enum ggml_op op); @@ -937,18 +756,6 @@ extern "C" { struct ggml_tensor * b, struct ggml_tensor * ids); - GGML_DEPRECATED(GGML_API struct ggml_tensor * ggml_add1( - struct ggml_context * ctx, - struct ggml_tensor * a, - struct ggml_tensor * b), - "use ggml_add instead"); - - GGML_DEPRECATED(GGML_API struct ggml_tensor * ggml_add1_inplace( - struct ggml_context * ctx, - struct ggml_tensor * a, - struct ggml_tensor * b), - "use ggml_add_inplace instead"); - // dst = a // view(dst, nb1, nb2, nb3, offset) += b // return dst @@ -1500,13 +1307,6 @@ extern "C" { struct ggml_tensor * a, struct ggml_tensor * b); - // change the precision of a matrix multiplication - // set to GGML_PREC_F32 for higher precision (useful for phi-2) - GGML_DEPRECATED(GGML_API void ggml_mul_mat_set_prec( - struct ggml_tensor * a, - enum ggml_prec prec), - "use ggml_prec_set_acc() instead"); - // change the hint of a matrix multiplication GGML_API void ggml_mul_mat_set_hint( struct ggml_tensor * a, @@ -1519,6 +1319,7 @@ extern "C" { // note: this op has undefined behaviour if a given `a` is referenced more than once for a given `b` // (i.e. duplicate experts per token are not guaranteed to be supported in all backends) // TODO: add backend support: https://github.com/ggml-org/llama.cpp/pull/29998#issuecomment-6153742533 + // the CPU backend skips negative ids, their rows of the result are zero GGML_API struct ggml_tensor * ggml_mul_mat_id( struct ggml_context * ctx, struct ggml_tensor * as, @@ -1777,30 +1578,6 @@ extern "C" { struct ggml_context * ctx, struct ggml_tensor * a); - // set elements above the diagonal to -INF - GGML_API struct ggml_tensor * ggml_diag_mask_inf( - struct ggml_context * ctx, - struct ggml_tensor * a, - int n_past); - - // in-place, returns view(a) - GGML_API struct ggml_tensor * ggml_diag_mask_inf_inplace( - struct ggml_context * ctx, - struct ggml_tensor * a, - int n_past); - - // set elements above the diagonal to 0 - GGML_API struct ggml_tensor * ggml_diag_mask_zero( - struct ggml_context * ctx, - struct ggml_tensor * a, - int n_past); - - // in-place, returns view(a) - GGML_API struct ggml_tensor * ggml_diag_mask_zero_inplace( - struct ggml_context * ctx, - struct ggml_tensor * a, - int n_past); - GGML_API struct ggml_tensor * ggml_clamp( struct ggml_context * ctx, struct ggml_tensor * a, @@ -2004,40 +1781,6 @@ extern "C" { float beta_fast, float beta_slow); - GGML_DEPRECATED(GGML_API struct ggml_tensor * ggml_rope_custom( - struct ggml_context * ctx, - struct ggml_tensor * a, - struct ggml_tensor * b, - int n_dims, - int mode, - int n_ctx_orig, - float freq_base, - float freq_scale, - float ext_factor, - float attn_factor, - float beta_fast, - float beta_slow), - "use ggml_rope_ext instead"); - - GGML_DEPRECATED(GGML_API struct ggml_tensor * ggml_rope_custom_inplace( - struct ggml_context * ctx, - struct ggml_tensor * a, - struct ggml_tensor * b, - int n_dims, - int mode, - int n_ctx_orig, - float freq_base, - float freq_scale, - float ext_factor, - float attn_factor, - float beta_fast, - float beta_slow), - "use ggml_rope_ext_inplace instead"); - - // compute correction dims for YaRN RoPE scaling - GGML_API void ggml_rope_yarn_corr_dims( - int n_dims, int n_ctx_orig, float freq_base, float beta_fast, float beta_slow, float dims[2]); - // rotary position embedding backward, i.e compute dx from dy // a - dy GGML_API struct ggml_tensor * ggml_rope_ext_back( @@ -2080,6 +1823,11 @@ extern "C" { struct ggml_tensor * a, int n_offs); + // control rounding of YaRN correction bounds (default: true) + GGML_API struct ggml_tensor * ggml_rope_set_truncate( + struct ggml_tensor * a, + bool truncate); + // im2col // converts data into a format that effectively results in a convolution when combined with matrix multiplication GGML_API struct ggml_tensor * ggml_im2col( @@ -2338,7 +2086,7 @@ extern "C" { GGML_SCALE_MODE_BILINEAR = 1, GGML_SCALE_MODE_BICUBIC = 2, - GGML_SCALE_MODE_COUNT + GGML_SCALE_MODE_COUNT = 3 }; enum ggml_scale_flag { @@ -2354,18 +2102,6 @@ extern "C" { int scale_factor, enum ggml_scale_mode mode); - // interpolate - // interpolate scale to specified dimensions - GGML_DEPRECATED(GGML_API struct ggml_tensor * ggml_upscale_ext( - struct ggml_context * ctx, - struct ggml_tensor * a, - int ne0, - int ne1, - int ne2, - int ne3, - enum ggml_scale_mode mode), - "use ggml_interpolate instead"); - // Up- or downsamples the input to the specified size. // 2D scale modes (eg. bilinear) are applied to the first two dimensions. GGML_API struct ggml_tensor * ggml_interpolate( @@ -2534,11 +2270,6 @@ extern "C" { float max_bias, float logit_softcap); - GGML_DEPRECATED(GGML_API void ggml_flash_attn_ext_set_prec( - struct ggml_tensor * a, - enum ggml_prec prec), - "use ggml_prec_set_acc() instead"); - GGML_API enum ggml_prec ggml_flash_attn_ext_get_prec( const struct ggml_tensor * a); diff --git a/ggml/src/CMakeLists.txt b/ggml/src/CMakeLists.txt index fe001a042a6a..5ff78f240720 100644 --- a/ggml/src/CMakeLists.txt +++ b/ggml/src/CMakeLists.txt @@ -131,15 +131,9 @@ endif() # POSIX conformance # -# clock_gettime came in POSIX.1b (1993) -# CLOCK_MONOTONIC came in POSIX.1-2001 / SUSv3 as optional -# posix_memalign came in POSIX.1-2001 / SUSv3 -# M_PI is an XSI extension since POSIX.1-2001 / SUSv3, came in XPG1 (1985) - -# Somehow in OpenBSD whenever POSIX conformance is specified -# some string functions rely on locale_t availability, -# which was introduced in POSIX.1-2008, forcing us to go higher -if (CMAKE_SYSTEM_NAME MATCHES "OpenBSD") +# POSIX.1-2001 with XSI extensions provides clock_gettime, CLOCK_MONOTONIC, posix_memalign and M_PI. +# OpenBSD needs POSIX.1-2008 for locale_t. FreeBSD needs it for F_DUPFD_CLOEXEC and strnlen. +if (CMAKE_SYSTEM_NAME MATCHES "OpenBSD|FreeBSD") add_compile_definitions(_XOPEN_SOURCE=700) elseif (CMAKE_SYSTEM_NAME MATCHES "AIX") # Don't define _XOPEN_SOURCE. We need _ALL_SOURCE, which is the default, @@ -216,6 +210,12 @@ set_target_properties(ggml-base PROPERTIES configure_file(ggml-version.h.in ${CMAKE_CURRENT_BINARY_DIR}/ggml-version.h @ONLY) target_include_directories(ggml-base PRIVATE . ${CMAKE_CURRENT_BINARY_DIR}) + +# icx/icpx default to -fp-model=fast, which contracts and reassociates the reference quantizers, so their output +# depends on the compiler; keep ggml-base IEEE-exact so the CPU reference matches gcc/clang builds +if (CMAKE_C_COMPILER_ID STREQUAL "IntelLLVM") + target_compile_options(ggml-base PRIVATE -fp-model=precise) +endif() if (GGML_BACKEND_DL) target_compile_definitions(ggml-base PUBLIC GGML_BACKEND_DL) endif() diff --git a/ggml/src/ggml-alloc.c b/ggml/src/ggml-alloc.c index b0c067b49135..65d2bea27726 100644 --- a/ggml/src/ggml-alloc.c +++ b/ggml/src/ggml-alloc.c @@ -23,11 +23,8 @@ bool ggml_op_can_inplace(enum ggml_op op) { switch (op) { case GGML_OP_FILL: case GGML_OP_SCALE: - case GGML_OP_DIAG_MASK_ZERO: - case GGML_OP_DIAG_MASK_INF: case GGML_OP_ADD: case GGML_OP_ADD_ID: - case GGML_OP_ADD1: case GGML_OP_SUB: case GGML_OP_MUL: case GGML_OP_DIV: diff --git a/ggml/src/ggml-backend-meta.cpp b/ggml/src/ggml-backend-meta.cpp index 2641e1f3a87e..595a5824aafe 100644 --- a/ggml/src/ggml-backend-meta.cpp +++ b/ggml/src/ggml-backend-meta.cpp @@ -489,6 +489,32 @@ static struct ggml_tensor * ggml_backend_meta_buffer_simple_tensor(const struct return it->second[index]; } +// bytes spanned by all dims except skip_dim +static size_t ggml_backend_meta_inner_extent(const struct ggml_tensor * tensor, int skip_dim) { + size_t extent = 0; + for (int i = 0; i < GGML_MAX_DIMS; i++) { + if (i != skip_dim && tensor->ne[i] > 1) { + extent += (size_t) (tensor->ne[i] - 1) * tensor->nb[i]; + } + } + return extent; +} + +// the view dim that holds the split axis of src, or -1 +// all other dims of src and view must stay inside one slab of that axis +static int ggml_backend_meta_view_slab_dim(const struct ggml_tensor * view, const struct ggml_tensor * src, int axis) { + const size_t slab = src->nb[axis]; + if (ggml_backend_meta_inner_extent(src, axis) >= slab) { + return -1; + } + for (int dim = 0; dim < GGML_MAX_DIMS; dim++) { + if (view->ne[dim] == src->ne[axis] && view->nb[dim] == slab) { + return ggml_backend_meta_inner_extent(view, dim) < slab ? dim : -1; + } + } + return -1; +} + static struct ggml_backend_meta_split_state ggml_backend_meta_get_split_state(const struct ggml_tensor * tensor, bool assume_sync); static struct ggml_backend_meta_split_state ggml_backend_meta_get_split_state( @@ -700,9 +726,39 @@ static struct ggml_backend_meta_split_state ggml_backend_meta_get_split_state( } GGML_ABORT("fatal error"); } + // permuted view: split the view dim that spans the src split axis, other dims must be inside one element of it or outside the span + if (ggml_is_permuted(tensor) && !ggml_is_permuted(tensor->src[0]) && axis >= 0 && axis < GGML_MAX_DIMS-1) { + const size_t nb_axis = tensor->src[0]->nb[axis]; + const size_t nb_next = tensor->src[0]->nb[axis+1]; + for (int dim = 0; dim < GGML_MAX_DIMS; dim++) { + if (tensor->ne[dim] == 1 || tensor->nb[dim] < nb_axis || tensor->nb[dim]*tensor->ne[dim] != nb_next) { + continue; + } + bool others_ok = true; + for (int i = 0; i < GGML_MAX_DIMS; i++) { + if (i == dim || tensor->ne[i] == 1) { + continue; + } + others_ok = others_ok && (tensor->nb[i]*tensor->ne[i] <= tensor->nb[dim] || tensor->nb[i] % nb_next == 0); + } + if (others_ok) { + return {ggml_backend_meta_split_axis(dim), {0}, {1}, 1}; + } + } + } if (src_ss[0].axis == GGML_BACKEND_SPLIT_AXIS_MIRRORED || src_ss[0].axis == GGML_BACKEND_SPLIT_AXIS_PARTIAL) { return src_ss[0]; } + // a view that only regroups dims inside one slab of the split axis keeps the split, + // the per-device strides are only rescaled above nb[axis] (e.g. the KV streams of a permuted q) + if (axis >= 0 && axis < GGML_MAX_DIMS && tensor->view_offs == tensor->src[0]->view_offs) { + const int dim = ggml_backend_meta_view_slab_dim(tensor, tensor->src[0], axis); + if (dim >= 0) { + ggml_backend_meta_split_state ret = src_ss[0]; + ret.axis = ggml_backend_meta_split_axis(dim); + return ret; + } + } GGML_ABORT("view of permuted tensor not implemented"); //return {GGML_BACKEND_SPLIT_AXIS_UNKNOWN, {0}, {1}, 1}; }; @@ -883,7 +939,6 @@ static struct ggml_backend_meta_split_state ggml_backend_meta_get_split_state( case GGML_OP_ADD_ID: { split_state = handle_bin_bcast(src_ss); } break; - case GGML_OP_ADD1: case GGML_OP_ACC: { split_state = handle_generic(src_ss, /*scalar_only =*/ true); } break; @@ -964,9 +1019,7 @@ static struct ggml_backend_meta_split_state ggml_backend_meta_get_split_state( case GGML_OP_SET_ROWS: { split_state = handle_set_rows(src_ss); } break; - case GGML_OP_DIAG: - case GGML_OP_DIAG_MASK_INF: - case GGML_OP_DIAG_MASK_ZERO: { + case GGML_OP_DIAG: { split_state = handle_generic(src_ss, /*scalar_only =*/ true); } break; case GGML_OP_SOFT_MAX: @@ -1000,8 +1053,13 @@ static struct ggml_backend_meta_split_state ggml_backend_meta_get_split_state( split_state = handle_pad(src_ss); } break; case GGML_OP_PAD_REFLECT_1D: - case GGML_OP_ROLL: - case GGML_OP_ARANGE: + case GGML_OP_ROLL: { + split_state = handle_generic(src_ss, /*scalar_only =*/ true); + } break; + case GGML_OP_ARANGE: { + // no sources, every device computes the full range + split_state = {GGML_BACKEND_SPLIT_AXIS_MIRRORED, {0}, {1}, 1}; + } break; case GGML_OP_TIMESTEP_EMBEDDING: { split_state = handle_generic(src_ss, /*scalar_only =*/ true); } break; @@ -1747,13 +1805,14 @@ static ggml_backend_buffer_t ggml_backend_meta_buffer_type_alloc_buffer_n(ggml_b const size_t n_simple_bufts = ggml_backend_meta_buft_n_bufts(buft); constexpr size_t compute_headroom = 16; // Maximum number of views per statically allocated tensor that can be created between evals. + constexpr size_t compute_headroom_min = 256; // A buffer with few tensors can still get many views, e.g. a KV cache shared by many layers. const ggml_init_params params_static = { /*.mem_size =*/ n_tensors * ggml_tensor_overhead(), /*.mem_buffer =*/ nullptr, /*.no_alloc =*/ true, }; const ggml_init_params params_compute = { - /*.mem_size =*/ compute_headroom * n_tensors * ggml_tensor_overhead(), + /*.mem_size =*/ std::max(compute_headroom * n_tensors, compute_headroom_min) * ggml_tensor_overhead(), /*.mem_buffer =*/ nullptr, /*.no_alloc =*/ true, }; diff --git a/ggml/src/ggml-backend-reg.cpp b/ggml/src/ggml-backend-reg.cpp index 4e9681c9f5c3..c3bc92679ffd 100644 --- a/ggml/src/ggml-backend-reg.cpp +++ b/ggml/src/ggml-backend-reg.cpp @@ -575,21 +575,22 @@ void ggml_backend_load_all_from_path(const char * dir_path) { bool silent = false; #endif - ggml_backend_load_best("blas", silent, dir_path); - ggml_backend_load_best("zendnn", silent, dir_path); - ggml_backend_load_best("cann", silent, dir_path); - ggml_backend_load_best("cuda", silent, dir_path); - ggml_backend_load_best("hip", silent, dir_path); - ggml_backend_load_best("metal", silent, dir_path); - ggml_backend_load_best("rpc", silent, dir_path); - ggml_backend_load_best("sycl", silent, dir_path); - ggml_backend_load_best("vulkan", silent, dir_path); - ggml_backend_load_best("virtgpu", silent, dir_path); - ggml_backend_load_best("opencl", silent, dir_path); - ggml_backend_load_best("hexagon", silent, dir_path); - ggml_backend_load_best("musa", silent, dir_path); + ggml_backend_load_best("blas", silent, dir_path); + ggml_backend_load_best("zendnn", silent, dir_path); // AMD ZenDNN backend + ggml_backend_load_best("zdnn", silent, dir_path); // IBM zDNN backend + ggml_backend_load_best("cann", silent, dir_path); + ggml_backend_load_best("cuda", silent, dir_path); + ggml_backend_load_best("hip", silent, dir_path); + ggml_backend_load_best("metal", silent, dir_path); + ggml_backend_load_best("rpc", silent, dir_path); + ggml_backend_load_best("sycl", silent, dir_path); + ggml_backend_load_best("vulkan", silent, dir_path); + ggml_backend_load_best("virtgpu", silent, dir_path); + ggml_backend_load_best("opencl", silent, dir_path); + ggml_backend_load_best("hexagon", silent, dir_path); + ggml_backend_load_best("musa", silent, dir_path); ggml_backend_load_best("openvino", silent, dir_path); - ggml_backend_load_best("cpu", silent, dir_path); + ggml_backend_load_best("cpu", silent, dir_path); // check the environment variable GGML_BACKEND_PATH to load an out-of-tree backend const char * backend_path = std::getenv("GGML_BACKEND_PATH"); if (backend_path) { diff --git a/ggml/src/ggml-backend.cpp b/ggml/src/ggml-backend.cpp index 3520b434918d..b70d0d8741e9 100644 --- a/ggml/src/ggml-backend.cpp +++ b/ggml/src/ggml-backend.cpp @@ -1195,8 +1195,9 @@ struct ggml_backend_sched { // hash map of the nodes in the graph struct ggml_hash_set hash_set; - int * hv_tensor_backend_ids; // [hash_set.size] - struct ggml_tensor ** hv_tensor_copies; // [hash_set.size][n_backends][n_copies] + int * hv_tensor_backend_ids; // [hash_set.size] + struct ggml_tensor ** hv_tensor_copies; // [hash_set.size][n_backends][n_copies] + bool * hv_tensor_requires_copy; // [hash_set.size] decides if input needs to be copied to each split int * node_backend_ids; // [graph_size] int * leaf_backend_ids; // [graph_size] @@ -1246,6 +1247,14 @@ struct ggml_backend_sched { int prev_graph_size; int debug_realloc; + + // inputs of the next split read from the backend of the current split before it computes, see ggml_backend_sched_prefetch_inputs + ggml_backend_event_t prefetch_events[GGML_SCHED_MAX_BACKENDS]; // created on first use, NULL if the backend has no events + bool prefetch_tried [GGML_SCHED_MAX_BACKENDS]; + struct ggml_tensor * prefetched[GGML_SCHED_MAX_SPLIT_INPUTS]; + int n_prefetched; + int prefetch_split; // the split that reads the prefetched inputs + int prefetch_backend_id; // the backend that copies them }; #define hash_id(tensor) ggml_hash_find_or_insert(&sched->hash_set, tensor) @@ -1800,6 +1809,11 @@ void ggml_backend_sched_split_graph(ggml_backend_sched_t sched, struct ggml_cgra } split->inputs[n_inputs] = src; } + // inputs of nodes that are not computed (e.g. the unselected branches of ggml_build_forward_select) + // stay in the split so that the allocation does not change, but they do not need to be copied + if (node->flags & GGML_TENSOR_FLAG_COMPUTE) { + sched->hv_tensor_requires_copy[src_id] = true; + } node->src[j] = tensor_id_copy(src_id, cur_backend_id, sched->cur_copy); } } @@ -2086,6 +2100,16 @@ static void ggml_backend_sched_copy_input(ggml_backend_sched_t sched, struct ggm ggml_backend_t input_backend = ggml_backend_sched_get_tensor_backend(sched, input); struct ggml_tensor * input_cpy = tensor_copy(input, split_backend_id, sched->cur_copy); + // read before the previous split, wait for the copy only + if (split == &sched->splits[sched->prefetch_split]) { + for (int i = 0; i < sched->n_prefetched; i++) { + if (sched->prefetched[i] == input) { + ggml_backend_event_synchronize(sched->prefetch_events[sched->prefetch_backend_id]); + return; + } + } + } + if (input->flags & GGML_TENSOR_FLAG_INPUT) { // inputs from the user must be copied immediately to prevent the user overwriting the data before the copy is done if (sched->events[split_backend_id][sched->cur_copy] != NULL) { @@ -2122,6 +2146,67 @@ static void ggml_backend_sched_copy_input(ggml_backend_sched_t sched, struct ggm } } +// the inputs of the next split that are computed on the backend of this split before it, and copied to host memory, +// are copied before this split computes, so that the next split waits for these copies instead of this split +// e.g. a CPU split runs while a GPU split computes, if it reads only the results of the earlier splits +static void ggml_backend_sched_prefetch_inputs(ggml_backend_sched_t sched, int split_id) { + sched->n_prefetched = 0; + if (split_id + 1 >= sched->n_splits) { + return; + } + struct ggml_backend_sched_split * split = &sched->splits[split_id]; + struct ggml_backend_sched_split * next = &sched->splits[split_id + 1]; + const int backend_id = split->backend_id; + ggml_backend_t backend = sched->backends[backend_id]; + if (backend->iface.get_tensor_async == NULL) { + return; + } + + for (int i = 0; i < next->n_inputs && sched->n_prefetched < GGML_SCHED_MAX_SPLIT_INPUTS; i++) { + struct ggml_tensor * input = next->inputs[i]; + if ((input->flags & GGML_TENSOR_FLAG_INPUT) || tensor_backend_id(input) != backend_id) { + continue; + } + struct ggml_tensor * input_cpy = tensor_copy(input, next->backend_id, sched->cur_copy); + if (input_cpy->buffer == NULL || !ggml_backend_buffer_is_host(input_cpy->buffer)) { + continue; + } + + // only results of earlier splits, the copy callback of this split runs later and can write other tensors (e.g. the MoE cache) + const struct ggml_tensor * base = input->view_src ? input->view_src : input; + if (base->buffer == NULL || ggml_backend_buffer_get_usage(base->buffer) != GGML_BACKEND_BUFFER_USAGE_COMPUTE) { + continue; + } + + // not if this split computes or writes the input + bool written = false; + for (int j = 0; j < split->graph.n_nodes && !written; j++) { + const struct ggml_tensor * node = split->graph.nodes[j]; + written = !ggml_is_view_op(node->op) && (node == base || node->view_src == base); + } + if (written) { + continue; + } + + if (!sched->prefetch_tried[backend_id]) { + sched->prefetch_tried[backend_id] = true; + sched->prefetch_events[backend_id] = ggml_backend_event_new(backend->device); + } + if (sched->prefetch_events[backend_id] == NULL) { + return; + } + + ggml_backend_tensor_get_async(backend, input, input_cpy->data, 0, ggml_nbytes(input)); + sched->prefetched[sched->n_prefetched++] = input; + } + + if (sched->n_prefetched > 0) { + sched->prefetch_split = split_id + 1; + sched->prefetch_backend_id = backend_id; + ggml_backend_event_record(sched->prefetch_events[backend_id], backend); + } +} + static enum ggml_status ggml_backend_sched_compute_splits(ggml_backend_sched_t sched) { GGML_ASSERT(sched); struct ggml_backend_sched_split * splits = sched->splits; @@ -2146,13 +2231,18 @@ static enum ggml_status ggml_backend_sched_compute_splits(ggml_backend_sched_t s // copy the input tensors to the split backend // the weights in host memory are copied last, so that the copy callback can read the other inputs of the split for (int input_id = 0; input_id < split->n_inputs; input_id++) { - if (!ggml_backend_sched_is_host_weight(split->inputs[input_id])) { - ggml_backend_sched_copy_input(sched, split, split->inputs[input_id]); + struct ggml_tensor * input = split->inputs[input_id]; + if (!ggml_backend_sched_is_host_weight(input) && sched->hv_tensor_requires_copy[hash_id(input)]) { + ggml_backend_sched_copy_input(sched, split, input); } } + // after the copies above, which may read host memory reused by the copies of the next split + // before the weights, whose copy callback may queue large uploads + ggml_backend_sched_prefetch_inputs(sched, split_id); for (int input_id = 0; input_id < split->n_inputs; input_id++) { - if (ggml_backend_sched_is_host_weight(split->inputs[input_id])) { - ggml_backend_sched_copy_input(sched, split, split->inputs[input_id]); + struct ggml_tensor * input = split->inputs[input_id]; + if (ggml_backend_sched_is_host_weight(input) && sched->hv_tensor_requires_copy[hash_id(input)]) { + ggml_backend_sched_copy_input(sched, split, input); } } @@ -2235,8 +2325,9 @@ ggml_backend_sched_t ggml_backend_sched_new( // initialize hash table // FIXME: needs to be size*2 to account for leafs (do it in graph_split instead) sched->hash_set = ggml_hash_set_new(graph_size); - sched->hv_tensor_backend_ids = (int *) malloc(sched->hash_set.size * sizeof(sched->hv_tensor_backend_ids[0])); - sched->hv_tensor_copies = (ggml_tensor **) malloc(sched->hash_set.size * sched->n_backends * sched->n_copies * sizeof(struct ggml_tensor *)); + sched->hv_tensor_backend_ids = (int *) malloc(sched->hash_set.size * sizeof(sched->hv_tensor_backend_ids[0])); + sched->hv_tensor_copies = (ggml_tensor **) malloc(sched->hash_set.size * sched->n_backends * sched->n_copies * sizeof(struct ggml_tensor *)); + sched->hv_tensor_requires_copy = (bool *) malloc(sched->hash_set.size * sizeof(sched->hv_tensor_requires_copy[0])); const size_t ggml_sched_max_splits = graph_size; // at most there is one split for each node in the graph const size_t nodes_size = graph_size + ggml_sched_max_splits*GGML_SCHED_MAX_SPLIT_INPUTS*2; @@ -2286,6 +2377,7 @@ void ggml_backend_sched_free(ggml_backend_sched_t sched) { for (int c = 0; c < sched->n_copies; c++) { ggml_backend_event_free(sched->events[b][c]); } + ggml_backend_event_free(sched->prefetch_events[b]); } ggml_gallocr_free(sched->galloc); ggml_free(sched->ctx); @@ -2297,6 +2389,7 @@ void ggml_backend_sched_free(ggml_backend_sched_t sched) { free(sched->graph_inputs); free(sched->hv_tensor_backend_ids); free(sched->hv_tensor_copies); + free(sched->hv_tensor_requires_copy); free(sched->node_backend_ids); free(sched->leaf_backend_ids); free(sched->prev_node_backend_ids); @@ -2312,8 +2405,9 @@ void ggml_backend_sched_reset(ggml_backend_sched_t sched) { // reset state for the next run if (!sched->is_reset) { ggml_hash_set_reset(&sched->hash_set); - memset(sched->hv_tensor_backend_ids, -1, sched->hash_set.size * sizeof(sched->hv_tensor_backend_ids[0])); - memset(sched->hv_tensor_copies, 0, sched->hash_set.size * sched->n_backends * sched->n_copies * sizeof(struct ggml_tensor *)); + memset(sched->hv_tensor_backend_ids, -1, sched->hash_set.size * sizeof(sched->hv_tensor_backend_ids[0])); + memset(sched->hv_tensor_copies, 0, sched->hash_set.size * sched->n_backends * sched->n_copies * sizeof(struct ggml_tensor *)); + memset(sched->hv_tensor_requires_copy, 0, sched->hash_set.size * sizeof(sched->hv_tensor_requires_copy[0])); sched->is_reset = true; } sched->is_alloc = false; diff --git a/ggml/src/ggml-cann/aclnn_ops.cpp b/ggml/src/ggml-cann/aclnn_ops.cpp index 902d2eda6938..e602e4bf3b11 100644 --- a/ggml/src/ggml-cann/aclnn_ops.cpp +++ b/ggml/src/ggml-cann/aclnn_ops.cpp @@ -1347,31 +1347,6 @@ void ggml_cann_rms_norm(ggml_backend_cann_context & ctx, ggml_tensor * dst) { GGML_CANN_CALL_ACLNN_OP(ctx, RmsNorm, acl_src.get(), acl_gamma.get(), eps, acl_dst.get(), acl_rstd.get()); } -// TODO: performace is low. -void ggml_cann_diag_mask(ggml_backend_cann_context & ctx, ggml_tensor * dst, float value) { - ggml_tensor * src = dst->src[0]; - - acl_tensor_ptr acl_src = ggml_cann_create_tensor(src); - acl_tensor_ptr acl_dst = ggml_cann_create_tensor(dst); - - const int n_past = ((int32_t *) dst->op_params)[0]; - - ggml_cann_pool_alloc one_tensor_allocator(ctx.pool(), ggml_nbytes(src)); - void * buffer = one_tensor_allocator.get(); - - acl_tensor_ptr mask_tensor = ggml_cann_create_tensor(buffer, ggml_cann_type_mapping(src->type), - ggml_type_size(src->type), src->ne, src->nb, GGML_MAX_DIMS); - - aclnn_fill_scalar(ctx, value, mask_tensor.get()); - - float alphaValue = 1.0f; - acl_scalar_ptr alpha = ggml_cann_create_scalar(&alphaValue, aclDataType::ACL_FLOAT); - - GGML_CANN_CALL_ACLNN_OP(ctx, InplaceTriu, mask_tensor.get(), n_past + 1); - GGML_CANN_CALL_ACLNN_OP(ctx, Tril, acl_src.get(), n_past + 1, acl_dst.get()); - GGML_CANN_CALL_ACLNN_OP(ctx, InplaceAdd, acl_dst.get(), mask_tensor.get(), alpha.get()); -} - /** * @brief Permutes the dimensions of a tensor according to a specified order. * @@ -2934,7 +2909,7 @@ void ggml_cann_rope(ggml_backend_cann_context & ctx, ggml_tensor * dst) { const float theta_scale = powf(freq_base, -2.0f / n_dims); float corr_dims[2]; - ggml_rope_yarn_corr_dims(n_dims, n_ctx_orig, freq_base, beta_fast, beta_slow, corr_dims); + ggml_rope_yarn_corr_dims(n_dims, n_ctx_orig, freq_base, beta_fast, beta_slow, true, corr_dims); bool is_neox = mode & GGML_ROPE_TYPE_NEOX; const bool is_imrope = mode == GGML_ROPE_TYPE_IMROPE; // qwen3vl apply interleaved mrope @@ -3289,7 +3264,7 @@ void ggml_cann_rope_cache_preload(ggml_backend_cann_context & ctx, ggml_tensor * const float theta_scale = powf(freq_base, -2.0f / n_dims); float corr_dims[2]; - ggml_rope_yarn_corr_dims(n_dims, n_ctx_orig, freq_base, beta_fast, beta_slow, corr_dims); + ggml_rope_yarn_corr_dims(n_dims, n_ctx_orig, freq_base, beta_fast, beta_slow, true, corr_dims); bool is_neox = mode & GGML_ROPE_TYPE_NEOX; const bool is_imrope = mode == GGML_ROPE_TYPE_IMROPE; diff --git a/ggml/src/ggml-cann/aclnn_ops.h b/ggml/src/ggml-cann/aclnn_ops.h index 678f4d654e7f..f470042cf7af 100644 --- a/ggml/src/ggml-cann/aclnn_ops.h +++ b/ggml/src/ggml-cann/aclnn_ops.h @@ -466,21 +466,6 @@ void ggml_cann_dup(ggml_backend_cann_context & ctx, ggml_tensor * dst); */ void ggml_cann_rms_norm(ggml_backend_cann_context & ctx, ggml_tensor * dst); -/** - * @brief Applies a diagonal mask to the tensor with a specified value. - * - * @details This function creates a mask tensor filled with ones, then applies - * an upper triangular and lower triangular operation to it based on - * the number of past elements specified. Afterward, it adds the masked - * tensor to the destination tensor in-place. - * - * @param ctx The backend CANN context used for operations. - * @param dst The destination tensor where the result will be stored. dst->op is - * `GGML_OP_DIAG_MASK` - * @param value The value to use for masking. - */ -void ggml_cann_diag_mask(ggml_backend_cann_context & ctx, ggml_tensor * dst, float value); - /** * @brief Performs an image-to-column transformation on the input tensor. * diff --git a/ggml/src/ggml-cann/ggml-cann.cpp b/ggml/src/ggml-cann/ggml-cann.cpp index d3a47f83946c..0bfb960095c5 100644 --- a/ggml/src/ggml-cann/ggml-cann.cpp +++ b/ggml/src/ggml-cann/ggml-cann.cpp @@ -1398,34 +1398,35 @@ static void ggml_backend_cann_buffer_get_tensor(ggml_backend_buffer_t buffer, static bool ggml_backend_cann_buffer_cpy_tensor(ggml_backend_buffer_t buffer, const ggml_tensor * src, ggml_tensor * dst) { - if (ggml_backend_buft_is_cann(src->buffer->buft)) { - ggml_backend_cann_buffer_context * src_ctx = (ggml_backend_cann_buffer_context *) src->buffer->context; - ggml_backend_cann_buffer_context * dst_ctx = (ggml_backend_cann_buffer_context *) buffer->context; + if (!ggml_backend_buft_is_cann(src->buffer->buft)) { + return false; + } - size_t memcpy_size = ggml_nbytes(src); - // Same device. - if (src_ctx->device == dst_ctx->device) { - ACL_CHECK(aclrtMemcpy((char *) dst->data, memcpy_size, (const char *) src->data, memcpy_size, - ACL_MEMCPY_DEVICE_TO_DEVICE)); - return true; - } else { + ggml_backend_cann_buffer_context * src_ctx = (ggml_backend_cann_buffer_context *) src->buffer->context; + ggml_backend_cann_buffer_context * dst_ctx = (ggml_backend_cann_buffer_context *) buffer->context; + + size_t memcpy_size = ggml_nbytes(src); + // Same device. + if (src_ctx->device == dst_ctx->device) { + ACL_CHECK(aclrtMemcpy((char *) dst->data, memcpy_size, (const char *) src->data, memcpy_size, + ACL_MEMCPY_DEVICE_TO_DEVICE)); + return true; + } else { #ifdef ASCEND_310P - // TODO: Support 310p P2P copy - return false; + // TODO: Support 310p P2P copy + return false; #endif - // Different device but can access by peer. - int32_t canAccessPeer = 0; - ACL_CHECK(aclrtDeviceCanAccessPeer(&canAccessPeer, src_ctx->device, dst_ctx->device)); - if (canAccessPeer) { - ggml_cann_set_device(src_ctx->device); - ACL_CHECK(aclrtDeviceEnablePeerAccess(dst_ctx->device, 0)); - ACL_CHECK(aclrtMemcpy((char *) dst->data, memcpy_size, (const char *) src->data, memcpy_size, - ACL_MEMCPY_DEVICE_TO_DEVICE)); - return true; - } + // Different device but can access by peer. + int32_t canAccessPeer = 0; + ACL_CHECK(aclrtDeviceCanAccessPeer(&canAccessPeer, src_ctx->device, dst_ctx->device)); + if (canAccessPeer) { + ggml_cann_set_device(src_ctx->device); + ACL_CHECK(aclrtDeviceEnablePeerAccess(dst_ctx->device, 0)); + ACL_CHECK(aclrtMemcpy((char *) dst->data, memcpy_size, (const char *) src->data, memcpy_size, + ACL_MEMCPY_DEVICE_TO_DEVICE)); + return true; } } - return false; } /** @@ -1792,7 +1793,6 @@ static bool ggml_cann_compute_forward(ggml_backend_cann_context & ctx, struct gg ggml_cann_dup(ctx, dst); break; case GGML_OP_ADD: - case GGML_OP_ADD1: ggml_cann_binary_op(ctx, dst); break; case GGML_OP_SUB: @@ -1957,9 +1957,6 @@ static bool ggml_cann_compute_forward(ggml_backend_cann_context & ctx, struct gg case GGML_OP_PERMUTE: case GGML_OP_TRANSPOSE: break; - case GGML_OP_DIAG_MASK_INF: - ggml_cann_diag_mask(ctx, dst, -INFINITY); - break; case GGML_OP_SOFT_MAX: ggml_cann_softmax(ctx, dst); break; @@ -2446,6 +2443,9 @@ static bool ggml_backend_cann_supports_op(ggml_backend_dev_t dev, const ggml_ten break; case GGML_OP_MUL_MAT: { + if (ggml_prec_get_src(op, 1) == GGML_PREC_BF16 || ggml_prec_get_src(op, 1) == GGML_PREC_F32) { + return false; + } switch (op->src[0]->type) { #ifndef ASCEND_310P case GGML_TYPE_BF16: @@ -2547,6 +2547,9 @@ static bool ggml_backend_cann_supports_op(ggml_backend_dev_t dev, const ggml_ten } case GGML_OP_ROPE: { + if (!ggml_rope_get_truncate(op)) { + return false; + } if (((const int32_t *) op->op_params)[15] != 0) { return false; // FIXME: support ggml_rope_set_offset } @@ -2611,7 +2614,6 @@ static bool ggml_backend_cann_supports_op(ggml_backend_dev_t dev, const ggml_ten case GGML_OP_TRANSPOSE: case GGML_OP_NORM: case GGML_OP_ADD: - case GGML_OP_ADD1: case GGML_OP_SUB: case GGML_OP_MUL: case GGML_OP_DIV: @@ -2619,7 +2621,6 @@ static bool ggml_backend_cann_supports_op(ggml_backend_dev_t dev, const ggml_ten case GGML_OP_SQR: case GGML_OP_SQRT: case GGML_OP_CLAMP: - case GGML_OP_DIAG_MASK_INF: case GGML_OP_SUM_ROWS: case GGML_OP_ARGSORT: case GGML_OP_ACC: @@ -2861,12 +2862,13 @@ static ggml_backend_t ggml_backend_cann_device_init(ggml_backend_dev_t dev, cons * otherwise false. */ static bool ggml_backend_cann_supports_buft(ggml_backend_dev_t dev, ggml_backend_buffer_type_t buft) { - if (ggml_backend_buft_is_cann(buft)) { - ggml_backend_cann_device_context * dev_ctx = (ggml_backend_cann_device_context *) dev->context; - ggml_backend_cann_buffer_type_context * buft_ctx = (ggml_backend_cann_buffer_type_context *) buft->context; - return buft_ctx->device == dev_ctx->device; + if (!ggml_backend_buft_is_cann(buft)) { + return false; } - return false; + + ggml_backend_cann_device_context * dev_ctx = (ggml_backend_cann_device_context *) dev->context; + ggml_backend_cann_buffer_type_context * buft_ctx = (ggml_backend_cann_buffer_type_context *) buft->context; + return buft_ctx->device == dev_ctx->device; } static ggml_backend_buffer_type_t ggml_backend_cann_device_get_buffer_type(ggml_backend_dev_t dev) { diff --git a/ggml/src/ggml-cpu/arch/riscv/repack.cpp b/ggml/src/ggml-cpu/arch/riscv/repack.cpp index c37488cae545..49797df85485 100644 --- a/ggml/src/ggml-cpu/arch/riscv/repack.cpp +++ b/ggml/src/ggml-cpu/arch/riscv/repack.cpp @@ -29,7 +29,7 @@ void ggml_quantize_mat_q8_0_4x8(const float * GGML_RESTRICT x, void * GGML_RESTR assert(k % QK8_0 == 0); const int nb = k / QK8_0; -#if defined(__riscv_v_intrinsic) +#if defined(__riscv_v) block_q8_0x4 * GGML_RESTRICT y = (block_q8_0x4 *) vy; const size_t vl_calc = __riscv_vsetvl_e32m8(QK8_0); const size_t vl_save = __riscv_vsetvl_e64m2(4); diff --git a/ggml/src/ggml-cpu/ggml-cpu-impl.h b/ggml/src/ggml-cpu/ggml-cpu-impl.h index 865847a7785f..f8b363728b24 100644 --- a/ggml/src/ggml-cpu/ggml-cpu-impl.h +++ b/ggml/src/ggml-cpu/ggml-cpu-impl.h @@ -345,7 +345,7 @@ static inline int32x4_t ggml_nvfp4_dot8(const int8x8_t q4_lo, const int8x8_t q8_ #include #endif -#ifdef __riscv_v_intrinsic +#if defined(__riscv_v) #include #endif diff --git a/ggml/src/ggml-cpu/ggml-cpu.c b/ggml/src/ggml-cpu/ggml-cpu.c index f4eaea5dfe95..a76af8487e11 100644 --- a/ggml/src/ggml-cpu/ggml-cpu.c +++ b/ggml/src/ggml-cpu/ggml-cpu.c @@ -224,8 +224,8 @@ static const struct ggml_type_traits_cpu type_traits_cpu[GGML_TYPE_COUNT] = { }, [GGML_TYPE_F16] = { .from_float = (ggml_from_float_t) ggml_cpu_fp32_to_fp16, - .vec_dot = (ggml_vec_dot_t) ggml_vec_dot_f16, - .vec_dot_type = GGML_TYPE_F16, + .vec_dot = (ggml_vec_dot_t) ggml_vec_dot_f16_f32, + .vec_dot_type = GGML_TYPE_F32, .nrows = 1, }, [GGML_TYPE_Q1_0] = { @@ -746,7 +746,7 @@ static void ggml_init_arm_arch_features(void) {} #endif #endif // __ARM_ARCH -#if defined(__riscv) && defined(__riscv_v_intrinsic) +#if defined(__riscv) && defined(__riscv_v) #include static void ggml_init_riscv_arch_features(void) { ggml_riscv_arch_features.rvv_vlen = __riscv_vlenb(); @@ -1284,9 +1284,12 @@ void ggml_compute_forward_mul_mat( const int ith = params->ith; const int nth = params->nth; - enum ggml_type const vec_dot_type = type_traits_cpu[src0->type].vec_dot_type; - ggml_from_float_t const from_float = type_traits_cpu[vec_dot_type].from_float; - int64_t const vec_dot_num_rows = type_traits_cpu[src0->type].nrows; + enum ggml_type const vec_dot_type = type_traits_cpu[src0->type].vec_dot_type; + ggml_from_float_t const from_float = type_traits_cpu[vec_dot_type].from_float; + int64_t const vec_dot_num_rows = type_traits_cpu[src0->type].nrows; + + // TODO: a more efficient F16 x F16 path can be used here if ggml_prec_get_src(dst, 1) allows converting src1 to F16 + // the code needs to be refactored to enable this (src1 is currently converted based on vec_dot_type alone) GGML_ASSERT(ne0 == ne01); GGML_ASSERT(ne1 == ne11); @@ -1509,8 +1512,8 @@ static void ggml_compute_forward_mul_mat_id_one_chunk( const enum ggml_type type = src0->type; - ggml_vec_dot_t const vec_dot = type_traits_cpu[type].vec_dot; - enum ggml_type const vec_dot_type = type_traits_cpu[type].vec_dot_type; + ggml_vec_dot_t const vec_dot = type_traits_cpu[type].vec_dot; + enum ggml_type const vec_dot_type = type_traits_cpu[type].vec_dot_type; const int64_t blck_0 = 16; const int64_t blck_1 = 16; @@ -1577,8 +1580,11 @@ static void ggml_compute_forward_mul_mat_id( const bool src1_cont = ggml_is_contiguous(src1); - enum ggml_type const vec_dot_type = type_traits_cpu[type].vec_dot_type; - ggml_from_float_t const from_float = type_traits_cpu[vec_dot_type].from_float; + enum ggml_type const vec_dot_type = type_traits_cpu[type].vec_dot_type; + ggml_from_float_t const from_float = type_traits_cpu[vec_dot_type].from_float; + + // TODO: a more efficient F16 x F16 path can be used here if ggml_prec_get_src(dst, 1) allows converting src1 to F16 + // the code needs to be refactored to enable this (src1 is currently converted based on vec_dot_type alone) // we don't support permuted src0 or src1 GGML_ASSERT(nb00 == ggml_type_size(type)); @@ -1776,10 +1782,6 @@ static void ggml_compute_forward(struct ggml_compute_params * params, struct ggm { ggml_compute_forward_add_id(params, tensor); } break; - case GGML_OP_ADD1: - { - ggml_compute_forward_add1(params, tensor); - } break; case GGML_OP_ACC: { ggml_compute_forward_acc(params, tensor); @@ -1920,14 +1922,6 @@ static void ggml_compute_forward(struct ggml_compute_params * params, struct ggm { ggml_compute_forward_diag(params, tensor); } break; - case GGML_OP_DIAG_MASK_INF: - { - ggml_compute_forward_diag_mask_inf(params, tensor); - } break; - case GGML_OP_DIAG_MASK_ZERO: - { - ggml_compute_forward_diag_mask_zero(params, tensor); - } break; case GGML_OP_SOFT_MAX: { ggml_compute_forward_soft_max(params, tensor); @@ -2279,7 +2273,6 @@ static int ggml_get_n_tasks(struct ggml_tensor * node, int n_threads) { case GGML_OP_CONT: case GGML_OP_ADD: case GGML_OP_ADD_ID: - case GGML_OP_ADD1: case GGML_OP_ACC: case GGML_OP_CUMSUM: case GGML_OP_TRI: @@ -2401,8 +2394,6 @@ static int ggml_get_n_tasks(struct ggml_tensor * node, int n_threads) { { n_tasks = 1; } break; - case GGML_OP_DIAG_MASK_ZERO: - case GGML_OP_DIAG_MASK_INF: case GGML_OP_SOFT_MAX_BACK: case GGML_OP_ROPE: case GGML_OP_ROPE_BACK: @@ -2883,7 +2874,6 @@ struct ggml_cplan ggml_graph_plan( } break; case GGML_OP_ADD: case GGML_OP_ADD_ID: - case GGML_OP_ADD1: { if (ggml_is_quantized(node->src[0]->type) || node->src[0]->type == GGML_TYPE_F8_E4M3) { cur = ggml_type_size(GGML_TYPE_F32) * node->src[0]->ne[0] * n_tasks; @@ -2965,8 +2955,8 @@ struct ggml_cplan ggml_graph_plan( if ((node->src[0]->type == GGML_TYPE_F16 || node->src[0]->type == GGML_TYPE_BF16) && node->src[1]->type == GGML_TYPE_F32) { + cur += sizeof(float)*ne10*ne11; cur += sizeof(ggml_fp16_t)*ne00*ne01*ne02; - cur += sizeof(ggml_fp16_t)*ne10*ne11; } else if (node->src[0]->type == GGML_TYPE_F32 && node->src[1]->type == GGML_TYPE_F32) { cur += sizeof(float)*ne00*ne01*ne02; @@ -2995,8 +2985,8 @@ struct ggml_cplan ggml_graph_plan( GGML_ASSERT(node->src[0]->type == GGML_TYPE_F16 || node->src[0]->type == GGML_TYPE_F32); GGML_ASSERT(node->src[1]->type == GGML_TYPE_F32); + cur += sizeof(float) * ne10 * ne11 * ne12 * ne13; cur += ggml_type_size(node->src[0]->type) * ne00 * ne01 * ne02 * ne03; - cur += ggml_type_size(node->src[0]->type) * ne10 * ne11 * ne12 * ne13; } break; case GGML_OP_TOP_K: @@ -3556,7 +3546,7 @@ void ggml_cpu_fp16_to_fp32(const ggml_fp16_t * x, float * y, int64_t n) { _mm_storeu_ps(y + i, y_vec); } -#elif defined(__riscv_v_intrinsic) && defined(__riscv_zvfhmin) +#elif defined(__riscv_v) && defined(__riscv_zvfhmin) // calculate step size const int epr = __riscv_vsetvlmax_e16m2(); const int step = epr * 2; @@ -3626,7 +3616,7 @@ void ggml_cpu_bf16_to_fp32(const ggml_bf16_t * x, float * y, int64_t n) { (const __m128i *)(x + i))), 16))); } -#elif defined(__riscv_v_intrinsic) && defined(__riscv_zvfbfmin) +#elif defined(__riscv_v) && defined(__riscv_zvfbfmin) // calculate step size const int epr = __riscv_vsetvlmax_e16m2(); const int step = epr * 2; @@ -3746,7 +3736,7 @@ int ggml_cpu_has_arm_fma(void) { } int ggml_cpu_has_riscv_v(void) { -#if defined(__riscv_v_intrinsic) +#if defined(__riscv_v) return 1; #else return 0; @@ -3754,7 +3744,7 @@ int ggml_cpu_has_riscv_v(void) { } int ggml_cpu_get_rvv_vlen(void) { -#if defined(__riscv) && defined(__riscv_v_intrinsic) +#if defined(__riscv) && defined(__riscv_v) return ggml_riscv_arch_features.rvv_vlen; #else return 0; diff --git a/ggml/src/ggml-cpu/ggml-cpu.cpp b/ggml/src/ggml-cpu/ggml-cpu.cpp index 81ff5d79fa98..29783400186d 100644 --- a/ggml/src/ggml-cpu/ggml-cpu.cpp +++ b/ggml/src/ggml-cpu/ggml-cpu.cpp @@ -455,10 +455,11 @@ static bool ggml_backend_cpu_device_supports_op(ggml_backend_dev_t dev, const st src0->type == GGML_TYPE_F32 && op->type == GGML_TYPE_F32) { return src1->type == GGML_TYPE_F32 || src1->type == GGML_TYPE_F16; } - // BF16 in src1 is widened into the F32 work buffer + // F16 or BF16 in src1 is widened into the F32 work buffer return src1->type == GGML_TYPE_F32 || src1->type == ggml_get_type_traits_cpu(src0->type)->vec_dot_type || - (src1->type == GGML_TYPE_BF16 && ggml_get_type_traits_cpu(src0->type)->vec_dot_type == GGML_TYPE_F32); + ((src1->type == GGML_TYPE_F16 || src1->type == GGML_TYPE_BF16) && + ggml_get_type_traits_cpu(src0->type)->vec_dot_type == GGML_TYPE_F32); case GGML_OP_SOFT_MAX_BACK: { if (op->src[0]->type != GGML_TYPE_F32 || op->src[1]->type != GGML_TYPE_F32) { return false; diff --git a/ggml/src/ggml-cpu/llamafile/sgemm.cpp b/ggml/src/ggml-cpu/llamafile/sgemm.cpp index d28840a91504..384f7dbe280e 100644 --- a/ggml/src/ggml-cpu/llamafile/sgemm.cpp +++ b/ggml/src/ggml-cpu/llamafile/sgemm.cpp @@ -69,7 +69,7 @@ #define VECTOR_REGISTERS 16 #endif -#if defined(__riscv_v_intrinsic) +#if defined(__riscv_v) #define LMUL 4 #endif @@ -180,7 +180,7 @@ inline float32x4_t madd(float32x4_t a, float32x4_t b, float32x4_t c) { } #endif -#if defined(__riscv_v_intrinsic) +#if defined(__riscv_v) template <> inline vfloat32m1_t madd(vfloat32m1_t a, vfloat32m1_t b, vfloat32m1_t c) { return __riscv_vfmacc_vv_f32m1(c, a, b, __riscv_vsetvlmax_e32m1()); } @@ -277,7 +277,7 @@ inline float hsum(__m512 x) { } #endif // __AVX512F__ -#if defined(__riscv_v_intrinsic) +#if defined(__riscv_v) inline float hsum(vfloat32m1_t x) { return __riscv_vfmv_f_s_f32m1_f32( __riscv_vfredusum_vs_f32m1_f32m1(x, __riscv_vfmv_v_f_f32m1(0, 1), __riscv_vsetvlmax_e32m1())); @@ -326,6 +326,12 @@ template <> inline float32x4_t load(const ggml_fp16_t * p) { return vec_xl(0, (const float *)(tmp)); } +template <> inline float32x4_t load(const ggml_bf16_t * p) { + uint64_t bits; + memcpy(&bits, p, sizeof(bits)); + const uint64x2_t v_bits = { bits, bits }; + return GGML_BF16_TO_F32_HI((GGML_BF16_VEC) v_bits); +} template <> inline float32x4_t load(const float * p) { return vec_xl(0, p); } @@ -458,7 +464,7 @@ template <> inline __m512bh load_partial(const ggml_bf16_t *p, int n) { #endif #endif -#if defined(__riscv_v_intrinsic) +#if defined(__riscv_v) template <> inline vfloat32m1_t load(const float *p) { return __riscv_vle32_v_f32m1(p, __riscv_vsetvlmax_e32m1()); } @@ -503,7 +509,7 @@ template <> inline vbfloat16m4_t load(const ggml_bf16_t *p) { } #endif -#if defined(__riscv_v_intrinsic) +#if defined(__riscv_v) template T set_zero(); template <> inline vfloat32m1_t set_zero() { @@ -520,7 +526,7 @@ template <> inline vfloat32m8_t set_zero() { } #endif -#if defined(__riscv_v_intrinsic) +#if defined(__riscv_v) template size_t vlmax() { if constexpr (std::is_same_v) { return __riscv_vsetvlmax_e32m1(); } else if constexpr (std::is_same_v) { return __riscv_vsetvlmax_e32m2(); } @@ -732,7 +738,7 @@ class tinyBLAS { const int64_t ldc; }; -#if defined(__riscv_v_intrinsic) +#if defined(__riscv_v) template class tinyBLAS_RVV { public: @@ -1302,6 +1308,88 @@ class tinyBLAS_RVV { ////////////////////////////////////////////////////////////////////////////////////////// // QUANT ZERO MATRIX MULTIPLICATION +#if defined(__VXE__) || defined(__VXE2__) +template +class tinyBLAS_Q0_VXE { + static_assert(sizeof(TA::qs) == QK8_0, "VXE weight loads require 32 quantized bytes per block"); + + public: + tinyBLAS_Q0_VXE(int64_t k, + const TA * A, int64_t lda, + const block_q8_0 * B, int64_t ldb, + float * C, int64_t ldc, + int32_t ith, int32_t nth) + : A(A), B(B), C(C), k(k), lda(lda), ldb(ldb), ldc(ldc), ith(ith), nth(nth) { + } + + void matmul(int64_t m, int64_t n) { + constexpr int32_t tile_size = 2; + const int64_t row_tiles = m / tile_size; + const int64_t tiles = row_tiles * (n / tile_size); + const int64_t duty = (tiles + nth - 1) / nth; + const int64_t start = duty * ith; + const int64_t end = MIN(start + duty, tiles); + + for (int64_t job = start; job < end; ++job) { + const int64_t ii = tile_size * (job % row_tiles); + const int64_t jj = tile_size * (job / row_tiles); + float32x4_t v_sum[tile_size][tile_size] = {}; + + for (int64_t l = 0; l < k; ++l) { + int8x16_t v_xl[tile_size], v_xh[tile_size], v_yl[tile_size], v_yh[tile_size]; + float x_scale[tile_size], y_scale[tile_size]; + + for (int32_t i = 0; i < tile_size; ++i) { + const TA & x = A[lda * (ii + i) + l]; + const block_q8_0 & y = B[ldb * (jj + i) + l]; + + const int8_t * x_qs = x.qs; + const int8_t * y_qs = y.qs; + v_xl[i] = vec_xl(0, x_qs); + v_xh[i] = vec_xl(0, x_qs + QK8_0 / 2); + v_yl[i] = vec_xl(0, y_qs); + v_yh[i] = vec_xl(0, y_qs + QK8_0 / 2); + x_scale[i] = unhalf(x.d); + y_scale[i] = unhalf(y.d); + } + + for (int32_t j = 0; j < tile_size; ++j) { + for (int32_t i = 0; i < tile_size; ++i) { + const int32x4_t v_dot = dot_s8_wide(v_xl[i], v_yl[j]) + dot_s8_wide(v_xh[i], v_yh[j]); + const float32x4_t v_scale = vec_splats(x_scale[i] * y_scale[j]); + v_sum[j][i] = vec_madd(vec_float(v_dot), v_scale, v_sum[j][i]); + } + } + } + + for (int32_t j = 0; j < tile_size; ++j) { + for (int32_t i = 0; i < tile_size; ++i) { + C[ldc * (jj + j) + ii + i] = hsum(v_sum[j][i]); + } + } + } + } + + private: + static int32x4_t dot_s8_wide(int8x16_t v_x, int8x16_t v_y) { + const int16x8_t v_even = vec_mule(v_x, v_y); + const int16x8_t v_odd = vec_mulo(v_x, v_y); + // Two (-128) * (-128) products can overflow a signed 16-bit sum. + return vec_unpackh(v_even) + vec_unpackl(v_even) + vec_unpackh(v_odd) + vec_unpackl(v_odd); + } + + const TA * const A; + const block_q8_0 * const B; + float * const C; + const int64_t k; + const int64_t lda; + const int64_t ldb; + const int64_t ldc; + const int32_t ith; + const int32_t nth; +}; +#endif + #if defined(__ARM_FEATURE_DOTPROD) template class tinyBLAS_Q0_ARM { @@ -3958,7 +4046,7 @@ bool llamafile_sgemm(const struct ggml_compute_params * params, int64_t m, int64 params->ith, params->nth}; tb.matmul(m, n); return true; -#elif defined(__riscv_v_intrinsic) +#elif defined(__riscv_v) #if LMUL == 1 tinyBLAS_RVV tb{ params, k, (const float *)A, lda, @@ -4006,6 +4094,14 @@ bool llamafile_sgemm(const struct ggml_compute_params * params, int64_t m, int64 (float *)C, ldc}; return tb.matmul(m, n); } +#elif defined(__VXE__) || defined(__VXE2__) + if (Btype == GGML_TYPE_BF16) { + tinyBLAS<4, float32x4_t, float32x4_t, ggml_bf16_t, ggml_bf16_t, float> tb{ params, k, + (const ggml_bf16_t *)A, lda, + (const ggml_bf16_t *)B, ldb, + (float *)C, ldc}; + return tb.matmul(m, n); + } #elif defined(__MMA__) if (k % 8) { return false; @@ -4054,6 +4150,13 @@ bool llamafile_sgemm(const struct ggml_compute_params * params, int64_t m, int64 (float *)C, ldc}; return tb.matmul(m, n); } + if (Btype == GGML_TYPE_F32) { + tinyBLAS<16, __m512, __m512, ggml_fp16_t, float, float> tb{ params, k, + (const ggml_fp16_t *)A, lda, + (const float *)B, ldb, + (float *)C, ldc}; + return tb.matmul(m, n); + } #elif (defined(__AVX__) || defined(__AVX2__)) && defined(__F16C__) if (Btype == GGML_TYPE_F16) { tinyBLAS<8, __m256, __m256, ggml_fp16_t, ggml_fp16_t, float> tb{ params, k, @@ -4062,6 +4165,13 @@ bool llamafile_sgemm(const struct ggml_compute_params * params, int64_t m, int64 (float *)C, ldc}; return tb.matmul(m, n); } + if (Btype == GGML_TYPE_F32) { + tinyBLAS<8, __m256, __m256, ggml_fp16_t, float, float> tb{ params, k, + (const ggml_fp16_t *)A, lda, + (const float *)B, ldb, + (float *)C, ldc}; + return tb.matmul(m, n); + } #elif defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) && !defined(_MSC_VER) if (n < 8) return false; @@ -4148,6 +4258,17 @@ bool llamafile_sgemm(const struct ggml_compute_params * params, int64_t m, int64 params->ith, params->nth}; tb.matmul(m, n); return true; +#elif defined(__VXE__) || defined(__VXE2__) + if (m % 2 || n % 2) { + return false; + } + tinyBLAS_Q0_VXE tb{ + k, (const block_q8_0 *)A, lda, + (const block_q8_0 *)B, ldb, + (float *)C, ldc, + params->ith, params->nth}; + tb.matmul(m, n); + return true; #elif defined(__MMA__) //TO-DO: Remove this condition once gemv forwarding is enabled. if (n < 8 && n != 4) diff --git a/ggml/src/ggml-cpu/ops.cpp b/ggml/src/ggml-cpu/ops.cpp index dbdde6997f48..eed5d69e3c19 100644 --- a/ggml/src/ggml-cpu/ops.cpp +++ b/ggml/src/ggml-cpu/ops.cpp @@ -5704,94 +5704,6 @@ void ggml_compute_forward_diag( } } -// ggml_compute_forward_diag_mask_inf - -static void ggml_compute_forward_diag_mask_f32( - const ggml_compute_params * params, - ggml_tensor * dst, - const float value) { - - const ggml_tensor * src0 = dst->src[0]; - - const int ith = params->ith; - const int nth = params->nth; - - const int n_past = ((int32_t *) dst->op_params)[0]; - const bool inplace = src0->data == dst->data; - - GGML_ASSERT(n_past >= 0); - - if (!inplace) { - if (ith == 0) { - // memcpy needs to be synchronized across threads to avoid race conditions. - // => do it in INIT phase - GGML_ASSERT(ggml_nelements(dst) == ggml_nelements(src0)); - GGML_ASSERT(ggml_is_contiguous(dst) && ggml_is_contiguous(src0)); - memcpy( - ((char *) dst->data), - ((char *) src0->data), - ggml_nbytes(dst)); - } - ggml_barrier(params->threadpool); - } - - // TODO: handle transposed/permuted matrices - - const int n = ggml_nrows(src0); - const int nc = src0->ne[0]; - const int nr = src0->ne[1]; - const int nz = n/nr; - - GGML_ASSERT( dst->nb[0] == sizeof(float)); - GGML_ASSERT(src0->nb[0] == sizeof(float)); - - for (int k = 0; k < nz; k++) { - for (int j = ith; j < nr; j += nth) { - for (int i = n_past; i < nc; i++) { - if (i > n_past + j) { - *(float *)((char *) dst->data + k*dst->nb[2] + j*dst->nb[1] + i*dst->nb[0]) = value; - } - } - } - } -} - -void ggml_compute_forward_diag_mask_inf( - const ggml_compute_params * params, - ggml_tensor * dst) { - - const ggml_tensor * src0 = dst->src[0]; - - switch (src0->type) { - case GGML_TYPE_F32: - { - ggml_compute_forward_diag_mask_f32(params, dst, -INFINITY); - } break; - default: - { - GGML_ABORT("fatal error"); - } - } -} - -void ggml_compute_forward_diag_mask_zero( - const ggml_compute_params * params, - ggml_tensor * dst) { - - const ggml_tensor * src0 = dst->src[0]; - - switch (src0->type) { - case GGML_TYPE_F32: - { - ggml_compute_forward_diag_mask_f32(params, dst, 0); - } break; - default: - { - GGML_ABORT("fatal error"); - } - } -} - // ggml_compute_forward_soft_max static void ggml_compute_forward_soft_max_f32( @@ -6310,8 +6222,8 @@ static void ggml_compute_forward_rope_flt( //const int n_past = ((int32_t *) dst->op_params)[0]; const int n_dims = ((int32_t *) dst->op_params)[1]; const int mode = ((int32_t *) dst->op_params)[2]; - //const int n_ctx = ((int32_t *) dst->op_params)[3]; const int n_ctx_orig = ((int32_t *) dst->op_params)[4]; + const bool truncate = ggml_rope_get_truncate(dst); memcpy(&freq_base, (int32_t *) dst->op_params + 5, sizeof(float)); memcpy(&freq_scale, (int32_t *) dst->op_params + 6, sizeof(float)); @@ -6356,7 +6268,7 @@ static void ggml_compute_forward_rope_flt( const float theta_scale = powf(freq_base, -2.0f/n_dims); float corr_dims[2]; - ggml_rope_yarn_corr_dims(n_dims, n_ctx_orig, freq_base, beta_fast, beta_slow, corr_dims); + ggml_rope_yarn_corr_dims(n_dims, n_ctx_orig, freq_base, beta_fast, beta_slow, truncate, corr_dims); const bool is_imrope = mode == GGML_ROPE_TYPE_IMROPE; // qwen3vl apply interleaved mrope const bool mrope_used = mode & GGML_ROPE_TYPE_MROPE; // ggml_rope_multi, note: also true for vision (24 & 8 == true) and for imrope @@ -6514,17 +6426,28 @@ static void ggml_compute_forward_conv_transpose_1d_f16_f32( const int ith = params->ith; const int nth = params->nth; - const int nk = ne00*ne01*ne02; - GGML_ASSERT(nb00 == sizeof(ggml_fp16_t)); GGML_ASSERT(nb10 == sizeof(float)); if (ith == 0) { memset(params->wdata, 0, params->wsize); + // permute source data (src1) from (L x Cin) to (Cin x L) + { + float * const wdata = (float *) params->wdata + 0; + float * dst_data = wdata; + + for (int64_t i11 = 0; i11 < ne11; i11++) { + const float * const src = (float *)((char *) src1->data + i11*nb11); + for (int64_t i10 = 0; i10 < ne10; i10++) { + dst_data[i10*ne11 + i11] = src[i10]; + } + } + } + // permute kernel data (src0) from (K x Cout x Cin) to (Cin x K x Cout) { - ggml_fp16_t * const wdata = (ggml_fp16_t *) params->wdata + 0; + ggml_fp16_t * const wdata = (ggml_fp16_t *)((char *) params->wdata + ne10*ne11*sizeof(float)); for (int64_t i02 = 0; i02 < ne02; i02++) { for (int64_t i01 = 0; i01 < ne01; i01++) { @@ -6537,19 +6460,6 @@ static void ggml_compute_forward_conv_transpose_1d_f16_f32( } } - // permute source data (src1) from (L x Cin) to (Cin x L) - { - ggml_fp16_t * const wdata = (ggml_fp16_t *) params->wdata + nk; - ggml_fp16_t * dst_data = wdata; - - for (int64_t i11 = 0; i11 < ne11; i11++) { - const float * const src = (float *)((char *) src1->data + i11*nb11); - for (int64_t i10 = 0; i10 < ne10; i10++) { - dst_data[i10*ne11 + i11] = GGML_CPU_FP32_TO_FP16(src[i10]); - } - } - } - // need to zero dst since we are accumulating into it memset(dst->data, 0, ggml_nbytes(dst)); } @@ -6567,19 +6477,19 @@ static void ggml_compute_forward_conv_transpose_1d_f16_f32( const int ir0 = dr*ith; const int ir1 = MIN(ir0 + dr, nr); - ggml_fp16_t * const wdata = (ggml_fp16_t *) params->wdata + 0; - ggml_fp16_t * const wdata_src = wdata + nk; + float * const wdata_src = (float *) params->wdata + 0; + ggml_fp16_t * const wdata_kernel = (ggml_fp16_t *)((char *) params->wdata + ne10*ne11*sizeof(float)); for (int i1 = ir0; i1 < ir1; i1++) { float * dst_data = (float *)((char *) dst->data + i1*nb1); - ggml_fp16_t * wdata_kernel = wdata + i1*ne02*ne00; + ggml_fp16_t * wdata_kernel_row = wdata_kernel + i1*ne02*ne00; for (int i10 = 0; i10 < ne10; i10++) { const int i1n = i10*ne11; for (int i00 = 0; i00 < ne00; i00++) { float v = 0; - ggml_vec_dot_f16(ne02, &v, 0, - (ggml_fp16_t *) wdata_src + i1n, 0, - (ggml_fp16_t *) wdata_kernel + i00*ne02, 0, 1); + ggml_vec_dot_f16_f32(ne02, &v, 0, + wdata_kernel_row + i00*ne02, 0, + wdata_src + i1n, 0, 1); dst_data[i10*s0 + i00] += v; } } @@ -7172,29 +7082,30 @@ void ggml_compute_forward_im2col_3d( } } -static void ggml_call_mul_mat(ggml_type type, const ggml_compute_params * params, int64_t m, int64_t n, int64_t k, +static void ggml_call_mul_mat(ggml_type src0_type, ggml_type src1_type, const ggml_compute_params * params, int64_t m, int64_t n, int64_t k, void * a, void * b, float * c) { - const ggml_type_traits * traits = ggml_get_type_traits(type); + const ggml_type_traits * traits0 = ggml_get_type_traits(src0_type); + const ggml_type_traits * traits1 = ggml_get_type_traits(src1_type); struct ggml_tensor src1 = {}; - src1.type = type; + src1.type = src1_type; src1.ne[0] = k; src1.ne[1] = m; src1.ne[2] = 1; src1.ne[3] = 1; - src1.nb[0] = traits->type_size; - src1.nb[1] = k * traits->type_size; + src1.nb[0] = traits1->type_size; + src1.nb[1] = k * traits1->type_size; src1.nb[2] = src1.nb[1]; src1.nb[3] = src1.nb[2]; src1.data = a; struct ggml_tensor src0 = {}; - src0.type = type; + src0.type = src0_type; src0.ne[0] = k; src0.ne[1] = n; src0.ne[2] = 1; src0.ne[3] = 1; - src0.nb[0] = traits->type_size; - src0.nb[1] = k * traits->type_size; + src0.nb[0] = traits0->type_size; + src0.nb[1] = k * traits0->type_size; src0.nb[2] = src0.nb[1]; src0.nb[3] = src0.nb[2]; src0.data = b; @@ -7304,8 +7215,6 @@ static void ggml_compute_forward_conv_2d_impl(const ggml_compute_params * params GGML_ASSERT(kernel_type == GGML_TYPE_F16 || kernel_type == GGML_TYPE_F32); GGML_ASSERT(kernel->type == kernel_type); - const ggml_type_traits * traits = ggml_get_type_traits(kernel_type); - const int32_t stride_x = dst->op_params[0]; const int32_t stride_y = dst->op_params[1]; const int32_t pad_x = dst->op_params[2]; @@ -7331,7 +7240,7 @@ static void ggml_compute_forward_conv_2d_impl(const ggml_compute_params * params const int64_t knl_n = knl_w * knl_h * c_in; const int64_t patch_total = dst->ne[3] * dst_w * dst_h; - const int64_t space_per_patch = knl_n * traits->type_size + c_out * sizeof(float); + const int64_t space_per_patch = knl_n * sizeof(float) + c_out * sizeof(float); const int64_t batch_size = params->wsize / space_per_patch; const int64_t patches_per_batch = batch_size > 8 ? (batch_size / 8) * 8 : batch_size; const int64_t batch_n = (patch_total + patches_per_batch - 1) / patches_per_batch; @@ -7358,7 +7267,7 @@ static void ggml_compute_forward_conv_2d_impl(const ggml_compute_params * params const int64_t src_y = p % dst_w; const float * src_base = (const float *)((const char *)src_data + batch_n * src->nb[3]); - char * dst_row = (char *) tmp + (p % patches_per_batch) * knl_n * traits->type_size; + float * dst_row = (float *) tmp + (p % patches_per_batch) * knl_n; for (int64_t ic = 0; ic < c_in; ++ic) { for (int64_t ky = 0; ky < knl_h; ++ky) { @@ -7376,12 +7285,7 @@ static void ggml_compute_forward_conv_2d_impl(const ggml_compute_params * params src_val = *src_ptr; } - char * element_ptr = dst_row + dst_idx * traits->type_size; - if (kernel_type == GGML_TYPE_F32) { - *(float *) element_ptr = src_val; - } else if (kernel_type == GGML_TYPE_F16) { - *(ggml_fp16_t *) element_ptr = GGML_CPU_FP32_TO_FP16(src_val); - } + dst_row[dst_idx] = src_val; } } } @@ -7389,12 +7293,12 @@ static void ggml_compute_forward_conv_2d_impl(const ggml_compute_params * params ggml_barrier(params->threadpool); - float * gemm_output = (float *) ((char *) tmp + patches_per_batch * knl_n * traits->type_size); + float * gemm_output = (float *) ((char *) tmp + patches_per_batch * knl_n * sizeof(float)); GGML_ASSERT(gemm_output + patch_n * c_out <= (float*)tmp + params->wsize); // GEMM: patches[patch_n, knl_n] × kernel[knl_n, c_out] = output[patch_n, c_out] - ggml_call_mul_mat(kernel_type, params, patch_n, c_out, knl_n, tmp, knl_data, gemm_output); + ggml_call_mul_mat(kernel_type, GGML_TYPE_F32, params, patch_n, c_out, knl_n, tmp, knl_data, gemm_output); ggml_barrier(params->threadpool); @@ -7441,8 +7345,6 @@ static void ggml_compute_forward_conv_3d_impl(const ggml_compute_params * params GGML_ASSERT(kernel_type == GGML_TYPE_F16 || kernel_type == GGML_TYPE_F32); GGML_ASSERT(kernel->type == kernel_type); - const ggml_type_traits * traits = ggml_get_type_traits(kernel_type); - const int32_t s0 = dst->op_params[0]; const int32_t s1 = dst->op_params[1]; const int32_t s2 = dst->op_params[2]; @@ -7474,7 +7376,7 @@ static void ggml_compute_forward_conv_3d_impl(const ggml_compute_params * params const int64_t knl_n_total = knl_n_per_channel * c; const int64_t patch_total = n * dst_w * dst_h * dst_d; - const int64_t space_per_patch = knl_n_total * traits->type_size + oc * sizeof(float); + const int64_t space_per_patch = knl_n_total * sizeof(float) + oc * sizeof(float); const int64_t batch_size = params->wsize / space_per_patch; const int64_t patches_per_batch = batch_size > 8 ? (batch_size / 8) * 8 : batch_size; const int64_t batch_n = (patch_total + patches_per_batch - 1) / patches_per_batch; @@ -7500,7 +7402,7 @@ static void ggml_compute_forward_conv_3d_impl(const ggml_compute_params * params const int64_t dst_y = p_in_depth / dst_w; const int64_t dst_x = p_in_depth % dst_w; - char * dst_row = (char *) tmp + (p % patches_per_batch) * knl_n_total * traits->type_size; + float * dst_row = (float *) tmp + (p % patches_per_batch) * knl_n_total; for (int64_t ic = 0; ic < c; ++ic) { for (int64_t kz = 0; kz < knl_d; ++kz) { @@ -7521,12 +7423,7 @@ static void ggml_compute_forward_conv_3d_impl(const ggml_compute_params * params src_val = *src_ptr; } - char * element_ptr = dst_row + dst_idx * traits->type_size; - if (kernel_type == GGML_TYPE_F32) { - *(float *)element_ptr = src_val; - } else if (kernel_type == GGML_TYPE_F16) { - *(ggml_fp16_t *)element_ptr = GGML_CPU_FP32_TO_FP16(src_val); - } + dst_row[dst_idx] = src_val; } } } @@ -7535,8 +7432,8 @@ static void ggml_compute_forward_conv_3d_impl(const ggml_compute_params * params ggml_barrier(params->threadpool); - float * gemm_output = (float *) ((char *) tmp + patches_per_batch * knl_n_total * traits->type_size); - ggml_call_mul_mat(kernel_type, params, patch_n_in_batch, oc, knl_n_total, tmp, knl_data, gemm_output); + float * gemm_output = (float *) ((char *) tmp + patches_per_batch * knl_n_total * sizeof(float)); + ggml_call_mul_mat(kernel_type, GGML_TYPE_F32, params, patch_n_in_batch, oc, knl_n_total, tmp, knl_data, gemm_output); ggml_barrier(params->threadpool); @@ -7588,7 +7485,7 @@ static void ggml_compute_forward_conv_transpose_2d_impl( const int ith = params->ith; const int nth = params->nth; - const int nk = ne00*ne01*ne02*ne03; + const int nsrc = ne10*ne11*ne12*ne13; GGML_ASSERT(nb00 == ggml_type_size(src0->type)); GGML_ASSERT(nb10 == sizeof(float)); @@ -7596,9 +7493,26 @@ static void ggml_compute_forward_conv_transpose_2d_impl( if (ith == 0) { memset(params->wdata, 0, params->wsize); + // permute source data (src1) from (Sw x Sh x Cin) to (Cin x Sw x Sh), for all batches + { + float * const wdata = (float *) params->wdata + 0; + for (int i13 = 0; i13 < ne13; i13++) { + float * const wdata_b = wdata + i13*ne10*ne11*ne12; + for (int i12 = 0; i12 < ne12; i12++) { + for (int i11 = 0; i11 < ne11; i11++) { + const float * const src = (float *)((char *) src1->data + i13*nb13 + i12*nb12 + i11*nb11); + float * dst_data = wdata_b + i11*ne10*ne12; + for (int i10 = 0; i10 < ne10; i10++) { + dst_data[i10*ne12 + i12] = src[i10]; + } + } + } + } + } + // permute kernel data (src0) from (Kw x Kh x Cout x Cin) to (Cin x Kw x Kh x Cout) { - kernel_t * const wdata = (kernel_t *) params->wdata + 0; + kernel_t * const wdata = (kernel_t *)((char *) params->wdata + nsrc*sizeof(float)); for (int64_t i03 = 0; i03 < ne03; i03++) { for (int64_t i02 = 0; i02 < ne02; i02++) { @@ -7613,27 +7527,6 @@ static void ggml_compute_forward_conv_transpose_2d_impl( } } - // permute source data (src1) from (Sw x Sh x Cin) to (Cin x Sw x Sh), for all batches - { - kernel_t * const wdata = (kernel_t *) params->wdata + nk; - for (int i13 = 0; i13 < ne13; i13++) { - kernel_t * const wdata_b = wdata + i13*ne10*ne11*ne12; - for (int i12 = 0; i12 < ne12; i12++) { - for (int i11 = 0; i11 < ne11; i11++) { - const float * const src = (float *)((char *) src1->data + i13*nb13 + i12*nb12 + i11*nb11); - kernel_t * dst_data = wdata_b + i11*ne10*ne12; - for (int i10 = 0; i10 < ne10; i10++) { - if constexpr (std::is_same_v) { - dst_data[i10*ne12 + i12] = GGML_CPU_FP32_TO_FP16(src[i10]); - } else { - dst_data[i10*ne12 + i12] = src[i10]; - } - } - } - } - } - } - memset(dst->data, 0, ggml_nbytes(dst)); } ggml_barrier(params->threadpool); @@ -7650,14 +7543,14 @@ static void ggml_compute_forward_conv_transpose_2d_impl( const int ip0 = dp*ith; const int ip1 = MIN(ip0 + dp, np); - kernel_t * const wdata = (kernel_t *) params->wdata + 0; - kernel_t * const wdata_src = wdata + nk; + float * const wdata_src = (float *) params->wdata + 0; + kernel_t * const wdata_kernel = (kernel_t *)((char *) params->wdata + nsrc*sizeof(float)); for (int i2 = ip0; i2 < ip1; i2++) { // Cout - kernel_t * wdata_kernel = wdata + i2*ne01*ne00*ne03; + kernel_t * wdata_kernel_c = wdata_kernel + i2*ne01*ne00*ne03; for (int i3 = 0; i3 < ne3; i3++) { // batch float * dst_data = (float *)((char *) dst->data + i3*nb3 + i2*nb2); - kernel_t * wdata_src_b = wdata_src + i3*ne10*ne11*ne12; + float * wdata_src_b = wdata_src + i3*ne10*ne11*ne12; for (int i11 = 0; i11 < ne11; i11++) { for (int i10 = 0; i10 < ne10; i10++) { const int i1n = i11*ne10*ne12 + i10*ne12; @@ -7665,13 +7558,13 @@ static void ggml_compute_forward_conv_transpose_2d_impl( for (int i00 = 0; i00 < ne00; i00++) { float v = 0; if constexpr (std::is_same_v) { - ggml_vec_dot_f16(ne03, &v, 0, - wdata_src_b + i1n, 0, - wdata_kernel + i01*ne00*ne03 + i00*ne03, 0, 1); + ggml_vec_dot_f16_f32(ne03, &v, 0, + wdata_kernel_c + i01*ne00*ne03 + i00*ne03, 0, + wdata_src_b + i1n, 0, 1); } else { ggml_vec_dot_f32(ne03, &v, 0, - wdata_src_b + i1n, 0, - wdata_kernel + i01*ne00*ne03 + i00*ne03, 0, 1); + wdata_kernel_c + i01*ne00*ne03 + i00*ne03, 0, + wdata_src_b + i1n, 0, 1); } dst_data[(i11*stride + i01)*ne0 + i10*stride + i00] += v; } @@ -8930,14 +8823,9 @@ static void ggml_compute_forward_flash_attn_ext_f16_one_chunk( float * VKQ32 = (float *) params->wdata + ith*(1*DK + 2*DV + CACHE_LINE_SIZE_F32); // FP32 VKQ accumulator float * V32 = (VKQ32 + 1*DV); // (temporary) FP32 V buffer - ggml_fp16_t * VKQ16 = (ggml_fp16_t *) (VKQ32 + 1*DV); // (temporary) FP16 VKQ accumulator ggml_fp16_t * Q_q = (ggml_fp16_t *) (VKQ32 + 2*DV); // (temporary) buffer for Q converted to quantized/FP16 - if (v->type == GGML_TYPE_F16) { - memset(VKQ16, 0, DV*sizeof(ggml_fp16_t)); - } else { - memset(VKQ32, 0, DV*sizeof(float)); - } + memset(VKQ32, 0, DV*sizeof(float)); const ggml_fp16_t * mp = mask ? (ggml_fp16_t *)((char *) mask->data + iq1*mask->nb[1] + (iq2%mask->ne[2])*mask->nb[2] + (iq3%mask->ne[3])*mask->nb[3]) : NULL; @@ -8989,53 +8877,32 @@ static void ggml_compute_forward_flash_attn_ext_f16_one_chunk( const char * v_data = ((const char *) v->data + (ik1*nbv1 + iv2*nbv2 + iv3*nbv3)); - if (v->type == GGML_TYPE_F16) { - if (s > M) { - // s is new maximum, ms < 1.0f, vs == expf(s - s) == 1.0f - M = s; - ms = expf(Mold - M); - - // V = V*expf(Mold - M) - ggml_vec_scale_f16(DV, VKQ16, ms); - } else { - // no new maximum, ms == 1.0f, vs != 1.0f - vs = expf(s - M); - } + if (s > M) { + // s is new maximum, ms < 1.0f, vs == expf(s - s) == 1.0f + M = s; + ms = expf(Mold - M); - // V += v*expf(s - M) - ggml_vec_mad_f16(DV, VKQ16, (const ggml_fp16_t *) v_data, vs); + // V = V*expf(Mold - M) + ggml_vec_scale_f32(DV, VKQ32, ms); } else { - if (s > M) { - // s is new maximum, ms < 1.0f, vs == expf(s - s) == 1.0f - M = s; - ms = expf(Mold - M); - - // V = V*expf(Mold - M) - ggml_vec_scale_f32(DV, VKQ32, ms); - } else { - // no new maximum, ms == 1.0f, vs != 1.0f - vs = expf(s - M); - } + // no new maximum, ms == 1.0f, vs != 1.0f + vs = expf(s - M); + } - // V += v*expf(s - M) - if (v_to_float) { - v_to_float(v_data, V32, DV); - ggml_vec_mad_f32(DV, VKQ32, V32, vs); - } else { - // V is F32 - ggml_vec_mad_f32(DV, VKQ32, (const float *) v_data, vs); - } + // V += v*expf(s - M) + if (v->type == GGML_TYPE_F16) { + ggml_vec_mad_f16_f32(DV, VKQ32, (const ggml_fp16_t *) v_data, vs); + } else if (v_to_float) { + v_to_float(v_data, V32, DV); + ggml_vec_mad_f32(DV, VKQ32, V32, vs); + } else { + // V is F32 + ggml_vec_mad_f32(DV, VKQ32, (const float *) v_data, vs); } S = S*ms + vs; // scale and increment sum with partial sum } - if (v->type == GGML_TYPE_F16) { - for (int64_t d = 0; d < DV; ++d) { - VKQ32[d] = GGML_CPU_FP16_TO_FP32(VKQ16[d]); - } - } - // sinks - apply only on the first kv-chunk if (sinks && ic_start == 0) { const float s = ((float *)((char *) sinks->data))[h]; @@ -9588,9 +9455,10 @@ static void ggml_compute_forward_flash_attn_ext_f16( void ggml_compute_forward_flash_attn_ext( const ggml_compute_params * params, ggml_tensor * dst) { - switch (dst->op_params[3]) { - case GGML_PREC_DEFAULT: + switch (ggml_prec_get_acc(dst)) { + case GGML_PREC_UNDEFINED: case GGML_PREC_F32: + case GGML_PREC_BF16: { // uses F32 accumulators ggml_compute_forward_flash_attn_ext_f16(params, dst); @@ -10114,7 +9982,7 @@ static void ggml_compute_forward_ssm_scan_f32( } sumf = GGML_F32xt_REDUCE_ONE(sum); - #elif defined(__riscv_v_intrinsic) + #elif defined(__riscv_v) // todo: RVV implementation const int np = 0; #else @@ -11778,7 +11646,7 @@ static void ggml_compute_forward_rwkv_wkv7_f32( int64_t h_stride_2d = head_size * head_size; #if defined(GGML_SIMD) - #if defined(__ARM_FEATURE_SVE) || defined(__riscv_v_intrinsic) + #if defined(__ARM_FEATURE_SVE) || defined(__riscv_v) // scalar Route to scalar implementation //TODO: Write SVE code and RVV code for (int64_t t = 0; t < T; t++) { int64_t t_offset = t * t_stride; diff --git a/ggml/src/ggml-cpu/ops.h b/ggml/src/ggml-cpu/ops.h index 00a2ea7dc2de..aac527a64c54 100644 --- a/ggml/src/ggml-cpu/ops.h +++ b/ggml/src/ggml-cpu/ops.h @@ -56,8 +56,6 @@ void ggml_compute_forward_get_rows(const struct ggml_compute_params * params, st void ggml_compute_forward_get_rows_back(const struct ggml_compute_params * params, struct ggml_tensor * dst); void ggml_compute_forward_set_rows(const struct ggml_compute_params * params, struct ggml_tensor * dst); void ggml_compute_forward_diag(const struct ggml_compute_params * params, struct ggml_tensor * dst); -void ggml_compute_forward_diag_mask_inf(const struct ggml_compute_params * params, struct ggml_tensor * dst); -void ggml_compute_forward_diag_mask_zero(const struct ggml_compute_params * params, struct ggml_tensor * dst); void ggml_compute_forward_soft_max(const struct ggml_compute_params * params, struct ggml_tensor * dst); void ggml_compute_forward_soft_max_ext_back(const struct ggml_compute_params * params, struct ggml_tensor * dst); void ggml_compute_forward_rope(const struct ggml_compute_params * params, struct ggml_tensor * dst); diff --git a/ggml/src/ggml-cpu/simd-gemm.h b/ggml/src/ggml-cpu/simd-gemm.h index 4b9396d54066..969ffd13ff02 100644 --- a/ggml/src/ggml-cpu/simd-gemm.h +++ b/ggml/src/ggml-cpu/simd-gemm.h @@ -5,11 +5,11 @@ #include "simd-mappings.h" // TODO: add support for sizeless vector types -#if defined(GGML_SIMD) && !defined(__ARM_FEATURE_SVE) && !defined(__riscv_v_intrinsic) +#if defined(GGML_SIMD) && !defined(__ARM_FEATURE_SVE) && !defined(__riscv_v) // TODO: untested on avx512 // These are in units of GGML_F32_EPR -#if defined(__AVX512F__) || defined (__ARM_NEON__) +#if defined(__AVX512F__) || defined(__ARM_NEON) static constexpr int GEMM_RM = 4; static constexpr int GEMM_RN = 4; // 16+4+1 = 25/32 #elif defined(__AVX2__) || defined(__AVX__) @@ -149,7 +149,7 @@ static void simd_gemm( C += N; } } -#elif defined(GGML_SIMD) && defined(__riscv_v_intrinsic) +#elif defined(GGML_SIMD) && defined(__riscv_v) // RM accumulators + 1 B vector = RM + 1 <= 8 => RM <= 7 // Microkernel: C[RM x vl] += A[RM x K] * B[K x N] template diff --git a/ggml/src/ggml-cpu/simd-mappings.h b/ggml/src/ggml-cpu/simd-mappings.h index 7b86e8ad9a79..d3d00bf6f717 100644 --- a/ggml/src/ggml-cpu/simd-mappings.h +++ b/ggml/src/ggml-cpu/simd-mappings.h @@ -14,7 +14,9 @@ #include #endif -#if defined(__riscv_v_intrinsic) +// __riscv_v_intrinsic reports compiler API support and can be defined without +// enabling V; use the ISA feature macro to select RVV code. +#if defined(__riscv_v) #include #endif @@ -1306,7 +1308,7 @@ static inline void __lzs_f16cx4_store(ggml_fp16_t * x, float32x4_t v_y) { #define GGML_BF16_FMA_HI(acc, x, y) \ (acc) = GGML_F32x4_FMA((acc), GGML_BF16_TO_F32_HI(x), GGML_BF16_TO_F32_HI(y)) -#elif defined(__riscv_v_intrinsic) +#elif defined(__riscv_v) // compatible with vlen >= 128 diff --git a/ggml/src/ggml-cpu/spacemit/ime.cpp b/ggml/src/ggml-cpu/spacemit/ime.cpp index 5f114f587a49..dc51ad0aa4dd 100644 --- a/ggml/src/ggml-cpu/spacemit/ime.cpp +++ b/ggml/src/ggml-cpu/spacemit/ime.cpp @@ -198,7 +198,7 @@ template class tensor_ case GGML_TYPE_Q5_0: case GGML_TYPE_Q5_1: case GGML_TYPE_Q5_K: - //case GGML_TYPE_MXFP4: + case GGML_TYPE_MXFP4: forward_mul_mat(params, op); return true; default: @@ -1154,9 +1154,17 @@ class tensor_traits_common : public tensor_traits_base { ggml_compute_forward_concat(params, op); } return true; - // TODO For GGML_OP_GATED_DELTA_NET - // case GGML_OP_GATED_DELTA_NET: - // return true; + case GGML_OP_GATED_DELTA_NET: + if (params->use_ref) { + return false; + } + switch (op->src[0]->type) { + case GGML_TYPE_F32: + spacemit_kernels::rvv::forward_gated_delta_net_f32(params, op); + return true; + default: + GGML_ABORT("fatal error"); + } default: break; } @@ -1180,7 +1188,7 @@ class tensor_traits_common : public tensor_traits_base { const int64_t DK = nek0; const int64_t DV = nev0; - const bool supported_prec = (dst->op_params[3] == GGML_PREC_F32 || dst->op_params[3] == GGML_PREC_DEFAULT); + const bool supported_prec = (ggml_prec_get_acc(dst) == GGML_PREC_F32 || ggml_prec_get_acc(dst) == GGML_PREC_BF16 || ggml_prec_get_acc(dst) == GGML_PREC_UNDEFINED); const bool supported_types = (q->type == GGML_TYPE_F32 && k->type == GGML_TYPE_F16 && v->type == GGML_TYPE_F16); const bool supported_shape = (DK > 0 && DK <= 128 && DV > 0 && DV <= 128); const bool supported_vlen = (__riscv_vlenb() == 128); @@ -1315,11 +1323,10 @@ static const ggml::cpu::tensor_traits * ggml_riscv64_spacemit_get_optimal_repack case GGML_TYPE_Q4_1: { #if defined(RISCV64_SPACEMIT_IME2) - // TODO - // if (cur->ne[1] % 32 == 0 && cur->ne[0] % 256 == 0 && - // (ggml::cpu::riscv64_spacemit::global_spine_env_info.use_ime2)) { - // return &ggml::cpu::riscv64_spacemit::q4_1_32x256_q8_0; - // } + if (cur->ne[1] % 32 == 0 && cur->ne[0] % 256 == 0 && + (ggml::cpu::riscv64_spacemit::global_spine_env_info.use_ime2)) { + return &ggml::cpu::riscv64_spacemit::q4_1_32x256_q8_0; + } if (cur->ne[1] % 32 == 0 && (ggml::cpu::riscv64_spacemit::global_spine_env_info.use_ime2)) { return &ggml::cpu::riscv64_spacemit::q4_1_32x32_q8_0; @@ -1375,10 +1382,10 @@ static const ggml::cpu::tensor_traits * ggml_riscv64_spacemit_get_optimal_repack case GGML_TYPE_MXFP4: { #if defined(RISCV64_SPACEMIT_IME2) - // TODO - // if (cur->ne[1] % 32 == 0 && (ggml::cpu::riscv64_spacemit::global_spine_env_info.use_ime2)) { - // return &ggml::cpu::riscv64_spacemit::mxfp4_32x32_q8_0; - // } + if (ggml_n_dims(cur) == 2 && cur->ne[0] % 32 == 0 && cur->ne[1] % 32 == 0 && + (ggml::cpu::riscv64_spacemit::global_spine_env_info.use_ime2)) { + return &ggml::cpu::riscv64_spacemit::mxfp4_32x32_q8_0; + } #endif } break; @@ -1613,6 +1620,9 @@ class extra_buffer_type : ggml::cpu::extra_buffer_type { if (op->src[1]->buffer && !ggml_backend_buft_is_host(op->src[1]->buffer->buft)) { return false; } + if (op->src[0]->type == GGML_TYPE_MXFP4 && !ggml_is_contiguous(op->src[1])) { + return false; + } if (op->src[1]->type == GGML_TYPE_F32) { return true; } @@ -1658,7 +1668,7 @@ class extra_buffer_type : ggml::cpu::extra_buffer_type { case GGML_OP_SUM_ROWS: case GGML_OP_GET_ROWS: case GGML_OP_CONCAT: - // case GGML_OP_GATED_DELTA_NET: + case GGML_OP_GATED_DELTA_NET: return (ggml::cpu::tensor_traits *) (&ggml::cpu::riscv64_spacemit::rvv_impl); default: // GGML_ABORT("fatal error"); diff --git a/ggml/src/ggml-cpu/spacemit/ime2_kernels.cpp b/ggml/src/ggml-cpu/spacemit/ime2_kernels.cpp index 0c7a036a92af..92b84a90b0fb 100644 --- a/ggml/src/ggml-cpu/spacemit/ime2_kernels.cpp +++ b/ggml/src/ggml-cpu/spacemit/ime2_kernels.cpp @@ -2999,8 +2999,107 @@ void gemm_kernel_i8i4_hp_m1(size_t blk_len, "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "fa0", "fa1", "ft0", "ft1"); } } else { - // TODO: support quant_b_zp for i8i4 hp kernel - GGML_ABORT("gemm_kernel_i8i4_hp_m1 with quant_b_zp is not supported yet"); + const _Float16 hp_scale_0125 = 0.125; + for (size_t ni = 0; ni < count_n; ni += 32) { + const size_t nb_real = std::min(NB_COLS, count_n - ni); + if (nb_real != NB_COLS) { + break; + } + + uint8_t * b_data = (uint8_t *) quant_b_data + (ni / NB_COLS) * b_tile_stride; + int8_t * a_data = (int8_t *) quant_a_ptr; + float * dst_c = c_ptr + ni; + + asm volatile( + "vsetvli t0, x0, e16, m1 \n\t" + "vxor.vv v31, v31, v31 \n\t" + "mv t4, %[BK] \n\t" + "li t0, 0x4c00 \n\t" + "fmv.h.x fa0, t0 \n\t" + + ".align 4 \n\t" + "BLK_LOOP%=: \n\t" + "li t1, 4608 \n\t" + "add s6, %[B], t1 \n\t" + "li t5, 8 \n\t" + "addi t6, %[A], 288 \n\t" + "flh ft1, (t6) \n\t" + "addi t6, %[A], 272 \n\t" + + "vsetvli t0, x0, e16, m1 \n\t" + "vxor.vv v16, v18, v18 \n\t" + "vxor.vv v17, v18, v18 \n\t" + "vxor.vv v18, v18, v18 \n\t" + "vxor.vv v19, v18, v18 \n\t" + + "INNER_BLK_LOOP%=: \n\t" + "flh fa1, (t6) \n\t" + "addi t6, t6, 2 \n\t" + "flh ft0, (%[A]) \n\t" + "addi %[A], %[A], 2 \n\t" + "vsetvli t0, x0, e8, mf4 \n\t" + "vle8.v v3, (%[A]) \n\t" + "addi %[A], %[A], 32 \n\t" + + "vsetvli t0, x0, e16, mf2 \n\t" + "vle16.v v8, (%[B]) \n\t" + "addi %[B], %[B], 64 \n\t" + "vl4r.v v4, (%[B]) \n\t" + "addi %[B], %[B], 512 \n\t" + "vfmul.vf v8, v8, ft0 \n\t" + "vfmul.vf v9, v8, fa0 \n\t" + "vsetvli t0, x0, e8, mf4 \n\t" + "vle8.v v11, (s6) \n\t" + "addi s6, s6, 32 \n\t" + "vfwcvt.f.xu.v v13, v11 \n\t" + "fmul.h ft2, fa1, %[HP0125] \n\t" + "vsetvli t0, x0, e16, mf2 \n\t" + "vfmul.vv v10, v13, v8 \n\t" + "vfmul.vf v10, v10, ft2 \n\t" + "vfwmacc.vf v31, ft1, v10 \n\t" + + "vsetvli t0, x0, e8, m1 \n\t" + "vpack.vv v0, v8, v9, 3 \n\t" + "vsrl.vi v28, v3, 4 \n\t" + + "vsetvli t0, x0, e16, m1 \n\t" + "vnpack4.vv v2, v3, v3, 3 \n\t" + "vnpack4.vv v3, v28, v28, 3 \n\t" + + "vsetvli t0, x0, e16, m1 \n\t" + "vmadotsu.hp v16, v3, v4, v0, 4, i4 \n\t" + "vmadotsu.hp v17, v3, v5, v0, 5, i4 \n\t" + "vmadotsu.hp v18, v3, v6, v0, 6, i4 \n\t" + "vmadotsu.hp v19, v3, v7, v0, 7, i4 \n\t" + "vmadotu.hp v16, v2, v4, v0, 0, i4 \n\t" + "vmadotu.hp v17, v2, v5, v0, 1, i4 \n\t" + "vmadotu.hp v18, v2, v6, v0, 2, i4 \n\t" + "vmadotu.hp v19, v2, v7, v0, 3, i4 \n\t" + + "addi t5, t5, -1 \n\t" + "bgtz t5, INNER_BLK_LOOP%= \n\t" + + "vpack.vv v8, v16, v17, 1 \n\t" + "vpack.vv v12, v18, v19, 1 \n\t" + "vpack.vv v20, v8, v12, 2 \n\t" + + "vsetvli t0, x0, e16, mf2 \n\t" + "addi t4, t4, -1 \n\t" + "vfwmacc.vf v31, ft1, v20 \n\t" + + "addi %[A], t6, 2 \n\t" + "mv %[B], s6 \n\t" + + "bgtz t4, BLK_LOOP%= \n\t" + + "vsetvli t0, x0, e32, m1 \n\t" + "vse32.v v31, (%[DST]) \n\t" + : [A] "+r"(a_data), [B] "+r"(b_data) + : [DST] "r"(dst_c), [BK] "r"(k_blks), [HP0125] "f"(hp_scale_0125) + : "t0", "t1", "t2", "t3", "t4", "t5", "t6", "s6", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", + "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", + "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "fa0", "fa1", "ft0", "ft1", "ft2"); + } } } @@ -3754,7 +3853,7 @@ void gemm_kernel_i8mxfp4_m1(size_t blk_len, using blk_type = nrow_block_mxfp4; GGML_ASSERT(blk_len == K_TILE); - GGML_ASSERT(count_m == 1); + GGML_ASSERT(count_m >= 1); GGML_UNUSED(quant_b_zp); const size_t a_blk_stride = q8_blk_size(blk_len, true); @@ -3923,7 +4022,7 @@ void gemm_kernel_i8mxfp4_m1(size_t blk_len, : : [BCK] "r"(cnt), [NBLKS] "r"(nblks), [pA] "r"(quant_a_ptr), [pB] "r"(QuantBDataPtr), [pC] "r"(CPtr) : "cc", "memory", "t0", "t1", "t2", "t3", "t4", "f0", "s2", "s3", "s4", "s5", "s6", "s7", "v0", "v1", - "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v16", "v17", "v18", "v19", + "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31"); } } @@ -3943,7 +4042,7 @@ void gemm_kernel_i8mxfp4_m4(size_t blk_len, using blk_type = nrow_block_mxfp4; GGML_ASSERT(blk_len == K_TILE); - GGML_ASSERT(count_m == 4); + GGML_ASSERT(count_m >= 4); GGML_UNUSED(quant_b_zp); const size_t a_blk_stride = q8_blk_size(blk_len, true); @@ -5687,20 +5786,10 @@ size_t gemm_kernel_i8mxfp4(size_t blk_len, size_t k_blks, size_t ldc) { if (count_m >= 4) { -#if 1 - gemm_kernel_i8mxfp4_mrow_ref<4, 32>(blk_len, quant_a_ptr, quant_b_data, quant_b_zp, c_ptr, count_m, count_n, - k_blks, ldc); -#else gemm_kernel_i8mxfp4_m4(blk_len, quant_a_ptr, quant_b_data, quant_b_zp, c_ptr, count_m, count_n, k_blks, ldc); -#endif return 4; } else { -#if 1 - gemm_kernel_i8mxfp4_mrow_ref<1, 32>(blk_len, quant_a_ptr, quant_b_data, quant_b_zp, c_ptr, count_m, count_n, - k_blks, ldc); -#else gemm_kernel_i8mxfp4_m1(blk_len, quant_a_ptr, quant_b_data, quant_b_zp, c_ptr, count_m, count_n, k_blks, ldc); -#endif return 1; } } diff --git a/ggml/src/ggml-cpu/spacemit/rvv_kernels.cpp b/ggml/src/ggml-cpu/spacemit/rvv_kernels.cpp index 13b84dcbe72b..e27a006eab0d 100644 --- a/ggml/src/ggml-cpu/spacemit/rvv_kernels.cpp +++ b/ggml/src/ggml-cpu/spacemit/rvv_kernels.cpp @@ -3175,4 +3175,199 @@ template void forward_get_rows(ggml_compute_params * params, ggml_tenso template void forward_concat(ggml_compute_params * params, ggml_tensor * op); template void forward_concat(ggml_compute_params * params, ggml_tensor * op); +static inline float rvv_dot_f32(const float * x, const float * y, int64_t n) { + size_t vl = __riscv_vsetvlmax_e32m4(); + vfloat32m4_t vsum = __riscv_vfmv_v_f_f32m4(0.0f, vl); + for (int64_t i = 0; i < n; i += vl) { + vl = __riscv_vsetvl_e32m4(n - i); + vfloat32m4_t ax = __riscv_vle32_v_f32m4(x + i, vl); + vfloat32m4_t ay = __riscv_vle32_v_f32m4(y + i, vl); + vsum = __riscv_vfmacc_vv_f32m4_tu(vsum, ax, ay, vl); + } + vl = __riscv_vsetvlmax_e32m4(); + vfloat32m1_t vs = __riscv_vfmv_v_f_f32m1(0.0f, 1); + vs = __riscv_vfredusum_vs_f32m4_f32m1(vsum, vs, vl); + return __riscv_vfmv_f_s_f32m1_f32(vs); +} + +static void forward_gated_delta_net_one_chunk( + const ggml_compute_params * params, + ggml_tensor * dst, + int64_t ir0, + int64_t ir1) { + + ggml_tensor * src_q = dst->src[0]; + ggml_tensor * src_k = dst->src[1]; + ggml_tensor * src_v = dst->src[2]; + ggml_tensor * src_g = dst->src[3]; + ggml_tensor * src_beta = dst->src[4]; + ggml_tensor * src_state = dst->src[5]; + + const int64_t S_v = src_v->ne[0]; + const int64_t H = src_v->ne[1]; + const int64_t n_tokens = src_v->ne[2]; + const int64_t n_seqs = src_v->ne[3]; + + GGML_ASSERT(ggml_is_contiguous_rows(src_q)); + GGML_ASSERT(ggml_is_contiguous_rows(src_k)); + GGML_ASSERT(ggml_is_contiguous_rows(src_v)); + GGML_ASSERT(ggml_is_contiguous(src_g)); + GGML_ASSERT(ggml_is_contiguous(src_beta)); + GGML_ASSERT(ggml_is_contiguous(src_state)); + + GGML_ASSERT(src_g->ne[0] == 1 || src_g->ne[0] == S_v); + GGML_ASSERT(src_beta->ne[0] == 1); + + GGML_TENSOR_LOCALS(int64_t, neq, src_q, ne); + GGML_TENSOR_LOCALS(size_t, nbq, src_q, nb); + GGML_TENSOR_LOCALS(int64_t, nek, src_k, ne); + GGML_TENSOR_LOCALS(size_t, nbk, src_k, nb); + GGML_TENSOR_LOCALS(int64_t, nev, src_v, ne); + GGML_TENSOR_LOCALS(size_t, nbv, src_v, nb); + GGML_TENSOR_LOCALS(int64_t, neg, src_g, ne); + GGML_TENSOR_LOCALS(size_t, nbg, src_g, nb); + GGML_TENSOR_LOCALS(size_t, nbb, src_beta, nb); + + const bool kda = (neg0 == S_v); + + const int64_t K = ggml_get_op_params_i32(dst, 0); + GGML_ASSERT(K >= 1); + const int64_t state_seq_stride = src_state->nb[3] / sizeof(float); + + const int64_t per_thread = S_v + (K > 1 ? S_v * S_v : 0); + const int ith = params->ith; + + float * delta = (float *) params->wdata + ith * per_thread + CACHE_LINE_SIZE_F32; + float * state_work = K > 1 ? (delta + S_v) : nullptr; + + const int64_t attn_score_elems = S_v * H * n_tokens * n_seqs; + const int64_t state_size_per_snap = S_v * S_v * H * n_seqs; + float * attn_out_base = (float *) dst->data; + float * state_out_base = (float *) dst->data + attn_score_elems; + + const float * state_in_base = (const float *) src_state->data; + + const int64_t rq3 = nev3 / neq3; + const int64_t rk3 = nev3 / nek3; + + const float scale = 1.0f / sqrtf((float) S_v); + + for (int64_t ir = ir0; ir < ir1; ++ir) { + const int64_t iv1 = ir % H; + const int64_t iv3 = ir / H; + + const int64_t iq1 = iv1 % neq1; + const int64_t ik1 = iv1 % nek1; + + const int64_t iq3 = iv3 / rq3; + const int64_t ik3 = iv3 / rk3; + + float * s_out = (K > 1) + ? state_work + : state_out_base + (iv3 * H + iv1) * S_v * S_v; + + const float * s_in = state_in_base + iv3 * state_seq_stride + iv1 * S_v * S_v; + memcpy(s_out, s_in, S_v * S_v * sizeof(float)); + + float * attn_data = attn_out_base + (iv3 * n_tokens * H + iv1) * S_v; + + for (int64_t t = 0; t < n_tokens; t++) { + const float * q_d = (const float *) ((const char *) src_q->data + iq3 * nbq3 + t * nbq2 + iq1 * nbq1); + const float * k_d = (const float *) ((const char *) src_k->data + ik3 * nbk3 + t * nbk2 + ik1 * nbk1); + const float * v_d = (const float *) ((const char *) src_v->data + iv3 * nbv3 + t * nbv2 + iv1 * nbv1); + + const float beta_val = *(const float *) ((const char *) src_beta->data + iv3 * nbb3 + t * nbb2 + iv1 * nbb1); + const float * g_d = (const float *) ((const char *) src_g->data + iv3 * nbg3 + t * nbg2 + iv1 * nbg1); + + if (kda) { + for (int64_t i = 0; i < S_v; ++i) { + delta[i] = expf(g_d[i]); + } + for (int64_t j = 0; j < S_v; ++j) { + float * row = s_out + j * S_v; + const float * d = delta; + int64_t remaining = S_v; + while (remaining > 0) { + const size_t vl = __riscv_vsetvl_e32m4(remaining); + vfloat32m4_t vr = __riscv_vle32_v_f32m4(row, vl); + vfloat32m4_t vd = __riscv_vle32_v_f32m4(d, vl); + vr = __riscv_vfmul_vv_f32m4(vr, vd, vl); + __riscv_vse32_v_f32m4(row, vr, vl); + row += vl; + d += vl; + remaining -= vl; + } + } + } else { + const float gate = expf(g_d[0]); + float * p = s_out; + int64_t remaining = S_v * S_v; + while (remaining > 0) { + const size_t vl = __riscv_vsetvl_e32m4(remaining); + vfloat32m4_t v = __riscv_vle32_v_f32m4(p, vl); + v = __riscv_vfmul_vf_f32m4(v, gate, vl); + __riscv_vse32_v_f32m4(p, v, vl); + p += vl; + remaining -= vl; + } + } + + for (int64_t j = 0; j < S_v; ++j) { + const float sum = rvv_dot_f32(s_out + j * S_v, k_d, S_v); + delta[j] = (v_d[j] - sum) * beta_val; + } + + for (int64_t j = 0; j < S_v; ++j) { + float * row = s_out + j * S_v; + const float * k = k_d; + const float dj = delta[j]; + int64_t remaining = S_v; + while (remaining > 0) { + const size_t vl = __riscv_vsetvl_e32m4(remaining); + vfloat32m4_t vr = __riscv_vle32_v_f32m4(row, vl); + vfloat32m4_t vk = __riscv_vle32_v_f32m4(k, vl); + vr = __riscv_vfmacc_vf_f32m4(vr, dj, vk, vl); + __riscv_vse32_v_f32m4(row, vr, vl); + row += vl; + k += vl; + remaining -= vl; + } + } + + for (int64_t j = 0; j < S_v; ++j) { + const float sum = rvv_dot_f32(s_out + j * S_v, q_d, S_v); + attn_data[j] = sum * scale; + } + + attn_data += S_v * H; + + if (K > 1) { + const int64_t target_slot = n_tokens - 1 - t; + if (target_slot >= 0 && target_slot < K) { + float * curr_state_o = state_out_base + target_slot * state_size_per_snap + + (iv3 * H + iv1) * S_v * S_v; + memcpy(curr_state_o, s_out, S_v * S_v * sizeof(float)); + } + } + } + } +} + +void forward_gated_delta_net_f32(ggml_compute_params * params, ggml_tensor * dst) { + ggml_tensor * src_v = dst->src[2]; + + int ith = params->ith; + int nth = params->nth; + + const auto nr = src_v->ne[1] * src_v->ne[3]; + const auto task_per_thread = (nr + nth - 1) / nth; + + const auto task_begin = ith * task_per_thread; + const auto task_end = std::min((ith + 1) * task_per_thread, nr); + + if (task_begin < task_end) { + forward_gated_delta_net_one_chunk(params, dst, task_begin, task_end); + } +} + } // namespace spacemit_kernels::rvv diff --git a/ggml/src/ggml-cpu/spacemit/rvv_kernels.h b/ggml/src/ggml-cpu/spacemit/rvv_kernels.h index edddf957c21d..a085613a3071 100644 --- a/ggml/src/ggml-cpu/spacemit/rvv_kernels.h +++ b/ggml/src/ggml-cpu/spacemit/rvv_kernels.h @@ -62,6 +62,8 @@ void forward_rms_norm_f32(ggml_compute_params * params, ggml_tensor * op); void forward_norm_f32(ggml_compute_params * params, ggml_tensor * op); +void forward_gated_delta_net_f32(ggml_compute_params * params, ggml_tensor * op); + void forward_cont_with_permute(ggml_compute_params * params, ggml_tensor * op); void forward_cpy_with_permute(ggml_compute_params * params, ggml_tensor * op); diff --git a/ggml/src/ggml-cpu/vec.cpp b/ggml/src/ggml-cpu/vec.cpp index 8f0bf4999cfb..9ff7335b8f7b 100644 --- a/ggml/src/ggml-cpu/vec.cpp +++ b/ggml/src/ggml-cpu/vec.cpp @@ -84,7 +84,7 @@ void ggml_vec_dot_f32(int n, float * GGML_RESTRICT s, size_t bs, const float * G } // reduce sum1,sum2 to sum1 GGML_F32_VEC_REDUCE(sumf, sum1, sum2, sum3, sum4, sum5, sum6, sum7, sum8); - #elif defined(__riscv_v_intrinsic) + #elif defined(__riscv_v) int vl = __riscv_vsetvlmax_e32m8(); vfloat32m1_t vs = __riscv_vfmv_v_f_f32m1(0.0f, 1); vfloat32m8_t vsum; @@ -195,7 +195,7 @@ void ggml_vec_dot_bf16(int n, float * GGML_RESTRICT s, size_t bs, ggml_bf16_t * sumf += (ggml_float)_mm_cvtss_f32(g); #undef LOAD -#elif defined(__riscv_v_intrinsic) && defined(__riscv_zvfbfwma) +#elif defined(__riscv_v) && defined(__riscv_zvfbfwma) size_t vl = __riscv_vsetvlmax_e32m4(); // initialize accumulators to all zeroes @@ -318,7 +318,7 @@ void ggml_vec_dot_f16(int n, float * GGML_RESTRICT s, size_t bs, ggml_fp16_t * G sum1_hi = svadd_f32_m(DEFAULT_PG32, sum1_hi, sum3_hi); sumf = ggml_sve_sum_f32x2(sum1_lo, sum1_hi); - #elif defined(__riscv_v_intrinsic) + #elif defined(__riscv_v) #if defined(__riscv_zvfh) int vl = __riscv_vsetvlmax_e32m2(); vfloat32m1_t vs = __riscv_vfmv_v_f_f32m1(0.0f, 1); @@ -377,6 +377,85 @@ void ggml_vec_dot_f16(int n, float * GGML_RESTRICT s, size_t bs, ggml_fp16_t * G *s = sumf; } +void ggml_vec_dot_f16_f32(int n, float * GGML_RESTRICT s, size_t bs, const ggml_fp16_t * GGML_RESTRICT x, size_t bx, const float * GGML_RESTRICT y, size_t by, int nrc) { + assert(nrc == 1); + GGML_UNUSED(nrc); + GGML_UNUSED(bx); + GGML_UNUSED(by); + GGML_UNUSED(bs); + + ggml_float sumf = 0.0; + int i = 0; + + // on AVX512 and AVX with F16C, x is widened to F32 in-register and accumulated in F32 over 4 independent sums +#if defined(__AVX512F__) + __m512 acc0 = _mm512_setzero_ps(); + __m512 acc1 = _mm512_setzero_ps(); + __m512 acc2 = _mm512_setzero_ps(); + __m512 acc3 = _mm512_setzero_ps(); + for (; i + 64 <= n; i += 64) { + acc0 = _mm512_fmadd_ps(_mm512_cvtph_ps(_mm256_loadu_si256((const __m256i *)(x + i + 0))), _mm512_loadu_ps(y + i + 0), acc0); + acc1 = _mm512_fmadd_ps(_mm512_cvtph_ps(_mm256_loadu_si256((const __m256i *)(x + i + 16))), _mm512_loadu_ps(y + i + 16), acc1); + acc2 = _mm512_fmadd_ps(_mm512_cvtph_ps(_mm256_loadu_si256((const __m256i *)(x + i + 32))), _mm512_loadu_ps(y + i + 32), acc2); + acc3 = _mm512_fmadd_ps(_mm512_cvtph_ps(_mm256_loadu_si256((const __m256i *)(x + i + 48))), _mm512_loadu_ps(y + i + 48), acc3); + } + for (; i + 16 <= n; i += 16) { + acc0 = _mm512_fmadd_ps(_mm512_cvtph_ps(_mm256_loadu_si256((const __m256i *)(x + i))), _mm512_loadu_ps(y + i), acc0); + } + sumf = _mm512_reduce_add_ps(_mm512_add_ps(_mm512_add_ps(acc0, acc1), _mm512_add_ps(acc2, acc3))); +#elif defined(__AVX__) && defined(__F16C__) + __m256 acc0 = _mm256_setzero_ps(); + __m256 acc1 = _mm256_setzero_ps(); + __m256 acc2 = _mm256_setzero_ps(); + __m256 acc3 = _mm256_setzero_ps(); + for (; i + 32 <= n; i += 32) { + acc0 = GGML_F32x8_FMA(acc0, _mm256_cvtph_ps(_mm_loadu_si128((const __m128i *)(x + i + 0))), _mm256_loadu_ps(y + i + 0)); + acc1 = GGML_F32x8_FMA(acc1, _mm256_cvtph_ps(_mm_loadu_si128((const __m128i *)(x + i + 8))), _mm256_loadu_ps(y + i + 8)); + acc2 = GGML_F32x8_FMA(acc2, _mm256_cvtph_ps(_mm_loadu_si128((const __m128i *)(x + i + 16))), _mm256_loadu_ps(y + i + 16)); + acc3 = GGML_F32x8_FMA(acc3, _mm256_cvtph_ps(_mm_loadu_si128((const __m128i *)(x + i + 24))), _mm256_loadu_ps(y + i + 24)); + } + for (; i + 8 <= n; i += 8) { + acc0 = GGML_F32x8_FMA(acc0, _mm256_cvtph_ps(_mm_loadu_si128((const __m128i *)(x + i))), _mm256_loadu_ps(y + i)); + } + const __m256 acc = _mm256_add_ps(_mm256_add_ps(acc0, acc1), _mm256_add_ps(acc2, acc3)); + __m128 r = _mm_add_ps(_mm256_castps256_ps128(acc), _mm256_extractf128_ps(acc, 1)); + r = _mm_add_ps(r, _mm_movehl_ps(r, r)); + r = _mm_add_ss(r, _mm_movehdup_ps(r)); + sumf = _mm_cvtss_f32(r); +#elif defined(GGML_SIMD) + // convert chunks of x to F32 and reuse the F32 SIMD dot + // TODO: convert F16 to F32 in-register and fuse the dot (needs a new SIMD macro); also avoid the per-chunk call overhead for tiny k + float tx[64]; + for (; i + 64 <= n; i += 64) { + const ggml_fp16_t * GGML_RESTRICT xr = x + i; + float * GGML_RESTRICT tr = tx; + for (int j = 0; j < 64; ++j) { + tr[j] = GGML_CPU_FP16_TO_FP32(xr[j]); + } + float v; + ggml_vec_dot_f32(64, &v, 0, tx, 0, y + i, 0, 1); + sumf += v; + } + if (i < n) { + const int nr = n - i; + const ggml_fp16_t * GGML_RESTRICT xr = x + i; + float * GGML_RESTRICT tr = tx; + for (int j = 0; j < nr; ++j) { + tr[j] = GGML_CPU_FP16_TO_FP32(xr[j]); + } + float v; + ggml_vec_dot_f32(nr, &v, 0, tx, 0, y + i, 0, 1); + sumf += v; + i = n; + } +#endif + for (; i < n; ++i) { + sumf += (ggml_float)(GGML_CPU_FP16_TO_FP32(x[i])*y[i]); + } + + *s = sumf; +} + void ggml_vec_silu_f32(const int n, float * y, const float * x) { int i = 0; #if defined(__AVX512F__) && defined(__AVX512DQ__) @@ -405,7 +484,7 @@ void ggml_vec_silu_f32(const int n, float * y, const float * x) { for (; i + 3 < n; i += 4) { vec_xst(ggml_v_silu(vec_xl(0, x + i)), 0, y + i); } -#elif defined(__riscv_v_intrinsic) +#elif defined(__riscv_v) for (int vl; i < n; i += vl) { vl = __riscv_vsetvl_e32m2(n - i); vfloat32m2_t vx = __riscv_vle32_v_f32m2(&x[i], vl); @@ -446,7 +525,7 @@ void ggml_vec_swiglu_f32(const int n, float * y, const float * x, const float * for (; i + 3 < n; i += 4) { vec_xst(vec_mul(ggml_v_silu(vec_xl(0, x + i)), vec_xl(0, g + i)), 0, y + i); } -#elif defined(__riscv_v_intrinsic) +#elif defined(__riscv_v) for (int vl; i < n; i += vl) { vl = __riscv_vsetvl_e32m2(n - i); vfloat32m2_t vx = __riscv_vle32_v_f32m2(&x[i], vl); @@ -602,7 +681,7 @@ ggml_float ggml_vec_cvar_f32(const int n, float * y, const float * x, const floa val = vec_mul(val, val); sum += (ggml_float)vec_hsum_f32x4(val); } -#elif defined(__riscv_v_intrinsic) +#elif defined(__riscv_v) vfloat64m1_t vsum = __riscv_vfmv_v_f_f64m1(0, 1); for (int vl; i < n; i += vl) { vl = __riscv_vsetvl_e32m2(n - i); @@ -681,7 +760,7 @@ ggml_float ggml_vec_soft_max_f32(const int n, float * y, const float * x, float vec_xst(v_val, 0, y + i); sum += (ggml_float)vec_hsum_f32x4(v_val); } -#elif defined(__riscv_v_intrinsic) +#elif defined(__riscv_v) vfloat64m1_t vsum = __riscv_vfmv_v_f_f64m1(0, 1); for (int avl; i < n; i += avl) { avl = __riscv_vsetvl_e32m2(n - i); diff --git a/ggml/src/ggml-cpu/vec.h b/ggml/src/ggml-cpu/vec.h index f24fa00f2b0d..c54ed3f07787 100644 --- a/ggml/src/ggml-cpu/vec.h +++ b/ggml/src/ggml-cpu/vec.h @@ -70,6 +70,7 @@ extern ggml_fp16_t ggml_table_gelu_quick_f16[1 << 16]; void ggml_vec_dot_f32(int n, float * GGML_RESTRICT s, size_t bs, const float * GGML_RESTRICT x, size_t bx, const float * GGML_RESTRICT y, size_t by, int nrc); void ggml_vec_dot_bf16(int n, float * GGML_RESTRICT s, size_t bs, ggml_bf16_t * GGML_RESTRICT x, size_t bx, ggml_bf16_t * GGML_RESTRICT y, size_t by, int nrc); void ggml_vec_dot_f16(int n, float * GGML_RESTRICT s, size_t bs, ggml_fp16_t * GGML_RESTRICT x, size_t bx, ggml_fp16_t * GGML_RESTRICT y, size_t by, int nrc); +void ggml_vec_dot_f16_f32(int n, float * GGML_RESTRICT s, size_t bs, const ggml_fp16_t * GGML_RESTRICT x, size_t bx, const float * GGML_RESTRICT y, size_t by, int nrc); void ggml_vec_silu_f32(const int n, float * y, const float * x); ggml_float ggml_vec_cvar_f32(const int n, float * y, const float * x, const float mean); //it will also center y ( y = y - mean ) @@ -206,7 +207,7 @@ inline static void ggml_vec_dot_f16_unroll(const int n, const int xs, float * GG sumf[0] = ggml_sve_sum_f32x2(sum_0_lo, sum_0_hi); sumf[1] = ggml_sve_sum_f32x2(sum_1_lo, sum_1_hi); np = n; - #elif defined(__riscv_v_intrinsic) + #elif defined(__riscv_v) #if defined(__riscv_zvfh) size_t vl = __riscv_vsetvlmax_e32m4(); @@ -396,7 +397,7 @@ inline static void ggml_vec_mad_f32(const int n, float * GGML_RESTRICT y, const svst1_f32(pg, y + np2, ay1); } - #elif defined(__riscv_v_intrinsic) + #elif defined(__riscv_v) for (int i = 0, avl; i < n; i += avl) { avl = __riscv_vsetvl_e32m8(n - i); vfloat32m8_t ax = __riscv_vle32_v_f32m8(&x[i], avl); @@ -435,6 +436,41 @@ inline static void ggml_vec_mad_f32(const int n, float * GGML_RESTRICT y, const #endif } +// y (F32) += x (F16)*v +// on AVX512, AVX with F16C, NEON and RVV, x is widened to F32 in-register and fused into the multiply-add +inline static void ggml_vec_mad_f16_f32(const int n, float * GGML_RESTRICT y, const ggml_fp16_t * GGML_RESTRICT x, const float v) { + int i = 0; +#if defined(__AVX512F__) + const __m512 vx = _mm512_set1_ps(v); + for (; i + 16 <= n; i += 16) { + const __m512 ax = _mm512_cvtph_ps(_mm256_loadu_si256((const __m256i *)(x + i))); + _mm512_storeu_ps(y + i, _mm512_fmadd_ps(ax, vx, _mm512_loadu_ps(y + i))); + } +#elif defined(__AVX__) && defined(__F16C__) + const __m256 vx = _mm256_set1_ps(v); + for (; i + 8 <= n; i += 8) { + const __m256 ax = _mm256_cvtph_ps(_mm_loadu_si128((const __m128i *)(x + i))); + _mm256_storeu_ps(y + i, GGML_F32x8_FMA(_mm256_loadu_ps(y + i), ax, vx)); + } +#elif defined(__ARM_NEON) && defined(__ARM_FEATURE_FMA) && defined(__ARM_FP16_FORMAT_IEEE) + const float32x4_t vx = vdupq_n_f32(v); + for (; i + 8 <= n; i += 8) { + const float16x8_t ax = vld1q_f16((const __fp16 *)(x + i)); + vst1q_f32(y + i + 0, vfmaq_f32(vld1q_f32(y + i + 0), vcvt_f32_f16(vget_low_f16 (ax)), vx)); + vst1q_f32(y + i + 4, vfmaq_f32(vld1q_f32(y + i + 4), vcvt_f32_f16(vget_high_f16(ax)), vx)); + } +#elif defined(__riscv_v_intrinsic) && defined(__riscv_zvfhmin) + for (int vl; i < n; i += vl) { + vl = __riscv_vsetvl_e16m2(n - i); + const vfloat32m4_t ax = __riscv_vfwcvt_f_f_v_f32m4(__riscv_vle16_v_f16m2((const _Float16 *)x + i, vl), vl); + __riscv_vse32_v_f32m4(y + i, __riscv_vfmacc_vf_f32m4(__riscv_vle32_v_f32m4(y + i, vl), v, ax, vl), vl); + } +#endif + for (; i < n; ++i) { + y[i] += GGML_CPU_FP16_TO_FP32(x[i])*v; + } +} + inline static void ggml_vec_mad_f16(const int n, ggml_fp16_t * GGML_RESTRICT y, const ggml_fp16_t * GGML_RESTRICT x, const float v) { #if defined(GGML_SIMD) && defined(__ARM_FEATURE_SVE) const int sve_register_length = svcntb() * 8; @@ -513,7 +549,7 @@ inline static void ggml_vec_mad_f16(const int n, ggml_fp16_t * GGML_RESTRICT y, svst1_f16(pg, (__fp16 *)(y + np2), hy); } np = n; -#elif defined(__riscv_v_intrinsic) // implies __riscv_v_intrinsic +#elif defined(__riscv_v) #if defined (__riscv_zvfh) const ggml_fp16_t s = GGML_CPU_FP32_TO_FP16(v); const _Float16 scale = *(const _Float16*)(&s); @@ -599,7 +635,7 @@ inline static void ggml_vec_mad_f32_unroll(const int n, const int xs, const int y[i] += x[k][i]*v[k][0]; } } - #elif defined(__riscv_v_intrinsic) + #elif defined(__riscv_v) for (int i = 0, avl; i < n; i += avl) { avl = __riscv_vsetvl_e32m8(n - i); vfloat32m8_t ay = __riscv_vle32_v_f32m8(&y[i], avl); @@ -660,7 +696,7 @@ inline static void ggml_vec_mad1_f32(const int n, float * y, const float * x, co for (int i = 0; i < n; ++i) { y[i] = x[i]*s + b; } - #elif defined(__riscv_v_intrinsic) + #elif defined(__riscv_v) for (int i = 0, avl; i < n; i += avl) { avl = __riscv_vsetvl_e32m8(n - i); vfloat32m8_t ax = __riscv_vle32_v_f32m8(&x[i], avl); @@ -729,7 +765,7 @@ inline static void ggml_vec_scale_f32(const int n, float * y, const float v) { ay1 = svmul_f32_m(pg, ay1, vx); svst1_f32(pg, y + i, ay1); } - #elif defined(__riscv_v_intrinsic) + #elif defined(__riscv_v) for (int i = 0, avl; i < n; i += avl) { avl = __riscv_vsetvl_e32m8(n - i); vfloat32m8_t ay = __riscv_vle32_v_f32m8(&y[i], avl); @@ -793,7 +829,7 @@ inline static void ggml_vec_scale_f16(const int n, ggml_fp16_t * y, const float svst1_f16(pg, (__fp16 *)(y + np), out); } np = n; -#elif defined(__riscv_v_intrinsic) +#elif defined(__riscv_v) #if defined(__riscv_zvfh) const ggml_fp16_t s = GGML_CPU_FP32_TO_FP16(v); const _Float16 scale = *(const _Float16*)(&s); @@ -1411,7 +1447,7 @@ inline static float32x4_t ggml_v_gelu_quick(float32x4_t x) { return vec_div(x, vec_add(one, exp_cx)); } -#elif defined(__riscv_v_intrinsic) +#elif defined(__riscv_v) // adapted from arm limited optimized routine // the maximum error is 1.45358 plus 0.5 ulps @@ -1469,7 +1505,7 @@ inline static vfloat32m2_t ggml_v_gelu_quick_m2(vfloat32m2_t x, int vl) { return __riscv_vfdiv_vv_f32m2(x, __riscv_vfadd_vf_f32m2(exp_cx, 1.0f, vl), vl); } -#endif // __ARM_NEON / __AVX2__ / __SSE2__ / __VXE__ / __riscv_v_intrinsic +#endif // __ARM_NEON / __AVX2__ / __SSE2__ / __VXE__ / __riscv_v inline static void ggml_vec_silu_f16(const int n, ggml_fp16_t * y, const ggml_fp16_t * x) { for (int i = 0; i < n; ++i) { diff --git a/ggml/src/ggml-cuda/CMakeLists.txt b/ggml/src/ggml-cuda/CMakeLists.txt index 297d9ffc9d4a..dce8827ae8f1 100644 --- a/ggml/src/ggml-cuda/CMakeLists.txt +++ b/ggml/src/ggml-cuda/CMakeLists.txt @@ -163,7 +163,7 @@ if (CUDAToolkit_FOUND) if (GGML_STATIC) if (WIN32) # As of 12.3.1 CUDA Toolkit for Windows does not offer a static cublas library - target_link_libraries(ggml-cuda PRIVATE CUDA::cudart_static CUDA::cublas) + target_link_libraries(ggml-cuda PRIVATE CUDA::cudart_static CUDA::cublas CUDA::cublasLt) else () if (CUDAToolkit_VERSION VERSION_GREATER_EQUAL "10.1") target_link_libraries(ggml-cuda PRIVATE CUDA::cudart_static CUDA::cublas_static CUDA::cublasLt_static) @@ -172,7 +172,7 @@ if (CUDAToolkit_FOUND) endif() endif() else() - target_link_libraries(ggml-cuda PRIVATE CUDA::cudart CUDA::cublas) + target_link_libraries(ggml-cuda PRIVATE CUDA::cudart CUDA::cublas CUDA::cublasLt) endif() if (GGML_CUDA_NO_VMM) diff --git a/ggml/src/ggml-cuda/batch-get.cu b/ggml/src/ggml-cuda/batch-get.cu new file mode 100644 index 000000000000..60c4747b967c --- /dev/null +++ b/ggml/src/ggml-cuda/batch-get.cu @@ -0,0 +1,198 @@ +#include "batch-get.cuh" +#include "common.cuh" +#include "ggml-cpp.h" + +#include + +#if !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA) +struct ggml_cuda_batch_get_descriptor { + const unsigned char * src; + unsigned char * dst; + size_t size; +}; + +static __global__ void ggml_cuda_batch_get_kernel(const ggml_cuda_batch_get_descriptor * copies, size_t count) { + // Blocks along y cooperate on each descriptor. + const size_t index = blockIdx.x; + if (index >= count) { + return; + } + const size_t first = (size_t) blockIdx.y * blockDim.x + threadIdx.x; + const size_t stride = (size_t) gridDim.y * blockDim.x; + const auto copy = copies[index]; + // Use 16-, 8-, 4-, or 2-byte copies when both pointers meet that alignment; threads copy separate chunks. + const uintptr_t alignment = (uintptr_t) copy.src | (uintptr_t) copy.dst; + size_t tail = 0; + if (alignment % 16 == 0) { + for (size_t i = first; i < copy.size / 16; i += stride) { + ggml_cuda_memcpy_1<16>(copy.dst + 16*i, copy.src + 16*i); + } + tail = copy.size / 16 * 16; + } else if (alignment % 8 == 0) { + for (size_t i = first; i < copy.size / 8; i += stride) { + ggml_cuda_memcpy_1<8>(copy.dst + 8*i, copy.src + 8*i); + } + tail = copy.size / 8 * 8; + } else if (alignment % 4 == 0) { + for (size_t i = first; i < copy.size / 4; i += stride) { + ggml_cuda_memcpy_1<4>(copy.dst + 4*i, copy.src + 4*i); + } + tail = copy.size / 4 * 4; + } else if (alignment % 2 == 0) { + for (size_t i = first; i < copy.size / 2; i += stride) { + ggml_cuda_memcpy_1<2>(copy.dst + 2*i, copy.src + 2*i); + } + tail = copy.size / 2 * 2; + } + // Copy trailing bytes, or the entire range when no wider alignment is available. + for (size_t i = tail + first; i < copy.size; i += stride) { + ggml_cuda_memcpy_1<1>(copy.dst + i, copy.src + i); + } +} + +struct ggml_cuda_batch_get_mapping { + void * host = nullptr; + unsigned char * device = nullptr; + + ggml_cuda_batch_get_mapping(void * host, size_t size) { + const cudaError_t err = cudaHostRegister(host, size, cudaHostRegisterMapped | cudaHostRegisterPortable); + if (err != cudaSuccess) { + (void) cudaGetLastError(); + GGML_LOG_DEBUG("%s: failed to register %.2f MiB of pinned memory: %s\n", __func__, + size / 1024.0 / 1024.0, cudaGetErrorString(err)); + return; + } + this->host = host; + if (cudaHostGetDevicePointer((void **) &device, host, 0) != cudaSuccess) { + (void) cudaGetLastError(); + device = nullptr; + return; + } + } + + ggml_cuda_batch_get_mapping(const ggml_cuda_batch_get_mapping &) = delete; + ggml_cuda_batch_get_mapping & operator=(const ggml_cuda_batch_get_mapping &) = delete; + + ~ggml_cuda_batch_get_mapping() { + if (host) { + // Release only the registration acquired by this object. + CUDA_CHECK(cudaHostUnregister(host)); + } + } +}; +#endif + +bool ggml_cuda_get_tensor_n(int device, + const ggml_tensor * const * tensors, void * const * data, const size_t * offsets, + const size_t * sizes, size_t n_tensors) { +#if defined(GGML_USE_HIP) || defined(GGML_USE_MUSA) + GGML_UNUSED(device); + GGML_UNUSED(tensors); + GGML_UNUSED(data); + GGML_UNUSED(offsets); + GGML_UNUSED(sizes); + GGML_UNUSED(n_tensors); + return false; +#else + // Provisional dispatch values based on experiments. + constexpr size_t min_batch_reads = 128; + constexpr size_t max_gather_read_bytes = 128 * 1024; + // Collect non-empty CPU destination ranges. + std::vector> ranges; + ranges.reserve(n_tensors); + size_t payload_bytes = 0; + // Number of non-empty reads whose size is at most max_gather_read_bytes. + size_t n_reads_small = 0; + for (size_t i = 0; i < n_tensors; ++i) { + if (sizes[i] == 0) { + continue; + } + const uintptr_t address = (uintptr_t) data[i]; + GGML_ASSERT(address <= UINTPTR_MAX - sizes[i]); + GGML_ASSERT(payload_bytes <= SIZE_MAX - sizes[i]); + payload_bytes += sizes[i]; + n_reads_small += sizes[i] <= max_gather_read_bytes; + ranges.emplace_back(address, address + sizes[i]); + } + if (ranges.empty()) { + return true; + } + // Total number of non-empty reads. + const size_t n_reads = ranges.size(); + if (n_reads > SIZE_MAX / sizeof(ggml_cuda_batch_get_descriptor) || n_reads > INT_MAX) { + return false; + } + if (n_reads < 2 || (n_reads < min_batch_reads && n_reads_small == n_reads)) { + return false; + } + std::sort(ranges.begin(), ranges.end()); + for (size_t i = 1; i < ranges.size(); ++i) { + // Preserve the ordered copy path when CPU destinations overlap. + if (ranges[i].first < ranges[i - 1].second) { + return false; + } + } + // Find the CPU span to register, including gaps between destinations. + const uintptr_t begin = ranges.front().first; + const uintptr_t end = ranges.back().second; + if (end - begin - payload_bytes > payload_bytes) { + return false; + } + // Register and map the existing CPU destination so the kernel can write to it directly. + ggml_cuda_batch_get_mapping mapping((void *) begin, end - begin); + if (!mapping.host) { + return false; + } + // Reads selected for gather; zero means all reads use DMA. + size_t n_reads_gather = mapping.device && n_reads_small >= min_batch_reads ? n_reads_small : 0; + // Keep descriptor storage alive until the final stream synchronization. + constexpr size_t max_batch_descriptors = 128 * 1024; + const size_t bytes = std::min(n_reads_gather, max_batch_descriptors) * sizeof(ggml_cuda_batch_get_descriptor); + ggml_backend_buffer_ptr descriptors(n_reads_gather ? ggml_backend_buft_alloc_buffer(ggml_backend_cuda_buffer_type(device), bytes) : nullptr); + if (!descriptors) { + n_reads_gather = 0; + } + const auto host_type = ggml_backend_cuda_host_buffer_type(); + ggml_backend_buffer_ptr host_descriptors(n_reads_gather ? ggml_backend_buft_alloc_buffer(host_type, bytes) : nullptr); + if (!host_descriptors || ggml_backend_buffer_get_type(host_descriptors.get()) != host_type) { + n_reads_gather = 0; + } + if (n_reads_gather) { + auto * copies = static_cast(ggml_backend_buffer_get_base(host_descriptors.get())); + auto * device_copies = static_cast(ggml_backend_buffer_get_base(descriptors.get())); + const int threads = 256; + const size_t nsm = ggml_cuda_info().devices[device].nsm; + size_t i = 0; + for (size_t first = 0; first < n_reads_gather; first += max_batch_descriptors) { + const size_t n_batch = std::min(n_reads_gather - first, max_batch_descriptors); + size_t count = 0; + for (; i < n_tensors && count < n_batch; ++i) { + if (sizes[i] == 0 || sizes[i] > max_gather_read_bytes) { + continue; + } + copies[count++] = {(const unsigned char *) tensors[i]->data + offsets[i], + mapping.device + ((uintptr_t) data[i] - begin), sizes[i]}; + } + const size_t blocks_per_copy = std::min(UINT16_MAX, std::max(1, (nsm + n_batch - 1) / n_batch)); + const dim3 blocks((unsigned int) n_batch, (unsigned int) blocks_per_copy); + // Stream ordering keeps each kernel ahead of the next descriptor overwrite. + CUDA_CHECK(cudaMemcpyAsync(device_copies, copies, n_batch * sizeof(copies[0]), cudaMemcpyHostToDevice, cudaStreamPerThread)); + ggml_cuda_batch_get_kernel<<>>(device_copies, n_batch); + CUDA_CHECK(cudaGetLastError()); + if (first + n_batch < n_reads_gather) { + // Wait before refilling the pinned host descriptor buffer. + CUDA_CHECK(cudaStreamSynchronize(cudaStreamPerThread)); + } + } + } + // Keep the destination pinned for DMA too; overlapping destinations already declined above. + for (size_t i = 0; i < n_tensors; ++i) { + if (sizes[i] && (!n_reads_gather || sizes[i] > max_gather_read_bytes)) { + CUDA_CHECK(cudaMemcpyAsync(data[i], (const char *) tensors[i]->data + offsets[i], sizes[i], cudaMemcpyDeviceToHost, cudaStreamPerThread)); + } + } + // Wait for GPU writes before releasing the CPU mapping and descriptor buffers. + CUDA_CHECK(cudaStreamSynchronize(cudaStreamPerThread)); + return true; +#endif +} diff --git a/ggml/src/ggml-cuda/batch-get.cuh b/ggml/src/ggml-cuda/batch-get.cuh new file mode 100644 index 000000000000..eb9bf0cc81bd --- /dev/null +++ b/ggml/src/ggml-cuda/batch-get.cuh @@ -0,0 +1,7 @@ +#pragma once + +#include "ggml-backend.h" + +bool ggml_cuda_get_tensor_n(int device, + const ggml_tensor * const * tensors, void * const * data, const size_t * offsets, + const size_t * sizes, size_t n_tensors); diff --git a/ggml/src/ggml-cuda/common.cuh b/ggml/src/ggml-cuda/common.cuh index a46c4d3f096f..41417730827c 100644 --- a/ggml/src/ggml-cuda/common.cuh +++ b/ggml/src/ggml-cuda/common.cuh @@ -7,7 +7,6 @@ #include #include #include -#include #if defined(GGML_USE_HIP) #define GGML_COMMON_DECL_HIP @@ -133,6 +132,14 @@ static __device__ __forceinline__ void ggml_cuda_syncwarp() { #endif // GGML_USE_HIP } +static __device__ __forceinline__ void ggml_cuda_prefetch_l2(const void * p) { +#if !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA) + asm volatile("prefetch.global.L2 [%0];" :: "l"(p)); +#else + GGML_UNUSED(p); +#endif // !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA) +} + static __device__ __forceinline__ void ggml_cuda_pdl_sync() { #if defined(GGML_CUDA_USE_PDL) && __CUDA_ARCH__ >= GGML_CUDA_CC_HOPPER cudaGridDependencySynchronize(); @@ -389,6 +396,10 @@ static bool blackwell_mma_available(const int cc) { ggml_cuda_highest_compiled_arch(cc) < GGML_CUDA_CC_RUBIN; } +static bool fp8_mma_hardware_available(const int cc) { + return GGML_CUDA_CC_IS_NVIDIA(cc) && cc >= GGML_CUDA_CC_ADA_LOVELACE && cc < GGML_CUDA_CC_RUBIN; +} + // Checks whether the tensor's base data pointer and higher-dimensional strides are byte-aligned to `alignment` bytes. static bool ggml_cuda_is_aligned(const ggml_tensor * tensor, const size_t alignment) { GGML_ASSERT(tensor != nullptr); @@ -884,6 +895,67 @@ static __device__ __forceinline__ float ggml_cuda_ue4m3_to_fp32(uint8_t x) { #endif // defined(GGML_USE_HIP) && defined(CDNA3) && defined(FP8_AVAILABLE) && HIP_VERSION >= 60200000 } +static __host__ __device__ __forceinline__ float ggml_cuda_f8_e4m3_to_fp32(uint8_t x) { +#if defined(FP8_AVAILABLE) && !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA) + __nv_fp8_e4m3 xf; + xf.__x = x; + return static_cast(xf); +#else + const uint8_t magnitude = x & 0x7F; + if (magnitude == 0x7F) { + return NAN; + } + + const int exp = (magnitude >> 3) & 0x0F; + const int man = magnitude & 0x07; + float value; + if (exp == 0) { + value = ldexpf((float) man, -9); + } else { + value = ldexpf(1.0f + (float) man / 8.0f, exp - 7); + } + return x & 0x80 ? -value : value; +#endif // defined(FP8_AVAILABLE) && !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA) +} + +static __host__ __device__ __forceinline__ uint8_t ggml_cuda_fp32_to_f8_e4m3(float x) { +#if defined(FP8_AVAILABLE) && !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA) + return __nv_cvt_float_to_fp8(x, __NV_SATFINITE, __NV_E4M3); +#elif !defined(__CUDA_ARCH__) && !defined(__HIP_DEVICE_COMPILE__) && !defined(__MUSA_ARCH__) + // ggml_cuda_cast also runs on the host, where __float2int_rn is not available. + return ggml_fp32_to_f8_e4m3(x); +#else + const uint8_t sign = signbit(x) ? 0x80 : 0; + x = fabsf(x); + + if (isnan(x)) { + return sign | 0x7F; + } + if (x == 0.0f) { + return sign; + } + if (isinf(x) || x >= 448.0f) { + return sign | 0x7E; + } + if (x < 0.015625f) { + return sign | (uint8_t) __float2int_rn(x * 512.0f); + } + + int exp; + const float mantissa = frexpf(x, &exp) * 2.0f; + int encoded_exp = exp + 6; + int encoded_man = __float2int_rn((mantissa - 1.0f) * 8.0f); + if (encoded_man == 8) { + encoded_man = 0; + encoded_exp++; + } + if (encoded_exp > 15 || (encoded_exp == 15 && encoded_man > 6)) { + return sign | 0x7E; + } + return sign | (uint8_t) (encoded_exp << 3) | (uint8_t) encoded_man; +#endif // defined(FP8_AVAILABLE) && !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA) +} + static __device__ __forceinline__ uint8_t ggml_cuda_fp32_to_ue4m3(float x) { #if defined(BLACKWELL_MMA_AVAILABLE) // This is used for NVFP4 subblock scale quantizations only if (!(x > 0.0f)) { @@ -1207,6 +1279,8 @@ const ggml_cuda_device_info & ggml_cuda_info(); void ggml_cuda_set_device(int device); int ggml_cuda_get_device(); +bool ggml_cuda_pool_debug_enabled(); + struct ggml_cuda_pool { virtual ~ggml_cuda_pool() = default; @@ -1271,186 +1345,9 @@ struct ggml_tensor_extra_gpu { #define USE_CUDA_GRAPH #endif -struct ggml_cuda_graph { -#ifdef USE_CUDA_GRAPH - ~ggml_cuda_graph() { - for (cudaGraphExec_t instance : instances) { - CUDA_CHECK(cudaGraphExecDestroy(instance)); - } - } - // the ggml graph is captured as a series of CUDA graphs, so the GPU can run one chunk while the next is enqueued - std::vector instances; - size_t num_nodes = 0; - std::vector nodes; - bool disable_due_to_gpu_arch = false; - bool warmup_complete = false; - uint64_t uid = 0; - int64_t last_used_time = 0; - struct node_properties { - ggml_tensor node; - void * node_src_data_ptrs[GGML_MAX_SRC]; - int64_t node_src_ne[GGML_MAX_SRC][GGML_MAX_DIMS]; - size_t node_src_nb[GGML_MAX_SRC][GGML_MAX_DIMS]; - }; - std::vector node_props; - - bool is_enabled() const { - static const bool disable_cuda_graphs_due_to_env = (getenv("GGML_CUDA_DISABLE_GRAPHS") != nullptr); - return !(disable_due_to_gpu_arch || disable_cuda_graphs_due_to_env); - } -#endif -}; - -struct ggml_cuda_concurrent_event { - std::vector join_events; - cudaEvent_t fork_event = nullptr; - - int n_streams = 0; - std::unordered_map stream_mapping; - - // Original order of nodes in this concurrent region (before interleaving) - // Used to restore grouping for fusion within streams - std::vector original_order; - - const ggml_tensor * join_node; - - ggml_cuda_concurrent_event() = default; - - ggml_cuda_concurrent_event(const ggml_cuda_concurrent_event &) = delete; - ggml_cuda_concurrent_event & operator=(const ggml_cuda_concurrent_event &) = delete; - - explicit ggml_cuda_concurrent_event(int n_streams) : n_streams(n_streams) { - join_events.resize(n_streams); - - for (size_t i = 0; i < join_events.size(); ++i) { - CUDA_CHECK(cudaEventCreateWithFlags(&join_events[i], cudaEventDisableTiming)); - } - - CUDA_CHECK(cudaEventCreateWithFlags(&fork_event, cudaEventDisableTiming)); - } - - ggml_cuda_concurrent_event(ggml_cuda_concurrent_event && other) noexcept - : join_events(std::move(other.join_events)) - , fork_event(other.fork_event) - , n_streams(other.n_streams) - , stream_mapping(std::move(other.stream_mapping)) - , original_order(std::move(other.original_order)) - , join_node(other.join_node) { - other.fork_event = nullptr; - } - - // 1. check if any branches write to overlapping memory ranges (except the join node) - // 2. check whether all srcs are either within the branch or outside the nodes covered by ggml_cuda_concurrent_event - // we assume all nodes have the same buffer - bool is_valid() const { - std::vector>> write_ranges; - write_ranges.resize(n_streams); - - // get join_node's memory range to exclude from overlap checking. - // multiple nodes can use join_node's buffer; we synchronize on the join node. - const ggml_tensor * join_t = join_node->view_src ? join_node->view_src : join_node; - const int64_t join_start = (int64_t) join_t->data; - const int64_t join_end = join_start + ggml_nbytes(join_t); - - for (const auto & [tensor, stream] : stream_mapping) { - const ggml_tensor * t = tensor->view_src ? tensor->view_src : tensor; - const int64_t t_start = (int64_t) t->data; - const int64_t t_end = t_start + ggml_nbytes(t); - - // skip tensors that overlap with join_node's buffer. - if ((t_start <= join_start && join_start < t_end) || (join_start <= t_start && t_start < join_end)) { - continue; - } - - // concurrent streams begin from 1 - write_ranges[stream - 1].emplace_back(t_start, t_end); - } - - for (int i = 0; i < n_streams; ++i) { - // sorts first by start then by end of write range - std::sort(write_ranges[i].begin(), write_ranges[i].end()); - } - - bool writes_overlap = false; - bool dependent_srcs = false; - for (const auto & [tensor, stream] : stream_mapping) { - const ggml_tensor * t = tensor->view_src ? tensor->view_src : tensor; - const int64_t t_start = (int64_t) t->data; - const int64_t t_end = t_start + ggml_nbytes(t); - - // skip tensors that overlap with join_node's buffer - if ((t_start <= join_start && join_start < t_end) || (join_start <= t_start && t_start < join_end)) { - continue; - } - - // check if this buffer's write data overlaps with another stream's - std::pair data_range = std::make_pair(t_start, t_end); - for (int i = 0; i < n_streams; ++i) { - if (i == stream - 1) { - continue; - } - auto it = std::lower_bound(write_ranges[i].begin(), write_ranges[i].end(), data_range); - - if (it != write_ranges[i].end()) { - const std::pair & other = *it; - - // std::lower_bound returns the first element where other >= data_range (lexicographically). - // This guarantees other.first >= data_range.first. - // Therefore, overlap occurs iff other.first < data_range.second - // (i.e., the other range starts before this range ends). - if (other.first < data_range.second) { - GGML_LOG_DEBUG("Writes overlap for %s", tensor->name); - writes_overlap = true; - break; - } - } - } - - //check if all srcs are either in branch or don't have a branch - for (int i = 0; i < GGML_MAX_SRC; ++i) { - if (!tensor->src[i]) { - continue; - } - - auto it = stream_mapping.find(tensor->src[i]); - - if (it == stream_mapping.end()) { - continue; - } - - if (it->second != stream) { - dependent_srcs = true; - break; - } - } - - if (dependent_srcs || writes_overlap) { - break; - } - } - - return !writes_overlap && !dependent_srcs; - } - - ~ggml_cuda_concurrent_event() { - if (fork_event != nullptr) { - CUDA_CHECK(cudaEventDestroy(fork_event)); - } - for (cudaEvent_t e : join_events) { - if (e != nullptr) { - CUDA_CHECK(cudaEventDestroy(e)); - } - } - } -}; - -struct ggml_cuda_stream_context { - std::unordered_map concurrent_events; - - void reset() { - concurrent_events.clear(); - } -}; +// defined in graph.cuh +struct ggml_cuda_graph; +struct ggml_cuda_stream_context; struct ggml_backend_cuda_context { int device; @@ -1461,6 +1358,9 @@ struct ggml_backend_cuda_context { cublasHandle_t cublas_handles[GGML_CUDA_MAX_DEVICES][GGML_CUDA_MAX_STREAMS] = {}; void * cublas_workspaces[GGML_CUDA_MAX_DEVICES][GGML_CUDA_MAX_STREAMS] = {}; size_t cublas_workspace_sizes[GGML_CUDA_MAX_DEVICES] = {}; +#if !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA) && CUDART_VERSION >= 11080 + cublasLtHandle_t cublaslt_handles[GGML_CUDA_MAX_DEVICES] = {}; +#endif int curr_stream_no = 0; @@ -1469,59 +1369,23 @@ struct ggml_backend_cuda_context { // when the computation is split across CPU/GPU (e.g., with --n-cpu-moe) std::unordered_map> cuda_graphs; - int64_t last_graph_eviction_sweep = 0; - - ggml_cuda_graph * cuda_graph(const void * first_node_ptr) { - const int64_t time_now = ggml_time_us(); - - // sweep every 5s, evicting cuda graphs unused for >=10s - if (time_now - last_graph_eviction_sweep >= 5'000'000) { - last_graph_eviction_sweep = time_now; - for (auto it = cuda_graphs.begin(); it != cuda_graphs.end(); ) { - if (time_now - it->second->last_used_time >= 10'000'000) { - it = cuda_graphs.erase(it); - } else { - ++it; - } - } - } + int64_t next_graph_eviction_time = 0; + int64_t last_graph_use_time = 0; - auto it = cuda_graphs.find(first_node_ptr); - if (it == cuda_graphs.end()) { - it = cuda_graphs.emplace(first_node_ptr, std::make_unique()).first; - } - it->second->last_used_time = time_now; - return it->second.get(); - } + // defined in ggml-cuda.cu, which includes graph.cuh + ggml_cuda_graph * cuda_graph(const void * first_node_ptr, int64_t time_now = ggml_time_us()); // Check if any CUDA graph is enabled for this context (used by kernels that need to know // if graphs are in use without having access to the specific graph key) - bool any_cuda_graph_enabled() const { - for (const auto & [key, graph] : cuda_graphs) { - if (graph && graph->is_enabled()) { - return true; - } - } - return false; - } + bool any_cuda_graph_enabled() const; // Check if any CUDA graph has an instance for this context - bool any_cuda_graph_has_instance() const { - for (const auto & [key, graph] : cuda_graphs) { - if (graph && !graph->instances.empty()) { - return true; - } - } - return false; - } + bool any_cuda_graph_has_instance() const; #endif // USE_CUDA_GRAPH - explicit ggml_backend_cuda_context(int device) : - device(device), - name(GGML_CUDA_NAME + std::to_string(device)) { - } + explicit ggml_backend_cuda_context(int device); - ggml_cuda_stream_context concurrent_stream_context; + std::unique_ptr concurrent_stream_context; ~ggml_backend_cuda_context(); @@ -1535,7 +1399,7 @@ struct ggml_backend_cuda_context { cudaStream_t stream() { return stream(device, curr_stream_no); } - ggml_cuda_stream_context & stream_context() { return concurrent_stream_context; } + ggml_cuda_stream_context & stream_context() { return *concurrent_stream_context; } cublasHandle_t cublas_handle() { if (cublas_handles[device][curr_stream_no] == nullptr) { @@ -1543,10 +1407,14 @@ struct ggml_backend_cuda_context { CUBLAS_CHECK(cublasCreate(&cublas_handles[device][curr_stream_no])); CUBLAS_CHECK(cublasSetMathMode(cublas_handles[device][curr_stream_no], CUBLAS_TF32_TENSOR_OP_MATH)); CUBLAS_CHECK(cublasSetStream(cublas_handles[device][curr_stream_no], stream())); -#if !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA) && (CUBLAS_VER_MAJOR > 11 || (CUBLAS_VER_MAJOR == 11 && CUBLAS_VER_MINOR >= 2)) +#if (defined(GGML_USE_HIP) && HIP_VERSION >= 70000000) || (!defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA) && (CUBLAS_VER_MAJOR > 11 || (CUBLAS_VER_MAJOR == 11 && CUBLAS_VER_MINOR >= 2))) if (cublas_workspace_sizes[device] == 0) { +#if defined(GGML_USE_HIP) + cublas_workspace_sizes[device] = 32 * 1024 * 1024; +#else const int cc = ggml_cuda_info().devices[device].cc; cublas_workspace_sizes[device] = (cc >= GGML_CUDA_CC_HOPPER) ? 32 * 1024 * 1024 : 4 * 1024 * 1024; +#endif } CUDA_CHECK(cudaMalloc(&cublas_workspaces[device][curr_stream_no], cublas_workspace_sizes[device])); CUBLAS_CHECK(cublasSetWorkspace(cublas_handles[device][curr_stream_no], cublas_workspaces[device][curr_stream_no], cublas_workspace_sizes[device])); @@ -1555,6 +1423,16 @@ struct ggml_backend_cuda_context { return cublas_handles[device][curr_stream_no]; } +#if !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA) && CUDART_VERSION >= 11080 + cublasLtHandle_t cublaslt_handle() { + if (cublaslt_handles[device] == nullptr) { + ggml_cuda_set_device(device); + CUBLAS_CHECK(cublasLtCreate(&cublaslt_handles[device])); + } + return cublaslt_handles[device]; + } +#endif + // pool std::unique_ptr pools[GGML_CUDA_MAX_DEVICES][GGML_CUDA_MAX_STREAMS]; @@ -1580,6 +1458,7 @@ struct ggml_cuda_mm_fusion_args_host { const ggml_tensor * gate_scale = nullptr; ggml_glu_op glu_op; float glu_limit = 0.0f; + ggml_prec prec_src1 = GGML_PREC_UNDEFINED; // src1 precision requested by the fused MUL_MATs (MMQ) const ggml_tensor * shared_up = nullptr; const ggml_tensor * shared_gate = nullptr; ggml_tensor * shared_dst = nullptr; @@ -1638,60 +1517,8 @@ struct ggml_cuda_pdl_config { }; -static bool ggml_cuda_kernel_can_use_pdl(const void * kernel) { - const int device = ggml_cuda_get_device(); - - struct cache_key { - int device; - const void * kernel; - - bool operator==(const cache_key & other) const { return device == other.device && kernel == other.kernel; } - }; - - struct cache_key_hash { - // MurmurHash3 mixing function for better hash distribution (vs. just std::hash which in some implementations simply returns the identity) - static size_t hash_mix(size_t x) { - std::uint64_t y = x; - const std::uint64_t m = 0xe9846af9b1a615d; - - y ^= y >> 32; - y *= m; - y ^= y >> 32; - y *= m; - y ^= y >> 28; - - return static_cast(y); - } - - size_t operator()(const cache_key & key) const { - // Use a nonzero seed to avoid mapping all-zero keys to zero - size_t h = 42; - h = hash_mix(h + key.device); - h = hash_mix(h + reinterpret_cast(key.kernel)); - return h; - } - }; - - static std::mutex cache_mutex; - static std::unordered_map cache; - - const cache_key key = { device, kernel }; - std::lock_guard lock(cache_mutex); - const auto it = cache.find(key); - if (it != cache.end()) { - return it->second; - } - - cudaFuncAttributes attr = {}; - CUDA_CHECK(cudaFuncGetAttributes(&attr, kernel)); - - // PDL device-side primitives are emitted only for PTX versions >= 90. - // We have to guard on a loaded kernel's PTX version so a kernel forward-JIT'ed - // from pre-Hopper PTX to a Hopper-or-newer GPU does not opt into PDL. - const bool can_use_pdl = attr.ptxVersion >= 90; - cache.emplace(key, can_use_pdl); - return can_use_pdl; -} +// whether kernel can be launched with PDL on the current device (also honors GGML_CUDA_PDL=0), defined in ggml-cuda.cu +bool ggml_cuda_kernel_can_use_pdl(const void * kernel); #endif //defined(GGML_CUDA_USE_PDL) @@ -1705,13 +1532,7 @@ static bool ggml_cuda_kernel_can_use_pdl(const void * kernel) { template static __inline__ void ggml_cuda_kernel_launch(Kernel kernel, const ggml_cuda_kernel_launch_params & launch_params, Args&&... args) { #if defined(GGML_CUDA_USE_PDL) - - static const bool env_pdl_enabled = []() { - const char * env = getenv("GGML_CUDA_PDL"); - return env == nullptr || std::atoi(env) != 0; - }(); - - if (env_pdl_enabled && ggml_cuda_kernel_can_use_pdl(reinterpret_cast(kernel))) { + if (ggml_cuda_kernel_can_use_pdl(reinterpret_cast(kernel))) { auto pdl_cfg = ggml_cuda_pdl_config(launch_params); CUDA_CHECK(cudaLaunchKernelEx(&pdl_cfg.cfg, kernel, std::forward(args)... )); diff --git a/ggml/src/ggml-cuda/convert.cu b/ggml/src/ggml-cuda/convert.cu index 77fd28894516..a8f5047a2302 100644 --- a/ggml/src/ggml-cuda/convert.cu +++ b/ggml/src/ggml-cuda/convert.cu @@ -541,6 +541,8 @@ to_bf16_cuda_t ggml_get_to_bf16_cuda(ggml_type type) { return dequantize_row_mxfp4_cuda; case GGML_TYPE_NVFP4: return dequantize_row_nvfp4_cuda; + case GGML_TYPE_F8_E4M3: + return convert_unary_cont_cuda; case GGML_TYPE_F32: return convert_unary_cont_cuda; case GGML_TYPE_F16: @@ -601,6 +603,8 @@ to_fp16_cuda_t ggml_get_to_fp16_cuda(ggml_type type) { return dequantize_row_mxfp4_cuda; case GGML_TYPE_NVFP4: return dequantize_row_nvfp4_cuda; + case GGML_TYPE_F8_E4M3: + return convert_unary_cont_cuda; case GGML_TYPE_F32: return convert_unary_cont_cuda; case GGML_TYPE_BF16: @@ -658,6 +662,8 @@ to_fp32_cuda_t ggml_get_to_fp32_cuda(ggml_type type) { return dequantize_row_mxfp4_cuda; case GGML_TYPE_NVFP4: return dequantize_row_nvfp4_cuda; + case GGML_TYPE_F8_E4M3: + return convert_unary_cont_cuda; case GGML_TYPE_F16: return convert_unary_cont_cuda; case GGML_TYPE_BF16: @@ -685,6 +691,8 @@ to_fp16_nc_cuda_t ggml_get_to_fp16_nc_cuda(ggml_type type) { return dequantize_block_cuda; case GGML_TYPE_Q8_0: return dequantize_block_cuda; + case GGML_TYPE_F8_E4M3: + return convert_unary_cuda; case GGML_TYPE_BF16: return convert_unary_cuda; default: @@ -710,6 +718,8 @@ to_bf16_nc_cuda_t ggml_get_to_bf16_nc_cuda(ggml_type type) { return dequantize_block_cuda; case GGML_TYPE_Q8_0: return dequantize_block_cuda; + case GGML_TYPE_F8_E4M3: + return convert_unary_cuda; case GGML_TYPE_F16: return convert_unary_cuda; default: @@ -735,6 +745,8 @@ to_fp32_nc_cuda_t ggml_get_to_fp32_nc_cuda(ggml_type type) { return dequantize_block_cuda; case GGML_TYPE_Q8_0: return dequantize_block_cuda; + case GGML_TYPE_F8_E4M3: + return convert_unary_cuda; case GGML_TYPE_BF16: return convert_unary_cuda; default: diff --git a/ggml/src/ggml-cuda/convert.cuh b/ggml/src/ggml-cuda/convert.cuh index 91255ccf31aa..fb2a1b3bc09e 100644 --- a/ggml/src/ggml-cuda/convert.cuh +++ b/ggml/src/ggml-cuda/convert.cuh @@ -35,6 +35,36 @@ template __host__ __device__ inline dst_t ggml_cuda_cast(src_t x) { if constexpr (std::is_same_v) { return x; + } else if constexpr (std::is_same_v) { + return { ggml_cuda_fp32_to_f8_e4m3(ggml_cuda_cast(x)) }; + } else if constexpr (std::is_same_v) { +#if defined(FP8_AVAILABLE) && !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA) +#if CUDART_VERSION >= 13040 + if constexpr (std::is_same_v) { + return static_cast(__nv_cvt_fp8_to_bf16raw(x.bits, __NV_E4M3)); + } +#endif + if constexpr (std::is_same_v) { + return static_cast(__nv_cvt_fp8_to_halfraw(x.bits, __NV_E4M3)); + } +#endif + return ggml_cuda_cast(ggml_cuda_f8_e4m3_to_fp32(x.bits)); +#if defined(FP8_AVAILABLE) && !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA) + } else if constexpr (std::is_same_v && std::is_same_v) { +#if CUDART_VERSION >= 13040 + return static_cast(__nv_cvt_fp8x2_to_bf162raw(x.__x, __NV_E4M3)); +#else + return make_bfloat162( + ggml_cuda_cast(ggml_cuda_f8_e4m3_to_fp32(x.__x & 0xFF)), + ggml_cuda_cast(ggml_cuda_f8_e4m3_to_fp32(x.__x >> 8))); +#endif + } else if constexpr (std::is_same_v && std::is_same_v) { +#if CUDART_VERSION >= 13040 + return ggml_cuda_cast(ggml_cuda_cast(x)); +#else + return __half22float2(static_cast(__nv_cvt_fp8x2_to_halfraw2(x.__x, __NV_E4M3))); +#endif +#endif } else if constexpr(std::is_same_v) { return __float2bfloat16(float(x)); } else if constexpr(std::is_same_v) { diff --git a/ggml/src/ggml-cuda/cp-async.cuh b/ggml/src/ggml-cuda/cp-async.cuh index 63d0c482ff72..1071352e7b94 100644 --- a/ggml/src/ggml-cuda/cp-async.cuh +++ b/ggml/src/ggml-cuda/cp-async.cuh @@ -55,3 +55,23 @@ static __device__ __forceinline__ void cp_async_wait_all() { NO_DEVICE_CODE; #endif // CP_ASYNC_AVAILABLE } + +// Commits the copies issued since the last commit as one group. +static __device__ __forceinline__ void cp_async_commit_group() { +#ifdef CP_ASYNC_AVAILABLE + asm volatile("cp.async.commit_group;"); +#else + NO_DEVICE_CODE; +#endif // CP_ASYNC_AVAILABLE +} + +// Makes each thread wait until at most n of its copy groups are pending. +// Like cp_async_wait_all, this does NOT provide any additional synchronization. +template +static __device__ __forceinline__ void cp_async_wait_group() { +#ifdef CP_ASYNC_AVAILABLE + asm volatile("cp.async.wait_group %0;" : : "n"(n)); +#else + NO_DEVICE_CODE; +#endif // CP_ASYNC_AVAILABLE +} diff --git a/ggml/src/ggml-cuda/cpy.cu b/ggml/src/ggml-cuda/cpy.cu index 90a06b97bbc6..f1f1bf2dabcc 100644 --- a/ggml/src/ggml-cuda/cpy.cu +++ b/ggml/src/ggml-cuda/cpy.cu @@ -477,6 +477,41 @@ void ggml_cuda_cpy(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, gg } else if (ggml_cuda_cpy_as_memcpy_2d(src0, src1, mc_width, mc_height, mc_spitch, mc_dpitch)) { CUDA_CHECK(cudaMemcpy2DAsync(src1_ddc, mc_dpitch, src0_ddc, mc_spitch, mc_width, mc_height, cudaMemcpyDeviceToDevice, main_stream)); + } else if (src0->type == GGML_TYPE_F8_E4M3 && src1->type == GGML_TYPE_F8_E4M3) { + ggml_cpy_scalar_cuda + (src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream); + } else if (src0->type == GGML_TYPE_F32 && src1->type == GGML_TYPE_F8_E4M3) { + if (contiguous_srcs) { + ggml_cpy_scalar_contiguous_cuda + (src0_ddc, src1_ddc, ne, main_stream); + } else { + ggml_cpy_scalar_cuda + (src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream); + } + } else if (src0->type == GGML_TYPE_F16 && src1->type == GGML_TYPE_F8_E4M3) { + if (contiguous_srcs) { + ggml_cpy_scalar_contiguous_cuda + (src0_ddc, src1_ddc, ne, main_stream); + } else { + ggml_cpy_scalar_cuda + (src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream); + } + } else if (src0->type == GGML_TYPE_BF16 && src1->type == GGML_TYPE_F8_E4M3) { + if (contiguous_srcs) { + ggml_cpy_scalar_contiguous_cuda + (src0_ddc, src1_ddc, ne, main_stream); + } else { + ggml_cpy_scalar_cuda + (src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream); + } + } else if (src0->type == GGML_TYPE_F8_E4M3 && src1->type == GGML_TYPE_F32) { + if (contiguous_srcs) { + ggml_cpy_scalar_contiguous_cuda + (src0_ddc, src1_ddc, ne, main_stream); + } else { + ggml_cpy_scalar_cuda + (src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream); + } } else if (src0->type == GGML_TYPE_F32 && src1->type == GGML_TYPE_F32) { if (can_be_transposed) { ggml_cpy_scalar_cuda diff --git a/ggml/src/ggml-cuda/diagmask.cu b/ggml/src/ggml-cuda/diagmask.cu deleted file mode 100644 index 4b713ba22eb5..000000000000 --- a/ggml/src/ggml-cuda/diagmask.cu +++ /dev/null @@ -1,40 +0,0 @@ -#include "diagmask.cuh" - -static __global__ void diag_mask_inf_f32(const float * x, float * dst, const int ncols, const int rows_per_channel, const int n_past) { - const int col = blockDim.y*blockIdx.y + threadIdx.y; - const int row = blockDim.x*blockIdx.x + threadIdx.x; - - if (col >= ncols) { - return; - } - - const int i = row*ncols + col; - //dst[i] = col > (n_past + row % rows_per_channel) ? -INFINITY : x[i]; - //dst[i] = x[i] - (col > n_past + row % rows_per_channel) * INT_MAX; // equivalent within rounding error but slightly faster on GPU - dst[i] = x[i] - (col > n_past + row % rows_per_channel) * FLT_MAX; -} - -static void diag_mask_inf_f32_cuda(const float * x, float * dst, const int ncols_x, const int nrows_x, const int rows_per_channel, const int n_past, cudaStream_t stream) { - const dim3 block_dims(1, CUDA_DIAG_MASK_INF_BLOCK_SIZE, 1); - const int block_num_x = (ncols_x + CUDA_DIAG_MASK_INF_BLOCK_SIZE - 1) / CUDA_DIAG_MASK_INF_BLOCK_SIZE; - const dim3 block_nums(nrows_x, block_num_x, 1); - diag_mask_inf_f32<<>>(x, dst, ncols_x, rows_per_channel, n_past); -} - -void ggml_cuda_op_diag_mask_inf(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { - const ggml_tensor * src0 = dst->src[0]; - const float * src0_d = (const float *)src0->data; - float * dst_d = (float *)dst->data; - cudaStream_t stream = ctx.stream(); - - GGML_ASSERT(src0->type == GGML_TYPE_F32); - GGML_ASSERT( dst->type == GGML_TYPE_F32); - - const int64_t ne00 = src0->ne[0]; - const int64_t ne01 = src0->ne[1]; - const int nrows0 = ggml_nrows(src0); - - const int n_past = ((int32_t *) dst->op_params)[0]; - - diag_mask_inf_f32_cuda(src0_d, dst_d, ne00, nrows0, ne01, n_past, stream); -} diff --git a/ggml/src/ggml-cuda/diagmask.cuh b/ggml/src/ggml-cuda/diagmask.cuh deleted file mode 100644 index 6cdbef17e345..000000000000 --- a/ggml/src/ggml-cuda/diagmask.cuh +++ /dev/null @@ -1,5 +0,0 @@ -#include "common.cuh" - -#define CUDA_DIAG_MASK_INF_BLOCK_SIZE 32 - -void ggml_cuda_op_diag_mask_inf(ggml_backend_cuda_context & ctx, ggml_tensor * dst); diff --git a/ggml/src/ggml-cuda/fattn-common.cuh b/ggml/src/ggml-cuda/fattn-common.cuh index beb877941c75..6667ca40080a 100644 --- a/ggml/src/ggml-cuda/fattn-common.cuh +++ b/ggml/src/ggml-cuda/fattn-common.cuh @@ -146,6 +146,34 @@ static __device__ __forceinline__ float vec_dot_fattn_vec_KQ_bf16( return sum; } +template +static __device__ __forceinline__ float vec_dot_fattn_vec_KQ_f8_e4m3( + const char * __restrict__ K_c, const void * __restrict__ Q_v, const int * __restrict__ Q_q8, const void * __restrict__ Q_ds_v) { + + const uint8_t * K_f8 = (const uint8_t *) K_c; + GGML_UNUSED(Q_q8); + GGML_UNUSED(Q_ds_v); + + float sum = 0.0f; + +#pragma unroll + for (int k_KQ_0 = 0; k_KQ_0 < D; k_KQ_0 += nthreads*4) { + uint8_t tmp[4]; + ggml_cuda_memcpy_1(tmp, K_f8 + k_KQ_0 + (threadIdx.x % nthreads)*4); +#pragma unroll + for (int k_KQ_1 = 0; k_KQ_1 < 4; ++k_KQ_1) { + const float k = ggml_cuda_f8_e4m3_to_fp32(tmp[k_KQ_1]); +#ifdef V_DOT2_F32_F16_AVAILABLE + sum += k * __half2float(((const half *) Q_v)[k_KQ_0/nthreads + k_KQ_1]); +#else + sum += k * ((const float *) Q_v)[k_KQ_0/nthreads + k_KQ_1]; +#endif // V_DOT2_F32_F16_AVAILABLE + } + } + + return sum; +} + template static __device__ __forceinline__ float vec_dot_fattn_vec_KQ_q4_0( const char * __restrict__ K_c, const void * __restrict__ Q_v, const int * __restrict__ Q_q8, const void * __restrict__ Q_ds_v) { @@ -406,6 +434,19 @@ static __device__ __forceinline__ void dequantize_V_bf16(const void * __restrict } } +template +static __device__ __forceinline__ void dequantize_V_f8_e4m3(const void * __restrict__ vx, void * __restrict__ dst, const int64_t i0) { + static_assert(ne == 2 || ne == 4, "bad ne"); + static_assert(std::is_same_v || std::is_same_v, "unsupported type"); + ggml_fp8_e4m3_t tmp[ne]; + ggml_cuda_memcpy_1(tmp, (const ggml_fp8_e4m3_t *) vx + i0); + +#pragma unroll + for (int l = 0; l < ne; ++l) { + ((T *) dst)[l] = ggml_cuda_cast(tmp[l]); + } +} + template static __device__ __forceinline__ void dequantize_V_q4_0(const void * __restrict__ vx, void * __restrict__ dst, const int64_t i0) { const block_q4_0 * x = (const block_q4_0 *) vx; @@ -428,17 +469,18 @@ static __device__ __forceinline__ void dequantize_V_q4_0(const void * __restrict const int8_t * q8 = (const int8_t *) &q; -#ifdef FP16_AVAILABLE if constexpr (std::is_same_v) { +#ifdef FP16_AVAILABLE const half2 d = __half2half2(x[ib].d); #pragma unroll for (int l0 = 0; l0 < ne; l0 += 2) { ((half2 *) dst)[l0/2] = d * make_half2(q8[l0 + 0], q8[l0 + 1]); } - } else +#else + NO_DEVICE_CODE; #endif // FP16_AVAILABLE - if constexpr (std::is_same_v) { + } else if constexpr (std::is_same_v) { const float d = x[ib].d; #pragma unroll @@ -466,8 +508,8 @@ static __device__ __forceinline__ void dequantize_V_q4_1(const void * __restrict const int8_t * q8 = (const int8_t *) &q; -#ifdef FP16_AVAILABLE if constexpr (std::is_same_v) { +#ifdef FP16_AVAILABLE const half2 dm = x[ib].dm; const half2 d = __half2half2( __low2half(dm)); const half2 m = __half2half2(__high2half(dm)); @@ -476,9 +518,10 @@ static __device__ __forceinline__ void dequantize_V_q4_1(const void * __restrict for (int l0 = 0; l0 < ne; l0 += 2) { ((half2 *) dst)[l0/2] = d * make_half2(q8[l0 + 0], q8[l0 + 1]) + m; } - } else +#else + NO_DEVICE_CODE; #endif // FP16_AVAILABLE - if constexpr (std::is_same_v) { + } else if constexpr (std::is_same_v) { const float2 dm = __half22float2(x[ib].dm); #pragma unroll @@ -518,17 +561,18 @@ static __device__ __forceinline__ void dequantize_V_q5_0(const void * __restrict const int8_t * q8 = (const int8_t *) &q; -#ifdef FP16_AVAILABLE if constexpr (std::is_same_v) { +#ifdef FP16_AVAILABLE const half2 d = __half2half2(x[ib].d); #pragma unroll for (int l0 = 0; l0 < ne; l0 += 2) { ((half2 *) dst)[l0/2] = d * make_half2(q8[l0 + 0], q8[l0 + 1]); } - } else +#else + NO_DEVICE_CODE; #endif // FP16_AVAILABLE - if constexpr (std::is_same_v) { + } else if constexpr (std::is_same_v) { const float d = x[ib].d; #pragma unroll @@ -566,8 +610,8 @@ static __device__ __forceinline__ void dequantize_V_q5_1(const void * __restrict const int8_t * q8 = (const int8_t *) &q; -#ifdef FP16_AVAILABLE if constexpr (std::is_same_v) { +#ifdef FP16_AVAILABLE const half2 dm = x[ib].dm; const half2 d = __half2half2( __low2half(dm)); const half2 m = __half2half2(__high2half(dm)); @@ -576,9 +620,10 @@ static __device__ __forceinline__ void dequantize_V_q5_1(const void * __restrict for (int l0 = 0; l0 < ne; l0 += 2) { ((half2 *) dst)[l0/2] = d * make_half2(q8[l0 + 0], q8[l0 + 1]) + m; } - } else +#else + NO_DEVICE_CODE; #endif // FP16_AVAILABLE - if constexpr (std::is_same_v) { + } else if constexpr (std::is_same_v) { const float2 dm = __half22float2(x[ib].dm); #pragma unroll @@ -601,17 +646,18 @@ static __device__ __forceinline__ void dequantize_V_q8_0(const void * __restrict int8_t qs[ne]; ggml_cuda_memcpy_1(qs, x[ib].qs + iqs); -#ifdef FP16_AVAILABLE if constexpr (std::is_same::value) { +#ifdef FP16_AVAILABLE const half2 d = __half2half2(x[ib].d); #pragma unroll for (int l0 = 0; l0 < ne; l0 += 2) { ((half2 *) dst)[l0/2] = d * make_half2(qs[l0 + 0], qs[l0 + 1]); } - } else +#else + NO_DEVICE_CODE; #endif // FP16_AVAILABLE - if constexpr (std::is_same::value) { + } else if constexpr (std::is_same::value) { const float d = x[ib].d; #pragma unroll @@ -639,6 +685,8 @@ constexpr __device__ vec_dot_KQ_t get_vec_dot_KQ() { return vec_dot_fattn_vec_KQ_q8_0; } else if constexpr (type_K == GGML_TYPE_BF16) { return vec_dot_fattn_vec_KQ_bf16; + } else if constexpr (type_K == GGML_TYPE_F8_E4M3) { + return vec_dot_fattn_vec_KQ_f8_e4m3; } else { static_assert(type_K == -1, "bad type"); return nullptr; @@ -661,6 +709,8 @@ constexpr __device__ dequantize_V_t get_dequantize_V() { return dequantize_V_q8_0; } else if constexpr (type_V == GGML_TYPE_BF16) { return dequantize_V_bf16; + } else if constexpr (type_V == GGML_TYPE_F8_E4M3) { + return dequantize_V_f8_e4m3; } else { static_assert(type_V == -1, "bad type"); return nullptr; diff --git a/ggml/src/ggml-cuda/fattn-mma-f16.cuh b/ggml/src/ggml-cuda/fattn-mma-f16.cuh index 96b596ea6fb5..07a6db842c31 100644 --- a/ggml/src/ggml-cuda/fattn-mma-f16.cuh +++ b/ggml/src/ggml-cuda/fattn-mma-f16.cuh @@ -36,11 +36,21 @@ struct fattn_mma_config { } \ static constexpr __host__ __device__ fattn_mma_config ggml_cuda_fattn_mma_get_config_ampere(const int DKQ, const int DV, const int ncols) { + GGML_CUDA_FATTN_MMA_CONFIG_CASE( 40, 40, 8, 128, 2, 128, 24, 24, 24, 2, true); + GGML_CUDA_FATTN_MMA_CONFIG_CASE( 40, 40, 16, 128, 2, 64, 24, 24, 24, 2, true); + GGML_CUDA_FATTN_MMA_CONFIG_CASE( 40, 40, 32, 128, 2, 64, 24, 24, 24, 2, true); + GGML_CUDA_FATTN_MMA_CONFIG_CASE( 40, 40, 64, 128, 2, 64, 24, 24, 24, 2, true); + GGML_CUDA_FATTN_MMA_CONFIG_CASE( 64, 64, 8, 128, 2, 128, 32, 32, 32, 2, true); GGML_CUDA_FATTN_MMA_CONFIG_CASE( 64, 64, 16, 128, 2, 64, 32, 32, 32, 2, true); GGML_CUDA_FATTN_MMA_CONFIG_CASE( 64, 64, 32, 128, 2, 64, 32, 32, 32, 2, true); GGML_CUDA_FATTN_MMA_CONFIG_CASE( 64, 64, 64, 128, 2, 64, 32, 32, 32, 2, true); + GGML_CUDA_FATTN_MMA_CONFIG_CASE( 72, 72, 8, 128, 2, 128, 40, 40, 40, 2, true); + GGML_CUDA_FATTN_MMA_CONFIG_CASE( 72, 72, 16, 128, 2, 64, 40, 40, 40, 2, true); + GGML_CUDA_FATTN_MMA_CONFIG_CASE( 72, 72, 32, 128, 2, 64, 40, 40, 40, 2, true); + GGML_CUDA_FATTN_MMA_CONFIG_CASE( 72, 72, 64, 128, 2, 64, 40, 40, 40, 2, true); + GGML_CUDA_FATTN_MMA_CONFIG_CASE( 80, 80, 8, 128, 2, 128, 40, 40, 40, 2, true); GGML_CUDA_FATTN_MMA_CONFIG_CASE( 80, 80, 16, 128, 2, 64, 40, 40, 40, 2, true); GGML_CUDA_FATTN_MMA_CONFIG_CASE( 80, 80, 32, 128, 2, 64, 40, 40, 40, 2, true); @@ -123,16 +133,29 @@ static constexpr __host__ __device__ fattn_mma_config ggml_cuda_fattn_mma_get_co GGML_CUDA_FATTN_MMA_CONFIG_CASE(576, 512, 32, 128, 2, 32, 160, 128, 64, 1, false); GGML_CUDA_FATTN_MMA_CONFIG_CASE(576, 512, 64, 256, 1, 32, 160, 128, 64, 1, false); + GGML_CUDA_FATTN_MMA_CONFIG_CASE(256, 256, 32, 128, 2, 32, 128, 128, 64, 1, false); + GGML_CUDA_FATTN_MMA_CONFIG_CASE(256, 256, 64, 128, 2, 32, 128, 64, 128, 2, false); + // TODO tune specifically for Volta return ggml_cuda_fattn_mma_get_config_ampere(DKQ, DV, ncols); } static constexpr __host__ __device__ fattn_mma_config ggml_cuda_fattn_mma_get_config_rdna(const int DKQ, const int DV, const int ncols) { + GGML_CUDA_FATTN_MMA_CONFIG_CASE( 40, 40, 8, 128, 2, 64, 24, 24, 24, 1, true); + GGML_CUDA_FATTN_MMA_CONFIG_CASE( 40, 40, 16, 128, 2, 64, 24, 24, 24, 1, true); + GGML_CUDA_FATTN_MMA_CONFIG_CASE( 40, 40, 32, 128, 2, 64, 24, 24, 24, 1, true); + GGML_CUDA_FATTN_MMA_CONFIG_CASE( 40, 40, 64, 128, 2, 64, 24, 24, 24, 1, true); + GGML_CUDA_FATTN_MMA_CONFIG_CASE( 64, 64, 8, 128, 2, 64, 32, 32, 32, 1, true); GGML_CUDA_FATTN_MMA_CONFIG_CASE( 64, 64, 16, 128, 2, 64, 32, 32, 32, 1, true); GGML_CUDA_FATTN_MMA_CONFIG_CASE( 64, 64, 32, 128, 2, 64, 32, 32, 32, 1, true); GGML_CUDA_FATTN_MMA_CONFIG_CASE( 64, 64, 64, 128, 2, 64, 32, 32, 32, 1, true); + GGML_CUDA_FATTN_MMA_CONFIG_CASE( 72, 72, 8, 64, 2, 32, 40, 40, 40, 1, true); + GGML_CUDA_FATTN_MMA_CONFIG_CASE( 72, 72, 16, 64, 2, 32, 40, 40, 40, 1, true); + GGML_CUDA_FATTN_MMA_CONFIG_CASE( 72, 72, 32, 128, 2, 64, 40, 40, 40, 1, true); + GGML_CUDA_FATTN_MMA_CONFIG_CASE( 72, 72, 64, 128, 2, 64, 40, 40, 40, 1, true); + GGML_CUDA_FATTN_MMA_CONFIG_CASE( 80, 80, 8, 64, 2, 32, 40, 40, 40, 1, true); GGML_CUDA_FATTN_MMA_CONFIG_CASE( 80, 80, 16, 64, 2, 32, 40, 40, 40, 1, true); GGML_CUDA_FATTN_MMA_CONFIG_CASE( 80, 80, 32, 128, 2, 64, 40, 40, 40, 1, true); @@ -180,11 +203,21 @@ static constexpr __host__ __device__ fattn_mma_config ggml_cuda_fattn_mma_get_co } static constexpr __host__ __device__ fattn_mma_config ggml_cuda_fattn_mma_get_config_cdna(const int DKQ, const int DV, const int ncols) { + GGML_CUDA_FATTN_MMA_CONFIG_CASE( 40, 40, 8, 256, 2, 64, 24, 24, 24, 1, true); + GGML_CUDA_FATTN_MMA_CONFIG_CASE( 40, 40, 16, 256, 2, 64, 24, 24, 24, 1, true); + GGML_CUDA_FATTN_MMA_CONFIG_CASE( 40, 40, 32, 256, 2, 64, 24, 24, 24, 1, true); + GGML_CUDA_FATTN_MMA_CONFIG_CASE( 40, 40, 64, 256, 2, 64, 24, 24, 24, 1, true); + GGML_CUDA_FATTN_MMA_CONFIG_CASE( 64, 64, 8, 128, 1, 64, 32, 32, 32, 1, true); GGML_CUDA_FATTN_MMA_CONFIG_CASE( 64, 64, 16, 256, 2, 64, 32, 32, 32, 1, true); GGML_CUDA_FATTN_MMA_CONFIG_CASE( 64, 64, 32, 256, 2, 64, 32, 32, 32, 1, true); GGML_CUDA_FATTN_MMA_CONFIG_CASE( 64, 64, 64, 256, 3, 64, 32, 32, 32, 1, true); + GGML_CUDA_FATTN_MMA_CONFIG_CASE( 72, 72, 8, 256, 2, 64, 40, 40, 40, 1, true); + GGML_CUDA_FATTN_MMA_CONFIG_CASE( 72, 72, 16, 256, 2, 64, 40, 40, 40, 1, true); + GGML_CUDA_FATTN_MMA_CONFIG_CASE( 72, 72, 32, 256, 2, 64, 40, 40, 40, 1, true); + GGML_CUDA_FATTN_MMA_CONFIG_CASE( 72, 72, 64, 256, 2, 64, 40, 40, 40, 1, true); + GGML_CUDA_FATTN_MMA_CONFIG_CASE( 80, 80, 8, 256, 2, 64, 40, 40, 40, 1, true); GGML_CUDA_FATTN_MMA_CONFIG_CASE( 80, 80, 16, 256, 2, 64, 40, 40, 40, 1, true); GGML_CUDA_FATTN_MMA_CONFIG_CASE( 80, 80, 32, 256, 2, 64, 40, 40, 40, 1, true); @@ -619,6 +652,9 @@ static __device__ __forceinline__ void flash_attn_ext_f16_iter( constexpr bool Q_in_reg = ggml_cuda_fattn_mma_get_Q_in_reg (DKQ, DV, ncols); constexpr int nstages = ggml_cuda_fattn_mma_get_nstages (DKQ, DV, ncols1, ncols2, use_sparse); + constexpr int DKQp = GGML_PAD(DKQ, 16); + constexpr int DVp = GGML_PAD(DV, 16); + constexpr bool swz = ggml_cuda_fattn_mma_get_swizzled(DKQ, DV, ncols); constexpr int stride_tile_K = swz ? nbatch_K2 : nbatch_K2 + 4; constexpr int stride_tile_V = V_is_K_view ? stride_tile_K : (swz ? nbatch_V2 : nbatch_V2 + 4); @@ -635,12 +671,12 @@ static __device__ __forceinline__ void flash_attn_ext_f16_iter( if constexpr (nstages > 1) { static_assert(!oob_check, "OOB check incompatible with multi-stage pipeline"); static_assert(!V_is_K_view, "K data reuse not implemented multi-stage loading"); - static_assert(nbatch_K2 == DKQ/2, "batching not implemented for multi stage loading"); + static_assert(nbatch_K2 == DKQp/2, "batching not implemented for multi stage loading"); constexpr bool use_cp_async = true; cp_async_wait_all(); __syncthreads(); flash_attn_ext_f16_load_tile - (V_h2, tile_V, nbatch_V2, stride_V, k_VKQ_0, k_VKQ_sup, indices); + (V_h2, tile_V, min(nbatch_V2, DV/2), stride_V, k_VKQ_0, k_VKQ_sup, indices); } else { // the sparse mask values are gathered per element, always load them synchronously constexpr bool use_cp_async = nstages == 1 && !use_sparse; @@ -653,11 +689,11 @@ static __device__ __forceinline__ void flash_attn_ext_f16_iter( // For MLA K and V have the same data. // Therefore, iterate over K in reverse and later re-use the data if possible. #pragma unroll - for (int k0_start = (DKQ/2-1) - (DKQ/2-1) % nbatch_K2; k0_start >= 0; k0_start -= nbatch_K2) { - const int k0_stop = k0_start + nbatch_K2 < DKQ/2 ? k0_start + nbatch_K2 : DKQ/2; + for (int k0_start = (DKQp/2-1) - (DKQp/2-1) % nbatch_K2; k0_start >= 0; k0_start -= nbatch_K2) { + const int k0_stop = k0_start + nbatch_K2 < DKQp/2 ? k0_start + nbatch_K2 : DKQp/2; if constexpr (nstages <= 1) { - const int k0_diff = k0_stop - k0_start; + const int k0_diff = min(k0_stop, DKQ/2) - k0_start; // Only load the real head size from global memory. constexpr bool use_cp_async = nstages == 1; flash_attn_ext_f16_load_tile (K_h2 + k0_start, tile_K, k0_diff, stride_K, k_VKQ_0, k_VKQ_sup, indices); @@ -691,7 +727,7 @@ static __device__ __forceinline__ void flash_attn_ext_f16_iter( } } } else { - constexpr int stride_tile_Q = DKQ/2 + 4; + constexpr int stride_tile_Q = DKQp/2 + 4; #pragma unroll for (int k_KQ_0 = k0_start; k_KQ_0 < k0_stop; k_KQ_0 += T_A_KQ::J) { load_ldmatrix(Q_B[0], tile_Q + (threadIdx.y / np)*(T_B_KQ::I*stride_tile_Q) + k_KQ_0, stride_tile_Q); @@ -918,7 +954,7 @@ static __device__ __forceinline__ void flash_attn_ext_f16_iter( if constexpr (cols_per_warp == 8) { const half2 KQ_max_scale_h2 = make_half2(KQ_max_scale[0], KQ_max_scale[cols_per_thread - 1]); #pragma unroll - for (int i = 0; i < DV/T_C_VKQ::I; ++i) { + for (int i = 0; i < DVp/T_C_VKQ::I; ++i) { #pragma unroll for (int l = 0; l < T_C_VKQ::ne; ++l) { VKQ_C[i].x[l] *= KQ_max_scale_h2; @@ -929,7 +965,7 @@ static __device__ __forceinline__ void flash_attn_ext_f16_iter( for (int col = 0; col < cols_per_thread; ++col) { const half2 KQ_max_scale_h2 = make_half2(KQ_max_scale[col], KQ_max_scale[col]); #pragma unroll - for (int i = 0; i < (DV/2)/T_C_VKQ::J; ++i) { + for (int i = 0; i < (DVp/2)/T_C_VKQ::J; ++i) { #pragma unroll for (int l0 = 0; l0 < T_C_VKQ::ne; l0 += 2) { VKQ_C[i].x[l0 + col] *= KQ_max_scale_h2; @@ -941,7 +977,7 @@ static __device__ __forceinline__ void flash_attn_ext_f16_iter( if constexpr (std::is_same_v) { const half2 KQ_max_scale_h2 = make_half2(KQ_max_scale[0], KQ_max_scale[0]); #pragma unroll - for (int i = 0; i < (DV/2)/T_C_VKQ::J; ++i) { + for (int i = 0; i < (DVp/2)/T_C_VKQ::J; ++i) { #pragma unroll for (int l = 0; l < T_C_VKQ::ne; ++l) { VKQ_C[i].x[l] *= KQ_max_scale_h2; @@ -950,7 +986,7 @@ static __device__ __forceinline__ void flash_attn_ext_f16_iter( } else { static_assert(std::is_same_v, "bad VKQ type"); #pragma unroll - for (int i = 0; i < DV/T_C_VKQ::J; ++i) { + for (int i = 0; i < DVp/T_C_VKQ::J; ++i) { #pragma unroll for (int l = 0; l < T_C_VKQ::ne; ++l) { VKQ_C[i].x[l] *= KQ_max_scale[0]; @@ -961,7 +997,7 @@ static __device__ __forceinline__ void flash_attn_ext_f16_iter( const half2 KQ_max_scale_h2 = make_half2( KQ_max_scale[(threadIdx.x / 2) % 2], KQ_max_scale[(threadIdx.x / 2) % 2]); #pragma unroll - for (int i = 0; i < (DV/2)/T_C_VKQ::J; ++i) { + for (int i = 0; i < (DVp/2)/T_C_VKQ::J; ++i) { #pragma unroll for (int l = 0; l < T_C_VKQ::ne; ++l) { VKQ_C[i].x[l] *= KQ_max_scale_h2; @@ -997,19 +1033,19 @@ static __device__ __forceinline__ void flash_attn_ext_f16_iter( (mask_h, tile_mask, stride_mask, k_VKQ_0 + nbatch_fa, k_VKQ_sup, jt*ncols1, ne01, nullptr); } flash_attn_ext_f16_load_tile - (K_h2, tile_K, nbatch_K2, stride_K, k_VKQ_0 + nbatch_fa, k_VKQ_sup, indices); + (K_h2, tile_K, min(nbatch_K2, DKQ/2), stride_K, k_VKQ_0 + nbatch_fa, k_VKQ_sup, indices); } } // Calculate VKQ tile, need to use logical rather than physical elements for i0 due to transposition of V: #pragma unroll - for (int i0_start = 0; i0_start < DV; i0_start += 2*nbatch_V2) { - static_assert(DV % (2*nbatch_V2) == 0, "bad loop size"); + for (int i0_start = 0; i0_start < DVp; i0_start += 2*nbatch_V2) { + static_assert(DVp % (2*nbatch_V2) == 0, "bad loop size"); const int i0_stop = i0_start + 2*nbatch_V2; if constexpr (nstages <= 1) { - const int i0_diff = i0_stop - i0_start; + const int i0_diff = min(i0_stop, DV) - i0_start; // Only load the real head size from global memory. if (!V_is_K_view || i0_stop > 2*nbatch_K2) { constexpr bool use_cp_async = nstages == 1; flash_attn_ext_f16_load_tile @@ -1146,6 +1182,22 @@ template struct mma_tile_sizes<112, ncols> { using T_B_VKQ = tile<16, 8, half2, DATA_LAYOUT_I_MAJOR_MIRRORED>; // column-major using T_C_VKQ = tile<16, 16, float, DATA_LAYOUT_I_MAJOR>; // column-major }; +template struct mma_tile_sizes<40, ncols> { + using T_A_KQ = tile<16, 8, half2, DATA_LAYOUT_I_MAJOR_MIRRORED>; // row-major + using T_B_KQ = tile<16, 8, half2, DATA_LAYOUT_I_MAJOR_MIRRORED>; // column-major + using T_C_KQ = tile<16, 16, float, DATA_LAYOUT_I_MAJOR>; // column-major + using T_A_VKQ = tile<16, 8, half2, DATA_LAYOUT_I_MAJOR_MIRRORED>; // row-major + using T_B_VKQ = tile<16, 8, half2, DATA_LAYOUT_I_MAJOR_MIRRORED>; // column-major + using T_C_VKQ = tile<16, 16, float, DATA_LAYOUT_I_MAJOR>; // column-major +}; +template struct mma_tile_sizes<72, ncols> { + using T_A_KQ = tile<16, 8, half2, DATA_LAYOUT_I_MAJOR_MIRRORED>; // row-major + using T_B_KQ = tile<16, 8, half2, DATA_LAYOUT_I_MAJOR_MIRRORED>; // column-major + using T_C_KQ = tile<16, 16, float, DATA_LAYOUT_I_MAJOR>; // column-major + using T_A_VKQ = tile<16, 8, half2, DATA_LAYOUT_I_MAJOR_MIRRORED>; // row-major + using T_B_VKQ = tile<16, 8, half2, DATA_LAYOUT_I_MAJOR_MIRRORED>; // column-major + using T_C_VKQ = tile<16, 16, float, DATA_LAYOUT_I_MAJOR>; // column-major +}; #else template struct mma_tile_sizes { using T_A_KQ = tile<16, 8, half2, DATA_LAYOUT_I_MAJOR>; // row-major @@ -1171,6 +1223,22 @@ template struct mma_tile_sizes<112, ncols> { using T_B_VKQ = tile<16, 8, half2>; // column-major using T_C_VKQ = tile<16, 8, half2>; // column-major }; +template struct mma_tile_sizes<40, ncols> { + using T_A_KQ = tile<16, 8, half2>; // row-major + using T_B_KQ = tile<16, 8, half2>; // column-major + using T_C_KQ = tile<16, 16, float>; // column-major + using T_A_VKQ = tile<16, 8, half2>; // row-major + using T_B_VKQ = tile<16, 8, half2>; // column-major + using T_C_VKQ = tile<16, 8, half2>; // column-major +}; +template struct mma_tile_sizes<72, ncols> { + using T_A_KQ = tile<16, 8, half2>; // row-major + using T_B_KQ = tile<16, 8, half2>; // column-major + using T_C_KQ = tile<16, 16, float>; // column-major + using T_A_VKQ = tile<16, 8, half2>; // row-major + using T_B_VKQ = tile<16, 8, half2>; // column-major + using T_C_VKQ = tile<16, 8, half2>; // column-major +}; #endif // RDNA3 #elif defined(AMD_MFMA_AVAILABLE) template struct mma_tile_sizes { @@ -1240,6 +1308,9 @@ static __device__ __forceinline__ void flash_attn_ext_f16_process_tile( constexpr bool Q_in_reg = ggml_cuda_fattn_mma_get_Q_in_reg (DKQ, DV, ncols); constexpr int nstages = ggml_cuda_fattn_mma_get_nstages (DKQ, DV, ncols1, ncols2, use_sparse); + constexpr int DKQp = GGML_PAD(DKQ, 16); + constexpr int DVp = GGML_PAD(DV, 16); + if (cols_per_warp > ncols) { NO_DEVICE_CODE; return; @@ -1248,7 +1319,7 @@ static __device__ __forceinline__ void flash_attn_ext_f16_process_tile( static_assert(nwarps * (cols_per_warp/ncols2) % ncols1 == 0, "bad nwarps"); constexpr bool swz = ggml_cuda_fattn_mma_get_swizzled(DKQ, DV, ncols); - constexpr int stride_tile_Q = DKQ/2 + 4; + constexpr int stride_tile_Q = DKQp/2 + 4; constexpr int stride_tile_K = swz ? nbatch_K2 : nbatch_K2 + 4; constexpr int stride_tile_V = V_is_K_view ? stride_tile_K : (swz ? nbatch_V2 : nbatch_V2 + 4); constexpr int stride_tile_KV_max = stride_tile_K > stride_tile_V ? stride_tile_K : stride_tile_V; @@ -1258,17 +1329,17 @@ static __device__ __forceinline__ void flash_attn_ext_f16_process_tile( half2 * tile_V = nstages > 1 ? tile_K + nbatch_fa * stride_tile_K : tile_K; half * tile_mask = (half *) (nstages > 1 ? tile_V + nbatch_fa * stride_tile_V : tile_V + nbatch_fa * stride_tile_KV_max); - T_B_KQ Q_B[(Q_in_reg ? DKQ/(2*T_B_KQ::J) : 1)]; + T_B_KQ Q_B[(Q_in_reg ? DKQp/(2*T_B_KQ::J) : 1)]; #if defined(TURING_MMA_AVAILABLE) - T_C_VKQ VKQ_C[cols_per_warp == 8 ? DV/T_C_VKQ::I : DV/(2*T_C_VKQ::J)]; + T_C_VKQ VKQ_C[cols_per_warp == 8 ? DVp/T_C_VKQ::I : DVp/(2*T_C_VKQ::J)]; #elif defined(AMD_WMMA_256B_AVAILABLE) - T_C_VKQ VKQ_C[DV % 32 != 0 ? DV/T_C_VKQ::J : DV/(2*T_C_VKQ::J)]; + T_C_VKQ VKQ_C[DVp % 32 != 0 ? DVp/T_C_VKQ::J : DVp/(2*T_C_VKQ::J)]; #elif defined(AMD_MFMA_AVAILABLE) - T_C_VKQ VKQ_C[ DV/T_C_VKQ::J]; + T_C_VKQ VKQ_C[ DVp/T_C_VKQ::J]; #elif defined(AMD_WMMA_AVAILABLE) - T_C_VKQ VKQ_C[ DV/(2*T_C_VKQ::J)]; + T_C_VKQ VKQ_C[ DVp/(2*T_C_VKQ::J)]; #else // Volta - T_C_VKQ VKQ_C[ DV/(2*T_C_VKQ::J)]; + T_C_VKQ VKQ_C[ DVp/(2*T_C_VKQ::J)]; #endif // defined(TURING_MMA_AVAILABLE) float KQ_rowsum[cols_per_thread] = {0.0f}; @@ -1284,8 +1355,8 @@ static __device__ __forceinline__ void flash_attn_ext_f16_process_tile( const half2 scale_h2 = make_half2(scale, scale); #pragma unroll for (int stride_k : {warp_size, warp_size/2, warp_size/4, warp_size/8}) { - const int k0_start = stride_k == warp_size ? 0 : DKQ/2 - (DKQ/2) % (2*stride_k); - const int k0_stop = DKQ/2 - (DKQ/2) % (1*stride_k); + const int k0_start = stride_k == warp_size ? 0 : DKQp/2 - (DKQp/2) % (2*stride_k); + const int k0_stop = DKQp/2 - (DKQp/2) % (1*stride_k); const int stride_jc = warp_size / stride_k; if (k0_start == k0_stop) { @@ -1308,7 +1379,8 @@ static __device__ __forceinline__ void flash_attn_ext_f16_process_tile( for (int k0 = k0_start; k0 < k0_stop; k0 += stride_k) { const int k = k0 + (stride_k == warp_size ? threadIdx.x : threadIdx.x % stride_k); - const float2 tmp = Q_f2[(jt*ncols1 + j)*stride_Q1 + c*stride_Q2 + k]; + const float2 tmp = DKQp == DKQ || k < DKQ/2 ? + Q_f2[(jt*ncols1 + j)*stride_Q1 + c*stride_Q2 + k] : make_float2(0.0f, 0.0f); tile_Q[jc*stride_tile_Q + k] = scale_h2 * make_half2(tmp.x, tmp.y); } } else { @@ -1328,19 +1400,50 @@ static __device__ __forceinline__ void flash_attn_ext_f16_process_tile( const int j0 = (threadIdx.y / np) * cols_per_warp; #pragma unroll - for (int k0 = 0; k0 < DKQ/2; k0 += T_B_KQ::J) { + for (int k0 = 0; k0 < DKQp/2; k0 += T_B_KQ::J) { load_ldmatrix(Q_B[k0/T_B_KQ::J], tile_Q + j0*stride_tile_Q + k0, stride_tile_Q); } } __syncthreads(); + // Zero the pad columns of the K/V tiles once, the tile loads only write the first DKQ/2 (DV/2) half2 of a row. + // Done after Q_B is loaded because tile_K may alias tile_Q. + if constexpr (DKQp != DKQ || DVp != DV) { + static_assert(!swz, "pad zero-fill assumes unswizzled tiles"); + static_assert(nbatch_K2 == DKQp/2 && nbatch_V2 == DVp/2, "K/V tile batching not implemented for padded head sizes"); + static_assert(DKQ == DV, "pad zero-fill for DKQ != DV requires separate handling of the aliased V tile"); + + const half2 zero[4] = {{0.0f, 0.0f}, {0.0f, 0.0f}, {0.0f, 0.0f}, {0.0f, 0.0f}}; + const int tid = threadIdx.y*warp_size + threadIdx.x; +#pragma unroll + for (int i0 = 0; i0 < nbatch_fa; i0 += nwarps*warp_size) { + const int i = i0 + tid; + + if (i0 + nwarps*warp_size > nbatch_fa && i >= nbatch_fa) { + break; + } + +#pragma unroll + for (int k = DKQ/2; k < DKQp/2; k += 4) { + ggml_cuda_memcpy_1<16>(tile_K + i*stride_tile_K + k, zero); + } + if constexpr (nstages > 1) { // tile_V is only a separate buffer with multi-stage loading. +#pragma unroll + for (int k = DV/2; k < DVp/2; k += 4) { + ggml_cuda_memcpy_1<16>(tile_V + i*stride_tile_V + k, zero); + } + } + } + __syncthreads(); + } + int kb0 = kb0_start; // Preload mask and K data for first iteration when using cp_async with multiple stages: if constexpr (nstages > 1) { static_assert(!use_sparse, "sparse gather not implemented for multi-stage loading"); - static_assert(nbatch_K2 == DKQ/2, "batching not implemented for multi-stage pipeline"); + static_assert(nbatch_K2 == DKQp/2, "batching not implemented for multi-stage pipeline"); constexpr bool use_cp_async = true; constexpr bool oob_check = false; constexpr int k_VKQ_sup = nbatch_fa; @@ -1349,7 +1452,7 @@ static __device__ __forceinline__ void flash_attn_ext_f16_process_tile( (mask_h, tile_mask, stride_mask, kb0*nbatch_fa, k_VKQ_sup, jt*ncols1, ne01, nullptr); } flash_attn_ext_f16_load_tile - (K_h2, tile_K, nbatch_K2, stride_K, kb0*nbatch_fa, k_VKQ_sup, indices); + (K_h2, tile_K, min(nbatch_K2, DKQ/2), stride_K, kb0*nbatch_fa, k_VKQ_sup, indices); } // kb0_start is always < kb0_stop so the last iter can be executed unconditionally. @@ -1454,7 +1557,7 @@ static __device__ __forceinline__ void flash_attn_ext_f16_process_tile( if constexpr (cols_per_warp == 8) { const half2 KQ_max_scale_h2 = make_half2(KQ_max_scale[0], KQ_max_scale[cols_per_thread - 1]); #pragma unroll - for (int i = 0; i < DV/T_C_VKQ::I; ++i) { + for (int i = 0; i < DVp/T_C_VKQ::I; ++i) { #pragma unroll for (int l = 0; l < T_C_VKQ::ne; ++l) { VKQ_C[i].x[l] *= KQ_max_scale_h2; @@ -1465,7 +1568,7 @@ static __device__ __forceinline__ void flash_attn_ext_f16_process_tile( for (int col = 0; col < cols_per_thread; ++col) { const half2 KQ_max_scale_h2 = make_half2(KQ_max_scale[col], KQ_max_scale[col]); #pragma unroll - for (int i = 0; i < (DV/2)/T_C_VKQ::J; ++i) { + for (int i = 0; i < (DVp/2)/T_C_VKQ::J; ++i) { #pragma unroll for (int l0 = 0; l0 < T_C_VKQ::ne; l0 += 2) { VKQ_C[i].x[l0 + col] *= KQ_max_scale_h2; @@ -1477,7 +1580,7 @@ static __device__ __forceinline__ void flash_attn_ext_f16_process_tile( if constexpr (std::is_same_v) { const half2 KQ_max_scale_h2 = make_half2(KQ_max_scale[0], KQ_max_scale[0]); #pragma unroll - for (int i = 0; i < (DV/2)/T_C_VKQ::J; ++i) { + for (int i = 0; i < (DVp/2)/T_C_VKQ::J; ++i) { #pragma unroll for (int l = 0; l < T_C_VKQ::ne; ++l) { VKQ_C[i].x[l] *= KQ_max_scale_h2; @@ -1486,7 +1589,7 @@ static __device__ __forceinline__ void flash_attn_ext_f16_process_tile( } else { static_assert(std::is_same_v, "bad VKQ type"); #pragma unroll - for (int i = 0; i < DV/T_C_VKQ::J; ++i) { + for (int i = 0; i < DVp/T_C_VKQ::J; ++i) { #pragma unroll for (int l = 0; l < T_C_VKQ::ne; ++l) { VKQ_C[i].x[l] *= KQ_max_scale[0]; @@ -1497,7 +1600,7 @@ static __device__ __forceinline__ void flash_attn_ext_f16_process_tile( const int col = (threadIdx.x / 2) % 2; const half2 KQ_max_scale_h2 = make_half2(KQ_max_scale[col], KQ_max_scale[col]); #pragma unroll - for (int i = 0; i < (DV/2)/T_C_VKQ::J; ++i) { + for (int i = 0; i < (DVp/2)/T_C_VKQ::J; ++i) { #pragma unroll for (int l = 0; l < T_C_VKQ::ne; ++l) { VKQ_C[i].x[l] *= KQ_max_scale_h2; @@ -1511,7 +1614,7 @@ static __device__ __forceinline__ void flash_attn_ext_f16_process_tile( // So also write VKQ accumulators to shared memory in column-major format if np == 1. constexpr int tile_stride = nbatch_combine + 4; - static_assert((DV/2) % nbatch_combine == 0, "bad nbatch_combine"); + static_assert((DVp/2) % nbatch_combine == 0, "bad nbatch_combine"); if constexpr (cols_per_warp == 8) { const int jc_cwmo = (threadIdx.x % (2*T_C_VKQ::J)) / T_C_VKQ::J; // jc combine write meta offset @@ -1655,7 +1758,7 @@ static __device__ __forceinline__ void flash_attn_ext_f16_process_tile( } #pragma unroll - for (int k00 = 0; k00 < DV/2; k00 += nbatch_combine) { + for (int k00 = 0; k00 < DVp/2; k00 += nbatch_combine) { if constexpr (cols_per_warp == 8) { static_assert(std::is_same_v, "bad VKQ type"); const int jc_cwd = threadIdx.y*T_B_KQ::I + T_B_KQ::get_i(-1); // jc combine write data @@ -1754,6 +1857,10 @@ static __device__ __forceinline__ void flash_attn_ext_f16_process_tile( for (int k0 = k0_start; k0 < k0_stop; k0 += stride_k) { const int k = k0 + (stride_k == warp_size ? threadIdx.x : threadIdx.x % stride_k); + if (DVp != DV && k00 + k >= DV/2) { + continue; + } + float2 dstk_val = make_float2(0.0f, 0.0f); #pragma unroll for (int ip = 0; ip < np; ++ip) { @@ -2034,6 +2141,7 @@ void ggml_cuda_flash_attn_ext_mma_f16_case(ggml_backend_cuda_context & ctx, ggml const int nwarps = nthreads / warp_size_host; constexpr bool V_is_K_view = DKQ == 576; // Guaranteed by the kernel selection logic in fattn.cu + constexpr int DKQp = GGML_PAD(DKQ, 16); // KV tile strides must match flash_attn_ext_f16_iter / _process_tile. const bool swz = ggml_cuda_fattn_mma_get_swizzled(DKQ, DV, ncols, cc); @@ -2041,7 +2149,7 @@ void ggml_cuda_flash_attn_ext_mma_f16_case(ggml_backend_cuda_context & ctx, ggml const int stride_tile_V = V_is_K_view ? stride_tile_K : (swz ? nbatch_V2 : nbatch_V2 + 4); const size_t nbytes_shared_KV_1stage = nbatch_fa * std::max(stride_tile_K, stride_tile_V) * sizeof(half2); const size_t nbytes_shared_KV_2stage = nbatch_fa * (stride_tile_K + stride_tile_V) * sizeof(half2); - const size_t nbytes_shared_Q = ncols * (DKQ/2 + 4) * sizeof(half2); + const size_t nbytes_shared_Q = ncols * (DKQp/2 + 4) * sizeof(half2); const size_t nbytes_shared_mask = ncols1 * (nbatch_fa/2 + 4) * sizeof(half2); const size_t nbytes_shared_combine = nwarps*cols_per_warp * (nbatch_combine + 4) * sizeof(half2); @@ -2126,28 +2234,36 @@ void ggml_cuda_flash_attn_ext_mma_f16_case(ggml_backend_cuda_context & ctx, ggml extern DECL_FATTN_MMA_F16_CASE(DKQ, DV, (ncols)/ 8, 8); \ extern DECL_FATTN_MMA_F16_CASE(DKQ, DV, (ncols)/16, 16); \ +DECL_FATTN_MMA_F16_CASE_ALL_NCOLS2( 40, 40, 8) DECL_FATTN_MMA_F16_CASE_ALL_NCOLS2( 64, 64, 8) +DECL_FATTN_MMA_F16_CASE_ALL_NCOLS2( 72, 72, 8) DECL_FATTN_MMA_F16_CASE_ALL_NCOLS2( 80, 80, 8) DECL_FATTN_MMA_F16_CASE_ALL_NCOLS2( 96, 96, 8) DECL_FATTN_MMA_F16_CASE_ALL_NCOLS2(112, 112, 8) DECL_FATTN_MMA_F16_CASE_ALL_NCOLS2(128, 128, 8) DECL_FATTN_MMA_F16_CASE_ALL_NCOLS2(256, 256, 8) +DECL_FATTN_MMA_F16_CASE_ALL_NCOLS2( 40, 40, 16) DECL_FATTN_MMA_F16_CASE_ALL_NCOLS2( 64, 64, 16) +DECL_FATTN_MMA_F16_CASE_ALL_NCOLS2( 72, 72, 16) DECL_FATTN_MMA_F16_CASE_ALL_NCOLS2( 80, 80, 16) DECL_FATTN_MMA_F16_CASE_ALL_NCOLS2( 96, 96, 16) DECL_FATTN_MMA_F16_CASE_ALL_NCOLS2(112, 112, 16) DECL_FATTN_MMA_F16_CASE_ALL_NCOLS2(128, 128, 16) DECL_FATTN_MMA_F16_CASE_ALL_NCOLS2(256, 256, 16) +DECL_FATTN_MMA_F16_CASE_ALL_NCOLS2( 40, 40, 32) DECL_FATTN_MMA_F16_CASE_ALL_NCOLS2( 64, 64, 32) +DECL_FATTN_MMA_F16_CASE_ALL_NCOLS2( 72, 72, 32) DECL_FATTN_MMA_F16_CASE_ALL_NCOLS2( 80, 80, 32) DECL_FATTN_MMA_F16_CASE_ALL_NCOLS2( 96, 96, 32) DECL_FATTN_MMA_F16_CASE_ALL_NCOLS2(112, 112, 32) DECL_FATTN_MMA_F16_CASE_ALL_NCOLS2(128, 128, 32) DECL_FATTN_MMA_F16_CASE_ALL_NCOLS2(256, 256, 32) +DECL_FATTN_MMA_F16_CASE_ALL_NCOLS2( 40, 40, 64) DECL_FATTN_MMA_F16_CASE_ALL_NCOLS2( 64, 64, 64) +DECL_FATTN_MMA_F16_CASE_ALL_NCOLS2( 72, 72, 64) DECL_FATTN_MMA_F16_CASE_ALL_NCOLS2( 80, 80, 64) DECL_FATTN_MMA_F16_CASE_ALL_NCOLS2( 96, 96, 64) DECL_FATTN_MMA_F16_CASE_ALL_NCOLS2(112, 112, 64) diff --git a/ggml/src/ggml-cuda/fattn-vec.cuh b/ggml/src/ggml-cuda/fattn-vec.cuh index 9df15128f0ca..02c3c327fd07 100644 --- a/ggml/src/ggml-cuda/fattn-vec.cuh +++ b/ggml/src/ggml-cuda/fattn-vec.cuh @@ -96,7 +96,7 @@ static __global__ void flash_attn_ext_vec( constexpr int V_cols_per_iter = WARP_SIZE / nthreads_V; constexpr vec_dot_KQ_t vec_dot_KQ = get_vec_dot_KQ(); - constexpr bool Q_q8_1 = type_K != GGML_TYPE_F16 && type_K != GGML_TYPE_BF16; + constexpr bool Q_q8_1 = type_K != GGML_TYPE_F16 && type_K != GGML_TYPE_BF16 && type_K != GGML_TYPE_F8_E4M3; #ifdef V_DOT2_F32_F16_AVAILABLE constexpr dequantize_V_t dequantize_V = get_dequantize_V(); #else @@ -203,6 +203,27 @@ static __global__ void flash_attn_ext_vec( } __syncthreads(); + } else if constexpr (type_K == GGML_TYPE_F8_E4M3) { +#pragma unroll + for (int j = 0; j < ncols; ++j) { + const float * Q_j = (const float *) (Q + j*nb01); +#pragma unroll + for (int i0 = 0; i0 < D; i0 += nthreads_KQ*4) { + const int i = i0 + (threadIdx.x % nthreads_KQ)*4; + float4 tmp = make_float4(0.0f, 0.0f, 0.0f, 0.0f); + if (ncols == 1 || ic0 + j < int(ne01.z)) { + ggml_cuda_memcpy_1(&tmp, Q_j + i); + } +#ifdef V_DOT2_F32_F16_AVAILABLE + const half2 scale_h2 = make_half2(scale, scale); + Q_reg[j][i0/(2*nthreads_KQ) + 0] = __hmul2(make_half2(tmp.x, tmp.y), scale_h2); + Q_reg[j][i0/(2*nthreads_KQ) + 1] = __hmul2(make_half2(tmp.z, tmp.w), scale_h2); +#else + Q_reg[j][i0/(2*nthreads_KQ) + 0] = make_float2(tmp.x*scale, tmp.y*scale); + Q_reg[j][i0/(2*nthreads_KQ) + 1] = make_float2(tmp.z*scale, tmp.w*scale); +#endif + } + } } else { #ifdef V_DOT2_F32_F16_AVAILABLE const half2 scale_h2 = make_half2(scale, scale); @@ -587,6 +608,7 @@ void ggml_cuda_flash_attn_ext_vec_case(ggml_backend_cuda_context & ctx, ggml_ten extern DECL_FATTN_VEC_CASE(D, type_K, GGML_TYPE_Q5_1); \ extern DECL_FATTN_VEC_CASE(D, type_K, GGML_TYPE_Q8_0); \ extern DECL_FATTN_VEC_CASE(D, type_K, GGML_TYPE_BF16); \ + extern DECL_FATTN_VEC_CASE(D, type_K, GGML_TYPE_F8_E4M3); \ EXTERN_DECL_FATTN_VEC_CASES( 64, GGML_TYPE_F16) EXTERN_DECL_FATTN_VEC_CASES( 64, GGML_TYPE_Q4_0) @@ -595,6 +617,7 @@ EXTERN_DECL_FATTN_VEC_CASES( 64, GGML_TYPE_Q5_0) EXTERN_DECL_FATTN_VEC_CASES( 64, GGML_TYPE_Q5_1) EXTERN_DECL_FATTN_VEC_CASES( 64, GGML_TYPE_Q8_0) EXTERN_DECL_FATTN_VEC_CASES( 64, GGML_TYPE_BF16) +EXTERN_DECL_FATTN_VEC_CASES( 64, GGML_TYPE_F8_E4M3) EXTERN_DECL_FATTN_VEC_CASES(128, GGML_TYPE_F16) EXTERN_DECL_FATTN_VEC_CASES(128, GGML_TYPE_Q4_0) @@ -603,6 +626,7 @@ EXTERN_DECL_FATTN_VEC_CASES(128, GGML_TYPE_Q5_0) EXTERN_DECL_FATTN_VEC_CASES(128, GGML_TYPE_Q5_1) EXTERN_DECL_FATTN_VEC_CASES(128, GGML_TYPE_Q8_0) EXTERN_DECL_FATTN_VEC_CASES(128, GGML_TYPE_BF16) +EXTERN_DECL_FATTN_VEC_CASES(128, GGML_TYPE_F8_E4M3) EXTERN_DECL_FATTN_VEC_CASES(256, GGML_TYPE_F16) EXTERN_DECL_FATTN_VEC_CASES(256, GGML_TYPE_Q4_0) @@ -611,3 +635,4 @@ EXTERN_DECL_FATTN_VEC_CASES(256, GGML_TYPE_Q5_0) EXTERN_DECL_FATTN_VEC_CASES(256, GGML_TYPE_Q5_1) EXTERN_DECL_FATTN_VEC_CASES(256, GGML_TYPE_Q8_0) EXTERN_DECL_FATTN_VEC_CASES(256, GGML_TYPE_BF16) +EXTERN_DECL_FATTN_VEC_CASES(256, GGML_TYPE_F8_E4M3) diff --git a/ggml/src/ggml-cuda/fattn.cu b/ggml/src/ggml-cuda/fattn.cu index 5d7ebf966378..cf44ab4f504b 100644 --- a/ggml/src/ggml-cuda/fattn.cu +++ b/ggml/src/ggml-cuda/fattn.cu @@ -314,10 +314,18 @@ static void ggml_cuda_flash_attn_ext_mma_f16(ggml_backend_cuda_context & ctx, gg const ggml_tensor * mask = dst->src[3]; switch (Q->ne[0]) { + case 40: + GGML_ASSERT(V->ne[0] == 40); + ggml_cuda_flash_attn_ext_mma_f16_switch_ncols2< 40, 40>(ctx, dst); + break; case 64: GGML_ASSERT(V->ne[0] == 64); ggml_cuda_flash_attn_ext_mma_f16_switch_ncols2< 64, 64>(ctx, dst); break; + case 72: + GGML_ASSERT(V->ne[0] == 72); + ggml_cuda_flash_attn_ext_mma_f16_switch_ncols2< 72, 72>(ctx, dst); + break; case 80: GGML_ASSERT(V->ne[0] == 80); ggml_cuda_flash_attn_ext_mma_f16_switch_ncols2< 80, 80>(ctx, dst); @@ -462,6 +470,7 @@ static fattn_vec_case_t ggml_cuda_get_fattn_vec_case(const int64_t head_size, co FATTN_VEC_CASES_ALL_D(Q5_1, F16) FATTN_VEC_CASES_ALL_D(Q8_0, F16) FATTN_VEC_CASES_ALL_D(BF16, F16) + FATTN_VEC_CASES_ALL_D(F8_E4M3, F16) FATTN_VEC_CASES_ALL_D(F16, Q4_0) FATTN_VEC_CASES_ALL_D(Q4_0, Q4_0) @@ -470,6 +479,7 @@ static fattn_vec_case_t ggml_cuda_get_fattn_vec_case(const int64_t head_size, co FATTN_VEC_CASES_ALL_D(Q5_1, Q4_0) FATTN_VEC_CASES_ALL_D(Q8_0, Q4_0) FATTN_VEC_CASES_ALL_D(BF16, Q4_0) + FATTN_VEC_CASES_ALL_D(F8_E4M3, Q4_0) FATTN_VEC_CASES_ALL_D(F16, Q4_1) FATTN_VEC_CASES_ALL_D(Q4_0, Q4_1) @@ -478,6 +488,7 @@ static fattn_vec_case_t ggml_cuda_get_fattn_vec_case(const int64_t head_size, co FATTN_VEC_CASES_ALL_D(Q5_1, Q4_1) FATTN_VEC_CASES_ALL_D(Q8_0, Q4_1) FATTN_VEC_CASES_ALL_D(BF16, Q4_1) + FATTN_VEC_CASES_ALL_D(F8_E4M3, Q4_1) FATTN_VEC_CASES_ALL_D(F16, Q5_0) FATTN_VEC_CASES_ALL_D(Q4_0, Q5_0) @@ -486,6 +497,7 @@ static fattn_vec_case_t ggml_cuda_get_fattn_vec_case(const int64_t head_size, co FATTN_VEC_CASES_ALL_D(Q5_1, Q5_0) FATTN_VEC_CASES_ALL_D(Q8_0, Q5_0) FATTN_VEC_CASES_ALL_D(BF16, Q5_0) + FATTN_VEC_CASES_ALL_D(F8_E4M3, Q5_0) FATTN_VEC_CASES_ALL_D(F16, Q5_1) FATTN_VEC_CASES_ALL_D(Q4_0, Q5_1) @@ -494,6 +506,7 @@ static fattn_vec_case_t ggml_cuda_get_fattn_vec_case(const int64_t head_size, co FATTN_VEC_CASES_ALL_D(Q5_1, Q5_1) FATTN_VEC_CASES_ALL_D(Q8_0, Q5_1) FATTN_VEC_CASES_ALL_D(BF16, Q5_1) + FATTN_VEC_CASES_ALL_D(F8_E4M3, Q5_1) FATTN_VEC_CASES_ALL_D(F16, Q8_0) FATTN_VEC_CASES_ALL_D(Q4_0, Q8_0) @@ -502,6 +515,7 @@ static fattn_vec_case_t ggml_cuda_get_fattn_vec_case(const int64_t head_size, co FATTN_VEC_CASES_ALL_D(Q5_1, Q8_0) FATTN_VEC_CASES_ALL_D(Q8_0, Q8_0) FATTN_VEC_CASES_ALL_D(BF16, Q8_0) + FATTN_VEC_CASES_ALL_D(F8_E4M3, Q8_0) FATTN_VEC_CASES_ALL_D(F16, BF16) FATTN_VEC_CASES_ALL_D(Q4_0, BF16) @@ -510,6 +524,16 @@ static fattn_vec_case_t ggml_cuda_get_fattn_vec_case(const int64_t head_size, co FATTN_VEC_CASES_ALL_D(Q5_1, BF16) FATTN_VEC_CASES_ALL_D(Q8_0, BF16) FATTN_VEC_CASES_ALL_D(BF16, BF16) + FATTN_VEC_CASES_ALL_D(F8_E4M3, BF16) + + FATTN_VEC_CASES_ALL_D(F16, F8_E4M3) + FATTN_VEC_CASES_ALL_D(Q4_0, F8_E4M3) + FATTN_VEC_CASES_ALL_D(Q4_1, F8_E4M3) + FATTN_VEC_CASES_ALL_D(Q5_0, F8_E4M3) + FATTN_VEC_CASES_ALL_D(Q5_1, F8_E4M3) + FATTN_VEC_CASES_ALL_D(Q8_0, F8_E4M3) + FATTN_VEC_CASES_ALL_D(BF16, F8_E4M3) + FATTN_VEC_CASES_ALL_D(F8_E4M3, F8_E4M3) return nullptr; } @@ -553,6 +577,7 @@ static bool ggml_cuda_fattn_kv_type_supported(const ggml_type type) { case GGML_TYPE_Q5_0: case GGML_TYPE_Q5_1: case GGML_TYPE_Q8_0: + case GGML_TYPE_F8_E4M3: return true; default: return false; @@ -666,7 +691,7 @@ static best_fattn_kernel ggml_cuda_get_best_fattn_kernel(const int device, const const bool can_use_vector_kernel = Q->ne[0] <= 256 && Q->ne[0] % 64 == 0 && Q->ne[0] != 192 && K->ne[1] % FATTN_KQ_STRIDE == 0; // If Turing tensor cores are available, use them: - if (turing_mma_available(cc) && Q->ne[0] != 40 && Q->ne[0] != 72) { + if (turing_mma_available(cc)) { if (can_use_vector_kernel) { if (!ggml_is_quantized(K->type) && !ggml_is_quantized(V->type)) { // the sparse gather exists only in the MMA kernel: (DKQ, DV, 1, 8) with GQA > 4 diff --git a/ggml/src/ggml-cuda/fp8.cu b/ggml/src/ggml-cuda/fp8.cu new file mode 100644 index 000000000000..5fdec42208c8 --- /dev/null +++ b/ggml/src/ggml-cuda/fp8.cu @@ -0,0 +1,206 @@ +#include "fp8.cuh" + +#if !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA) && CUDART_VERSION >= 11080 + +struct fp8_abs_src { + const float * x; + int64_t ne0; + int64_t ne1; + int64_t ne2; + int64_t s1; + int64_t s2; + int64_t s3; + + __device__ float operator()(int64_t i) const { + const int64_t i0 = i % ne0; + i /= ne0; + const int64_t i1 = i % ne1; + i /= ne1; + const int64_t i2 = i % ne2; + const int64_t i3 = i / ne2; + const float value = fabsf(x[i0 + i1*s1 + i2*s2 + i3*s3]); + return isfinite(value) ? value : 448.0f; + } +}; + +static __global__ void fp8_amax_partials(fp8_abs_src src, int64_t ne, float * partials) { + float amax = 0.0f; + for (int64_t i = (int64_t) blockIdx.x*blockDim.x + threadIdx.x; i < ne; i += (int64_t) blockDim.x*gridDim.x) { + amax = fmaxf(amax, src(i)); + } + + __shared__ float shared[WARP_SIZE]; + amax = block_reduce(amax, shared); + if (threadIdx.x == 0) { + partials[blockIdx.x] = amax; + } +} + +static __global__ void fp8_amax_final(const float * partials, int n, float * amax) { + float value = 0.0f; + for (int i = threadIdx.x; i < n; i += blockDim.x) { + value = fmaxf(value, partials[i]); + } + + __shared__ float shared[WARP_SIZE]; + value = block_reduce(value, shared); + if (threadIdx.x == 0) { + *amax = value; + } +} + +static __global__ void quantize_fp8_e4m3( + const float * __restrict__ x, uint8_t * __restrict__ y, const float * __restrict__ amax, + float * __restrict__ scale, int64_t ne0, int64_t ne1, int64_t ne2, int64_t ne, int64_t s1, int64_t s2, int64_t s3) { + const int64_t i = (int64_t) blockIdx.x*blockDim.x + threadIdx.x; + if (i >= ne) { + return; + } + + int64_t tmp = i / ne0; + const int64_t i0 = i - tmp*ne0; + const int64_t i1 = tmp % ne1; + tmp /= ne1; + const int64_t i2 = tmp % ne2; + const int64_t i3 = tmp / ne2; + + const float d = *amax > 0.0f ? *amax / 448.0f : 1.0f; + y[i] = ggml_cuda_fp32_to_f8_e4m3(x[i0 + i1*s1 + i2*s2 + i3*s3] / d); + if (i == 0) { + *scale = d; + } +} + +static void fp8_set_batch(cublasLtMatrixLayout_t desc, int32_t count, int64_t stride) { + CUBLAS_CHECK(cublasLtMatrixLayoutSetAttribute(desc, CUBLASLT_MATRIX_LAYOUT_BATCH_COUNT, &count, sizeof(count))); + CUBLAS_CHECK(cublasLtMatrixLayoutSetAttribute(desc, CUBLASLT_MATRIX_LAYOUT_STRIDED_BATCH_OFFSET, &stride, sizeof(stride))); +} + +static void fp8_destroy_matmul( + cublasLtMatmulDesc_t op_desc, cublasLtMatrixLayout_t a_desc, cublasLtMatrixLayout_t b_desc, + cublasLtMatrixLayout_t d_desc) { + CUBLAS_CHECK(cublasLtMatrixLayoutDestroy(d_desc)); + CUBLAS_CHECK(cublasLtMatrixLayoutDestroy(b_desc)); + CUBLAS_CHECK(cublasLtMatrixLayoutDestroy(a_desc)); + CUBLAS_CHECK(cublasLtMatmulDescDestroy(op_desc)); +} + +bool ggml_cuda_mul_mat_fp8( + ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst, + const ggml_tensor * weight_scale) { + const int cc = ggml_cuda_info().devices[ctx.device].cc; + if (!fp8_mma_hardware_available(cc) || src0->type != GGML_TYPE_F8_E4M3 || src1->type != GGML_TYPE_F32 || + dst->type != GGML_TYPE_F32 || !ggml_is_contiguous(dst) || src0->ne[0] % 16 != 0 || src0->ne[1] % 16 != 0 || + src0->nb[0] != sizeof(uint8_t) || src0->nb[1] != (size_t) src0->ne[0] || src1->nb[0] != sizeof(float) || + !ggml_cuda_is_aligned(src0, 16) || !ggml_cuda_is_aligned(dst, 16)) { + return false; + } + + GGML_ASSERT(!weight_scale || (weight_scale->type == GGML_TYPE_F32 && ggml_is_contiguous(weight_scale) && ggml_nelements(weight_scale) == 1)); + + GGML_TENSOR_BINARY_OP_LOCALS + GGML_ASSERT(ne10 == ne00); + GGML_ASSERT(ne0 == ne01); + GGML_ASSERT(ne12 % ne02 == 0); + GGML_ASSERT(ne13 % ne03 == 0); + + cudaStream_t stream = ctx.stream(); + const int64_t ne_src1 = ggml_nelements(src1); + ggml_cuda_pool_alloc src1_fp8(ctx.pool(), ne_src1); + ggml_cuda_pool_alloc src1_scale(ctx.pool(), 1); + ggml_cuda_pool_alloc src1_amax(ctx.pool(), 1); + + const fp8_abs_src abs_src = { + (const float *) src1->data, ne10, ne11, ne12, + (int64_t) (nb11 / sizeof(float)), (int64_t) (nb12 / sizeof(float)), (int64_t) (nb13 / sizeof(float)) + }; + const int reduce_blocks = std::min((ne_src1 + 255)/256, 1024); + ggml_cuda_pool_alloc reduce_tmp(ctx.pool(), reduce_blocks); + fp8_amax_partials<<>>(abs_src, ne_src1, reduce_tmp.ptr); + fp8_amax_final<<<1, 256, 0, stream>>>(reduce_tmp.ptr, reduce_blocks, src1_amax.ptr); + + quantize_fp8_e4m3<<<(ne_src1 + 255)/256, 256, 0, stream>>>( + (const float *) src1->data, src1_fp8.ptr, src1_amax.ptr, src1_scale.ptr, ne10, ne11, ne12, ne_src1, + nb11 / sizeof(float), nb12 / sizeof(float), nb13 / sizeof(float)); + CUDA_CHECK(cudaGetLastError()); + + cublasLtMatmulDesc_t op_desc; + cublasLtMatrixLayout_t a_desc; + cublasLtMatrixLayout_t b_desc; + cublasLtMatrixLayout_t d_desc; + CUBLAS_CHECK(cublasLtMatmulDescCreate(&op_desc, CUBLAS_COMPUTE_32F, CUDA_R_32F)); + const cublasOperation_t trans_a = CUBLAS_OP_T; + CUBLAS_CHECK(cublasLtMatmulDescSetAttribute(op_desc, CUBLASLT_MATMUL_DESC_TRANSA, &trans_a, sizeof(trans_a))); + CUBLAS_CHECK(cublasLtMatmulDescSetAttribute( + op_desc, CUBLASLT_MATMUL_DESC_B_SCALE_POINTER, &src1_scale.ptr, sizeof(src1_scale.ptr))); + if (weight_scale) { + const float * scale_ptr = (const float *) weight_scale->data; + CUBLAS_CHECK(cublasLtMatmulDescSetAttribute( + op_desc, CUBLASLT_MATMUL_DESC_A_SCALE_POINTER, &scale_ptr, sizeof(scale_ptr))); + } + const int64_t r2 = ne12 / ne02; + const int64_t r3 = ne13 / ne03; + + // src1 batches that share a src0 matrix are adjacent in the compact src1 copy and in dst, fold them into N + const int64_t n = ne11*(ne02 == 1 ? r2*r3 : r2); + + // batch the remaining matmuls with a fixed src0 stride, or issue one batched call per i3 if there is none + int64_t batch_count = ne02; + int64_t stride_a = nb02; + int64_t n_calls = ne13; + if (ne02 == 1) { + batch_count = ne03; + stride_a = nb03; + n_calls = 1; + } else if (r3 == 1 && (ne03 == 1 || nb03 == ne02*nb02)) { + batch_count = ne02*ne03; + n_calls = 1; + } + + CUBLAS_CHECK(cublasLtMatrixLayoutCreate(&a_desc, CUDA_R_8F_E4M3, ne00, ne01, ne00)); + CUBLAS_CHECK(cublasLtMatrixLayoutCreate(&b_desc, CUDA_R_8F_E4M3, ne10, n, ne10)); + CUBLAS_CHECK(cublasLtMatrixLayoutCreate(&d_desc, CUDA_R_32F, ne0, n, ne0)); + if (batch_count > 1) { + fp8_set_batch(a_desc, batch_count, stride_a); + fp8_set_batch(b_desc, batch_count, ne10*n); + fp8_set_batch(d_desc, batch_count, ne0*n); + } + + // TODO: Allow workspace in the heuristic search for potential performance gains. + // cuBLASLt takes caller-owned workspace per matmul call. + cublasLtMatmulPreference_t preference; + CUBLAS_CHECK(cublasLtMatmulPreferenceCreate(&preference)); + cublasLtMatmulHeuristicResult_t heuristic; + int returned = 0; + const cublasStatus_t status = cublasLtMatmulAlgoGetHeuristic( + ctx.cublaslt_handle(), op_desc, a_desc, b_desc, d_desc, d_desc, preference, 1, &heuristic, &returned); + CUBLAS_CHECK(cublasLtMatmulPreferenceDestroy(preference)); + if (status != CUBLAS_STATUS_SUCCESS || returned == 0) { + fp8_destroy_matmul(op_desc, a_desc, b_desc, d_desc); + return false; + } + + const float alpha = 1.0f; + const float beta = 0.0f; + for (int64_t i3 = 0; i3 < n_calls; ++i3) { + const char * a = (const char *) src0->data + (i3/r3)*nb03; + const uint8_t * b = src1_fp8.ptr + i3*ne12*ne11*ne10; + float * d = (float *) ((char *) dst->data + i3*dst->nb[3]); + CUBLAS_CHECK(cublasLtMatmul(ctx.cublaslt_handle(), op_desc, &alpha, a, a_desc, b, b_desc, + &beta, d, d_desc, d, d_desc, &heuristic.algo, nullptr, 0, stream)); + } + + fp8_destroy_matmul(op_desc, a_desc, b_desc, d_desc); + return true; +} + +#else + +bool ggml_cuda_mul_mat_fp8( + ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst, + const ggml_tensor * weight_scale) { + GGML_UNUSED_VARS(ctx, src0, src1, dst, weight_scale); + return false; +} + +#endif diff --git a/ggml/src/ggml-cuda/fp8.cuh b/ggml/src/ggml-cuda/fp8.cuh new file mode 100644 index 000000000000..069349ed9ddb --- /dev/null +++ b/ggml/src/ggml-cuda/fp8.cuh @@ -0,0 +1,7 @@ +#pragma once + +#include "common.cuh" + +bool ggml_cuda_mul_mat_fp8( + ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst, + const ggml_tensor * weight_scale = nullptr); diff --git a/ggml/src/ggml-cuda/getrows.cu b/ggml/src/ggml-cuda/getrows.cu index d97e8fdb0f40..09b7f02b8f85 100644 --- a/ggml/src/ggml-cuda/getrows.cu +++ b/ggml/src/ggml-cuda/getrows.cu @@ -316,6 +316,10 @@ static void ggml_cuda_get_rows_switch_src0_type( get_rows_cuda_float((const nv_bfloat16 *) src0_d, src1_d, dst_d, ne00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb1, nb2, nb3, stream); break; + case GGML_TYPE_F8_E4M3: + get_rows_cuda_float((const ggml_fp8_e4m3_t *) src0_d, src1_d, dst_d, + ne00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb1, nb2, nb3, stream); + break; case GGML_TYPE_Q1_0: get_rows_cuda_q(src0_d, src1_d, dst_d, ne00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb1, nb2, nb3, stream); diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu index 06956ce7a3d9..fed14ddfc4a9 100644 --- a/ggml/src/ggml-cuda/ggml-cuda.cu +++ b/ggml/src/ggml-cuda/ggml-cuda.cu @@ -4,11 +4,13 @@ #include "ggml-cuda/allreduce.cuh" #include "ggml-cuda/common.cuh" +#include "ggml-cuda/graph.cuh" #include "ggml-cuda/acc.cuh" #include "ggml-cuda/add-id.cuh" #include "ggml-cuda/arange.cuh" #include "ggml-cuda/argmax.cuh" #include "ggml-cuda/argsort.cuh" +#include "ggml-cuda/batch-get.cuh" #include "ggml-cuda/binbcast.cuh" #include "ggml-cuda/clamp.cuh" #include "ggml-cuda/col2im-1d.cuh" @@ -23,14 +25,15 @@ #include "ggml-cuda/cpy.cuh" #include "ggml-cuda/cross-entropy-loss.cuh" #include "ggml-cuda/cumsum.cuh" -#include "ggml-cuda/diagmask.cuh" #include "ggml-cuda/diag.cuh" #include "ggml-cuda/fattn.cuh" +#include "ggml-cuda/fp8.cuh" #include "ggml-cuda/fwht.cuh" #include "ggml-cuda/getrows.cuh" #include "ggml-cuda/im2col.cuh" #include "ggml-cuda/mmf.cuh" #include "ggml-cuda/mmq.cuh" +#include "ggml-cuda/mmq-few.cuh" #include "ggml-cuda/mmvf.cuh" #include "ggml-cuda/mmvq.cuh" #include "ggml-cuda/moe-weighted-reduction.cuh" @@ -178,6 +181,11 @@ static bool ggml_cuda_p2p_enabled() { return enabled; } +bool ggml_cuda_pool_debug_enabled() { + static const bool enabled = ggml_cuda_env_flag("GGML_CUDA_POOL_DEBUG"); + return enabled; +} + static int ggml_cuda_op_offload_min_batch() { static const int min_batch = ggml_cuda_env_int("GGML_OP_OFFLOAD_MIN_BATCH", 32); return min_batch; @@ -582,6 +590,7 @@ struct ggml_cuda_pool_vmm : public ggml_cuda_pool { CUdeviceptr pool_addr = 0; size_t pool_used = 0; size_t pool_size = 0; + size_t max_size = CUDA_POOL_VMM_MAX_SIZE; size_t granularity; #if defined(GGML_USE_HIP) std::vector> mappings; @@ -603,7 +612,7 @@ struct ggml_cuda_pool_vmm : public ggml_cuda_pool { #else CU_CHECK(cuMemUnmap(pool_addr, pool_size)); #endif - CU_CHECK(cuMemAddressFree(pool_addr, CUDA_POOL_VMM_MAX_SIZE)); + CU_CHECK(cuMemAddressFree(pool_addr, max_size)); } } @@ -619,7 +628,21 @@ struct ggml_cuda_pool_vmm : public ggml_cuda_pool { size_t reserve_size = size - avail; reserve_size = granularity * ((reserve_size + granularity - 1) / granularity); - GGML_ASSERT(pool_size + reserve_size <= CUDA_POOL_VMM_MAX_SIZE); + // reserve virtual address space (if not already reserved) + if (pool_addr == 0) { + CUresult err = cuMemAddressReserve(&pool_addr, max_size, 0, 0, 0); + while (err == CUDA_ERROR_OUT_OF_MEMORY && (max_size >> 1) >= reserve_size && (max_size >> 1) >= granularity) { + max_size >>= 1; + err = cuMemAddressReserve(&pool_addr, max_size, 0, 0, 0); + } + CU_CHECK(err); + if (max_size < CUDA_POOL_VMM_MAX_SIZE) { + GGML_LOG_WARN(GGML_CUDA_NAME " pool[%d]: reserved %.2f GiB of virtual memory instead of %.2f GiB\n", + device, max_size / (1024.0 * 1024.0 * 1024.0), CUDA_POOL_VMM_MAX_SIZE / (1024.0 * 1024.0 * 1024.0)); + } + } + + GGML_ASSERT(pool_size + reserve_size <= max_size); // allocate more physical memory CUmemAllocationProp prop = {}; @@ -629,11 +652,6 @@ struct ggml_cuda_pool_vmm : public ggml_cuda_pool { CUmemGenericAllocationHandle handle; CU_CHECK(cuMemCreate(&handle, reserve_size, &prop, 0)); - // reserve virtual address space (if not already reserved) - if (pool_addr == 0) { - CU_CHECK(cuMemAddressReserve(&pool_addr, CUDA_POOL_VMM_MAX_SIZE, 0, 0, 0)); - } - // map at the end of the pool CUdeviceptr start_ptr = (CUdeviceptr)((char *)(pool_addr) + pool_size); CU_CHECK(cuMemMap(start_ptr, reserve_size, 0, handle, 0)); @@ -723,9 +741,43 @@ struct ggml_cuda_pool_vmm : public ggml_cuda_pool { }; #endif // defined(GGML_USE_VMM) +// Debug pool for NVIDIA compute-sanitizer, e.g. +// `GGML_CUDA_POOL_DEBUG=1 compute-sanitizer --tool memcheck ./build/bin/test-backend-ops -b CUDA0` +// Allocations are exact-size and freed on release, +// allowing memcheck to catch out-of-bounds and use-after-free. +struct ggml_cuda_pool_debug : public ggml_cuda_pool { + int device; + size_t pool_size = 0; // leak check (error if not zero at destruction) + + explicit ggml_cuda_pool_debug(int device) : + device(device) { + } + + ~ggml_cuda_pool_debug() { + GGML_ASSERT(pool_size == 0); + } + + void * alloc(size_t size, size_t * actual_size) override { + void * ptr; + CUDA_CHECK(ggml_cuda_device_malloc(&ptr, size, device)); + *actual_size = size; + pool_size += size; + return ptr; + } + + void free(void * ptr, size_t size) override { + ggml_cuda_set_device(device); + CUDA_CHECK(cudaFree(ptr)); + pool_size -= size; + } +}; + std::unique_ptr ggml_backend_cuda_context::new_pool_for_device(ggml_backend_cuda_context & ctx, int device, [[maybe_unused]] int stream_no) { + if (ggml_cuda_pool_debug_enabled()) { + return std::unique_ptr(new ggml_cuda_pool_debug(device)); + } #if defined(GGML_USE_VMM) if (ggml_cuda_info().devices[device].vmm) { return std::unique_ptr(new ggml_cuda_pool_vmm(device)); @@ -734,6 +786,124 @@ std::unique_ptr ggml_backend_cuda_context::new_pool_for_device(g return std::unique_ptr(new ggml_cuda_pool_leg(ctx, device)); } +ggml_backend_cuda_context::ggml_backend_cuda_context(int device) : + device(device), + name(GGML_CUDA_NAME + std::to_string(device)), + concurrent_stream_context(std::make_unique()) { +} + +#ifdef USE_CUDA_GRAPH +ggml_cuda_graph * ggml_backend_cuda_context::cuda_graph(const void * first_node_ptr, int64_t time_now) { + // Give all cached graphs 10s to be reused after an idle period, without updating each graph. + if (!cuda_graphs.empty() && time_now - last_graph_use_time >= 10'000'000) { + next_graph_eviction_time = time_now + 10'000'000; + } + last_graph_use_time = time_now; + + // Sweep every 5s during active use, evicting graphs unused for at least 10s. + if (time_now >= next_graph_eviction_time) { + next_graph_eviction_time = time_now + 5'000'000; + for (auto it = cuda_graphs.begin(); it != cuda_graphs.end(); ) { + if (it->first != first_node_ptr && time_now - it->second->last_used_time >= 10'000'000) { + it = cuda_graphs.erase(it); + } else { + ++it; + } + } + } + + auto it = cuda_graphs.find(first_node_ptr); + if (it == cuda_graphs.end()) { + it = cuda_graphs.emplace(first_node_ptr, std::make_unique()).first; + } + it->second->last_used_time = time_now; + return it->second.get(); +} + +bool ggml_backend_cuda_context::any_cuda_graph_enabled() const { + for (const auto & [key, graph] : cuda_graphs) { + if (graph && graph->is_enabled()) { + return true; + } + } + return false; +} + +bool ggml_backend_cuda_context::any_cuda_graph_has_instance() const { + for (const auto & [key, graph] : cuda_graphs) { + if (graph && !graph->instances.empty()) { + return true; + } + } + return false; +} +#endif // USE_CUDA_GRAPH + +#if defined(GGML_CUDA_USE_PDL) +bool ggml_cuda_kernel_can_use_pdl(const void * kernel) { + static const bool env_pdl_enabled = []() { + const char * env = getenv("GGML_CUDA_PDL"); + return env == nullptr || std::atoi(env) != 0; + }(); + if (!env_pdl_enabled) { + return false; + } + + const int device = ggml_cuda_get_device(); + + struct cache_key { + int device; + const void * kernel; + + bool operator==(const cache_key & other) const { return device == other.device && kernel == other.kernel; } + }; + + struct cache_key_hash { + // MurmurHash3 mixing function for better hash distribution (vs. just std::hash which in some implementations simply returns the identity) + static size_t hash_mix(size_t x) { + std::uint64_t y = x; + const std::uint64_t m = 0xe9846af9b1a615d; + + y ^= y >> 32; + y *= m; + y ^= y >> 32; + y *= m; + y ^= y >> 28; + + return static_cast(y); + } + + size_t operator()(const cache_key & key) const { + // Use a nonzero seed to avoid mapping all-zero keys to zero + size_t h = 42; + h = hash_mix(h + key.device); + h = hash_mix(h + reinterpret_cast(key.kernel)); + return h; + } + }; + + static std::mutex cache_mutex; + static std::unordered_map cache; + + const cache_key key = { device, kernel }; + std::lock_guard lock(cache_mutex); + const auto it = cache.find(key); + if (it != cache.end()) { + return it->second; + } + + cudaFuncAttributes attr = {}; + CUDA_CHECK(cudaFuncGetAttributes(&attr, kernel)); + + // PDL device-side primitives are emitted only for PTX versions >= 90. + // We have to guard on a loaded kernel's PTX version so a kernel forward-JIT'ed + // from pre-Hopper PTX to a Hopper-or-newer GPU does not opt into PDL. + const bool can_use_pdl = attr.ptxVersion >= 90; + cache.emplace(key, can_use_pdl); + return can_use_pdl; +} +#endif // defined(GGML_CUDA_USE_PDL) + // destroying a cuBLAS handle while a graph is being captured in a different thread can result in a CUDA error // this lock is used to ensure that no cuBLAS handle is destroyed while a graph is being captured @@ -760,6 +930,11 @@ ggml_backend_cuda_context::~ggml_backend_cuda_context() { CUDA_CHECK(cudaFree(cublas_workspaces[i][j])); } } +#if !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA) && CUDART_VERSION >= 11080 + if (cublaslt_handles[i] != nullptr) { + CUBLAS_CHECK(cublasLtDestroy(cublaslt_handles[i])); + } +#endif } } @@ -886,6 +1061,11 @@ static void ggml_backend_cuda_buffer_get_tensor_n(ggml_backend_buffer_t buffer, ggml_cuda_set_device(ctx->device); + if (!ggml_cuda_pinned_memory_disabled() && + ggml_cuda_get_tensor_n(ctx->device, tensors, data, offsets, sizes, n_tensors)) { + return; + } + for (size_t i = 0; i < n_tensors; i++) { if (sizes[i] == 0) { continue; @@ -1117,6 +1297,18 @@ struct ggml_backend_cuda_comm_context { ggml_cuda_ar_pipeline * ar_pipeline = nullptr; + // Number of ranks in the communicator when each rank is a separate process. Zero unless + // comm_init_rank created this context. + int world_size = 0; + + // Number of ranks taking part in the reduction. For a communicator from comm_init_rank this is + // world_size. Otherwise it is the number of backends this process holds. The NCCL reduction + // uses it to choose the tensor size below which it reduces in F32 rather than converting to + // BF16 first. + size_t n_ranks() const { + return world_size > 0 ? (size_t) world_size : backends.size(); + } + #ifdef GGML_USE_NCCL std::vector comms; #endif // GGML_USE_NCCL @@ -1144,6 +1336,7 @@ static bool ggml_backend_cuda_comm_allreduce_nccl( } const size_t n_backends = comm_ctx->backends.size(); + const size_t n_ranks = comm_ctx->n_ranks(); for (size_t i = 0; i < n_backends; ++i) { GGML_ASSERT(tensors[i] != nullptr); @@ -1153,7 +1346,7 @@ static bool ggml_backend_cuda_comm_allreduce_nccl( // For small tensors, simply reduce them as FP32. // The following heuristic for how "small" a tensor should be is based on RTX 4090s connected via 16x PCIe 4.0. - if ((n_backends <= 2 && ne < 32768) || (n_backends == 3 && ne < 131072) || (n_backends >= 4 && ne < 262144)) { + if ((n_ranks <= 2 && ne < 32768) || (n_ranks == 3 && ne < 131072) || (n_ranks >= 4 && ne < 262144)) { for (size_t i = 0; i < n_backends; ++i) { if ((tensors[i]->flags & GGML_TENSOR_FLAG_COMPUTE) == 0) { ggml_backend_cuda_context * cuda_ctx = (ggml_backend_cuda_context *) comm_ctx->backends[i]->context; @@ -1384,6 +1577,90 @@ static void * ggml_backend_cuda_comm_init(ggml_backend_t * backends, size_t n_ba return ret; } +#ifdef GGML_USE_NCCL +// Builds the communicator with ncclCommInitRank, for the case where each rank is its own +// process. Blocks until all world_size ranks have joined. +static void * ggml_backend_cuda_comm_init_rank(ggml_backend_t backend, int rank, int world_size, + ggml_backend_comm_bootstrap_callback bootstrap_callback, void * bootstrap_callback_data) { + if (world_size < 1 || rank < 0 || rank >= world_size) { + GGML_LOG_ERROR("%s: invalid rank %d for world size %d\n", __func__, rank, world_size); + return nullptr; + } + + // Every rank sends whether its local checks passed, and rank 0 also sends the NCCL unique id. + // So all ranks either join together or fail together, and no rank waits in ncclCommInitRank alone. + struct bootstrap_msg { + uint8_t ok; + ncclUniqueId id; + }; + bootstrap_msg local; + memset(&local, 0, sizeof(local)); + local.ok = 1; + + if (!ggml_backend_is_cuda(backend)) { + GGML_LOG_ERROR("%s: backend is not a CUDA backend\n", __func__); + local.ok = 0; + } else { + // NCCL requires one distinct physical GPU per rank, see ggml_backend_cuda_comm_init_nccl. + const ggml_cuda_device_info & info = ggml_cuda_info(); + if (info.device_count > info.physical_device_count) { + GGML_LOG_ERROR("%s: CUDA virtual devices are in use, which NCCL does not support\n", __func__); + local.ok = 0; + } + } + if (local.ok && rank == 0) { + const ncclResult_t rc = ncclGetUniqueId(&local.id); + if (rc != ncclSuccess) { + GGML_LOG_ERROR("%s: ncclGetUniqueId failed (%s)\n", __func__, ncclGetErrorString(rc)); + local.ok = 0; + } + } + + std::vector all(world_size); + if (!bootstrap_callback(bootstrap_callback_data, &local, all.data(), sizeof(local))) { + GGML_LOG_ERROR("%s: bootstrap exchange failed on rank %d\n", __func__, rank); + return nullptr; + } + for (int i = 0; i < world_size; i++) { + if (!all[i].ok) { + GGML_LOG_ERROR("%s: rank %d failed its setup checks\n", __func__, i); + return nullptr; + } + } + + ggml_backend_cuda_context * cuda_ctx = (ggml_backend_cuda_context *) backend->context; + + auto * ret = new ggml_backend_cuda_comm_context; + ret->backends.push_back(backend); + ret->dev_ids.push_back(cuda_ctx->device); + ret->world_size = world_size; + ret->comms.resize(1); + ret->try_allreduce = ggml_backend_cuda_comm_try_allreduce_nccl; + + // On DGX Spark, NCCL picks 64 channels by default, which is slower and uses more memory than 8. + // NCCL_MAX_CTAS still overrides this. + ncclConfig_t config = NCCL_CONFIG_INITIALIZER; + const int cc = ggml_cuda_info().devices[cuda_ctx->device].cc; + if (GGML_CUDA_CC_IS_NVIDIA(cc) && cc == GGML_CUDA_CC_DGX_SPARK) { + config.maxCTAs = 8; + } + + ggml_cuda_set_device(cuda_ctx->device); + const ncclResult_t rc = ncclCommInitRankConfig(&ret->comms[0], world_size, all[0].id, rank, &config); + if (rc != ncclSuccess) { + GGML_LOG_ERROR("%s: ncclCommInitRankConfig failed for rank %d/%d (%s)\n", + __func__, rank, world_size, ncclGetErrorString(rc)); + ret->comms.clear(); + delete ret; + return nullptr; + } + + GGML_LOG_INFO("%s: joined NCCL communicator as rank %d/%d on device %d\n", + __func__, rank, world_size, cuda_ctx->device); + return ret; +} +#endif // GGML_USE_NCCL + // Top-level dispatch -- calls the function pointer chosen by comm_init. // Returns false to let the meta-backend's butterfly run. static bool ggml_backend_cuda_comm_allreduce_tensor(void * comm_ctx_v, struct ggml_tensor ** tensors) { @@ -1766,9 +2043,9 @@ static void ggml_cuda_mul_mat_cublas_impl(ggml_backend_cuda_context & ctx, const static void ggml_cuda_mul_mat_cublas(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { const int cc = ggml_cuda_info().devices[ctx.device].cc; - const ggml_prec prec = (ggml_prec) ggml_get_op_params_i32(dst, 0); + const ggml_prec prec = ggml_prec_get_acc(dst); ggml_type compute_type = src0->type; - if (ggml_is_quantized(compute_type)) { + if (ggml_is_quantized(compute_type) || compute_type == GGML_TYPE_F8_E4M3) { compute_type = fast_fp16_hardware_available(cc) ? GGML_TYPE_F16 : GGML_TYPE_F32; } else if (compute_type == GGML_TYPE_F16 && !fast_fp16_hardware_available(cc)) { compute_type = GGML_TYPE_F32; @@ -1939,7 +2216,7 @@ static bool ggml_cuda_should_fuse_mul_mat(const ggml_tensor * ffn_up, return true; } -static bool ggml_cuda_should_fuse_mul_mat_vec_f(const ggml_tensor * tensor) { +static bool ggml_cuda_should_fuse_mul_mat_vec_f(const ggml_tensor * tensor, const ggml_tensor * gate = nullptr) { ggml_tensor * src0 = tensor->src[0]; ggml_tensor * src1 = tensor->src[1]; const ggml_tensor * dst = tensor; @@ -1947,12 +2224,17 @@ static bool ggml_cuda_should_fuse_mul_mat_vec_f(const ggml_tensor * tensor) { const bool is_mul_mat_id = tensor->op == GGML_OP_MUL_MAT_ID; bool use_mul_mat_vec_f = - (src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16 || src0->type == GGML_TYPE_BF16) && + (src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16 || src0->type == GGML_TYPE_BF16 || + src0->type == GGML_TYPE_F8_E4M3) && src1->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32; const int cc = ggml_cuda_info().devices[ggml_cuda_get_device()].cc; const int warp_size = ggml_cuda_info().devices[ggml_cuda_get_device()].warp_size; - use_mul_mat_vec_f = use_mul_mat_vec_f && ggml_cuda_should_use_mmvf(src0->type, cc, warp_size, src0->ne, src0->nb, is_mul_mat_id ? src1->ne[2] : src1->ne[1]); + use_mul_mat_vec_f = use_mul_mat_vec_f && ggml_cuda_should_use_mmvf(src0, cc, warp_size, is_mul_mat_id ? src1->ne[2] : src1->ne[1]); + + if (gate && !ggml_cuda_should_use_mmvf(gate->src[0], cc, warp_size, is_mul_mat_id ? src1->ne[2] : src1->ne[1])) { + return false; + } //we only support fusion for ncols_dst = 1 if (tensor->op == GGML_OP_MUL_MAT && dst->ne[1] != 1) { @@ -1976,11 +2258,11 @@ static bool ggml_cuda_should_fuse_mul_mat_vec_q(const ggml_tensor * tensor) { ggml_nbytes(src0) != ggml_backend_buffer_get_alloc_size(src0->buffer, src0) && src0->view_src; - bool use_mul_mat_vec_q = ggml_is_quantized(src0->type) && !bad_padding_clear && src1->type == GGML_TYPE_F32 && - dst->type == GGML_TYPE_F32 && src1->ne[1] <= MMVQ_MAX_BATCH_SIZE; + const int cc = ggml_cuda_info().devices[ggml_cuda_get_device()].cc; + bool use_mul_mat_vec_q = ggml_cuda_should_use_mmvq(src0->type, cc, src1->ne[1]) && !bad_padding_clear && + src1->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32; // fusion is not universally faster on Pascal - const int cc = ggml_cuda_info().devices[ggml_cuda_get_device()].cc; if (cc <= GGML_CUDA_CC_PASCAL) { return false; } @@ -2048,8 +2330,50 @@ static bool ggml_cuda_match_shared_expert(const ggml_cgraph * graph, int routed_ return true; } +// The weight scale that scale_node (MUL) applies to the output of mm_node (MUL_MAT), or nullptr. +// Shared by ggml_cuda_try_fuse and graph_optimize so that both accept the same patterns. +static const ggml_tensor * ggml_cuda_get_mul_mat_scale(const ggml_tensor * scale_node, const ggml_tensor * mm_node) { + const bool scale_lhs_mm = scale_node->src[0] == mm_node; + const bool scale_rhs_mm = scale_node->src[1] == mm_node; + if (!scale_lhs_mm && !scale_rhs_mm) { + return nullptr; + } + + const ggml_tensor * scale = scale_lhs_mm ? scale_node->src[1] : scale_node->src[0]; + if ((mm_node->src[0]->type != GGML_TYPE_NVFP4 && mm_node->src[0]->type != GGML_TYPE_F8_E4M3) || scale_node->type != GGML_TYPE_F32 || + scale->type != GGML_TYPE_F32 || !ggml_is_contiguous(scale) || ggml_nelements(scale) != 1 || + !ggml_are_same_shape(scale_node, mm_node)) { + return nullptr; + } + + return scale; +} + +// The same for MUL_MAT_ID with one scale per expert: RESHAPE, REPEAT and GET_ROWS (by the ids) of the scales, then the MUL. +static const ggml_tensor * ggml_cuda_get_mul_mat_id_scale(const ggml_tensor * reshape, const ggml_tensor * repeat, const ggml_tensor * getrows, + const ggml_tensor * scale_node, const ggml_tensor * mm_node) { + if (repeat->src[0] != reshape || getrows->src[0] != repeat || getrows->src[1] != mm_node->src[2]) { + return nullptr; + } + if (!((scale_node->src[0] == mm_node && scale_node->src[1] == getrows) || + (scale_node->src[0] == getrows && scale_node->src[1] == mm_node))) { + return nullptr; + } + + const ggml_tensor * scale = reshape->src[0]; + if ((mm_node->src[0]->type != GGML_TYPE_NVFP4 && mm_node->src[0]->type != GGML_TYPE_F8_E4M3) || scale_node->type != GGML_TYPE_F32 || + scale->type != GGML_TYPE_F32 || !ggml_is_contiguous(scale) || ggml_nelements(scale) != mm_node->src[0]->ne[2] || + !ggml_are_same_shape(scale_node, mm_node)) { + return nullptr; + } + + return scale; +} + // Buffer- and batch-size independent, so graph_optimize produces the same graph topology for every ubatch size (see #29986). -static bool ggml_cuda_mul_mat_q_fusion_matches(const ggml_tensor * up, const ggml_tensor * gate, const ggml_tensor * glu, const int device) { +// up_scale/gate_scale: the MUL nodes that apply the NVFP4 weight scales (per tensor or per expert) between the MUL_MATs and the GLU. +static bool ggml_cuda_mul_mat_q_fusion_matches(const ggml_tensor * up, const ggml_tensor * gate, const ggml_tensor * glu, const int device, + const ggml_tensor * up_scale = nullptr, const ggml_tensor * gate_scale = nullptr) { const ggml_tensor * x_up = up->src[0]; const ggml_tensor * y = up->src[1]; @@ -2062,7 +2386,12 @@ static bool ggml_cuda_mul_mat_q_fusion_matches(const ggml_tensor * up, const ggm return false; } - if (!ggml_cuda_should_fuse_mul_mat(up, gate, glu)) { + if (!ggml_cuda_should_fuse_mul_mat(up, gate, glu, nullptr, nullptr, up_scale, gate_scale)) { + return false; + } + + // The fused kernel runs both MUL_MATs at one src1 precision, see ggml_cuda_mul_mat_q. + if (ggml_prec_get_src(up, 1) != ggml_prec_get_src(gate, 1)) { return false; } @@ -2075,7 +2404,8 @@ static bool ggml_cuda_mul_mat_q_fusion_matches(const ggml_tensor * up, const ggm return turing_mma_available(ggml_cuda_info().devices[device].cc); } -static bool ggml_cuda_should_fuse_mul_mat_q(const ggml_tensor * up, const ggml_tensor * gate, const ggml_tensor * glu) { +static bool ggml_cuda_should_fuse_mul_mat_q(const ggml_tensor * up, const ggml_tensor * gate, const ggml_tensor * glu, + const ggml_tensor * up_scale = nullptr, const ggml_tensor * gate_scale = nullptr) { // A hinted matmul can be fused, but its unfused path may be faster (e.g. FWHT for GGML_HINT_SRC0_IS_HADAMARD). if (ggml_get_op_params_i32(up, 1) != GGML_HINT_NONE || ggml_get_op_params_i32(gate, 1) != GGML_HINT_NONE) { return false; @@ -2087,7 +2417,7 @@ static bool ggml_cuda_should_fuse_mul_mat_q(const ggml_tensor * up, const ggml_t ggml_nbytes(x) != ggml_backend_buffer_get_alloc_size(x->buffer, x) && x->view_src; }; const int device = ggml_cuda_get_device(); - if (!ggml_cuda_mul_mat_q_fusion_matches(up, gate, glu, device) || bad_padding_clear(up->src[0]) || bad_padding_clear(gate->src[0])) { + if (!ggml_cuda_mul_mat_q_fusion_matches(up, gate, glu, device, up_scale, gate_scale) || bad_padding_clear(up->src[0]) || bad_padding_clear(gate->src[0])) { return false; } @@ -2101,12 +2431,16 @@ static bool ggml_cuda_should_fuse_mul_mat_q(const ggml_tensor * up, const ggml_t const int64_t ncols = is_id ? y->ne[2] : y->ne[1]; const int64_t nchannels = is_id ? x_up->ne[2] : y->ne[2]*y->ne[3]; const bool use_mmvq = is_id ? ncols <= get_mmvq_mmid_max_batch(x_up->type, cc) : ggml_cuda_should_use_mmvq(x_up->type, cc, ncols); + // do not fuse the matmuls that few-row MMQ takes in ggml_cuda_mul_mat + if (!is_id && ggml_cuda_should_use_mmq_few(x_up, y, cc)) { + return false; + } if (use_mmvq || !ggml_cuda_should_use_mmq(x_up->type, cc, ncols, /*n_experts =*/ is_id ? x_up->ne[2] : 0)) { return false; } // The fused kernel needs one block per tile (no stream-k split), so only fuse if the tiling is efficient. - return ggml_cuda_mmq_fusion_is_efficient(x_up->type, x_up->ne[1], ncols, nchannels, device); + return ggml_cuda_mmq_fusion_is_efficient(x_up->type, ggml_prec_get_src(up, 1), x_up->ne[1], ncols, nchannels, device); } static void ggml_cuda_mul_mat(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { @@ -2129,17 +2463,17 @@ static void ggml_cuda_mul_mat(ggml_backend_cuda_context & ctx, const ggml_tensor const int cc = ggml_cuda_info().devices[ctx.device].cc; const int warp_size = ggml_cuda_info().devices[ctx.device].warp_size; - if (ggml_cuda_should_use_mmvf(src0->type, cc, warp_size, src0->ne, src0->nb, ne11)) { - // The custom F16 vector kernel can be used over batched cuBLAS GEMM. + if (ggml_cuda_should_use_mmvf(src0, cc, warp_size, ne11)) { + // The custom vector kernel can be used over batched cuBLAS GEMM. // But this is only faster for GPUs without tensor cores or with a thin src0 matrix (particularly KQV in attention) ggml_cuda_mul_mat_vec_f(ctx, src0, src1, nullptr, dst); return; } - // A transposed vector can still use MMVQ (i.e. ne01 == 1) + // A transposed vector can still use MMVF (i.e. ne01 == 1) if (ne01 == 1 && ne11 > MMVF_MAX_BATCH_SIZE && ne2 == 1 && ne3 == 1 && src0->type == GGML_TYPE_F32 && ggml_is_contiguous(src0) && ggml_is_contiguous(src1) && ggml_is_contiguous(dst) - && ggml_cuda_should_use_mmvf(src1->type, cc, warp_size, src1->ne, src1->nb, /*ne11 =*/ 1)) { + && ggml_cuda_should_use_mmvf(src1, cc, warp_size, /*ne11 =*/ 1)) { ggml_tensor dst_vec = *dst; dst_vec.ne[0] = ne11; dst_vec.ne[1] = 1; @@ -2153,14 +2487,23 @@ static void ggml_cuda_mul_mat(ggml_backend_cuda_context & ctx, const ggml_tensor ggml_cuda_mul_mat_f(ctx, src0, src1, nullptr, dst); return; } - if (ggml_cuda_should_use_mmvq(src0->type, cc, ne11)) { + // from a per-type number of src1 columns up to 24, few-row MMQ is faster than MMVQ and MMQ + const bool use_mmq_few = ggml_cuda_should_use_mmq_few(src0, src1, cc); + if (!use_mmq_few && ggml_cuda_should_use_mmvq(src0->type, cc, ne11)) { ggml_cuda_mul_mat_vec_q(ctx, src0, src1, nullptr, dst); return; } + if (use_mmq_few) { + ggml_cuda_mul_mat_q_few(ctx, src0, src1, dst); + return; + } if (ggml_cuda_should_use_mmq(src0->type, cc, ne11, /*n_experts =*/ 0)) { ggml_cuda_mul_mat_q(ctx, src0, src1, nullptr, dst); return; } + if (src0->type == GGML_TYPE_F8_E4M3 && ggml_cuda_mul_mat_fp8(ctx, src0, src1, dst)) { + return; + } ggml_cuda_mul_mat_cublas(ctx, src0, src1, dst); } @@ -2169,7 +2512,6 @@ static void ggml_cuda_mul_mat(ggml_backend_cuda_context & ctx, const ggml_tensor static bool ggml_cuda_mul_mat_id_needs_sync(const ggml_tensor * dst, const int cc) { const ggml_tensor * src0 = dst->src[0]; const ggml_tensor * src1 = dst->src[1]; - const int warp_size = ggml_cuda_info().devices[ggml_cuda_get_device()].warp_size; if (src1->type != GGML_TYPE_F32 || dst->type != GGML_TYPE_F32) { @@ -2181,7 +2523,10 @@ static bool ggml_cuda_mul_mat_id_needs_sync(const ggml_tensor * dst, const int c if (dst->ne[2] <= get_mmvq_mmid_max_batch(src0->type, cc)) { return false; } - } else if (GGML_CUDA_CC_IS_AMD(cc)) { + } else if (src0->type == GGML_TYPE_F8_E4M3 && + ggml_cuda_should_use_mmvf(src0, cc, warp_size, dst->ne[2])) { + return false; + } else if (src0->type != GGML_TYPE_F8_E4M3 && GGML_CUDA_CC_IS_AMD(cc)) { return false; } } @@ -2207,7 +2552,7 @@ static void ggml_cuda_mul_mat_id(ggml_backend_cuda_context & ctx, ggml_tensor * GGML_TENSOR_BINARY_OP_LOCALS - const int cc = ggml_cuda_info().devices[ggml_cuda_get_device()].cc; + const int cc = ggml_cuda_info().devices[ggml_cuda_get_device()].cc; const int warp_size = ggml_cuda_info().devices[ggml_cuda_get_device()].warp_size; // [TAG_MUL_MAT_ID_CUDA_GRAPHS] @@ -2220,8 +2565,12 @@ static void ggml_cuda_mul_mat_id(ggml_backend_cuda_context & ctx, ggml_tensor * ggml_cuda_mul_mat_vec_q(ctx, src0, src1, ids, dst); return; } + } else if (src0->type == GGML_TYPE_F8_E4M3 && + ggml_cuda_should_use_mmvf(src0, cc, warp_size, ne2)) { + ggml_cuda_mul_mat_vec_f(ctx, src0, src1, ids, dst); + return; } else { - if (GGML_CUDA_CC_IS_AMD(cc)) { + if (src0->type != GGML_TYPE_F8_E4M3 && GGML_CUDA_CC_IS_AMD(cc)) { ggml_cuda_mul_mat_vec_f(ctx, src0, src1, ids, dst); return; } @@ -2247,7 +2596,7 @@ static void ggml_cuda_mul_mat_id(ggml_backend_cuda_context & ctx, ggml_tensor * GGML_ASSERT(nb2 % nb1 == 0); const ggml_type type_src1_sorted = (src0->type == GGML_TYPE_F16 && !fast_fp16_hardware_available(cc)) - || ggml_is_quantized(src0->type) ? GGML_TYPE_F32 : src0->type; + || ggml_is_quantized(src0->type) || src0->type == GGML_TYPE_F8_E4M3 ? GGML_TYPE_F32 : src0->type; const ggml_type type_dst_sorted = GGML_TYPE_F32; const size_t ts_src1_sorted = ggml_type_size(type_src1_sorted); const size_t ts_dst_sorted = ggml_type_size(type_dst_sorted); @@ -2392,7 +2741,6 @@ static bool ggml_cuda_compute_forward(ggml_backend_cuda_context & ctx, struct gg ggml_cuda_dup(ctx, dst); break; case GGML_OP_ADD: - case GGML_OP_ADD1: // TODO: more efficient implementation ggml_cuda_op_add(ctx, dst); break; case GGML_OP_ADD_ID: @@ -2587,9 +2935,6 @@ static bool ggml_cuda_compute_forward(ggml_backend_cuda_context & ctx, struct gg case GGML_OP_DIAG: ggml_cuda_op_diag(ctx, dst); break; - case GGML_OP_DIAG_MASK_INF: - ggml_cuda_op_diag_mask_inf(ctx, dst); - break; case GGML_OP_SOFT_MAX: ggml_cuda_op_soft_max(ctx, dst); break; @@ -2939,7 +3284,54 @@ static bool ggml_cuda_graph_update_required(ggml_backend_cuda_context * cuda_ctx return res; } -static void ggml_cuda_graph_update_executable(cudaGraphExec_t & instance, cudaGraph_t cuda_graph) { +#ifdef USE_CUDA_GRAPH_CLUSTER_CHECK +static std::vector ggml_cuda_graph_get_clusters(cudaGraph_t cuda_graph, int device) { + std::vector clusters; + static const bool runtime_affected = [] { + int version = 0; + CUDA_CHECK(cudaRuntimeGetVersion(&version)); + // TODO: upper bound once fixed in the CUDA runtime + return version >= 11080; + }(); + if (!runtime_affected || ggml_cuda_info().devices[device].cc < GGML_CUDA_CC_HOPPER) { + return clusters; + } + size_t n_nodes = 0; + CUDA_CHECK(cudaGraphGetNodes(cuda_graph, nullptr, &n_nodes)); + std::vector nodes(n_nodes); + CUDA_CHECK(cudaGraphGetNodes(cuda_graph, nodes.data(), &n_nodes)); + for (size_t i = 0; i < n_nodes; ++i) { + cudaGraphNodeType type; + CUDA_CHECK(cudaGraphNodeGetType(nodes[i], &type)); + if (type != cudaGraphNodeTypeKernel) { + continue; + } + cudaLaunchAttributeValue val = {}; + CUDA_CHECK(cudaGraphKernelNodeGetAttribute(nodes[i], cudaLaunchAttributeClusterDimension, &val)); + if (val.clusterDim.x * val.clusterDim.y * val.clusterDim.z > 1) { + clusters.push_back({ i, val.clusterDim.x, val.clusterDim.y, val.clusterDim.z }); + } + } + return clusters; +} +#endif // USE_CUDA_GRAPH_CLUSTER_CHECK + +static void ggml_cuda_graph_update_executable(ggml_cuda_graph_instance & instance, cudaGraph_t cuda_graph, int device) { +#ifdef USE_CUDA_GRAPH_CLUSTER_CHECK + std::vector clusters = ggml_cuda_graph_get_clusters(cuda_graph, device); + if (instance.clusters != clusters) { + if (ggml_cuda_graph_debug_enabled()) { + GGML_LOG_DEBUG("%s: cluster dims changed, re-instantiating CUDA graph\n", __func__); + } + CUDA_CHECK(cudaGraphExecDestroy(instance)); + CUDA_CHECK(cudaGraphInstantiate(&instance.handle, cuda_graph, nullptr, nullptr, 0)); + instance.clusters = std::move(clusters); + return; + } +#else + GGML_UNUSED(device); +#endif // USE_CUDA_GRAPH_CLUSTER_CHECK + #if CUDART_VERSION >= 12000 cudaGraphExecUpdateResultInfo result_info; cudaError_t stat = cudaGraphExecUpdate(instance, cuda_graph, &result_info); @@ -2958,8 +3350,8 @@ static void ggml_cuda_graph_update_executable(cudaGraphExec_t & instance, cudaGr // so instead clear error and re-instantiate (void)cudaGetLastError(); CUDA_CHECK(cudaGraphExecDestroy(instance)); - instance = nullptr; - CUDA_CHECK(cudaGraphInstantiate(&instance, cuda_graph, nullptr, nullptr, 0)); + instance.handle = nullptr; + CUDA_CHECK(cudaGraphInstantiate(&instance.handle, cuda_graph, nullptr, nullptr, 0)); } else { GGML_ASSERT(stat == cudaSuccess); } @@ -2980,11 +3372,14 @@ static void ggml_cuda_graph_launch_chunk(ggml_backend_cuda_context * cuda_ctx, g CUDA_CHECK(cudaStreamEndCapture(cuda_ctx->stream(), &cuda_graph)); if (idx < graph->instances.size()) { - ggml_cuda_graph_update_executable(graph->instances[idx], cuda_graph); + ggml_cuda_graph_update_executable(graph->instances[idx], cuda_graph, cuda_ctx->device); } else { GGML_ASSERT(idx == graph->instances.size()); - graph->instances.emplace_back(nullptr); - CUDA_CHECK(cudaGraphInstantiate(&graph->instances[idx], cuda_graph, nullptr, nullptr, 0)); + graph->instances.emplace_back(); + CUDA_CHECK(cudaGraphInstantiate(&graph->instances[idx].handle, cuda_graph, nullptr, nullptr, 0)); +#ifdef USE_CUDA_GRAPH_CLUSTER_CHECK + graph->instances[idx].clusters = ggml_cuda_graph_get_clusters(cuda_graph, cuda_ctx->device); +#endif // USE_CUDA_GRAPH_CLUSTER_CHECK } CUDA_CHECK(cudaGraphDestroy(cuda_graph)); @@ -4181,43 +4576,6 @@ static int ggml_cuda_try_fuse(ggml_backend_cuda_context * cuda_ctx, ggml_cgraph bool fused_mul_mat_vec = false; int fused_node_count = 0; - auto get_mul_mat_scale = [](const ggml_tensor * scale_node, const ggml_tensor * mm_node) -> const ggml_tensor * { - const bool scale_lhs_mm = scale_node->src[0] == mm_node; - const bool scale_rhs_mm = scale_node->src[1] == mm_node; - if (!scale_lhs_mm && !scale_rhs_mm) { - return nullptr; - } - - const ggml_tensor * scale = scale_lhs_mm ? scale_node->src[1] : scale_node->src[0]; - if (mm_node->src[0]->type != GGML_TYPE_NVFP4 || scale_node->type != GGML_TYPE_F32 || - scale->type != GGML_TYPE_F32 || !ggml_is_contiguous(scale) || ggml_nelements(scale) != 1 || - !ggml_are_same_shape(scale_node, mm_node)) { - return nullptr; - } - - return scale; - }; - - auto get_mul_mat_id_scale = [](const ggml_tensor * reshape, const ggml_tensor * repeat, const ggml_tensor * getrows, - const ggml_tensor * scale_node, const ggml_tensor * mm_node) -> const ggml_tensor * { - if (repeat->src[0] != reshape || getrows->src[0] != repeat || getrows->src[1] != mm_node->src[2]) { - return nullptr; - } - if (!((scale_node->src[0] == mm_node && scale_node->src[1] == getrows) || - (scale_node->src[0] == getrows && scale_node->src[1] == mm_node))) { - return nullptr; - } - - const ggml_tensor * scale = reshape->src[0]; - if (mm_node->src[0]->type != GGML_TYPE_NVFP4 || scale_node->type != GGML_TYPE_F32 || - scale->type != GGML_TYPE_F32 || !ggml_is_contiguous(scale) || ggml_nelements(scale) != mm_node->src[0]->ne[2] || - !ggml_are_same_shape(scale_node, mm_node)) { - return nullptr; - } - - return scale; - }; - auto get_bias_tensor = [](const ggml_tensor * bias_node, const ggml_tensor * mul_node, ggml_op op_bias) -> const ggml_tensor * { if (op_bias == GGML_OP_ADD) { if (bias_node->src[0] == mul_node) { @@ -4284,8 +4642,8 @@ static int ggml_cuda_try_fuse(ggml_backend_cuda_context * cuda_ctx, ggml_cgraph continue; } - const ggml_tensor * gate_scale = get_mul_mat_scale(gate_scale_n, gate_n); - const ggml_tensor * up_scale = get_mul_mat_scale(up_scale_n, up_n); + const ggml_tensor * gate_scale = ggml_cuda_get_mul_mat_scale(gate_scale_n, gate_n); + const ggml_tensor * up_scale = ggml_cuda_get_mul_mat_scale(up_scale_n, up_n); if (!gate_scale || !up_scale) { continue; } @@ -4310,12 +4668,27 @@ static int ggml_cuda_try_fuse(ggml_backend_cuda_context * cuda_ctx, ggml_cgraph fusion_data.glu_op = ggml_get_glu_op(glu); fusion_data.glu_limit = ggml_get_op_params_f32(glu, 3); + if (ggml_cuda_should_fuse_mul_mat_vec_f(up_n, gate_n)) { + ggml_cuda_mul_mat_vec_f(*cuda_ctx, src0, src1, ids, cgraph->nodes[glu_idx], &fusion_data); + fused_mul_mat_vec = true; + fused_node_count = n_ops; + break; + } + if (ggml_cuda_should_fuse_mul_mat_vec_q(up_n)) { ggml_cuda_mul_mat_vec_q(*cuda_ctx, src0, src1, ids, cgraph->nodes[glu_idx], &fusion_data); fused_mul_mat_vec = true; fused_node_count = n_ops; break; } + + if (!with_bias && ggml_cuda_should_fuse_mul_mat_q(up_n, gate_n, glu, up_scale_n, gate_scale_n)) { + fusion_data.prec_src1 = ggml_prec_get_src(up_n, 1); + ggml_cuda_mul_mat_q(*cuda_ctx, src0, src1, ids, cgraph->nodes[glu_idx], &fusion_data); + fused_mul_mat_vec = true; + fused_node_count = n_ops; + break; + } } if (fused_mul_mat_vec) { @@ -4380,9 +4753,9 @@ static int ggml_cuda_try_fuse(ggml_backend_cuda_context * cuda_ctx, ggml_cgraph continue; } - const ggml_tensor * gate_scale = get_mul_mat_id_scale(cgraph->nodes[gate_idx + 1], cgraph->nodes[gate_idx + 2], + const ggml_tensor * gate_scale = ggml_cuda_get_mul_mat_id_scale(cgraph->nodes[gate_idx + 1], cgraph->nodes[gate_idx + 2], cgraph->nodes[gate_idx + 3], gate_scale_n, gate_n); - const ggml_tensor * up_scale = get_mul_mat_id_scale(cgraph->nodes[up_idx + 1], cgraph->nodes[up_idx + 2], + const ggml_tensor * up_scale = ggml_cuda_get_mul_mat_id_scale(cgraph->nodes[up_idx + 1], cgraph->nodes[up_idx + 2], cgraph->nodes[up_idx + 3], up_scale_n, up_n); if (!gate_scale || !up_scale) { continue; @@ -4404,12 +4777,27 @@ static int ggml_cuda_try_fuse(ggml_backend_cuda_context * cuda_ctx, ggml_cgraph fusion_data.glu_op = ggml_get_glu_op(glu); fusion_data.glu_limit = ggml_get_op_params_f32(glu, 3); + if (ggml_cuda_should_fuse_mul_mat_vec_f(up_n, gate_n)) { + ggml_cuda_mul_mat_vec_f(*cuda_ctx, src0, src1, ids, cgraph->nodes[glu_idx], &fusion_data); + fused_mul_mat_vec = true; + fused_node_count = n_ops; + break; + } + if (ggml_cuda_should_fuse_mul_mat_vec_q(up_n)) { ggml_cuda_mul_mat_vec_q(*cuda_ctx, src0, src1, ids, cgraph->nodes[glu_idx], &fusion_data); fused_mul_mat_vec = true; fused_node_count = n_ops; break; } + + if (!with_bias && ggml_cuda_should_fuse_mul_mat_q(up_n, gate_n, glu, up_scale_n, gate_scale_n)) { + fusion_data.prec_src1 = ggml_prec_get_src(up_n, 1); + ggml_cuda_mul_mat_q(*cuda_ctx, src0, src1, ids, cgraph->nodes[glu_idx], &fusion_data); + fused_mul_mat_vec = true; + fused_node_count = n_ops; + break; + } } if (fused_mul_mat_vec) { @@ -4453,7 +4841,7 @@ static int ggml_cuda_try_fuse(ggml_backend_cuda_context * cuda_ctx, ggml_cgraph const ggml_tensor * src1 = up_n->src[1]; const ggml_tensor * ids = up_n->src[2]; - if (ggml_cuda_should_fuse_mul_mat_vec_f(up_n)) { + if (ggml_cuda_should_fuse_mul_mat_vec_f(up_n, gate_n)) { ggml_cuda_mm_fusion_args_host fusion_data{}; fusion_data.gate = gate_n->src[0]; fusion_data.x_bias = up_bias_tensor; @@ -4496,7 +4884,7 @@ static int ggml_cuda_try_fuse(ggml_backend_cuda_context * cuda_ctx, ggml_cgraph const ggml_tensor * src1 = up->src[1]; const ggml_tensor * ids = up->src[2]; - if (ggml_cuda_should_fuse_mul_mat_vec_f(up)) { + if (ggml_cuda_should_fuse_mul_mat_vec_f(up, gate)) { ggml_cuda_mm_fusion_args_host fusion_data{}; fusion_data.gate = gate->src[0]; fusion_data.glu_op = ggml_get_glu_op(glu); @@ -4525,6 +4913,7 @@ static int ggml_cuda_try_fuse(ggml_backend_cuda_context * cuda_ctx, ggml_cgraph fusion_data.gate = gate->src[0]; fusion_data.glu_op = ggml_get_glu_op(glu); fusion_data.glu_limit = ggml_get_op_params_f32(glu, 3); + fusion_data.prec_src1 = ggml_prec_get_src(up, 1); ggml_cuda_mul_mat_q(*cuda_ctx, src0, src1, ids, glu, &fusion_data); fused_mul_mat_vec = true; @@ -4586,9 +4975,9 @@ static int ggml_cuda_try_fuse(ggml_backend_cuda_context * cuda_ctx, ggml_cgraph const ggml_tensor * scale = nullptr; if (op == GGML_OP_MUL_MAT) { - scale = get_mul_mat_scale(scale_node, mm_node); + scale = ggml_cuda_get_mul_mat_scale(scale_node, mm_node); } else { - scale = get_mul_mat_id_scale(cgraph->nodes[i + 1], cgraph->nodes[i + 2], cgraph->nodes[i + 3], scale_node, mm_node); + scale = ggml_cuda_get_mul_mat_id_scale(cgraph->nodes[i + 1], cgraph->nodes[i + 2], cgraph->nodes[i + 3], scale_node, mm_node); } if (!scale) { continue; @@ -4613,6 +5002,36 @@ static int ggml_cuda_try_fuse(ggml_backend_cuda_context * cuda_ctx, ggml_cgraph fusion_data.x_bias = bias; fusion_data.x_scale = scale; + if (ggml_cuda_should_fuse_mul_mat_vec_f(mm_node)) { + ggml_cuda_mul_mat_vec_f(*cuda_ctx, src0, src1, ids, out_node, &fusion_data); + fused_mul_mat_vec = true; + fused_node_count = n_ops; + break; + } + + if (op == GGML_OP_MUL_MAT && !with_bias && src0->type == GGML_TYPE_F8_E4M3 && + ggml_get_op_params_i32(mm_node, 3) != GGML_PREC_F32 && + ggml_cuda_mul_mat_fp8(*cuda_ctx, src0, src1, scale_node, scale)) { + return 1; + } + + if (!with_bias && src0->type == GGML_TYPE_NVFP4 && src1->type == GGML_TYPE_F32 && mm_node->type == GGML_TYPE_F32) { + const int cc = ggml_cuda_info().devices[cuda_ctx->device].cc; + const bool is_id = op == GGML_OP_MUL_MAT_ID; + const int64_t ncols = is_id ? src1->ne[2] : src1->ne[1]; + const bool use_mmvq = is_id ? ncols <= get_mmvq_mmid_max_batch(src0->type, cc) : + ggml_cuda_should_use_mmvq(src0->type, cc, ncols); + const bool bad_padding_clear = ggml_backend_buffer_get_usage(src0->buffer) == GGML_BACKEND_BUFFER_USAGE_COMPUTE && + ggml_nbytes(src0) != ggml_backend_buffer_get_alloc_size(src0->buffer, src0) && + src0->view_src; + if (!use_mmvq && !bad_padding_clear && + ggml_cuda_should_use_mmq(src0->type, cc, ncols, /*n_experts =*/ is_id ? src0->ne[2] : 0)) { + fusion_data.prec_src1 = ggml_prec_get_src(mm_node, 1); + ggml_cuda_mul_mat_q(*cuda_ctx, src0, src1, ids, out_node, &fusion_data); + return n_ops - 1; + } + } + if (ggml_cuda_should_fuse_mul_mat_vec_q(mm_node)) { ggml_cuda_mul_mat_vec_q(*cuda_ctx, src0, src1, ids, out_node, &fusion_data); fused_mul_mat_vec = true; @@ -5012,7 +5431,9 @@ static enum ggml_status ggml_backend_cuda_graph_compute(ggml_backend_t backend, ggml_cuda_graph * graph = cuda_ctx->cuda_graph(graph_key); if (graph->is_enabled()) { - const bool graph_compatible = ggml_cuda_graph_check_compability(cgraph); + // graph->uid is set only for a compatible graph (see ggml_cuda_graph_update_required), so a reused graph needs no new check + const bool graph_reused = cgraph->uid != 0 && cgraph->uid == graph->uid; + const bool graph_compatible = graph_reused || ggml_cuda_graph_check_compability(cgraph); if (graph_compatible) { const bool properties_changed = ggml_cuda_graph_update_required(cuda_ctx, cgraph); @@ -5139,6 +5560,33 @@ static void ggml_backend_cuda_graph_optimize(ggml_backend_t backend, ggml_cgraph continue; } } + // Same with the NVFP4 scales: MUL_MAT(gate), MUL(gate scale), MUL_MAT(up), MUL(up scale), GLU. + if (i + 4 < cgraph->n_nodes && cgraph->nodes[i + 4]->op == GGML_OP_GLU) { + ggml_tensor * glu = cgraph->nodes[i + 4]; + ggml_tensor * gate_scale = cgraph->nodes[i + 1]; + ggml_tensor * up_scale = cgraph->nodes[i + 3]; + if (glu->src[0] == gate_scale && glu->src[1] == up_scale && + ggml_cuda_mul_mat_q_fusion_matches(cgraph->nodes[i + 2], cgraph->nodes[i], glu, cuda_ctx->device, up_scale, gate_scale) && + ggml_cuda_get_mul_mat_scale(gate_scale, cgraph->nodes[i]) && ggml_cuda_get_mul_mat_scale(up_scale, cgraph->nodes[i + 2])) { + add_alloc_deps(i, i + 4); + i += 4; + continue; + } + } + // Same with per-expert NVFP4 scales: MUL_MAT_ID, RESHAPE, REPEAT, GET_ROWS, MUL for gate and for up, GLU. + if (i + 10 < cgraph->n_nodes && cgraph->nodes[i + 10]->op == GGML_OP_GLU) { + ggml_tensor * glu = cgraph->nodes[i + 10]; + ggml_tensor * gate_scale = cgraph->nodes[i + 4]; + ggml_tensor * up_scale = cgraph->nodes[i + 9]; + if (glu->src[0] == gate_scale && glu->src[1] == up_scale && + ggml_cuda_mul_mat_q_fusion_matches(cgraph->nodes[i + 5], cgraph->nodes[i], glu, cuda_ctx->device, up_scale, gate_scale) && + ggml_cuda_get_mul_mat_id_scale(cgraph->nodes[i + 1], cgraph->nodes[i + 2], cgraph->nodes[i + 3], gate_scale, cgraph->nodes[i]) && + ggml_cuda_get_mul_mat_id_scale(cgraph->nodes[i + 6], cgraph->nodes[i + 7], cgraph->nodes[i + 8], up_scale, cgraph->nodes[i + 5])) { + add_alloc_deps(i, i + 10); + i += 10; + continue; + } + } ggml_cuda_moe_weighted_reduction_match match; if (ggml_cuda_match_moe_weighted_reduction(cgraph, i, match)) { @@ -5820,7 +6268,16 @@ static bool ggml_backend_cuda_device_supports_op(ggml_backend_dev_t dev, const g if (b->type == GGML_TYPE_F16 && a->type != GGML_TYPE_F16 && !ggml_cuda_op_mul_mat_use_fwht(op)) { return false; } - if (op->op == GGML_OP_MUL_MAT_ID && ggml_get_op_params_i32(op, 3) == GGML_PREC_F32) { + const int cc_dev = ggml_cuda_info().devices[dev_ctx->device].cc; + const bool f32_src1_ok = GGML_CUDA_CC_IS_NVIDIA(cc_dev) && cc_dev < GGML_CUDA_CC_AMPERE; + const enum ggml_prec prec_src1 = ggml_prec_get_src(op, 1); + const bool fp8_id_mmvf = op->op == GGML_OP_MUL_MAT_ID && + a->type == GGML_TYPE_F8_E4M3 && b->type == GGML_TYPE_F32 && op->type == GGML_TYPE_F32 && + ggml_cuda_should_use_mmvf(a, cc_dev, ggml_cuda_info().devices[dev_ctx->device].warp_size, b->ne[2]); + if (prec_src1 == GGML_PREC_F32 && !fp8_id_mmvf && (op->op == GGML_OP_MUL_MAT_ID || !f32_src1_ok || a->type != GGML_TYPE_F32)) { + return false; + } + if (prec_src1 == GGML_PREC_BF16 && (op->op == GGML_OP_MUL_MAT_ID || a->type != GGML_TYPE_F32)) { return false; } #ifdef GGML_USE_MUSA @@ -5848,6 +6305,7 @@ static bool ggml_backend_cuda_device_supports_op(ggml_backend_dev_t dev, const g case GGML_TYPE_Q8_0: case GGML_TYPE_MXFP4: case GGML_TYPE_NVFP4: + case GGML_TYPE_F8_E4M3: case GGML_TYPE_Q2_K: case GGML_TYPE_Q3_K: case GGML_TYPE_Q4_K: @@ -5885,6 +6343,7 @@ static bool ggml_backend_cuda_device_supports_op(ggml_backend_dev_t dev, const g case GGML_TYPE_Q5_0: case GGML_TYPE_Q5_1: case GGML_TYPE_Q8_0: + case GGML_TYPE_F8_E4M3: case GGML_TYPE_Q2_K: case GGML_TYPE_Q3_K: case GGML_TYPE_Q4_K: @@ -5914,7 +6373,7 @@ static bool ggml_backend_cuda_device_supports_op(ggml_backend_dev_t dev, const g { return ( ( - (op->type == GGML_TYPE_F32 || op->type == GGML_TYPE_F16 || op->type == GGML_TYPE_BF16 || + (op->type == GGML_TYPE_F32 || op->type == GGML_TYPE_F16 || op->type == GGML_TYPE_BF16 || op->type == GGML_TYPE_F8_E4M3 || op->type == GGML_TYPE_Q4_0 || op->type == GGML_TYPE_Q4_1 || op->type == GGML_TYPE_Q5_0 || op->type == GGML_TYPE_Q5_1 || op->type == GGML_TYPE_Q8_0 || op->type == GGML_TYPE_IQ4_NL) && op->src[0]->type == GGML_TYPE_F32 @@ -5935,6 +6394,16 @@ static bool ggml_backend_cuda_device_supports_op(ggml_backend_dev_t dev, const g { ggml_type src0_type = op->src[0]->type; ggml_type src1_type = op->src[1]->type; + if (src0_type == GGML_TYPE_F8_E4M3 && src1_type == GGML_TYPE_F8_E4M3) { + return true; + } + if ((src0_type == GGML_TYPE_F32 || src0_type == GGML_TYPE_F16 || src0_type == GGML_TYPE_BF16) && + src1_type == GGML_TYPE_F8_E4M3) { + return true; + } + if (src0_type == GGML_TYPE_F8_E4M3 && src1_type == GGML_TYPE_F32) { + return true; + } if ((src0_type == GGML_TYPE_F32 || src0_type == GGML_TYPE_BF16 || src0_type == GGML_TYPE_F16) && (src1_type == GGML_TYPE_F32 || src1_type == GGML_TYPE_BF16 || src1_type == GGML_TYPE_F16) ) { @@ -6066,7 +6535,6 @@ static bool ggml_backend_cuda_device_supports_op(ggml_backend_dev_t dev, const g case GGML_OP_PERMUTE: case GGML_OP_TRANSPOSE: case GGML_OP_ADD_ID: - case GGML_OP_ADD1: case GGML_OP_SQR: case GGML_OP_SQRT: case GGML_OP_SIN: @@ -6094,9 +6562,9 @@ static bool ggml_backend_cuda_device_supports_op(ggml_backend_dev_t dev, const g if (op->src[3]->ne[0] == 1) { // Mamba2 - // (kernel only supports (d_state == 96 || d_state == 128 || d_state == 256) && d_head % 16 == 0) + // (kernel only supports (d_state == 64 || d_state == 96 || d_state == 128 || d_state == 256) && d_head % 16 == 0) const int64_t d_state = op->src[0]->ne[0]; - return (d_state == 96 || d_state == 128 || d_state == 256) && op->src[0]->ne[1] % 16 == 0; + return (d_state == 64 || d_state == 96 || d_state == 128 || d_state == 256) && op->src[0]->ne[1] % 16 == 0; } else { if (K > 1) { return false; @@ -6113,8 +6581,6 @@ static bool ggml_backend_cuda_device_supports_op(ggml_backend_dev_t dev, const g } case GGML_OP_CONT: return true; - case GGML_OP_DIAG_MASK_INF: - return true; case GGML_OP_SOFT_MAX: return true; case GGML_OP_SOFT_MAX_BACK: { @@ -6352,12 +6818,21 @@ static ggml_backend_feature * ggml_backend_cuda_get_features(ggml_backend_reg_t { const auto & info = ggml_cuda_info(); + bool has_fp8 = false; + bool has_fp4 = false; for (int id = 0; id < info.device_count; ++id) { - if (blackwell_mma_available(info.devices[id].cc)) { - features.push_back({ "BLACKWELL_NATIVE_FP4", "1"}); + has_fp8 |= fp8_mma_hardware_available(info.devices[id].cc); + has_fp4 |= blackwell_mma_available(info.devices[id].cc); + if (has_fp8 && has_fp4) { break; } } + if (has_fp8) { + features.push_back({ "NATIVE_FP8", "1"}); + } + if (has_fp4) { + features.push_back({ "BLACKWELL_NATIVE_FP4", "1"}); + } } #undef _STRINGIFY @@ -6384,6 +6859,11 @@ static void * ggml_backend_cuda_reg_get_proc_address(ggml_backend_reg_t reg, con if (strcmp(name, "ggml_backend_comm_allreduce_tensor") == 0) { return (void *)ggml_backend_cuda_comm_allreduce_tensor; } +#ifdef GGML_USE_NCCL + if (strcmp(name, "ggml_backend_comm_init_rank") == 0) { + return (void *)ggml_backend_cuda_comm_init_rank; + } +#endif // GGML_USE_NCCL if (strcmp(name, "ggml_backend_register_host_buffer") == 0) { return (void *)ggml_backend_cuda_register_host_buffer; } diff --git a/ggml/src/ggml-cuda/graph.cuh b/ggml/src/ggml-cuda/graph.cuh new file mode 100644 index 000000000000..7b9c277898e1 --- /dev/null +++ b/ggml/src/ggml-cuda/graph.cuh @@ -0,0 +1,221 @@ +#pragma once + +// CUDA graph capture and concurrent stream state, only needed by ggml-cuda.cu + +#include "common.cuh" + +#include +#include +#include +#include + +#ifdef USE_CUDA_GRAPH +// cudaGraphExecUpdate keeps stale kernel cluster dims (cuBLAS sm90), re-instantiate when they change +// TODO: add a test, fails on H100 (driver 590.48.01, CUDA runtime 12.9.79, cuBLAS 12.9.1.4) for F16 MUL_MAT with +// ne01 1200 <-> 1040 (k=768, n=1024) between graph updates, see https://github.com/ggml-org/llama.cpp/pull/31284 +#if !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA) && CUDART_VERSION >= 11080 +#define USE_CUDA_GRAPH_CLUSTER_CHECK +#endif + +#ifdef USE_CUDA_GRAPH_CLUSTER_CHECK +// cluster dims of a clustered kernel node +struct ggml_cuda_graph_cluster { + size_t node; + uint32_t x, y, z; + + bool operator==(const ggml_cuda_graph_cluster & other) const { + return node == other.node && x == other.x && y == other.y && z == other.z; + } +}; +#endif + +struct ggml_cuda_graph_instance { + cudaGraphExec_t handle = nullptr; +#ifdef USE_CUDA_GRAPH_CLUSTER_CHECK + std::vector clusters; +#endif + + operator cudaGraphExec_t() const { return handle; } +}; +#endif // USE_CUDA_GRAPH + +struct ggml_cuda_graph { +#ifdef USE_CUDA_GRAPH + ~ggml_cuda_graph() { + for (cudaGraphExec_t instance : instances) { + CUDA_CHECK(cudaGraphExecDestroy(instance)); + } + } + // the ggml graph is captured as a series of CUDA graphs, so the GPU can run one chunk while the next is enqueued + std::vector instances; + size_t num_nodes = 0; + std::vector nodes; + bool disable_due_to_gpu_arch = false; + bool warmup_complete = false; + uint64_t uid = 0; + int64_t last_used_time = 0; + struct node_properties { + ggml_tensor node; + void * node_src_data_ptrs[GGML_MAX_SRC]; + int64_t node_src_ne[GGML_MAX_SRC][GGML_MAX_DIMS]; + size_t node_src_nb[GGML_MAX_SRC][GGML_MAX_DIMS]; + }; + std::vector node_props; + + bool is_enabled() const { + static const bool disable_cuda_graphs_due_to_env = (getenv("GGML_CUDA_DISABLE_GRAPHS") != nullptr); + return !(disable_due_to_gpu_arch || disable_cuda_graphs_due_to_env || ggml_cuda_pool_debug_enabled()); + } +#endif +}; + +struct ggml_cuda_concurrent_event { + std::vector join_events; + cudaEvent_t fork_event = nullptr; + + int n_streams = 0; + std::unordered_map stream_mapping; + + // Original order of nodes in this concurrent region (before interleaving) + // Used to restore grouping for fusion within streams + std::vector original_order; + + const ggml_tensor * join_node; + + ggml_cuda_concurrent_event() = default; + + ggml_cuda_concurrent_event(const ggml_cuda_concurrent_event &) = delete; + ggml_cuda_concurrent_event & operator=(const ggml_cuda_concurrent_event &) = delete; + + explicit ggml_cuda_concurrent_event(int n_streams) : n_streams(n_streams) { + join_events.resize(n_streams); + + for (size_t i = 0; i < join_events.size(); ++i) { + CUDA_CHECK(cudaEventCreateWithFlags(&join_events[i], cudaEventDisableTiming)); + } + + CUDA_CHECK(cudaEventCreateWithFlags(&fork_event, cudaEventDisableTiming)); + } + + ggml_cuda_concurrent_event(ggml_cuda_concurrent_event && other) noexcept + : join_events(std::move(other.join_events)) + , fork_event(other.fork_event) + , n_streams(other.n_streams) + , stream_mapping(std::move(other.stream_mapping)) + , original_order(std::move(other.original_order)) + , join_node(other.join_node) { + other.fork_event = nullptr; + } + + // 1. check if any branches write to overlapping memory ranges (except the join node) + // 2. check whether all srcs are either within the branch or outside the nodes covered by ggml_cuda_concurrent_event + // we assume all nodes have the same buffer + bool is_valid() const { + std::vector>> write_ranges; + write_ranges.resize(n_streams); + + // get join_node's memory range to exclude from overlap checking. + // multiple nodes can use join_node's buffer; we synchronize on the join node. + const ggml_tensor * join_t = join_node->view_src ? join_node->view_src : join_node; + const int64_t join_start = (int64_t) join_t->data; + const int64_t join_end = join_start + ggml_nbytes(join_t); + + for (const auto & [tensor, stream] : stream_mapping) { + const ggml_tensor * t = tensor->view_src ? tensor->view_src : tensor; + const int64_t t_start = (int64_t) t->data; + const int64_t t_end = t_start + ggml_nbytes(t); + + // skip tensors that overlap with join_node's buffer. + if ((t_start <= join_start && join_start < t_end) || (join_start <= t_start && t_start < join_end)) { + continue; + } + + // concurrent streams begin from 1 + write_ranges[stream - 1].emplace_back(t_start, t_end); + } + + for (int i = 0; i < n_streams; ++i) { + // sorts first by start then by end of write range + std::sort(write_ranges[i].begin(), write_ranges[i].end()); + } + + bool writes_overlap = false; + bool dependent_srcs = false; + for (const auto & [tensor, stream] : stream_mapping) { + const ggml_tensor * t = tensor->view_src ? tensor->view_src : tensor; + const int64_t t_start = (int64_t) t->data; + const int64_t t_end = t_start + ggml_nbytes(t); + + // skip tensors that overlap with join_node's buffer + if ((t_start <= join_start && join_start < t_end) || (join_start <= t_start && t_start < join_end)) { + continue; + } + + // check if this buffer's write data overlaps with another stream's + std::pair data_range = std::make_pair(t_start, t_end); + for (int i = 0; i < n_streams; ++i) { + if (i == stream - 1) { + continue; + } + auto it = std::lower_bound(write_ranges[i].begin(), write_ranges[i].end(), data_range); + + if (it != write_ranges[i].end()) { + const std::pair & other = *it; + + // std::lower_bound returns the first element where other >= data_range (lexicographically). + // This guarantees other.first >= data_range.first. + // Therefore, overlap occurs iff other.first < data_range.second + // (i.e., the other range starts before this range ends). + if (other.first < data_range.second) { + GGML_LOG_DEBUG("Writes overlap for %s", tensor->name); + writes_overlap = true; + break; + } + } + } + + //check if all srcs are either in branch or don't have a branch + for (int i = 0; i < GGML_MAX_SRC; ++i) { + if (!tensor->src[i]) { + continue; + } + + auto it = stream_mapping.find(tensor->src[i]); + + if (it == stream_mapping.end()) { + continue; + } + + if (it->second != stream) { + dependent_srcs = true; + break; + } + } + + if (dependent_srcs || writes_overlap) { + break; + } + } + + return !writes_overlap && !dependent_srcs; + } + + ~ggml_cuda_concurrent_event() { + if (fork_event != nullptr) { + CUDA_CHECK(cudaEventDestroy(fork_event)); + } + for (cudaEvent_t e : join_events) { + if (e != nullptr) { + CUDA_CHECK(cudaEventDestroy(e)); + } + } + } +}; + +struct ggml_cuda_stream_context { + std::unordered_map concurrent_events; + + void reset() { + concurrent_events.clear(); + } +}; diff --git a/ggml/src/ggml-cuda/lightning-indexer.cu b/ggml/src/ggml-cuda/lightning-indexer.cu index 429d2e200c84..b7939243b74b 100644 --- a/ggml/src/ggml-cuda/lightning-indexer.cu +++ b/ggml/src/ggml-cuda/lightning-indexer.cu @@ -2,21 +2,22 @@ #include "lightning-indexer.cuh" #include "fattn-common.cuh" #include "convert.cuh" +#include "mma.cuh" -#if !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA) -#if defined(TURING_MMA_AVAILABLE) +// tokens scored per block by the tile kernel +#define LIGHTNING_INDEXER_TILE_TOKENS 8 -typedef union { - int2 i2; - half2 h2[2]; -} half4; +// heads whose queries the tile kernel stages per pass, MUSA arch 21 caps static shared memory +// at 28 KB and the queries of four heads do not fit there next to the key tile +#if defined(GGML_USE_MUSA) && defined(__MUSA_ARCH__) && __MUSA_ARCH__ < 220 +#define LIGHTNING_INDEXER_TILE_HEADS_PER_PASS 2 +#else +#define LIGHTNING_INDEXER_TILE_HEADS_PER_PASS 4 +#endif -// TODO add support for AMD cards via rocWMMA -#include -namespace wmma = nvcuda::wmma; template -static __global__ void lightning_indexer_kernel_wmma( +static __global__ void lightning_indexer_kernel_mma( const float * Q, const char * K, const float * W, const half * M, float * dst, int64_t n_stream, int64_t n_batch, int64_t n_kv, size_t nb1, size_t nb2, size_t nb3, @@ -26,227 +27,161 @@ static __global__ void lightning_indexer_kernel_wmma( size_t nbm1, size_t nbm2, size_t nbm3, int64_t nem3 ) { - - constexpr int THREADS_PER_BLOCK = WARPS_PER_BLOCK * WARP_SIZE; - constexpr int HEADS_PER_INNER_LOOP = 8; - constexpr int K_EMBD_PER_INNER_LOOP = 16; + constexpr int MMA_DIM = 16; + constexpr int WAVE_SIZE = ggml_cuda_get_physical_warp_size(); + constexpr int THREADS_PER_BLOCK = WARPS_PER_BLOCK * WAVE_SIZE; constexpr int N_EMBD_PADDED = N_EMBD + 8; + constexpr int K_VECS_PADDED = K_VECS_PER_BLOCK + 8; + constexpr int HEAD_CHUNK = MMA_DIM; + constexpr int N_HEAD_CHUNKS = N_HEAD / HEAD_CHUNK; + constexpr int N_KV_TILES = K_VECS_PER_BLOCK / MMA_DIM; + constexpr int N_EMBD_TILES = N_EMBD / MMA_DIM; + + static_assert(N_HEAD % MMA_DIM == 0, "N_HEAD must be a multiple of 16"); + static_assert(N_EMBD % MMA_DIM == 0, "N_EMBD must be a multiple of 16"); + static_assert(K_VECS_PER_BLOCK % MMA_DIM == 0, "K_VECS_PER_BLOCK must be a multiple of 16"); + +#if defined(AMD_WMMA_AVAILABLE) || defined(AMD_MFMA_AVAILABLE) + using tile_qk = ggml_cuda_mma::tile; +#else + using tile_qk = ggml_cuda_mma::tile; +#endif const int i_batch = blockIdx.y; const int i_stream = blockIdx.z; const int i_warp = threadIdx.y; const int i_lane = threadIdx.x; - const int tid = i_warp * WARP_SIZE + i_lane; + const int tid = i_warp * WAVE_SIZE + i_lane; - // each block processes K_VECS_PER_BLOCK K vectors const int start_kv = blockIdx.x * K_VECS_PER_BLOCK; const char * q_base = (const char *) Q + i_batch*nbq2 + i_stream*nbq3; const float * w_base = (const float *) ((const char *) W + i_batch*nbw1 + i_stream*nbw3); - // phase 1 - load weights and first Q tile to shared memory - __shared__ float w_shared[N_HEAD]; - __shared__ int2 q_shared_h[HEADS_PER_INNER_LOOP][N_EMBD_PADDED / 4]; + __shared__ half q_shared[2][HEAD_CHUNK][N_EMBD_PADDED]; + __shared__ half k_shared[K_VECS_PER_BLOCK][N_EMBD_PADDED]; + __shared__ float qk_chunk[HEAD_CHUNK][K_VECS_PADDED]; + __shared__ float score_shared[K_VECS_PER_BLOCK]; - if (tid < N_HEAD) { - w_shared[tid] = w_base[tid]; + for (int i = tid; i < N_HEAD; i += THREADS_PER_BLOCK) { + w_shared[i] = w_base[i]; } - - // total number of half4 elements in HEADS_PER_INNER_LOOP x N_EMBD Q tile - constexpr int N_Q_TILE = HEADS_PER_INNER_LOOP * (N_EMBD / 4); - // number of registers needed in each thread to store Q tile in thread block - constexpr int N_Q_NEXT = (N_Q_TILE + THREADS_PER_BLOCK - 1) / THREADS_PER_BLOCK; - -#pragma unroll - for (int i_q = tid; i_q < N_Q_TILE; i_q += THREADS_PER_BLOCK) { - const int i_head = i_q / (N_EMBD / 4); - const int i_embd = i_q % (N_EMBD / 4); - const float4 q = *(const float4 *) (q_base + i_head*nbq1 + i_embd*sizeof(float4)); - half4 q_packed; - q_packed.h2[0] = __float22half2_rn(make_float2(q.x, q.y)); - q_packed.h2[1] = __float22half2_rn(make_float2(q.z, q.w)); - q_shared_h[i_head][i_embd] = q_packed.i2; + for (int i = tid; i < K_VECS_PER_BLOCK; i += THREADS_PER_BLOCK) { + score_shared[i] = 0.0f; } - // phase 2 - load (and dequantize if needed) K to shared mem - - __shared__ half2 k_shared_h[K_VECS_PER_BLOCK][N_EMBD_PADDED / 4][2]; - - constexpr int n_k = K_VECS_PER_BLOCK * (N_EMBD / 4); - + constexpr int N_K_VEC = K_VECS_PER_BLOCK * (N_EMBD / 4); if constexpr (TYPE_K == GGML_TYPE_F16) { -#pragma unroll - for (int i_k = tid; i_k < n_k; i_k += THREADS_PER_BLOCK) { - const int i_k_vec = i_k / (N_EMBD / 4); - const int i_embd = i_k % (N_EMBD / 4); - const int i_kv = start_kv + i_k_vec; + for (int i = tid; i < N_K_VEC; i += THREADS_PER_BLOCK) { + const int i_k_vec = i / (N_EMBD / 4); + const int i_embd4 = i % (N_EMBD / 4); + const int i_kv = start_kv + i_k_vec; if (i_kv < n_kv) { - const int2 * k_base = (const int2 *) ((const char *) K + i_kv*nbk2 + i_stream*nbk3); - *(int2*) &k_shared_h[i_k_vec][i_embd] = k_base[i_embd]; + const int2 * k_ptr = (const int2 *) ((const char *) K + i_kv*nbk2 + i_stream*nbk3); + *(int2 *) &k_shared[i_k_vec][i_embd4*4] = k_ptr[i_embd4]; } else { - *(int2*) &k_shared_h[i_k_vec][i_embd] = make_int2(0, 0); + *(int2 *) &k_shared[i_k_vec][i_embd4*4] = make_int2(0, 0); } } } else { constexpr dequantize_V_t dequantize_k = get_dequantize_V(); -#pragma unroll - for (int i_k = tid; i_k < n_k; i_k += THREADS_PER_BLOCK) { - const int i_k_vec = i_k / (N_EMBD / 4); - const int i_embd = i_k % (N_EMBD / 4); - const int i_kv = start_kv + i_k_vec; + for (int i = tid; i < N_K_VEC; i += THREADS_PER_BLOCK) { + const int i_k_vec = i / (N_EMBD / 4); + const int i_embd4 = i % (N_EMBD / 4); + const int i_kv = start_kv + i_k_vec; if (i_kv < n_kv) { - const void * k_base = (const void *) ((const char *) K + i_kv*nbk2 + i_stream*nbk3); - dequantize_k(k_base, &k_shared_h[i_k_vec][i_embd][0], i_embd * 4); + const void * k_ptr = (const void *) ((const char *) K + i_kv*nbk2 + i_stream*nbk3); + dequantize_k(k_ptr, &k_shared[i_k_vec][i_embd4*4], i_embd4*4); } else { - *(int2*) &k_shared_h[i_k_vec][i_embd] = make_int2(0, 0); + *(int2 *) &k_shared[i_k_vec][i_embd4*4] = make_int2(0, 0); } } } - __syncthreads(); - - // phase 3 - calculate lightning indexer scores - - __shared__ float qk_shared[WARPS_PER_BLOCK][HEADS_PER_INNER_LOOP][K_VECS_PER_BLOCK]; - - // load K fragment - wmma::fragment frag_k; - wmma::load_matrix_sync(frag_k, (half*) &k_shared_h[0][i_warp * K_EMBD_PER_INNER_LOOP / 4], N_EMBD_PADDED); + constexpr int N_Q_CHUNK_VEC = HEAD_CHUNK * (N_EMBD / 4); + constexpr int N_Q_NEXT = (N_Q_CHUNK_VEC + THREADS_PER_BLOCK - 1) / THREADS_PER_BLOCK; - float score_k = 0.0f; + for (int i = tid; i < N_Q_CHUNK_VEC; i += THREADS_PER_BLOCK) { + const int i_head = i / (N_EMBD / 4); + const int i_embd4 = i % (N_EMBD / 4); + const float4 q = *(const float4 *) (q_base + i_head*nbq1 + i_embd4*sizeof(float4)); + const half2 qh[2] = {make_half2(q.x, q.y), make_half2(q.z, q.w)}; + ggml_cuda_memcpy_1(&q_shared[0][i_head][i_embd4*4], qh); + } - for (int i_head_0 = 0; i_head_0 < N_HEAD; i_head_0 += HEADS_PER_INNER_LOOP) { - const int i_head_next = i_head_0 + HEADS_PER_INNER_LOOP; + __syncthreads(); - // we don't use accumulator for anything, fill it with zeros - wmma::fragment frag_acc; - wmma::fill_fragment(frag_acc, 0.0f); + constexpr ggml_cuda_mma::data_layout input_layout = ggml_cuda_mma::get_input_data_layout(); - // load Q fragment - wmma::fragment frag_q; - wmma::load_matrix_sync(frag_q, (half*) &q_shared_h[0][i_warp * K_EMBD_PER_INNER_LOOP / 4], N_EMBD_PADDED); + for (int hc = 0; hc < N_HEAD_CHUNKS; ++hc) { + const int buf = hc & 1; + const int nbuf = (hc + 1) & 1; + const bool has_next = (hc + 1) < N_HEAD_CHUNKS; - // preload next Q tile to registers during matrix multiplication float4 q_next[N_Q_NEXT]; - - if (i_head_next < N_HEAD) { + if (has_next) { #pragma unroll - for (int i_q = tid, i_q_next = 0; i_q < N_Q_TILE; i_q += THREADS_PER_BLOCK) { - const int i_head = i_head_next + i_q / (N_EMBD / 4); - const int i_embd = i_q % (N_EMBD / 4); - q_next[i_q_next++] = *(const float4 *) (q_base + i_head*nbq1 + i_embd*sizeof(float4)); + for (int i = tid, j = 0; i < N_Q_CHUNK_VEC; i += THREADS_PER_BLOCK, ++j) { + const int i_head = (hc + 1) * HEAD_CHUNK + i / (N_EMBD / 4); + const int i_embd4 = i % (N_EMBD / 4); + q_next[j] = *(const float4 *) (q_base + i_head*nbq1 + i_embd4*sizeof(float4)); } } - // perform matrix multiplication - wmma::mma_sync(frag_acc, frag_q, frag_k, frag_acc); - wmma::store_matrix_sync((float*) &qk_shared[i_warp][0][0], frag_acc, K_VECS_PER_BLOCK, wmma::mem_row_major); - - // make sure all threads finished using q_shared_h so we can store next tile - __syncthreads(); - - // write preloaded Q tile to shared memory - if (i_head_next < N_HEAD) { + for (int kt = i_warp; kt < N_KV_TILES; kt += WARPS_PER_BLOCK) { + tile_qk Dqk; #pragma unroll - for (int i_q = tid, i_q_next = 0; i_q < N_Q_TILE; i_q += THREADS_PER_BLOCK) { - const int i_head = i_q / (N_EMBD / 4); - const int i_embd = i_q % (N_EMBD / 4); - half4 q_packed; - q_packed.h2[0] = __float22half2_rn(make_float2(q_next[i_q_next].x, q_next[i_q_next].y)); - q_packed.h2[1] = __float22half2_rn(make_float2(q_next[i_q_next].z, q_next[i_q_next].w)); - q_shared_h[i_head][i_embd] = q_packed.i2; - ++i_q_next; + for (int et = 0; et < N_EMBD_TILES; ++et) { + ggml_cuda_mma::tile Aq; + ggml_cuda_mma::tile Bk; + ggml_cuda_mma::load_generic(Aq, (const half2 *) &q_shared[buf][0][et*MMA_DIM], N_EMBD_PADDED/2); + ggml_cuda_mma::load_generic(Bk, (const half2 *) &k_shared[kt*MMA_DIM][et*MMA_DIM], N_EMBD_PADDED/2); + ggml_cuda_mma::mma(Dqk, Aq, Bk); } - } - - // accumulate QK multiplication results from all block warps - // (there are 256 threads in block and 256 matmul outputs) - // TODO it will break if WARP_SIZE is not 32 - const int h = tid / K_VECS_PER_BLOCK; - const int k = tid % K_VECS_PER_BLOCK; - const float w_val = w_shared[i_head_0 + h]; - - float sum = 0.0f; #pragma unroll - for (int w = 0; w < WARPS_PER_BLOCK; ++w) { - sum += qk_shared[w][h][k]; + for (int l = 0; l < Dqk.ne; ++l) { + const int head_local = Dqk.get_i(l); + const int kv_local = kt*MMA_DIM + Dqk.get_j(l); + qk_chunk[head_local][kv_local] = Dqk.x[l]; + } } - // ReLU, weight - sum = sum > 0.0f ? sum : 0.0f; - sum *= w_val; - - // wait until qk_shared[0] is no longer used __syncthreads(); - // reuse qk_shared[0] for storing partial results - qk_shared[0][h][k] = sum; - - // wait until all threads write their results - __syncthreads(); + if (has_next) { +#pragma unroll + for (int i = tid, j = 0; i < N_Q_CHUNK_VEC; i += THREADS_PER_BLOCK, ++j) { + const int i_head = i / (N_EMBD / 4); + const int i_embd4 = i % (N_EMBD / 4); + const half2 qh[2] = {make_half2(q_next[j].x, q_next[j].y), make_half2(q_next[j].z, q_next[j].w)}; + ggml_cuda_memcpy_1(&q_shared[nbuf][i_head][i_embd4*4], qh); + } + } - // accumulate result over heads - if (tid < K_VECS_PER_BLOCK) { + for (int kv = tid; kv < K_VECS_PER_BLOCK; kv += THREADS_PER_BLOCK) { + float s = 0.0f; #pragma unroll - for (int i_head = 0; i_head < HEADS_PER_INNER_LOOP; ++i_head) { - score_k += qk_shared[0][i_head][tid]; + for (int h = 0; h < HEAD_CHUNK; ++h) { + const float qk = qk_chunk[h][kv]; + s += (qk > 0.0f ? qk : 0.0f) * w_shared[hc*HEAD_CHUNK + h]; } + score_shared[kv] += s; } - // make sure all threads finished using qk_shared __syncthreads(); } - // phase 4 - store output to VRAM - - if (tid < K_VECS_PER_BLOCK) { - const int i_kv = start_kv + tid; + for (int i_kv_local = tid; i_kv_local < K_VECS_PER_BLOCK; i_kv_local += THREADS_PER_BLOCK) { + const int i_kv = start_kv + i_kv_local; if (i_kv < n_kv) { - const half * m_base = (const half *) ((const char *) M + i_batch*nbm1 + (i_stream%nem3)*nbm3); - float * dst_base = (float *) ((char *) dst + i_batch*nb1 + i_stream*nb3); - dst_base[i_kv] = score_k + __half2float(m_base[i_kv]); + const half * m_base = (const half *) ((const char *) M + i_batch*nbm1 + (i_stream%nem3)*nbm3); + float * dst_base = (float *) ((char *) dst + i_batch*nb1 + i_stream*nb3); + dst_base[i_kv] = score_shared[i_kv_local] + __half2float(m_base[i_kv]); } } } -#else // defined(TURING_MMA_AVAILABLE) - -template -static __global__ void lightning_indexer_kernel_wmma( - const float * Q, const char * K, const float * W, const half * M, float * dst, - int64_t n_stream, int64_t n_batch, int64_t n_kv, - size_t nb1, size_t nb2, size_t nb3, - size_t nbq1, size_t nbq2, size_t nbq3, - size_t nbk1, size_t nbk2, size_t nbk3, - size_t nbw1, size_t nbw2, size_t nbw3, - size_t nbm1, size_t nbm2, size_t nbm3, - int64_t nem3 - ) { - GGML_UNUSED_VARS(Q, K, W, M, dst, - n_stream, n_batch, n_kv, - nb1, nb2, nb3, - nbq1, nbq2, nbq3, - nbk1, nbk2, nbk3, - nbw1, nbw2, nbw3, - nem3); - NO_DEVICE_CODE; -} - -#endif // defined(TURING_MMA_AVAILABLE) -#endif // !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA) - -// tokens scored per block by the tile kernel -#define LIGHTNING_INDEXER_TILE_TOKENS 8 - -// heads whose queries the tile kernel stages per pass, MUSA arch 21 caps static shared memory -// at 28 KB and the queries of four heads do not fit there next to the key tile -#if defined(GGML_USE_MUSA) && defined(__MUSA_ARCH__) && __MUSA_ARCH__ < 220 -#define LIGHTNING_INDEXER_TILE_HEADS_PER_PASS 2 -#else -#define LIGHTNING_INDEXER_TILE_HEADS_PER_PASS 4 -#endif - // TODO there is one ugly assumption used in this kernel - that WARP_SIZE is equal to 32 // thanks to that one warp operating on float4 processes whole indexer K/Q vectors // 32 * 4 = 128 (N_EMBD) @@ -616,27 +551,22 @@ void ggml_cuda_lightning_indexer(ggml_backend_cuda_context & ctx, ggml_tensor * const int cc = ggml_cuda_info().devices[device].cc; if (n_embd == 128 && n_head == 64) { -#if !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA) - if (GGML_CUDA_CC_IS_NVIDIA(cc) && turing_mma_available(cc) && k->type != GGML_TYPE_F32 && k->type != GGML_TYPE_BF16) { - // use wmma kernel + if ((turing_mma_available(cc) || amd_mfma_available(cc) || amd_wmma_available(cc)) && k->type != GGML_TYPE_F32 && k->type != GGML_TYPE_BF16) { constexpr int K_VECS_PER_BLOCK = 32; - constexpr int WARPS_PER_BLOCK = 8; + constexpr int WARPS_PER_BLOCK = 4; - dim3 block(32, WARPS_PER_BLOCK); + dim3 block(ggml_cuda_info().devices[device].warp_size, WARPS_PER_BLOCK); int num_kv_blocks = (n_kv + (K_VECS_PER_BLOCK) - 1) / (K_VECS_PER_BLOCK); dim3 grid(num_kv_blocks, n_batch, n_stream); - LIGHTNING_INDEXER_CASE(lightning_indexer_kernel_wmma, 128, 64, k, GGML_TYPE_F16) - LIGHTNING_INDEXER_CASE(lightning_indexer_kernel_wmma, 128, 64, k, GGML_TYPE_Q4_0) - LIGHTNING_INDEXER_CASE(lightning_indexer_kernel_wmma, 128, 64, k, GGML_TYPE_Q4_1) - LIGHTNING_INDEXER_CASE(lightning_indexer_kernel_wmma, 128, 64, k, GGML_TYPE_Q5_0) - LIGHTNING_INDEXER_CASE(lightning_indexer_kernel_wmma, 128, 64, k, GGML_TYPE_Q5_1) - LIGHTNING_INDEXER_CASE(lightning_indexer_kernel_wmma, 128, 64, k, GGML_TYPE_Q8_0) + LIGHTNING_INDEXER_CASE(lightning_indexer_kernel_mma, 128, 64, k, GGML_TYPE_F16) + LIGHTNING_INDEXER_CASE(lightning_indexer_kernel_mma, 128, 64, k, GGML_TYPE_Q4_0) + LIGHTNING_INDEXER_CASE(lightning_indexer_kernel_mma, 128, 64, k, GGML_TYPE_Q4_1) + LIGHTNING_INDEXER_CASE(lightning_indexer_kernel_mma, 128, 64, k, GGML_TYPE_Q5_0) + LIGHTNING_INDEXER_CASE(lightning_indexer_kernel_mma, 128, 64, k, GGML_TYPE_Q5_1) + LIGHTNING_INDEXER_CASE(lightning_indexer_kernel_mma, 128, 64, k, GGML_TYPE_Q8_0) GGML_ABORT("fatal error"); } else { -#else // !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA) - { -#endif // !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA) // use vector kernel constexpr int K_VECS_PER_WARP = 8; constexpr int WARPS_PER_BLOCK = 8; @@ -657,27 +587,22 @@ void ggml_cuda_lightning_indexer(ggml_backend_cuda_context & ctx, ggml_tensor * GGML_ABORT("fatal error"); } } else if (n_embd == 128 && n_head == 32) { -#if !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA) - if (GGML_CUDA_CC_IS_NVIDIA(cc) && turing_mma_available(cc) && k->type != GGML_TYPE_F32 && k->type != GGML_TYPE_BF16) { - // use wmma kernel + if ((turing_mma_available(cc) || amd_mfma_available(cc) || amd_wmma_available(cc)) && k->type != GGML_TYPE_F32 && k->type != GGML_TYPE_BF16) { constexpr int K_VECS_PER_BLOCK = 32; - constexpr int WARPS_PER_BLOCK = 8; + constexpr int WARPS_PER_BLOCK = 4; - dim3 block(32, WARPS_PER_BLOCK); + dim3 block(ggml_cuda_info().devices[device].warp_size, WARPS_PER_BLOCK); int num_kv_blocks = (n_kv + (K_VECS_PER_BLOCK) - 1) / (K_VECS_PER_BLOCK); dim3 grid(num_kv_blocks, n_batch, n_stream); - LIGHTNING_INDEXER_CASE(lightning_indexer_kernel_wmma, 128, 32, k, GGML_TYPE_F16) - LIGHTNING_INDEXER_CASE(lightning_indexer_kernel_wmma, 128, 32, k, GGML_TYPE_Q4_0) - LIGHTNING_INDEXER_CASE(lightning_indexer_kernel_wmma, 128, 32, k, GGML_TYPE_Q4_1) - LIGHTNING_INDEXER_CASE(lightning_indexer_kernel_wmma, 128, 32, k, GGML_TYPE_Q5_0) - LIGHTNING_INDEXER_CASE(lightning_indexer_kernel_wmma, 128, 32, k, GGML_TYPE_Q5_1) - LIGHTNING_INDEXER_CASE(lightning_indexer_kernel_wmma, 128, 32, k, GGML_TYPE_Q8_0) + LIGHTNING_INDEXER_CASE(lightning_indexer_kernel_mma, 128, 32, k, GGML_TYPE_F16) + LIGHTNING_INDEXER_CASE(lightning_indexer_kernel_mma, 128, 32, k, GGML_TYPE_Q4_0) + LIGHTNING_INDEXER_CASE(lightning_indexer_kernel_mma, 128, 32, k, GGML_TYPE_Q4_1) + LIGHTNING_INDEXER_CASE(lightning_indexer_kernel_mma, 128, 32, k, GGML_TYPE_Q5_0) + LIGHTNING_INDEXER_CASE(lightning_indexer_kernel_mma, 128, 32, k, GGML_TYPE_Q5_1) + LIGHTNING_INDEXER_CASE(lightning_indexer_kernel_mma, 128, 32, k, GGML_TYPE_Q8_0) GGML_ABORT("fatal error"); } else { -#else // !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA) - { -#endif // !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA) // use vector kernel constexpr int K_VECS_PER_WARP = 8; constexpr int WARPS_PER_BLOCK = 8; diff --git a/ggml/src/ggml-cuda/mmf.cu b/ggml/src/ggml-cuda/mmf.cu index 646a5899c803..040d390e0cc5 100644 --- a/ggml/src/ggml-cuda/mmf.cu +++ b/ggml/src/ggml-cuda/mmf.cu @@ -50,7 +50,8 @@ void ggml_cuda_mul_mat_f(ggml_backend_cuda_context & ctx, const ggml_tensor * sr mmf_ids_data * ids_info_ptr = nullptr; ggml_cuda_pool_alloc ids_src_compact_dev; ggml_cuda_pool_alloc ids_dst_compact_dev; - ggml_cuda_pool_alloc expert_bounds_dev; + + ggml_cuda_pool_alloc tile_spec_dev; // For MUL_MAT_ID the memory layout is different than for MUL_MAT: const int64_t ncols_dst = ids ? ne2 : ne1; @@ -74,25 +75,28 @@ void ggml_cuda_mul_mat_f(ggml_backend_cuda_context & ctx, const ggml_tensor * sr const int64_t n_experts = ne02; const int64_t n_tokens = ne12; const int64_t ne_get_rows = n_tokens * n_expert_used; + const int64_t max_tiles = (ne_get_rows + 16 - 1) / 16 + ne02; ids_src_compact_dev.alloc(ctx.pool(), ne_get_rows); ids_dst_compact_dev.alloc(ctx.pool(), ne_get_rows); - expert_bounds_dev.alloc(ctx.pool(), n_experts + 1); + tile_spec_dev.alloc(ctx.pool(), max_tiles); const int si1 = static_cast(ids_s1); const int sis1 = static_cast(src1->nb[2] / src1->nb[1]); GGML_ASSERT(sis1 > 0); - ggml_cuda_launch_mm_ids_helper(ids_d, ids_src_compact_dev.get(), ids_dst_compact_dev.get(), expert_bounds_dev.get(), - static_cast(n_experts), static_cast(n_tokens), static_cast(n_expert_used), static_cast(ne11), si1, sis1, /*write_inverse =*/ false, ctx.stream()); + ggml_cuda_launch_mm_ids_helper(ids_d, ids_src_compact_dev.get(), ids_dst_compact_dev.get(), tile_spec_dev.get(), + static_cast(n_experts), static_cast(n_tokens), static_cast(n_expert_used), static_cast(ne11), 16, + si1, sis1, /*write_inverse =*/ false, ctx.stream()); CUDA_CHECK(cudaGetLastError()); - ids_info.ids_src_compact = ids_src_compact_dev.get(); - ids_info.ids_dst_compact = ids_dst_compact_dev.get(); - ids_info.expert_bounds_dev = expert_bounds_dev.get(); - ids_info.n_experts = static_cast(n_experts); - ids_info.sis1 = sis1; + ids_info.ids_src_compact = ids_src_compact_dev.get(); + ids_info.ids_dst_compact = ids_dst_compact_dev.get(); + ids_info.tile_spec_dev = tile_spec_dev.get(); + ids_info.n_experts = static_cast(n_experts); + ids_info.max_tiles = static_cast(max_tiles); + ids_info.sis1 = sis1; ids_info_ptr = &ids_info; } diff --git a/ggml/src/ggml-cuda/mmf.cuh b/ggml/src/ggml-cuda/mmf.cuh index 429c8874ce89..c49f6c47ad1d 100644 --- a/ggml/src/ggml-cuda/mmf.cuh +++ b/ggml/src/ggml-cuda/mmf.cuh @@ -1,7 +1,8 @@ #pragma once -#include "mma.cuh" #include "common.cuh" +#include "mma.cuh" +#include "mmid.cuh" #include "convert.cuh" using namespace ggml_cuda_mma; @@ -36,8 +37,9 @@ static constexpr __device__ int mmf_get_padding() { struct mmf_ids_data { const int32_t * ids_src_compact = nullptr; const int32_t * ids_dst_compact = nullptr; - const int32_t * expert_bounds_dev = nullptr; + const ggml_cuda_mm_id_tile_spec * tile_spec_dev = nullptr; int n_experts = 0; + int max_tiles = 0; int sis1 = 0; }; @@ -308,7 +310,7 @@ __launch_bounds__(ggml_cuda_get_physical_warp_size()*nwarps, 1) static __global__ void mul_mat_f_ids( const T * __restrict__ x, const float * __restrict__ y, const int32_t * __restrict__ ids_src_compact, const int32_t * __restrict__ ids_dst_compact, - const int32_t * __restrict__ expert_bounds, float * __restrict__ dst, + const ggml_cuda_mm_id_tile_spec * __restrict__ tile_spec, float * __restrict__ dst, const int ncols, const int ncols_dst_total, const int nchannels_dst, const int stride_row, const int stride_col_y, const int stride_col_dst, const int channel_ratio, const int stride_channel_x, const int stride_channel_y, const int stride_channel_dst, const int sample_ratio, const int stride_sample_x, const int stride_sample_y, const int stride_sample_dst, @@ -349,21 +351,20 @@ static __global__ void mul_mat_f_ids( constexpr int ntA = rows_per_block / tile_A::I; constexpr int ntB = (cols_per_block + tile_B::I - 1) / tile_B::I; - const int row0 = blockIdx.x * rows_per_block; + const int row0 = blockIdx.x * rows_per_block; + const int tile_idx = blockIdx.y; - const int expert_idx = blockIdx.y; - const int expert_start = expert_bounds[expert_idx]; - const int expert_end = expert_bounds[expert_idx + 1]; + const ggml_cuda_mm_id_tile_spec ts = tile_spec[tile_idx]; + + const int expert_idx = ts.expert; + const int expert_start = ts.col_low; + const int expert_end = ts.col_high; const int ncols_expert = expert_end - expert_start; - const int tiles_for_expert = (ncols_expert + cols_per_block - 1) / cols_per_block; - const int tile_idx = blockIdx.z; - if (tile_idx >= tiles_for_expert) { + if (ncols_expert == 0) { return; } - const int col_base = tile_idx * cols_per_block; - GGML_UNUSED(channel_ratio); const int channel_x = expert_idx; @@ -409,10 +410,9 @@ static __global__ void mul_mat_f_ids( #pragma unroll for (int j0 = 0; j0 < tile_B::I; ++j0) { const int j = j0 + tile_idx_local*tile_B::I; - const int global_j = col_base + j; float val = 0.0f; - if (j < cols_per_block && global_j < ncols_expert) { - const int src_entry = ids_src_expert[global_j]; + if (j < cols_per_block && j < ncols_expert) { + const int src_entry = ids_src_expert[j]; const uint2 qrm = fast_div_modulo((uint32_t) src_entry, sis1_fd); const int token = (int) qrm.x; const int channel = (int) qrm.y; @@ -462,10 +462,9 @@ static __global__ void mul_mat_f_ids( #pragma unroll for (int j0 = 0; j0 < tile_B::I; ++j0) { const int j = j0 + tile_idx_local*tile_B::I; - const int global_j = col_base + j; float2 tmp = make_float2(0.0f, 0.0f); - if (j < cols_per_block && global_j < ncols_expert) { - const int src_entry = ids_src_expert[global_j]; + if (j < cols_per_block && j < ncols_expert) { + const int src_entry = ids_src_expert[j]; const uint2 qrm = fast_div_modulo((uint32_t) src_entry, sis1_fd); const int token = (int) qrm.x; const int channel = (int) qrm.y; @@ -564,9 +563,8 @@ static __global__ void mul_mat_f_ids( } } - const int global_j = col_base + j; - if (j < cols_per_block && global_j < ncols_expert && nchannels_dst > 0) { - const int dst_entry = ids_dst_expert[global_j]; + if (j < cols_per_block && j < ncols_expert && nchannels_dst > 0) { + const int dst_entry = ids_dst_expert[j]; const uint2 qrm = fast_div_modulo((uint32_t) dst_entry, nch_fd); const int token = (int) qrm.x; if (token < ncols_dst_total) { @@ -580,7 +578,7 @@ static __global__ void mul_mat_f_ids( } } #else - GGML_UNUSED_VARS(x, y, ids_src_compact, ids_dst_compact, expert_bounds, dst, + GGML_UNUSED_VARS(x, y, ids_src_compact, ids_dst_compact, tile_spec, dst, ncols, ncols_dst_total, nchannels_dst, stride_row, stride_col_y, stride_col_dst, channel_ratio, stride_channel_x, stride_channel_y, stride_channel_dst, sample_ratio, stride_sample_x, stride_sample_y, stride_sample_dst, sis1_fd, nch_fd); @@ -603,17 +601,16 @@ static inline void mul_mat_f_switch_ids( // Use the compact-ids kernel only for larger tiles; for small ncols_dst (< 16) // we prefer the normal mul_mat_f path with has_ids=true. if (has_ids_data && ncols_dst > 16) { - const int max_tiles = (int) ((ncols_dst + cols_per_block - 1) / cols_per_block); - if (max_tiles == 0) { + if (ids_data->max_tiles == 0) { return; } - dim3 block_nums_ids(block_nums.x, ids_data->n_experts, max_tiles); + dim3 block_nums_ids(block_nums.x, ids_data->max_tiles, 1); const uint3 sis1_fd = ids_data->sis1 > 0 ? init_fastdiv_values((uint32_t) ids_data->sis1) : make_uint3(0, 0, 1); const uint3 nch_fd = init_fastdiv_values((uint32_t) nchannels_dst); mul_mat_f_ids<<>> - (x, y, ids_data->ids_src_compact, ids_data->ids_dst_compact, ids_data->expert_bounds_dev, dst, + (x, y, ids_data->ids_src_compact, ids_data->ids_dst_compact, ids_data->tile_spec_dev, dst, ncols_x, ncols_dst, nchannels_dst, stride_row, stride_col_y, stride_col_dst, channel_ratio, stride_channel_x, stride_channel_y, stride_channel_dst, sample_ratio, stride_sample_x, stride_sample_y, stride_sample_dst, diff --git a/ggml/src/ggml-cuda/mmid.cu b/ggml/src/ggml-cuda/mmid.cu index 12ae14ce9a17..64de4fb27b60 100644 --- a/ggml/src/ggml-cuda/mmid.cu +++ b/ggml/src/ggml-cuda/mmid.cu @@ -27,12 +27,16 @@ template <> struct mm_ids_pow2<0> { static constexpr int value = 1; }; // Helper function for mul_mat_id, converts ids to a more convenient format. // ids_src1 describes how to permute the flattened column indices of src1 in order to get a compact src1 tensor sorted by expert. // ids_dst describes the same mapping but for the dst tensor. -// The upper and lower bounds for the ith expert in the compact src1 tensor are stored in expert_bounds[i:i+1]. +// The first (n_tokens*n_expert_used + J - 1) / tile_width elements of tile_spec contain the specication for +// matrix multiplication tiles at token positions which are exactly divided by tile_width. +// Additionally the following n_expert elements of tile_spec may contain additional tiles that fall between these seams. template __launch_bounds__(ggml_cuda_get_physical_warp_size(), 1) static __global__ void mm_ids_helper( - const int32_t * __restrict__ ids, int32_t * __restrict__ ids_src1, int32_t * __restrict__ ids_dst, int32_t * __restrict__ expert_bounds, - const int n_tokens, const int n_expert_used_var, const int nchannels_y, const int si1, const int sis1, const bool write_inverse) { + const int32_t * __restrict__ ids, + int32_t * __restrict__ ids_src1, int32_t * __restrict__ ids_dst, ggml_cuda_mm_id_tile_spec * __restrict__ tile_spec, + const int n_tokens, const int n_expert_used_var, const int nchannels_y, const uint3 tile_width, + const int si1, const int sis1, const bool write_inverse) { constexpr int warp_size = ggml_cuda_get_physical_warp_size(); const int n_expert_used = n_expert_used_template == 0 ? n_expert_used_var : n_expert_used_template; const int expert = blockIdx.x; @@ -118,19 +122,29 @@ static __global__ void mm_ids_helper( return; } - expert_bounds[expert] = nex_prev; + int col_low = nex_prev; + int col_high = nex_prev + min(int(tile_width.z), it_compact); - if (expert < static_cast(gridDim.x) - 1) { - return; + // The while loop covers all tiles at the seams containing the column with % tile_width == 0. + while (col_high > fastdiv(col_low + tile_width.z - 1, tile_width) * tile_width.z) { + tile_spec[fastdiv(col_low + tile_width.z - 1, tile_width)] + = {expert, col_low, col_high, /*padding =*/ 0}; + + col_low = col_high; + col_high = min(col_high + int(tile_width.z), nex_prev + it_compact); } - expert_bounds[gridDim.x] = nex_prev + it_compact; + // Finally, at most one extra tile_spec between the seams per expert. + tile_spec[fastdiv(n_tokens*n_expert_used + tile_width.z - 1, tile_width) + expert] + = {expert, col_low, col_high, /*padding =*/ 0}; } template static void launch_mm_ids_helper( - const int32_t * __restrict__ ids, int32_t * __restrict__ ids_src1, int32_t * __restrict__ ids_dst, int32_t * __restrict__ expert_bounds, - const int n_experts, const int n_tokens, const int n_expert_used_var, const int nchannels_y, const int si1, const int sis1, const bool write_inverse, cudaStream_t stream) { + const int32_t * __restrict__ ids, + int32_t * __restrict__ ids_src1, int32_t * __restrict__ ids_dst, ggml_cuda_mm_id_tile_spec * __restrict__ tile_spec, + const int n_experts, const int n_tokens, const int n_expert_used_var, const int nchannels_y, const int tile_width, + const int si1, const int sis1, const bool write_inverse, cudaStream_t stream) { GGML_ASSERT(n_tokens < (1 << 22) && "too few bits in mm_ids_helper_store"); GGML_ASSERT(n_expert_used_var < (1 << 10) && "too few bits in mm_ids_helper_store"); @@ -139,41 +153,53 @@ static void launch_mm_ids_helper( const size_t smpbo = ggml_cuda_info().devices[id].smpbo; CUDA_SET_SHARED_MEMORY_LIMIT(mm_ids_helper, smpbo); + const uint3 tile_width_fd = init_fastdiv_values(tile_width); + const dim3 num_blocks(n_experts, 1, 1); const dim3 block_size(warp_size, 1, 1); const size_t nbytes_shared = n_tokens*sizeof(mm_ids_helper_store); GGML_ASSERT(nbytes_shared <= smpbo); mm_ids_helper<<>> - (ids, ids_src1, ids_dst, expert_bounds, n_tokens, n_expert_used_var, nchannels_y, si1, sis1, write_inverse); + (ids, ids_src1, ids_dst, tile_spec, n_tokens, n_expert_used_var, nchannels_y, tile_width_fd, si1, sis1, write_inverse); } void ggml_cuda_launch_mm_ids_helper( - const int32_t * __restrict__ ids, int32_t * __restrict__ ids_src1, int32_t * __restrict__ ids_dst, int32_t * __restrict__ expert_bounds, - const int n_experts, const int n_tokens, const int n_expert_used, const int nchannels_y, const int si1, const int sis1, const bool write_inverse, cudaStream_t stream) { + const int32_t * __restrict__ ids, + int32_t * __restrict__ ids_src1, int32_t * __restrict__ ids_dst, ggml_cuda_mm_id_tile_spec * __restrict__ tile_spec, + const int n_experts, const int n_tokens, const int n_expert_used, const int nchannels_y, const int tile_width, + const int si1, const int sis1, const bool write_inverse, cudaStream_t stream) { switch (n_expert_used) { case 2: - launch_mm_ids_helper< 2>(ids, ids_src1, ids_dst, expert_bounds, n_experts, n_tokens, n_expert_used, nchannels_y, si1, sis1, write_inverse, stream); + launch_mm_ids_helper< 2>(ids, ids_src1, ids_dst, tile_spec, + n_experts, n_tokens, n_expert_used, nchannels_y, tile_width, si1, sis1, write_inverse, stream); break; case 4: - launch_mm_ids_helper< 4>(ids, ids_src1, ids_dst, expert_bounds, n_experts, n_tokens, n_expert_used, nchannels_y, si1, sis1, write_inverse, stream); + launch_mm_ids_helper< 4>(ids, ids_src1, ids_dst, tile_spec, + n_experts, n_tokens, n_expert_used, nchannels_y, tile_width, si1, sis1, write_inverse, stream); break; case 6: - launch_mm_ids_helper< 6>(ids, ids_src1, ids_dst, expert_bounds, n_experts, n_tokens, n_expert_used, nchannels_y, si1, sis1, write_inverse, stream); + launch_mm_ids_helper< 6>(ids, ids_src1, ids_dst, tile_spec, + n_experts, n_tokens, n_expert_used, nchannels_y, tile_width, si1, sis1, write_inverse, stream); break; case 8: - launch_mm_ids_helper< 8>(ids, ids_src1, ids_dst, expert_bounds, n_experts, n_tokens, n_expert_used, nchannels_y, si1, sis1, write_inverse, stream); + launch_mm_ids_helper< 8>(ids, ids_src1, ids_dst, tile_spec, + n_experts, n_tokens, n_expert_used, nchannels_y, tile_width, si1, sis1, write_inverse, stream); break; case 10: - launch_mm_ids_helper<10>(ids, ids_src1, ids_dst, expert_bounds, n_experts, n_tokens, n_expert_used, nchannels_y, si1, sis1, write_inverse, stream); + launch_mm_ids_helper<10>(ids, ids_src1, ids_dst, tile_spec, + n_experts, n_tokens, n_expert_used, nchannels_y, tile_width, si1, sis1, write_inverse, stream); break; case 16: - launch_mm_ids_helper<16>(ids, ids_src1, ids_dst, expert_bounds, n_experts, n_tokens, n_expert_used, nchannels_y, si1, sis1, write_inverse, stream); + launch_mm_ids_helper<16>(ids, ids_src1, ids_dst, tile_spec, + n_experts, n_tokens, n_expert_used, nchannels_y, tile_width, si1, sis1, write_inverse, stream); break; case 32: - launch_mm_ids_helper<32>(ids, ids_src1, ids_dst, expert_bounds, n_experts, n_tokens, n_expert_used, nchannels_y, si1, sis1, write_inverse, stream); + launch_mm_ids_helper<32>(ids, ids_src1, ids_dst, tile_spec, + n_experts, n_tokens, n_expert_used, nchannels_y, tile_width, si1, sis1, write_inverse, stream); break; default: - launch_mm_ids_helper< 0>(ids, ids_src1, ids_dst, expert_bounds, n_experts, n_tokens, n_expert_used, nchannels_y, si1, sis1, write_inverse, stream); + launch_mm_ids_helper< 0>(ids, ids_src1, ids_dst, tile_spec, + n_experts, n_tokens, n_expert_used, nchannels_y, tile_width, si1, sis1, write_inverse, stream); break; } } diff --git a/ggml/src/ggml-cuda/mmid.cuh b/ggml/src/ggml-cuda/mmid.cuh index 74c2db43385e..dc878a71cd1d 100644 --- a/ggml/src/ggml-cuda/mmid.cuh +++ b/ggml/src/ggml-cuda/mmid.cuh @@ -1,5 +1,16 @@ #pragma once +struct ggml_cuda_mm_id_tile_spec { + int32_t expert; + int32_t col_low; + int32_t col_high; + + int32_t padding; +}; +static_assert(sizeof(ggml_cuda_mm_id_tile_spec) == 16, "bad ggml_cuda_mm_id_tile_spec size"); + void ggml_cuda_launch_mm_ids_helper( - const int32_t * ids, int32_t * ids_src1, int32_t * ids_dst, int32_t * expert_bounds, - int n_experts, int n_tokens, int n_expert_used, int nchannels_y, int si1, int sis1, bool write_inverse, cudaStream_t stream); + const int32_t * __restrict__ ids, + int32_t * __restrict__ ids_src1, int32_t * __restrict__ ids_dst, ggml_cuda_mm_id_tile_spec * __restrict__ tile_spec, + const int n_experts, const int n_tokens, const int n_expert_used, const int nchannels_y, const int tile_width, + const int si1, const int sis1, const bool write_inverse, cudaStream_t stream); diff --git a/ggml/src/ggml-cuda/mmq-few.cu b/ggml/src/ggml-cuda/mmq-few.cu new file mode 100644 index 000000000000..2c2a2745e3ec --- /dev/null +++ b/ggml/src/ggml-cuda/mmq-few.cu @@ -0,0 +1,876 @@ +#include "mmq-few.cuh" +#include "cp-async.cuh" +#include "mma.cuh" +#include "quantize.cuh" +#include "vecdotq.cuh" + +#include + +// NVIDIA only: the kernel uses the NVIDIA int8 MMA fragment layout and cp.async. +#if !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA) + +using namespace ggml_cuda_mma; + +// Few-row MMQ: int8 MMA mat-mul for 2-24 src1 columns. A CUDA block has 16 src0 rows (32 for 2 column groups on large matrices). +// Its warps split K in units of 256 values and copy raw src0 rows into their own cp.async ring, src1 is quantized to the MMA fragment layout. + +static constexpr int MMQ_FEW_MAX_NCG = 3; // max. number of column groups +static constexpr int MMQ_FEW_NROWS = 16; // src0 rows of one MMA tile +static constexpr int MMQ_FEW_NWARPS = 4; + +template +static constexpr __host__ __device__ int mmq_few_unit_size_of() { + return QK_K/ggml_cuda_type_traits::qk*ggml_cuda_type_traits::bs; +} + +// src0 bytes per 256 values, 0 for the types without a few-row decoder. +static constexpr __host__ __device__ int mmq_few_unit_size(const ggml_type type) { + switch (type) { + case GGML_TYPE_Q8_0: return mmq_few_unit_size_of(); + case GGML_TYPE_Q3_K: return mmq_few_unit_size_of(); + case GGML_TYPE_Q4_K: return mmq_few_unit_size_of(); + case GGML_TYPE_Q5_K: return mmq_few_unit_size_of(); + case GGML_TYPE_Q6_K: return mmq_few_unit_size_of(); + case GGML_TYPE_IQ3_S: return mmq_few_unit_size_of(); + case GGML_TYPE_IQ4_NL: return mmq_few_unit_size_of(); + case GGML_TYPE_IQ4_XS: return mmq_few_unit_size_of(); + default: return 0; + } +} + +// Q4_K and Q5_K also need the src1 sums for their mins. +static constexpr __host__ __device__ mmq_q8_1_ds_layout mmq_few_ds_layout(const ggml_type type) { + return type == GGML_TYPE_Q4_K || type == GGML_TYPE_Q5_K ? MMQ_Q8_1_DS_LAYOUT_DS4 : MMQ_Q8_1_DS_LAYOUT_D4; +} + +// Units of 256 values per ring stage: 1 for Q6_K, 3 column groups (fewer free registers) and 32-row tiles (2 blocks per SM), else 2. +static constexpr __host__ __device__ int mmq_few_nunits_stage(const ggml_type type, const int ncg, const int nmt) { + return type == GGML_TYPE_Q6_K || ncg > 2 || nmt > 1 ? 1 : 2; +} + +// A third ring stage only makes Q6_K faster, the other types get slower on large matrices. +static constexpr __host__ __device__ int mmq_few_nstages(const ggml_type type) { + return type == GGML_TYPE_Q6_K ? 3 : 2; +} + +// The src0 data of a ring stage does not fill whole 16 byte chunks, it is copied from the preceding 16 byte boundary. +static constexpr __host__ __device__ bool mmq_few_unaligned(const ggml_type type, const int nunits_stage) { + return (nunits_stage*mmq_few_unit_size(type)) % 16 != 0; +} + +// for unaligned data the max. over the 8 possible start offsets +static constexpr __host__ __device__ int mmq_few_nchunks(const ggml_type type, const int nunits_stage) { + const int nbytes = nunits_stage*mmq_few_unit_size(type); + return mmq_few_unaligned(type, nunits_stage) ? (14 + nbytes + 15)/16 : nbytes/16; +} + +// Unaligned rows are padded to an odd number of 16 byte chunks against bank conflicts, for aligned rows the padding costs more than it saves. +static constexpr __host__ __device__ int mmq_few_sram_stride(const ggml_type type, const int nunits_stage) { + const int nchunks = mmq_few_nchunks(type, nunits_stage); + return mmq_few_unaligned(type, nunits_stage) && nchunks % 2 == 0 ? 16*(nchunks + 1) : 16*nchunks; +} + +static constexpr __host__ __device__ int mmq_few_nbytes_shared(const ggml_type type, const int ncg, const int nmt) { + return MMQ_FEW_NWARPS*mmq_few_nstages(type)*nmt*MMQ_FEW_NROWS*mmq_few_sram_stride(type, mmq_few_nunits_stage(type, ncg, nmt)); +} + +// With Q6_K and Q8_0 two CUDA blocks with 32-row tiles would not fit into an SM. +static constexpr __host__ __device__ bool mmq_few_tall_supported(const ggml_type type) { + return type == GGML_TYPE_Q3_K || type == GGML_TYPE_Q4_K || type == GGML_TYPE_Q5_K || + type == GGML_TYPE_IQ3_S || type == GGML_TYPE_IQ4_NL || type == GGML_TYPE_IQ4_XS; +} + +#if defined(CP_ASYNC_AVAILABLE) + +// The src1 fragments of a lane for one unit of 256 values, see quantize.cuh. +struct mmq_few_y_frag { + int4 qs[4]; + int4 ds[4]; +}; + +static __device__ __forceinline__ void mmq_few_load_y(mmq_few_y_frag & yf, const int4 * y) { +#pragma unroll + for (int j = 0; j < 4; ++j) { + yf.qs[j] = y[j*WARP_SIZE + threadIdx.x]; + } +#pragma unroll + for (int j = 0; j < 4; ++j) { + yf.ds[j] = y[4*WARP_SIZE + 4*(threadIdx.x % 4) + j]; + } +} + +static __device__ __forceinline__ int2 mmq_few_get_b(const mmq_few_y_frag & yf, const int s) { + const int4 q = yf.qs[s/2]; + return s % 2 == 0 ? make_int2(q.x, q.y) : make_int2(q.z, q.w); +} + +template +static __device__ __forceinline__ float2 mmq_few_get_d(const mmq_few_y_frag & yf, const int s) { + if constexpr (ds_layout == MMQ_Q8_1_DS_LAYOUT_D4) { + const float * d = (const float *) &yf.ds[s/2]; + return make_float2(d[2*(s % 2) + 0], d[2*(s % 2) + 1]); + } else { + const half2 * ds = (const half2 *) &yf.ds[s/2]; + return make_float2(__low2float(ds[2*(s % 2) + 0]), __low2float(ds[2*(s % 2) + 1])); + } +} + +static __device__ __forceinline__ float2 mmq_few_get_sum(const mmq_few_y_frag & yf, const int s) { + const half2 * ds = (const half2 *) &yf.ds[s/2]; + return make_float2(__high2float(ds[2*(s % 2) + 0]), __high2float(ds[2*(s % 2) + 1])); +} + +// C.x[0], C.x[1]: row g, columns 2t and 2t+1. C.x[2], C.x[3]: row g+8. +static __device__ __forceinline__ tile<16, 8, int> mmq_few_mma(const int (&lo)[2], const int (&hi)[2], const int2 b) { + tile<16, 8, int> A; + A.x[0] = lo[0]; + A.x[1] = lo[1]; + A.x[2] = hi[0]; + A.x[3] = hi[1]; + tile<8, 8, int> B; + B.x[0] = b.x; + B.x[1] = b.y; + tile<16, 8, int> C; + mma(C, A, B); + return C; +} + +static __device__ __forceinline__ tile<16, 8, int> mmq_few_mma_k16(const int (&a)[2], const int b) { + tile<16, 4, int> A; + A.x[0] = a[0]; + A.x[1] = a[1]; + tile<8, 4, int> B; + B.x[0] = b; + tile<16, 8, int> C; + mma(C, A, B); + return C; +} + +// sum_d += d * (C * ls) of sub-block s for all column groups, ls are the src0 row scales +template +static __device__ __forceinline__ void mmq_few_mma_add( + const int (&lo)[2], const int (&hi)[2], const scale_t (&ls)[2], const mmq_few_y_frag (&yf)[ncg], const int s, float (&sum_d)[ncg][4]) { +#pragma unroll + for (int c = 0; c < ncg; ++c) { + const tile<16, 8, int> C = mmq_few_mma(lo, hi, mmq_few_get_b(yf[c], s)); + + const float2 d = mmq_few_get_d(yf[c], s); + sum_d[c][0] += d.x * float(C.x[0]*ls[0]); + sum_d[c][1] += d.y * float(C.x[1]*ls[0]); + sum_d[c][2] += d.x * float(C.x[2]*ls[1]); + sum_d[c][3] += d.y * float(C.x[3]*ls[1]); + } +} + +// Same with one k=16 MMA per half of sub-block s, the int8 scales of the halves are bytes 0 and 1 of sc[r]. +template +static __device__ __forceinline__ void mmq_few_mma_k16_add( + const int (&lo)[2], const int (&hi)[2], const int (&sc)[2], const mmq_few_y_frag (&yf)[ncg], const int s, float (&sum_d)[ncg][4]) { + const int sc_lo[2] = {(int8_t) sc[0], (int8_t) sc[1]}; + const int sc_hi[2] = {(int8_t) (sc[0] >> 8), (int8_t) (sc[1] >> 8)}; +#pragma unroll + for (int c = 0; c < ncg; ++c) { + const int2 b = mmq_few_get_b(yf[c], s); + const tile<16, 8, int> C_lo = mmq_few_mma_k16(lo, b.x); + const tile<16, 8, int> C_hi = mmq_few_mma_k16(hi, b.y); + + const float2 d = mmq_few_get_d(yf[c], s); + sum_d[c][0] += d.x * float(C_lo.x[0]*sc_lo[0] + C_hi.x[0]*sc_hi[0]); + sum_d[c][1] += d.y * float(C_lo.x[1]*sc_lo[0] + C_hi.x[1]*sc_hi[0]); + sum_d[c][2] += d.x * float(C_lo.x[2]*sc_lo[1] + C_hi.x[2]*sc_hi[1]); + sum_d[c][3] += d.y * float(C_lo.x[3]*sc_lo[1] + C_hi.x[3]*sc_hi[1]); + } +} + +template +static __device__ __forceinline__ void mmq_few_add_scaled(float (&sum)[ncg][4], const float (&d)[2], const float (&sum_d)[ncg][4]) { +#pragma unroll + for (int c = 0; c < ncg; ++c) { +#pragma unroll + for (int l = 0; l < 4; ++l) { + sum[c][l] += d[l/2]*sum_d[c][l]; + } + } +} + +// vec_dot: one unit of 256 values of the 16 rows of a warp, x[0] and x[1] are the rows g and g+8 of lane (g, t). +// The lane decodes the values [4t, 4t + 4) and [16 + 4t, 16 + 4t + 4) of each sub-block, its A fragments. + +template +static __device__ __forceinline__ void mmq_few_vec_dot_q45_K(const char * const (&x)[2], const mmq_few_y_frag (&yf)[ncg], float (&sum)[ncg][4]) { + using block_t = std::conditional_t; + const int t = threadIdx.x % 4; + + int qs[2][4][2]; + [[maybe_unused]] int qh[2][2]; // Q5_K: bit s holds the high bit of sub-block s + int sc[2][2]; + int mn[2][2]; + float d[2]; + float dmin[2]; +#pragma unroll + for (int r = 0; r < 2; ++r) { + const block_t * bx = (const block_t *) x[r]; +#pragma unroll + for (int j = 0; j < 4; ++j) { + qs[r][j][0] = get_int_b4(bx->qs + 32*j, t); + qs[r][j][1] = get_int_b4(bx->qs + 32*j + 16, t); + } + if constexpr (type == GGML_TYPE_Q5_K) { + qh[r][0] = get_int_b4(bx->qh, t); + qh[r][1] = get_int_b4(bx->qh + 16, t); + } + const int4 hdr = *(const int4 *) bx; // dm and the 12 scale bytes + const int scales[3] = {hdr.y, hdr.z, hdr.w}; + sc[r][0] = unpack_scales_q45_K(scales, 0); + sc[r][1] = unpack_scales_q45_K(scales, 1); + mn[r][0] = unpack_scales_q45_K(scales, 2); + mn[r][1] = unpack_scales_q45_K(scales, 3); + const float2 dm = __half22float2(bx->dm); + d[r] = dm.x; + dmin[r] = dm.y; + } + + float sum_d[ncg][4] = {{0.0f}}; + float sum_m[ncg][4] = {{0.0f}}; +#pragma unroll + for (int s = 0; s < 8; ++s) { + const int j = s/2; + const int shift = 4*(s % 2); + int lo[2]; + int hi[2]; +#pragma unroll + for (int r = 0; r < 2; ++r) { + lo[r] = (qs[r][j][0] >> shift) & 0x0F0F0F0F; + hi[r] = (qs[r][j][1] >> shift) & 0x0F0F0F0F; + if constexpr (type == GGML_TYPE_Q5_K) { + lo[r] |= ((qh[r][0] >> s) & 0x01010101) << 4; + hi[r] |= ((qh[r][1] >> s) & 0x01010101) << 4; + } + } + const int ls[2] = {(sc[0][s/4] >> (8*(s % 4))) & 0xFF, (sc[1][s/4] >> (8*(s % 4))) & 0xFF}; + const float m0 = (mn[0][s/4] >> (8*(s % 4))) & 0xFF; + const float m1 = (mn[1][s/4] >> (8*(s % 4))) & 0xFF; + mmq_few_mma_add(lo, hi, ls, yf, s, sum_d); +#pragma unroll + for (int c = 0; c < ncg; ++c) { + const float2 sm = mmq_few_get_sum(yf[c], s); + sum_m[c][0] += m0 * sm.x; + sum_m[c][1] += m0 * sm.y; + sum_m[c][2] += m1 * sm.x; + sum_m[c][3] += m1 * sm.y; + } + } +#pragma unroll + for (int c = 0; c < ncg; ++c) { +#pragma unroll + for (int l = 0; l < 4; ++l) { + sum[c][l] += d[l/2]*sum_d[c][l] - dmin[l/2]*sum_m[c][l]; + } + } +} + +// Q6_K and Q3_K have a scale per 16 values: one k=16 MMA per half of a sub-block. +template +static __device__ __forceinline__ void mmq_few_vec_dot_q6_K(const char * const (&x)[2], const mmq_few_y_frag (&yf)[ncg], float (&sum)[ncg][4]) { + const int t = threadIdx.x % 4; + + float d[2]; +#pragma unroll + for (int r = 0; r < 2; ++r) { + d[r] = __half2float(((const block_q6_K *) x[r])->d); + } + + float sum_d[ncg][4] = {{0.0f}}; +#pragma unroll + for (int n = 0; n < 2; ++n) { + int ql[2][2][2]; // [row][ql offset 0 or 32][lo or hi] + int qh[2][2]; + int sc[2][2]; +#pragma unroll + for (int r = 0; r < 2; ++r) { + const block_q6_K * bx = (const block_q6_K *) x[r]; +#pragma unroll + for (int k = 0; k < 2; ++k) { + ql[r][k][0] = get_int_b2(bx->ql + 64*n + 32*k, t); + ql[r][k][1] = get_int_b2(bx->ql + 64*n + 32*k + 16, t); + } + qh[r][0] = get_int_b2(bx->qh + 32*n, t); + qh[r][1] = get_int_b2(bx->qh + 32*n + 16, t); + sc[r][0] = get_int_b2(bx->scales + 8*n, 0); + sc[r][1] = get_int_b2(bx->scales + 8*n, 1); + } +#pragma unroll + for (int j = 0; j < 4; ++j) { + const int s = 4*n + j; + int lo[2]; + int hi[2]; +#pragma unroll + for (int r = 0; r < 2; ++r) { + lo[r] = __vsub4(((ql[r][j % 2][0] >> (4*(j/2))) & 0x0F0F0F0F) | (((qh[r][0] >> (2*j)) & 0x03030303) << 4), 0x20202020); + hi[r] = __vsub4(((ql[r][j % 2][1] >> (4*(j/2))) & 0x0F0F0F0F) | (((qh[r][1] >> (2*j)) & 0x03030303) << 4), 0x20202020); + } + const int sc_s[2] = {sc[0][j/2] >> (16*(j % 2)), sc[1][j/2] >> (16*(j % 2))}; // scales 2s and 2s+1 + mmq_few_mma_k16_add(lo, hi, sc_s, yf, s, sum_d); + } + } + mmq_few_add_scaled(sum, d, sum_d); +} + +template +static __device__ __forceinline__ void mmq_few_vec_dot_q3_K(const char * const (&x)[2], const mmq_few_y_frag (&yf)[ncg], float (&sum)[ncg][4]) { + const int t = threadIdx.x % 4; + + int hm[2][2]; + int sc[2][4]; // the 16 scales of a row as int8, minus 32 + float d[2]; +#pragma unroll + for (int r = 0; r < 2; ++r) { + const block_q3_K * bx = (const block_q3_K *) x[r]; + hm[r][0] = ~get_int_b2(bx->hmask, t); // a set bit means that 4 is subtracted + hm[r][1] = ~get_int_b2(bx->hmask + 16, t); +#pragma unroll + for (int ksc = 0; ksc < 4; ++ksc) { + sc[r][ksc] = unpack_scales_q3_K(bx->scales, ksc); + } + d[r] = __half2float(bx->d); + } + + float sum_d[ncg][4] = {{0.0f}}; +#pragma unroll + for (int n = 0; n < 2; ++n) { + int qs[2][2]; +#pragma unroll + for (int r = 0; r < 2; ++r) { + const block_q3_K * bx = (const block_q3_K *) x[r]; + qs[r][0] = get_int_b2(bx->qs + 32*n, t); + qs[r][1] = get_int_b2(bx->qs + 32*n + 16, t); + } +#pragma unroll + for (int j = 0; j < 4; ++j) { + const int s = 4*n + j; + int lo[2]; + int hi[2]; +#pragma unroll + for (int r = 0; r < 2; ++r) { + lo[r] = __vsub4((qs[r][0] >> (2*j)) & 0x03030303, ((hm[r][0] >> s) & 0x01010101) << 2); + hi[r] = __vsub4((qs[r][1] >> (2*j)) & 0x03030303, ((hm[r][1] >> s) & 0x01010101) << 2); + } + const int sc_s[2] = {sc[0][s/2] >> (16*(s % 2)), sc[1][s/2] >> (16*(s % 2))}; // scales 2s and 2s+1 + mmq_few_mma_k16_add(lo, hi, sc_s, yf, s, sum_d); + } + } + mmq_few_add_scaled(sum, d, sum_d); +} + +template +static __device__ __forceinline__ void mmq_few_vec_dot_iq4_xs(const char * const (&x)[2], const mmq_few_y_frag (&yf)[ncg], float (&sum)[ncg][4]) { + const int t = threadIdx.x % 4; + + float d[2]; + int scales_h[2]; + int scales_l[2]; +#pragma unroll + for (int r = 0; r < 2; ++r) { + const block_iq4_xs * bx = (const block_iq4_xs *) x[r]; + d[r] = __half2float(bx->d); + scales_h[r] = bx->scales_h; + scales_l[r] = get_int_b4(bx->scales_l, 0); + } + + float sum_d[ncg][4] = {{0.0f}}; +#pragma unroll + for (int s = 0; s < 8; ++s) { + int lo[2]; + int hi[2]; + int ls[2]; +#pragma unroll + for (int r = 0; r < 2; ++r) { + const block_iq4_xs * bx = (const block_iq4_xs *) x[r]; + const int2 v = get_int_from_table_16(get_int_b4(bx->qs + 16*s, t), kvalues_iq4nl); + lo[r] = v.x; + hi[r] = v.y; + ls[r] = (((scales_l[r] >> (4*s)) & 0x0F) | (((scales_h[r] >> (2*s)) & 0x03) << 4)) - 32; + } + mmq_few_mma_add(lo, hi, ls, yf, s, sum_d); + } + mmq_few_add_scaled(sum, d, sum_d); +} + +// Q8_0 and IQ4_NL have a block with its own scale per 32 values. +template +static __device__ __forceinline__ void mmq_few_vec_dot_q8_0_iq4_nl(const char * const (&x)[2], const mmq_few_y_frag (&yf)[ncg], float (&sum)[ncg][4]) { + const int t = threadIdx.x % 4; + +#pragma unroll + for (int s = 0; s < 8; ++s) { + int lo[2]; + int hi[2]; + float d[2]; +#pragma unroll + for (int r = 0; r < 2; ++r) { + if constexpr (type == GGML_TYPE_Q8_0) { + const block_q8_0 * bx = (const block_q8_0 *) x[r] + s; + lo[r] = get_int_b2(bx->qs, t); + hi[r] = get_int_b2(bx->qs + 16, t); + d[r] = __half2float(bx->d); + } else { + const block_iq4_nl * bx = (const block_iq4_nl *) x[r] + s; + const int2 v = get_int_from_table_16(get_int_b2(bx->qs, t), kvalues_iq4nl); + lo[r] = v.x; + hi[r] = v.y; + d[r] = __half2float(bx->d); + } + } + mmq_few_mma_add(lo, hi, d, yf, s, sum); + } +} + +template +static __device__ __forceinline__ void mmq_few_vec_dot_iq3_s(const char * const (&x)[2], const mmq_few_y_frag (&yf)[ncg], float (&sum)[ncg][4]) { + const int t = threadIdx.x % 4; + + float d[2]; +#pragma unroll + for (int r = 0; r < 2; ++r) { + d[r] = __half2float(((const block_iq3_s *) x[r])->d); + } + + float sum_d[ncg][4] = {{0.0f}}; +#pragma unroll + for (int s = 0; s < 8; ++s) { + int lo[2]; + int hi[2]; + int ls[2]; +#pragma unroll + for (int r = 0; r < 2; ++r) { + const block_iq3_s * bx = (const block_iq3_s *) x[r]; + const int qh = bx->qh[s]; + // the values [4k, 4k + 4) of a sub-block come from grid entry k, with the signs in bits 4*(k%2) of byte k/2 + const int k_lo = t; + const int k_hi = 4 + t; + const int grid_lo = iq3s_grid[bx->qs[8*s + k_lo] | (((qh >> k_lo) & 1) << 8)]; + const int grid_hi = iq3s_grid[bx->qs[8*s + k_hi] | (((qh >> k_hi) & 1) << 8)]; + const int sign_lo = (bx->signs[4*s + k_lo/2] >> (4*(k_lo % 2))) & 0x0F; + const int sign_hi = (bx->signs[4*s + k_hi/2] >> (4*(k_hi % 2))) & 0x0F; + const int mask_lo = __vcmpne4(((sign_lo & 0x03) << 7) | ((sign_lo & 0x0C) << 21), 0x00000000); + const int mask_hi = __vcmpne4(((sign_hi & 0x03) << 7) | ((sign_hi & 0x0C) << 21), 0x00000000); + lo[r] = __vsub4(grid_lo ^ mask_lo, mask_lo); + hi[r] = __vsub4(grid_hi ^ mask_hi, mask_hi); + ls[r] = 1 + 2*((bx->scales[s/2] >> (4*(s % 2))) & 0x0F); + } + mmq_few_mma_add(lo, hi, ls, yf, s, sum_d); + } + mmq_few_add_scaled(sum, d, sum_d); +} + +template +static __device__ __forceinline__ void mmq_few_vec_dot(const char * const (&x)[2], const mmq_few_y_frag (&yf)[ncg], float (&sum)[ncg][4]) { + if constexpr (type == GGML_TYPE_Q8_0 || type == GGML_TYPE_IQ4_NL) { + mmq_few_vec_dot_q8_0_iq4_nl(x, yf, sum); + } else if constexpr (type == GGML_TYPE_Q3_K) { + mmq_few_vec_dot_q3_K(x, yf, sum); + } else if constexpr (type == GGML_TYPE_Q4_K || type == GGML_TYPE_Q5_K) { + mmq_few_vec_dot_q45_K(x, yf, sum); + } else if constexpr (type == GGML_TYPE_Q6_K) { + mmq_few_vec_dot_q6_K(x, yf, sum); + } else if constexpr (type == GGML_TYPE_IQ3_S) { + mmq_few_vec_dot_iq3_s(x, yf, sum); + } else if constexpr (type == GGML_TYPE_IQ4_XS) { + mmq_few_vec_dot_iq4_xs(x, yf, sum); + } else { + static_assert(type == GGML_TYPE_COUNT, "unsupported type"); + } +} + +#endif // defined(CP_ASYNC_AVAILABLE) + +template +__launch_bounds__(MMQ_FEW_NWARPS*ggml_cuda_get_physical_warp_size(), 1) +static __global__ void mul_mat_q_few( + const char * x, const int4 * y_ptr, float * dst, + const int nunits, const int nrows_x, const int ncols_dst, const int64_t stride_row_x, const int stride_col_dst, + const int channel_ratio, const int64_t stride_channel_x, const int stride_channel_dst, + const int sample_ratio, const int64_t stride_sample_x, const int stride_sample_dst, const bool x_is_constant) { +#if defined(CP_ASYNC_AVAILABLE) + constexpr int nwarps = MMQ_FEW_NWARPS; + constexpr int warp_size = ggml_cuda_get_physical_warp_size(); + static_assert(warp_size == WARP_SIZE, "unsupported warp size"); + + constexpr int unit_size = mmq_few_unit_size(type); + constexpr int nrows = nmt*MMQ_FEW_NROWS; + constexpr int nunits_stage = mmq_few_nunits_stage(type, ncg, nmt); + constexpr int nstages = mmq_few_nstages(type); + constexpr bool unaligned = mmq_few_unaligned(type, nunits_stage); + constexpr int nchunks = mmq_few_nchunks(type, nunits_stage); + constexpr int sram_stride = mmq_few_sram_stride(type, nunits_stage); + constexpr int nbytes_stage = nrows*sram_stride; + constexpr int ncopies = nrows*nchunks; + static_assert(unit_size > 0, "unsupported type"); + + extern __shared__ char data_mmq_few[]; + char * ring = data_mmq_few + threadIdx.y*nstages*nbytes_stage; + + const int row0 = blockIdx.x*nrows; + const int channel_dst = blockIdx.y; + const int sample_dst = blockIdx.z; + const int nchannels = gridDim.y; + + x += int64_t(sample_dst/sample_ratio)*stride_sample_x + int64_t(channel_dst/channel_ratio)*stride_channel_x; + dst += int64_t(sample_dst)*stride_sample_dst + int64_t(channel_dst)*stride_channel_dst; + const int4 * GGML_CUDA_RESTRICT y = y_ptr + (int64_t(sample_dst)*nchannels + channel_dst)*nunits*ncg*MMQ_FEW_Q8_1_INT4; + + // No bounds check for the units, the gate makes nunits a multiple of nunits_stage. + // The extra bytes of unaligned data are inside the allocation, which ggml aligns and pads to 128 bytes. + auto load_stage = [&](const int stage, const int iu0) { + const unsigned int dst_stage = ggml_cuda_cvta_generic_to_shared(ring + stage*nbytes_stage); +#pragma unroll + for (int c0 = 0; c0 < ncopies; c0 += warp_size) { + const int c = c0 + threadIdx.x; + const int i = c / nchunks; + const int ic = c % nchunks; + if (ncopies % warp_size == 0 || c < ncopies) { + const int row = min(row0 + i, nrows_x - 1); + const char * src = x + int64_t(row)*stride_row_x + int64_t(iu0)*unit_size; + if constexpr (unaligned) { + const int off = (uintptr_t) src % 16; + if (16*ic < off + nunits_stage*unit_size) { + cp_async_cg_16<0>(dst_stage + i*sram_stride + 16*ic, src - off + 16*ic); + } + } else { + cp_async_cg_16<0>(dst_stage + i*sram_stride + 16*ic, src + 16*ic); + } + } + } + }; + + // offsets of the rows of this lane from their 16 byte boundary + int off_row[2*nmt] = {0}; + if constexpr (unaligned) { +#pragma unroll + for (int r = 0; r < 2*nmt; ++r) { + const int row = min(row0 + int(threadIdx.x)/4 + 8*r, nrows_x - 1); + off_row[r] = (uintptr_t) (x + int64_t(row)*stride_row_x) % 16; + } + } + + // Only weights, which no preceding kernel writes, are read before the PDL wait. + if (!x_is_constant) { + ggml_cuda_pdl_sync(); + } + +#if __CUDA_ARCH__ == GGML_CUDA_CC_DGX_SPARK + // DGX Spark DRAM serves the short row pieces of the stage copies slowly, so the rows of the block are first prefetched into L2 in address order. + { + const int tid = threadIdx.y*warp_size + threadIdx.x; + const int nrows_blk = min(nrows, nrows_x - row0); + const int64_t row_bytes = int64_t(nunits)*unit_size; + const bool contig = stride_row_x == row_bytes; + for (int ir = 0; ir < (contig ? 1 : nrows_blk); ++ir) { + const uintptr_t p0 = (uintptr_t) (x + int64_t(row0 + ir)*stride_row_x); + const uintptr_t p1 = p0 + (contig ? nrows_blk*row_bytes : row_bytes); + for (uintptr_t p = (p0 & ~uintptr_t(127)) + 128*tid; p < p1; p += 128*nwarps*warp_size) { + ggml_cuda_prefetch_l2((const void *) p); + } + } + } +#endif // __CUDA_ARCH__ == GGML_CUDA_CC_DGX_SPARK + + const int iu_step = nunits_stage*nwarps; + +#pragma unroll + for (int stage = 0; stage < nstages - 1; ++stage) { + const int iu0 = nunits_stage*threadIdx.y + stage*iu_step; + if (iu0 < nunits) { + load_stage(stage, iu0); + } + cp_async_commit_group(); + } + if (x_is_constant) { + ggml_cuda_pdl_sync(); + } + + auto load_y = [&](mmq_few_y_frag (&yf)[ncg], const int iu) { +#pragma unroll + for (int c = 0; c < ncg; ++c) { + mmq_few_load_y(yf[c], y + (int64_t(iu)*ncg + c)*MMQ_FEW_Q8_1_INT4); + } + }; + + // The next src1 fragments are loaded right after the MMAs that use the current ones, an earlier load is slower. + mmq_few_y_frag yf[nunits_stage][ncg]; + if (nunits_stage*threadIdx.y < nunits) { +#pragma unroll + for (int k = 0; k < nunits_stage; ++k) { + load_y(yf[k], nunits_stage*threadIdx.y + k); + } + } + + float sum[nmt][ncg][4] = {{{0.0f}}}; + + for (int iter = 0, iu0 = nunits_stage*threadIdx.y; iu0 < nunits; ++iter, iu0 += iu_step) { + const int iu_load = iu0 + (nstages - 1)*iu_step; + if (iu_load < nunits) { + load_stage((iter + nstages - 1) % nstages, iu_load); + } + cp_async_commit_group(); + + const bool has_next = iu0 + iu_step < nunits; + + cp_async_wait_group(); + ggml_cuda_syncwarp(); + + const char * xs = ring + (iter % nstages)*nbytes_stage; + const int off_stage = unaligned ? (iu0*unit_size) % 16 : 0; +#pragma unroll + for (int k = 0; k < nunits_stage; ++k) { +#pragma unroll + for (int m = 0; m < nmt; ++m) { + const char * const xr[2] = { + xs + (MMQ_FEW_NROWS*m + threadIdx.x/4 + 0)*sram_stride + (off_row[2*m + 0] + off_stage) % 16 + k*unit_size, + xs + (MMQ_FEW_NROWS*m + threadIdx.x/4 + 8)*sram_stride + (off_row[2*m + 1] + off_stage) % 16 + k*unit_size, + }; + mmq_few_vec_dot(xr, yf[k], sum[m]); + } + if (has_next) { + load_y(yf[k], iu0 + iu_step + k); + } + } + + ggml_cuda_syncwarp(); // the stage is overwritten in the next iteration + } + cp_async_wait_all(); + + ggml_cuda_pdl_lc(); + + // reduce the sums over the warps as one flat array + constexpr int nsum = nmt*ncg*4; + float * sum_flat = &sum[0][0][0]; + + float * buf_sum = (float *) data_mmq_few; + __syncthreads(); + if (threadIdx.y > 0) { +#pragma unroll + for (int i = 0; i < nsum; ++i) { + buf_sum[((threadIdx.y - 1)*nsum + i)*warp_size + threadIdx.x] = sum_flat[i]; + } + } + __syncthreads(); + if (threadIdx.y > 0) { + return; + } +#pragma unroll + for (int w = 0; w < nwarps - 1; ++w) { +#pragma unroll + for (int i = 0; i < nsum; ++i) { + sum_flat[i] += buf_sum[(w*nsum + i)*warp_size + threadIdx.x]; + } + } + +#pragma unroll + for (int m = 0; m < nmt; ++m) { +#pragma unroll + for (int c = 0; c < ncg; ++c) { +#pragma unroll + for (int l = 0; l < 4; ++l) { + const int i = row0 + MMQ_FEW_NROWS*m + tile<16, 8, int>::get_i(l); + const int j = c*MMQ_FEW_NCOLS + tile<16, 8, int>::get_j(l); + if (i < nrows_x && j < ncols_dst) { + dst[int64_t(j)*stride_col_dst + i] = sum[m][c][l]; + } + } + } + } +#else + GGML_UNUSED_VARS(x, y_ptr, dst, nunits, nrows_x, ncols_dst, stride_row_x, stride_col_dst, + channel_ratio, stride_channel_x, stride_channel_dst, + sample_ratio, stride_sample_x, stride_sample_dst, x_is_constant); + NO_DEVICE_CODE; +#endif // defined(CP_ASYNC_AVAILABLE) +} + +// ggml_is_constant (static in ggml.c) on the tensor that a view points to +static bool mmq_few_is_constant(const ggml_tensor * src0) { + const ggml_tensor * x_root = src0->view_src ? src0->view_src : src0; + return x_root->buffer && ggml_backend_buffer_get_usage(x_root->buffer) == GGML_BACKEND_BUFFER_USAGE_WEIGHTS && + !(x_root->flags & GGML_TENSOR_FLAG_PARAM); +} + +template +static void launch_mul_mat_q_few(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst, + const int4 * src1_q, cudaStream_t stream) { + GGML_TENSOR_BINARY_OP_LOCALS; + + const int nunits = ne00 / QK_K; + GGML_ASSERT(nunits % mmq_few_nunits_stage(type, ncg, nmt) == 0); + + const int warp_size = ggml_cuda_info().devices[ggml_cuda_get_device()].warp_size; + const int nbytes_shared = mmq_few_nbytes_shared(type, ncg, nmt); + CUDA_SET_SHARED_MEMORY_LIMIT((mul_mat_q_few), nbytes_shared); + + const bool x_is_constant = mmq_few_is_constant(src0); + + const dim3 block_nums((ne01 + nmt*MMQ_FEW_NROWS - 1) / (nmt*MMQ_FEW_NROWS), ne12, ne13); + const dim3 block_dims(warp_size, MMQ_FEW_NWARPS, 1); + const ggml_cuda_kernel_launch_params params(block_nums, block_dims, nbytes_shared, stream); + ggml_cuda_kernel_launch(mul_mat_q_few, params, + (const char *) src0->data, src1_q, (float *) dst->data, + nunits, (int) ne01, (int) ne1, (int64_t) nb01, (int) (nb1/sizeof(float)), + (int) (ne12/ne02), (int64_t) nb02, (int) (nb2/sizeof(float)), + (int) (ne13/ne03), (int64_t) nb03, (int) (nb3/sizeof(float)), x_is_constant); +} + +static int mmq_few_ncg(const int64_t ncols) { + return (ncols + MMQ_FEW_NCOLS - 1) / MMQ_FEW_NCOLS; +} + +// With 2 column groups, 32-row tiles halve the src1 loads per src0 byte. With 3 column groups they spill registers. +static int mmq_few_nmt(const ggml_type type, const int ncg, const int64_t nrows_x) { + const int nsm = ggml_cuda_info().devices[ggml_cuda_get_device()].nsm; + return mmq_few_tall_supported(type) && ncg == 2 && nrows_x >= int64_t(nsm)*2*MMQ_FEW_NROWS ? 2 : 1; +} + +template +static void launch_mul_mat_q_few_tall(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst, + const int4 * src1_q, cudaStream_t stream) { + if constexpr (mmq_few_tall_supported(type)) { + if (mmq_few_nmt(type, 2, src0->ne[1]) == 2) { + launch_mul_mat_q_few(src0, src1, dst, src1_q, stream); + return; + } + } + launch_mul_mat_q_few(src0, src1, dst, src1_q, stream); +} + +template +static void launch_mul_mat_q_few_switch_ncg(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst, + const int4 * src1_q, const int ncg, cudaStream_t stream) { + switch (ncg) { + case 1: + launch_mul_mat_q_few(src0, src1, dst, src1_q, stream); + break; + case 2: + launch_mul_mat_q_few_tall(src0, src1, dst, src1_q, stream); + break; + case 3: + launch_mul_mat_q_few(src0, src1, dst, src1_q, stream); + break; + default: + GGML_ABORT("fatal error"); + } +} + +void ggml_cuda_mul_mat_q_few(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { + GGML_TENSOR_BINARY_OP_LOCALS; + + GGML_ASSERT(src1->type == GGML_TYPE_F32); + GGML_ASSERT( dst->type == GGML_TYPE_F32); + GGML_ASSERT(nb00 == ggml_type_size(src0->type)); + GGML_ASSERT(nb10 == sizeof(float)); + GGML_ASSERT(nb0 == sizeof(float)); + GGML_ASSERT(ne11 <= MMQ_FEW_MAX_NCG*MMQ_FEW_NCOLS); + GGML_ASSERT(ne00 % QK_K == 0); + GGML_ASSERT(ne12 % ne02 == 0 && ne13 % ne03 == 0); + + cudaStream_t stream = ctx.stream(); + + const int nunits = ne00 / QK_K; + const int ncg = mmq_few_ncg(ne11); + + ggml_cuda_pool_alloc src1_q(ctx.pool(), int64_t(ne12)*ne13*nunits*ncg*MMQ_FEW_Q8_1_INT4); + quantize_mmq_few_q8_1_cuda((const float *) src1->data, src1_q.get(), mmq_few_ds_layout(src0->type), ne10, nb11/sizeof(float), nb12/sizeof(float), nb13/sizeof(float), + ne11, ne12, ne13, stream); + + switch (src0->type) { + case GGML_TYPE_Q8_0: + launch_mul_mat_q_few_switch_ncg(src0, src1, dst, src1_q.get(), ncg, stream); + break; + case GGML_TYPE_Q3_K: + launch_mul_mat_q_few_switch_ncg(src0, src1, dst, src1_q.get(), ncg, stream); + break; + case GGML_TYPE_Q4_K: + launch_mul_mat_q_few_switch_ncg(src0, src1, dst, src1_q.get(), ncg, stream); + break; + case GGML_TYPE_Q5_K: + launch_mul_mat_q_few_switch_ncg(src0, src1, dst, src1_q.get(), ncg, stream); + break; + case GGML_TYPE_Q6_K: + launch_mul_mat_q_few_switch_ncg(src0, src1, dst, src1_q.get(), ncg, stream); + break; + case GGML_TYPE_IQ3_S: + launch_mul_mat_q_few_switch_ncg(src0, src1, dst, src1_q.get(), ncg, stream); + break; + case GGML_TYPE_IQ4_NL: + launch_mul_mat_q_few_switch_ncg(src0, src1, dst, src1_q.get(), ncg, stream); + break; + case GGML_TYPE_IQ4_XS: + launch_mul_mat_q_few_switch_ncg(src0, src1, dst, src1_q.get(), ncg, stream); + break; + default: + GGML_ABORT("unsupported type for few-row MMQ"); + } +} + +// Min. number of src1 columns for the few-row MMQ, 0 if it is not used for this type, GPU and row length. +static int ggml_cuda_mmq_few_min_ncols(const ggml_type type, const int cc, const int64_t ne00) { +#ifdef GGML_CUDA_FORCE_CUBLAS + return 0; +#endif // GGML_CUDA_FORCE_CUBLAS + + // NVIDIA Ampere to Blackwell + if (!ampere_mma_available(cc) || cc >= GGML_CUDA_CC_RUBIN) { + return 0; + } + // on DGX Spark the kernel needs its L2 prefetch, which is only in the DGX Spark device code + if (cc == GGML_CUDA_CC_DGX_SPARK && ggml_cuda_highest_compiled_arch(cc) != GGML_CUDA_CC_DGX_SPARK) { + return 0; + } + // with fewer than 8 units of 256 values per row the warps idle, MMVQ and MMQ are better there + if (mmq_few_unit_size(type) == 0 || ne00 < 8*QK_K) { + return 0; + } + if (cc == GGML_CUDA_CC_DGX_SPARK) { // tuned on DGX Spark GB10 + return type == GGML_TYPE_IQ4_XS || type == GGML_TYPE_IQ4_NL ? 8 : 2; + } + switch (type) { // all GPUs except DGX Spark, tuned on RTX 5090 + case GGML_TYPE_Q3_K: return 3; + case GGML_TYPE_Q5_K: return 3; + case GGML_TYPE_Q4_K: return 4; + case GGML_TYPE_IQ4_NL: return 4; + case GGML_TYPE_Q6_K: return 4; + case GGML_TYPE_Q8_0: return 5; + case GGML_TYPE_IQ3_S: return 5; + case GGML_TYPE_IQ4_XS: return 6; + default: return 0; + } +} + +bool ggml_cuda_should_use_mmq_few(const ggml_tensor * src0, const ggml_tensor * src1, const int cc) { + const int min_ncols = ggml_cuda_mmq_few_min_ncols(src0->type, cc, src0->ne[0]); + if (min_ncols == 0 || src1->ne[1] < min_ncols || src1->ne[1] > MMQ_FEW_MAX_NCG*MMQ_FEW_NCOLS) { + return false; + } + // the row must hold a whole number of ring stages of the launched kernel + const int ncg = mmq_few_ncg(src1->ne[1]); + const int nmt = mmq_few_nmt(src0->type, ncg, src0->ne[1]); + const int nunits_stage = mmq_few_nunits_stage(src0->type, ncg, nmt); + if (src0->ne[0] % (nunits_stage*QK_K) != 0) { + return false; + } + // unaligned data needs the alignment of the decoder loads + const int x_alignment = mmq_few_unaligned(src0->type, nunits_stage) ? (ggml_type_size(src0->type) % 4 == 0 ? 4 : 2) : 16; + return ggml_cuda_is_aligned(src0, x_alignment) && ggml_cuda_is_aligned(src1, 16); +} + +#else // !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA) + +void ggml_cuda_mul_mat_q_few(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { + GGML_UNUSED_VARS(ctx, src0, src1, dst); + GGML_ABORT("few-row MMQ is not supported on HIP or MUSA"); +} + +bool ggml_cuda_should_use_mmq_few(const ggml_tensor * src0, const ggml_tensor * src1, const int cc) { + GGML_UNUSED_VARS(src0, src1, cc); + return false; +} + +#endif // !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA) diff --git a/ggml/src/ggml-cuda/mmq-few.cuh b/ggml/src/ggml-cuda/mmq-few.cuh new file mode 100644 index 000000000000..016834998f72 --- /dev/null +++ b/ggml/src/ggml-cuda/mmq-few.cuh @@ -0,0 +1,7 @@ +#pragma once + +#include "common.cuh" + +bool ggml_cuda_should_use_mmq_few(const ggml_tensor * src0, const ggml_tensor * src1, int cc); + +void ggml_cuda_mul_mat_q_few(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst); diff --git a/ggml/src/ggml-cuda/mmq-load-tiles.cuh b/ggml/src/ggml-cuda/mmq-load-tiles.cuh index 4951db72ebfc..9a5a6a40532a 100644 --- a/ggml/src/ggml-cuda/mmq-load-tiles.cuh +++ b/ggml/src/ggml-cuda/mmq-load-tiles.cuh @@ -629,6 +629,13 @@ template static __device__ __forceinline_ } } +// The 6-bit scales 4*ksc .. 4*ksc + 3 of a q3_K block, minus 32, as int8. +static __device__ __forceinline__ int unpack_scales_q3_K(const uint8_t * scales, const int ksc) { + const int sc_low = (get_int_b2(scales, ksc % 2) >> (4 * (ksc / 2))) & 0x0F0F0F0F; + const int sc_high = ((get_int_b2(scales, 2) >> (2 * ksc)) << 4) & 0x30303030; + return __vsub4(sc_low | sc_high, 0x20202020); +} + template static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_q3_K( const char * __restrict__ x, const char * __restrict__ x_gate, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) { GGML_UNUSED(x_gate); @@ -695,15 +702,7 @@ template static __device__ __forceinline_ const int ksc = threadIdx.x % 4; - const int ksc_low = ksc % (QI3_K/8); - const int shift_low = 4 * (ksc / (QI3_K/8)); - const int sc_low = (get_int_b2(bxi->scales, ksc_low) >> shift_low) & 0x0F0F0F0F; - - const int ksc_high = QI3_K/8; - const int shift_high = 2 * ksc; - const int sc_high = ((get_int_b2(bxi->scales, ksc_high) >> shift_high) << 4) & 0x30303030; - - const int sc = __vsub4(sc_low | sc_high, 0x20202020); + const int sc = unpack_scales_q3_K(bxi->scales, ksc); #if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) const int8_t * sc8 = (const int8_t *) ≻ @@ -1724,10 +1723,8 @@ template static __device__ __forceinline_ } } -template static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_nvfp4( +template static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_nvfp4( const char * __restrict__ x, const char * __restrict__ x_gate, int * __restrict__ x_tile, const int kb0, const int i_max, const int stride) { - GGML_UNUSED(x_gate); - constexpr int warp_size = ggml_cuda_get_physical_warp_size(); constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback, prec_src1) / warp_size; constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback, prec_src1); @@ -1755,7 +1752,7 @@ template static __de i = min(i, i_max); } - const block_nvfp4 * bxi = (const block_nvfp4 *) x + kb0 + i * stride + kbx; + const block_nvfp4 * bxi = ggml_cuda_mmq_x_row(x, x_gate, kb0, i, stride) + kbx; const uint32_t * __restrict__ src_qs = reinterpret_cast(bxi->qs); const int kqs = 16 * kbx; const int ksc = 4 * kbx; @@ -1782,10 +1779,8 @@ template static __de } } -template static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_nvfp4_nvfp4( +template static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_nvfp4_nvfp4( const char * __restrict__ x, const char * __restrict__ x_gate, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) { - GGML_UNUSED(x_gate); - constexpr int warp_size = ggml_cuda_get_physical_warp_size(); constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback, GGML_PREC_Q4) / warp_size; constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback, GGML_PREC_Q4); @@ -1800,7 +1795,6 @@ template static __device__ __forceinline_ const int kbx = txi % threads_per_row; const int row_in_warp = txi / threads_per_row; - const block_nvfp4 * bxi_base = (const block_nvfp4 *) x + kbx0 + kbx; uint32_t * x_u32_scale = x_u32 + 64 + kbx; #pragma unroll @@ -1811,7 +1805,7 @@ template static __device__ __forceinline_ i = min(i, i_max); } - const block_nvfp4 * bxi = bxi_base + i * stride; + const block_nvfp4 * bxi = ggml_cuda_mmq_x_row(x, x_gate, kbx0, i, stride) + kbx; const uint32_t * src_qs = reinterpret_cast(bxi->qs); diff --git a/ggml/src/ggml-cuda/mmq.cu b/ggml/src/ggml-cuda/mmq.cu index 49e0ce9050c0..2e6199766839 100644 --- a/ggml/src/ggml-cuda/mmq.cu +++ b/ggml/src/ggml-cuda/mmq.cu @@ -114,17 +114,17 @@ static ggml_prec ggml_cuda_mmq_get_prec_env() { // src1 is quantized to Q8_1 unless the FP4 types can use 4-bit activations, in which case they // default to the native W4A4 instructions on Blackwell. -static ggml_prec ggml_cuda_mmq_get_prec_src1(const ggml_tensor * src0, const ggml_tensor * dst, const int cc) { +static ggml_prec ggml_cuda_mmq_get_prec_src1(const ggml_type type, const ggml_prec prec_requested, const int cc) { static const ggml_prec prec_env = ggml_cuda_mmq_get_prec_env(); ggml_prec prec = prec_env; if (prec == GGML_PREC_UNDEFINED) { - prec = (ggml_prec) ggml_get_op_params_i32(dst, 3); + prec = prec_requested; } // Q4 only for the FP4 types on Blackwell GGML_ASSERT(prec == GGML_PREC_UNDEFINED || prec == GGML_PREC_Q8 || prec == GGML_PREC_Q4); - const bool can_use_q4 = (src0->type == GGML_TYPE_NVFP4 || src0->type == GGML_TYPE_MXFP4) && blackwell_mma_available(cc); + const bool can_use_q4 = (type == GGML_TYPE_NVFP4 || type == GGML_TYPE_MXFP4) && blackwell_mma_available(cc); if (prec == GGML_PREC_Q8 || !can_use_q4) { return GGML_PREC_Q8; } @@ -169,14 +169,27 @@ void ggml_cuda_mul_mat_q( GGML_ASSERT(!ids || ids->type == GGML_TYPE_I32); // Optional, used for batched GGML_MUL_MAT_ID. // Fused gate/up/GLU: src0 is the up weight, dst the GLU output. + // Fused scale (no gate): dst is the MUL that applies the weight scale to the MUL_MAT. const ggml_tensor * gate = fusion ? fusion->gate : nullptr; ggml_cuda_mm_fusion_args_device fusion_device = {}; if (fusion) { - GGML_ASSERT(gate && !fusion->x_bias && !fusion->gate_bias && !fusion->x_scale && !fusion->gate_scale); - GGML_ASSERT(gate->type == src0->type && ggml_are_same_shape(gate, src0) && ggml_are_same_stride(gate, src0)); - fusion_device.gate = gate->data; - fusion_device.glu_op = fusion->glu_op; - fusion_device.glu_limit = fusion->glu_limit; + GGML_ASSERT(!fusion->x_bias && !fusion->gate_bias); + // NVFP4 weight scales (only applied for NVFP4): one value per tensor, or one per expert with ids. + for (const ggml_tensor * scale : { fusion->x_scale, fusion->gate_scale }) { + GGML_ASSERT(!scale || (src0->type == GGML_TYPE_NVFP4 && scale->type == GGML_TYPE_F32 && ggml_is_contiguous(scale) && + ggml_nelements(scale) == (ids ? src0->ne[2] : 1))); + } + if (gate) { + GGML_ASSERT(!fusion->x_scale == !fusion->gate_scale); + GGML_ASSERT(gate->type == src0->type && ggml_are_same_shape(gate, src0) && ggml_are_same_stride(gate, src0)); + fusion_device.gate = gate->data; + fusion_device.gate_scale = fusion->gate_scale ? fusion->gate_scale->data : nullptr; + fusion_device.glu_op = fusion->glu_op; + fusion_device.glu_limit = fusion->glu_limit; + } else { + GGML_ASSERT(!fusion->gate_scale && fusion->x_scale); + } + fusion_device.x_scale = fusion->x_scale ? fusion->x_scale->data : nullptr; } GGML_TENSOR_BINARY_OP_LOCALS; @@ -224,9 +237,8 @@ void ggml_cuda_mul_mat_q( const bool fallback = ggml_cuda_mmq_needs_fallback(ne01); - // With fusion dst is the GLU output, not a MUL_MAT; the fused types always quantize src1 to Q8_1. - // TODO support for Q4 precision if both tensors allow it - const ggml_prec prec_src1 = gate ? GGML_PREC_Q8 : ggml_cuda_mmq_get_prec_src1(src0, dst, cc); + // With fusion dst is the GLU output or the scale node, not a MUL_MAT; the src1 precision requested by the fused MUL_MATs is in the fusion args. + const ggml_prec prec_src1 = ggml_cuda_mmq_get_prec_src1(src0->type, fusion ? fusion->prec_src1 : ggml_prec_get_src(dst, 1), cc); const bool use_native_fp4 = prec_src1 == GGML_PREC_Q4; const size_t y_block_size = use_native_fp4 ? sizeof(block_fp4_mmq) : sizeof(block_q8_1_mmq); @@ -241,10 +253,7 @@ void ggml_cuda_mul_mat_q( ncols_opt = ne12; // Each expert only sees ne12*n_expert_used/ne02 tokens on average. - // On RDNA3 and RDNA4 it is faster to pick the tile size against this value instead of ne12. - if (GGML_CUDA_CC_IS_RDNA3(cc) || GGML_CUDA_CC_IS_RDNA4_M(cc) || GGML_CUDA_CC_IS_RDNA4(cc)) { - ncols_opt = (ne12*n_expert_used + ne02 - 1) / ne02; - } + ncols_opt = (ne12*n_expert_used + ne02 - 1) / ne02; } J_best = ggml_cuda_mmq_get_J_best(src0->type, ncols_opt, fallback, cc, smpbo, prec_src1, &nthreads_best); @@ -258,6 +267,8 @@ void ggml_cuda_mul_mat_q( * src1_load_chunk_size; if (!ids) { + const int ntj = (ne1 + J_best - 1) / J_best; + const size_t nbytes_src1_q8_1 = ne13*ne12 * ne11*ne10_padded * y_block_size/y_values_per_block + src1_q8_1_padding; ggml_cuda_pool_alloc src1_q8_1(ctx.pool(), nbytes_src1_q8_1); ggml_cuda_pool_alloc src1_scale(ctx.pool()); @@ -295,7 +306,7 @@ void ggml_cuda_mul_mat_q( ne00, ne01, ne1, s01, ne11, s1, ne02, ne12, s02, s12, s2, ne03, ne13, s03, s13, s3, - ne1, J_best, fusion_device}; + ntj, J_best, fusion_device}; ggml_cuda_mul_mat_q_switch_type(ctx, args, stream, prec_src1); return; } @@ -310,7 +321,9 @@ void ggml_cuda_mul_mat_q( ggml_cuda_pool_alloc ids_src1(ctx.pool(), ne_get_rows); ggml_cuda_pool_alloc ids_dst(ctx.pool(), ne_get_rows + J_best-1); // Needs to be padded for unconditional memory access. - ggml_cuda_pool_alloc expert_bounds(ctx.pool(), ne02 + 1); + + const int ntj = (ne_get_rows + J_best - 1) / J_best + ne02; + ggml_cuda_pool_alloc tile_spec(ctx.pool(), ntj); // gate/up activations are broadcast across experts (ne11 == 1): quantize each token once and // scatter to its slots. ids_src1 then holds the inverse map (token slot -> compact row). @@ -321,8 +334,8 @@ void ggml_cuda_mul_mat_q( const int si1 = ids->nb[1] / ggml_element_size(ids); const int sis1 = nb12 / nb11; - ggml_cuda_launch_mm_ids_helper((const int32_t *) ids->data, ids_src1.get(), ids_dst.get(), expert_bounds.get(), - ne02, ne12, n_expert_used, ne11, si1, sis1, /*write_inverse =*/ dedup_bcast, stream); + ggml_cuda_launch_mm_ids_helper((const int32_t *) ids->data, ids_src1.get(), ids_dst.get(), tile_spec.get(), + ne02, ne12, n_expert_used, ne11, J_best, si1, sis1, /*write_inverse =*/ dedup_bcast, stream); CUDA_CHECK(cudaGetLastError()); } @@ -369,12 +382,12 @@ void ggml_cuda_mul_mat_q( // Note that ne02 is used instead of ne12 because the number of y channels determines the z dimension of the CUDA grid. const mmq_args args = { - src0_d, src0->type, (const int *) src1_q8_1.get(), ids_dst.get(), expert_bounds.get(), dst_d, + src0_d, src0->type, (const int *) src1_q8_1.get(), ids_dst.get(), tile_spec.get(), dst_d, src1_scale.ptr, ne00, ne01, ne_get_rows, s01, ne_get_rows, s1, - ne02, ne02, s02, s12, s2, + ne02, 1, s02, s12, s2, ne03, ne13, s03, s13, s3, - ne12, J_best, fusion_device}; + ntj, J_best, fusion_device}; ggml_cuda_mul_mat_q_switch_type(ctx, args, stream, prec_src1); } @@ -513,18 +526,20 @@ bool ggml_cuda_should_use_mmq(enum ggml_type type, int cc, int64_t ne11, int64_t return (!GGML_CUDA_CC_IS_CDNA(cc)) || ne11 < MMQ_DP4A_MAX_BATCH_SIZE; } -bool ggml_cuda_mmq_fusion_is_efficient(const ggml_type type, const int64_t nrows_x, const int64_t ncols, const int64_t nchannels, const int device) { +bool ggml_cuda_mmq_fusion_is_efficient( + const ggml_type type, const ggml_prec prec_requested, const int64_t nrows_x, const int64_t ncols, const int64_t nchannels, const int device) { const int cc = ggml_cuda_info().devices[device].cc; const int nsm = ggml_cuda_info().devices[device].nsm; const size_t smpbo = ggml_cuda_info().devices[device].smpbo; - // Same tile width as ggml_cuda_mul_mat_q; the fused kernel has no fallback, quantizes src1 to Q8_1 and writes I/2 output rows per tile. - const bool fallback = false; - const int J_best = ggml_cuda_mmq_get_J_best(type, ncols, fallback, cc, smpbo, GGML_PREC_Q8); + // Same tile width as ggml_cuda_mul_mat_q; the fused kernel has no fallback and writes I/2 output rows per tile. + const bool fallback = false; + const ggml_prec prec_src1 = ggml_cuda_mmq_get_prec_src1(type, prec_requested, cc); + const int J_best = ggml_cuda_mmq_get_J_best(type, ncols, fallback, cc, smpbo, prec_src1); if (J_best == 0) { return false; } - const int64_t nrows_tile = ggml_cuda_mmq_get_config(type, J_best, fallback, cc, GGML_PREC_Q8).I / 2; + const int64_t nrows_tile = ggml_cuda_mmq_get_config(type, J_best, fallback, cc, prec_src1).I / 2; const int64_t ntiles = (ncols + J_best - 1)/J_best * ((nrows_x + nrows_tile - 1)/nrows_tile) * nchannels; // Same condition as launch_mul_mat_q uses to run the stream-k kernel with one block per tile. const int64_t tiles_nwaves = (ntiles + nsm - 1) / nsm; diff --git a/ggml/src/ggml-cuda/mmq.cuh b/ggml/src/ggml-cuda/mmq.cuh index 308968be9dcb..d645a8aacd4d 100644 --- a/ggml/src/ggml-cuda/mmq.cuh +++ b/ggml/src/ggml-cuda/mmq.cuh @@ -1,6 +1,7 @@ #pragma once #include "common.cuh" +#include "mmid.cuh" #include "unary.cuh" #include @@ -14,9 +15,9 @@ typedef void (*ggml_cuda_mmq_load_tiles_t)(const char * __restrict__ x, const char * __restrict__ x_gate, int * x_tile, const int kbx0, const int i_max, const int stride); typedef void (*ggml_cuda_mmq_vec_dot_t)(const int * __restrict__ x, const int * __restrict__ y, float * __restrict__ sum, const int k00); -typedef void (*ggml_cuda_mmq_write_back_t)(const float * __restrict__ sum, const int32_t * __restrict__ get_rows_to_sorted, - float * __restrict__ dst, const float * __restrict__ y_scale, const int stride, const int i_max, const int j_max, - const ggml_glu_op glu_op, const float glu_limit); +typedef void (*ggml_cuda_mmq_write_back_t)(const float * __restrict__ sum, const int32_t * __restrict__ ids_dst, + float * __restrict__ dst, const float * __restrict__ y_scale, const int stride, const int i_max, const int j_sup, + const ggml_glu_op glu_op, const float glu_limit, const float * __restrict__ x_scale, const float * __restrict__ gate_scale); enum mmq_q8_1_ds_layout { MMQ_Q8_1_DS_LAYOUT_D4, @@ -436,7 +437,8 @@ static __host__ int ggml_cuda_mmq_get_nbytes_shared_x(const ggml_cuda_mmq_config // Weight types with a fused gate/up/GLU path (see ggml_cuda_mmq_get_util_funcs). static constexpr __host__ __device__ bool ggml_cuda_mmq_fusion_supported(const ggml_type type) { - return type == GGML_TYPE_Q4_K || type == GGML_TYPE_Q5_K || type == GGML_TYPE_Q6_K || type == GGML_TYPE_Q8_0 || type == GGML_TYPE_IQ4_XS; + return type == GGML_TYPE_Q4_K || type == GGML_TYPE_Q5_K || type == GGML_TYPE_Q6_K || type == GGML_TYPE_Q8_0 || type == GGML_TYPE_IQ4_XS || + type == GGML_TYPE_NVFP4; } // Fused gate/up/GLU: the weight tile alternates chunks of this many up and gate rows, so rows r and r + 8 of an MMA tile (same thread) pair up. @@ -461,22 +463,25 @@ static __device__ __forceinline__ const block_t * ggml_cuda_mmq_x_row( template static __device__ __forceinline__ void ggml_cuda_mmq_write_back_dp4a( const float * __restrict__ sum, const int32_t * __restrict__ ids_dst, float * __restrict__ dst, - const float * __restrict__ y_scale, const int stride, const int i_max, const int j_max, - const ggml_glu_op glu_op, const float glu_limit) { - GGML_UNUSED_VARS(glu_op, glu_limit); + const float * __restrict__ y_scale, const int stride, const int i_max, const int j_sup, + const ggml_glu_op glu_op, const float glu_limit, const float * __restrict__ x_scale, const float * __restrict__ gate_scale) { + GGML_UNUSED_VARS(glu_op, glu_limit, gate_scale); + + // NVFP4 weight scale of a fused MUL (per tensor, or of the expert of this tile), 1 without. + const float scale = type == GGML_TYPE_NVFP4 && x_scale ? *x_scale : 1.0f; constexpr int warp_size = ggml_cuda_get_physical_warp_size(); constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback, prec_src1) / warp_size; constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback, prec_src1); static_assert(I % warp_size == 0, "DP4A MMQ tile height must be a multiple of the physical warp size"); - const bool y_scale_used = y_scale != nullptr; + const bool y_scale_used = type == GGML_TYPE_NVFP4 && y_scale != nullptr; #pragma unroll for (int j0 = 0; j0 < J; j0 += nwarps) { const int j = j0 + threadIdx.y; - if (j > j_max) { + if (j >= j_sup) { return; } @@ -488,26 +493,26 @@ template static __de continue; } + float value = sum[(j0/nwarps) * (I/warp_size) + i0/warp_size]; if constexpr (type == GGML_TYPE_NVFP4) { if (y_scale_used) { - dst[ids_dst[j]*stride + i] = y_scale[j] * sum[(j0/nwarps) * (I/warp_size) + i0/warp_size]; - } else { - dst[ids_dst[j]*stride + i] = sum[(j0/nwarps) * (I/warp_size) + i0/warp_size]; + value *= y_scale[j]; } } else { - dst[ids_dst[j]*stride + i] = sum[(j0/nwarps) * (I/warp_size) + i0/warp_size]; GGML_UNUSED(y_scale_used); } + dst[ids_dst[j]*stride + i] = value * scale; } } } // has_fusion: accumulators l and l + ne/2 hold the up and gate values of one output element, the GLU of the two is written. +// x_scale/gate_scale: optional NVFP4 scales of the up and gate weights (per tensor, or of the expert of this tile). template static __device__ __forceinline__ void ggml_cuda_mmq_write_back_mma( const float * __restrict__ sum, const int * __restrict__ ids_dst, float * __restrict__ dst, - const float * __restrict__ y_scale, const int stride, const int i_max, const int j_max, - const ggml_glu_op glu_op, const float glu_limit) { + const float * __restrict__ y_scale, const int stride, const int i_max, const int j_sup, + const ggml_glu_op glu_op, const float glu_limit, const float * __restrict__ x_scale, const float * __restrict__ gate_scale) { #if defined(AMD_MFMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) typedef tile<16, 16, int, DATA_LAYOUT_J_MAJOR> tile_C; @@ -521,7 +526,12 @@ static __device__ __forceinline__ void ggml_cuda_mmq_write_back_mma( const int i0 = (threadIdx.y / ntx) * (ntx*tile_C::I); - const bool y_scale_used = y_scale != nullptr; + const bool y_scale_used = type == GGML_TYPE_NVFP4 && y_scale != nullptr; + + // NVFP4: load the weight scales once, 1 without scales. Without fusion x_scale is the scale of a fused MUL. + constexpr bool fused_nvfp4 = has_fusion && type == GGML_TYPE_NVFP4; + const float scale_up = type == GGML_TYPE_NVFP4 && x_scale ? *x_scale : 1.0f; + const float scale_gate = fused_nvfp4 && gate_scale ? *gate_scale : 1.0f; #pragma unroll for (int j0 = 0; j0 < J; j0 += ntx*tile_C::J) { @@ -531,7 +541,7 @@ static __device__ __forceinline__ void ggml_cuda_mmq_write_back_mma( for (int l = 0; l < (has_fusion ? tile_C::ne/2 : tile_C::ne); ++l) { const int j = j0 + (threadIdx.y % ntx) * tile_C::J + tile_C::get_j(l); - if (j > j_max) { + if (j >= j_sup) { continue; } @@ -541,24 +551,35 @@ static __device__ __forceinline__ void ggml_cuda_mmq_write_back_mma( continue; } - if constexpr (has_fusion) { + if (has_fusion) { const int sum_index = (j0/tile_C::J + n)*tile_C::ne + l; - dst[ids_dst[j]*stride + i] = ggml_cuda_op_glu_single(glu_op, sum[sum_index + tile_C::ne/2], sum[sum_index], glu_limit); - } else if constexpr (type == GGML_TYPE_NVFP4) { - if (y_scale_used) { - dst[ids_dst[j]*stride + i] = y_scale[j] * sum[(j0/tile_C::J + n)*tile_C::ne + l]; - } else { - dst[ids_dst[j]*stride + i] = sum[(j0/tile_C::J + n)*tile_C::ne + l]; + float up = sum[sum_index]; + float gate = sum[sum_index + tile_C::ne/2]; + if constexpr (type == GGML_TYPE_NVFP4) { + if (y_scale_used) { + up *= y_scale[j]; + gate *= y_scale[j]; + } + up *= scale_up; + gate *= scale_gate; } + dst[ids_dst[j]*stride + i] = ggml_cuda_op_glu_single(glu_op, gate, up, glu_limit); } else { - dst[ids_dst[j]*stride + i] = sum[(j0/tile_C::J + n)*tile_C::ne + l]; - GGML_UNUSED(y_scale_used); + float value = sum[(j0/tile_C::J + n)*tile_C::ne + l]; + if constexpr (type == GGML_TYPE_NVFP4) { + if (y_scale_used) { + value *= y_scale[j]; + } + } else { + GGML_UNUSED(y_scale_used); + } + dst[ids_dst[j]*stride + i] = value * scale_up; } } } } - GGML_UNUSED_VARS(y_scale_used, glu_op, glu_limit); + GGML_UNUSED_VARS(y_scale_used, glu_op, glu_limit, x_scale, gate_scale, scale_up, scale_gate); } // ------------------------------------------------------------------------------------------------------------------------------------- @@ -712,7 +733,7 @@ static constexpr __device__ ggml_cuda_mmq_util_funcs ggml_cuda_mmq_get_util_func case GGML_TYPE_NVFP4: return ggml_cuda_mmq_util_funcs( VDR_NVFP4_Q8_1_MMQ, - ggml_cuda_mmq_load_tiles_nvfp4, + ggml_cuda_mmq_load_tiles_nvfp4, ggml_cuda_mmq_vec_dot_q8_0_16_q8_1_dp4a, ggml_cuda_mmq_write_back_dp4a); default: @@ -737,9 +758,9 @@ static constexpr __device__ ggml_cuda_mmq_util_funcs ggml_cuda_mmq_get_util_func if (prec_src1 == GGML_PREC_Q4) { return ggml_cuda_mmq_util_funcs( -1, - ggml_cuda_mmq_load_tiles_nvfp4_nvfp4, + ggml_cuda_mmq_load_tiles_nvfp4_nvfp4, ggml_cuda_mmq_vec_dot_fp4_fp4_mma, - ggml_cuda_mmq_write_back_mma); + ggml_cuda_mmq_write_back_mma); } break; default: @@ -882,9 +903,9 @@ static constexpr __device__ ggml_cuda_mmq_util_funcs ggml_cuda_mmq_get_util_func case GGML_TYPE_NVFP4: return ggml_cuda_mmq_util_funcs( -1, - ggml_cuda_mmq_load_tiles_nvfp4, + ggml_cuda_mmq_load_tiles_nvfp4, ggml_cuda_mmq_vec_dot_q8_0_16_q8_1_mma, - ggml_cuda_mmq_write_back_mma); + ggml_cuda_mmq_write_back_mma); default: return ggml_cuda_mmq_util_funcs(1, nullptr, nullptr, nullptr); } @@ -918,8 +939,9 @@ static __device__ __forceinline__ void mul_mat_q_process_tile( const char * __restrict__ x, const char * __restrict__ x_gate, const int offset_x, const int * __restrict__ y, const int * __restrict__ ids_dst, float * __restrict__ dst, float * __restrict__ tmp_fixup, const float * __restrict__ y_scale, const ggml_glu_op glu_op, const float glu_limit, + const float * __restrict__ x_scale, const float * __restrict__ gate_scale, const int stride_row_x, const int ncols_y, const int stride_col_dst, - const int tile_x_max_i, const int tile_y_max_j, const int kb0_start, const int kb0_stop) { + const int tile_x_i_max, const int tile_y_j_sup, const int kb0_start, const int kb0_stop) { static_assert(!(fixup && has_fusion), "the fused gate/up path does not split K"); static_assert(!(fallback && has_fusion), "the fused gate/up path needs nrows_x % 128 == 0"); @@ -952,7 +974,7 @@ static __device__ __forceinline__ void mul_mat_q_process_tile( constexpr int sz = sizeof(block_q8_1_mmq) / sizeof(int); for (int kb0 = kb0_start; kb0 < kb0_stop; kb0 += blocks_per_iter) { - load_tiles(x, x_gate, tile_x, offset_x + kb0, tile_x_max_i, stride_row_x); + load_tiles(x, x_gate, tile_x, offset_x + kb0, tile_x_i_max, stride_row_x); { const int * by0 = y + ncols_y * (kb0 * qk / ne_block) * sz; #pragma unroll @@ -987,9 +1009,9 @@ static __device__ __forceinline__ void mul_mat_q_process_tile( } if (fixup) { - write_back(sum, ids_dst, tmp_fixup + blockIdx.x*(J*I), y_scale, I, I, J, glu_op, glu_limit); + write_back(sum, ids_dst, tmp_fixup + blockIdx.x*(J*I), y_scale, I, I, J, glu_op, glu_limit, x_scale, gate_scale); } else { - write_back(sum, ids_dst, dst, y_scale, stride_col_dst, tile_x_max_i, tile_y_max_j, glu_op, glu_limit); + write_back(sum, ids_dst, dst, y_scale, stride_col_dst, tile_x_i_max, tile_y_j_sup, glu_op, glu_limit, x_scale, gate_scale); } } @@ -1000,21 +1022,28 @@ template ::qk; constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback, prec_src1); - const uint32_t nty = (nrows_x + I - 1) / I; // Number of tiles y + const uint32_t nti = (nrows_x + I - 1) / I; // Number of tiles in I direction // Initialize the ids for writing back data with just the index. // For regular matrix multiplications this is never changed. @@ -1049,35 +1078,28 @@ static __global__ void mul_mat_q( const uint2 tmp2 = fast_div_modulo(blockIdx.z, nchannels_y); const int wt = tmp2.x; - const int zt = tmp2.y; + int zt = tmp2.y; const int jt = blockIdx.x; const int it = blockIdx.y; // Defaults for regular matrix multiplication: - int col_low = 0; - int col_high = ncols_dst; - int col_diff = ncols_dst; + int col_low = jt*J; + int col_high = min(col_low + J, ncols_dst); int offset_y = wt*stride_sample_y + zt*stride_channel_y; + int offset_y_scale = wt*nchannels_y.z*ncols_y + zt*ncols_y; int offset_dst = wt*stride_sample_dst + zt*stride_channel_dst + jt*J*stride_col_dst; - int offset_y_scale; - if constexpr (type == GGML_TYPE_NVFP4) { - offset_y_scale = wt*nchannels_y.z*ncols_y + zt*ncols_y; - } else { - GGML_UNUSED(offset_y_scale); - } if (ids_dst) { - col_low = expert_bounds[zt + 0]; - col_high = expert_bounds[zt + 1]; - col_diff = col_high - col_low; + const ggml_cuda_mm_id_tile_spec tsj = tile_spec[jt]; + zt = tsj.expert; + col_low = tsj.col_low; + col_high = tsj.col_high; - offset_y = 0; - offset_dst = 0; - if constexpr (type == GGML_TYPE_NVFP4) { - offset_y_scale = 0; - } + offset_y = 0; + offset_y_scale = 0; + offset_dst = 0; - if (jt*J >= col_diff) { + if (col_low == col_high) { return; } @@ -1090,29 +1112,30 @@ static __global__ void mul_mat_q( break; } - ids_dst_shared[j] = ids_dst[col_low + jt*J + j]; + ids_dst_shared[j] = ids_dst[col_low + j]; } __syncthreads(); } - offset_y += (col_low + jt*J)*(sizeof(block_q8_1_mmq)/sizeof(int)); - offset_dst += it*I_dst; - const float * y_scale_tile = nullptr; - if constexpr (type == GGML_TYPE_NVFP4) { - offset_y_scale += col_low + jt*J; - y_scale_tile = y_scale ? y_scale + offset_y_scale : nullptr; - } + offset_y += col_low * (sizeof(block_q8_1_mmq)/sizeof(int)); + offset_y_scale += col_low; + offset_dst += it*I_dst; + + const float * y_scale_tile = type == GGML_TYPE_NVFP4 && y_scale ? y_scale + offset_y_scale : nullptr; - const int tile_x_max_i = nrows_x - it*I_dst - 1; - const int tile_y_max_j = col_diff - jt*J - 1; + const int tile_x_i_max = nrows_x - it*I_dst - 1; const int offset_x = fastdiv(wt, sample_ratio)*stride_sample_x + fastdiv(zt, channel_ratio)*stride_channel_x + it*I_dst*stride_row_x; + // With ids the NVFP4 weight scales are per expert. + const float * x_scale_tile = x_scale && ids_dst ? x_scale + zt : x_scale; + const float * gate_scale_tile = gate_scale && ids_dst ? gate_scale + zt : gate_scale; + constexpr bool fixup = false; mul_mat_q_process_tile (x, x_gate, offset_x, y + offset_y, ids_dst_shared, dst + offset_dst, tmp_fixup, y_scale_tile, - glu_op, glu_limit, stride_row_x, ncols_y, stride_col_dst, - tile_x_max_i, tile_y_max_j, 0, blocks_per_ne00.z); + glu_op, glu_limit, x_scale_tile, gate_scale_tile, stride_row_x, ncols_y, stride_col_dst, + tile_x_i_max, col_high - col_low, 0, blocks_per_ne00.z); return; } @@ -1120,8 +1143,8 @@ static __global__ void mul_mat_q( constexpr int blocks_per_iter = ITER_K / qk; // kbc == k block continuous, current index in continuous ijk space. - int kbc = int64_t(blockIdx.x) *(nsamples_y.z*nchannels_y.z*ntx.z*nty*blocks_per_ne00.z) / gridDim.x; - int kbc_stop = int64_t(blockIdx.x + 1)*(nsamples_y.z*nchannels_y.z*ntx.z*nty*blocks_per_ne00.z) / gridDim.x; + int kbc = int64_t(blockIdx.x) *(nsamples_y.z*nchannels_y.z*ntj.z*nti*blocks_per_ne00.z) / gridDim.x; + int kbc_stop = int64_t(blockIdx.x + 1)*(nsamples_y.z*nchannels_y.z*ntj.z*nti*blocks_per_ne00.z) / gridDim.x; kbc -= fastmodulo(kbc, blocks_per_ne00) % blocks_per_iter; kbc_stop -= fastmodulo(kbc_stop, blocks_per_ne00) % blocks_per_iter; @@ -1131,41 +1154,34 @@ static __global__ void mul_mat_q( int kb0_stop = min(blocks_per_ne00.z, uint32_t(kb0_start + kbc_stop - kbc)); while (kbc < kbc_stop && kb0_stop == int(blocks_per_ne00.z)) { int tmp = fastdiv(kbc, blocks_per_ne00); - uint2 tmp2 = fast_div_modulo(tmp, ntx); + uint2 tmp2 = fast_div_modulo(tmp, ntj); const int jt = tmp2.y; tmp = tmp2.x; tmp2 = fast_div_modulo(tmp, nchannels_y); - const int zt = tmp2.y; + int zt = tmp2.y; tmp = tmp2.x; tmp2 = fast_div_modulo(tmp, nsamples_y); const int wt = tmp2.y; const int it = tmp2.x; // Defaults for regular matrix multiplication: - int col_low = 0; - int col_high = ncols_dst; - int col_diff = ncols_dst; + int col_low = jt*J; + int col_high = min(col_low + J, ncols_dst); int offset_y = wt*stride_sample_y + zt*stride_channel_y; + int offset_y_scale = wt*nchannels_y.z*ncols_y + zt*ncols_y; int offset_dst = wt*stride_sample_dst + zt*stride_channel_dst + jt*J*stride_col_dst; - int offset_y_scale; - if constexpr (type == GGML_TYPE_NVFP4) { - offset_y_scale = wt*nchannels_y.z*ncols_y + zt*ncols_y; - } else { - GGML_UNUSED(offset_y_scale); - } if (ids_dst) { - col_low = expert_bounds[zt + 0]; - col_high = expert_bounds[zt + 1]; - col_diff = col_high - col_low; + const ggml_cuda_mm_id_tile_spec tsj = tile_spec[jt]; + zt = tsj.expert; + col_low = tsj.col_low; + col_high = tsj.col_high; - offset_y = 0; - offset_dst = 0; - if constexpr (type == GGML_TYPE_NVFP4) { - offset_y_scale = 0; - } + offset_y = 0; + offset_y_scale = 0; + offset_dst = 0; - if (jt*J >= col_diff) { + if (col_low == col_high) { kbc += blocks_per_ne00.z; kbc -= fastmodulo(kbc, blocks_per_ne00); @@ -1184,29 +1200,28 @@ static __global__ void mul_mat_q( break; } - ids_dst_shared[j] = ids_dst[col_low + jt*J + j]; + ids_dst_shared[j] = ids_dst[col_low + j]; } __syncthreads(); } - offset_y += (col_low + jt * J) * (sizeof(block_q8_1_mmq) / sizeof(int)); - offset_dst += it*I; - const float * y_scale_tile = nullptr; - if constexpr (type == GGML_TYPE_NVFP4) { - offset_y_scale += col_low + jt * J; - y_scale_tile = y_scale ? y_scale + offset_y_scale : nullptr; - } + offset_y += col_low * (sizeof(block_q8_1_mmq) / sizeof(int)); + offset_y_scale += col_low; + offset_dst += it*I; + + const float * y_scale_tile = type == GGML_TYPE_NVFP4 && y_scale ? y_scale + offset_y_scale : nullptr; - const int tile_x_max_i = nrows_x - it*I - 1; - const int tile_y_max_j = col_diff - jt*J - 1; + const int tile_x_i_max = nrows_x - it*I - 1; const int offset_x = fastdiv(wt, sample_ratio)*stride_sample_x + fastdiv(zt, channel_ratio)*stride_channel_x + it*I*stride_row_x; + const float * x_scale_tile = x_scale && ids_dst ? x_scale + zt : x_scale; + constexpr bool fixup = false; // All but (potentially) the last iterations write their data to dst rather than the fixup buffer. mul_mat_q_process_tile (x, x_gate, offset_x, y + offset_y, ids_dst_shared, dst + offset_dst, tmp_fixup, y_scale_tile, - glu_op, glu_limit, stride_row_x, ncols_y, stride_col_dst, - tile_x_max_i, tile_y_max_j, kb0_start, kb0_stop); + glu_op, glu_limit, x_scale_tile, gate_scale, stride_row_x, ncols_y, stride_col_dst, + tile_x_i_max, col_high - col_low, kb0_start, kb0_stop); kbc += blocks_per_ne00.z; kbc -= fastmodulo(kbc, blocks_per_ne00); @@ -1220,41 +1235,34 @@ static __global__ void mul_mat_q( } int tmp = fastdiv(kbc, blocks_per_ne00); - uint2 tmp2 = fast_div_modulo(tmp, ntx); + uint2 tmp2 = fast_div_modulo(tmp, ntj); const int jt = tmp2.y; tmp = tmp2.x; tmp2 = fast_div_modulo(tmp, nchannels_y); - const int zt = tmp2.y; + int zt = tmp2.y; tmp = tmp2.x; tmp2 = fast_div_modulo(tmp, nsamples_y); const int wt = tmp2.y; const int it = tmp2.x; // Defaults for regular matrix multiplication: - int col_low = 0; - int col_high = ncols_dst; - int col_diff = ncols_dst; + int col_low = jt*J; + int col_high = min(col_low + J, ncols_dst); int offset_y = wt*stride_sample_y + zt*stride_channel_y; + int offset_y_scale = wt*nchannels_y.z*ncols_y + zt*ncols_y; int offset_dst = wt*stride_sample_dst + zt*stride_channel_dst + jt*J*stride_col_dst; - int offset_y_scale; - if constexpr (type == GGML_TYPE_NVFP4) { - offset_y_scale = wt*nchannels_y.z*ncols_y + zt*ncols_y; - } else { - GGML_UNUSED(offset_y_scale); - } if (ids_dst) { - col_low = expert_bounds[zt + 0]; - col_high = expert_bounds[zt + 1]; - col_diff = col_high - col_low; + const ggml_cuda_mm_id_tile_spec tsj = tile_spec[jt]; + zt = tsj.expert; + col_low = tsj.col_low; + col_high = tsj.col_high; - offset_y = 0; - offset_dst = 0; - if constexpr (type == GGML_TYPE_NVFP4) { - offset_y_scale = 0; - } + offset_y = 0; + offset_y_scale = 0; + offset_dst = 0; - if (jt*J >= col_diff) { + if (col_low == col_high) { return; } @@ -1273,33 +1281,32 @@ static __global__ void mul_mat_q( __syncthreads(); } - offset_y += (col_low + jt * J) * (sizeof(block_q8_1_mmq) / sizeof(int)); - offset_dst += it*I; - const float * y_scale_tile = nullptr; - if constexpr (type == GGML_TYPE_NVFP4) { - offset_y_scale += col_low + jt * J; - y_scale_tile = y_scale ? y_scale + offset_y_scale : nullptr; - } + offset_y += col_low * (sizeof(block_q8_1_mmq) / sizeof(int)); + offset_y_scale += col_low; + offset_dst += it*I; + + const float * y_scale_tile = type == GGML_TYPE_NVFP4 && y_scale ? y_scale + offset_y_scale : nullptr; - const int tile_x_max_i = nrows_x - it*I - 1; - const int tile_y_max_j = col_diff - jt*J - 1; + const int tile_x_i_max = nrows_x - it*I - 1; const int offset_x = fastdiv(wt, sample_ratio)*stride_sample_x + fastdiv(zt, channel_ratio)*stride_channel_x + it*I*stride_row_x; + const float * x_scale_tile = x_scale && ids_dst ? x_scale + zt : x_scale; + constexpr bool fixup = true; // Last index writes its data to fixup buffer to avoid data races with other blocks. mul_mat_q_process_tile (x, x_gate, offset_x, y + offset_y, ids_dst_shared, dst + offset_dst, tmp_fixup, y_scale_tile, - glu_op, glu_limit, stride_row_x, ncols_y, stride_col_dst, - tile_x_max_i, tile_y_max_j, kb0_start, kb0_stop); + glu_op, glu_limit, x_scale_tile, gate_scale, stride_row_x, ncols_y, stride_col_dst, + tile_x_i_max, col_high - col_low, kb0_start, kb0_stop); } template __launch_bounds__(ggml_cuda_mmq_get_nthreads(type, J, fallback, prec_src1)/2, 1) static __global__ void mul_mat_q_stream_k_fixup( - const int32_t * __restrict__ ids_dst, const int32_t * __restrict__ expert_bounds, float * __restrict__ dst, + const int32_t * __restrict__ ids_dst, const ggml_cuda_mm_id_tile_spec * __restrict__ tile_spec, float * __restrict__ dst, float * __restrict__ tmp_last_tile, const uint3 blocks_per_ne00, const int nrows_x, const int ncols_dst, const int stride_col_dst, const uint3 nchannels_y, const int stride_channel_dst, const uint3 nsamples_y, - const int stride_sample_dst, const uint3 ntx) { + const int stride_sample_dst, const uint3 ntj) { constexpr int warp_size = ggml_cuda_get_physical_warp_size(); constexpr int nwarps = (ggml_cuda_mmq_get_nthreads(type, J, fallback, prec_src1) / 2) / warp_size; constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback, prec_src1); @@ -1310,13 +1317,13 @@ static __global__ void mul_mat_q_stream_k_fixup( float sum[J / nwarps] = {0.0f}; const int i = blockIdx.y*warp_size + threadIdx.x; - const int nty = (nrows_x + I - 1) / I; + const int nti = (nrows_x + I - 1) / I; const int bidx0 = blockIdx.x; // kbc == k block continuous, current index in continuous ijk space. - int kbc0 = int64_t(blockIdx.x) *(nsamples_y.z*nchannels_y.z*ntx.z*nty*blocks_per_ne00.z) / gridDim.x; - int kbc0_stop = int64_t(blockIdx.x + 1)*(nsamples_y.z*nchannels_y.z*ntx.z*nty*blocks_per_ne00.z) / gridDim.x; + int kbc0 = int64_t(blockIdx.x) *(nsamples_y.z*nchannels_y.z*ntj.z*nti*blocks_per_ne00.z) / gridDim.x; + int kbc0_stop = int64_t(blockIdx.x + 1)*(nsamples_y.z*nchannels_y.z*ntj.z*nti*blocks_per_ne00.z) / gridDim.x; kbc0 -= fastmodulo(kbc0, blocks_per_ne00) % blocks_per_iter; kbc0_stop -= fastmodulo(kbc0_stop, blocks_per_ne00) % blocks_per_iter; @@ -1335,7 +1342,7 @@ static __global__ void mul_mat_q_stream_k_fixup( int bidx = bidx0 - 1; int kbc_stop = kbc0; while(true) { - int kbc = int64_t(bidx)*(nsamples_y.z*nchannels_y.z*ntx.z*nty*blocks_per_ne00.z) / gridDim.x; + int kbc = int64_t(bidx)*(nsamples_y.z*nchannels_y.z*ntj.z*nti*blocks_per_ne00.z) / gridDim.x; kbc -= fastmodulo(kbc, blocks_per_ne00) % blocks_per_iter; if (kbc == kbc_stop) { // Did not have any data. @@ -1367,11 +1374,11 @@ static __global__ void mul_mat_q_stream_k_fixup( } int tmp = fastdiv(kbc0, blocks_per_ne00); - uint2 tmp2 = fast_div_modulo(tmp, ntx); + uint2 tmp2 = fast_div_modulo(tmp, ntj); const int jt = tmp2.y; tmp = tmp2.x; tmp2 = fast_div_modulo(tmp, nchannels_y); - const int zt = tmp2.y; + int zt = tmp2.y; tmp = tmp2.x; tmp2 = fast_div_modulo(tmp, nsamples_y); const int wt = tmp2.y; @@ -1382,7 +1389,7 @@ static __global__ void mul_mat_q_stream_k_fixup( dst += offset_dst; const int i_max = nrows_x - it*I - 1; - const int j_max = ncols_dst - jt*J - 1; + const int j_sup = ncols_dst - jt*J; if (fallback && i > i_max) { return; } @@ -1391,7 +1398,7 @@ static __global__ void mul_mat_q_stream_k_fixup( for (int j0 = 0; j0 < J; j0 += nwarps) { const int j = j0 + threadIdx.y; - if (j > j_max) { + if (j >= j_sup) { return; } @@ -1401,12 +1408,18 @@ static __global__ void mul_mat_q_stream_k_fixup( } __shared__ int ids_dst_shared[J]; - const int col_low = expert_bounds[zt + 0]; - const int col_high = expert_bounds[zt + 1]; - const int col_diff = col_high - col_low; + const ggml_cuda_mm_id_tile_spec tsj = tile_spec[jt]; + zt = tsj.expert; + const int col_low = tsj.col_low; + const int col_high = tsj.col_high; + const int j_sup = col_high - col_low; + + if (j_sup <= 0) { + return; + } for (int j = threadIdx.y*warp_size + threadIdx.x; j < J; j += nwarps*warp_size) { - ids_dst_shared[j] = ids_dst[col_low + jt*J + j]; + ids_dst_shared[j] = ids_dst[col_low + j]; } __syncthreads(); @@ -1414,7 +1427,6 @@ static __global__ void mul_mat_q_stream_k_fixup( dst += offset_dst; const int i_max = nrows_x - it*I - 1; - const int j_max = col_diff - jt*J - 1; if (fallback && i > i_max) { return; } @@ -1423,7 +1435,7 @@ static __global__ void mul_mat_q_stream_k_fixup( for (int j0 = 0; j0 < J; j0 += nwarps) { const int j = j0 + threadIdx.y; - if (j > j_max) { + if (j >= j_sup) { return; } @@ -1432,12 +1444,12 @@ static __global__ void mul_mat_q_stream_k_fixup( } struct mmq_args { - const char * x; ggml_type type_x; const int * y; const int32_t * ids_dst; const int32_t * expert_bounds; float * dst; - const float * y_scale; + const char * x; ggml_type type_x; const int * y; const int32_t * ids_dst; + const ggml_cuda_mm_id_tile_spec * tile_spec; float * dst; const float * y_scale; int64_t ncols_x; int64_t nrows_x; int64_t ncols_dst; int64_t stride_row_x; int64_t ncols_y; int64_t nrows_dst; int64_t nchannels_x; int64_t nchannels_y; int64_t stride_channel_x; int64_t stride_channel_y; int64_t stride_channel_dst; int64_t nsamples_x; int64_t nsamples_y; int64_t stride_sample_x; int64_t stride_sample_y; int64_t stride_sample_dst; - int64_t ncols_max; + int64_t ntj; int J_best; // Tile width in ne11(dense)/ne12(MoE) direction to use for optimal performance. ggml_cuda_mm_fusion_args_device fusion = {}; // gate != nullptr selects the fused gate/up/GLU path }; @@ -1469,38 +1481,38 @@ static void launch_mul_mat_q(ggml_backend_cuda_context & ctx, const mmq_args & a } const int I_dst = has_fusion ? config.I/2 : config.I; // Rows of dst per tile, see mul_mat_q. - const int nty = (args.nrows_x + I_dst - 1) / I_dst; - const int ntx = (args.ncols_max + config.J - 1) / config.J; - const int ntzw = args.nchannels_y * args.nsamples_y; + const int nti = (args.nrows_x + I_dst - 1) / I_dst; + const int ntzw = args.nchannels_y * args.nsamples_y; // Column tiles innermost so that the concurrently running blocks share the same tile of x and read it from L2. - const dim3 block_nums_xy_tiling(ntx, nty, ntzw); + const dim3 block_nums_xy_tiling(args.ntj, nti, ntzw); - GGML_ASSERT(args.nchannels_y % args.nchannels_x == 0); - GGML_ASSERT(args.nsamples_y % args.nsamples_x == 0); - const int channel_ratio = args.nchannels_y / args.nchannels_x; - const int sample_ratio = args.nsamples_y / args.nsamples_x; + GGML_ASSERT(args.ids_dst || args.nchannels_y % args.nchannels_x == 0); + GGML_ASSERT( args.nsamples_y % args.nsamples_x == 0); + const int channel_ratio = args.ids_dst ? 1 : args.nchannels_y / args.nchannels_x; + const int sample_ratio = args.nsamples_y / args.nsamples_x; const uint3 blocks_per_ne00_fd = init_fastdiv_values(args.ncols_x / ggml_cuda_type_traits::qk); - const uint3 ntx_fd = init_fastdiv_values(ntx); + const uint3 ntj_fd = init_fastdiv_values(args.ntj); const uint3 nchannels_y_fd = init_fastdiv_values(args.nchannels_y); const uint3 nsamples_y_fd = init_fastdiv_values(args.nsamples_y); const uint3 channel_ratio_fd = init_fastdiv_values(channel_ratio); const uint3 sample_ratio_fd = init_fastdiv_values(sample_ratio); if (has_fusion || !config.stream_k) { - GGML_ASSERT(nty <= 65535); // gridDim.y limit + GGML_ASSERT(nti <= 65535); // gridDim.y limit mul_mat_q<<>> - (args.x, (const char *) args.fusion.gate, args.y, args.ids_dst, args.expert_bounds, args.dst, nullptr, args.y_scale, args.fusion.glu_op, args.fusion.glu_limit, + (args.x, (const char *) args.fusion.gate, args.y, args.ids_dst, args.tile_spec, args.dst, nullptr, + args.y_scale, args.fusion.glu_op, args.fusion.glu_limit, (const float *) args.fusion.x_scale, (const float *) args.fusion.gate_scale, blocks_per_ne00_fd, args.nrows_x, args.ncols_dst, args.stride_row_x, args.ncols_y, args.nrows_dst, channel_ratio_fd, nchannels_y_fd, args.stride_channel_x, args.stride_channel_y, args.stride_channel_dst, sample_ratio_fd, nsamples_y_fd, args.stride_sample_x, args.stride_sample_y, args.stride_sample_dst, - ntx_fd); + ntj_fd); return; } // For the stream-k kernel it is possible to run it with tiling by setting the number of CUDA blocks equal to the number of tiles. // This is worthwhile if the efficiency of tiling is high and skipping the fixup kernel is more important. - const int ntiles_dst = ntx * nty * ntzw; + const int ntiles_dst = args.ntj * nti * ntzw; const int tiles_nwaves = (ntiles_dst + nsm - 1) / nsm; const int tiles_efficiency_percent = 100 * ntiles_dst / (nsm*tiles_nwaves); @@ -1521,11 +1533,12 @@ static void launch_mul_mat_q(ggml_backend_cuda_context & ctx, const mmq_args & a const dim3 block_dims_fixup(block_dims.x, block_dims.y/2, block_dims.z); mul_mat_q<<>> - (args.x, (const char *) args.fusion.gate, args.y, args.ids_dst, args.expert_bounds, args.dst, tmp_fixup.ptr, args.y_scale, args.fusion.glu_op, args.fusion.glu_limit, + (args.x, (const char *) args.fusion.gate, args.y, args.ids_dst, args.tile_spec, args.dst, tmp_fixup.ptr, + args.y_scale, args.fusion.glu_op, args.fusion.glu_limit, (const float *) args.fusion.x_scale, (const float *) args.fusion.gate_scale, blocks_per_ne00_fd, args.nrows_x, args.ncols_dst, args.stride_row_x, args.ncols_y, args.nrows_dst, channel_ratio_fd, nchannels_y_fd, args.stride_channel_x, args.stride_channel_y, args.stride_channel_dst, sample_ratio_fd, nsamples_y_fd, args.stride_sample_x, args.stride_sample_y, args.stride_sample_dst, - ntx_fd); + ntj_fd); if (!fixup_needed) { return; @@ -1533,9 +1546,9 @@ static void launch_mul_mat_q(ggml_backend_cuda_context & ctx, const mmq_args & a CUDA_CHECK(cudaGetLastError()); mul_mat_q_stream_k_fixup<<>> - (args.ids_dst, args.expert_bounds, args.dst, tmp_fixup.ptr, blocks_per_ne00_fd, args.nrows_x, args.ncols_dst, + (args.ids_dst, args.tile_spec, args.dst, tmp_fixup.ptr, blocks_per_ne00_fd, args.nrows_x, args.ncols_dst, args.nrows_dst, nchannels_y_fd, args.stride_channel_dst, nsamples_y_fd, args.stride_sample_dst, - ntx_fd); + ntj_fd); } template @@ -1661,4 +1674,5 @@ void ggml_cuda_mul_mat_q( bool ggml_cuda_should_use_mmq(enum ggml_type type, int cc, int64_t ne11, int64_t n_experts); -bool ggml_cuda_mmq_fusion_is_efficient(ggml_type type, int64_t nrows_x, int64_t ncols, int64_t nchannels, int device); +// prec_requested: the src1 precision requested by the fused MUL_MATs. +bool ggml_cuda_mmq_fusion_is_efficient(ggml_type type, ggml_prec prec_requested, int64_t nrows_x, int64_t ncols, int64_t nchannels, int device); diff --git a/ggml/src/ggml-cuda/mmvf.cu b/ggml/src/ggml-cuda/mmvf.cu index a62bf53e8472..154a621fac92 100644 --- a/ggml/src/ggml-cuda/mmvf.cu +++ b/ggml/src/ggml-cuda/mmvf.cu @@ -5,6 +5,14 @@ #include "mmf.cuh" #include "convert.cuh" +static __device__ __forceinline__ float2 mmvf_load_fp8x2(const ggml_fp8_e4m3_t * x, const int col2) { +#if defined(FP8_AVAILABLE) && !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA) && defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 800 + return ggml_cuda_cast(reinterpret_cast(x)[col2]); +#else + return make_float2(ggml_cuda_cast(x[2*col2]), ggml_cuda_cast(x[2*col2 + 1])); +#endif +} + template static __global__ void mul_mat_vec_f( const T * x_ptr, const float * y_ptr, const int32_t * ids_ptr, const ggml_cuda_mm_fusion_args_device fusion, float * dst_ptr, @@ -56,16 +64,24 @@ static __global__ void mul_mat_vec_f( bool use_gate = false; bool use_bias = false; bool use_gate_bias = false; + bool use_scale = false; + bool use_gate_scale = false; ggml_glu_op glu_op = ggml_glu_op::GGML_GLU_OP_SWIGLU; float glu_limit = 0.0f; const T * gate_x = nullptr; const float * x_bias = nullptr; const float * gate_bias = nullptr; + const float * x_scale = nullptr; + const float * gate_scale = nullptr; if constexpr (has_fusion) { use_gate = fusion.gate != nullptr; use_bias = fusion.x_bias != nullptr; use_gate_bias = fusion.gate_bias != nullptr; + if constexpr (std::is_same_v) { + use_scale = fusion.x_scale != nullptr; + use_gate_scale = fusion.gate_scale != nullptr && use_gate; + } glu_op = fusion.glu_op; glu_limit = fusion.glu_limit; @@ -81,6 +97,12 @@ static __global__ void mul_mat_vec_f( } else { use_gate_bias = false; } + if (use_scale) { + x_scale = static_cast(fusion.x_scale); + } + if (use_gate_scale) { + gate_scale = static_cast(fusion.gate_scale); + } } if (use_gate) { @@ -97,8 +119,6 @@ static __global__ void mul_mat_vec_f( } } - const float2 * y2 = (const float2 *) y; - extern __shared__ char data_mmv[]; float * buf_iw = (float *) data_mmv; [[maybe_unused]] float * buf_iw_gate = nullptr; @@ -128,6 +148,7 @@ static __global__ void mul_mat_vec_f( } if constexpr (std::is_same_v) { + const float2 * y2 = (const float2 *) y; const float2 * x2 = (const float2 *) x; [[maybe_unused]] const float2 * gate_x2 = nullptr; if constexpr (has_fusion) { @@ -160,6 +181,7 @@ static __global__ void mul_mat_vec_f( } } } else if constexpr (std::is_same_v) { + const float2 * y2 = (const float2 *) y; const half2 * x2 = (const half2 *) x; [[maybe_unused]] const half2 * gate_x2 = nullptr; if constexpr (has_fusion) { @@ -235,6 +257,7 @@ static __global__ void mul_mat_vec_f( #endif // FP16_AVAILABLE } } else if constexpr (std::is_same_v) { + const float2 * y2 = (const float2 *) y; //TODO: add support for ggml_cuda_mad for hip_bfloat162 #if defined(GGML_USE_HIP) const int * x2 = (const int *) x; @@ -301,6 +324,28 @@ static __global__ void mul_mat_vec_f( } } #endif + } else if constexpr (std::is_same_v) { + const float2 * y2 = (const float2 *) y; + for (int col2 = tid; col2 < ncols2; col2 += block_size) { + const float2 tmpx = mmvf_load_fp8x2(x, col2); + float2 tmpx_gate = {}; + if constexpr (has_fusion) { + if (use_gate) { + tmpx_gate = mmvf_load_fp8x2(gate_x, col2); + } + } +#pragma unroll + for (int j = 0; j < ncols_dst; ++j) { + const float2 tmpy = y2[j*stride_col_y2 + col2]; + ggml_cuda_mad(sumf[j], tmpx, tmpy); + + if constexpr (has_fusion) { + if (use_gate) { + ggml_cuda_mad(sumf_gate[j], tmpx_gate, tmpy); + } + } + } + } } else { static_assert(std::is_same_v, "unsupported type"); } @@ -348,12 +393,18 @@ static __global__ void mul_mat_vec_f( float value = sumf[tid]; if constexpr (has_fusion) { + if (use_scale) { + value *= x_scale[ids ? channel_x : 0]; + } if (use_bias) { value += x_bias[tid*stride_col_dst + row]; } if (use_gate) { float gate_value = sumf_gate[tid]; + if (use_gate_scale) { + gate_value *= gate_scale[ids ? channel_x : 0]; + } if (use_gate_bias) { gate_value += gate_bias[tid*stride_col_dst + row]; } @@ -380,7 +431,11 @@ static __global__ void mul_mat_vec_f( dst[tid*stride_col_dst + row] = value; if constexpr (!has_fusion) { - GGML_UNUSED_VARS(use_gate, use_bias, use_gate_bias, glu_op, glu_limit, gate_x, x_bias, gate_bias, sumf_gate); + GGML_UNUSED_VARS(use_gate, use_bias, use_gate_bias, use_scale, use_gate_scale, glu_op, glu_limit, gate_x, x_bias, gate_bias, + x_scale, gate_scale, sumf_gate); + } + if constexpr (!std::is_same_v) { + GGML_UNUSED_VARS(use_scale, use_gate_scale, x_scale, gate_scale); } } @@ -395,7 +450,8 @@ static void mul_mat_vec_f_switch_fusion( const ggml_cuda_kernel_launch_params launch_params = {block_nums, block_dims, nbytes_shared, stream}; - const bool has_fusion = fusion.gate != nullptr || fusion.x_bias != nullptr || fusion.gate_bias != nullptr; + const bool has_fusion = fusion.gate != nullptr || fusion.x_bias != nullptr || fusion.gate_bias != nullptr || + fusion.x_scale != nullptr || fusion.gate_scale != nullptr; if constexpr (ncols_dst == 1) { if (has_fusion) { ggml_cuda_kernel_launch(mul_mat_vec_f, launch_params, @@ -616,7 +672,7 @@ static void mul_mat_vec_f_cuda( const int64_t ids_stride, enum ggml_prec prec, cudaStream_t stream) { if constexpr(std::is_same_v) { - if (prec == GGML_PREC_DEFAULT) { + if (prec == GGML_PREC_UNDEFINED) { mul_mat_vec_f_cuda_switch_ncols_dst (x, y, ids, fusion, dst, ncols, nrows, ncols_dst, stride_row, stride_col_y, stride_col_dst, nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, @@ -649,9 +705,11 @@ void ggml_cuda_mul_mat_vec_f(ggml_backend_cuda_context & ctx, const ggml_tensor GGML_ASSERT( nb10 == ts_src1); GGML_ASSERT(!ids || ids->nb[0] == ggml_type_size(ids->type)); GGML_ASSERT( nb0 == ts_dst); + GGML_ASSERT(src0->type != GGML_TYPE_F8_E4M3 || (uintptr_t) src0->data % alignof(uint16_t) == 0); const int cc = ggml_cuda_info().devices[ggml_cuda_get_device()].cc; - const enum ggml_prec prec = fast_fp16_available(cc) ? ggml_prec(dst->op_params[0]) : GGML_PREC_F32; + // the fused GLU path gets the GLU node as dst, which carries no matmul precision, so it accumulates in F32 + const enum ggml_prec prec = fast_fp16_available(cc) && !(fusion && fusion->gate) ? ggml_prec_get_acc(dst) : GGML_PREC_F32; const float * src1_d = (const float *) src1->data; const int32_t * ids_d = ids ? (const int32_t *) ids->data : nullptr; @@ -662,6 +720,7 @@ void ggml_cuda_mul_mat_vec_f(ggml_backend_cuda_context & ctx, const ggml_tensor if (fusion) { GGML_ASSERT( !ids || dst->ne[2] == 1); GGML_ASSERT( ids || dst->ne[1] == 1); + GGML_ASSERT((fusion->x_scale == nullptr && fusion->gate_scale == nullptr) || src0->type == GGML_TYPE_F8_E4M3); if (fusion->x_bias) { GGML_ASSERT(fusion->x_bias->type == GGML_TYPE_F32); GGML_ASSERT(fusion->x_bias->ne[0] == dst->ne[0]); @@ -670,6 +729,7 @@ void ggml_cuda_mul_mat_vec_f(ggml_backend_cuda_context & ctx, const ggml_tensor } if (fusion->gate) { GGML_ASSERT(fusion->gate->type == src0->type && ggml_are_same_stride(fusion->gate, src0)); + GGML_ASSERT(src0->type != GGML_TYPE_F8_E4M3 || (uintptr_t) fusion->gate->data % alignof(uint16_t) == 0); fusion_local.gate = fusion->gate->data; } if (fusion->gate_bias) { @@ -678,6 +738,18 @@ void ggml_cuda_mul_mat_vec_f(ggml_backend_cuda_context & ctx, const ggml_tensor GGML_ASSERT(!ids || fusion->gate_bias->ne[1] == src0->ne[2]); fusion_local.gate_bias = fusion->gate_bias->data; } + if (fusion->x_scale) { + GGML_ASSERT(fusion->x_scale->type == GGML_TYPE_F32); + GGML_ASSERT(ggml_is_contiguous(fusion->x_scale)); + GGML_ASSERT(ggml_nelements(fusion->x_scale) == (ids ? src0->ne[2] : 1)); + fusion_local.x_scale = fusion->x_scale->data; + } + if (fusion->gate_scale) { + GGML_ASSERT(fusion->gate_scale->type == GGML_TYPE_F32); + GGML_ASSERT(ggml_is_contiguous(fusion->gate_scale)); + GGML_ASSERT(ggml_nelements(fusion->gate_scale) == (ids ? src0->ne[2] : 1)); + fusion_local.gate_scale = fusion->gate_scale->data; + } fusion_local.glu_op = fusion->glu_op; fusion_local.glu_limit = fusion->glu_limit; } @@ -722,6 +794,12 @@ void ggml_cuda_mul_mat_vec_f(ggml_backend_cuda_context & ctx, const ggml_tensor ne02, nchannels_y, nchannels_dst, s02, stride_channel_y, stride_channel_dst, ne03, ne3, s03, s13, s3, ids_stride, prec, ctx.stream()); } break; + case GGML_TYPE_F8_E4M3: { + const ggml_fp8_e4m3_t * src0_d = (const ggml_fp8_e4m3_t *) src0->data; + mul_mat_vec_f_cuda(src0_d, src1_d, ids_d, fusion_local, dst_d, ne00, ne01, ncols_dst, s01, stride_col_y, stride_col_dst, + ne02, nchannels_y, nchannels_dst, s02, stride_channel_y, stride_channel_dst, + ne03, ne3, s03, s13, s3, ids_stride, prec, ctx.stream()); + } break; default: GGML_ABORT("unsupported type: %s", ggml_type_name(src0->type)); } @@ -743,7 +821,7 @@ void ggml_cuda_op_mul_mat_vec_f( const int id = ggml_cuda_get_device(); const int cc = ggml_cuda_info().devices[id].cc; - const enum ggml_prec prec = fast_fp16_available(cc) ? ggml_prec(dst->op_params[0]) : GGML_PREC_F32; + const enum ggml_prec prec = fast_fp16_available(cc) ? ggml_prec_get_acc(dst) : GGML_PREC_F32; // ggml_cuda_op provides single, contiguous matrices const int64_t stride_row = ne00; @@ -788,7 +866,11 @@ void ggml_cuda_op_mul_mat_vec_f( GGML_UNUSED_VARS(ctx, src1, dst, src1_ddq_i, src1_ncols, src1_padded_row_size); } -bool ggml_cuda_should_use_mmvf(enum ggml_type type, int cc, int warp_size, const int64_t * src0_ne, const size_t * src0_nb, int64_t ne11) { +bool ggml_cuda_should_use_mmvf(const ggml_tensor * src0, int cc, int warp_size, int64_t ne11) { + const ggml_type type = src0->type; + const int64_t * src0_ne = src0->ne; + const size_t * src0_nb = src0->nb; + if (src0_ne[0] % 2 != 0) { return false; } @@ -798,11 +880,8 @@ bool ggml_cuda_should_use_mmvf(enum ggml_type type, int cc, int warp_size, const return false; } - // Pointers not aligned to the size of half2/nv_bfloat162/float2 would result in a crash: - for (size_t i = 1; i < GGML_MAX_DIMS; ++i) { - if (src0_nb[i] % (2*ts) != 0) { - return false; - } + if (!ggml_cuda_is_aligned(src0, 2*ts)) { + return false; } switch (type) { @@ -862,6 +941,18 @@ bool ggml_cuda_should_use_mmvf(enum ggml_type type, int cc, int warp_size, const if (src0_small && !ggml_cuda_should_use_mmf(type, cc, warp_size, src0_ne, src0_nb, ne11, /*mul_mat_id =*/ false)) { return ne11 <= MMVF_MAX_BATCH_SIZE; } + // MMF tiles 32 rows per block, so a matrix of few rows leaves most SMs idle while MMVF runs a block + // per row: with a long enough K, MMVF wins for the small batches of speculative decoding as long as + // its work (rows x batch) stays small relative to the SM count, with a budget that grows with K + if (src0_small && src0_ne[0] >= 1024) { + const bool blackwell = cc >= GGML_CUDA_CC_BLACKWELL; + const int nsm = ggml_cuda_info().devices[ggml_cuda_get_device()].nsm; + const int64_t max_ncols = blackwell ? MMVF_MAX_BATCH_SIZE : 5; + const int64_t max_k = blackwell ? 6144 : 4096; + if (ne11 <= max_ncols && src0_ne[1]*ne11*256 <= nsm*std::min(src0_ne[0], max_k)) { + return true; + } + } if (ampere_mma_available(cc)) { return src0_small && ne11 == 1; } @@ -882,6 +973,8 @@ bool ggml_cuda_should_use_mmvf(enum ggml_type type, int cc, int warp_size, const return ne11 <= MMVF_MAX_BATCH_SIZE; } return ne11 <= MMVF_MAX_BATCH_SIZE; + case GGML_TYPE_F8_E4M3: + return ne11 <= MMVF_MAX_BATCH_SIZE; default: return false; } diff --git a/ggml/src/ggml-cuda/mmvf.cuh b/ggml/src/ggml-cuda/mmvf.cuh index 350b5cc064c6..5a2477b079b2 100644 --- a/ggml/src/ggml-cuda/mmvf.cuh +++ b/ggml/src/ggml-cuda/mmvf.cuh @@ -11,4 +11,4 @@ void ggml_cuda_op_mul_mat_vec_f( const char * src1_ddq_i, float * dst_dd_i, const int64_t row_low, const int64_t row_high, const int64_t src1_ncols, const int64_t src1_padded_row_size, cudaStream_t stream); -bool ggml_cuda_should_use_mmvf(enum ggml_type type, int cc, int warp_size, const int64_t * src0_ne, const size_t * src0_nb, int64_t ne11); +bool ggml_cuda_should_use_mmvf(const ggml_tensor * src0, int cc, int warp_size, int64_t ne11); diff --git a/ggml/src/ggml-cuda/mmvq.cu b/ggml/src/ggml-cuda/mmvq.cu index 1a9cf55cb915..c6d895db4e77 100644 --- a/ggml/src/ggml-cuda/mmvq.cu +++ b/ggml/src/ggml-cuda/mmvq.cu @@ -29,10 +29,6 @@ static constexpr __host__ __device__ bool mmvq_should_prefetch(ggml_type type) { return false; } } - -static __device__ __forceinline__ void mmvq_prefetch_l2(const void * p) { - asm volatile("prefetch.global.L2 [%0];" :: "l"(p)); -} #endif typedef float (*vec_dot_q_cuda_t)(const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & kbx, const int & iqs); @@ -285,6 +281,7 @@ static constexpr __host__ __device__ int get_mmvq_mmid_max_batch_rdna4(ggml_type int get_mmvq_mmid_max_batch(ggml_type type, int cc) { // NVIDIA: P100, Volta, Ada Lovelace, and Blackwell always use MMVQ for MUL_MAT_ID. if (GGML_CUDA_CC_IS_NVIDIA(cc)) { + cc = ggml_cuda_highest_compiled_arch(cc); if (cc == GGML_CUDA_CC_PASCAL || cc == GGML_CUDA_CC_VOLTA || cc >= GGML_CUDA_CC_ADA_LOVELACE) { return MMVQ_MAX_BATCH_SIZE; } @@ -756,10 +753,10 @@ static __global__ void mul_mat_vec_q( #pragma unroll for (int i = 0; i < rows_per_cuda_block; ++i) { const size_t off = (size_t)(kbx_offset + i*stride_row_x + kbx_pf) * ggml_cuda_type_traits::bs; - mmvq_prefetch_l2((const char *) vx + off); + ggml_cuda_prefetch_l2((const char *) vx + off); if constexpr (has_fusion) { if (use_gate) { - mmvq_prefetch_l2((const char *) vgate + off); + ggml_cuda_prefetch_l2((const char *) vgate + off); } } } @@ -1554,7 +1551,7 @@ void ggml_cuda_mul_mat_vec_q( const int64_t s11 = src1->nb[1] / ts_src1; const int64_t s12 = src1->nb[2] / ts_src1; const int64_t s13 = src1->nb[3] / ts_src1; - quantize_row_q8_1_cuda(src1_d, nullptr, src1_q8_1.get(), src0->type, ne10, s11, s12, s13, ne10_padded, ne11, ne12, ne13, stream); + quantize_row_q8_1_cuda(src1_d, src1_q8_1.get(), ne10, s11, s12, s13, ne10_padded, ne11, ne12, ne13, stream); } const int64_t s01 = src0->nb[1] / ts_src0; diff --git a/ggml/src/ggml-cuda/quantize.cu b/ggml/src/ggml-cuda/quantize.cu index ea089a8c6249..2acf1f6d0e21 100644 --- a/ggml/src/ggml-cuda/quantize.cu +++ b/ggml/src/ggml-cuda/quantize.cu @@ -83,21 +83,21 @@ static __global__ void quantize_q8_1( ggml_cuda_pdl_sync(); const float xi = i0 < ne00 ? x[i03*s03 + i02*s02 + i01*s01 + i00] : 0.0f; float amax = fabsf(xi); - float sum = xi; amax = warp_reduce_max(amax); - sum = warp_reduce_sum(sum); const float d = amax / 127.0f; const int8_t q = amax == 0.0f ? 0 : roundf(xi / d); + const int sum = warp_reduce_sum((int) q); + y[ib].qs[iqs] = q; if (iqs > 0) { return; } - y[ib].ds = make_half2(d, sum); + y[ib].ds = make_half2(d, d * sum); } __device__ __forceinline__ uint8_t compute_e8m0_scale(float amax) { @@ -500,17 +500,6 @@ static __global__ void quantize_mmq_q8_1( amax = fmaxf(amax, __shfl_xor_sync(0xFFFFFFFF, amax, offset, WARP_SIZE)); } - float sum; - if (ds_layout != MMQ_Q8_1_DS_LAYOUT_D4) { - sum = xi.x + xi.y + xi.z + xi.w; - - // Calculate sums across vals_per_sum/4 threads. -#pragma unroll - for (int offset = vals_per_sum/8; offset > 0; offset >>= 1) { - sum += __shfl_xor_sync(0xFFFFFFFF, sum, offset, WARP_SIZE); - } - } - const float d_inv = 127.0f / amax; char4 q; q.x = roundf(xi.x*d_inv); @@ -519,6 +508,19 @@ static __global__ void quantize_mmq_q8_1( q.w = roundf(xi.w*d_inv); const float d = 1.0f / d_inv; + float sum = 0.0f; + if (ds_layout != MMQ_Q8_1_DS_LAYOUT_D4) { + int sum_q = int(q.x) + int(q.y) + int(q.z) + int(q.w); + + // Calculate sums across vals_per_sum/4 threads. +#pragma unroll + for (int offset = vals_per_sum/8; offset > 0; offset >>= 1) { + sum_q += __shfl_xor_sync(0xFFFFFFFF, sum_q, offset, WARP_SIZE); + } + + sum = d * float(sum_q); + } + // write the block once (normal) or to each of the token's compact rows (scatter) const int nwrite = scatter ? n_expert_used : 1; #pragma unroll @@ -555,10 +557,9 @@ static __global__ void quantize_mmq_q8_1( } void quantize_row_q8_1_cuda( - const float * x, const int32_t * ids, void * vy, const ggml_type type_src0, + const float * x, void * vy, const int64_t ne00, const int64_t s01, const int64_t s02, const int64_t s03, const int64_t ne0, const int64_t ne1, const int64_t ne2, const int64_t ne3, cudaStream_t stream) { - GGML_ASSERT(!ids); GGML_ASSERT(ne0 % QK8_1 == 0); const uint3 ne2_fastdiv = init_fastdiv_values(ne2); @@ -568,7 +569,6 @@ void quantize_row_q8_1_cuda( const dim3 block_size(CUDA_QUANTIZE_BLOCK_SIZE, 1, 1); const ggml_cuda_kernel_launch_params launch_params = ggml_cuda_kernel_launch_params(num_blocks, block_size, 0, stream); ggml_cuda_kernel_launch(quantize_q8_1, launch_params, x, vy, ne00, s01, s02, s03, ne0, ne1, ne2_fastdiv); - GGML_UNUSED(type_src0); } void quantize_mmq_q8_1_cuda( @@ -600,6 +600,81 @@ void quantize_mmq_q8_1_cuda( } } +// Each thread quantizes 4 values of one column, see quantize.cuh for the layout. +template +static __global__ void quantize_mmq_few_q8_1( + const float * x_ptr, void * vy_ptr, const int64_t ne00, const int64_t s01, const int64_t s02, const int64_t s03, + const int ne1, const int ne2) { + ggml_cuda_pdl_lc(); + const float * GGML_CUDA_RESTRICT x = x_ptr; + int4 * GGML_CUDA_RESTRICT vy = (int4 *) vy_ptr; + const int64_t i0 = ((int64_t)blockDim.x*blockIdx.y + threadIdx.x)*4; + + if (i0 >= ne00) { + return; + } + ggml_cuda_pdl_sync(); + + const int64_t i2 = blockIdx.z % ne2; + const int64_t i3 = blockIdx.z / ne2; + const float4 xi = (int) blockIdx.x < ne1 ? *(const float4 *) (x + i3*s03 + i2*s02 + blockIdx.x*s01 + i0) : make_float4(0.0f, 0.0f, 0.0f, 0.0f); + + // the 32 values of a sub-block are held by 8 threads + float amax = fmaxf(fmaxf(fabsf(xi.x), fabsf(xi.y)), fmaxf(fabsf(xi.z), fabsf(xi.w))); + amax = warp_reduce_max<8>(amax); + + const float d_inv = 127.0f / amax; + char4 q; + q.x = roundf(xi.x*d_inv); + q.y = roundf(xi.y*d_inv); + q.z = roundf(xi.z*d_inv); + q.w = roundf(xi.w*d_inv); + const float d = 1.0f / d_inv; + + // as in quantize_mmq_q8_1: the sum of the quantized values; the int warp_reduce_sum reduces the whole warp on Ampere+ + [[maybe_unused]] float sum = 0.0f; + if constexpr (ds_layout != MMQ_Q8_1_DS_LAYOUT_D4) { + sum = d*warp_reduce_sum<8>(float(q.x + q.y + q.z + q.w)); + } + + const int s = i0 % QK_K / QK8_1; // sub-block + const int k = i0 % QK8_1; // first value in sub-block + const int g = blockIdx.x % MMQ_FEW_NCOLS; // column in group + int4 * yg = vy + ((blockIdx.z*(ne00/QK_K) + i0/QK_K)*(gridDim.x/MMQ_FEW_NCOLS) + blockIdx.x/MMQ_FEW_NCOLS)*MMQ_FEW_Q8_1_INT4; + ((int *) (yg + (s/2)*WARP_SIZE + 4*g + (k % 16)/4))[2*(s % 2) + k/16] = *(const int *) &q; + if (k == 0) { + int4 * ds = yg + 4*WARP_SIZE + 4*(g/2) + s/2; + if constexpr (ds_layout == MMQ_Q8_1_DS_LAYOUT_D4) { + ((float *) ds)[2*(s % 2) + g % 2] = d; + } else { + ((half2 *) ds)[2*(s % 2) + g % 2] = make_half2(d, sum); + } + } +} + +void quantize_mmq_few_q8_1_cuda( + const float * x, void * vy, const mmq_q8_1_ds_layout ds_layout, const int64_t ne00, const int64_t s01, const int64_t s02, const int64_t s03, + const int64_t ne1, const int64_t ne2, const int64_t ne3, cudaStream_t stream) { + GGML_ASSERT(ne00 % QK_K == 0); + + const int64_t block_num_y = (ne00 + 4*CUDA_QUANTIZE_BLOCK_SIZE_MMQ - 1) / (4*CUDA_QUANTIZE_BLOCK_SIZE_MMQ); + const dim3 num_blocks(GGML_PAD(ne1, MMQ_FEW_NCOLS), block_num_y, ne2*ne3); + const dim3 block_size(CUDA_QUANTIZE_BLOCK_SIZE_MMQ, 1, 1); + const ggml_cuda_kernel_launch_params launch_params = ggml_cuda_kernel_launch_params(num_blocks, block_size, 0, stream); + switch (ds_layout) { + case MMQ_Q8_1_DS_LAYOUT_D4: + ggml_cuda_kernel_launch(quantize_mmq_few_q8_1, launch_params, + x, vy, ne00, s01, s02, s03, (int) ne1, (int) ne2); + break; + case MMQ_Q8_1_DS_LAYOUT_DS4: + ggml_cuda_kernel_launch(quantize_mmq_few_q8_1, launch_params, + x, vy, ne00, s01, s02, s03, (int) ne1, (int) ne2); + break; + default: + GGML_ABORT("fatal error"); + } +} + // scatter=true reuses the quant kernel: grid over tokens, ids = inverse map (token slot -> compact row) void quantize_scatter_mmq_q8_1_cuda( const float * x, const int32_t * ids_src1_inv, void * vy, const ggml_type type_src0, diff --git a/ggml/src/ggml-cuda/quantize.cuh b/ggml/src/ggml-cuda/quantize.cuh index 5f08dcbfe331..11720ef93bad 100644 --- a/ggml/src/ggml-cuda/quantize.cuh +++ b/ggml/src/ggml-cuda/quantize.cuh @@ -11,14 +11,9 @@ static_assert(MATRIX_ROW_PADDING % CUDA_QUANTIZE_BLOCK_SIZE == 0, "Risk of out-of-bounds access."); static_assert(MATRIX_ROW_PADDING % (4*CUDA_QUANTIZE_BLOCK_SIZE_MMQ) == 0, "Risk of out-of-bounds access."); -typedef void (*quantize_cuda_t)( - const float * x, const int32_t * ids, void * vy, - ggml_type type_src0, int64_t ne00, int64_t s01, int64_t s02, int64_t s03, - int64_t ne0, int64_t ne1, int64_t ne2, int64_t ne3, cudaStream_t stream); - void quantize_row_q8_1_cuda( - const float * x, const int32_t * ids, void * vy, - ggml_type type_src0, int64_t ne00, int64_t s01, int64_t s02, int64_t s03, + const float * x, void * vy, + int64_t ne00, int64_t s01, int64_t s02, int64_t s03, int64_t ne0, int64_t ne1, int64_t ne2, int64_t ne3, cudaStream_t stream); void quantize_mmq_q8_1_cuda( @@ -26,6 +21,17 @@ void quantize_mmq_q8_1_cuda( ggml_type type_src0, int64_t ne00, int64_t s01, int64_t s02, int64_t s03, int64_t ne0, int64_t ne1, int64_t ne2, int64_t ne3, cudaStream_t stream); +// src1 of the few-row MMQ in the layout of the MMA fragments. Per channel, 256 values and group of MMQ_FEW_NCOLS columns: +// qs: int4[4][32], [j][lane] holds the B fragments of sub-blocks 2j and 2j+1 of lane (g, t): the values [4t, 4t + 4) and [16 + 4t, 16 + 4t + 4) of column g. +// ds: int4[4][4], [t][j] holds d (D4, float) or (d, sum) (DS4, half2) of the columns 2t and 2t+1 for sub-block 2j, then 2j+1. +// The column groups of a unit are adjacent, columns beyond ne1 are zero. +static constexpr int MMQ_FEW_NCOLS = 8; // src1 columns per column group +static constexpr int MMQ_FEW_Q8_1_INT4 = 4*WARP_SIZE + 4*4; // int4 per 256 values and column group + +void quantize_mmq_few_q8_1_cuda( + const float * x, void * vy, mmq_q8_1_ds_layout ds_layout, int64_t ne00, int64_t s01, int64_t s02, int64_t s03, + int64_t ne1, int64_t ne2, int64_t ne3, cudaStream_t stream); + void quantize_mmq_fp4_cuda(const float * x, const int32_t * ids, void * vy, diff --git a/ggml/src/ggml-cuda/rope.cu b/ggml/src/ggml-cuda/rope.cu index 45e813406597..2604d014f854 100644 --- a/ggml/src/ggml-cuda/rope.cu +++ b/ggml/src/ggml-cuda/rope.cu @@ -591,7 +591,6 @@ void ggml_cuda_op_rope_impl(ggml_backend_cuda_context & ctx, //const int n_past = ((int32_t *) dst->op_params)[0]; const int n_dims = ((int32_t *) dst->op_params)[1]; const int mode = ((int32_t *) dst->op_params)[2]; - //const int n_ctx = ((int32_t *) dst->op_params)[3]; const int n_ctx_orig = ((int32_t *) dst->op_params)[4]; const int n_offs = ((int32_t *) dst->op_params)[15]; mrope_sections sections; @@ -637,7 +636,7 @@ void ggml_cuda_op_rope_impl(ggml_backend_cuda_context & ctx, } rope_corr_dims corr_dims; - ggml_rope_yarn_corr_dims(n_dims, n_ctx_orig, freq_base, beta_fast, beta_slow, corr_dims.v); + ggml_rope_yarn_corr_dims(n_dims, n_ctx_orig, freq_base, beta_fast, beta_slow, ggml_rope_get_truncate(dst), corr_dims.v); // compute if (is_neox) { @@ -929,7 +928,7 @@ void ggml_cuda_op_rms_norm_mul_rope_fused(ggml_backend_cuda_context & ctx, const float * freq_factors = rope->src[2] != nullptr ? (const float *) rope->src[2]->data : nullptr; rope_corr_dims corr_dims; - ggml_rope_yarn_corr_dims(n_dims, n_ctx_orig, freq_base, beta_fast, beta_slow, corr_dims.v); + ggml_rope_yarn_corr_dims(n_dims, n_ctx_orig, freq_base, beta_fast, beta_slow, ggml_rope_get_truncate(rope), corr_dims.v); const size_t ts0 = ggml_type_size(x->type); GGML_ASSERT(x->nb[0] == ts0); diff --git a/ggml/src/ggml-cuda/set-rows.cu b/ggml/src/ggml-cuda/set-rows.cu index 4659970651e5..4be3e649b73b 100644 --- a/ggml/src/ggml-cuda/set-rows.cu +++ b/ggml/src/ggml-cuda/set-rows.cu @@ -257,6 +257,16 @@ static void set_rows_cuda(ggml_backend_cuda_context & ctx, const ggml_tensor * s nb1, nb2, nb3, stream ); + } else if (dst->type == GGML_TYPE_F8_E4M3) { + set_rows_cuda( + src0_d, src1_d, (ggml_fp8_e4m3_t *) dst->data, + ne00, ne01, ne02, ne03, + ne10, ne11, ne12, ne13, + nb01, nb02, nb03, + nb10, nb11, nb12, + nb1, nb2, nb3, + stream + ); } else if (dst->type == GGML_TYPE_Q4_0) { set_rows_cuda_quant( src0_d, src1_d, (block_q4_0*)dst->data, diff --git a/ggml/src/ggml-cuda/ssm-scan.cu b/ggml/src/ggml-cuda/ssm-scan.cu index dcc199f734ff..b88c9d6ea749 100644 --- a/ggml/src/ggml-cuda/ssm-scan.cu +++ b/ggml/src/ggml-cuda/ssm-scan.cu @@ -256,7 +256,17 @@ static void ssm_scan_f32_cuda(const float * src0, const float * src1, const floa // NOTE: if you change conditions here, be sure to update the corresponding supports_op condition! if (src3_nb1 == sizeof(float)) { // Mamba-2 - if (d_state == 96) { + if (d_state == 64) { // Zamba2 + constexpr int threads = 64; + constexpr int num_warps = threads/WARP_SIZE; + + const dim3 blocks((n_head * head_dim + (num_warps - 1)) / num_warps, n_seq, 1); + const ggml_cuda_kernel_launch_params launch_params = ggml_cuda_kernel_launch_params(blocks, threads, 0, stream); + ggml_cuda_kernel_launch(ssm_scan_f32_group<64/WARP_SIZE, 64>, launch_params, + src0, src1, src2, src3, src4, src5, src6, dst, + src0_nb2, src0_nb3, src1_nb2, src1_nb3, src2_nb1, src2_nb2, src3_nb1, + src4_nb2, src4_nb3, src5_nb2, src5_nb3, s_base, s_slot_bytes, n_head, head_dim, n_group, n_tok, K); + } else if (d_state == 96) { constexpr int threads = 96; constexpr int num_warps = threads/WARP_SIZE; @@ -287,7 +297,7 @@ static void ssm_scan_f32_cuda(const float * src0, const float * src1, const floa src0_nb2, src0_nb3, src1_nb2, src1_nb3, src2_nb1, src2_nb2, src3_nb1, src4_nb2, src4_nb3, src5_nb2, src5_nb3, s_base, s_slot_bytes, n_head, head_dim, n_group, n_tok, K); } else { - GGML_ABORT("doesn't support d_state!=(96, 128 or 256)."); + GGML_ABORT("doesn't support d_state!=(64, 96, 128 or 256)."); } } else { // Mamba-1 diff --git a/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_1-ncols2_8.cu b/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_1-ncols2_8.cu index 6ae77bec8958..3c861c2c0912 100644 --- a/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_1-ncols2_8.cu +++ b/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_1-ncols2_8.cu @@ -2,7 +2,9 @@ #include "../fattn-mma-f16.cuh" +DECL_FATTN_MMA_F16_CASE(40, 40, 1, 8); DECL_FATTN_MMA_F16_CASE(64, 64, 1, 8); +DECL_FATTN_MMA_F16_CASE(72, 72, 1, 8); DECL_FATTN_MMA_F16_CASE(80, 80, 1, 8); DECL_FATTN_MMA_F16_CASE(96, 96, 1, 8); DECL_FATTN_MMA_F16_CASE(112, 112, 1, 8); diff --git a/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_16-ncols2_1.cu b/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_16-ncols2_1.cu index 9d3cfd8edf74..86501536489e 100644 --- a/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_16-ncols2_1.cu +++ b/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_16-ncols2_1.cu @@ -2,7 +2,9 @@ #include "../fattn-mma-f16.cuh" +DECL_FATTN_MMA_F16_CASE(40, 40, 16, 1); DECL_FATTN_MMA_F16_CASE(64, 64, 16, 1); +DECL_FATTN_MMA_F16_CASE(72, 72, 16, 1); DECL_FATTN_MMA_F16_CASE(80, 80, 16, 1); DECL_FATTN_MMA_F16_CASE(96, 96, 16, 1); DECL_FATTN_MMA_F16_CASE(112, 112, 16, 1); diff --git a/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_16-ncols2_2.cu b/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_16-ncols2_2.cu index 0c1a37fd0086..8cf304825901 100644 --- a/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_16-ncols2_2.cu +++ b/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_16-ncols2_2.cu @@ -2,7 +2,9 @@ #include "../fattn-mma-f16.cuh" +DECL_FATTN_MMA_F16_CASE(40, 40, 16, 2); DECL_FATTN_MMA_F16_CASE(64, 64, 16, 2); +DECL_FATTN_MMA_F16_CASE(72, 72, 16, 2); DECL_FATTN_MMA_F16_CASE(80, 80, 16, 2); DECL_FATTN_MMA_F16_CASE(96, 96, 16, 2); DECL_FATTN_MMA_F16_CASE(112, 112, 16, 2); diff --git a/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_16-ncols2_4.cu b/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_16-ncols2_4.cu index d2415bfa957f..51c91c2044b6 100644 --- a/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_16-ncols2_4.cu +++ b/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_16-ncols2_4.cu @@ -2,7 +2,9 @@ #include "../fattn-mma-f16.cuh" +DECL_FATTN_MMA_F16_CASE(40, 40, 16, 4); DECL_FATTN_MMA_F16_CASE(64, 64, 16, 4); +DECL_FATTN_MMA_F16_CASE(72, 72, 16, 4); DECL_FATTN_MMA_F16_CASE(80, 80, 16, 4); DECL_FATTN_MMA_F16_CASE(96, 96, 16, 4); DECL_FATTN_MMA_F16_CASE(112, 112, 16, 4); diff --git a/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_2-ncols2_4.cu b/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_2-ncols2_4.cu index 8eec1d74e293..b6f99eb72e52 100644 --- a/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_2-ncols2_4.cu +++ b/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_2-ncols2_4.cu @@ -2,7 +2,9 @@ #include "../fattn-mma-f16.cuh" +DECL_FATTN_MMA_F16_CASE(40, 40, 2, 4); DECL_FATTN_MMA_F16_CASE(64, 64, 2, 4); +DECL_FATTN_MMA_F16_CASE(72, 72, 2, 4); DECL_FATTN_MMA_F16_CASE(80, 80, 2, 4); DECL_FATTN_MMA_F16_CASE(96, 96, 2, 4); DECL_FATTN_MMA_F16_CASE(112, 112, 2, 4); diff --git a/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_2-ncols2_8.cu b/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_2-ncols2_8.cu index 9f4bef11a443..14c120243035 100644 --- a/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_2-ncols2_8.cu +++ b/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_2-ncols2_8.cu @@ -2,7 +2,9 @@ #include "../fattn-mma-f16.cuh" +DECL_FATTN_MMA_F16_CASE(40, 40, 2, 8); DECL_FATTN_MMA_F16_CASE(64, 64, 2, 8); +DECL_FATTN_MMA_F16_CASE(72, 72, 2, 8); DECL_FATTN_MMA_F16_CASE(80, 80, 2, 8); DECL_FATTN_MMA_F16_CASE(96, 96, 2, 8); DECL_FATTN_MMA_F16_CASE(112, 112, 2, 8); diff --git a/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_32-ncols2_1.cu b/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_32-ncols2_1.cu index 0543532ea347..dbc58be55e54 100644 --- a/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_32-ncols2_1.cu +++ b/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_32-ncols2_1.cu @@ -2,7 +2,9 @@ #include "../fattn-mma-f16.cuh" +DECL_FATTN_MMA_F16_CASE(40, 40, 32, 1); DECL_FATTN_MMA_F16_CASE(64, 64, 32, 1); +DECL_FATTN_MMA_F16_CASE(72, 72, 32, 1); DECL_FATTN_MMA_F16_CASE(80, 80, 32, 1); DECL_FATTN_MMA_F16_CASE(96, 96, 32, 1); DECL_FATTN_MMA_F16_CASE(112, 112, 32, 1); diff --git a/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_32-ncols2_2.cu b/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_32-ncols2_2.cu index d5a4d160a498..784a1f919506 100644 --- a/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_32-ncols2_2.cu +++ b/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_32-ncols2_2.cu @@ -2,7 +2,9 @@ #include "../fattn-mma-f16.cuh" +DECL_FATTN_MMA_F16_CASE(40, 40, 32, 2); DECL_FATTN_MMA_F16_CASE(64, 64, 32, 2); +DECL_FATTN_MMA_F16_CASE(72, 72, 32, 2); DECL_FATTN_MMA_F16_CASE(80, 80, 32, 2); DECL_FATTN_MMA_F16_CASE(96, 96, 32, 2); DECL_FATTN_MMA_F16_CASE(112, 112, 32, 2); diff --git a/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_2.cu b/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_2.cu index d8e7588cfc3c..7ace51905334 100644 --- a/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_2.cu +++ b/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_2.cu @@ -2,7 +2,9 @@ #include "../fattn-mma-f16.cuh" +DECL_FATTN_MMA_F16_CASE(40, 40, 4, 2); DECL_FATTN_MMA_F16_CASE(64, 64, 4, 2); +DECL_FATTN_MMA_F16_CASE(72, 72, 4, 2); DECL_FATTN_MMA_F16_CASE(80, 80, 4, 2); DECL_FATTN_MMA_F16_CASE(96, 96, 4, 2); DECL_FATTN_MMA_F16_CASE(112, 112, 4, 2); diff --git a/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_4.cu b/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_4.cu index 3475dfea08a1..f27640e84efc 100644 --- a/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_4.cu +++ b/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_4.cu @@ -2,7 +2,9 @@ #include "../fattn-mma-f16.cuh" +DECL_FATTN_MMA_F16_CASE(40, 40, 4, 4); DECL_FATTN_MMA_F16_CASE(64, 64, 4, 4); +DECL_FATTN_MMA_F16_CASE(72, 72, 4, 4); DECL_FATTN_MMA_F16_CASE(80, 80, 4, 4); DECL_FATTN_MMA_F16_CASE(96, 96, 4, 4); DECL_FATTN_MMA_F16_CASE(112, 112, 4, 4); diff --git a/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_8.cu b/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_8.cu index 859bea5c5253..28a961b6b9b9 100644 --- a/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_8.cu +++ b/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_8.cu @@ -2,7 +2,9 @@ #include "../fattn-mma-f16.cuh" +DECL_FATTN_MMA_F16_CASE(40, 40, 4, 8); DECL_FATTN_MMA_F16_CASE(64, 64, 4, 8); +DECL_FATTN_MMA_F16_CASE(72, 72, 4, 8); DECL_FATTN_MMA_F16_CASE(80, 80, 4, 8); DECL_FATTN_MMA_F16_CASE(96, 96, 4, 8); DECL_FATTN_MMA_F16_CASE(112, 112, 4, 8); diff --git a/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_64-ncols2_1.cu b/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_64-ncols2_1.cu index 0d7a9c728537..1ad67c0d8f3f 100644 --- a/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_64-ncols2_1.cu +++ b/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_64-ncols2_1.cu @@ -2,7 +2,9 @@ #include "../fattn-mma-f16.cuh" +DECL_FATTN_MMA_F16_CASE(40, 40, 64, 1); DECL_FATTN_MMA_F16_CASE(64, 64, 64, 1); +DECL_FATTN_MMA_F16_CASE(72, 72, 64, 1); DECL_FATTN_MMA_F16_CASE(80, 80, 64, 1); DECL_FATTN_MMA_F16_CASE(96, 96, 64, 1); DECL_FATTN_MMA_F16_CASE(112, 112, 64, 1); diff --git a/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_1.cu b/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_1.cu index 9d5a9976f0ed..695782ca4be0 100644 --- a/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_1.cu +++ b/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_1.cu @@ -2,7 +2,9 @@ #include "../fattn-mma-f16.cuh" +DECL_FATTN_MMA_F16_CASE(40, 40, 8, 1); DECL_FATTN_MMA_F16_CASE(64, 64, 8, 1); +DECL_FATTN_MMA_F16_CASE(72, 72, 8, 1); DECL_FATTN_MMA_F16_CASE(80, 80, 8, 1); DECL_FATTN_MMA_F16_CASE(96, 96, 8, 1); DECL_FATTN_MMA_F16_CASE(112, 112, 8, 1); diff --git a/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_2.cu b/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_2.cu index e48e28c4e48b..de265981544e 100644 --- a/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_2.cu +++ b/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_2.cu @@ -2,7 +2,9 @@ #include "../fattn-mma-f16.cuh" +DECL_FATTN_MMA_F16_CASE(40, 40, 8, 2); DECL_FATTN_MMA_F16_CASE(64, 64, 8, 2); +DECL_FATTN_MMA_F16_CASE(72, 72, 8, 2); DECL_FATTN_MMA_F16_CASE(80, 80, 8, 2); DECL_FATTN_MMA_F16_CASE(96, 96, 8, 2); DECL_FATTN_MMA_F16_CASE(112, 112, 8, 2); diff --git a/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_4.cu b/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_4.cu index 684cd25ce0de..9d9c748aef71 100644 --- a/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_4.cu +++ b/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_4.cu @@ -2,7 +2,9 @@ #include "../fattn-mma-f16.cuh" +DECL_FATTN_MMA_F16_CASE(40, 40, 8, 4); DECL_FATTN_MMA_F16_CASE(64, 64, 8, 4); +DECL_FATTN_MMA_F16_CASE(72, 72, 8, 4); DECL_FATTN_MMA_F16_CASE(80, 80, 8, 4); DECL_FATTN_MMA_F16_CASE(96, 96, 8, 4); DECL_FATTN_MMA_F16_CASE(112, 112, 8, 4); diff --git a/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_8.cu b/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_8.cu index c975ce6b9b74..50f17f2f8d0d 100644 --- a/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_8.cu +++ b/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_8.cu @@ -2,7 +2,9 @@ #include "../fattn-mma-f16.cuh" +DECL_FATTN_MMA_F16_CASE(40, 40, 8, 8); DECL_FATTN_MMA_F16_CASE(64, 64, 8, 8); +DECL_FATTN_MMA_F16_CASE(72, 72, 8, 8); DECL_FATTN_MMA_F16_CASE(80, 80, 8, 8); DECL_FATTN_MMA_F16_CASE(96, 96, 8, 8); DECL_FATTN_MMA_F16_CASE(112, 112, 8, 8); diff --git a/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-bf16-f8_e4m3.cu b/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-bf16-f8_e4m3.cu new file mode 100644 index 000000000000..ae9672a9e567 --- /dev/null +++ b/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-bf16-f8_e4m3.cu @@ -0,0 +1,7 @@ +// This file has been autogenerated by generate_cu_files.py, do not edit manually. + +#include "../fattn-vec.cuh" + +DECL_FATTN_VEC_CASE( 64, GGML_TYPE_BF16, GGML_TYPE_F8_E4M3); +DECL_FATTN_VEC_CASE(128, GGML_TYPE_BF16, GGML_TYPE_F8_E4M3); +DECL_FATTN_VEC_CASE(256, GGML_TYPE_BF16, GGML_TYPE_F8_E4M3); diff --git a/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f16-f8_e4m3.cu b/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f16-f8_e4m3.cu new file mode 100644 index 000000000000..2afaefa838bd --- /dev/null +++ b/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f16-f8_e4m3.cu @@ -0,0 +1,7 @@ +// This file has been autogenerated by generate_cu_files.py, do not edit manually. + +#include "../fattn-vec.cuh" + +DECL_FATTN_VEC_CASE( 64, GGML_TYPE_F16, GGML_TYPE_F8_E4M3); +DECL_FATTN_VEC_CASE(128, GGML_TYPE_F16, GGML_TYPE_F8_E4M3); +DECL_FATTN_VEC_CASE(256, GGML_TYPE_F16, GGML_TYPE_F8_E4M3); diff --git a/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f8_e4m3-bf16.cu b/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f8_e4m3-bf16.cu new file mode 100644 index 000000000000..9dfb9c45e7a9 --- /dev/null +++ b/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f8_e4m3-bf16.cu @@ -0,0 +1,7 @@ +// This file has been autogenerated by generate_cu_files.py, do not edit manually. + +#include "../fattn-vec.cuh" + +DECL_FATTN_VEC_CASE( 64, GGML_TYPE_F8_E4M3, GGML_TYPE_BF16); +DECL_FATTN_VEC_CASE(128, GGML_TYPE_F8_E4M3, GGML_TYPE_BF16); +DECL_FATTN_VEC_CASE(256, GGML_TYPE_F8_E4M3, GGML_TYPE_BF16); diff --git a/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f8_e4m3-f16.cu b/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f8_e4m3-f16.cu new file mode 100644 index 000000000000..9e7570d36fee --- /dev/null +++ b/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f8_e4m3-f16.cu @@ -0,0 +1,7 @@ +// This file has been autogenerated by generate_cu_files.py, do not edit manually. + +#include "../fattn-vec.cuh" + +DECL_FATTN_VEC_CASE( 64, GGML_TYPE_F8_E4M3, GGML_TYPE_F16); +DECL_FATTN_VEC_CASE(128, GGML_TYPE_F8_E4M3, GGML_TYPE_F16); +DECL_FATTN_VEC_CASE(256, GGML_TYPE_F8_E4M3, GGML_TYPE_F16); diff --git a/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f8_e4m3-f8_e4m3.cu b/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f8_e4m3-f8_e4m3.cu new file mode 100644 index 000000000000..6c380e54c8d8 --- /dev/null +++ b/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f8_e4m3-f8_e4m3.cu @@ -0,0 +1,7 @@ +// This file has been autogenerated by generate_cu_files.py, do not edit manually. + +#include "../fattn-vec.cuh" + +DECL_FATTN_VEC_CASE( 64, GGML_TYPE_F8_E4M3, GGML_TYPE_F8_E4M3); +DECL_FATTN_VEC_CASE(128, GGML_TYPE_F8_E4M3, GGML_TYPE_F8_E4M3); +DECL_FATTN_VEC_CASE(256, GGML_TYPE_F8_E4M3, GGML_TYPE_F8_E4M3); diff --git a/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f8_e4m3-q4_0.cu b/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f8_e4m3-q4_0.cu new file mode 100644 index 000000000000..fe22bb964257 --- /dev/null +++ b/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f8_e4m3-q4_0.cu @@ -0,0 +1,7 @@ +// This file has been autogenerated by generate_cu_files.py, do not edit manually. + +#include "../fattn-vec.cuh" + +DECL_FATTN_VEC_CASE( 64, GGML_TYPE_F8_E4M3, GGML_TYPE_Q4_0); +DECL_FATTN_VEC_CASE(128, GGML_TYPE_F8_E4M3, GGML_TYPE_Q4_0); +DECL_FATTN_VEC_CASE(256, GGML_TYPE_F8_E4M3, GGML_TYPE_Q4_0); diff --git a/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f8_e4m3-q4_1.cu b/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f8_e4m3-q4_1.cu new file mode 100644 index 000000000000..317273afc206 --- /dev/null +++ b/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f8_e4m3-q4_1.cu @@ -0,0 +1,7 @@ +// This file has been autogenerated by generate_cu_files.py, do not edit manually. + +#include "../fattn-vec.cuh" + +DECL_FATTN_VEC_CASE( 64, GGML_TYPE_F8_E4M3, GGML_TYPE_Q4_1); +DECL_FATTN_VEC_CASE(128, GGML_TYPE_F8_E4M3, GGML_TYPE_Q4_1); +DECL_FATTN_VEC_CASE(256, GGML_TYPE_F8_E4M3, GGML_TYPE_Q4_1); diff --git a/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f8_e4m3-q5_0.cu b/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f8_e4m3-q5_0.cu new file mode 100644 index 000000000000..e05b1ee381ab --- /dev/null +++ b/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f8_e4m3-q5_0.cu @@ -0,0 +1,7 @@ +// This file has been autogenerated by generate_cu_files.py, do not edit manually. + +#include "../fattn-vec.cuh" + +DECL_FATTN_VEC_CASE( 64, GGML_TYPE_F8_E4M3, GGML_TYPE_Q5_0); +DECL_FATTN_VEC_CASE(128, GGML_TYPE_F8_E4M3, GGML_TYPE_Q5_0); +DECL_FATTN_VEC_CASE(256, GGML_TYPE_F8_E4M3, GGML_TYPE_Q5_0); diff --git a/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f8_e4m3-q5_1.cu b/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f8_e4m3-q5_1.cu new file mode 100644 index 000000000000..6cc6386873cc --- /dev/null +++ b/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f8_e4m3-q5_1.cu @@ -0,0 +1,7 @@ +// This file has been autogenerated by generate_cu_files.py, do not edit manually. + +#include "../fattn-vec.cuh" + +DECL_FATTN_VEC_CASE( 64, GGML_TYPE_F8_E4M3, GGML_TYPE_Q5_1); +DECL_FATTN_VEC_CASE(128, GGML_TYPE_F8_E4M3, GGML_TYPE_Q5_1); +DECL_FATTN_VEC_CASE(256, GGML_TYPE_F8_E4M3, GGML_TYPE_Q5_1); diff --git a/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f8_e4m3-q8_0.cu b/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f8_e4m3-q8_0.cu new file mode 100644 index 000000000000..0a88e240b0ce --- /dev/null +++ b/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f8_e4m3-q8_0.cu @@ -0,0 +1,7 @@ +// This file has been autogenerated by generate_cu_files.py, do not edit manually. + +#include "../fattn-vec.cuh" + +DECL_FATTN_VEC_CASE( 64, GGML_TYPE_F8_E4M3, GGML_TYPE_Q8_0); +DECL_FATTN_VEC_CASE(128, GGML_TYPE_F8_E4M3, GGML_TYPE_Q8_0); +DECL_FATTN_VEC_CASE(256, GGML_TYPE_F8_E4M3, GGML_TYPE_Q8_0); diff --git a/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-f8_e4m3.cu b/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-f8_e4m3.cu new file mode 100644 index 000000000000..44b4d4fc9dc9 --- /dev/null +++ b/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-f8_e4m3.cu @@ -0,0 +1,7 @@ +// This file has been autogenerated by generate_cu_files.py, do not edit manually. + +#include "../fattn-vec.cuh" + +DECL_FATTN_VEC_CASE( 64, GGML_TYPE_Q4_0, GGML_TYPE_F8_E4M3); +DECL_FATTN_VEC_CASE(128, GGML_TYPE_Q4_0, GGML_TYPE_F8_E4M3); +DECL_FATTN_VEC_CASE(256, GGML_TYPE_Q4_0, GGML_TYPE_F8_E4M3); diff --git a/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_1-f8_e4m3.cu b/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_1-f8_e4m3.cu new file mode 100644 index 000000000000..f16490953e44 --- /dev/null +++ b/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_1-f8_e4m3.cu @@ -0,0 +1,7 @@ +// This file has been autogenerated by generate_cu_files.py, do not edit manually. + +#include "../fattn-vec.cuh" + +DECL_FATTN_VEC_CASE( 64, GGML_TYPE_Q4_1, GGML_TYPE_F8_E4M3); +DECL_FATTN_VEC_CASE(128, GGML_TYPE_Q4_1, GGML_TYPE_F8_E4M3); +DECL_FATTN_VEC_CASE(256, GGML_TYPE_Q4_1, GGML_TYPE_F8_E4M3); diff --git a/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-f8_e4m3.cu b/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-f8_e4m3.cu new file mode 100644 index 000000000000..6dab7b408a68 --- /dev/null +++ b/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-f8_e4m3.cu @@ -0,0 +1,7 @@ +// This file has been autogenerated by generate_cu_files.py, do not edit manually. + +#include "../fattn-vec.cuh" + +DECL_FATTN_VEC_CASE( 64, GGML_TYPE_Q5_0, GGML_TYPE_F8_E4M3); +DECL_FATTN_VEC_CASE(128, GGML_TYPE_Q5_0, GGML_TYPE_F8_E4M3); +DECL_FATTN_VEC_CASE(256, GGML_TYPE_Q5_0, GGML_TYPE_F8_E4M3); diff --git a/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_1-f8_e4m3.cu b/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_1-f8_e4m3.cu new file mode 100644 index 000000000000..2cdaa289f39c --- /dev/null +++ b/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_1-f8_e4m3.cu @@ -0,0 +1,7 @@ +// This file has been autogenerated by generate_cu_files.py, do not edit manually. + +#include "../fattn-vec.cuh" + +DECL_FATTN_VEC_CASE( 64, GGML_TYPE_Q5_1, GGML_TYPE_F8_E4M3); +DECL_FATTN_VEC_CASE(128, GGML_TYPE_Q5_1, GGML_TYPE_F8_E4M3); +DECL_FATTN_VEC_CASE(256, GGML_TYPE_Q5_1, GGML_TYPE_F8_E4M3); diff --git a/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-f8_e4m3.cu b/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-f8_e4m3.cu new file mode 100644 index 000000000000..2556c8c82a3a --- /dev/null +++ b/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-f8_e4m3.cu @@ -0,0 +1,7 @@ +// This file has been autogenerated by generate_cu_files.py, do not edit manually. + +#include "../fattn-vec.cuh" + +DECL_FATTN_VEC_CASE( 64, GGML_TYPE_Q8_0, GGML_TYPE_F8_E4M3); +DECL_FATTN_VEC_CASE(128, GGML_TYPE_Q8_0, GGML_TYPE_F8_E4M3); +DECL_FATTN_VEC_CASE(256, GGML_TYPE_Q8_0, GGML_TYPE_F8_E4M3); diff --git a/ggml/src/ggml-cuda/template-instances/generate_cu_files.py b/ggml/src/ggml-cuda/template-instances/generate_cu_files.py index 7be409bd36ab..f49a015ae1f5 100755 --- a/ggml/src/ggml-cuda/template-instances/generate_cu_files.py +++ b/ggml/src/ggml-cuda/template-instances/generate_cu_files.py @@ -8,7 +8,7 @@ # DKQ -> DV override for asymmetric head dims. HEAD_SIZES_V_OVERRIDE = {576: 512, 320: 256, 192: 128} -TYPES_KV = ["GGML_TYPE_F16", "GGML_TYPE_Q4_0", "GGML_TYPE_Q4_1", "GGML_TYPE_Q5_0", "GGML_TYPE_Q5_1", "GGML_TYPE_Q8_0", "GGML_TYPE_BF16"] +TYPES_KV = ["GGML_TYPE_F16", "GGML_TYPE_Q4_0", "GGML_TYPE_Q4_1", "GGML_TYPE_Q5_0", "GGML_TYPE_Q5_1", "GGML_TYPE_Q8_0", "GGML_TYPE_BF16", "GGML_TYPE_F8_E4M3"] SOURCE_FATTN_TILE = """// This file has been autogenerated by generate_cu_files.py, do not edit manually. @@ -90,10 +90,6 @@ def get_short_name(long_quant_name): f.write(SOURCE_FATTN_MMA_START) for head_size_kq in HEAD_SIZES_KQ: - if head_size_kq == 40: - continue - if head_size_kq == 72: - continue # Skip compilation of unused ncols2 values for niche head sizes: if head_size_kq == 192 and ncols2 not in (8, 16): # MiMo-V2.5 continue diff --git a/ggml/src/ggml-cuda/top-k.cu b/ggml/src/ggml-cuda/top-k.cu index b0f60999021f..19cb45d1f93c 100644 --- a/ggml/src/ggml-cuda/top-k.cu +++ b/ggml/src/ggml-cuda/top-k.cu @@ -88,18 +88,18 @@ static __device__ __forceinline__ uint32_t top_k_float_to_ordered(float value) { return bits ^ mask; } -struct top_k_radix_state { +// one state per cache line, so the gather atomics of different rows do not compete +struct alignas(128) top_k_radix_state { uint32_t prefix; uint32_t prefix_mask; int rank; int greater_count; - int equal_count; }; static __global__ void top_k_radix_init(top_k_radix_state * states, int nrows, int k) { const int row = blockIdx.x * blockDim.x + threadIdx.x; if (row < nrows) { - states[row] = {0, 0, k, 0, 0}; + states[row] = {0, 0, k, 0}; } } @@ -109,6 +109,7 @@ static __global__ void top_k_radix_histogram( const top_k_radix_state * __restrict__ states, int * __restrict__ block_histograms, int ncols, + int block_cols, int blocks_per_row, int shift) { constexpr int NBINS = 1 << RADIX_BITS; @@ -117,15 +118,15 @@ static __global__ void top_k_radix_histogram( const int row_block = blockIdx.x % blocks_per_row; const int tid = threadIdx.x; const float * row_src = src + (size_t) row * ncols; + const int64_t col_begin = (int64_t) row_block * block_cols; + const int64_t col_end = min(col_begin + block_cols, (int64_t) ncols); __shared__ int histogram[NBINS]; histogram[tid] = 0; __syncthreads(); const top_k_radix_state state = states[row]; - for (int64_t col = row_block * BLOCK_SIZE + tid; - col < ncols; - col += blocks_per_row * BLOCK_SIZE) { + for (int64_t col = col_begin + tid; col < col_end; col += BLOCK_SIZE) { const uint32_t key = top_k_float_to_ordered(row_src[col]); if ((key & state.prefix_mask) == state.prefix) { atomicAdd(&histogram[(key >> shift) & (NBINS - 1)], 1); @@ -170,42 +171,90 @@ static __global__ void top_k_radix_select( } } -static __global__ void top_k_radix_reset_counters(top_k_radix_state * states, int nrows) { - const int row = blockIdx.x * blockDim.x + threadIdx.x; - if (row < nrows) { - states[row].greater_count = 0; - states[row].equal_count = 0; - } -} - -template +template static __global__ void top_k_radix_gather( const float * __restrict__ src, int * __restrict__ dst, + const int * __restrict__ block_histograms, top_k_radix_state * __restrict__ states, int ncols, + int block_cols, int k, int blocks_per_row) { + constexpr int NBINS = 1 << RADIX_BITS; + constexpr int warp_size = ggml_cuda_get_physical_warp_size(); + constexpr int nwarps = BLOCK_SIZE / warp_size; + using lane_mask_t = std::conditional_t; + // HIP requires a 64-bit mask also for a 32-wide warp + constexpr uint64_t ballot_mask = ~(lane_mask_t) 0; + const auto popc = [](const lane_mask_t mask) { + if constexpr (warp_size == 64) { + return __popcll(mask); + } else { + return __popc(mask); + } + }; + __shared__ int warp_ties[nwarps]; + const int row = blockIdx.x / blocks_per_row; const int row_block = blockIdx.x % blocks_per_row; - const int tid = threadIdx.x; + const int lane = threadIdx.x % warp_size; + const int warp = threadIdx.x / warp_size; const float * row_src = src + (size_t) row * ncols; int * row_dst = dst + (size_t) row * k; top_k_radix_state * state = &states[row]; + const uint32_t threshold = state->prefix; + const int rank = state->rank; + + // each warp reads a contiguous part of the contiguous range of the block + const int warp_cols = block_cols / nwarps; + const int64_t warp_begin = (int64_t) row_block * block_cols + warp * warp_cols; + const int64_t warp_end = min(warp_begin + warp_cols, (int64_t) ncols); - for (int64_t col = row_block * BLOCK_SIZE + tid; - col < ncols; - col += blocks_per_row * BLOCK_SIZE) { + int nties = 0; + for (int64_t col = warp_begin + lane; col < warp_end; col += warp_size) { const uint32_t key = top_k_float_to_ordered(row_src[col]); - if (key > state->prefix) { + if (key > threshold) { const int pos = atomicAdd(&state->greater_count, 1); row_dst[pos] = col; - } else if (key == state->prefix) { - const int pos = atomicAdd(&state->equal_count, 1); - if (pos < state->rank) { - row_dst[k - state->rank + pos] = col; - } + } else if (key == threshold) { + nties++; + } + } + + // the ties fill the last rank slots in column order, the lowest columns win. + // the histograms of the last pass count the ties of each block in the bin of the threshold. + const int * block_ties = block_histograms + (size_t) row * blocks_per_row * NBINS + (threshold & (NBINS - 1)); + int pos = 0; + for (int b = lane; b < row_block; b += warp_size) { + pos += block_ties[b * NBINS]; + } + pos = warp_reduce_sum(pos); + if (pos >= rank || block_ties[row_block * NBINS] == 0) { + return; // the same result in all threads of the block + } + + nties = warp_reduce_sum(nties); + if (lane == 0) { + warp_ties[warp] = nties; + } + __syncthreads(); + for (int w = 0; w < warp; ++w) { + pos += warp_ties[w]; + } + + // pos is the same in all lanes of the warp, so the lanes leave the loop together + const int pos_end = min(pos + nties, rank); + for (int64_t tile = warp_begin; tile < warp_end && pos < pos_end; tile += warp_size) { + const int64_t col = tile + lane; + const bool is_tie = col < warp_end && top_k_float_to_ordered(row_src[col]) == threshold; + + const lane_mask_t ties = (lane_mask_t) __ballot_sync(ballot_mask, is_tie); + const int tie_pos = pos + popc(ties & (((lane_mask_t) 1 << lane) - 1)); + if (is_tie && tie_pos < pos_end) { + row_dst[k - rank + tie_pos] = col; } + pos += popc(ties); } } @@ -216,6 +265,8 @@ static void top_k_radix_cuda( constexpr int RADIX_BITS = 8; constexpr int NBINS = 1 << RADIX_BITS; const int blocks_per_row = (int) std::min(((int64_t) ncols + 1023) / 1024, 64); + // each block of a row reads a contiguous range of whole tiles + const int block_cols = (int) (((int64_t) ncols + blocks_per_row * BLOCK_SIZE - 1) / (blocks_per_row * BLOCK_SIZE) * BLOCK_SIZE); // chunk the rows to bound the histogram memory to 64 MB const int64_t chunk_nrows = ggml_cuda_chunk_nrows((size_t) blocks_per_row * NBINS * sizeof(int), nrows); @@ -234,16 +285,14 @@ static void top_k_radix_cuda( for (int shift = 32 - RADIX_BITS; shift >= 0; shift -= RADIX_BITS) { top_k_radix_histogram <<>>( - src, states, histograms, ncols, blocks_per_row, shift); + src, states, histograms, ncols, block_cols, blocks_per_row, shift); top_k_radix_select <<>>(histograms, states, blocks_per_row, shift); } - top_k_radix_reset_counters - <<<(iter_nrows + BLOCK_SIZE - 1) / BLOCK_SIZE, BLOCK_SIZE, 0, stream>>>(states, iter_nrows); - top_k_radix_gather + top_k_radix_gather <<>>( - src, dst, states, ncols, k, blocks_per_row); + src, dst, histograms, states, ncols, block_cols, k, blocks_per_row); src += (size_t) ncols * iter_nrows; dst += (size_t) k * iter_nrows; diff --git a/ggml/src/ggml-cuda/topk-moe.cu b/ggml/src/ggml-cuda/topk-moe.cu index a93d3a9785d8..e99b2e84a3cb 100644 --- a/ggml/src/ggml-cuda/topk-moe.cu +++ b/ggml/src/ggml-cuda/topk-moe.cu @@ -217,6 +217,9 @@ __global__ void topk_moe_cuda(const float * logits, const int expert = i + threadIdx.x; selection_wt[i / WARP_SIZE] = (n_experts % WARP_SIZE == 0 || expert < n_experts) ? wt[i / WARP_SIZE] + bias[expert] : -INFINITY; + if (__isnanf(selection_wt[i / WARP_SIZE])) { + selection_wt[i / WARP_SIZE] = -FLT_MAX; + } } } diff --git a/ggml/src/ggml-cuda/vendors/cuda.h b/ggml/src/ggml-cuda/vendors/cuda.h index 323c98019347..86c1ed9ce9b6 100644 --- a/ggml/src/ggml-cuda/vendors/cuda.h +++ b/ggml/src/ggml-cuda/vendors/cuda.h @@ -3,6 +3,7 @@ #include #include #include +#include #include #include diff --git a/ggml/src/ggml-cuda/vendors/hip.h b/ggml/src/ggml-cuda/vendors/hip.h index 0db8e97c97a1..da159dea13dd 100644 --- a/ggml/src/ggml-cuda/vendors/hip.h +++ b/ggml/src/ggml-cuda/vendors/hip.h @@ -43,6 +43,9 @@ #define cublasHandle_t hipblasHandle_t #define cublasSetMathMode(handle, mode) CUBLAS_STATUS_SUCCESS #define cublasSetStream hipblasSetStream +#if HIP_VERSION >= 70000000 +#define cublasSetWorkspace hipblasSetWorkspace +#endif // HIP_VERSION >= 70000000 #define cublasSgemm hipblasSgemm #define cublasSgemmBatched hipblasSgemmBatched #define cublasSgemmStridedBatched hipblasSgemmStridedBatched @@ -57,7 +60,9 @@ #define cudaDeviceProp hipDeviceProp_t #define cudaDeviceSynchronize hipDeviceSynchronize #define cudaError_t hipError_t +#define CUresult hipError_t #define cudaErrorMemoryAllocation hipErrorOutOfMemory +#define CUDA_ERROR_OUT_OF_MEMORY hipErrorOutOfMemory #define cudaErrorPeerAccessAlreadyEnabled hipErrorPeerAccessAlreadyEnabled #define cudaErrorPeerAccessNotEnabled hipErrorPeerAccessNotEnabled #define cudaEventCreateWithFlags hipEventCreateWithFlags diff --git a/ggml/src/ggml-cuda/vendors/musa.h b/ggml/src/ggml-cuda/vendors/musa.h index 22126decfbd8..ff13fa052543 100644 --- a/ggml/src/ggml-cuda/vendors/musa.h +++ b/ggml/src/ggml-cuda/vendors/musa.h @@ -129,6 +129,7 @@ // Additional mappings for MUSA graphs #define CUDA_SUCCESS MUSA_SUCCESS +#define CUDA_ERROR_OUT_OF_MEMORY MUSA_ERROR_OUT_OF_MEMORY #define CUresult MUresult #define cuGetErrorString muGetErrorString #define cudaErrorGraphExecUpdateFailure musaErrorGraphExecUpdateFailure diff --git a/ggml/src/ggml-et/ggml-et-ops.cpp b/ggml/src/ggml-et/ggml-et-ops.cpp index 8765138672a8..c51d16f829aa 100644 --- a/ggml/src/ggml-et/ggml-et-ops.cpp +++ b/ggml/src/ggml-et/ggml-et-ops.cpp @@ -1523,7 +1523,7 @@ bool ggml_et_op_flash_attn_ext(ggml_backend_et_device_context * dev_ctx, const g } const ggml_prec prec = ggml_flash_attn_ext_get_prec(node); - if (prec != GGML_PREC_F32 && prec != GGML_PREC_DEFAULT) { + if (prec != GGML_PREC_F32 && prec != GGML_PREC_BF16 && prec != GGML_PREC_UNDEFINED) { GGML_LOG_ERROR("ET: FLASH_ATTN_EXT baseline kernel only supports F32 precision\n"); return false; } diff --git a/ggml/src/ggml-et/ggml-et.cpp b/ggml/src/ggml-et/ggml-et.cpp index cdd947c5221c..53e64b323e5a 100644 --- a/ggml/src/ggml-et/ggml-et.cpp +++ b/ggml/src/ggml-et/ggml-et.cpp @@ -941,6 +941,9 @@ static bool ggml_backend_et_device_supports_op(ggml_backend_dev_t dev, const ggm op->nb[1] == op->ne[0] * sizeof(float); break; case GGML_OP_MUL_MAT: + if (ggml_prec_get_src(op, 1) == GGML_PREC_BF16 || ggml_prec_get_src(op, 1) == GGML_PREC_F32) { + break; + } // Support Q8_0 x F32 -> F32, F16 x F32 -> F32, F16 x F16 -> F32, and F32 x F32 -> F32 matrix multiplication // Stride requirements: first dimension must be contiguous for all tensors if (op->type == GGML_TYPE_F32 && @@ -1069,7 +1072,7 @@ static bool ggml_backend_et_device_supports_op(ggml_backend_dev_t dev, const ggm const bool zero_rot_offset = ggml_get_op_params_i32(op, 15) == 0; supported = - zero_view_offset && zero_rot_offset && ndims <= 512 && + ggml_rope_get_truncate(op) && zero_view_offset && zero_rot_offset && ndims <= 512 && (is_normal || (is_neox && ndims % 16 == 0) || (is_imrope && ndims % 16 == 0 && has_sections)); } else { supported = false; @@ -1299,7 +1302,7 @@ static bool ggml_backend_et_device_supports_op(ggml_backend_dev_t dev, const ggm const bool me_eligible = op->src[1]->type == GGML_TYPE_F16 && op->src[2]->type == GGML_TYPE_F16 && (op->src[0]->ne[0] % 32) == 0; - supported = me_eligible && mask_ok && (prec == GGML_PREC_F32 || prec == GGML_PREC_DEFAULT) && + supported = me_eligible && mask_ok && (prec == GGML_PREC_F32 || prec == GGML_PREC_BF16 || prec == GGML_PREC_UNDEFINED) && max_bias == 0.0f && logit_softcap == 0.0f && op->src[0]->nb[0] == sizeof(float) && op->src[1]->nb[0] == k_elem && op->src[2]->nb[0] == v_elem && op->nb[0] == sizeof(float) && op->src[0]->ne[0] == op->src[1]->ne[0] && // dk matches diff --git a/ggml/src/ggml-hexagon/ggml-hexagon.cpp b/ggml/src/ggml-hexagon/ggml-hexagon.cpp index 6cbc475a0e1d..03da443c8318 100644 --- a/ggml/src/ggml-hexagon/ggml-hexagon.cpp +++ b/ggml/src/ggml-hexagon/ggml-hexagon.cpp @@ -90,7 +90,7 @@ size_t opt_vmem = HTP_OP_MAX_VMEM_DEFAULT; // max available va space for int opt_etm = 0; int opt_verbose = 0; int opt_profile = 0; // profiling mode (0-disabled, 1-basic, 2-pmu) -bool opt_hostbuf = false; +bool opt_hostbuf = true; bool opt_dma64 = false; int opt_power_mode = HTP_POWER_MODE_BURST; // DCVS/HMX power profile requested at session start @@ -135,6 +135,7 @@ size_t opt_mbuf_total = 0; // total buffer space limit ( int opt_mm_select = 2; // 2 = HMX -> HVX -> CPU, 1 = HVX -> CPU, 0 = CPU (unsupported) int opt_fa_select = 2; // 2 = HMX -> HVX -> CPU, 1 = HVX -> CPU, 0 = CPU (unsupported) int opt_fa_split = 1; // 1 = head-split + team-split when n_kv_heads < N_cores (default), 0 = off +int opt_li_select = 2; // 2 = HMX -> HVX, 1 = HVX, 0 = CPU (unsupported) int opt_gdn_select = 2; // 2 = HMX -> HVX, 1 = HVX, 0 = CPU (unsupported) int opt_ar_select = 2; // 2 = fused ALLREDUCE+ADD (default), 1 = unfused ALLREDUCE, 0 = fallback to CPY+FENCE int opt_ar_scatter = 1; // 1 = reduce-scatter the fused ALLREDUCE+ADD (default), 0 = full reduction @@ -149,7 +150,7 @@ int opt_opqueue = 8; // max number of pending batches int opt_optrace = 0; // trace buffer size per thread (0 means default) int opt_oppoll = 0; // polling for batch completions int opt_opfusion = 1; // enable/disable op fusion -int opt_rt_repack = 1; // runtime repacking: 0 = disabled, 1 = quantized only (default), 2 = always +int opt_rt_repack = 1; // runtime repacking: 0 = disabled, 1 = all tileable weights (default) static std::regex* opt_opfilter = NULL; // regex of ops to not claim @@ -241,10 +242,11 @@ static void ggml_hexagon_dump_batch_prof(const char * sess_name, const htp_opbat char evt_str[256] = "----"; if (opt_profile == 3) { - snprintf(evt_str, sizeof(evt_str), "evt-cnt %u,%u,%u,%u,%u,%u,%u,%u,%u,%u,%u", + static_assert(HTP_OP_MAX_NTHREADS == 10, "current implementation assumes 10 trace counters"); + snprintf(evt_str, sizeof(evt_str), "evt-cnt %u,%u,%u,%u,%u,%u,%u,%u,%u,%u", rsp.n_traces[0], rsp.n_traces[1], rsp.n_traces[2], rsp.n_traces[3], rsp.n_traces[4], rsp.n_traces[5], rsp.n_traces[6], rsp.n_traces[7], - rsp.n_traces[8], rsp.n_traces[9], rsp.n_traces[10]); + rsp.n_traces[8], rsp.n_traces[9]); } GGML_LOG_DEBUG("ggml-hex: %s profile-op OPBATCH|----|n-ops %u|%s|----|----|usec %u cycles %llu start %llu mhz %.1f\n", @@ -252,16 +254,11 @@ static void ggml_hexagon_dump_batch_prof(const char * sess_name, const htp_opbat } static void ggml_hexagon_dump_trace_events(const char * sess_name, const htp_opbatch_rsp & rsp, - const htp_trace_desc * trace_events, uint32_t n_traces) { + const htp_trace_desc * trace_events, uint32_t n_threads, uint32_t n_traces) { if (opt_profile == 3 && trace_events) { - uint32_t valid_cnt[HTP_MAX_NTHREADS + 1] = {0}; - for (uint32_t t = 0; t <= HTP_MAX_NTHREADS; t++) { - uint32_t count = rsp.n_traces[t]; - valid_cnt[t] = count > n_traces ? n_traces : count; - } - - for (uint32_t t = 0; t <= HTP_MAX_NTHREADS; t++) { - for (uint32_t idx = 0; idx < valid_cnt[t]; idx++) { + for (uint32_t t = 0; t < n_threads; t++) { + uint32_t count = std::min(rsp.n_traces[t], n_traces); + for (uint32_t idx = 0; idx < count; idx++) { const auto & e = trace_events[t * n_traces + idx]; bool is_stop = (e.info & 0x8000) != 0; uint16_t info = e.info & 0x7FFF; @@ -392,7 +389,7 @@ struct ggml_hexagon_session { const char * htp_session_name(const struct ggml_hexagon_session * sess) { return sess->c_name(); } size_t htp_session_vtcm_size(const struct ggml_hexagon_session * sess) { return sess->vtcm_size; } -uint32_t htp_session_n_threads(const struct ggml_hexagon_session * sess) { return sess->n_threads; } +uint32_t htp_session_n_hvx(const struct ggml_hexagon_session * sess) { return sess->n_hvx; } uint32_t htp_session_n_hmx(const struct ggml_hexagon_session * sess) { return sess->n_hmx; } uint32_t htp_session_mdev_count(const struct ggml_hexagon_session * sess) { return sess->mdev.count; } @@ -462,7 +459,6 @@ struct ggml_hexagon_rpcmem_block { struct ggml_hexagon_shared_buffer { ggml_hexagon_session * sess; std::shared_ptr mem; - std::vector tensor_extra; bool mapped; bool pinned; bool extended; @@ -485,6 +481,9 @@ struct ggml_hexagon_shared_buffer { if (err != 0) { GGML_LOG_ERROR("ggml-hex: %s buffer mapping failed : domain_id %d size %zu fd %d error 0x%08x\n", sess->c_name(), sess->domain_id, size(), fd(), (unsigned) err); + if (!this->pinned) { + GGML_LOG_ERROR("ggml-hex: %s possibly out of IOVA space, try --no-host\n", sess->c_name()); + } throw std::runtime_error("ggml-hex: fastrpc_mmap failed (see log for details)"); } @@ -560,9 +559,6 @@ struct ggml_hexagon_shared_buffer { ~ggml_hexagon_shared_buffer() { free(); - for (auto * extra : tensor_extra) { - delete extra; - } } }; @@ -621,11 +617,9 @@ inline void ggml_hexagon_session::free_fence(void * ptr, uint32_t n_slots) { } inline ggml_tensor ggml_hexagon_session::make_fence_tensor(const volatile void * slot, uint32_t n_slots) const { - static htp_tensor_extra fence_extra { {}, 0, HTP_TENSOR_FENCE }; - ggml_tensor t {}; t.buffer = &this->fence_buf->backend_buffer; - t.extra = &fence_extra; + t.extra = (void *) (uintptr_t) HTP_TENSOR_FENCE; t.data = const_cast(slot); t.type = GGML_TYPE_I8; t.ne[0] = HTP_FENCE_SLOT_SIZE; @@ -658,10 +652,7 @@ static enum ggml_status ggml_backend_hexagon_buffer_init_tensor(ggml_backend_buf HEX_VERBOSE("ggml-hex: %s init-tensor %s : base %p data %p nbytes %zu\n", sess->c_name(), tensor->name, (void *) sbuf->base(), tensor->data, ggml_nbytes(tensor)); - auto extra = new htp_tensor_extra(); - sbuf->tensor_extra.push_back(extra); - - tensor->extra = extra; + tensor->extra = (void *) (uintptr_t) 0; return GGML_STATUS_SUCCESS; } @@ -671,42 +662,19 @@ static void ggml_backend_hexagon_buffer_set_tensor(ggml_backend_buffer_t buffer, const void * data, size_t offset, size_t size) { - auto extra = (htp_tensor_extra *) tensor->extra; - auto sbuf = (ggml_hexagon_shared_buffer *) buffer->context; - auto sess = sbuf->sess; + auto sbuf = (ggml_hexagon_shared_buffer *) buffer->context; + auto sess = sbuf->sess; + + sess->flush_sync(); if (ggml_backend_buffer_get_usage(buffer) == GGML_BACKEND_BUFFER_USAGE_WEIGHTS) { - extra->flags |= HTP_TENSOR_WEIGHT; - if (htp_is_tiled_weight_tensor(tensor)) { - extra->flags |= HTP_TENSOR_TILED; - } + ggml_hexagon_tensor_set_flags(tensor, HTP_TENSOR_WEIGHT); } HEX_VERBOSE("ggml-hex: %s set-tensor %s : data %p offset %zu size %zu usage %d flags 0x%x\n", - sess->c_name(), tensor->name, data, offset, size, (int) buffer->usage, extra->flags); - - if ((extra->flags & HTP_TENSOR_TILED) == 0) { - memcpy((char *) tensor->data + offset, data, size); - return; - } - - if (offset == 0 && size == ggml_nbytes(tensor) && extra->shadow_buf.empty()) { - htp_repack_tensor_tiled(tensor, data, size); - return; - } + sess->c_name(), tensor->name, data, offset, size, (int) buffer->usage, ggml_hexagon_tensor_get_flags(tensor)); - if (extra->shadow_buf.size() < ggml_nbytes(tensor)) { - extra->shadow_buf.resize(ggml_nbytes(tensor)); - } - memcpy(extra->shadow_buf.data() + offset, data, size); - extra->shadow_size += size; - - if (extra->shadow_size >= ggml_nbytes(tensor)) { - htp_repack_tensor_tiled(tensor, extra->shadow_buf.data(), extra->shadow_buf.size()); - extra->shadow_buf.clear(); - extra->shadow_buf.shrink_to_fit(); - extra->shadow_size = 0; - } + memcpy((char *) tensor->data + offset, data, size); } static void ggml_backend_hexagon_buffer_get_tensor(ggml_backend_buffer_t buffer, @@ -714,14 +682,14 @@ static void ggml_backend_hexagon_buffer_get_tensor(ggml_backend_buffer_t buffer, void * data, size_t offset, size_t size) { - auto extra = (htp_tensor_extra *) tensor->extra; - auto sbuf = (ggml_hexagon_shared_buffer *) buffer->context; - auto sess = sbuf->sess; + auto sbuf = (ggml_hexagon_shared_buffer *) buffer->context; + auto sess = sbuf->sess; + const uint32_t flags = ggml_hexagon_tensor_get_flags(tensor); HEX_VERBOSE("ggml-hex: %s get-tensor %s : data %p offset %zu size %zu usage %d flags 0x%x\n", - sess->c_name(), tensor->name, data, offset, size, (int) buffer->usage, extra->flags); + sess->c_name(), tensor->name, data, offset, size, (int) buffer->usage, flags); - if ((extra->flags & HTP_TENSOR_TILED) == 0) { + if ((flags & HTP_TENSOR_TILED) == 0) { memcpy(data, (const char *) tensor->data + offset, size); return; } @@ -741,47 +709,27 @@ static bool ggml_backend_hexagon_buffer_cpy_tensor(ggml_backend_buffer_t bu } static void ggml_backend_hexagon_buffer_set_tensor_2d(ggml_backend_buffer_t buffer, - ggml_tensor * tensor, - const void * data, - size_t offset, - size_t size, - size_t n_copies, - size_t stride_tensor, - size_t stride_data) { - auto extra = (htp_tensor_extra *) tensor->extra; - auto sbuf = (ggml_hexagon_shared_buffer *) buffer->context; - auto sess = sbuf->sess; + ggml_tensor * tensor, + const void * data, + size_t offset, + size_t size, + size_t n_copies, + size_t stride_tensor, + size_t stride_data) { + auto sbuf = (ggml_hexagon_shared_buffer *) buffer->context; + auto sess = sbuf->sess; + + sess->flush_sync(); if (ggml_backend_buffer_get_usage(buffer) == GGML_BACKEND_BUFFER_USAGE_WEIGHTS) { - extra->flags |= HTP_TENSOR_WEIGHT; - if (htp_is_tiled_weight_tensor(tensor)) { - extra->flags |= HTP_TENSOR_TILED; - } + ggml_hexagon_tensor_set_flags(tensor, HTP_TENSOR_WEIGHT); } HEX_VERBOSE("ggml-hex: %s set-tensor-2d %s : data %p offset %zu size %zu n_copies %zu stride_tensor %zu stride_data %zu usage %d flags 0x%x\n", - sess->c_name(), tensor->name, data, offset, size, n_copies, stride_tensor, stride_data, (int) buffer->usage, extra->flags); + sess->c_name(), tensor->name, data, offset, size, n_copies, stride_tensor, stride_data, (int) buffer->usage, ggml_hexagon_tensor_get_flags(tensor)); - if ((extra->flags & HTP_TENSOR_TILED) == 0) { - for (size_t i = 0; i < n_copies; i++) { - memcpy((uint8_t *) tensor->data + offset + i * stride_tensor, (const uint8_t *) data + i * stride_data, size); - } - return; - } - - if (extra->shadow_buf.size() < ggml_nbytes(tensor)) { - extra->shadow_buf.resize(ggml_nbytes(tensor)); - } for (size_t i = 0; i < n_copies; i++) { - memcpy(extra->shadow_buf.data() + offset + i * stride_tensor, (const uint8_t *) data + i * stride_data, size); - } - extra->shadow_size += n_copies * size; - - if (extra->shadow_size >= ggml_nbytes(tensor)) { - htp_repack_tensor_tiled(tensor, extra->shadow_buf.data(), extra->shadow_buf.size()); - extra->shadow_buf.clear(); - extra->shadow_buf.shrink_to_fit(); - extra->shadow_size = 0; + memcpy((uint8_t *) tensor->data + offset + i * stride_tensor, (const uint8_t *) data + i * stride_data, size); } } @@ -793,14 +741,14 @@ static void ggml_backend_hexagon_buffer_get_tensor_2d(ggml_backend_buffer_t buff size_t n_copies, size_t stride_tensor, size_t stride_data) { - auto extra = (htp_tensor_extra *) tensor->extra; - auto sbuf = (ggml_hexagon_shared_buffer *) buffer->context; - auto sess = sbuf->sess; + auto sbuf = (ggml_hexagon_shared_buffer *) buffer->context; + auto sess = sbuf->sess; + const uint32_t flags = ggml_hexagon_tensor_get_flags(tensor); HEX_VERBOSE("ggml-hex: %s get-tensor-2d %s : data %p offset %zu size %zu n_copies %zu stride_tensor %zu stride_data %zu usage %d\n", sess->c_name(), tensor->name, data, offset, size, n_copies, stride_tensor, stride_data, (int) buffer->usage); - if ((extra->flags & HTP_TENSOR_TILED) == 0) { + if ((flags & HTP_TENSOR_TILED) == 0) { for (size_t i = 0; i < n_copies; i++) { memcpy((uint8_t *)data + i * stride_data, (const uint8_t *)tensor->data + offset + i * stride_tensor, size); } @@ -824,6 +772,29 @@ static void ggml_backend_hexagon_buffer_get_tensor_2d(ggml_backend_buffer_t buff } } +static void ggml_backend_hexagon_buffer_set_tensor_n(ggml_backend_buffer_t buffer, + struct ggml_tensor * const * tensors, + const void * const * data, + const size_t * offsets, + const size_t * sizes, + size_t n_tensors) { + const bool is_weights = ggml_backend_buffer_get_usage(buffer) == GGML_BACKEND_BUFFER_USAGE_WEIGHTS; + auto sbuf = (ggml_hexagon_shared_buffer *) buffer->context; + auto sess = sbuf->sess; + + sess->flush_sync(); + + for (size_t i = 0; i < n_tensors; i++) { + if (sizes[i] == 0) { + continue; + } + if (is_weights) { + ggml_hexagon_tensor_set_flags(tensors[i], HTP_TENSOR_WEIGHT); + } + memcpy((char *) tensors[i]->data + offsets[i], data[i], sizes[i]); + } +} + static void ggml_backend_hexagon_buffer_clear(ggml_backend_buffer_t buffer, uint8_t value) { auto sbuf = (ggml_hexagon_shared_buffer *) buffer->context; auto sess = sbuf->sess; @@ -843,7 +814,7 @@ static ggml_backend_buffer_i ggml_backend_hexagon_buffer_interface = { /* .cpy_tensor = */ ggml_backend_hexagon_buffer_cpy_tensor, /* .clear = */ ggml_backend_hexagon_buffer_clear, /* .reset = */ NULL, - /* .set_tensor_n = */ NULL, + /* .set_tensor_n = */ ggml_backend_hexagon_buffer_set_tensor_n, /* .get_tensor_n = */ NULL, /* .cpy_tensor_n = */ NULL, }; @@ -855,8 +826,12 @@ static void ggml_backend_hexagon_host_buffer_set_tensor(ggml_backend_buffer_t bu const void * data, size_t offset, size_t size) { + auto sbuf = (ggml_hexagon_shared_buffer *) buffer->context; + auto sess = sbuf->sess; + + sess->flush_sync(); + memcpy((char *) tensor->data + offset, data, size); - GGML_UNUSED(buffer); } static void ggml_backend_hexagon_host_buffer_get_tensor(ggml_backend_buffer_t buffer, @@ -868,6 +843,25 @@ static void ggml_backend_hexagon_host_buffer_get_tensor(ggml_backend_buffer_t bu GGML_UNUSED(buffer); } +static void ggml_backend_hexagon_host_buffer_set_tensor_n(ggml_backend_buffer_t buffer, + struct ggml_tensor * const * tensors, + const void * const * data, + const size_t * offsets, + const size_t * sizes, + size_t n_tensors) { + auto sbuf = (ggml_hexagon_shared_buffer *) buffer->context; + auto sess = sbuf->sess; + + sess->flush_sync(); + + for (size_t i = 0; i < n_tensors; i++) { + if (sizes[i] == 0) { + continue; + } + memcpy((char *) tensors[i]->data + offsets[i], data[i], sizes[i]); + } +} + static ggml_backend_buffer_i ggml_backend_hexagon_host_buffer_interface = { /* .free_buffer = */ ggml_backend_hexagon_buffer_free_buffer, /* .get_base = */ ggml_backend_hexagon_buffer_get_base, @@ -880,7 +874,7 @@ static ggml_backend_buffer_i ggml_backend_hexagon_host_buffer_interface = { /* .cpy_tensor = */ ggml_backend_hexagon_buffer_cpy_tensor, /* .clear = */ ggml_backend_hexagon_buffer_clear, /* .reset = */ NULL, - /* .set_tensor_n = */ NULL, + /* .set_tensor_n = */ ggml_backend_hexagon_host_buffer_set_tensor_n, /* .get_tensor_n = */ NULL, /* .cpy_tensor_n = */ NULL, }; @@ -895,7 +889,7 @@ static ggml_backend_buffer_t ggml_backend_hexagon_buffer_type_alloc_buffer( ggml_backend_buffer_type_t buffer_type, size_t size) { auto dev_ctx = static_cast(buffer_type->context)->dev_ctx; auto sess = dev_ctx->session(); - if (sess && sess->max_vmem && size > sess->max_vmem) { + if (size > sess->max_vmem) { GGML_LOG_ERROR("ggml-hex: %s buffer size %zu exceeds max_vmem %zu\n", dev_ctx->c_name(), size, sess->max_vmem); return nullptr; @@ -913,7 +907,7 @@ static ggml_backend_buffer_t ggml_backend_hexagon_host_buffer_type_alloc_buffer( ggml_backend_buffer_type_t buffer_type, size_t size) { auto dev_ctx = static_cast(buffer_type->context)->dev_ctx; auto sess = dev_ctx->session(); - if (sess && sess->max_vmem && size > sess->max_vmem) { + if (size > sess->max_vmem) { GGML_LOG_ERROR("ggml-hex: %s host buffer size %zu exceeds max_vmem %zu\n", dev_ctx->c_name(), size, sess->max_vmem); return nullptr; @@ -933,7 +927,7 @@ static size_t ggml_backend_hexagon_buffer_type_get_alignment(ggml_backend_buffer } static size_t ggml_backend_hexagon_buffer_type_get_alloc_size(ggml_backend_buffer_type_t buft, const struct ggml_tensor * t) { - if (!ggml_backend_buft_is_host(buft) && htp_is_tiled_weight_tensor(t)) { + if (opt_rt_repack != 0 && !ggml_backend_buft_is_host(buft) && ggml_op_is_empty(t->op) && htp_is_tiled_weight_tensor(t)) { int64_t ne0 = hex_round_up(t->ne[0], 32); int64_t ne1 = hex_round_up(t->ne[1], 32); int64_t ne2 = t->ne[2]; @@ -1211,8 +1205,7 @@ struct ggml_hexagon_opbatch { std::vector h_tens; // htp tensor descriptors std::vector h_ops; // htp op descriptors - std::unordered_map t_map; // tensor ptr to index - std::unordered_multimap d_map; // tensor data to index + std::unordered_map t_map; // tensor hash to index unsigned int n_bufs; // num buffers in the batch unsigned int n_tens; // num tensors ... @@ -1231,7 +1224,6 @@ struct ggml_hexagon_opbatch { b_vmem = 0; t_map.clear(); - d_map.clear(); ops.resize(n_ops_max); } @@ -1251,7 +1243,6 @@ struct ggml_hexagon_opbatch { h_ops.resize(n_ops_max); t_map.reserve(n_tens_max); - d_map.reserve(n_tens_max); GGML_LOG_INFO("ggml-hex: %s op batching: n-bufs %u n-tensors %u n-ops %u vmem %zu\n", sess->c_name(), n_bufs_max, n_tens_max, n_ops_max, b_vmem_max); @@ -1297,7 +1288,7 @@ struct ggml_hexagon_opbatch { } static void init_tensor_shape(htp_tensor & h, const ggml_tensor * t) { - auto extra = (htp_tensor_extra *) t->extra; + const uint32_t flags = ggml_hexagon_tensor_get_flags(t); h.type = t->type; h.ne[0] = t->ne[0]; @@ -1305,7 +1296,7 @@ struct ggml_hexagon_opbatch { h.ne[2] = t->ne[2]; h.ne[3] = t->ne[3]; - if (extra->flags & HTP_TENSOR_TILED) { + if (flags & HTP_TENSOR_TILED) { if (t->type == GGML_TYPE_F32 || t->type == GGML_TYPE_BF16) { h.type = GGML_TYPE_F16; h.nb[0] = sizeof(ggml_fp16_t); @@ -1326,31 +1317,42 @@ struct ggml_hexagon_opbatch { h.size = ggml_nbytes(t); } - h.flags = extra->flags & (HTP_TENSOR_WEIGHT | HTP_TENSOR_TILED | HTP_TENSOR_FENCE); + h.flags = flags & (HTP_TENSOR_WEIGHT | HTP_TENSOR_TILED | HTP_TENSOR_FENCE); } - static bool same_shape(const htp_tensor & a, const htp_tensor & b) { - return (a.type == b.type) && - (a.ne[0] == b.ne[0]) && (a.ne[1] == b.ne[1]) && (a.ne[2] == b.ne[2]) && (a.ne[3] == b.ne[3]) && - (a.nb[0] == b.nb[0]) && (a.nb[1] == b.nb[1]) && (a.nb[2] == b.nb[2]) && (a.nb[3] == b.nb[3]); + static uint64_t hash_tensor(const ggml_tensor * t, const htp_tensor & shape) { + uint64_t hash = 1469598103934665603ull; + auto hash_value = [&hash](uint64_t value) { + hash ^= value; + hash *= 1099511628211ull; + }; + + hash_value((uintptr_t) t->data); + hash_value(shape.type); + hash_value(shape.size); + hash_value(shape.flags); + for (int i = 0; i < GGML_MAX_DIMS; ++i) { + hash_value(shape.ne[i]); + hash_value(shape.nb[i]); + } + + // avalanche high bits down to low bits for power-of-two table indexing + hash ^= hash >> 33; + hash *= 0xff51afd7ed558ccdull; + hash ^= hash >> 33; + + return hash; } // add tensor and return its index int add_tensor(const ggml_tensor * t) { - // Lookup by tensor ptr - auto it = t_map.find(t); - if (it != t_map.end()) { return it->second; } - htp_tensor shape {}; init_tensor_shape(shape, t); - // Lookup by tensor data and shape - auto range = d_map.equal_range(t->data); - for (auto dit = range.first; dit != range.second; ++dit) { - if (same_shape(h_tens[dit->second], shape)) { - t_map.insert({t, dit->second}); - return dit->second; - } + const uint64_t hash = hash_tensor(t, shape); + auto it = t_map.find(hash); + if (it != t_map.end()) { + return it->second; } auto sbuf = static_cast(t->buffer->context); @@ -1359,8 +1361,7 @@ struct ggml_hexagon_opbatch { int ti = n_tens++; GGML_ASSERT(n_tens <= n_tens_max); - t_map.insert({t, ti}); - d_map.insert({t->data, ti}); + t_map[hash] = ti; uint64_t t_offset = (uint8_t *) t->data - sbuf->base(); @@ -1390,8 +1391,11 @@ struct ggml_hexagon_opbatch { auto fit_tensor = [&](const ggml_tensor * t) { if (!t) return; + if (check_tens) { - if (t_map.count(t)) return; + htp_tensor shape {}; + init_tensor_shape(shape, t); + if (t_map.count(hash_tensor(t, shape))) return; extra_tens++; } @@ -1539,7 +1543,7 @@ struct ggml_hexagon_opqueue { size_t tr_size = 0; if (opt_profile == 3) { - tr_size = (HTP_MAX_NTHREADS + 1) * opt_optrace * sizeof(htp_trace_desc); + tr_size = sess->n_threads * opt_optrace * sizeof(htp_trace_desc); } shm_blk_size = sizeof(htp_buf_desc) * n_bufs + @@ -1594,7 +1598,7 @@ struct ggml_hexagon_opqueue { size_t tr_size = 0; if (opt_profile == 3) { req.n_traces = opt_optrace; - tr_size = (HTP_MAX_NTHREADS + 1) * req.n_traces * sizeof(htp_trace_desc); + tr_size = shm_buf->sess->n_threads * req.n_traces * sizeof(htp_trace_desc); } else { req.n_traces = 0; } @@ -1649,7 +1653,7 @@ struct ggml_hexagon_opqueue { uint32_t n_traces = 0; if (opt_profile == 3) { n_traces = opt_optrace; - tr_size = (HTP_MAX_NTHREADS + 1) * n_traces * sizeof(htp_trace_desc); + tr_size = shm_buf->sess->n_threads * n_traces * sizeof(htp_trace_desc); } const size_t m_size = b_size + t_size + o_size + p_size + tr_size; @@ -1678,7 +1682,7 @@ struct ggml_hexagon_opqueue { ggml_hexagon_dump_op_prof(shm_buf->sess->c_name(), ops[i], pd[i]); } - ggml_hexagon_dump_trace_events(shm_buf->sess->c_name(), rsp, trace_events, n_traces); + ggml_hexagon_dump_trace_events(shm_buf->sess->c_name(), rsp, trace_events, shm_buf->sess->n_threads, n_traces); } } }; @@ -2113,34 +2117,80 @@ void ggml_hexagon_session::allocate(const ggml_hexagon_device_config & config) n } } - char session_uri[256]; - { - char htp_uri[256]; - snprintf(htp_uri, sizeof(htp_uri), "file:///libggml-htp-v%u.so?htp_iface_skel_handle_invoke&_modver=1.0", opt_arch); + auto open_session = [&](const std::string & prefix, std::string & uri_out) -> int { + const std::string htp_uri = prefix + "libggml-htp-v" + std::to_string(opt_arch) + + ".so?htp_iface_skel_handle_invoke&_modver=1.0"; + + std::vector session_uri(htp_uri.size() + 256); struct remote_rpc_get_uri u = {}; u.session_id = this->session_id; u.domain_name = const_cast(dom_name.c_str()); u.domain_name_len = dom_name.size(); - u.module_uri = const_cast(htp_uri); - u.module_uri_len = strlen(htp_uri); - u.uri = session_uri; - u.uri_len = sizeof(session_uri); + u.module_uri = const_cast(htp_uri.c_str()); + u.module_uri_len = htp_uri.size(); + u.uri = session_uri.data(); + u.uri_len = session_uri.size(); int err = remote_session_control(FASTRPC_GET_URI, (void *) &u, sizeof(u)); if (err != AEE_SUCCESS) { - snprintf(session_uri, sizeof(session_uri), "%s&_dom=%s&_session=%u", - htp_uri, dom_name.c_str(), this->session_id); + snprintf(session_uri.data(), session_uri.size(), "%s&_dom=%s&_session=%u", + htp_uri.c_str(), dom_name.c_str(), this->session_id); - GGML_LOG_WARN("ggml-hex: %s failed to get URI (physical %d, virtual %d) : error 0x%x. Falling back to single session URI: %s\n", - this->c_name(), phys_idx, virt_idx, err, session_uri); + HEX_VERBOSE("ggml-hex: %s failed to get URI (physical %d, virtual %d) : error 0x%x. Falling back to single session URI: %s\n", + this->c_name(), phys_idx, virt_idx, err, session_uri.data()); } + + uri_out = session_uri.data(); + return htp_iface_open(session_uri.data(), &this->handle); + }; + + const std::string lib_dir = htp_lib_dir(); + + std::string session_uri; + int err = AEE_EFAILED; + + // 1. Try opening session with absolute path in URI + if (!lib_dir.empty()) { + std::string abs_prefix = "file:///" + lib_dir; + if (abs_prefix.back() != '/' && abs_prefix.back() != '\\') { + abs_prefix += "/"; + } + + HEX_VERBOSE("ggml-hex: %s trying absolute path URI: %s\n", this->c_name(), abs_prefix.c_str()); + err = open_session(abs_prefix, session_uri); + if (err != AEE_SUCCESS) { + HEX_VERBOSE("ggml-hex: %s failed to open session with absolute path : uri %s error 0x%x\n", + this->c_name(), session_uri.c_str(), err); + } + } + + // 2. Fallback to DSP search path via FASTRPC_MANAGE_DSP_LIBRARY_PATH + if (err != AEE_SUCCESS && !lib_dir.empty()) { + int sp_err = htpdrv_set_dsp_search_path(this->domain_id, lib_dir, this->c_name()); + if (sp_err == AEE_SUCCESS) { + err = open_session("file:///", session_uri); + if (err != AEE_SUCCESS) { + HEX_VERBOSE("ggml-hex: %s failed to open session with search path : uri %s error 0x%x\n", + this->c_name(), session_uri.c_str(), err); + } + } + } + + // 3. Fallback to process environment variables + if (err != AEE_SUCCESS && !lib_dir.empty()) { + HEX_VERBOSE("ggml-hex: %s falling back to process env for DSP search path\n", this->c_name()); + htpdrv_set_dsp_env_path(lib_dir, this->c_name()); + err = open_session("file:///", session_uri); + } + + // Bare name attempt if lib_dir is empty and nothing was tried yet + if (err != AEE_SUCCESS && lib_dir.empty()) { + err = open_session("file:///", session_uri); } - // Open session - int err = htp_iface_open(session_uri, &this->handle); if (err != AEE_SUCCESS) { - GGML_LOG_ERROR("ggml-hex: %s failed to open session : uri %s error 0x%x\n", this->c_name(), session_uri, err); + GGML_LOG_ERROR("ggml-hex: %s failed to open session : uri %s error 0x%x\n", this->c_name(), session_uri.c_str(), err); throw std::runtime_error("ggml-hex: failed to open session (see log for details)"); } @@ -2154,11 +2204,11 @@ void ggml_hexagon_session::allocate(const ggml_hexagon_device_config & config) n unsigned long long hw_vtcm_size = 0; int hw_err = htp_iface_hwinfo(this->handle, &hw_n_threads, &hw_n_hvx, &hw_n_hmx, &hw_vtcm_size); if (hw_err == 0) { - const uint32_t max_n_threads = (std::min)((uint32_t) HTP_MAX_NTHREADS, (uint32_t) hw_n_threads); - this->n_threads = opt_nhvx > 0 ? (uint32_t) (std::min)(opt_nhvx, (size_t) max_n_threads) : max_n_threads; - this->n_hvx = this->n_threads; - this->n_hmx = (opt_nhmx != 0) ? (uint32_t)hw_n_hmx : 0; - this->vtcm_size = (uint64_t)hw_vtcm_size; + this->n_threads = (std::min)((uint32_t) HTP_OP_MAX_NTHREADS, (uint32_t) hw_n_threads); + const uint32_t max_n_hvx = (std::min)({this->n_threads - 1, (uint32_t) hw_n_hvx, (uint32_t) HTP_OP_MAX_NHVX}); + this->n_hvx = opt_nhvx > 0 ? (uint32_t) (std::min)(opt_nhvx, (size_t) max_n_hvx) : max_n_hvx; + this->n_hmx = (opt_nhmx != 0) ? (uint32_t)hw_n_hmx : 0; + this->vtcm_size = (uint64_t)hw_vtcm_size; if (opt_vtcm_size > 0) { if (opt_vtcm_size > hw_vtcm_size) { GGML_LOG_WARN("ggml-hex: %s VTCM override %zu exceeds HW size %llu\n", this->c_name(), opt_vtcm_size, hw_vtcm_size); @@ -2170,11 +2220,11 @@ void ggml_hexagon_session::allocate(const ggml_hexagon_device_config & config) n (unsigned long long)(this->vtcm_size / (1024 * 1024))); } else { GGML_LOG_WARN("ggml-hex: %s failed to query hwinfo (0x%x), using defaults\n", this->c_name(), hw_err); - const uint32_t default_n_threads = (std::min)(8u, (uint32_t) HTP_MAX_NTHREADS); - this->n_threads = opt_nhvx > 0 ? (uint32_t) (std::min)(opt_nhvx, (size_t) HTP_MAX_NTHREADS) : default_n_threads; - this->n_hvx = this->n_threads; - this->n_hmx = (opt_nhmx != 0) ? 1 : 0; - this->vtcm_size = opt_vtcm_size > 0 ? opt_vtcm_size : 8 * 1024 * 1024; + this->n_threads = (std::min)(8u, (uint32_t) HTP_OP_MAX_NTHREADS); + const uint32_t max_n_hvx = (std::min)(this->n_threads - 1, (uint32_t) HTP_OP_MAX_NHVX); + this->n_hvx = opt_nhvx > 0 ? (uint32_t) (std::min)(opt_nhvx, (size_t) max_n_hvx) : (std::min)(4u, max_n_hvx); + this->n_hmx = (opt_nhmx != 0) ? 1 : 0; + this->vtcm_size = opt_vtcm_size > 0 ? opt_vtcm_size : 8 * 1024 * 1024; } } @@ -2190,7 +2240,7 @@ void ggml_hexagon_session::allocate(const ggml_hexagon_device_config & config) n } GGML_LOG_INFO("ggml-hex: %s new session : session-id %d domain-id %d uri %s handle 0x%lx\n", this->c_name(), - this->session_id, this->domain_id, session_uri, (unsigned long) this->handle); + this->session_id, this->domain_id, session_uri.c_str(), (unsigned long) this->handle); const size_t req_q_size = (sizeof(htp_opbatch_req) * opt_opqueue * 2) + 1024; const size_t rsp_q_size = (sizeof(htp_opbatch_rsp) * opt_opqueue * 2) + 1024; @@ -2243,7 +2293,7 @@ void ggml_hexagon_session::allocate(const ggml_hexagon_device_config & config) n this->op_batch = new ggml_hexagon_opbatch(this, opt_opbatch, this->max_vmem); // Start dspqueue/opbatch processing - err = htp_iface_start(this->handle, this->session_id, this->queue_id, this->n_threads, opt_nhmx, opt_vtcm_size, this->max_vmem, + err = htp_iface_start(this->handle, this->session_id, this->queue_id, this->n_hvx, opt_nhmx, opt_vtcm_size, this->max_vmem, (uint32_t) opt_power_mode); if (err != 0) { GGML_LOG_ERROR("ggml-hex: %s failed to start session: 0x%08x\n", this->c_name(), (unsigned) err); @@ -2351,6 +2401,7 @@ ggml_hexagon_session::~ggml_hexagon_session() noexcept(true) { // ** backend interface + static const char * ggml_backend_hexagon_name(ggml_backend_t backend) { auto sess = static_cast(backend->context); return sess->c_name(); @@ -2362,11 +2413,33 @@ static void ggml_backend_hexagon_free(ggml_backend_t backend) { delete backend; } -static inline bool op_is_compute(ggml_tensor *node) -{ +static inline bool ggml_hexagon_op_is_compute(const struct ggml_tensor * node) { return !ggml_op_is_empty(node->op) && !ggml_is_empty(node) && (node->flags & GGML_TENSOR_FLAG_COMPUTE); } +static void ggml_hexagon_tensor_repack(ggml_tensor * tensor, ggml_hexagon_session * sess) { + if (tensor->data == nullptr || ggml_hexagon_tensor_is_tiled(tensor) || !ggml_hexagon_tensor_can_tile(tensor)) { + return; + } + HEX_VERBOSE("ggml-hex: %s rt-repack %s|size %zu\n", sess->c_name(), htp_format_tensor(tensor, HTP_OPFMT_TENSOR).c_str(), ggml_nbytes(tensor)); + std::vector raw(ggml_nbytes(tensor)); + memcpy(raw.data(), tensor->data, raw.size()); + htp_repack_tensor_tiled(tensor, raw.data(), raw.size()); + ggml_hexagon_tensor_set_flags(tensor, HTP_TENSOR_TILED); +} + +static void ggml_hexagon_graph_repack(const ggml_cgraph * graph, ggml_hexagon_session * sess) { + for (int i = 0; i < graph->n_nodes; ++i) { + ggml_tensor * n = graph->nodes[i]; + if (!ggml_hexagon_op_is_compute(n)) { + continue; + } + if (n->op == GGML_OP_MUL_MAT || n->op == GGML_OP_MUL_MAT_ID) { + ggml_hexagon_tensor_repack(n->src[0], sess); + } + } +} + static ggml_status ggml_backend_hexagon_graph_compute(ggml_backend_t backend, ggml_cgraph * graph) { auto sess = static_cast(backend->context); @@ -2383,13 +2456,15 @@ static ggml_status ggml_backend_hexagon_graph_compute(ggml_backend_t backend, gg if (cache_hit) { nodes_ptr = &sess->cached_nodes; } else { + ggml_hexagon_graph_repack(graph, sess); + htp_fusion_tag_graph(graph); computed_nodes.reserve(graph->n_nodes); for (int i = 0; i < graph->n_nodes; ++i) { ggml_tensor * n = graph->nodes[i]; - if (!op_is_compute(n)) { + if (!ggml_hexagon_op_is_compute(n)) { continue; } @@ -2399,22 +2474,7 @@ static ggml_status ggml_backend_hexagon_graph_compute(ggml_backend_t backend, gg } htp_opnode node(opcode, n); - if (node.opcode == HTP_OP_MUL_MAT || node.opcode == HTP_OP_MUL_MAT_ID || node.opcode == HTP_OP_GET_ROWS) { - ggml_tensor * src0 = node.node->src[0]; - if (htp_needs_repack(src0)) { - HEX_VERBOSE("ggml-hex: %s rt-repack %s|size %zu\n", sess->c_name(), htp_format_tensor(src0, HTP_OPFMT_TENSOR).c_str(), ggml_nbytes(src0)); - std::vector raw(ggml_nbytes(src0)); - memcpy(raw.data(), src0->data, raw.size()); - htp_repack_tensor_tiled(src0, raw.data(), raw.size()); - auto * extra = (htp_tensor_extra *) src0->extra; - extra->flags |= HTP_TENSOR_TILED; - } - } - GGML_ASSERT(htp_kparams_solve(sess, node)); - if (node.opcode == HTP_OP_CPY && node.kparams()->total_elems == 0) { - continue; - } computed_nodes.push_back(std::move(node)); } @@ -2624,7 +2684,11 @@ static std::vector ggml_hexagon_graph_optimize_reorder(const std::vector(backend->context); + + HEX_VERBOSE("ggml-hex: %s graph-optimize n_nodes %d\n", sess->c_name(), gf->n_nodes); + + ggml_hexagon_graph_repack(gf, sess); const int n = gf->n_nodes; @@ -2659,7 +2723,7 @@ static void ggml_backend_hexagon_graph_optimize(ggml_backend_t backend, ggml_cgr } } - GGML_UNUSED(backend); + GGML_UNUSED(params); } static uint64_t ggml_hexagon_session_key(const ggml_hexagon_session * sess) { @@ -2721,14 +2785,7 @@ static bool ggml_backend_hexagon_cpy_tensor_async(ggml_backend_t backend_src, gg return false; } - // FIXME: ggml-meta needs to call init_tensor on auxiliary tensors - if (!dst->extra) { - ggml_backend_buffer_init_tensor(dst->buffer, dst); - } - - auto * dst_extra = static_cast(dst->extra); - const auto * src_extra = static_cast(src->extra); - dst_extra->flags = src_extra->flags & ~HTP_TENSOR_FUSEABLE; + dst->extra = (void *) (uintptr_t) (ggml_hexagon_tensor_get_flags(src) & ~HTP_TENSOR_FUSEABLE); auto sess_src = static_cast(backend_src->context); auto sess_dst = static_cast(backend_dst->context); @@ -3001,18 +3058,18 @@ static bool ggml_backend_hexagon_device_supports_op(ggml_backend_dev_t dev, cons } bool supp = false; - switch (op->op) { - case GGML_OP_NONE: - case GGML_OP_RESHAPE: - case GGML_OP_VIEW: - case GGML_OP_PERMUTE: - case GGML_OP_TRANSPOSE: - supp = true; - break; - - default: + if (ggml_op_is_empty(op->op) || ggml_is_empty(op)) { + supp = true; + } else { + supp = ggml_backend_hexagon_buffer_type_get_alloc_size(&dev_ctx->buffer_type, op) <= sess->max_vmem; + for (int i = 0; supp && i < GGML_MAX_SRC; i++) { + if (op->src[i] && ggml_backend_hexagon_buffer_type_get_alloc_size(&dev_ctx->buffer_type, op->src[i]) > sess->max_vmem) { + supp = false; + } + } + if (supp) { supp = htp_kparams_check(sess, op); - break; + } } ggml_hexagon_dump_op_supp(sess->c_name(), op, supp); @@ -3338,8 +3395,8 @@ static void ggml_hexagon_discover_devices() { discovery_supported = true; const int n_domains = std::min(domain_info.sys.num_domains, (int) domains.size()); for (int i = 0; i < n_domains; i++) { - GGML_LOG_INFO("ggml-hex: FASTRPC_GET_DOMAINS[%d]: type %d id %d name '%s' status %d instance-id %d\n", - i, (int) domains[i].type, domains[i].id, domains[i].name, domains[i].status, domains[i].instance_id); + HEX_VERBOSE("ggml-hex: FASTRPC_GET_DOMAINS[%d]: type %d id %d name '%s' status %d instance-id %d\n", + i, (int) domains[i].type, domains[i].id, domains[i].name, domains[i].status, domains[i].instance_id); if (domains[i].type != FASTRPC_NSP) { GGML_LOG_DEBUG("ggml-hex: skipping non-CDSP domain (type=%d)\n", (int) domains[i].type); continue; @@ -3353,10 +3410,10 @@ static void ggml_hexagon_discover_devices() { domains[i].instance_id, domains[i].id, domains[i].name); } } else { - GGML_LOG_WARN("ggml-hex: FASTRPC_GET_DOMAINS fetch failed (0x%x), using static CDSP domains\n", (unsigned) err); + HEX_VERBOSE("ggml-hex: FASTRPC_GET_DOMAINS fetch failed (0x%x), using static CDSP domains\n", (unsigned) err); } } else if (err != AEE_SUCCESS) { - GGML_LOG_DEBUG("ggml-hex: FASTRPC_GET_DOMAINS query failed (0x%x), using static CDSP domains\n", (unsigned) err); + HEX_VERBOSE("ggml-hex: FASTRPC_GET_DOMAINS query failed (0x%x), using static CDSP domains\n", (unsigned) err); } // Populate domain IDs and names for all configured devices @@ -3371,6 +3428,10 @@ static void ggml_hexagon_init(ggml_backend_reg * reg) { "please update hexagon_type to match ggml_type"); static_assert((unsigned int) HTP_TYPE_Q4_1 == (unsigned int) GGML_TYPE_Q4_1, "please update hexagon_type to match ggml_type"); + static_assert((unsigned int) HTP_TYPE_Q5_0 == (unsigned int) GGML_TYPE_Q5_0, + "please update hexagon_type to match ggml_type"); + static_assert((unsigned int) HTP_TYPE_Q5_1 == (unsigned int) GGML_TYPE_Q5_1, + "please update hexagon_type to match ggml_type"); static_assert((unsigned int) HTP_TYPE_Q8_0 == (unsigned int) GGML_TYPE_Q8_0, "please update hexagon_type to match ggml_type"); static_assert((unsigned int) HTP_TYPE_MXFP4 == (unsigned int) GGML_TYPE_MXFP4, @@ -3409,6 +3470,7 @@ static void ggml_hexagon_init(ggml_backend_reg * reg) { const char * str_mm_select = getenv("GGML_HEXAGON_MM_SELECT"); const char * str_fa_select = getenv("GGML_HEXAGON_FA_SELECT"); const char * str_fa_split = getenv("GGML_HEXAGON_FA_SPLIT"); + const char * str_li_select = getenv("GGML_HEXAGON_LI_SELECT"); const char * str_gdn_select = getenv("GGML_HEXAGON_GDN_SELECT"); const char * str_ar_select = getenv("GGML_HEXAGON_AR_SELECT"); const char * str_ar_scatter = getenv("GGML_HEXAGON_AR_SCATTER"); @@ -3475,6 +3537,7 @@ static void ggml_hexagon_init(ggml_backend_reg * reg) { opt_mm_select = str_mm_select ? atoi(str_mm_select) : opt_mm_select; opt_fa_select = str_fa_select ? atoi(str_fa_select) : opt_fa_select; opt_fa_split = str_fa_split ? atoi(str_fa_split) : opt_fa_split; + opt_li_select = str_li_select ? atoi(str_li_select) : opt_li_select; opt_gdn_select = str_gdn_select ? atoi(str_gdn_select) : opt_gdn_select; opt_ar_select = str_ar_select ? atoi(str_ar_select) : opt_ar_select; opt_ar_scatter = str_ar_scatter ? atoi(str_ar_scatter) : opt_ar_scatter; diff --git a/ggml/src/ggml-hexagon/htp-common.h b/ggml/src/ggml-hexagon/htp-common.h index e51340d8a278..4e9e54415409 100644 --- a/ggml/src/ggml-hexagon/htp-common.h +++ b/ggml/src/ggml-hexagon/htp-common.h @@ -4,12 +4,6 @@ #include "htp-opnode.h" #include "htp-repack.h" -struct htp_tensor_extra { - std::vector shadow_buf; - size_t shadow_size { 0 }; - uint32_t flags { 0 }; -}; - extern int opt_arch; extern size_t opt_ndev; extern size_t opt_nhvx; @@ -29,6 +23,7 @@ extern size_t opt_mbuf_total; extern int opt_mm_select; extern int opt_fa_select; extern int opt_fa_split; +extern int opt_li_select; extern int opt_gdn_select; extern int opt_ar_select; extern int opt_ar_scatter; @@ -45,36 +40,41 @@ extern int opt_rt_repack; if (opt_verbose) GGML_LOG_DEBUG(__VA_ARGS__) #endif -static inline bool htp_tensor_is_fuseable(const struct ggml_tensor * t) { - if (!t || !t->extra) return false; - const auto * extra = (const struct htp_tensor_extra *) t->extra; - return (extra->flags & HTP_TENSOR_FUSEABLE) != 0; +static inline uint32_t ggml_hexagon_tensor_get_flags(const struct ggml_tensor * t) { + return (uint32_t) (uintptr_t) t->extra; } -static inline bool htp_can_rt_repack(const struct ggml_tensor * t) { - if (opt_rt_repack == 0 || !htp_is_tiled_weight_tensor(t)) { - return false; - } - if (t->buffer && ggml_backend_buft_is_host(t->buffer->buft)) { - return false; - } - return opt_rt_repack > 1 || ggml_is_quantized(t->type); +static inline void ggml_hexagon_tensor_set_flags(struct ggml_tensor * t, uint32_t flags) { + t->extra = (void *) (uintptr_t) (ggml_hexagon_tensor_get_flags(t) | flags); } -static inline bool htp_tensor_is_tiled(const struct ggml_tensor * t) { - const auto * extra = (const struct htp_tensor_extra *) t->extra; - if (extra && (extra->flags & HTP_TENSOR_TILED)) { - return true; - } - return htp_can_rt_repack(t); +static inline void ggml_hexagon_tensor_clear_flags(struct ggml_tensor * t, uint32_t flags) { + t->extra = (void *) (uintptr_t) (ggml_hexagon_tensor_get_flags(t) & ~flags); } -static inline bool htp_needs_repack(const struct ggml_tensor * t) { - const auto * extra = (const struct htp_tensor_extra *) t->extra; - if (extra && (extra->flags & HTP_TENSOR_TILED)) { +static inline bool ggml_hexagon_tensor_is_fuseable(const struct ggml_tensor * t) { + return (ggml_hexagon_tensor_get_flags(t) & HTP_TENSOR_FUSEABLE) != 0; +} + +static inline bool ggml_hexagon_tensor_is_tiled(const struct ggml_tensor * t) { + return (ggml_hexagon_tensor_get_flags(t) & HTP_TENSOR_TILED) != 0; +} + +static inline bool ggml_hexagon_tensor_can_tile(const struct ggml_tensor * t) { + if (opt_rt_repack == 0) { return false; } - return htp_can_rt_repack(t); + if (t->buffer) { + if (ggml_backend_buft_is_host(t->buffer->buft) || + ggml_backend_buffer_get_usage(t->buffer) == GGML_BACKEND_BUFFER_USAGE_COMPUTE) { + return false; + } + } + return htp_is_tiled_weight_tensor(t); +} + +static inline bool ggml_hexagon_tensor_will_tile(const struct ggml_tensor * t) { + return ggml_hexagon_tensor_is_tiled(t) || ggml_hexagon_tensor_can_tile(t); } static inline bool htp_tensors_overlap(const struct ggml_tensor * a, const struct ggml_tensor * b) { @@ -97,7 +97,7 @@ struct ggml_hexagon_session; const char * htp_session_name(const struct ggml_hexagon_session * sess); size_t htp_session_vtcm_size(const struct ggml_hexagon_session * sess); -uint32_t htp_session_n_threads(const struct ggml_hexagon_session * sess); +uint32_t htp_session_n_hvx(const struct ggml_hexagon_session * sess); uint32_t htp_session_n_hmx(const struct ggml_hexagon_session * sess); uint32_t htp_session_mdev_count(const struct ggml_hexagon_session * sess); diff --git a/ggml/src/ggml-hexagon/htp-drv.cpp b/ggml/src/ggml-hexagon/htp-drv.cpp index 437e367c9d35..9ae4ea7275ac 100644 --- a/ggml/src/ggml-hexagon/htp-drv.cpp +++ b/ggml/src/ggml-hexagon/htp-drv.cpp @@ -1,7 +1,9 @@ +#include #include #include #include #include +#include #ifdef _WIN32 # define WIN32_LEAN_AND_MEAN @@ -28,6 +30,13 @@ #include +extern int opt_verbose; + +#ifndef HEX_VERBOSE +#define HEX_VERBOSE(...) \ + if (opt_verbose) GGML_LOG_DEBUG(__VA_ARGS__) +#endif + // // Driver API types // @@ -69,7 +78,7 @@ typedef int (*fastrpc_munmap_pfn_t)(int domain, int fd, void *addr, size_t lengt typedef int (*remote_handle64_open_pfn_t)(const char* name, remote_handle64 *ph); typedef int (*remote_handle64_invoke_pfn_t)(remote_handle64 h, uint32_t dwScalars, remote_arg *pra); -typedef int (*remote_handle64_close_pfn_t)(remote_handle h); +typedef int (*remote_handle64_close_pfn_t)(remote_handle64 h); typedef int (*remote_handle_control_pfn_t)(uint32_t req, void* data, uint32_t datalen); typedef int (*remote_handle64_control_pfn_t)(remote_handle64 h, uint32_t req, void* data, uint32_t datalen); typedef int (*remote_session_control_pfn_t)(uint32_t req, void *data, uint32_t datalen); @@ -423,3 +432,188 @@ int htpdrv_get_arch(int domain, int * arch) { *arch = (int) ((val >> 4) * 10 + (val & 0x0f)); return 0; } + +// Directory of the HTP library as a path with a trailing slash, or empty if it cannot be resolved. +// Checks GGML_HEXAGON_LIBHTP_DIR first, then the module directory and ../lib. +// On Windows, non-ASCII directories or paths with spaces use the 8.3 short name. +std::string htp_lib_dir() { + namespace fs = std::filesystem; + + std::vector candidates; + + const char * env_dir = getenv("GGML_HEXAGON_LIBHTP_DIR"); + if (env_dir && *env_dir) { + std::error_code ec; + fs::path p = fs::absolute(env_dir, ec); + if (!ec) { + candidates.push_back(p); + } + } + +#ifdef _WIN32 + HMODULE hm = nullptr; + if (GetModuleHandleExW(GET_MODULE_HANDLE_EX_FLAG_FROM_ADDRESS | GET_MODULE_HANDLE_EX_FLAG_UNCHANGED_REFCOUNT, + (LPCWSTR) &htp_lib_dir, &hm)) { + std::wstring buf(32768, L'\0'); + DWORD n = GetModuleFileNameW(hm, buf.data(), (DWORD) buf.size()); + if (n > 0 && n < buf.size()) { + buf.resize(n); + std::error_code ec; + fs::path mod_path = fs::canonical(buf, ec); + if (ec) { + mod_path = fs::absolute(buf, ec); + } + const fs::path mod_dir = mod_path.parent_path(); + candidates.push_back(mod_dir); + candidates.push_back(mod_dir / "lib"); + candidates.push_back(mod_dir / ".." / "lib"); + } + } +#elif defined(__linux__) || defined(__ANDROID__) + Dl_info info; + if (dladdr((void *) &htp_lib_dir, &info) && info.dli_fname) { + std::error_code ec; + fs::path mod_path = fs::canonical(info.dli_fname, ec); + if (ec) { + mod_path = fs::absolute(info.dli_fname, ec); + } + const fs::path mod_dir = mod_path.parent_path(); + candidates.push_back(mod_dir); + candidates.push_back(mod_dir / "lib"); + candidates.push_back(mod_dir / ".." / "lib"); + } +#endif + + for (size_t i = 0; i < candidates.size(); ++i) { + const fs::path & d = candidates[i]; + std::error_code ec; + if (!fs::is_directory(d, ec)) { + continue; + } + + // For auto-discovered directories, verify it contains an HTP library + const bool is_env = (env_dir && i == 0); + if (!is_env) { + bool found = false; + for (const auto & entry : fs::directory_iterator(d, ec)) { + if (ec) { + break; + } + const std::string fn = entry.path().filename().string(); + if (fn.rfind("libggml-htp-", 0) == 0) { + found = true; + break; + } + } + if (!found) { + continue; + } + } + + std::error_code ec_canon; + fs::path abs_d = fs::canonical(d, ec_canon); + if (ec_canon) { + abs_d = fs::absolute(d, ec_canon); + } + +#ifdef _WIN32 + std::wstring dir = abs_d.wstring(); + auto is_ascii = [](const std::wstring & s) { + return std::all_of(s.begin(), s.end(), [](wchar_t c) { return c < 0x80; }); + }; + auto has_spaces = [](const std::wstring & s) { + return std::any_of(s.begin(), s.end(), [](wchar_t c) { return c == L' '; }); + }; + + if (!is_ascii(dir) || has_spaces(dir)) { + DWORD m = GetShortPathNameW(dir.c_str(), nullptr, 0); + if (m > 0) { + std::wstring sp(m, L'\0'); + DWORD res = GetShortPathNameW(dir.c_str(), sp.data(), (DWORD) sp.size()); + if (res > 0 && res < sp.size()) { + sp.resize(res); + if (is_ascii(sp)) { + dir = sp; + } + } + } + if (!is_ascii(dir)) { + continue; + } + } + if (dir.back() != L'\\' && dir.back() != L'/') { + dir += L'\\'; + } + return std::string(dir.begin(), dir.end()); +#else + std::string dir = abs_d.string(); + if (dir.back() != '/') { + dir += '/'; + } + return dir; +#endif + } + + return {}; +} + +int htpdrv_set_dsp_search_path(int domain_id, const std::string & path, const char * dev_name) { + if (path.empty()) { + return AEE_SUCCESS; + } + + std::string clean_path = path; + while (clean_path.size() > 1 && (clean_path.back() == '/' || clean_path.back() == '\\')) { + clean_path.pop_back(); + } + + fastrpc_manage_dsp_lib_path p {}; + p.effec_domain_id = domain_id; + p.type = FASTRPC_SET_PATH; + p.path = const_cast(clean_path.c_str()); + p.pathlen = clean_path.size() + 1; + + int err = remote_session_control(FASTRPC_MANAGE_DSP_LIBRARY_PATH, (void *) &p, sizeof(p)); + if (err != AEE_SUCCESS && domain_id != -1) { + p.effec_domain_id = -1; + err = remote_session_control(FASTRPC_MANAGE_DSP_LIBRARY_PATH, (void *) &p, sizeof(p)); + } + + const char * name = dev_name ? dev_name : "driver"; + if (err == AEE_SUCCESS) { + GGML_LOG_INFO("ggml-hex: %s set DSP search path (domain %d) to %s\n", name, p.effec_domain_id, clean_path.c_str()); + return AEE_SUCCESS; + } + + HEX_VERBOSE("ggml-hex: %s FASTRPC_MANAGE_DSP_LIBRARY_PATH failed (0x%x)\n", name, err); + return err; +} + +void htpdrv_set_dsp_env_path(const std::string & path, const char * dev_name) { + if (path.empty()) { + return; + } + + std::string clean_path = path; + while (clean_path.size() > 1 && (clean_path.back() == '/' || clean_path.back() == '\\')) { + clean_path.pop_back(); + } + + const char * name = dev_name ? dev_name : "driver"; + HEX_VERBOSE("ggml-hex: %s setting DSP search path via process env to %s\n", name, clean_path.c_str()); + + static const char * const env_names[] = { + "ADSP_LIBRARY_PATH", + "CDSP_LIBRARY_PATH", + "CDSP1_LIBRARY_PATH", + "CDSP2_LIBRARY_PATH", + "CDSP3_LIBRARY_PATH", + }; + for (const char * env_name : env_names) { +#ifdef _WIN32 + SetEnvironmentVariableA(env_name, clean_path.c_str()); +#else + setenv(env_name, clean_path.c_str(), 1); +#endif + } +} diff --git a/ggml/src/ggml-hexagon/htp-drv.h b/ggml/src/ggml-hexagon/htp-drv.h index 8232780e7fdc..c88a72940adb 100644 --- a/ggml/src/ggml-hexagon/htp-drv.h +++ b/ggml/src/ggml-hexagon/htp-drv.h @@ -120,4 +120,10 @@ HTPDRV_API int remote_system_request(system_req_payload * req); #ifdef __cplusplus } + +#include + +std::string htp_lib_dir(); +int htpdrv_set_dsp_search_path(int domain_id, const std::string & path, const char * dev_name = nullptr); +void htpdrv_set_dsp_env_path(const std::string & path, const char * dev_name = nullptr); #endif diff --git a/ggml/src/ggml-hexagon/htp-fusion.cpp b/ggml/src/ggml-hexagon/htp-fusion.cpp index ba2dd64106c9..eb08626692f4 100644 --- a/ggml/src/ggml-hexagon/htp-fusion.cpp +++ b/ggml/src/ggml-hexagon/htp-fusion.cpp @@ -35,7 +35,7 @@ static inline bool htp_match_consumer( return false; } const ggml_tensor * prev_out = prev.dst(); - if (!htp_tensor_is_fuseable(prev_out)) { + if (!ggml_hexagon_tensor_is_fuseable(prev_out)) { return false; } if (htp_same_tensor_data(cur.src0(), prev_out)) { @@ -53,14 +53,14 @@ static bool is_supported_mul_mat_nx_kernel(const ggml_tensor * src0, const struc if (kparams->n_hmx) { return kparams->kernel_type == HTP_MM_KERNEL_HMX_2D; } - return htp_tensor_is_tiled(src0) && (kparams->kernel_type == HTP_MM_KERNEL_HVX_QUANT_ROW); + return ggml_hexagon_tensor_is_tiled(src0) && (kparams->kernel_type == HTP_MM_KERNEL_HVX_QUANT_ROW); } static bool is_supported_mul_mat_id_nx_kernel(const ggml_tensor * src0, const struct htp_mm_kernel_params * kparams) { if (kparams->n_hmx) { return kparams->kernel_type == HTP_MM_KERNEL_HMX_2D; } - return htp_tensor_is_tiled(src0) && + return ggml_hexagon_tensor_is_tiled(src0) && (kparams->kernel_type == HTP_MM_KERNEL_HVX_QUANT_ROW || kparams->kernel_type == HTP_MM_KERNEL_HVX_QUANT_BLOCK); } @@ -74,7 +74,7 @@ static bool fuse_allreduce_add(const ggml_hexagon_session * sess, htp_opnode & p const uint32_t n_ranks = (uint32_t) ar_kparams->n_ranks; const ggml_tensor * ar_local = prev.inputs[rank]; - if (!htp_tensor_is_fuseable(ar_local)) return false; + if (!ggml_hexagon_tensor_is_fuseable(ar_local)) return false; const ggml_tensor * add_dst = cur.dst(); const ggml_tensor * res_tensor = nullptr; @@ -157,6 +157,30 @@ static bool fuse_rms_norm_mul(const ggml_hexagon_session * sess, htp_opnode & pr return true; } +// post-norm residual: dst = rms_norm(x) * w + r +static bool fuse_rms_norm_mul_add(const ggml_hexagon_session * sess, htp_opnode & prev, const htp_opnode & cur) { + if (prev.opcode != HTP_OP_RMS_NORM_MUL) return false; + + const ggml_tensor * r = nullptr; + if (!htp_match_consumer(prev, cur, /*commutative=*/ true, &r)) return false; + + const ggml_tensor * src0 = prev.src0(); + const ggml_tensor * weight = prev.inputs[1]; + const ggml_tensor * dst = cur.dst(); + if (!ggml_are_same_layout(src0, r)) return false; + if (htp_inputs_overlap_unsafe(prev, dst, r)) return false; + + struct htp_unary_kernel_params kparams; + if (!htp_kparams_solve_rms_norm_mul_add(sess, src0, weight, dst, &kparams)) return false; + + prev.opcode = HTP_OP_RMS_NORM_MUL_ADD; + prev.inputs.push_back(r); + prev.outputs[0] = dst; + prev.fused.push_back(cur.node); + prev.set_kparams(kparams); + return true; +} + static bool fuse_rms_norm_scale(const ggml_hexagon_session * sess, htp_opnode & prev, const htp_opnode & cur) { GGML_UNUSED(sess); if (prev.opcode != HTP_OP_RMS_NORM && prev.opcode != HTP_OP_RMS_NORM_MUL) return false; @@ -215,11 +239,11 @@ static bool fuse_mul_mat_add(const ggml_hexagon_session * sess, htp_opnode & pre if (kparams.n_hmx > 0 && orig_kparams->n_hmx > 0) { if (kparams.m_chunk < orig_kparams->m_chunk || kparams.n_chunk < orig_kparams->n_chunk || - kparams.n_act_threads < orig_kparams->n_act_threads) { - HEX_VERBOSE("ggml-hex: %s skip MUL_MAT_ADD fusion: HMX efficiency reduced (m %d->%d, n %d->%d, th %d->%d)\n", + kparams.n_act_hvx < orig_kparams->n_act_hvx) { + HEX_VERBOSE("ggml-hex: %s skip MUL_MAT_ADD fusion: HMX efficiency reduced (m %d->%d, n %d->%d, hvx %d->%d)\n", htp_session_name(sess), orig_kparams->m_chunk, kparams.m_chunk, orig_kparams->n_chunk, kparams.n_chunk, - orig_kparams->n_act_threads, kparams.n_act_threads); + orig_kparams->n_act_hvx, kparams.n_act_hvx); return false; } } @@ -250,7 +274,7 @@ static bool fuse_mul_mat_nx(const ggml_hexagon_session * sess, htp_opnode & prev if (x_in != x_orig || w_in->type != w0->type || w_in->ne[0] != w0->ne[0] || w0->ne[2] != 1 || w0->ne[3] != 1 || w_in->ne[2] != 1 || w_in->ne[3] != 1 || - htp_tensor_is_tiled(w_in) != htp_tensor_is_tiled(w0) || + ggml_hexagon_tensor_is_tiled(w_in) != ggml_hexagon_tensor_is_tiled(w0) || orig_kparams->n_hmx != cur_kparams->n_hmx) { return false; } @@ -295,7 +319,7 @@ static bool fuse_mul_mat_id_nx(const ggml_hexagon_session * sess, htp_opnode & p if (x_in != x || ids_in != ids || w_in->type != w0->type || w_in->ne[0] != w0->ne[0] || w_in->ne[2] != w0->ne[2] || - htp_tensor_is_tiled(w_in) != htp_tensor_is_tiled(w0) || + ggml_hexagon_tensor_is_tiled(w_in) != ggml_hexagon_tensor_is_tiled(w0) || orig_kparams->n_hmx != cur_kparams->n_hmx) { return false; } @@ -522,8 +546,8 @@ static bool fuse_binary_nx(const ggml_hexagon_session * sess, htp_opnode & prev, const uint32_t total_elems = (uint32_t) ggml_nelements(src0); const uint32_t elem_size = (uint32_t) ggml_type_size(src0->type); - const uint32_t max_threads = htp_session_n_threads(sess); - if (!htp_binary_solve_chunked(&L, &kparams, total_elems, elem_size, max_threads, n_srcs, false, vtcm_size)) { + const uint32_t max_hvx = htp_session_n_hvx(sess); + if (!htp_binary_solve_chunked(&L, &kparams, total_elems, elem_size, max_hvx, n_srcs, false, vtcm_size)) { return false; } } @@ -598,19 +622,17 @@ int htp_fusion_max(const struct ggml_cgraph * graph, int idx) { void htp_fusion_tag_graph(struct ggml_cgraph * graph) { for (int i = 0; i < graph->n_nodes; i++) { - auto * extra = (htp_tensor_extra *) graph->nodes[i]->extra; - if (!extra) continue; - - extra->flags &= ~HTP_TENSOR_FUSEABLE; + ggml_tensor * node = graph->nodes[i]; + ggml_hexagon_tensor_clear_flags(node, HTP_TENSOR_FUSEABLE); - const ggml_op op = graph->nodes[i]->op; + const ggml_op op = node->op; if ((op == GGML_OP_RMS_NORM || op == GGML_OP_SCALE || op == GGML_OP_REPEAT || op == GGML_OP_UNARY || op == GGML_OP_ADD || op == GGML_OP_SUB || op == GGML_OP_MUL || op == GGML_OP_DIV || op == GGML_OP_MUL_MAT_ID || op == GGML_OP_SSM_CONV || - (op == GGML_OP_MUL_MAT && !htp_is_hadamard(graph->nodes[i]))) && + (op == GGML_OP_MUL_MAT && !htp_is_hadamard(node))) && ggml_node_has_n_uses(graph, i, 1)) { - extra->flags |= HTP_TENSOR_FUSEABLE; + ggml_hexagon_tensor_set_flags(node, HTP_TENSOR_FUSEABLE); } } } @@ -622,6 +644,7 @@ bool htp_fusion_try_fuse(const struct ggml_hexagon_session * sess, struct htp_op case HTP_OP_ADD: return (htp_fusion_enabled(HTP_FUSE_ALLREDUCE_ADD) && fuse_allreduce_add(sess, prev, cur)) || (htp_fusion_enabled(HTP_FUSE_MUL_MAT_ADD) && fuse_mul_mat_add(sess, prev, cur)) || + (htp_fusion_enabled(HTP_FUSE_RMS_NORM_MUL_ADD) && fuse_rms_norm_mul_add(sess, prev, cur)) || (htp_fusion_enabled(HTP_FUSE_BINARY_NX) && fuse_binary_nx(sess, prev, cur)); case HTP_OP_MUL: return (htp_fusion_enabled(HTP_FUSE_RMS_NORM_MUL) && fuse_rms_norm_mul(sess, prev, cur)) || diff --git a/ggml/src/ggml-hexagon/htp-fusion.h b/ggml/src/ggml-hexagon/htp-fusion.h index e794c1daa301..e4f711c872f5 100644 --- a/ggml/src/ggml-hexagon/htp-fusion.h +++ b/ggml/src/ggml-hexagon/htp-fusion.h @@ -16,6 +16,7 @@ enum htp_fusion_flags { HTP_FUSE_BINARY_NX = (1 << 10), // 1024 HTP_FUSE_RMS_NORM_SCALE = (1 << 11), // 2048 HTP_FUSE_SSM_CONV_SILU = (1 << 12), // 4096 + HTP_FUSE_RMS_NORM_MUL_ADD = (1 << 13), // 8192 }; static inline bool htp_fusion_enabled(int flag) { diff --git a/ggml/src/ggml-hexagon/htp-kparams.cpp b/ggml/src/ggml-hexagon/htp-kparams.cpp index 03b5a3b506d3..8c75be289ba4 100644 --- a/ggml/src/ggml-hexagon/htp-kparams.cpp +++ b/ggml/src/ggml-hexagon/htp-kparams.cpp @@ -50,17 +50,22 @@ htp_op_code htp_op_remap(const struct ggml_tensor * t) { case GGML_OP_ROPE: return HTP_OP_ROPE; case GGML_OP_REPEAT: return HTP_OP_REPEAT; case GGML_OP_CUMSUM: return HTP_OP_CUMSUM; + case GGML_OP_TOP_K_MASK: return HTP_OP_TOP_K_MASK; case GGML_OP_FILL: return HTP_OP_FILL; case GGML_OP_DIAG: return HTP_OP_DIAG; case GGML_OP_SOLVE_TRI: return HTP_OP_SOLVE_TRI; + case GGML_OP_DSV4_HC_PRE: return HTP_OP_DSV4_HC_PRE; + case GGML_OP_DSV4_HC_POST: return HTP_OP_DSV4_HC_POST; case GGML_OP_TRI: return HTP_OP_TRI; case GGML_OP_PAD: return HTP_OP_PAD; case GGML_OP_IM2COL: return HTP_OP_IM2COL; case GGML_OP_IM2COL_3D: return HTP_OP_IM2COL_3D; case GGML_OP_CONV_2D: return HTP_OP_CONV_2D; case GGML_OP_CONV_2D_DW: return HTP_OP_CONV_2D_DW; + case GGML_OP_CONV_3D: return HTP_OP_CONV_3D; case GGML_OP_ROLL: return HTP_OP_ROLL; case GGML_OP_UPSCALE: return HTP_OP_UPSCALE; + case GGML_OP_LIGHTNING_INDEXER: return HTP_OP_LIGHTNING_INDEXER; case GGML_OP_UNARY: switch (ggml_get_unary_op(t)) { @@ -149,7 +154,7 @@ bool htp_kparams_solve_allreduce( kparams->n_ranks = (int32_t) n_ranks; kparams->is_row_bcast = (has_add && is_row_bcast) ? 1 : 0; - const uint32_t max_threads = htp_session_n_threads(sess); + const uint32_t max_hvx = htp_session_n_hvx(sess); const size_t vtcm_size = htp_session_vtcm_size(sess); const uint32_t nelem = (uint32_t) ggml_nelements(dst); @@ -203,27 +208,27 @@ bool htp_kparams_solve_allreduce( if (use_1d) { const uint32_t rank_nelem = (uint32_t) kparams->rank_nelem; - const uint32_t n_threads = (std::min)(max_threads, (std::max)(1u, rank_nelem / 128)); - kparams->n_threads = n_threads; - const size_t n_vtcm_buffers = htp_allreduce_vtcm_buffer_count(n_ranks, n_threads, has_add, is_row_bcast); + const uint32_t n_hvx = (std::min)(max_hvx, (std::max)(1u, rank_nelem / 128)); + kparams->n_hvx = n_hvx; + const size_t n_vtcm_buffers = htp_allreduce_vtcm_buffer_count(n_ranks, n_hvx, has_add, is_row_bcast); uint32_t block_elems = 65536; - if (block_elems > rank_nelem / n_threads && rank_nelem / n_threads > 128) { - block_elems = hex_round_up(rank_nelem / (n_threads * 2), 128); + if (block_elems > rank_nelem / n_hvx && rank_nelem / n_hvx > 128) { + block_elems = hex_round_up(rank_nelem / (n_hvx * 2), 128); } block_elems = (std::max)(128u, block_elems); kparams->block_elems = block_elems; - kparams->vtcm_size_per_thread = 2 * block_elems * elem_size; - kparams->vtcm_size = n_vtcm_buffers * kparams->vtcm_size_per_thread; + kparams->vtcm_size_per_hvx = 2 * block_elems * elem_size; + kparams->vtcm_size = n_vtcm_buffers * kparams->vtcm_size_per_hvx; while ((size_t) kparams->vtcm_size > vtcm_size && block_elems > 128) { const size_t max_bytes_per_buf = vtcm_size / (n_vtcm_buffers * 2); block_elems = (uint32_t) hex_align_down((size_t) (max_bytes_per_buf / elem_size), 128); if (block_elems < 128) break; kparams->block_elems = block_elems; - kparams->vtcm_size_per_thread = 2 * block_elems * elem_size; - kparams->vtcm_size = n_vtcm_buffers * kparams->vtcm_size_per_thread; + kparams->vtcm_size_per_hvx = 2 * block_elems * elem_size; + kparams->vtcm_size = n_vtcm_buffers * kparams->vtcm_size_per_hvx; } if (vtcm_size < (size_t) kparams->vtcm_size || block_elems < 128) { @@ -232,33 +237,33 @@ bool htp_kparams_solve_allreduce( return false; } - kparams->elems_per_thread = hex_round_up((rank_nelem + n_threads - 1) / n_threads, block_elems); + kparams->elems_per_hvx = hex_round_up((rank_nelem + n_hvx - 1) / n_hvx, block_elems); kparams->kernel_type = HTP_ALLREDUCE_KERNEL_DMA_1D; return true; } else { const uint32_t rank_nrows = (uint32_t) kparams->rank_nelem; - const uint32_t n_threads = (std::min)(max_threads, (std::max)(1u, rank_nrows)); - kparams->n_threads = n_threads; - const size_t n_vtcm_buffers = htp_allreduce_vtcm_buffer_count(n_ranks, n_threads, has_add, is_row_bcast); + const uint32_t n_hvx = (std::min)(max_hvx, (std::max)(1u, rank_nrows)); + kparams->n_hvx = n_hvx; + const size_t n_vtcm_buffers = htp_allreduce_vtcm_buffer_count(n_ranks, n_hvx, has_add, is_row_bcast); const uint32_t row_bytes = ne0 * elem_size; const uint32_t row_size_aligned = (uint32_t) hex_align_up(row_bytes, 128); kparams->row_size_aligned = row_size_aligned; - const uint32_t nrows_per_thread = (rank_nrows + n_threads - 1) / n_threads; - uint32_t block_rows = (std::min)(128u, nrows_per_thread); + const uint32_t nrows_per_hvx = (rank_nrows + n_hvx - 1) / n_hvx; + uint32_t block_rows = (std::min)(128u, nrows_per_hvx); block_rows = (std::max)(1u, block_rows); kparams->block_elems = block_rows; - kparams->vtcm_size_per_thread = 2 * (block_rows * row_size_aligned); - kparams->vtcm_size = n_vtcm_buffers * kparams->vtcm_size_per_thread; + kparams->vtcm_size_per_hvx = 2 * (block_rows * row_size_aligned); + kparams->vtcm_size = n_vtcm_buffers * kparams->vtcm_size_per_hvx; while ((size_t) kparams->vtcm_size > vtcm_size && block_rows > 1) { const size_t max_rows_per_buf = vtcm_size / (n_vtcm_buffers * 2 * row_size_aligned); block_rows = (std::max)(1u, (uint32_t) max_rows_per_buf); kparams->block_elems = block_rows; - kparams->vtcm_size_per_thread = 2 * (block_rows * row_size_aligned); - kparams->vtcm_size = n_vtcm_buffers * kparams->vtcm_size_per_thread; + kparams->vtcm_size_per_hvx = 2 * (block_rows * row_size_aligned); + kparams->vtcm_size = n_vtcm_buffers * kparams->vtcm_size_per_hvx; if (max_rows_per_buf == 0) break; } @@ -268,7 +273,7 @@ bool htp_kparams_solve_allreduce( return false; } - kparams->elems_per_thread = nrows_per_thread; + kparams->elems_per_hvx = nrows_per_hvx; kparams->kernel_type = HTP_ALLREDUCE_KERNEL_DMA_2D; return true; } @@ -308,12 +313,12 @@ static bool htp_flash_attn_is_hmx_eligible( return true; } - const uint32_t n_threads = htp_session_n_threads(sess); + const uint32_t n_hvx = htp_session_n_hvx(sess); const uint32_t n_head = q->ne[2]; const uint32_t n_kv_heads = k->ne[2]; const uint32_t G = n_kv_heads > 0 ? n_head / n_kv_heads : 1; const uint32_t S = k->ne[1]; - const uint32_t total_threads = n_threads * htp_session_mdev_count(sess); + const uint32_t total_threads = n_hvx * htp_session_mdev_count(sess); if (DK > 256 || (DK == 256 && (DV > 256 || n_kv_heads < total_threads))) { return true; @@ -321,7 +326,7 @@ static bool htp_flash_attn_is_hmx_eligible( const bool is_tile_aligned = (G > 0 && (32 % G == 0)); if (!is_tile_aligned) { - if (n_threads >= 6) { + if (n_hvx >= 6) { return false; } return S >= 1024; @@ -331,9 +336,9 @@ static bool htp_flash_attn_is_hmx_eligible( if (DK == 256) { s_cross = 4096; } else if (DK <= 64) { - s_cross = (n_threads >= 8) ? 2048 : ((n_threads >= 6) ? 768 : 512); + s_cross = (n_hvx >= 8) ? 2048 : ((n_hvx >= 6) ? 768 : 512); } else { - s_cross = (n_threads >= 8) ? 1024 : 512; + s_cross = (n_hvx >= 8) ? 1024 : 512; } return S >= s_cross; @@ -377,7 +382,7 @@ static bool htp_kparams_solve_flash_attn( return false; } - const uint32_t n_threads = htp_session_n_threads(sess); + const uint32_t n_hvx = htp_session_n_hvx(sess); const size_t vtcm_size = htp_session_vtcm_size(sess); const uint32_t neq0 = q->ne[0]; @@ -429,7 +434,7 @@ static bool htp_kparams_solve_flash_attn( const uint32_t DK_pad = hex_round_up(DK, 64); const uint32_t DV_pad = hex_round_up(DV, 64); size_t Br = 0, Bc = 0; - int ret = hmx_fa_find_chunk_size(&Br, &Bc, G, DK_pad, DV_pad, neq1, nek1, vtcm_size, n_threads, (q->type == GGML_TYPE_F32), sinks != nullptr, n_head, fa_team_size); + int ret = hmx_fa_find_chunk_size(&Br, &Bc, G, DK_pad, DV_pad, neq1, nek1, vtcm_size, n_hvx, (q->type == GGML_TYPE_F32), sinks != nullptr, n_head, fa_team_size); if (ret == 0) { kparams->kernel_type = HTP_FA_KERNEL_HMX; if (logit_softcap == 0.0f) { @@ -442,10 +447,10 @@ static bool htp_kparams_solve_flash_attn( kparams->Br = Br; kparams->Bc = Bc; kparams->n_kv_blocks = (nek1 + Bc - 1) / Bc; - kparams->n_threads = (kparams->n_kv_blocks >= 3 && n_threads >= 2) ? n_threads : 1; + kparams->n_hvx = (kparams->n_kv_blocks >= 3 && n_hvx >= 2) ? n_hvx : 1; kparams->u.hmx.g_br = hex_align_up(G * Br, 32); - kparams->u.hmx.pipeline = (kparams->n_kv_blocks >= 3 && n_threads >= 2) ? 1 : 0; - kparams->vtcm_size = hmx_fa_compute_vtcm_usage(G, DK_pad, DV_pad, Br, Bc, kparams->n_threads, kparams->u.hmx.pipeline != 0, (q->type == GGML_TYPE_F32), sinks != nullptr, n_head); + kparams->u.hmx.pipeline = (kparams->n_kv_blocks >= 3 && n_hvx >= 2) ? 1 : 0; + kparams->vtcm_size = hmx_fa_compute_vtcm_usage(G, DK_pad, DV_pad, Br, Bc, kparams->n_hvx, kparams->u.hmx.pipeline != 0, (q->type == GGML_TYPE_F32), sinks != nullptr, n_head); kparams->u.hmx.mask_broadcast = (mask != nullptr && mask->ne[2] == 1) ? 1 : 0; kparams->u.hmx.div_G = init_fastdiv_values(G); if (mask) { @@ -460,13 +465,13 @@ static bool htp_kparams_solve_flash_attn( kparams->Br = 1; kparams->Bc = 64; kparams->n_kv_blocks = (k->ne[1] + 64 - 1) / 64; - kparams->n_threads = n_threads; + kparams->n_hvx = n_hvx; const size_t size_q_row_padded = hex_round_up(q->ne[0] * ((q->type == GGML_TYPE_F32) ? 4 : 2), 128); const size_t size_k_row_padded = hex_round_up(k->ne[0] * 2, 128); const size_t size_v_row_padded = hex_round_up(v->ne[0] * 2, 128); - kparams->vtcm_size = hvx_fa_compute_vtcm_usage(DK, DV, (q->type == GGML_TYPE_F32), mask != nullptr, sinks != nullptr, n_head, n_threads); + kparams->vtcm_size = hvx_fa_compute_vtcm_usage(DK, DV, (q->type == GGML_TYPE_F32), mask != nullptr, sinks != nullptr, n_head, n_hvx); if ((size_t) kparams->vtcm_size > vtcm_size) { HEX_VERBOSE("ggml-hex: skip flash_attn_ext because VTCM needed (%d) > budget (%zu)\n", kparams->vtcm_size, vtcm_size); @@ -549,7 +554,7 @@ static bool htp_kparams_solve_gated_delta_net( const uint32_t rq3 = (uint32_t) (n_seqs / q->ne[3]); const uint32_t rk3 = (uint32_t) (n_seqs / k->ne[3]); const uint32_t total_rows = (uint32_t) (H * n_seqs); - const uint32_t n_threads = (std::min)(htp_session_n_threads(sess), total_rows); + const uint32_t n_hvx = (std::min)(htp_session_n_hvx(sess), total_rows); const size_t vtcm_size = htp_session_vtcm_size(sess); const bool can_use_hmx = (opt_gdn_select >= 2) && @@ -563,7 +568,7 @@ static bool htp_kparams_solve_gated_delta_net( struct htp_gdn_vtcm_layout hvx_layout; uint32_t n_heads_batch = 1; - if (can_use_hmx && htp_gdn_hmx_solve_layout(&hmx_layout, (uint32_t) S_v, HTP_GDN_CHUNK_SIZE, total_rows, vtcm_size, n_threads, true, &n_heads_batch)) { + if (can_use_hmx && htp_gdn_hmx_solve_layout(&hmx_layout, (uint32_t) S_v, HTP_GDN_CHUNK_SIZE, total_rows, vtcm_size, n_hvx, true, &n_heads_batch)) { kparams->kernel_type = HTP_GDN_KERNEL_HMX_CHUNKED; kparams->pipeline = hmx_layout.pipeline ? 1 : 0; kparams->chunk_size = HTP_GDN_CHUNK_SIZE; @@ -571,7 +576,7 @@ static bool htp_kparams_solve_gated_delta_net( kparams->n_heads_batch = (uint16_t) n_heads_batch; kparams->vtcm_size = (uint32_t) hmx_layout.total_bytes; } else { - htp_gdn_vtcm_layout_build(&hvx_layout, (uint32_t) S_v, n_threads); + htp_gdn_vtcm_layout_build(&hvx_layout, (uint32_t) S_v, n_hvx); if (hvx_layout.total_bytes > vtcm_size) { return false; } @@ -581,7 +586,7 @@ static bool htp_kparams_solve_gated_delta_net( kparams->vtcm_size = (uint32_t) hvx_layout.total_bytes; } - kparams->n_threads = n_threads; + kparams->n_hvx = n_hvx; kparams->S_v = (uint32_t) S_v; kparams->H = (uint32_t) H; kparams->n_tokens = (uint32_t) n_tokens; @@ -624,7 +629,7 @@ static bool htp_matmul_is_hmx_eligible( return false; } - if (!htp_is_hmx_weight_type((ggml_type) wtype)) { + if (!htp_is_tiled_weight_type((ggml_type) wtype)) { return false; } @@ -632,7 +637,7 @@ static bool htp_matmul_is_hmx_eligible( return false; } - const bool is_tiled = htp_tensor_is_tiled(src0); + const bool is_tiled = ggml_hexagon_tensor_will_tile(src0); if (ggml_is_quantized((ggml_type) wtype) && !is_tiled) { return false; } @@ -670,9 +675,9 @@ static bool htp_kparams_solve_hmx_mm( size_t vtcm_budget, struct htp_mm_kernel_params * kparams ) { - const bool is_tiled = htp_tensor_is_tiled(src0); + const bool is_tiled = ggml_hexagon_tensor_will_tile(src0); const int aligned_tile_size = is_tiled ? htp_tensor_get_tile_padded_size(wtype) : 0; - const int n_threads = (int) htp_session_n_threads(sess); + const int n_hvx = (int) htp_session_n_hvx(sess); const uint32_t mdev_count = htp_session_mdev_count(sess); const int ne10 = act->ne[0]; @@ -694,10 +699,10 @@ static bool htp_kparams_solve_hmx_mm( size_t n_chunk = 0; size_t vtcm_size = 0; bool use_grouped = false; - int act_threads_selected = 0; + int act_hvx_selected = 0; if (is_batched_val && wtype == GGML_TYPE_F16 && !is_tiled && group_size > 1) { - if (htp_mm_hmx_solve_batched_params(wtype, ne00_padded, ne01_padded, m_for_solver, group_size, n_threads, pipeline, src2_size, vtcm_budget, &m_chunk, &n_chunk, &act_threads_selected, &vtcm_size)) { + if (htp_mm_hmx_solve_batched_params(wtype, ne00_padded, ne01_padded, m_for_solver, group_size, n_hvx, pipeline, src2_size, vtcm_budget, &m_chunk, &n_chunk, &act_hvx_selected, &vtcm_size)) { use_grouped = true; } } @@ -714,7 +719,7 @@ static bool htp_kparams_solve_hmx_mm( m_id_rows = hex_round_up(std::max(m_per_expert, 32), 32); } const uint32_t cost_m = is_matmul_id ? (uint32_t) m_id_rows : (uint32_t) m_for_solver; - if (!htp_mm_hmx_solve_2d_params(wtype, ne00_padded, m_id_rows, ne01_padded, m_for_solver_padded, cost_m, n_threads, pipeline, is_matmul_id, aligned_tile_size, src2_size, vtcm_budget, &m_chunk, &n_chunk, &act_threads_selected, &vtcm_size)) { + if (!htp_mm_hmx_solve_2d_params(wtype, ne00_padded, m_id_rows, ne01_padded, m_for_solver_padded, cost_m, n_hvx, pipeline, is_matmul_id, aligned_tile_size, src2_size, vtcm_budget, &m_chunk, &n_chunk, &act_hvx_selected, &vtcm_size)) { return false; } } @@ -723,13 +728,13 @@ static bool htp_kparams_solve_hmx_mm( kparams->pipeline = pipeline ? 1 : 0; kparams->m_chunk = m_chunk; kparams->n_chunk = n_chunk; - kparams->n_threads = n_threads; - kparams->n_act_threads = act_threads_selected; + kparams->n_hvx = n_hvx; + kparams->n_act_hvx = act_hvx_selected; kparams->tile_size = is_tiled ? htp_tensor_get_tile_size(wtype) : 0; kparams->aligned_tile_size = aligned_tile_size; kparams->act_row_size = htp_mm_tiled_act_row_size(wtype, ne10); kparams->vtcm_size = vtcm_size; - kparams->div_n_act_threads = init_fastdiv_values(act_threads_selected); + kparams->div_n_act_hvx = init_fastdiv_values(act_hvx_selected); kparams->div_ne00_padded = init_fastdiv_values(ne00_padded); kparams->vtcm_bias_size = (int32_t) src2_size; kparams->kernel_type = (is_batched && !is_matmul_id) ? HTP_MM_KERNEL_HMX_BATCHED : HTP_MM_KERNEL_HMX_2D; @@ -759,9 +764,9 @@ static void htp_kparams_solve_hvx_mm( return; } - const uint32_t n_threads = htp_session_n_threads(sess); + const uint32_t n_hvx = htp_session_n_hvx(sess); kparams->n_hmx = 0; - kparams->n_threads = n_threads; + kparams->n_hvx = n_hvx; GGML_UNUSED(ne02); GGML_UNUSED(ne03); @@ -775,7 +780,7 @@ static void htp_kparams_solve_hvx_mm( const bool k_align = (ne10 % htp_tensor_get_tile_k(wtype) == 0); if (is_matmul_id) { - kparams->kernel_type = (src1_nrows < (int) n_threads) + kparams->kernel_type = (src1_nrows < (int) n_hvx) ? HTP_MM_KERNEL_HVX_QUANT_BLOCK : HTP_MM_KERNEL_HVX_QUANT_ROW; kparams->act_row_size = htp_mm_tiled_act_row_size(wtype, ne10); @@ -784,7 +789,7 @@ static void htp_kparams_solve_hvx_mm( uint32_t best_n_prefetch = 2; for (uint32_t d = max_prefetch; d >= 2; d /= 2) { htp_mm_hvx_vtcm_layout_build( - &L, kparams->kernel_type, wtype, ne10, src1_nrows, n_threads, + &L, kparams->kernel_type, wtype, ne10, src1_nrows, n_hvx, 0, src0->nb[1], kparams->act_row_size, 0, d, true, false ); if (L.total_bytes <= vtcm_budget) { @@ -803,7 +808,7 @@ static void htp_kparams_solve_hvx_mm( } else { if (k_align && opt_mm_select >= 1) { kparams->act_row_size = htp_mm_tiled_act_row_size(wtype, ne10); - kparams->kernel_type = (src1_nrows < (int) n_threads) + kparams->kernel_type = (src1_nrows < (int) n_hvx) ? HTP_MM_KERNEL_HVX_QUANT_BLOCK : HTP_MM_KERNEL_HVX_QUANT_ROW; struct htp_mm_hvx_vtcm_layout L; @@ -811,7 +816,7 @@ static void htp_kparams_solve_hvx_mm( uint32_t best_n_prefetch = 2; for (uint32_t d = max_prefetch; d >= 2; d /= 2) { htp_mm_hvx_vtcm_layout_build( - &L, kparams->kernel_type, wtype, ne10, src1_nrows, n_threads, + &L, kparams->kernel_type, wtype, ne10, src1_nrows, n_hvx, dst->nb[1], src0->nb[1], src1->nb[1], src2_row_size, d, false, false ); if (L.total_bytes <= vtcm_budget) { @@ -822,7 +827,7 @@ static void htp_kparams_solve_hvx_mm( uint32_t m_chunk = 0; if (htp_mm_hvx_solve_vtcm_params( - kparams->kernel_type, wtype, ne10, src1_nrows, n_threads, + kparams->kernel_type, wtype, ne10, src1_nrows, n_hvx, dst->nb[1], src0->nb[1], src1->nb[1], src2_row_size, best_n_prefetch, vtcm_budget, &L, &m_chunk)) { kparams->n_prefetch = best_n_prefetch; @@ -852,7 +857,7 @@ static void htp_kparams_solve_hvx_mm( struct htp_mm_hvx_vtcm_layout L; uint32_t m_chunk = 0; if (htp_mm_hvx_solve_vtcm_params( - ktype, wtype, ne10, src1_nrows, n_threads, + ktype, wtype, ne10, src1_nrows, n_hvx, dst->nb[1], src0->nb[1], src1->nb[1], src2_row_size, 16, vtcm_budget, &L, &m_chunk)) { kparams->kernel_type = ktype; @@ -889,7 +894,7 @@ static bool htp_kparams_solve_matmul_impl( const int ne12 = src1->ne[2]; const int ne13 = src1->ne[3]; - const bool is_tiled = htp_tensor_is_tiled(src0); + const bool is_tiled = ggml_hexagon_tensor_will_tile(src0); const int wtype = (is_tiled && (src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_BF16)) ? GGML_TYPE_F16 : src0->type; const int ne00_padded = is_tiled ? hex_round_up(ne00, htp_tensor_get_tile_k(wtype)) : ne00; const int ne11_padded = hex_round_up(ne11, 32); @@ -934,19 +939,22 @@ static bool htp_matmul_check_types( return false; } + if (ggml_prec_get_src(dst, 1) == GGML_PREC_BF16 || ggml_prec_get_src(dst, 1) == GGML_PREC_F32) { + return false; + } + if (is_matmul_id) { const struct ggml_tensor * src2 = dst->src[2]; if (src1->type != GGML_TYPE_F32 || !src2 || src2->type != GGML_TYPE_I32) { return false; } - if (ggml_get_op_params_i32(dst, 3) == GGML_PREC_F32) { - return false; - } switch (src0->type) { case GGML_TYPE_Q1_0: case GGML_TYPE_Q2_0: case GGML_TYPE_Q4_0: case GGML_TYPE_Q4_1: + case GGML_TYPE_Q5_0: + case GGML_TYPE_Q5_1: case GGML_TYPE_Q8_0: case GGML_TYPE_IQ4_NL: case GGML_TYPE_MXFP4: @@ -989,6 +997,8 @@ static bool htp_matmul_check_types( case GGML_TYPE_Q2_0: case GGML_TYPE_Q4_0: case GGML_TYPE_Q4_1: + case GGML_TYPE_Q5_0: + case GGML_TYPE_Q5_1: case GGML_TYPE_Q8_0: case GGML_TYPE_IQ4_NL: case GGML_TYPE_MXFP4: @@ -1088,12 +1098,12 @@ bool htp_kparams_solve_mmnx( int32_t n_weights, struct htp_mm_kernel_params * kparams ) { - const uint32_t n_threads = htp_session_n_threads(sess); + const uint32_t n_hvx = htp_session_n_hvx(sess); const uint32_t n_hmx = htp_session_n_hmx(sess); const size_t vtcm_budget = htp_session_vtcm_size(sess); memset(kparams, 0, sizeof(*kparams)); - kparams->n_threads = n_threads; + kparams->n_hvx = n_hvx; const int ne00 = src0->ne[0]; const int ne01 = src0->ne[1]; @@ -1105,7 +1115,7 @@ bool htp_kparams_solve_mmnx( const int ne12 = act->ne[2]; const int ne13 = act->ne[3]; - const bool is_tiled = htp_tensor_is_tiled(src0); + const bool is_tiled = ggml_hexagon_tensor_will_tile(src0); const int wtype = (is_tiled && (src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_BF16)) ? GGML_TYPE_F16 : src0->type; const int ne00_padded = is_tiled ? hex_round_up(ne00, 32) : ne00; const int ne11_padded = hex_round_up(ne11, 32); @@ -1135,7 +1145,7 @@ bool htp_kparams_solve_mmnx( for (uint32_t d = max_prefetch; d >= 2; d /= 2) { struct htp_mm_hvx_vtcm_layout L; htp_mm_hvx_vtcm_layout_build( - &L, HTP_MM_KERNEL_HVX_QUANT_ROW, wtype, ne10, act_nrows, n_threads, + &L, HTP_MM_KERNEL_HVX_QUANT_ROW, wtype, ne10, act_nrows, n_hvx, 0, src0_row_size, act_row_size, 0, d, false, true ); if (L.total_bytes <= vtcm_budget) { @@ -1146,7 +1156,7 @@ bool htp_kparams_solve_mmnx( struct htp_mm_hvx_vtcm_layout L; htp_mm_hvx_vtcm_layout_build( - &L, HTP_MM_KERNEL_HVX_QUANT_ROW, wtype, ne10, act_nrows, n_threads, + &L, HTP_MM_KERNEL_HVX_QUANT_ROW, wtype, ne10, act_nrows, n_hvx, 0, src0_row_size, act_row_size, 0, best_n_prefetch, false, true ); @@ -1239,7 +1249,7 @@ static bool htp_kparams_solve_binary( } } - const uint32_t max_threads_sess = htp_session_n_threads(sess); + const uint32_t max_hvx_sess = htp_session_n_hvx(sess); const size_t vtcm_size = htp_session_vtcm_size(sess); const size_t elem_size = ggml_type_size(src0->type); @@ -1273,14 +1283,14 @@ static bool htp_kparams_solve_binary( const uint32_t total_elems = (uint32_t) ggml_nelements(src0); const uint32_t max_chunk_elems = 32768 / elem_size; const uint32_t min_chunk_elems = 256; - const uint32_t target_chunk_elems = hex_round_up((total_elems + (2 * max_threads_sess) - 1) / (2 * max_threads_sess), 32); + const uint32_t target_chunk_elems = hex_round_up((total_elems + (2 * max_hvx_sess) - 1) / (2 * max_hvx_sess), 32); const uint32_t chunk_size = (std::min)(max_chunk_elems, (std::max)(target_chunk_elems, min_chunk_elems)); const uint32_t chunk_bytes = hex_round_up(chunk_size * elem_size, 128); const uint32_t total_chunks = (total_elems + chunk_size - 1) / chunk_size; - const uint32_t n_threads = (total_chunks >= 2) ? (std::min)(max_threads_sess, total_chunks) : 1; + const uint32_t n_hvx = (total_chunks >= 2) ? (std::min)(max_hvx_sess, total_chunks) : 1; kparams->kernel_type = HTP_BINARY_KERNEL_CHUNKED; - kparams->n_threads = n_threads; + kparams->n_hvx = n_hvx; kparams->n_srcs = 2; kparams->rows_per_buffer = 1; kparams->src0_row_size_aligned = src0_row_size_aligned; @@ -1341,11 +1351,11 @@ static bool htp_kparams_solve_binary( } const uint32_t src0_nrows = src0->ne[1] * src0->ne[2] * src0->ne[3]; - const uint32_t min_rows_per_thread = (uint32_t) (std::max(1, (128 + dst_row_size - 1) / dst_row_size)); - const uint32_t max_threads = (std::max)(1u, src0_nrows / min_rows_per_thread); + const uint32_t min_rows_per_hvx = (uint32_t) (std::max(1, (128 + dst_row_size - 1) / dst_row_size)); + const uint32_t max_hvx = (std::max)(1u, src0_nrows / min_rows_per_hvx); kparams->kernel_type = kernel_type; - kparams->n_threads = (std::min)(max_threads_sess, max_threads); + kparams->n_hvx = (std::min)(max_hvx_sess, max_hvx); kparams->n_srcs = 2; kparams->src0_row_size_aligned = src0_row_size_aligned; kparams->src1_row_size_aligned = src1_row_size_aligned; @@ -1398,7 +1408,7 @@ static bool htp_kparams_solve_binary( if (!is_add_id && is_contig && (ggml_are_same_shape(src0, src1) || is_scalar_broadcast)) { const uint32_t total_elems = (uint32_t) ggml_nelements(src0); return htp_binary_solve_chunked(&L, kparams, total_elems, (uint32_t) elem_size, - max_threads_sess, 2, is_scalar_broadcast, vtcm_size); + max_hvx_sess, 2, is_scalar_broadcast, vtcm_size); } return false; } @@ -1410,7 +1420,7 @@ static bool htp_kparams_solve_binary( } // collapse contiguous dims of elementwise ops so that rows fill at least one HVX vector -static void htp_collapse_dims(struct ggml_tensor ** tensors, size_t n_tensors, size_t elem_size, uint32_t n_threads) { +static void htp_collapse_dims(struct ggml_tensor ** tensors, size_t n_tensors, size_t elem_size, uint32_t n_hvx) { if (n_tensors == 0 || !tensors[0]) { return; } @@ -1492,9 +1502,9 @@ static void htp_collapse_dims(struct ggml_tensor ** tensors, size_t n_tensors, s break; } } - if (nrows < n_threads && rows_contig && row_size_curr >= 4096) { + if (nrows < n_hvx && rows_contig && row_size_curr >= 4096) { const size_t vec_elems = 128 / elem_size; - int64_t k = std::max(row_size_curr / 16384, (n_threads * 4 + nrows - 1) / nrows); + int64_t k = std::max(row_size_curr / 16384, (n_hvx * 4 + nrows - 1) / nrows); k = std::min(k, row_size_curr / 2048); while (k > 1 && t0->ne[0] % (k * vec_elems) != 0) { k--; @@ -1571,23 +1581,27 @@ static bool htp_kparams_solve_unary_impl( const size_t dst_data_row_size = dst->ne[0] * ggml_type_size(dst->type); const uint32_t src0_nrows = src0->ne[1] * src0->ne[2] * src0->ne[3]; - const uint32_t min_rows_per_thread = (uint32_t) (std::max(1, (128 + dst_data_row_size - 1) / dst_data_row_size)); - const uint32_t max_threads = (std::max)(1u, src0_nrows / min_rows_per_thread); - const uint32_t n_threads = (std::min)(htp_session_n_threads(sess), max_threads); + const uint32_t min_rows_per_hvx = (uint32_t) (std::max(1, (128 + dst_data_row_size - 1) / dst_data_row_size)); + const uint32_t max_hvx = (std::max)(1u, src0_nrows / min_rows_per_hvx); + const uint32_t n_hvx = (std::min)(htp_session_n_hvx(sess), max_hvx); const size_t vtcm_size = htp_session_vtcm_size(sess); - kparams->n_threads = n_threads; + kparams->n_hvx = n_hvx; - const bool broadcast_weight = (op == HTP_OP_RMS_NORM_MUL) && (src1->ne[1] * src1->ne[2] * src1->ne[3] == 1); + const bool has_weight = (op == HTP_OP_RMS_NORM_MUL || op == HTP_OP_RMS_NORM_MUL_ADD); + const bool broadcast_weight = has_weight && (src1->ne[1] * src1->ne[2] * src1->ne[3] == 1); + if (op == HTP_OP_RMS_NORM_MUL_ADD && !broadcast_weight) { + return false; + } kparams->broadcast_weight = broadcast_weight; - const bool src0_contig = ggml_is_contiguous_1(src0); - const bool src1_contig = (op == HTP_OP_RMS_NORM_MUL && !broadcast_weight) ? ggml_is_contiguous_1(src1) : true; - const uint32_t src0_nrows_per_thread = (src0_nrows + n_threads - 1) / n_threads; - const uint32_t min_block_rows = (uint32_t) (std::max(1, 2048 / dst_data_row_size)); - uint32_t max_block = (src0_contig && src1_contig) ? src0_nrows : (uint32_t) src0->ne[1]; - if (src0_nrows_per_thread >= 2 * min_block_rows) { - max_block = (std::min)(max_block, (std::max)(min_block_rows, src0_nrows_per_thread / 4)); + const bool src0_contig = ggml_is_contiguous_1(src0); + const bool src1_contig = (has_weight && !broadcast_weight) ? ggml_is_contiguous_1(src1) : true; + const uint32_t src0_nrows_per_hvx = (src0_nrows + n_hvx - 1) / n_hvx; + const uint32_t min_block_rows = (uint32_t) (std::max(1, 2048 / dst_data_row_size)); + uint32_t max_block = (src0_contig && src1_contig) ? src0_nrows : (uint32_t) src0->ne[1]; + if (src0_nrows_per_hvx >= 2 * min_block_rows) { + max_block = (std::min)(max_block, (std::max)(min_block_rows, src0_nrows_per_hvx / 4)); } kparams->src0_contig = src0_contig ? 1 : 0; @@ -1595,16 +1609,16 @@ static bool htp_kparams_solve_unary_impl( struct htp_unary_vtcm_layout L; uint32_t col_tile = 0; - uint32_t vtcm_row_per_thread = 0; + uint32_t vtcm_row_per_hvx = 0; htp_unary_solve_layout(&L, op, src0->ne[0], dst->ne[0], - op == HTP_OP_RMS_NORM_MUL ? src1->ne[0] : 0, - max_block, broadcast_weight, n_threads, vtcm_size, elem_size, - &col_tile, &vtcm_row_per_thread); + has_weight ? src1->ne[0] : 0, + max_block, broadcast_weight, n_hvx, vtcm_size, elem_size, + &col_tile, &vtcm_row_per_hvx); kparams->col_tile = col_tile; kparams->vtcm_size = L.total_bytes; - kparams->block = col_tile ? 0 : vtcm_row_per_thread; + kparams->block = col_tile ? 0 : vtcm_row_per_hvx; const uint32_t tiles_per_row = col_tile > 0 ? (src0->ne[0] + col_tile - 1) / col_tile : 1; kparams->tiles_per_row = tiles_per_row; @@ -1613,7 +1627,7 @@ static bool htp_kparams_solve_unary_impl( kparams->div_ne012 = init_fastdiv_values(src0->ne[1] * src0->ne[2]); kparams->div_tpr = init_fastdiv_values(tiles_per_row); - return (col_tile > 0 || vtcm_row_per_thread > 0) && L.total_bytes <= vtcm_size; + return (col_tile > 0 || vtcm_row_per_hvx > 0) && L.total_bytes <= vtcm_size; } static bool htp_kparams_solve_unary( @@ -1637,6 +1651,16 @@ bool htp_kparams_solve_rms_norm_mul( return htp_kparams_solve_unary_impl(sess, HTP_OP_RMS_NORM_MUL, src0, src1, dst, kparams); } +bool htp_kparams_solve_rms_norm_mul_add( + const struct ggml_hexagon_session * sess, + const struct ggml_tensor * src0, + const struct ggml_tensor * src1, + const struct ggml_tensor * dst, + struct htp_unary_kernel_params * kparams +) { + return htp_kparams_solve_unary_impl(sess, HTP_OP_RMS_NORM_MUL_ADD, src0, src1, dst, kparams); +} + bool htp_kparams_solve_repeat_get_rows( const struct ggml_hexagon_session * sess, const struct ggml_tensor * src0, @@ -1679,7 +1703,7 @@ bool htp_kparams_solve_repeat_get_rows( const uint32_t ne12 = src1->ne[2]; const uint32_t nr = ne10 * ne11 * ne12; - const bool tiled = htp_tensor_is_tiled(src0); + const bool tiled = ggml_hexagon_tensor_is_tiled(src0); const int wtype = (tiled && (src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_BF16)) ? GGML_TYPE_F16 : src0->type; if (tiled) { @@ -1691,22 +1715,22 @@ bool htp_kparams_solve_repeat_get_rows( } const uint32_t total_tasks = nr; - const uint32_t min_tasks_per_thread = (uint32_t) (std::max(1, (128 + dst->nb[1] - 1) / dst->nb[1])); - const uint32_t max_threads = (std::max)(1u, total_tasks / min_tasks_per_thread); + const uint32_t min_tasks_per_hvx = (uint32_t) (std::max(1, (128 + dst->nb[1] - 1) / dst->nb[1])); + const uint32_t max_hvx = (std::max)(1u, total_tasks / min_tasks_per_hvx); const size_t vtcm_size = htp_session_vtcm_size(sess); - kparams->n_threads = (std::min)(htp_session_n_threads(sess), max_threads); + kparams->n_hvx = (std::min)(htp_session_n_hvx(sess), max_hvx); struct htp_get_rows_vtcm_layout vtcm_layout = {}; - while (kparams->n_threads > 0) { - htp_get_rows_vtcm_layout_build(&vtcm_layout, kparams->kernel_type, wtype, ne00, total_tasks, src1->type, kparams->n_threads); + while (kparams->n_hvx > 0) { + htp_get_rows_vtcm_layout_build(&vtcm_layout, kparams->kernel_type, wtype, ne00, total_tasks, src1->type, kparams->n_hvx); if (vtcm_layout.total_bytes <= vtcm_size) { break; } - --kparams->n_threads; + --kparams->n_hvx; } - if (kparams->n_threads == 0 && total_tasks > 0) { + if (kparams->n_hvx == 0 && total_tasks > 0) { htp_get_rows_vtcm_layout_build(&vtcm_layout, kparams->kernel_type, wtype, ne00, total_tasks, src1->type, 1); } @@ -1721,7 +1745,7 @@ bool htp_kparams_solve_repeat_get_rows( if (total_tasks == 0) { return true; } - return kparams->n_threads > 0 && (size_t) kparams->vtcm_size <= vtcm_size; + return kparams->n_hvx > 0 && (size_t) kparams->vtcm_size <= vtcm_size; } static bool htp_kparams_solve_get_rows( @@ -1758,11 +1782,11 @@ static bool htp_kparams_solve_set_rows( const uint32_t nr = src0->ne[1]; - kparams->n_threads = (std::min)(htp_session_n_threads(sess), nr); + kparams->n_hvx = (std::min)(htp_session_n_hvx(sess), nr); kparams->total_tasks = nr; struct htp_set_rows_vtcm_layout vtcm_layout; - htp_set_rows_vtcm_layout_build(&vtcm_layout, dst->type, src0->ne[0], kparams->n_threads); + htp_set_rows_vtcm_layout_build(&vtcm_layout, dst->type, src0->ne[0], kparams->n_hvx); kparams->vtcm_size = vtcm_layout.total_bytes; return vtcm_layout.total_bytes <= htp_session_vtcm_size(sess); @@ -1800,9 +1824,9 @@ static bool htp_kparams_solve_softmax( const uint32_t src0_nrows = src0->ne[1] * src0->ne[2] * src0->ne[3]; const size_t dst_data_row_size = op->ne[0] * ggml_type_size(op->type); - const uint32_t min_rows_per_thread = (uint32_t) (std::max(1, (128 + dst_data_row_size - 1) / dst_data_row_size)); - const uint32_t max_threads = (std::max)(1u, src0_nrows / min_rows_per_thread); - const uint32_t n_threads = (std::min)(htp_session_n_threads(sess), max_threads); + const uint32_t min_rows_per_hvx = (uint32_t) (std::max(1, (128 + dst_data_row_size - 1) / dst_data_row_size)); + const uint32_t max_hvx = (std::max)(1u, src0_nrows / min_rows_per_hvx); + const uint32_t n_hvx = (std::min)(htp_session_n_hvx(sess), max_hvx); float scale = 1.0f; float max_bias = 0.0f; @@ -1832,12 +1856,12 @@ static bool htp_kparams_solve_softmax( const uint32_t ne10 = src1 ? src1->ne[0] : 1; struct htp_softmax_vtcm_layout layout; - htp_softmax_vtcm_layout_build(&layout, ne00, ne10, kparams->use_src1 != 0, kparams->use_f16 != 0, n_threads); + htp_softmax_vtcm_layout_build(&layout, ne00, ne10, kparams->use_src1 != 0, kparams->use_f16 != 0, n_hvx); if (layout.total_bytes > htp_session_vtcm_size(sess)) { return false; } - kparams->n_threads = n_threads; + kparams->n_hvx = n_hvx; kparams->vtcm_size = (uint32_t) layout.total_bytes; if (!kparams->use_src1) { @@ -1870,6 +1894,10 @@ static bool htp_kparams_solve_rope( const struct ggml_tensor * src2 = op->src[2]; const struct ggml_tensor * dst = op; + if (!ggml_rope_get_truncate(op)) { + return false; + } + if (!ggml_are_same_shape(src0, dst)) { return false; } @@ -1939,16 +1967,16 @@ static bool htp_kparams_solve_rope( return false; } - const uint32_t n_threads = (std::min)(htp_session_n_threads(sess), src0_nrows); + const uint32_t n_hvx = (std::min)(htp_session_n_hvx(sess), src0_nrows); const uint32_t n_freq_factors = src2 ? (uint32_t) src2->ne[0] : 0; struct htp_rope_vtcm_layout layout; - htp_rope_vtcm_layout_build(&layout, src0->ne[0], n_threads, n_freq_factors); + htp_rope_vtcm_layout_build(&layout, src0->ne[0], n_hvx, n_freq_factors); if (layout.total_bytes > htp_session_vtcm_size(sess)) { return false; } - kparams->n_threads = n_threads; + kparams->n_hvx = n_hvx; kparams->vtcm_size = (uint32_t) layout.total_bytes; kparams->div_ne2_ne1 = init_fastdiv_values(dst->ne[2] * dst->ne[1]); kparams->div_ne1 = init_fastdiv_values(dst->ne[1]); @@ -1982,26 +2010,26 @@ static bool htp_kparams_solve_sum_rows( const uint32_t ne00 = src0->ne[0]; const uint32_t src0_nrows = src0->ne[1] * src0->ne[2] * src0->ne[3]; - const uint32_t max_threads_sess = htp_session_n_threads(sess); + const uint32_t max_hvx_sess = htp_session_n_hvx(sess); if (op->op == GGML_OP_SUM) { const uint32_t total_elems = (uint32_t) ggml_nelements(src0); - const uint32_t n_threads = (total_elems >= 1024) ? (std::min)(max_threads_sess, (uint32_t) HTP_MAX_NTHREADS) : 1; - const uint32_t raw_chunk = (total_elems + n_threads - 1) / n_threads; + const uint32_t n_hvx = (total_elems >= 1024) ? max_hvx_sess : 1; + const uint32_t raw_chunk = (total_elems + n_hvx - 1) / n_hvx; - kparams->n_threads = n_threads; - kparams->elems_per_thread = hex_round_up(raw_chunk, 32); + kparams->n_hvx = n_hvx; + kparams->elems_per_hvx = hex_round_up(raw_chunk, 32); } else if (op->op == GGML_OP_ARGMAX && src0_nrows <= 1) { - const uint32_t n_threads = (ne00 >= 1024) ? (std::min)(max_threads_sess, (uint32_t) HTP_MAX_NTHREADS) : 1; - const uint32_t raw_chunk = (ne00 + n_threads - 1) / n_threads; + const uint32_t n_hvx = (ne00 >= 1024) ? max_hvx_sess : 1; + const uint32_t raw_chunk = (ne00 + n_hvx - 1) / n_hvx; - kparams->n_threads = n_threads; - kparams->elems_per_thread = hex_round_up(raw_chunk, 32); + kparams->n_hvx = n_hvx; + kparams->elems_per_hvx = hex_round_up(raw_chunk, 32); } else { const uint32_t min_rpt = (src0->nb[1] < 1024) ? 32 : 1; - const uint32_t max_threads = (std::max)(1u, src0_nrows / min_rpt); + const uint32_t max_hvx = (std::max)(1u, src0_nrows / min_rpt); - kparams->n_threads = (std::min)(max_threads_sess, max_threads); + kparams->n_hvx = (std::min)(max_hvx_sess, max_hvx); } return true; @@ -2042,28 +2070,28 @@ bool htp_kparams_solve_unary_mul_glu( const size_t dst_row_size = dst->ne[0] * elem_size; const uint32_t src0_nrows = src0->ne[1] * src0->ne[2] * src0->ne[3]; - const uint32_t min_rows_per_thread = (uint32_t) (std::max(1, (128 + dst_row_size - 1) / dst_row_size)); - const uint32_t max_threads = (std::max)(1u, src0_nrows / min_rows_per_thread); - const uint32_t n_threads = (std::min)(htp_session_n_threads(sess), max_threads); + const uint32_t min_rows_per_hvx = (uint32_t) (std::max(1, (128 + dst_row_size - 1) / dst_row_size)); + const uint32_t max_hvx = (std::max)(1u, src0_nrows / min_rows_per_hvx); + const uint32_t n_hvx = (std::min)(htp_session_n_hvx(sess), max_hvx); const size_t vtcm_size = htp_session_vtcm_size(sess); const size_t row_size_aligned = hex_round_up(dst_row_size, 128); - uint32_t vtcm_row_per_thread = (uint32_t) (vtcm_size / (n_threads * 3 * row_size_aligned)); - if (vtcm_row_per_thread < 2) { + uint32_t vtcm_row_per_hvx = (uint32_t) (vtcm_size / (n_hvx * 3 * row_size_aligned)); + if (vtcm_row_per_hvx < 2) { return false; } - const uint32_t src0_nrows_per_thread = (src0_nrows + n_threads - 1) / n_threads; - if (src0_nrows_per_thread >= 2 && vtcm_row_per_thread > src0_nrows_per_thread) { - uint32_t target_rpt = (src0_nrows_per_thread >= 4) ? (src0_nrows_per_thread / 2) & ~1u : 2; - vtcm_row_per_thread = (std::max)(2u, (std::min)(vtcm_row_per_thread, target_rpt)); + const uint32_t src0_nrows_per_hvx = (src0_nrows + n_hvx - 1) / n_hvx; + if (src0_nrows_per_hvx >= 2 && vtcm_row_per_hvx > src0_nrows_per_hvx) { + uint32_t target_rpt = (src0_nrows_per_hvx >= 4) ? (src0_nrows_per_hvx / 2) & ~1u : 2; + vtcm_row_per_hvx = (std::max)(2u, (std::min)(vtcm_row_per_hvx, target_rpt)); } struct htp_glu_vtcm_layout L; - htp_glu_vtcm_layout_build(&L, dst_row_size, n_threads, vtcm_row_per_thread); + htp_glu_vtcm_layout_build(&L, dst_row_size, n_hvx, vtcm_row_per_hvx); - kparams->n_threads = n_threads; - kparams->vtcm_row_per_thread = L.vtcm_row_per_thread; + kparams->n_hvx = n_hvx; + kparams->vtcm_row_per_hvx = L.vtcm_row_per_hvx; kparams->vtcm_size = (uint32_t) L.total_bytes; return true; @@ -2096,11 +2124,11 @@ static bool htp_kparams_solve_repeat( const uint32_t total_dst_rows = dst->ne[1] * dst->ne[2] * dst->ne[3]; const size_t dst_row_size = dst->ne[0] * ggml_type_size(dst->type); - const uint32_t min_rows_per_thread = (uint32_t) (std::max(1, (128 + dst_row_size - 1) / dst_row_size)); - const uint32_t max_threads = (std::max)(1u, total_dst_rows / min_rows_per_thread); - const uint32_t n_threads = (std::min)(htp_session_n_threads(sess), max_threads); + const uint32_t min_rows_per_hvx = (uint32_t) (std::max(1, (128 + dst_row_size - 1) / dst_row_size)); + const uint32_t max_hvx = (std::max)(1u, total_dst_rows / min_rows_per_hvx); + const uint32_t n_hvx = (std::min)(htp_session_n_hvx(sess), max_hvx); - kparams->n_threads = n_threads; + kparams->n_hvx = n_hvx; kparams->nr0 = dst->ne[0] / src0->ne[0]; kparams->nr1 = dst->ne[1] / src0->ne[1]; kparams->nr2 = dst->ne[2] / src0->ne[2]; @@ -2163,31 +2191,31 @@ static bool htp_kparams_solve_ssm_conv( const uint32_t n_s = (uint32_t) n_s_64; const uint32_t ncs = (uint32_t) src0->ne[0]; - const uint32_t n_threads = (std::min)(htp_session_n_threads(sess), (d_inner + 31) / 32); + const uint32_t n_hvx = (std::min)(htp_session_n_hvx(sess), (d_inner + 31) / 32); const size_t vtcm_size = htp_session_vtcm_size(sess); - kparams->n_threads = n_threads; + kparams->n_hvx = n_hvx; kparams->d_conv = d_conv; kparams->d_inner = d_inner; kparams->n_t = n_t; kparams->n_s = n_s; - const uint32_t raw_rpt = (d_inner + n_threads - 1) / n_threads; - const uint32_t d_inner_per_thread = hex_round_up(raw_rpt, 32); + const uint32_t raw_rpt = (d_inner + n_hvx - 1) / n_hvx; + const uint32_t d_inner_per_hvx = hex_round_up(raw_rpt, 32); if (n_t == 1) { - kparams->d_inner_tile = d_inner_per_thread; + kparams->d_inner_tile = d_inner_per_hvx; } else { - const uint32_t src1_raw_bytes = hex_round_up(d_inner_per_thread * d_conv * sizeof(float), 128) + 128; - const uint32_t src1_T_bytes = hex_round_up(d_conv * d_inner_per_thread * sizeof(float), 128); - const uint32_t vtcm_src1_per_thread = src1_raw_bytes + src1_T_bytes; + const uint32_t src1_raw_bytes = hex_round_up(d_inner_per_hvx * d_conv * sizeof(float), 128) + 128; + const uint32_t src1_T_bytes = hex_round_up(d_conv * d_inner_per_hvx * sizeof(float), 128); + const uint32_t vtcm_src1_per_hvx = src1_raw_bytes + src1_T_bytes; - const size_t vtcm_budget = (vtcm_size > 0 ? vtcm_size / n_threads : (1024 * 1024)); + const size_t vtcm_budget = (vtcm_size > 0 ? vtcm_size / n_hvx : (1024 * 1024)); const uint32_t src0_block_T = hex_round_up(ncs * 32 * sizeof(float), 128); - const size_t fixed_bytes = vtcm_src1_per_thread + src0_block_T; + const size_t fixed_bytes = vtcm_src1_per_hvx + src0_block_T; const size_t avail_for_tiles = vtcm_budget > fixed_bytes ? vtcm_budget - fixed_bytes : (128 * 1024); - uint32_t target_max_tile = hex_round_up((d_inner_per_thread + 3) / 4, 32); + uint32_t target_max_tile = hex_round_up((d_inner_per_hvx + 3) / 4, 32); target_max_tile = (std::max)(target_max_tile, 32u); target_max_tile = (std::min)(target_max_tile, 128u); @@ -2200,14 +2228,14 @@ static bool htp_kparams_solve_ssm_conv( if (d_inner_tile > target_max_tile) { d_inner_tile = target_max_tile; } - if (d_inner_tile > d_inner_per_thread) { - d_inner_tile = d_inner_per_thread; + if (d_inner_tile > d_inner_per_hvx) { + d_inner_tile = d_inner_per_hvx; } kparams->d_inner_tile = d_inner_tile; } struct htp_ssm_conv_vtcm_layout L; - htp_ssm_conv_vtcm_layout_build(&L, d_inner_per_thread, d_conv, n_t, kparams->d_inner_tile, n_threads); + htp_ssm_conv_vtcm_layout_build(&L, d_inner_per_hvx, d_conv, n_t, kparams->d_inner_tile, n_hvx); kparams->vtcm_size = (uint32_t) L.total_bytes; kparams->div_d_inner_tile = init_fastdiv_values(kparams->d_inner_tile); @@ -2233,7 +2261,7 @@ static bool htp_kparams_solve_sort( const uint32_t total_rows = src0->ne[1] * src0->ne[2] * src0->ne[3]; const uint32_t ne00 = src0->ne[0]; const uint32_t k = dst->ne[0]; - const uint32_t n_threads_max = htp_session_n_threads(sess) > 0 ? htp_session_n_threads(sess) : 4; + const uint32_t n_hvx_max = htp_session_n_hvx(sess) > 0 ? htp_session_n_hvx(sess) : 4; const size_t vtcm_budget = htp_session_vtcm_size(sess) > 0 ? htp_session_vtcm_size(sess) : (8 * 1024 * 1024); int32_t order = GGML_SORT_ORDER_DESC; @@ -2242,11 +2270,11 @@ static bool htp_kparams_solve_sort( } struct htp_sort_vtcm_layout layout; - if (!htp_sort_solve_layout(&layout, ne00, total_rows, k, n_threads_max, vtcm_budget, is_top_k)) { + if (!htp_sort_solve_layout(&layout, ne00, total_rows, k, n_hvx_max, vtcm_budget, is_top_k)) { return false; } - kparams->n_threads = layout.n_threads; + kparams->n_hvx = layout.n_hvx; kparams->order = (uint32_t) order; kparams->is_top_k = is_top_k ? 1 : 0; kparams->chunk_elems = layout.chunk_elems; @@ -2324,11 +2352,11 @@ static bool htp_kparams_solve_pool_2d( const uint32_t planes = (uint32_t) (is_pool_1d ? (uint64_t) src0->ne[1] * (uint64_t) src0->ne[2] * (uint64_t) src0->ne[3] : (uint64_t) src0->ne[2] * (uint64_t) src0->ne[3]); - const uint32_t n_threads = (std::min)(htp_session_n_threads(sess), planes > 0 ? planes : 1); + const uint32_t n_hvx = (std::min)(htp_session_n_hvx(sess), planes > 0 ? planes : 1); const uint32_t planes_per_chunk = (dst_plane_bytes > 0) ? (HEX_L2_LINE_SIZE / hex_gcd_u32(dst_plane_bytes, HEX_L2_LINE_SIZE)) : 1; struct htp_pool_vtcm_layout layout; - if (!htp_pool_solve_layout(&layout, src_x, src_y, dst_x, dst_y, n_threads, htp_session_vtcm_size(sess))) { + if (!htp_pool_solve_layout(&layout, src_x, src_y, dst_x, dst_y, n_hvx, htp_session_vtcm_size(sess))) { return false; } @@ -2342,7 +2370,7 @@ static bool htp_kparams_solve_pool_2d( kparams->stride_y = (uint32_t) stride_y; kparams->pad_x = pad_x; kparams->pad_y = pad_y; - kparams->n_threads = n_threads; + kparams->n_hvx = n_hvx; kparams->planes = planes; kparams->planes_per_chunk = planes_per_chunk; kparams->pool_op = (uint32_t) params[0]; @@ -2448,18 +2476,18 @@ bool htp_kparams_solve_upscale( const float invscale1 = antialias ? (1.0f / filter_radius1) : 0.0f; const uint32_t total_slices = dst_z * dst_w; - uint32_t n_threads = htp_session_n_threads(sess); + uint32_t n_hvx = htp_session_n_hvx(sess); struct htp_upscale_vtcm_layout layout; uint32_t dst_nrows_per_chunk = 0; - while (n_threads > 0) { - if (htp_upscale_solve_layout(&layout, src_x, src_y, dst_x, dst_y, inv_sf1, filter_radius1, n_threads, + while (n_hvx > 0) { + if (htp_upscale_solve_layout(&layout, src_x, src_y, dst_x, dst_y, inv_sf1, filter_radius1, n_hvx, htp_session_vtcm_size(sess), &dst_nrows_per_chunk)) { break; } - n_threads--; + n_hvx--; } - if (n_threads == 0) { + if (n_hvx == 0) { return false; } @@ -2486,8 +2514,8 @@ bool htp_kparams_solve_upscale( const uint32_t chunks_per_slice = (dst_y + dst_nrows_per_chunk - 1) / dst_nrows_per_chunk; const uint32_t total_chunks = total_slices * chunks_per_slice; - n_threads = (std::min)(n_threads, total_chunks > 0 ? total_chunks : 1); - htp_upscale_vtcm_layout_build(&layout, src_x, src_y, dst_x, dst_y, dst_nrows_per_chunk, inv_sf1, filter_radius1, n_threads); + n_hvx = (std::min)(n_hvx, total_chunks > 0 ? total_chunks : 1); + htp_upscale_vtcm_layout_build(&layout, src_x, src_y, dst_x, dst_y, dst_nrows_per_chunk, inv_sf1, filter_radius1, n_hvx); const uint32_t slice_dst_bytes = dst_x * dst_y * sizeof(float); const uint32_t chunk_dst_bytes = dst_x * dst_nrows_per_chunk * sizeof(float); @@ -2542,7 +2570,7 @@ bool htp_kparams_solve_upscale( kparams->invscale0 = invscale0; kparams->invscale1 = invscale1; - kparams->n_threads = n_threads; + kparams->n_hvx = n_hvx; kparams->chunks_per_group = chunks_per_group; kparams->dst_nrows_per_chunk = dst_nrows_per_chunk; @@ -2586,7 +2614,7 @@ static bool htp_kparams_solve_concat( } const uint32_t type_size = ggml_type_size(dst->type); - const uint32_t n_threads = htp_session_n_threads(sess); + const uint32_t n_hvx = htp_session_n_hvx(sess); const uint32_t mdev_count = htp_session_mdev_count(sess); const size_t vtcm_size = htp_session_vtcm_size(sess); @@ -2620,10 +2648,10 @@ static bool htp_kparams_solve_concat( (uint32_t) src0->ne[0], (uint32_t) src0->nb[1], (uint32_t) src1->ne[0], (uint32_t) src1->nb[1], b_dense, type_size, total_rows, - n_threads, mdev_count, vtcm_size)) { + n_hvx, mdev_count, vtcm_size)) { kparams->kernel_type = HTP_CONCAT_KERNEL_GATHER_ROWS; - kparams->n_threads = (uint8_t) n_threads; - kparams->vtcm_size = kparams->u.gather.slice * n_threads; + kparams->n_hvx = (uint8_t) n_hvx; + kparams->vtcm_size = kparams->u.gather.slice * n_hvx; return true; } } @@ -2631,7 +2659,7 @@ static bool htp_kparams_solve_concat( const bool dma_strides_ok = (src0->nb[0] == type_size && src1->nb[0] == type_size && dst->nb[0] == type_size); if (dma_strides_ok) { kparams->kernel_type = HTP_CONCAT_KERNEL_REGULAR; - kparams->n_threads = 1; + kparams->n_hvx = 1; return true; } @@ -2642,7 +2670,7 @@ static bool htp_kparams_solve_concat( if (dim == 0 && is_src1_transposed && !is_src0_transposed && transposed_rows_ok && (dst->type == GGML_TYPE_F32 || dst->type == GGML_TYPE_F16)) { - const uint32_t nth = n_threads > 0 ? n_threads : 8; + const uint32_t nth = n_hvx > 0 ? n_hvx : 8; struct htp_concat_transposed_vtcm_layout layout; htp_concat_transposed_vtcm_layout_build(&layout, (uint32_t) src0->ne[0], (uint32_t) src1->ne[0], type_size, nth); @@ -2651,7 +2679,7 @@ static bool htp_kparams_solve_concat( } kparams->kernel_type = HTP_CONCAT_KERNEL_TRANSPOSED; - kparams->n_threads = nth; + kparams->n_hvx = nth; kparams->vtcm_size = layout.total_bytes; kparams->u.transposed.div_ne2 = init_fastdiv_values((uint32_t) dst->ne[2]); return true; @@ -2660,6 +2688,129 @@ static bool htp_kparams_solve_concat( return false; } +// Permuted copy with strided dim 0 (e.g. channel-first <-> channel-last) +static bool htp_kparams_solve_cpy_transpose( + const struct ggml_hexagon_session * sess, + const struct ggml_tensor * src0, + const struct ggml_tensor * dst, + uint32_t type_size, + struct htp_copy_kernel_params * kparams) { + + // View contiguous dst with src shape to support CPY with reshape + const bool dst_cont = ggml_is_contiguous(dst); + if (!dst_cont && !ggml_are_same_shape(src0, dst)) { + return false; + } + + // Drop size-1 dims and merge dims adjacent in both tensors + size_t ne[GGML_MAX_DIMS]; + size_t nbs[GGML_MAX_DIMS]; + size_t nbd[GGML_MAX_DIMS]; + size_t dst_stride = type_size; + int n = 0; + for (int d = 0; d < GGML_MAX_DIMS; d++) { + const size_t sd = dst_cont ? dst_stride : dst->nb[d]; + dst_stride *= (size_t) src0->ne[d]; + if (src0->ne[d] == 1) { + continue; + } + if (n > 0 && src0->nb[d] == nbs[n - 1] * ne[n - 1] && sd == nbd[n - 1] * ne[n - 1]) { + ne[n - 1] *= (size_t) src0->ne[d]; + continue; + } + ne[n] = (size_t) src0->ne[d]; + nbs[n] = src0->nb[d]; + nbd[n] = sd; + n++; + } + + // Fold leading dense dim into element block + size_t es = type_size; + for (int i = 0; i < n; i++) { + if (nbs[i] == type_size && nbd[i] == type_size) { + es *= ne[i]; + for (int j = i; j + 1 < n; j++) { + ne[j] = ne[j + 1]; + nbs[j] = nbs[j + 1]; + nbd[j] = nbd[j + 1]; + } + n--; + break; + } + } + if (es > 128 || (es & (es - 1)) != 0) { + return false; + } + + // Transpose axes: dim a dense in dst, dim k dense in src + int a = -1; + int k = -1; + for (int i = 0; i < n; i++) { + if (a < 0 && nbd[i] == es) { + a = i; + } + if (k < 0 && nbs[i] == es) { + k = i; + } + } + if (a < 0 || k < 0 || a == k) { + return false; + } + + struct htp_copy_transpose_params tp = {}; + tp.elem_size = (uint8_t) es; + tp.ne[0] = (uint32_t) ne[a]; + tp.ne[1] = (uint32_t) ne[k]; + tp.ne[2] = 1; + tp.src_nb[0] = (uint32_t) nbs[a]; + tp.dst_nb[0] = (uint32_t) nbd[k]; + tp.packed = nbd[k] == ne[a] * es && ne[a] * es < 128 && 128 % (ne[a] * es) == 0; + + size_t ne3 = 1; + int m = 0; + for (int i = 0; i < n; i++) { + if (i == a || i == k) { + continue; + } + if (m == 0) { + tp.ne[2] = (uint32_t) ne[i]; + } else { + ne3 = ne[i]; + } + tp.src_nb[1 + m] = (uint32_t) nbs[i]; + tp.dst_nb[1 + m] = (uint32_t) nbd[i]; + m++; + } + + const uint32_t n_hvx = htp_session_n_hvx(sess); + const size_t vtcm_size = htp_session_vtcm_size(sess); + uint64_t vtcm = 0; + for (uint32_t blk_bytes = HTP_COPY_TRANSPOSE_BLK_BYTES; ; blk_bytes /= 2) { + if (!htp_copy_transpose_layout_build(&tp, blk_bytes)) { + return false; + } + vtcm = (uint64_t) n_hvx * 2 * (tp.in_buf_size + tp.out_buf_size); + if (vtcm <= vtcm_size) { + break; + } + } + + const uint64_t total_tiles = (uint64_t) tp.n0b * tp.nkb * tp.ne[2] * ne3; + if (total_tiles > UINT32_MAX) { + return false; + } + tp.total_tiles = (uint32_t) total_tiles; + tp.div_nkb = init_fastdiv_values(tp.nkb); + tp.div_n0b = init_fastdiv_values(tp.n0b); + tp.div_ne2 = init_fastdiv_values(tp.ne[2]); + + kparams->kernel_type = HTP_COPY_KERNEL_TRANSPOSE; + kparams->n_hvx = (uint8_t) n_hvx; + kparams->vtcm_size = (uint32_t) vtcm; + kparams->u.transpose = tp; + return true; +} + static bool htp_kparams_solve_cpy( const struct ggml_hexagon_session * sess, const struct ggml_tensor * op, @@ -2726,7 +2877,7 @@ static bool htp_kparams_solve_cpy( const bool src_is_contiguous = ggml_is_contiguous(src0); const bool dst_is_contiguous = ggml_is_contiguous(dst); - const uint32_t n_threads = htp_session_n_threads(sess); + const uint32_t n_hvx = htp_session_n_hvx(sess); const uint32_t mdev_count = htp_session_mdev_count(sess); const size_t vtcm_size = htp_session_vtcm_size(sess); @@ -2749,14 +2900,18 @@ static bool htp_kparams_solve_cpy( (uint32_t) src0->ne[0], (uint32_t) src0->nb[1], 0, 0, false, src_type_size, total_rows, - n_threads, mdev_count, vtcm_size)) { + n_hvx, mdev_count, vtcm_size)) { kparams->kernel_type = HTP_COPY_KERNEL_GATHER_ROWS; - kparams->n_threads = (uint8_t) n_threads; - kparams->vtcm_size = kparams->u.gather.slice * n_threads; + kparams->n_hvx = (uint8_t) n_hvx; + kparams->vtcm_size = kparams->u.gather.slice * n_hvx; return true; } } + if (htp_kparams_solve_cpy_transpose(sess, src0, dst, src_type_size, kparams)) { + return true; + } + if (sameshape) { kparams->kernel_type = HTP_COPY_KERNEL_SAMESHAPE_SAMETYPE; kparams->total_rows = (uint32_t) (src0->ne[1] * src0->ne[2] * src0->ne[3]); @@ -2764,7 +2919,7 @@ static bool htp_kparams_solve_cpy( } kparams->kernel_type = HTP_COPY_KERNEL_RESHAPE; - kparams->n_threads = n_threads > 0 ? (uint8_t) n_threads : 4; + kparams->n_hvx = n_hvx > 0 ? (uint8_t) n_hvx : 4; kparams->u.reshape.div_ne0 = init_fastdiv_values((uint32_t) dst->ne[0]); kparams->u.reshape.div_ne1_ne0 = init_fastdiv_values((uint32_t) (dst->ne[1] * dst->ne[0])); kparams->u.reshape.div_ne2_ne1_ne0 = init_fastdiv_values((uint32_t) (dst->ne[2] * dst->ne[1] * dst->ne[0])); @@ -2786,7 +2941,7 @@ static bool htp_kparams_solve_cpy( return false; } - const uint32_t nth = n_threads > 0 ? n_threads : 4; + const uint32_t nth = n_hvx > 0 ? n_hvx : 4; struct htp_copy_convert_vtcm_layout layout; htp_copy_convert_solve_layout(&layout, (uint32_t) src0->ne[0], (uint32_t) src0->ne[1], src_type_size, dst_type_size, nth); @@ -2796,7 +2951,7 @@ static bool htp_kparams_solve_cpy( kparams->kernel_type = HTP_COPY_KERNEL_SAMESHAPE_CONVERT; kparams->total_rows = (uint32_t) (src0->ne[1] * src0->ne[2] * src0->ne[3]); - kparams->n_threads = (uint8_t) nth; + kparams->n_hvx = (uint8_t) nth; kparams->vtcm_size = (uint32_t) layout.total_bytes; kparams->u.convert.blk_rows = layout.blk_rows; kparams->u.convert.div_ne01 = init_fastdiv_values((uint32_t) src0->ne[1]); @@ -2896,8 +3051,8 @@ static bool htp_kparams_solve_im2col( is_patchembed = is_patchembed && (s1 == (int32_t) KH && p1 == 0 && d1 == 1); } - const uint32_t max_threads = (sess && htp_session_n_threads(sess) > 0) ? htp_session_n_threads(sess) : 8; - const uint32_t n_threads = (total_rows < max_threads) ? total_rows : max_threads; + const uint32_t max_hvx = (sess && htp_session_n_hvx(sess) > 0) ? htp_session_n_hvx(sess) : 8; + const uint32_t n_hvx = (total_rows < max_hvx) ? total_rows : max_hvx; const uint32_t vtcm_size = (sess && htp_session_vtcm_size(sess) > 0) ? (uint32_t) htp_session_vtcm_size(sess) : 0; const uint32_t row_size = (uint32_t) dst->nb[2]; const uint32_t rows_per_chunk = (row_size > 0) ? (HEX_L2_LINE_SIZE / hex_gcd_u32(row_size, HEX_L2_LINE_SIZE)) : 1; @@ -2910,12 +3065,12 @@ static bool htp_kparams_solve_im2col( const uint32_t dst_row_bytes = hex_round_up((uint32_t) dst_row_bytes_raw, 256); struct htp_im2col_vtcm_layout layout; - htp_im2col_vtcm_layout_build(&layout, src_row_bytes, dst_row_bytes, n_threads, + htp_im2col_vtcm_layout_build(&layout, src_row_bytes, dst_row_bytes, n_hvx, htp_im2col_scratch_bytes(HTP_IM2COL_KERNEL_PATCHEMBED_DMA, KH, patch_stride)); if (vtcm_size == 0 || layout.total_bytes <= vtcm_size) { kparams->kernel_type = HTP_IM2COL_KERNEL_PATCHEMBED_DMA; - kparams->n_threads = (uint8_t) n_threads; + kparams->n_hvx = (uint8_t) n_hvx; kparams->vtcm_size = (uint32_t) layout.total_bytes; kparams->src_row_bytes = src_row_bytes; kparams->dst_row_bytes = dst_row_bytes; @@ -2963,7 +3118,7 @@ static bool htp_kparams_solve_im2col( // threads work on output-col blocks, so a single output row (1D) can use them all const uint32_t nbpr = (OW + owb - 1) / owb; const uint64_t nblocks = (uint64_t) total_rows * nbpr; - const uint32_t blk_threads = (uint32_t) (nblocks < max_threads ? nblocks : max_threads); + const uint32_t blk_threads = (uint32_t) (nblocks < max_hvx ? nblocks : max_hvx); struct htp_im2col_vtcm_layout layout; htp_im2col_vtcm_layout_build(&layout, src_row_bytes, dst_row_bytes, blk_threads, @@ -2971,7 +3126,7 @@ static bool htp_kparams_solve_im2col( if (vtcm_size == 0 || layout.total_bytes <= vtcm_size) { kparams->kernel_type = HTP_IM2COL_KERNEL_BLOCKED_DMA; - kparams->n_threads = (uint8_t) blk_threads; + kparams->n_hvx = (uint8_t) blk_threads; kparams->vtcm_size = (uint32_t) layout.total_bytes; kparams->src_row_bytes = src_row_bytes; kparams->dst_row_bytes = dst_row_bytes; @@ -3082,8 +3237,8 @@ static bool htp_kparams_solve_im2col_3d( p0 == 0 && p1 == 0 && p2 == 0 && d0 == 1 && d1 == 1 && d2 == 1); - const uint32_t max_threads = (sess && htp_session_n_threads(sess) > 0) ? htp_session_n_threads(sess) : 8; - const uint32_t n_threads = (total_rows < max_threads) ? total_rows : max_threads; + const uint32_t max_hvx = (sess && htp_session_n_hvx(sess) > 0) ? htp_session_n_hvx(sess) : 8; + const uint32_t n_hvx = (total_rows < max_hvx) ? total_rows : max_hvx; const uint32_t vtcm_size = (sess && htp_session_vtcm_size(sess) > 0) ? (uint32_t) htp_session_vtcm_size(sess) : 0; const uint32_t row_size = (uint32_t) dst->nb[2]; const uint32_t rows_per_chunk = (row_size > 0) ? (HEX_L2_LINE_SIZE / hex_gcd_u32(row_size, HEX_L2_LINE_SIZE)) : 1; @@ -3096,11 +3251,11 @@ static bool htp_kparams_solve_im2col_3d( const uint32_t dst_row_bytes = hex_round_up((uint32_t) dst_row_bytes_raw, 256); struct htp_im2col_vtcm_layout layout; - htp_im2col_vtcm_layout_build(&layout, src_row_bytes, dst_row_bytes, n_threads, 256); + htp_im2col_vtcm_layout_build(&layout, src_row_bytes, dst_row_bytes, n_hvx, 256); if (vtcm_size == 0 || layout.total_bytes <= vtcm_size) { kparams->kernel_type = HTP_IM2COL_KERNEL_PATCHEMBED_DMA; - kparams->n_threads = (uint8_t) n_threads; + kparams->n_hvx = (uint8_t) n_hvx; kparams->vtcm_size = (uint32_t) layout.total_bytes; kparams->src_row_bytes = src_row_bytes; kparams->dst_row_bytes = dst_row_bytes; @@ -3146,11 +3301,11 @@ static bool htp_kparams_solve_im2col_3d( const uint32_t dst_row_bytes = hex_round_up((uint32_t) dst_row_bytes_raw, 256); struct htp_im2col_vtcm_layout layout; - htp_im2col_vtcm_layout_build(&layout, src_row_bytes, dst_row_bytes, n_threads, 256); + htp_im2col_vtcm_layout_build(&layout, src_row_bytes, dst_row_bytes, n_hvx, 256); if (vtcm_size == 0 || layout.total_bytes <= vtcm_size) { kparams->kernel_type = HTP_IM2COL_KERNEL_BLOCKED_DMA; - kparams->n_threads = (uint8_t) n_threads; + kparams->n_hvx = (uint8_t) n_hvx; kparams->vtcm_size = (uint32_t) layout.total_bytes; kparams->src_row_bytes = src_row_bytes; kparams->dst_row_bytes = dst_row_bytes; @@ -3205,16 +3360,16 @@ static bool htp_kparams_solve_pad( const uint32_t type_size = (uint32_t) ggml_type_size(src0->type); const uint32_t total_dst_rows = (uint32_t) (dst->ne[1] * dst->ne[2] * dst->ne[3]); - const uint32_t max_threads = (std::max)(1u, total_dst_rows); - const uint32_t n_threads = (std::min)(htp_session_n_threads(sess), max_threads); + const uint32_t max_hvx = (std::max)(1u, total_dst_rows); + const uint32_t n_hvx = (std::min)(htp_session_n_hvx(sess), max_hvx); struct htp_pad_vtcm_layout layout; - htp_pad_vtcm_layout_build(&layout, (uint32_t) src0->ne[0] * type_size, (uint32_t) dst->ne[0] * type_size, n_threads); + htp_pad_vtcm_layout_build(&layout, (uint32_t) src0->ne[0] * type_size, (uint32_t) dst->ne[0] * type_size, n_hvx); const bool use_dma = (src0->nb[0] == type_size) && (src0->ne[0] >= 512) && (htp_session_vtcm_size(sess) >= layout.total_bytes); - kparams->n_threads = n_threads; + kparams->n_hvx = n_hvx; kparams->total_rows = total_dst_rows; kparams->use_dma = use_dma ? 1 : 0; kparams->elem_size = type_size; @@ -3257,15 +3412,15 @@ static bool htp_kparams_solve_cumsum( } const uint32_t total_rows = (uint32_t) (src0->ne[1] * src0->ne[2] * src0->ne[3]); - const uint32_t max_threads = (std::max)(1u, total_rows); - const uint32_t n_threads = (std::min)(htp_session_n_threads(sess), max_threads); + const uint32_t max_hvx = (std::max)(1u, total_rows); + const uint32_t n_hvx = (std::min)(htp_session_n_hvx(sess), max_hvx); struct htp_cumsum_vtcm_layout layout; - htp_cumsum_vtcm_layout_build(&layout, (uint32_t) src0->nb[1], (uint32_t) dst->nb[1], n_threads); + htp_cumsum_vtcm_layout_build(&layout, (uint32_t) src0->nb[1], (uint32_t) dst->nb[1], n_hvx); const bool use_dma = (htp_session_vtcm_size(sess) >= layout.total_bytes); - kparams->n_threads = n_threads; + kparams->n_hvx = n_hvx; kparams->total_rows = total_rows; kparams->use_dma = use_dma ? 1 : 0; kparams->vtcm_size = use_dma ? (uint32_t) layout.total_bytes : 0; @@ -3273,6 +3428,194 @@ static bool htp_kparams_solve_cumsum( return true; } +// Shrink until the layout fits: halve tile_cols down to 1024 columns, then drop threads. False below the VTCM floor. +static bool htp_top_k_mask_search(struct htp_top_k_mask_vtcm_layout * L, const struct htp_top_k_mask_shape * S, uint32_t kernel_type, + size_t vtcm_size, uint32_t * n_hvx, uint32_t * tile_cols) { + const uint32_t min_cols = 1024; + while (true) { + htp_top_k_mask_vtcm_layout_build(L, S, kernel_type, *n_hvx, *tile_cols); + if (L->total_bytes <= vtcm_size) { + return true; + } + if (*tile_cols > min_cols) { + *tile_cols = (std::max)(min_cols, (uint32_t) hex_round_up(*tile_cols / 2, 64)); + } else if (*n_hvx > 1) { + (*n_hvx)--; + } else { + return false; + } + } +} + +// Row r of t (same ne[1..3] as dst) is at r * nb[1]: the NPU needs one stride per input. +// A mask broadcasts by i mod ne: its rows are dense above nb[1], the divisors and dst ne[1..2] fit 16 bits. +static bool htp_top_k_mask_mask_ok(const struct ggml_tensor * m, const struct ggml_tensor * dst) { + return m->ne[1] <= UINT16_MAX && m->ne[2] <= UINT16_MAX && m->ne[3] <= UINT16_MAX && dst->ne[1] <= UINT16_MAX && dst->ne[2] <= UINT16_MAX && + (m->ne[2] == 1 || m->nb[2] == m->nb[1] * m->ne[1]) && + (m->ne[3] == 1 || m->nb[3] == m->nb[1] * m->ne[1] * m->ne[2]); +} + +static bool htp_top_k_mask_row_flat(const struct ggml_tensor * t, const struct ggml_tensor * dst) { + return t->ne[1] == dst->ne[1] && t->ne[2] == dst->ne[2] && t->ne[3] == dst->ne[3] && + (t->ne[2] == 1 || t->nb[2] == t->nb[1] * t->ne[1]) && + (t->ne[3] == 1 || t->nb[3] == t->nb[1] * t->ne[1] * t->ne[2]); +} + +static bool htp_kparams_solve_top_k_mask( + const struct ggml_hexagon_session * sess, + const struct ggml_tensor * op, + struct htp_top_k_mask_kernel_params * kparams +) { + memset(kparams, 0, sizeof(*kparams)); + + const struct ggml_tensor * mask = op->src[0]; + const struct ggml_tensor * top_k = op->src[1]; + const struct ggml_tensor * score = op->src[2]; + const struct ggml_tensor * blocks = op->src[3]; + const struct ggml_tensor * extra = op->src[4]; + const struct ggml_tensor * dst = op; + + if ((mask->type != GGML_TYPE_F16 && mask->type != GGML_TYPE_F32) || dst->type != mask->type) { + return false; + } + const size_t elem_size = ggml_type_size(dst->type); + if (mask->nb[0] != elem_size || dst->nb[0] != elem_size || !ggml_is_contiguous(dst)) { + return false; + } + if (top_k->type != GGML_TYPE_I32 || top_k->nb[0] != 4) { + return false; + } + if (score && (score->type != GGML_TYPE_F32 || score->nb[0] != 4)) { + return false; + } + if (extra && (extra->type != GGML_TYPE_I32 || extra->nb[0] != 4)) { + return false; + } + // flat row addressing: inputs follow the dst rows, a mask with more than one row too + if (!htp_top_k_mask_row_flat(top_k, dst) || (score && !htp_top_k_mask_row_flat(score, dst)) || + (extra && !htp_top_k_mask_row_flat(extra, dst)) || (ggml_nrows(mask) > 1 && !htp_top_k_mask_mask_ok(mask, dst))) { + return false; + } + // the kernel scales block ids by blocks->nb[1] with a 16-bit vector multiply + if (blocks && (blocks->type != GGML_TYPE_I32 || blocks->nb[0] != 4 || blocks->nb[1] != (size_t) blocks->ne[0] * 4 || blocks->nb[1] > INT16_MAX)) { + return false; + } + + struct htp_top_k_mask_shape S = {}; + S.n_kv = (uint32_t) dst->ne[0]; + S.elem_size = (uint32_t) elem_size; + S.n_top = (uint32_t) top_k->ne[0]; + S.n_blk = blocks ? (uint32_t) blocks->ne[0] : 0; + S.n_pick = (uint32_t) (blocks ? blocks->ne[1] : score ? score->ne[0] : dst->ne[0]); + S.n_extra = extra ? (uint32_t) extra->ne[0] : 0; + const uint32_t total_rows = (uint32_t) ggml_nrows(dst); + S.has_score = score ? 1 : 0; + S.has_blocks = blocks ? 1 : 0; + S.mask_resident = ggml_nrows(mask) == 1 ? 1 : 0; + + // empty inputs would give out-of-range gathers and zero-length DMA rows: leave them to the CPU + if (S.n_kv == 0 || total_rows == 0 || S.n_pick == 0 || S.n_top == 0 || (blocks && S.n_blk == 0) || (extra && S.n_extra == 0)) { + return false; + } + + // PICKS for one row with a resident mask when its single row tile fits, ROWS otherwise + const uint32_t sess_threads = (std::max)(1u, htp_session_n_hvx(sess)); + const size_t vtcm_size = htp_session_vtcm_size(sess); + + uint32_t kernel_type = HTP_TOP_K_MASK_KERNEL_ROWS; + uint32_t n_hvx = sess_threads; + uint32_t tile_cols = 0; + + struct htp_top_k_mask_vtcm_layout L; + bool found = false; + if (total_rows == 1 && S.mask_resident && sess_threads > 1) { + kernel_type = HTP_TOP_K_MASK_KERNEL_PICKS; + tile_cols = hex_round_up(S.n_kv, 64); + htp_top_k_mask_vtcm_layout_build(&L, &S, kernel_type, n_hvx, tile_cols); + found = L.total_bytes <= vtcm_size; + } + if (!found) { + kernel_type = HTP_TOP_K_MASK_KERNEL_ROWS; + n_hvx = (std::max)(1u, (std::min)(sess_threads, total_rows)); + tile_cols = hex_round_up(S.n_kv, 64); + found = htp_top_k_mask_search(&L, &S, kernel_type, vtcm_size, &n_hvx, &tile_cols); + } + if (!found) { + return false; + } + + // compact kernel_params fields: sizes and offsets in HTP_TOP_K_MASK_UNIT bytes as u16, all multiples of the unit + const uint64_t off_extra = htp_top_k_mask_rowin_off_extra(S.n_top); + const uint64_t off_score = htp_top_k_mask_rowin_off_score(S.n_top, S.n_extra); + const uint64_t u16_max = (uint64_t) UINT16_MAX * HTP_TOP_K_MASK_UNIT; + if (L.rowin_bytes > u16_max || L.tile_bytes > u16_max || off_extra > u16_max || off_score > u16_max || L.off_scratch > u16_max) { + return false; + } + const auto units = [](uint64_t bytes) { return (uint16_t) (bytes / HTP_TOP_K_MASK_UNIT); }; + + kparams->n_hvx = (uint8_t) n_hvx; + kparams->kernel_type = (uint8_t) kernel_type; + kparams->flags = (dst->type == GGML_TYPE_F16 ? HTP_TOP_K_MASK_F16 : 0) | + (score ? HTP_TOP_K_MASK_HAS_SCORE : 0) | + (blocks ? HTP_TOP_K_MASK_HAS_BLOCKS : 0) | + (extra ? HTP_TOP_K_MASK_HAS_EXTRA : 0) | + (S.mask_resident ? HTP_TOP_K_MASK_MASK_RESIDENT : 0); + kparams->elem_size = (uint8_t) elem_size; + kparams->vtcm_size = (uint32_t) L.total_bytes; + kparams->n_kv = S.n_kv; + kparams->n_top = S.n_top; + kparams->n_pick = S.n_pick; + kparams->n_extra = S.n_extra; + kparams->ninf = dst->type == GGML_TYPE_F16 ? 0xfc00fc00u : 0xff800000u; + kparams->blocks_bytes = (uint32_t) L.blocks_bytes; + kparams->score_bytes = score ? S.n_pick * 4 : 0; + kparams->tile_cols = tile_cols; + kparams->n_col_tiles = (S.n_kv + tile_cols - 1) / tile_cols; + kparams->n_blk = (uint16_t) S.n_blk; + kparams->blocks_nb1 = (uint16_t) (blocks ? blocks->nb[1] : 0); + kparams->off_blocks_u = units(L.off_blocks); + kparams->off_mask_u = units(L.off_mask); + kparams->off_rowin_u = units(L.off_rowin); + kparams->off_tiles_u = units(L.off_tiles); + kparams->off_mtiles_u = units(L.off_mtiles); + kparams->off_scratch_u = units(L.off_scratch); + kparams->rowin_units = units(L.rowin_bytes); + kparams->tile_units = units(L.tile_bytes); + kparams->off_extra_units = units(off_extra); + kparams->off_score_units = units(off_score); + + if (kernel_type == HTP_TOP_K_MASK_KERNEL_PICKS) { + // per thread slices of the static copies and of the flag clear, a multiple of the unit + const auto slice = [&](uint64_t n) { return units(hex_round_up((n + n_hvx - 1) / n_hvx, HTP_TOP_K_MASK_UNIT)); }; + kparams->u.picks.picks_per_hvx = (uint32_t) hex_round_up((S.n_top + n_hvx - 1) / n_hvx, 32); + kparams->u.picks.blocks_slice_u = slice(L.blocks_bytes); + kparams->u.picks.top_slice_u = slice((uint64_t) S.n_top * 4); + kparams->u.picks.extra_slice_u = slice((uint64_t) S.n_extra * 4); + kparams->u.picks.score_slice_u = slice(kparams->score_bytes); + kparams->u.picks.flags_slice_u = slice(L.tile_bytes); + } else { + kparams->u.rows.total_rows = total_rows; + kparams->u.rows.rs_top = (uint32_t) top_k->nb[1]; + kparams->u.rows.rs_extra = extra ? (uint32_t) extra->nb[1] : 0; + kparams->u.rows.rs_score = score ? (uint32_t) score->nb[1] : 0; + kparams->u.rows.rs_dst = (uint32_t) dst->nb[1]; + kparams->u.rows.mask_nb1 = (uint32_t) mask->nb[1]; + if (S.mask_resident) { + kparams->u.rows.m.mask_bytes = (uint32_t) L.mask_bytes; + } else { + const int64_t d[5] = { dst->ne[1], dst->ne[2], mask->ne[1], mask->ne[2], mask->ne[3] }; + for (int k = 0; k < 5; k++) { + const struct fastdiv_values fd = init_fastdiv_values((uint32_t) d[k]); + kparams->u.rows.m.dec.mp[k] = fd.mp; + kparams->u.rows.m.dec.l[k] = (uint8_t) fd.l; + kparams->u.rows.m.dec.ne[k] = (uint16_t) d[k]; + } + } + } + + return true; +} + static bool htp_kparams_solve_fill( const struct ggml_hexagon_session * sess, const struct ggml_tensor * op, @@ -3295,12 +3638,12 @@ static bool htp_kparams_solve_fill( const uint32_t nb1 = (uint32_t) dst->nb[1]; const uint32_t total_rows = ne1 * ne2 * ne3; - const uint32_t max_threads = (std::max)(1u, total_rows); - const uint32_t n_threads = (std::min)(htp_session_n_threads(sess), max_threads); + const uint32_t max_hvx = (std::max)(1u, total_rows); + const uint32_t n_hvx = (std::min)(htp_session_n_hvx(sess), max_hvx); const uint32_t elem_size = (uint32_t) ggml_type_size(dst->type); const uint32_t rows_per_chunk = (nb1 > 0) ? (HEX_L2_LINE_SIZE / hex_gcd_u32(nb1, HEX_L2_LINE_SIZE)) : 1; - kparams->n_threads = n_threads; + kparams->n_hvx = n_hvx; kparams->total_rows = total_rows; kparams->elem_size = elem_size; kparams->kernel_type = ggml_is_contiguous(dst) ? HTP_FILL_KERNEL_CONTIGUOUS : HTP_FILL_KERNEL_STRIDED; @@ -3347,10 +3690,10 @@ static bool htp_kparams_solve_roll( } const uint32_t total_rows = ne1 * ne2 * ne3; - const uint32_t max_threads = (std::max)(1u, total_rows); - const uint32_t n_threads = (std::min)(htp_session_n_threads(sess), max_threads); + const uint32_t max_hvx = (std::max)(1u, total_rows); + const uint32_t n_hvx = (std::min)(htp_session_n_hvx(sess), max_hvx); - kparams->n_threads = n_threads; + kparams->n_hvx = n_hvx; kparams->total_rows = total_rows; kparams->i0_src0 = i0_src0; kparams->n0 = ne0 - i0_src0; @@ -3393,8 +3736,8 @@ static bool htp_kparams_solve_diag( } const uint32_t total_batches = (uint32_t) (src0->ne[2] * src0->ne[3]); - const uint32_t max_threads = (std::max)(1u, total_batches); - const uint32_t n_threads = (std::min)(htp_session_n_threads(sess), max_threads); + const uint32_t max_hvx = (std::max)(1u, total_batches); + const uint32_t n_hvx = (std::min)(htp_session_n_hvx(sess), max_hvx); const uint32_t dst_batch_size = (uint32_t) (dst->ne[1] * dst->nb[1]); bool can_split = (dst->ne[0] == 1 || dst->nb[0] == sizeof(float)) && !ggml_is_permuted(dst); @@ -3415,11 +3758,11 @@ static bool htp_kparams_solve_diag( const uint32_t dst_row_size = (uint32_t) (dst->ne[0] * sizeof(float)); struct htp_diag_vtcm_layout layout; - htp_diag_vtcm_layout_build(&layout, src_batch_size, dst_row_size, n_threads); + htp_diag_vtcm_layout_build(&layout, src_batch_size, dst_row_size, n_hvx); const bool use_dma = (htp_session_vtcm_size(sess) >= layout.total_bytes); - kparams->n_threads = n_threads; + kparams->n_hvx = n_hvx; kparams->total_batches = total_batches; kparams->use_dma = use_dma ? 1 : 0; kparams->can_split = can_split ? 1 : 0; @@ -3450,19 +3793,19 @@ static bool htp_kparams_solve_solve_tri( return false; } - const uint32_t max_threads_sess = htp_session_n_threads(sess); + const uint32_t max_hvx_sess = htp_session_n_hvx(sess); const uint32_t k = (uint32_t) src1->ne[0]; const uint32_t col_block = 32; const uint32_t k_chunks = (k + col_block - 1) / col_block; const uint32_t total_batches = (uint32_t) (src0->ne[2] * src0->ne[3]); - const bool batched = total_batches >= max_threads_sess; + const bool batched = total_batches >= max_hvx_sess; const uint32_t total_jobs = batched ? total_batches : (total_batches * k_chunks); - const uint32_t max_threads = (std::max)(1u, total_jobs); - const uint32_t n_threads = (std::min)(max_threads_sess, max_threads); + const uint32_t max_hvx = (std::max)(1u, total_jobs); + const uint32_t n_hvx = (std::min)(max_hvx_sess, max_hvx); const uint32_t batch_size = (uint32_t) dst->nb[2]; const uint32_t batches_per_chunk = (batch_size > 0) ? (HEX_L2_LINE_SIZE / hex_gcd_u32(batch_size, HEX_L2_LINE_SIZE)) : 1; - kparams->n_threads = n_threads; + kparams->n_hvx = n_hvx; kparams->total_batches = total_batches; kparams->total_jobs = total_jobs; kparams->k_chunks = k_chunks; @@ -3539,7 +3882,7 @@ static bool htp_kparams_conv2d_win( const uint32_t halo = moff / 32 + 1; const uint32_t n_rt = (Cout + 31) / 32; const uint64_t n_q = (uint64_t) (N * IHS - (IHS - OH) - 1) * SR + OW; - const uint32_t nth = kp->n_threads; + const uint32_t nth = kp->n_hvx; const size_t tile = 2048; if ((uint64_t) N * IHS * SR + moff + 64 >= (1ull << 31)) { @@ -3562,14 +3905,14 @@ static bool htp_kparams_conv2d_win( } // enough packing units for all threads const uint32_t kpw2 = es == 2 ? 64 : 32; - while (!fold && w_cs > 2 && n_rt * ((Cin + 31) / 32) * (32 / w_cs) < kp->n_threads) { + while (!fold && w_cs > 2 && n_rt * ((Cin + 31) / 32) * (32 / w_cs) < kp->n_hvx) { w_cs /= 2; } wraw = (std::min)(wraw, (size_t) 32 * ((w_cs * T + kpw2 - 1) / kpw2) * 128); wscr = 3 * wraw + 64 * 128 + 128; } else { w_cs = fold ? KW : (std::min)(T, 16u); - while (!fold && w_cs > 1 && n_rt * ((Cin + 31) / 32) * ((T + w_cs - 1) / w_cs) < kp->n_threads) { + while (!fold && w_cs > 1 && n_rt * ((Cin + 31) / 32) * ((T + w_cs - 1) / w_cs) < kp->n_hvx) { w_cs = (w_cs + 1) / 2; } wscr = 2 * (size_t) w_cs * 32 * 128; @@ -3621,6 +3964,84 @@ static bool htp_kparams_conv2d_win( } } + uint32_t n_hvx = nth; + // Keep the fold schedule, where output traffic can dominate input packing. + if (!fold && rt_block < n_rt && ct > 0 && nth > 0) { + const uint32_t old_rt = rt_block; + const uint32_t ct_max = (std::min)((std::min)((std::max)(4 * halo, (n_tiles + 3) / 4), 512u), n_tiles); + const size_t budget = htp_session_vtcm_size(sess); + // Approximate chunk overhead in input crouton work units. + const size_t c_chunk = 8; + const uint32_t tail = n_tiles % ct; + const size_t work = + (size_t) (n_tiles / ct) * ((ct + halo + nth - 1) / nth) + (tail ? (tail + halo + nth - 1) / nth : 0); + size_t best_cost = + (size_t) ((n_rt + rt_block - 1) / rt_block) * (n_cc * work + c_chunk * (n_tiles / ct + (tail != 0))); + bool found = false; + + // Keep output DMA segments. Try larger kernel blocks to avoid rebuilding input croutons. + for (uint32_t rb = n_rt; rb > old_rt && !found; rb = (rb + 1) / 2) { + for (uint32_t nt = nth; nt > 0; nt /= 2) { + for (uint32_t sb = 8; sb > 0; sb /= 2) { + const size_t input_stage = hex_round_up(cwhn ? (size_t) sb * 32 * Cin * sizeof(float) + 256 : (size_t) n_ch * sb * 32 * sizeof(float), 128); + const size_t scratch = hex_round_up( + (std::max)(2 * input_stage + (size_t) HTP_CONV2D_WIN_OBUF * 32 * seg_t * 32 * sizeof(float), + wscr), + tile); + if (n_cc == 0 || n_wt > budget / tile / n_cc / rb || halo > budget / tile / n_cc || + scratch > budget / nt) { + continue; + } + const size_t weights = (size_t) rb * n_wt * n_cc * tile; + htp_conv2d_vtcm_layout fixed; + htp_conv2d_vtcm_layout_build(&fixed, rb, weights, (size_t) n_cc * halo * tile, 0, scratch, nt); + if (fixed.total_bytes >= budget) { + continue; + } + const size_t per_ct = 2 * ((size_t) n_cc + rb) * tile; + const uint32_t cap = (uint32_t) (std::min)((size_t) ct_max, (budget - fixed.total_bytes) / per_ct); + const uint32_t aligned = (cap + halo) / nt * nt; + const uint32_t candidates[2] = { cap, aligned > halo ? aligned - halo : 0 }; + for (uint32_t i = 0; i < 2; i++) { + const uint32_t candidate = candidates[i]; + if (candidate == 0 || (i > 0 && candidate == cap)) { + continue; + } + if (hex_round_up((size_t) T * sizeof(uint32_t), 128) + 2 * hex_round_up((size_t) candidate, 128) > HTP_DDR_SPAD_SIZE) { + continue; + } + const uint32_t rem = n_tiles % candidate; + const size_t input_work = (size_t) (n_tiles / candidate) * ((candidate + halo + nt - 1) / nt) + (rem ? (rem + halo + nt - 1) / nt : 0); + const size_t chunks = n_tiles / candidate + (rem != 0); + const size_t cost = (size_t) ((n_rt + rb - 1) / rb) * (n_cc * input_work + c_chunk * chunks); + if (cost < best_cost) { + const size_t input = (size_t) n_cc * (candidate + halo) * tile; + const size_t output = (size_t) rb * candidate * tile; + htp_conv2d_vtcm_layout L; + htp_conv2d_vtcm_layout_build(&L, rb, weights, input, output, scratch, nt); + if (L.total_bytes > budget) { + continue; + } + found = true; + best_cost = cost; + rt_block = rb; + ct = candidate; + seg_b = sb; + n_hvx = nt; + stage = input_stage; + thr = scratch; + act = input; + out = output; + w_size = weights; + total = L.total_bytes; + n_bc = candidate + halo; + } + } + } + } + } + } + if (hex_round_up((size_t) T * sizeof(uint32_t), 128) + 2 * hex_round_up((size_t) ct, 128) > HTP_DDR_SPAD_SIZE) { return false; } @@ -3631,7 +4052,7 @@ static bool htp_kparams_conv2d_win( const uint32_t n_cs = (32 + CS - 1) / CS; kp->kernel_type = (uint8_t) HTP_CONV2D_KERNEL_HMX_WINDOW; - kp->n_threads = (uint8_t) nth; + kp->n_hvx = (uint8_t) n_hvx; kp->cwhn = cwhn ? 1 : 0; kp->w_kmajor = !w_rows ? 1 : 0; kp->n_row_tiles = n_rt; @@ -3659,6 +4080,50 @@ static bool htp_kparams_conv2d_win( return true; } +struct htp_conv_norm_geom { + uint32_t KW, KH, Cin, Cout; + uint32_t OW, OH, N; + int32_t s0, d0; + size_t es; + bool is3d; +}; + +static bool htp_conv_normalize_geom(const struct ggml_tensor * dst, struct htp_conv_norm_geom * g) { + const struct ggml_tensor * src0 = dst->src[0]; + const bool is3d = dst->op == GGML_OP_CONV_3D; + const int32_t s0 = ggml_get_op_params_i32(dst, 0); + const int32_t d0 = ggml_get_op_params_i32(dst, is3d ? 6 : 4); + + const int64_t kh_w = is3d ? src0->ne[2] * src0->ne[1] : src0->ne[1]; + const int64_t cin_w = is3d ? ggml_get_op_params_i32(dst, 9) : src0->ne[2]; + const int64_t oh_w = is3d ? dst->ne[2] * dst->ne[1] : dst->ne[1]; + const int64_t n_w = is3d ? ggml_get_op_params_i32(dst, 10) : dst->ne[3]; + const int64_t k_w = src0->ne[0] * kh_w * cin_w; + const int64_t ct_w = n_w * oh_w * ((dst->ne[0] + 31) / 32); + + if (kh_w < 1 || kh_w > 255 || k_w < 1 || k_w > (1 << 24) || ct_w < 1 || ct_w > UINT32_MAX) { + return false; + } + if (s0 <= 0 || d0 <= 0 || dst->nb[0] != sizeof(float)) { + return false; + } + + *g = { + /* .KW = */ (uint32_t) src0->ne[0], + /* .KH = */ (uint32_t) kh_w, + /* .Cin = */ (uint32_t) cin_w, + /* .Cout = */ (uint32_t) (is3d ? ggml_get_op_params_i32(dst, 11) : src0->ne[3]), + /* .OW = */ (uint32_t) dst->ne[0], + /* .OH = */ (uint32_t) oh_w, + /* .N = */ (uint32_t) n_w, + /* .s0 = */ s0, + /* .d0 = */ d0, + /* .es = */ ggml_type_size(src0->type), + /* .is3d = */ is3d, + }; + return true; +} + // Tiled implicit GEMM, MMA on HMX if available, else HVX static bool htp_kparams_conv2d_tiled( const struct ggml_hexagon_session * sess, @@ -3669,92 +4134,129 @@ static bool htp_kparams_conv2d_tiled( return false; } - const struct ggml_tensor * src0 = dst->src[0]; - const struct ggml_tensor * src1 = dst->src[1]; - const int32_t s0 = ggml_get_op_params_i32(dst, 0); - const int32_t d0 = ggml_get_op_params_i32(dst, 4); - - const uint32_t KW = src0->ne[0]; - const uint32_t KH = src0->ne[1]; - const uint32_t Cin = src0->ne[2]; - const uint32_t Cout = src0->ne[3]; - const uint32_t OW = dst->ne[0]; - const uint32_t OH = dst->ne[1]; - const uint32_t N = dst->ne[3]; - const size_t es = ggml_type_size(src0->type); - - if (s0 != 1 || dst->nb[0] != sizeof(float)) { + struct htp_conv_norm_geom g; + if (!htp_conv_normalize_geom(dst, &g)) { return false; } + const struct ggml_tensor * src0 = dst->src[0]; + const struct ggml_tensor * src1 = dst->src[1]; + // kernel k order must match the staged input: (cin, kh, kw) for WHCN input, (kh, kw, cin) for CWHN input. // Kernel is either k-contiguous rows per cout (WHCN only) or cout-contiguous with a uniform k stride. - const bool whcn = src1->nb[0] == sizeof(float); - const bool cwhn = !whcn && src1->nb[2] == sizeof(float) && src1->nb[0] == Cin * sizeof(float); - const bool co_c = Cout == 1 || src0->nb[3] == es; - const bool w_rows = whcn && src0->nb[0] == es && (KH == 1 || src0->nb[1] == KW * es) && (Cin == 1 || src0->nb[2] == KW * KH * es); + const bool whcn = g.is3d || src1->nb[0] == sizeof(float); + const bool cwhn = !whcn && src1->nb[2] == sizeof(float) && src1->nb[0] == g.Cin * sizeof(float); + const bool co_c = g.Cout == 1 || src0->nb[3] == g.es; + const bool w_rows = g.is3d || (whcn && src0->nb[0] == g.es && (g.KH == 1 || src0->nb[1] == g.KW * g.es) && (g.Cin == 1 || src0->nb[2] == g.KW * g.KH * g.es)); const size_t ks = whcn ? src0->nb[0] : src0->nb[2]; bool kmajor = false; if (whcn) { - kmajor = co_c && ks >= Cout * es && (KH == 1 || src0->nb[1] == KW * ks) && (Cin == 1 || src0->nb[2] == KW * KH * ks); + kmajor = co_c && ks >= g.Cout * g.es && (g.KH == 1 || src0->nb[1] == g.KW * ks) && (g.Cin == 1 || src0->nb[2] == g.KW * g.KH * ks); } else if (cwhn) { - kmajor = co_c && ks >= Cout * es && src0->nb[0] == Cin * ks && (KH == 1 || src0->nb[1] == KW * Cin * ks); + kmajor = co_c && ks >= g.Cout * g.es && src0->nb[0] == g.Cin * ks && (g.KH == 1 || src0->nb[1] == g.KW * g.Cin * ks); } if (!w_rows && !kmajor) { return false; } kmajor = kmajor && !w_rows; - const uint32_t K = KW * KH * Cin; - const uint32_t kg = !cwhn ? K : d0 == 1 ? KW * Cin : Cin; + const uint32_t K = g.KW * g.KH * g.Cin; + const uint32_t kg = !cwhn ? K : g.d0 == 1 ? g.KW * g.Cin : g.Cin; const uint32_t n_grp = K / kg; const uint32_t kg_p = hex_round_up(kg, 32); const uint32_t k_pad = hex_round_up(n_grp * kg_p, 64); const uint32_t n_kt = k_pad / 32; - const uint32_t n_rt = (Cout + 31) / 32; - const uint32_t n_owt = (OW + 31) / 32; - const uint32_t n_ct = N * OH * n_owt; + const uint32_t n_rt = (g.Cout + 31) / 32; + const uint32_t n_owt = (g.OW + 31) / 32; + const uint32_t n_ct = g.N * g.OH * n_owt; const uint32_t wk = (std::min)(k_pad, 512u); - const uint32_t nth = htp_session_n_threads(sess); + const uint32_t max_hvx = htp_session_n_hvx(sess); const size_t tile = 2048; uint32_t rt_block = n_rt; - uint32_t ct = (std::min)(n_ct, 64u); - uint32_t seg = (std::min)(8u, n_owt); + uint32_t ct = 0; + uint32_t seg = 0; + uint32_t nth = 0; uint32_t n_stage = 1; uint32_t sr = 0; - size_t stage = 0, ostage = 0, prow = 0, wpack = 0; - size_t w_size = 0, act_size = 0, out_size = 0, thr_size = 0, total = 0; + size_t stage = 0, thr_size = 0; + size_t best_cost = SIZE_MAX; + const size_t wpack = kmajor ? 2 * (size_t) wk * 64 * sizeof(float) + 96 * 128 : 2 * (size_t) 32 * wk * (sizeof(float) + sizeof(uint16_t)); + const size_t budget = htp_session_vtcm_size(sess); + // Approximate chunk overhead in patch K-tile work units. + const size_t c_chunk = 8; + + // Keep the kernel resident to avoid regenerating the input for each rt block. + // Balance patch work per thread and chunk overhead before reducing rt. + for (uint32_t rb = n_rt; rb > 0 && ct == 0; rb = rb > 1 ? (rb + 1) / 2 : 0) { + for (uint32_t nt = max_hvx; nt > 0; nt /= 2) { + for (uint32_t sg = (std::min)(8u, n_owt); sg > 0; --sg) { + // Keep the unused final stride lane for full-vector loads during packing. + const uint64_t wb = (uint64_t) 32 * sg * g.s0 + (uint64_t) (g.KW - 1) * g.d0; + if (wb > budget / sizeof(float) || wb > INT32_MAX) { + continue; + } + const uint64_t width = cwhn ? wb * g.Cin : wb; + if (width > budget / sizeof(float) || width > UINT32_MAX - 63) { + continue; + } + const uint32_t row = hex_round_up((uint32_t) width, cwhn ? 64 : 32); + const uint64_t rows = cwhn ? g.KH : (uint64_t) g.Cin * g.KH; + if (rows > budget / ((uint64_t) row * sizeof(float))) { + continue; + } + const size_t input_stage = cwhn ? (size_t) g.KH * row * sizeof(float) : (size_t) g.Cin * g.KH * row * sizeof(float); + const size_t output_stage = 2 * 32 * (size_t) sg * 32 * sizeof(float); + const size_t transpose = cwhn ? input_stage / 2 + 256 + 64 * 128 : 0; + const size_t scratch = hex_round_up((std::max)(input_stage + output_stage + transpose, wpack), tile); + htp_conv2d_vtcm_layout fixed; + htp_conv2d_vtcm_layout_build(&fixed, rb, (size_t) rb * n_kt * tile, 0, 0, scratch, nt); + if (fixed.total_bytes >= budget) { + continue; + } - while (true) { - const uint32_t wb = 32 * seg + (KW - 1) * d0; - sr = cwhn ? hex_round_up(wb * Cin, 64) : hex_round_up(wb, 32); - stage = cwhn ? (size_t) KH * sr * sizeof(float) : (size_t) Cin * KH * sr * sizeof(float); - ostage = 2 * 32 * (size_t) seg * 32 * sizeof(float); - prow = cwhn ? stage / 2 + 256 + 64 * 128 : 0; - wpack = kmajor ? 2 * (size_t) wk * 64 * sizeof(float) + 96 * 128 : 2 * (size_t) 32 * wk * (sizeof(float) + sizeof(uint16_t)); - thr_size = hex_round_up((std::max)(stage + ostage + prow, wpack), tile); - w_size = (size_t) rt_block * n_kt * tile; - act_size = (size_t) ct * n_kt * tile; - out_size = (size_t) rt_block * ct * tile; - struct htp_conv2d_vtcm_layout L; - htp_conv2d_vtcm_layout_build(&L, rt_block, w_size, act_size, out_size, thr_size, nth); - total = L.total_bytes; - if (total <= htp_session_vtcm_size(sess)) { - break; - } - if (ct > 8) { - ct /= 2; - } else if (seg > 1) { - seg--; - } else if (rt_block > 1) { - rt_block = (rt_block + 1) / 2; - } else if (ct > 1) { - ct /= 2; - } else { - return false; + // Each spatial tile adds two activation tiles and two output tiles per rt. + const size_t per_ct = 2 * ((size_t) n_kt + rb) * tile; + const uint32_t nc = (uint32_t) (std::min)((size_t) (std::min)(n_ct, 64u), (budget - fixed.total_bytes) / per_ct); + const uint32_t candidates[2] = { nc, nc / nt * nt }; + for (uint32_t i = 0; i < 2; i++) { + const uint32_t candidate = candidates[i]; + if (candidate == 0 || (i > 0 && candidate == candidates[0])) { + continue; + } + const uint32_t per_hvx = (candidate + nt - 1) / nt; + if (sg > per_hvx) { + continue; + } + const size_t work = (size_t) (n_ct / candidate) * per_hvx + (n_ct % candidate + nt - 1) / nt; + const size_t chunks = n_ct / candidate + (n_ct % candidate != 0); + const size_t cost = work * n_kt + c_chunk * chunks; + if (cost < best_cost) { + best_cost = cost; + rt_block = rb; + ct = candidate; + seg = sg; + nth = nt; + sr = row; + stage = input_stage; + thr_size = scratch; + } + } + } } } + if (ct == 0) { + return false; + } + + const size_t w_size = (size_t) rt_block * n_kt * tile; + const size_t act_size = (size_t) ct * n_kt * tile; + const size_t out_size = (size_t) rt_block * ct * tile; + const size_t ostage = 2 * 32 * (size_t) seg * 32 * sizeof(float); + const size_t prow = cwhn ? stage / 2 + 256 + 64 * 128 : 0; + struct htp_conv2d_vtcm_layout L; + htp_conv2d_vtcm_layout_build(&L, rt_block, w_size, act_size, out_size, thr_size, nth); + size_t total = L.total_bytes; // Second stage buffer: DMA of the next segment overlaps HVX pack of this one. // Skip large stages. They already fill VTCM, and the extra DMA slows HMX. @@ -3762,7 +4264,7 @@ static bool htp_kparams_conv2d_tiled( const size_t thr2 = hex_round_up((std::max)(2 * stage + ostage + prow, wpack), tile); struct htp_conv2d_vtcm_layout L2; htp_conv2d_vtcm_layout_build(&L2, rt_block, w_size, act_size, out_size, thr2, nth); - if (stage <= 128 * 1024 && L2.total_bytes <= htp_session_vtcm_size(sess)) { + if (stage <= 128 * 1024 && 2 * g.KH <= 255 && L2.total_bytes <= htp_session_vtcm_size(sess)) { n_stage = 2; thr_size = thr2; total = L2.total_bytes; @@ -3770,7 +4272,7 @@ static bool htp_kparams_conv2d_tiled( } kp->kernel_type = (uint8_t) HTP_CONV2D_KERNEL_HMX_TILED; - kp->n_threads = (uint8_t) nth; + kp->n_hvx = (uint8_t) nth; kp->cwhn = cwhn ? 1 : 0; kp->w_kmajor = kmajor ? 1 : 0; kp->n_row_tiles = n_rt; @@ -3791,7 +4293,9 @@ static bool htp_kparams_conv2d_tiled( kp->tiled.n_stage = n_stage; kp->tiled.wk_chunk = wk; kp->tiled.div_ow_tiles = init_fastdiv_values(n_owt); - kp->tiled.div_oh = init_fastdiv_values(OH); + kp->tiled.div_oh = init_fastdiv_values(g.OH); + kp->tiled.div_khr = init_fastdiv_values(src0->ne[1]); + kp->tiled.div_ohr = init_fastdiv_values(dst->ne[1]); return true; } @@ -3804,7 +4308,7 @@ static bool htp_kparams_solve_conv2d_unchecked( const uint32_t total_rows = (uint32_t) (dst->ne[1] * dst->ne[2] * dst->ne[3]); kparams->total_rows = total_rows; - kparams->n_threads = total_rows == 0 ? 1 : (std::min)((uint32_t) htp_session_n_threads(sess), total_rows); + kparams->n_hvx = total_rows == 0 ? 1 : (std::min)((uint32_t) htp_session_n_hvx(sess), total_rows); if (htp_kparams_conv2d_win(sess, dst, kparams)) { return true; @@ -3840,6 +4344,10 @@ static bool htp_kparams_solve_conv2d( const int32_t d0 = ggml_get_op_params_i32(op, 4); const int32_t d1 = ggml_get_op_params_i32(op, 5); + if (s0 <= 0 || s1 <= 0 || d0 <= 0 || d1 <= 0) { + return false; + } + const int64_t ow = (src1->ne[0] + 2 * p0 - d0 * (src0->ne[0] - 1) - 1) / s0 + 1; const int64_t oh = (src1->ne[1] + 2 * p1 - d1 * (src0->ne[1] - 1) - 1) / s1 + 1; if (ow != dst->ne[0] || oh != dst->ne[1] || src0->ne[3] != dst->ne[2] || src1->ne[3] != dst->ne[3]) { @@ -3945,10 +4453,10 @@ static bool htp_kparams_solve_conv2d_dw( const uint32_t w_size = (uint32_t) (kw * kh * c * ggml_type_size(src0->type)); - uint32_t max_threads_sess = (uint32_t) htp_session_n_threads(sess); + uint32_t max_hvx_sess = (uint32_t) htp_session_n_hvx(sess); uint32_t oh_chunk = oh_u; - uint32_t n_threads = max_threads_sess; + uint32_t n_hvx = max_hvx_sess; struct htp_conv2d_dw_vtcm_layout layout; while (true) { @@ -3956,7 +4464,7 @@ static bool htp_kparams_solve_conv2d_dw( const uint32_t max_src_bytes = (max_ih_chunk + pad_rows) * row_src_bytes + 128; const uint32_t max_dst_bytes = oh_chunk * row_dst_bytes + 128; - htp_conv2d_dw_vtcm_layout_build(&layout, max_src_bytes, max_dst_bytes, w_size, n_threads); + htp_conv2d_dw_vtcm_layout_build(&layout, max_src_bytes, max_dst_bytes, w_size, n_hvx); if (layout.total_bytes <= vtcm_budget && layout.total_bytes > 0) { break; @@ -3964,8 +4472,8 @@ static bool htp_kparams_solve_conv2d_dw( if (oh_chunk > 1) { oh_chunk = (oh_chunk + 1) / 2; - } else if (n_threads > 1) { - n_threads--; + } else if (n_hvx > 1) { + n_hvx--; } else { return false; } @@ -3973,18 +4481,18 @@ static bool htp_kparams_solve_conv2d_dw( const uint32_t n_chunks = (oh_u + oh_chunk - 1) / oh_chunk; const uint32_t total_tasks = cwhn ? (n * n_chunks) : (c * n * n_chunks); - n_threads = (std::min)(n_threads, total_tasks); - if (n_threads == 0) { - n_threads = 1; + n_hvx = (std::min)(n_hvx, total_tasks); + if (n_hvx == 0) { + n_hvx = 1; } const uint32_t max_ih_chunk = (std::min)(h, (oh_chunk - 1) * (uint32_t) s1 + (kh - 1) * (uint32_t) d1 + 1); const uint32_t max_src_bytes = (max_ih_chunk + pad_rows) * row_src_bytes + 128; const uint32_t max_dst_bytes = oh_chunk * row_dst_bytes + 128; - htp_conv2d_dw_vtcm_layout_build(&layout, max_src_bytes, max_dst_bytes, w_size, n_threads); + htp_conv2d_dw_vtcm_layout_build(&layout, max_src_bytes, max_dst_bytes, w_size, n_hvx); kparams->kernel_type = kernel_type; - kparams->n_threads = (uint8_t) n_threads; + kparams->n_hvx = (uint8_t) n_hvx; kparams->cwhn = cwhn ? 1 : 0; kparams->is_f16 = (src0->type == GGML_TYPE_F16) ? 1 : 0; kparams->total_tasks = total_tasks; @@ -4014,6 +4522,663 @@ static bool htp_kparams_solve_conv2d_dw( return true; } +static bool htp_kparams_solve_conv3d( + const struct ggml_hexagon_session * sess, + const struct ggml_tensor * op, + struct htp_conv2d_kernel_params * kparams +) { + memset(kparams, 0, sizeof(*kparams)); + + const struct ggml_tensor * src0 = op->src[0]; + const struct ggml_tensor * src1 = op->src[1]; + const struct ggml_tensor * dst = op; + + if ((src0->type != GGML_TYPE_F16 && src0->type != GGML_TYPE_F32) || + src1->type != GGML_TYPE_F32 || dst->type != GGML_TYPE_F32) { + return false; + } + + const int32_t IC = ggml_get_op_params_i32(op, 9); + const int32_t N = ggml_get_op_params_i32(op, 10); + const int32_t OC = ggml_get_op_params_i32(op, 11); + if (IC <= 0 || N <= 0 || OC <= 0 || src0->ne[3] != (int64_t) IC * OC || src1->ne[3] != (int64_t) IC * N || + dst->ne[3] != (int64_t) OC * N) { + return false; + } + + for (int i = 0; i < 3; i++) { + const int32_t s = ggml_get_op_params_i32(op, i); + const int32_t p = ggml_get_op_params_i32(op, 3 + i); + const int32_t d = ggml_get_op_params_i32(op, 6 + i); + if (s <= 0 || d <= 0 || p < 0 || src0->ne[i] <= 0) { + return false; + } + const int64_t o = (src1->ne[i] + 2 * p - d * (src0->ne[i] - 1) - 1) / s + 1; + if (o != dst->ne[i]) { + return false; + } + } + + const size_t es = ggml_type_size(src0->type); + const bool w_rows = src0->nb[0] == es && src0->nb[1] == src0->ne[0] * es && src0->nb[2] == src0->ne[1] * src0->nb[1] && + ((int64_t) IC * OC == 1 || src0->nb[3] == src0->ne[2] * src0->nb[2]); + if (!w_rows || src1->nb[0] != sizeof(float) || dst->nb[2] != dst->ne[1] * dst->nb[1]) { + return false; + } + + const int64_t total_rows = dst->ne[1] * dst->ne[2] * dst->ne[3]; + if (total_rows > UINT32_MAX) { + return false; + } + kparams->total_rows = (uint32_t) total_rows; + kparams->n_hvx = total_rows == 0 ? 1 : (std::min)((uint32_t) htp_session_n_hvx(sess), (uint32_t) total_rows); + + return htp_kparams_conv2d_tiled(sess, op, kparams); +} + +static bool htp_dsv4_hc_pre_can_collapse( + const struct ggml_tensor * x, + const struct ggml_tensor * weights, + const struct ggml_tensor * dst +) { + if (!x || !weights || !dst) { + return false; + } + if (x->type != GGML_TYPE_F32 || weights->type != GGML_TYPE_F32 || dst->type != GGML_TYPE_F32) { + return false; + } + if (x->nb[0] != sizeof(float) || weights->nb[0] != sizeof(float) || dst->nb[0] != sizeof(float)) { + return false; + } + if (x->ne[1] != 4 || dst->ne[0] != x->ne[0]) { + return false; + } + + const bool gated = ggml_get_op_params_i32(dst, 1) != 0; + const int64_t x_tokens = x->ne[2] * x->ne[3]; + const int64_t d_tokens = dst->ne[1] * dst->ne[2] * dst->ne[3]; + const int64_t w_tokens = gated ? (weights->ne[2] * weights->ne[3]) : (weights->ne[1] * weights->ne[2] * weights->ne[3]); + + if (x_tokens != d_tokens || x_tokens != w_tokens) { + return false; + } + + if (x->ne[3] == 1 && dst->ne[2] == 1 && dst->ne[3] == 1 && + (gated ? (weights->ne[3] == 1) : (weights->ne[2] == 1 && weights->ne[3] == 1))) { + return false; + } + + if (x->ne[3] > 1 && x->nb[3] != x->ne[2] * x->nb[2]) return false; + + if (dst->ne[2] > 1 && dst->nb[2] != dst->ne[1] * dst->nb[1]) return false; + if (dst->ne[3] > 1 && dst->nb[3] != dst->ne[2] * dst->nb[2]) return false; + + if (gated) { + if (weights->ne[3] > 1 && weights->nb[3] != weights->ne[2] * weights->nb[2]) return false; + } else { + if (weights->ne[2] > 1 && weights->nb[2] != weights->ne[1] * weights->nb[1]) return false; + if (weights->ne[3] > 1 && weights->nb[3] != weights->ne[2] * weights->nb[2]) return false; + } + + return true; +} + +static void htp_dsv4_hc_pre_collapse_tensors( + const struct ggml_tensor * x, + const struct ggml_tensor * weights, + const struct ggml_tensor * dst, + struct ggml_tensor * x_c, + struct ggml_tensor * w_c, + struct ggml_tensor * d_c +) { + const int64_t n_tokens = x->ne[2] * x->ne[3]; + const bool gated = ggml_get_op_params_i32(dst, 1) != 0; + + *x_c = *x; + x_c->ne[2] = n_tokens; + x_c->ne[3] = 1; + x_c->nb[3] = x_c->ne[2] * x_c->nb[2]; + + *d_c = *dst; + d_c->ne[1] = n_tokens; + d_c->ne[2] = 1; + d_c->ne[3] = 1; + d_c->nb[2] = d_c->ne[1] * d_c->nb[1]; + d_c->nb[3] = d_c->nb[2]; + + *w_c = *weights; + if (gated) { + w_c->ne[2] = n_tokens; + w_c->ne[3] = 1; + w_c->nb[3] = w_c->ne[2] * w_c->nb[2]; + } else { + w_c->ne[1] = n_tokens; + w_c->ne[2] = 1; + w_c->ne[3] = 1; + w_c->nb[2] = w_c->ne[1] * w_c->nb[1]; + w_c->nb[3] = w_c->nb[2]; + } +} + +static bool htp_kparams_solve_dsv4_hc_pre_impl( + const struct ggml_hexagon_session * sess, + const struct ggml_tensor * x, + const struct ggml_tensor * weights, + const struct ggml_tensor * dst, + struct htp_dsv4_hc_pre_kernel_params * kparams +) { + memset(kparams, 0, sizeof(*kparams)); + + if (x->type != GGML_TYPE_F32 || weights->type != GGML_TYPE_F32 || dst->type != GGML_TYPE_F32) { + return false; + } + if (x->ne[1] != 4 || x->ne[3] != 1) { + return false; + } + if (x->ne[0] % HTP_DSV4_HC_CHUNK_ELEMS != 0) { + return false; + } + if (dst->ne[0] != x->ne[0] || dst->ne[1] != x->ne[2] || dst->ne[2] != 1 || dst->ne[3] != 1) { + return false; + } + if (x->nb[0] != sizeof(float) || weights->nb[0] != sizeof(float) || dst->nb[0] != sizeof(float)) { + return false; + } + if (ggml_is_permuted(x) || ggml_is_permuted(weights) || ggml_is_permuted(dst)) { + return false; + } + + const bool gated = ggml_get_op_params_i32(dst, 1) != 0; + if (gated) { + if (weights->ne[0] != x->ne[0] || weights->ne[1] != x->ne[1] || weights->ne[2] != x->ne[2] || weights->ne[3] != 1) { + return false; + } + } else { + if (weights->ne[0] != x->ne[1] || weights->ne[1] != x->ne[2] || weights->ne[2] != 1 || weights->ne[3] != 1) { + return false; + } + } + + const uint32_t n_tokens = (uint32_t) x->ne[2]; + kparams->kernel_type = gated ? HTP_DSV4_HC_PRE_KERNEL_GATED : HTP_DSV4_HC_PRE_KERNEL_PLAIN; + + kparams->n_chunks = ((uint32_t) x->ne[0] + HTP_DSV4_HC_CHUNK_ELEMS - 1) / HTP_DSV4_HC_CHUNK_ELEMS; + kparams->total_jobs = n_tokens * kparams->n_chunks; + if (kparams->total_jobs == 0) { + return true; + } + kparams->n_hvx = (std::min)(htp_session_n_hvx(sess), kparams->total_jobs); + kparams->jobs_per_hvx = (kparams->total_jobs + kparams->n_hvx - 1) / kparams->n_hvx; + + const size_t vtcm_budget = htp_session_vtcm_size(sess); + const uint32_t n_rows = gated ? HTP_DSV4_HC_PRE_GATED_ROWS : HTP_DSV4_HC_PRE_PLAIN_ROWS; + uint32_t block_chunks = (std::min)((uint32_t) HTP_DSV4_HC_MAX_CHUNKS, kparams->n_chunks); + while (block_chunks > 16) { + const uint32_t stride = block_chunks * 128; + const uint32_t buf_sz = n_rows * stride; + if (2 * buf_sz * kparams->n_hvx <= vtcm_budget) { + break; + } + block_chunks /= 2; + } + + kparams->dma_block_chunks = block_chunks; + kparams->dma_row_stride = block_chunks * 128; + kparams->dma_buffer_size = n_rows * kparams->dma_row_stride; + kparams->vtcm_size = 2 * kparams->dma_buffer_size * kparams->n_hvx; + if (kparams->vtcm_size > vtcm_budget) { + return false; + } + + kparams->div_n_chunks = init_fastdiv_values(kparams->n_chunks); + kparams->div_n_hvx = init_fastdiv_values(kparams->n_hvx); + + return true; +} + +static inline bool htp_kparams_solve_dsv4_hc_pre( + const struct ggml_hexagon_session * sess, + const struct ggml_tensor * op, + struct htp_dsv4_hc_pre_kernel_params * kparams +) { + if (htp_dsv4_hc_pre_can_collapse(op->src[0], op->src[1], op)) { + struct ggml_tensor x_c, w_c, d_c; + htp_dsv4_hc_pre_collapse_tensors(op->src[0], op->src[1], op, &x_c, &w_c, &d_c); + return htp_kparams_solve_dsv4_hc_pre_impl(sess, &x_c, &w_c, &d_c, kparams); + } + return htp_kparams_solve_dsv4_hc_pre_impl(sess, op->src[0], op->src[1], op, kparams); +} + +static bool htp_dsv4_hc_post_can_collapse( + const struct ggml_tensor * x, + const struct ggml_tensor * residual, + const struct ggml_tensor * post, + const struct ggml_tensor * comb, + const struct ggml_tensor * dst +) { + if (!x || !residual || !post || !dst) { + return false; + } + if (x->type != GGML_TYPE_F32 || residual->type != GGML_TYPE_F32 || post->type != GGML_TYPE_F32 || dst->type != GGML_TYPE_F32) { + return false; + } + if (comb && comb->type != GGML_TYPE_F32) { + return false; + } + if (x->nb[0] != sizeof(float) || residual->nb[0] != sizeof(float) || post->nb[0] != sizeof(float) || dst->nb[0] != sizeof(float)) { + return false; + } + if (comb && comb->nb[0] != sizeof(float)) { + return false; + } + if (residual->ne[0] != x->ne[0] || residual->ne[1] != 4 || dst->ne[0] != x->ne[0] || dst->ne[1] != 4 || post->ne[0] != 4) { + return false; + } + if (comb && (comb->ne[0] != 4 || comb->ne[1] != 4)) { + return false; + } + + const int64_t x_tokens = x->ne[1] * x->ne[2] * x->ne[3]; + const int64_t r_tokens = residual->ne[2] * residual->ne[3]; + const int64_t p_tokens = post->ne[1] * post->ne[2] * post->ne[3]; + const int64_t d_tokens = dst->ne[2] * dst->ne[3]; + if (x_tokens != r_tokens || x_tokens != p_tokens || x_tokens != d_tokens) { + return false; + } + if (comb && (comb->ne[2] * comb->ne[3] != x_tokens)) { + return false; + } + + if (x->ne[2] == 1 && x->ne[3] == 1 && + residual->ne[3] == 1 && + post->ne[2] == 1 && post->ne[3] == 1 && + dst->ne[3] == 1 && + (!comb || comb->ne[3] == 1)) { + return false; + } + + if (x->ne[2] > 1 && x->nb[2] != x->ne[1] * x->nb[1]) return false; + if (x->ne[3] > 1 && x->nb[3] != x->ne[2] * x->nb[2]) return false; + + if (residual->ne[3] > 1 && residual->nb[3] != residual->ne[2] * residual->nb[2]) return false; + if (dst->ne[3] > 1 && dst->nb[3] != dst->ne[2] * dst->nb[2]) return false; + + if (post->ne[2] > 1 && post->nb[2] != post->ne[1] * post->nb[1]) return false; + if (post->ne[3] > 1 && post->nb[3] != post->ne[2] * post->nb[2]) return false; + + if (comb && comb->ne[3] > 1 && comb->nb[3] != comb->ne[2] * comb->nb[2]) return false; + + return true; +} + +static void htp_dsv4_hc_post_collapse_tensors( + const struct ggml_tensor * x, + const struct ggml_tensor * residual, + const struct ggml_tensor * post, + const struct ggml_tensor * comb, + const struct ggml_tensor * dst, + struct ggml_tensor * x_c, + struct ggml_tensor * r_c, + struct ggml_tensor * p_c, + struct ggml_tensor * c_c, + struct ggml_tensor * d_c +) { + const int64_t n_tokens = x->ne[1] * x->ne[2] * x->ne[3]; + + *x_c = *x; + x_c->ne[1] = n_tokens; + x_c->ne[2] = 1; + x_c->ne[3] = 1; + x_c->nb[2] = x_c->ne[1] * x_c->nb[1]; + x_c->nb[3] = x_c->nb[2]; + + *r_c = *residual; + r_c->ne[2] = n_tokens; + r_c->ne[3] = 1; + r_c->nb[3] = r_c->ne[2] * r_c->nb[2]; + + *d_c = *dst; + d_c->ne[2] = n_tokens; + d_c->ne[3] = 1; + d_c->nb[3] = d_c->ne[2] * d_c->nb[2]; + + *p_c = *post; + p_c->ne[1] = n_tokens; + p_c->ne[2] = 1; + p_c->ne[3] = 1; + p_c->nb[2] = p_c->ne[1] * p_c->nb[1]; + p_c->nb[3] = p_c->nb[2]; + + if (comb && c_c) { + *c_c = *comb; + c_c->ne[2] = n_tokens; + c_c->ne[3] = 1; + c_c->nb[3] = c_c->ne[2] * c_c->nb[2]; + } +} + +static bool htp_kparams_solve_dsv4_hc_post_impl( + const struct ggml_hexagon_session * sess, + const struct ggml_tensor * x, + const struct ggml_tensor * residual, + const struct ggml_tensor * post, + const struct ggml_tensor * comb, + const struct ggml_tensor * dst, + struct htp_dsv4_hc_post_kernel_params * kparams +) { + memset(kparams, 0, sizeof(*kparams)); + + if (x->type != GGML_TYPE_F32 || residual->type != GGML_TYPE_F32 || post->type != GGML_TYPE_F32 || dst->type != GGML_TYPE_F32) { + return false; + } + if (comb && comb->type != GGML_TYPE_F32) { + return false; + } + if (x->ne[2] != 1 || x->ne[3] != 1) { + return false; + } + if (x->ne[0] % HTP_DSV4_HC_CHUNK_ELEMS != 0) { + return false; + } + if (residual->ne[0] != x->ne[0] || residual->ne[1] != 4 || residual->ne[2] != x->ne[1] || residual->ne[3] != 1) { + return false; + } + if (post->ne[0] != 4 || post->ne[1] != x->ne[1] || post->ne[2] != 1 || post->ne[3] != 1) { + return false; + } + if (comb && (comb->ne[0] != 4 || comb->ne[1] != 4 || comb->ne[2] != x->ne[1] || comb->ne[3] != 1)) { + return false; + } + if (dst->ne[0] != x->ne[0] || dst->ne[1] != 4 || dst->ne[2] != x->ne[1] || dst->ne[3] != 1) { + return false; + } + if (x->nb[0] != sizeof(float) || residual->nb[0] != sizeof(float) || post->nb[0] != sizeof(float) || dst->nb[0] != sizeof(float)) { + return false; + } + if (comb && comb->nb[0] != sizeof(float)) { + return false; + } + if (ggml_is_permuted(x) || ggml_is_permuted(residual) || ggml_is_permuted(post) || ggml_is_permuted(dst) || (comb && ggml_is_permuted(comb))) { + return false; + } + + const uint32_t n_tokens = (uint32_t) x->ne[1]; + + kparams->has_comb = comb != nullptr; + kparams->kernel_type = kparams->has_comb ? HTP_DSV4_HC_POST_KERNEL_COMB : HTP_DSV4_HC_POST_KERNEL_IDENTITY; + kparams->n_chunks = ((uint32_t) x->ne[0] + HTP_DSV4_HC_CHUNK_ELEMS - 1) / HTP_DSV4_HC_CHUNK_ELEMS; + kparams->total_jobs = n_tokens * kparams->n_chunks; + if (kparams->total_jobs == 0) { + return true; + } + kparams->n_hvx = (std::min)(htp_session_n_hvx(sess), kparams->total_jobs); + kparams->jobs_per_hvx = (kparams->total_jobs + kparams->n_hvx - 1) / kparams->n_hvx; + + const size_t vtcm_budget = htp_session_vtcm_size(sess); + uint32_t block_chunks = (std::min)((uint32_t) HTP_DSV4_HC_MAX_CHUNKS, kparams->n_chunks); + while (block_chunks > 16) { + const uint32_t stride = block_chunks * 128; + const uint32_t buf_sz = HTP_DSV4_HC_POST_ROWS * stride; + if (2 * buf_sz * kparams->n_hvx <= vtcm_budget) { + break; + } + block_chunks /= 2; + } + + kparams->dma_block_chunks = block_chunks; + kparams->dma_row_stride = block_chunks * 128; + kparams->dma_buffer_size = HTP_DSV4_HC_POST_ROWS * kparams->dma_row_stride; + kparams->vtcm_size = 2 * kparams->dma_buffer_size * kparams->n_hvx; + if (kparams->vtcm_size > vtcm_budget) { + return false; + } + + kparams->div_n_chunks = init_fastdiv_values(kparams->n_chunks); + kparams->div_n_hvx = init_fastdiv_values(kparams->n_hvx); + + return true; +} + +static inline bool htp_kparams_solve_dsv4_hc_post( + const struct ggml_hexagon_session * sess, + const struct ggml_tensor * op, + struct htp_dsv4_hc_post_kernel_params * kparams +) { + if (htp_dsv4_hc_post_can_collapse(op->src[0], op->src[1], op->src[2], op->src[3], op)) { + struct ggml_tensor x_c, r_c, p_c, c_c, d_c; + htp_dsv4_hc_post_collapse_tensors(op->src[0], op->src[1], op->src[2], op->src[3], op, &x_c, &r_c, &p_c, op->src[3] ? &c_c : nullptr, &d_c); + return htp_kparams_solve_dsv4_hc_post_impl(sess, &x_c, &r_c, &p_c, op->src[3] ? &c_c : nullptr, &d_c, kparams); + } + return htp_kparams_solve_dsv4_hc_post_impl(sess, op->src[0], op->src[1], op->src[2], op->src[3], op, kparams); +} + +static bool htp_lightning_indexer_can_collapse( + const struct ggml_tensor * q, + const struct ggml_tensor * k, + const struct ggml_tensor * w, + const struct ggml_tensor * m, + const struct ggml_tensor * dst +) { + if (!q || !k || !w || !m || !dst) { + return false; + } + + // 2D K normalization: ne[1] is n_kv, ne[2] is 1 + const bool can_collapse_k = (k->ne[1] > 1 && k->ne[2] == 1); + + // Stream-to-token collapse across q, w, dst (and m) + bool can_collapse_stream = false; + if (q->ne[3] > 1 && k->ne[3] == 1) { + const bool q_ok = (q->nb[3] == (size_t) q->ne[2] * q->nb[2]); + const bool dst_ok = (dst->ne[3] == q->ne[3] && dst->ne[2] == 1 && dst->ne[1] == q->ne[2] && + dst->nb[3] == (size_t) dst->ne[1] * dst->nb[1]); + const bool w_ok = (w->ne[3] == q->ne[3] && w->ne[2] == 1 && w->ne[1] == q->ne[2] && + w->nb[3] == (size_t) w->ne[1] * w->nb[1]) || + (w->ne[3] == 1 && w->ne[1] == q->ne[2] * q->ne[3]); + const bool m_ok = (m->ne[3] == q->ne[3] && m->ne[2] == 1 && m->ne[1] == q->ne[2] && + m->nb[3] == (size_t) m->ne[1] * m->nb[1]) || + (m->ne[3] == 1 && m->ne[1] == q->ne[2] * q->ne[3]); + can_collapse_stream = q_ok && dst_ok && w_ok && m_ok; + } + + return can_collapse_k || can_collapse_stream; +} + +static void htp_lightning_indexer_collapse_tensors( + const struct ggml_tensor * q, + const struct ggml_tensor * k, + const struct ggml_tensor * w, + const struct ggml_tensor * m, + const struct ggml_tensor * dst, + struct ggml_tensor * q_c, + struct ggml_tensor * k_c, + struct ggml_tensor * w_c, + struct ggml_tensor * m_c, + struct ggml_tensor * dst_c +) { + *q_c = *q; + *k_c = *k; + *w_c = *w; + *m_c = *m; + *dst_c = *dst; + + if (k->ne[1] > 1 && k->ne[2] == 1) { + k_c->ne[2] = k->ne[1]; + k_c->ne[1] = 1; + k_c->nb[1] = k->nb[1]; + k_c->nb[2] = k->nb[1]; + k_c->nb[3] = k->nb[2]; + } + + if (q->ne[3] > 1 && k->ne[3] == 1) { + const int64_t n_tokens_collapsed = (int64_t) q->ne[2] * q->ne[3]; + + q_c->ne[2] = n_tokens_collapsed; + q_c->ne[3] = 1; + q_c->nb[3] = (size_t) q_c->ne[2] * q_c->nb[2]; + + if (w->ne[3] == q->ne[3]) { + w_c->ne[1] = n_tokens_collapsed; + w_c->ne[3] = 1; + w_c->nb[2] = (size_t) w_c->ne[1] * w_c->nb[1]; + w_c->nb[3] = w_c->nb[2]; + } + + dst_c->ne[1] = n_tokens_collapsed; + dst_c->ne[3] = 1; + dst_c->nb[2] = (size_t) dst_c->ne[1] * dst_c->nb[1]; + dst_c->nb[3] = dst_c->nb[2]; + + if (m->ne[3] == q->ne[3] && m->ne[1] == q->ne[2]) { + m_c->ne[1] = n_tokens_collapsed; + m_c->ne[3] = 1; + m_c->nb[2] = (size_t) m_c->ne[1] * m_c->nb[1]; + m_c->nb[3] = m_c->nb[2]; + } + } +} + +static bool htp_kparams_solve_lightning_indexer_impl( + const struct ggml_hexagon_session * sess, + const struct ggml_tensor * q, + const struct ggml_tensor * k, + const struct ggml_tensor * w, + const struct ggml_tensor * m, + const struct ggml_tensor * op, + struct htp_lid_kernel_params * kparams +) { + if (opt_li_select < 1) { + return false; + } + + if (k->type != GGML_TYPE_F32 && k->type != GGML_TYPE_F16 && k->type != GGML_TYPE_Q8_0 && k->type != GGML_TYPE_Q4_0) { + return false; + } + + if (q->ne[0] % 32 != 0 || + !ggml_is_contiguous_rows(q) || !ggml_is_contiguous_rows(k) || + !ggml_is_contiguous_rows(w) || !ggml_is_contiguous_rows(m) || + !ggml_is_contiguous_rows(op)) { + return false; + } + + const uint32_t n_kv_blocks = (k->ne[2] + HTP_LID_KV_BLOCK - 1) / HTP_LID_KV_BLOCK; + if (n_kv_blocks == 0 || q->ne[2] == 0) { + return false; + } + + const uint32_t D = q->ne[0]; + const uint32_t n_head = q->ne[1]; + const uint32_t n_tokens = q->ne[2]; + const size_t k_row_size = ggml_row_size(k->type, D); + + kparams->k_row_size = (uint32_t) k_row_size; + kparams->div_n_head = init_fastdiv_values(n_head); + kparams->div_m_ne3 = init_fastdiv_values((uint32_t) m->ne[3]); + + if (opt_li_select >= 2 && htp_session_n_hmx(sess) > 0 && D % HMX_FP16_TILE_N_COLS == 0) { + const uint32_t n_hvx = htp_session_n_hvx(sess); + const uint32_t n_stream = q->ne[3]; + const uint32_t n_dot = D / HMX_FP16_TILE_N_COLS; + + auto vtcm_usage = [&](uint32_t n_tok, uint32_t kv_block, uint32_t n_qsets) { + return htp_lid_hmx_compute_vtcm_usage(D, n_head, n_tok, kv_block, n_qsets, n_hvx, k_row_size); + }; + + uint32_t n_tok = (std::min)(n_tokens, (uint32_t) HTP_LID_HMX_TOKENS_MAX); + while (n_tok > 32 && vtcm_usage(n_tok, 64, 1) > htp_session_vtcm_size(sess)) { + n_tok /= 2; + } + + uint32_t n_qsets = n_tok == n_tokens ? (std::min)(n_stream, (uint32_t) HTP_LID_HMX_MAX_QSETS) : 1; + while (n_qsets > 1 && vtcm_usage(n_tok, 64, n_qsets) > htp_session_vtcm_size(sess)) { + n_qsets /= 2; + } + + uint32_t n_qbuf = 1; + if (n_qsets == 1 && n_head >= 16 && n_stream * ((n_tokens + n_tok - 1) / n_tok) > 1) { + uint32_t nt2 = n_tok; + while (nt2 > 32 && vtcm_usage(nt2, 64, 2) > htp_session_vtcm_size(sess)) { + nt2 /= 2; + } + const size_t q_bytes = (size_t) nt2 * n_head * D * sizeof(float); + const size_t k_bytes = (size_t) k->ne[2] * ggml_row_size(k->type, D); + if (vtcm_usage(nt2, 64, 2) <= htp_session_vtcm_size(sess) && q_bytes >= k_bytes) { + n_tok = nt2; + n_qbuf = 2; + } + } + + const uint32_t n_ch = (n_tok * n_head + 31) / 32; + const uint32_t kv_seg = hex_round_up(k->ne[2], 64); + const uint32_t n_rows = n_qsets * kv_seg; + const uint32_t bc_max = n_rows <= 512 ? n_rows : (std::max)(256u, n_rows / 4); + uint32_t kv_block = 64; + while (kv_block < HTP_LID_HMX_KV_BLOCK_MAX && kv_block < bc_max && (kv_block / 32) * n_ch * (n_dot + 1) < 256 && + vtcm_usage(n_tok, kv_block * 2, n_qsets * n_qbuf) <= htp_session_vtcm_size(sess)) { + kv_block *= 2; + } + + const size_t vtcm_size = vtcm_usage(n_tok, kv_block, n_qsets * n_qbuf); + if (vtcm_size <= htp_session_vtcm_size(sess)) { + const uint32_t n_tt = (n_tokens + n_tok - 1) / n_tok; + kparams->kernel_type = HTP_LID_KERNEL_HMX; + kparams->n_hvx = n_hvx; + kparams->n_tokens_tile = n_tok; + kparams->n_kv_blocks = (n_rows + kv_block - 1) / kv_block; + kparams->blocks_per_hvx = 0; + kparams->kv_block = kv_block; + kparams->n_qsets = n_qsets; + kparams->n_qbuf = n_qbuf; + kparams->kv_seg = kv_seg; + kparams->n_tt = n_tt; + kparams->vtcm_size = (uint32_t) vtcm_size; + kparams->div_kv_seg = init_fastdiv_values(kv_seg); + kparams->div_n_tt = init_fastdiv_values(n_tt); + kparams->div_n_ch = init_fastdiv_values(n_ch); + return true; + } + } + + const uint32_t n_hvx = (std::min)(htp_session_n_hvx(sess), n_kv_blocks); + uint32_t n_tok = (std::min)((uint32_t) q->ne[2], (uint32_t) HTP_LID_TOKENS_MAX); + while (n_tok > 1 && htp_lid_compute_vtcm_usage(D, n_head, n_tok, n_hvx, k_row_size) > htp_session_vtcm_size(sess)) { + n_tok /= 2; + } + + const size_t vtcm_size = htp_lid_compute_vtcm_usage(D, n_head, n_tok, n_hvx, k_row_size); + if (vtcm_size > htp_session_vtcm_size(sess)) { + return false; + } + + kparams->kernel_type = HTP_LID_KERNEL_HVX; + kparams->n_hvx = n_hvx; + kparams->n_tokens_tile = n_tok; + kparams->n_kv_blocks = n_kv_blocks; + kparams->blocks_per_hvx = (n_kv_blocks + n_hvx - 1) / n_hvx; + kparams->kv_block = HTP_LID_KV_BLOCK; + kparams->n_qsets = 1; + kparams->n_qbuf = 1; + kparams->kv_seg = 0; + kparams->n_tt = 0; + kparams->vtcm_size = (uint32_t) vtcm_size; + + return true; +} + +static bool htp_kparams_solve_lightning_indexer( + const struct ggml_hexagon_session * sess, + const struct ggml_tensor * op, + struct htp_lid_kernel_params * kparams +) { + if (htp_lightning_indexer_can_collapse(op->src[0], op->src[1], op->src[2], op->src[3], op)) { + struct ggml_tensor q_c, k_c, w_c, m_c, d_c; + htp_lightning_indexer_collapse_tensors(op->src[0], op->src[1], op->src[2], op->src[3], op, + &q_c, &k_c, &w_c, &m_c, &d_c); + return htp_kparams_solve_lightning_indexer_impl(sess, &q_c, &k_c, &w_c, &m_c, &d_c, kparams); + } + return htp_kparams_solve_lightning_indexer_impl(sess, op->src[0], op->src[1], op->src[2], op->src[3], op, kparams); +} template static inline bool htp_kparams_invoke( const struct ggml_hexagon_session * sess, @@ -4026,7 +5191,7 @@ static inline bool htp_kparams_invoke( } void htp_opnode_collapse(const struct ggml_hexagon_session * sess, htp_opnode & node) { - const uint32_t n_threads = htp_session_n_threads(sess); + const uint32_t n_hvx = htp_session_n_hvx(sess); if (htp_op_is_unary(node.opcode) && htp_unary_is_elementwise(node.opcode)) { if (node.inputs.empty() || !node.inputs[0] || node.outputs.empty() || !node.outputs[0]) { @@ -4039,7 +5204,7 @@ void htp_opnode_collapse(const struct ggml_hexagon_session * sess, htp_opnode & struct ggml_tensor src0_c = *node.inputs[0]; struct ggml_tensor dst_c = *node.outputs[0]; struct ggml_tensor * tensors[2] = { &src0_c, &dst_c }; - htp_collapse_dims(tensors, 2, elem_size, n_threads); + htp_collapse_dims(tensors, 2, elem_size, n_hvx); if (src0_c.ne[0] != node.inputs[0]->ne[0] || src0_c.ne[1] != node.inputs[0]->ne[1]) { node.inputs[0] = node.add_dummy(src0_c); node.outputs[0] = node.add_dummy(dst_c); @@ -4060,7 +5225,7 @@ void htp_opnode_collapse(const struct ggml_hexagon_session * sess, htp_opnode & struct ggml_tensor src1_c = *node.inputs[1]; struct ggml_tensor dst_c = *node.outputs[0]; struct ggml_tensor * tensors[3] = { &src0_c, &src1_c, &dst_c }; - htp_collapse_dims(tensors, 3, elem_size, n_threads); + htp_collapse_dims(tensors, 3, elem_size, n_hvx); if (src0_c.ne[0] != node.inputs[0]->ne[0] || src0_c.ne[1] != node.inputs[0]->ne[1]) { node.inputs[0] = node.add_dummy(src0_c); node.inputs[1] = node.add_dummy(src1_c); @@ -4125,6 +5290,70 @@ void htp_opnode_collapse(const struct ggml_hexagon_session * sess, htp_opnode & } return; } + + if (node.opcode == HTP_OP_DSV4_HC_PRE) { + if (node.inputs.size() < 2 || !node.inputs[0] || !node.inputs[1] || node.outputs.empty() || !node.outputs[0]) { + return; + } + const struct ggml_tensor * x = node.inputs[0]; + const struct ggml_tensor * weights = node.inputs[1]; + const struct ggml_tensor * dst = node.outputs[0]; + + if (htp_dsv4_hc_pre_can_collapse(x, weights, dst)) { + struct ggml_tensor x_c, w_c, d_c; + htp_dsv4_hc_pre_collapse_tensors(x, weights, dst, &x_c, &w_c, &d_c); + node.inputs[0] = node.add_dummy(x_c); + node.inputs[1] = node.add_dummy(w_c); + node.outputs[0] = node.add_dummy(d_c); + } + return; + } + + if (node.opcode == HTP_OP_DSV4_HC_POST) { + if (node.inputs.size() < 3 || !node.inputs[0] || !node.inputs[1] || !node.inputs[2] || node.outputs.empty() || !node.outputs[0]) { + return; + } + const struct ggml_tensor * x = node.inputs[0]; + const struct ggml_tensor * residual = node.inputs[1]; + const struct ggml_tensor * post = node.inputs[2]; + const struct ggml_tensor * comb = node.inputs.size() > 3 ? node.inputs[3] : nullptr; + const struct ggml_tensor * dst = node.outputs[0]; + + if (htp_dsv4_hc_post_can_collapse(x, residual, post, comb, dst)) { + struct ggml_tensor x_c, r_c, p_c, c_c, d_c; + htp_dsv4_hc_post_collapse_tensors(x, residual, post, comb, dst, &x_c, &r_c, &p_c, comb ? &c_c : nullptr, &d_c); + node.inputs[0] = node.add_dummy(x_c); + node.inputs[1] = node.add_dummy(r_c); + node.inputs[2] = node.add_dummy(p_c); + if (comb) { + node.inputs[3] = node.add_dummy(c_c); + } + node.outputs[0] = node.add_dummy(d_c); + } + return; + } + + if (node.opcode == HTP_OP_LIGHTNING_INDEXER) { + if (node.inputs.size() < 4 || !node.inputs[0] || !node.inputs[1] || !node.inputs[2] || !node.inputs[3] || node.outputs.empty() || !node.outputs[0]) { + return; + } + const struct ggml_tensor * q = node.inputs[0]; + const struct ggml_tensor * k = node.inputs[1]; + const struct ggml_tensor * w = node.inputs[2]; + const struct ggml_tensor * m = node.inputs[3]; + const struct ggml_tensor * dst = node.outputs[0]; + + if (htp_lightning_indexer_can_collapse(q, k, w, m, dst)) { + struct ggml_tensor q_c, k_c, w_c, m_c, d_c; + htp_lightning_indexer_collapse_tensors(q, k, w, m, dst, &q_c, &k_c, &w_c, &m_c, &d_c); + node.inputs[0] = node.add_dummy(q_c); + node.inputs[1] = node.add_dummy(k_c); + node.inputs[2] = node.add_dummy(w_c); + node.inputs[3] = node.add_dummy(m_c); + node.outputs[0] = node.add_dummy(d_c); + } + return; + } } bool htp_kparams_solve( @@ -4157,6 +5386,19 @@ bool htp_kparams_solve( if (node.opcode == HTP_OP_UPSCALE) { return htp_kparams_solve_upscale(sess, src0, dst, node.kparams()); } + if (node.opcode == HTP_OP_DSV4_HC_PRE) { + return htp_kparams_solve_dsv4_hc_pre_impl(sess, src0, src1, dst, node.kparams()); + } + if (node.opcode == HTP_OP_DSV4_HC_POST) { + const ggml_tensor * src2 = node.inputs.size() > 2 ? node.inputs[2] : nullptr; + const ggml_tensor * src3 = node.inputs.size() > 3 ? node.inputs[3] : nullptr; + return htp_kparams_solve_dsv4_hc_post_impl(sess, src0, src1, src2, src3, dst, node.kparams()); + } + if (node.opcode == HTP_OP_LIGHTNING_INDEXER) { + const ggml_tensor * src2 = node.inputs.size() > 2 ? node.inputs[2] : nullptr; + const ggml_tensor * src3 = node.inputs.size() > 3 ? node.inputs[3] : nullptr; + return htp_kparams_solve_lightning_indexer_impl(sess, src0, src1, src2, src3, dst, node.kparams()); + } return htp_kparams_solve(sess, node.node, node.kernel_params); } @@ -4238,6 +5480,9 @@ bool htp_kparams_solve( case HTP_OP_CONV_2D_DW: return htp_kparams_invoke(sess, op, kparams); + case HTP_OP_CONV_3D: + return htp_kparams_invoke(sess, op, kparams); + case HTP_OP_SUM_ROWS: case HTP_OP_SUM: case HTP_OP_ARGMAX: @@ -4252,18 +5497,30 @@ bool htp_kparams_solve( case HTP_OP_CUMSUM: return htp_kparams_invoke(sess, op, kparams); + case HTP_OP_TOP_K_MASK: + return htp_kparams_invoke(sess, op, kparams); + case HTP_OP_FILL: return htp_kparams_invoke(sess, op, kparams); case HTP_OP_ROLL: return htp_kparams_invoke(sess, op, kparams); + case HTP_OP_LIGHTNING_INDEXER: + return htp_kparams_invoke(sess, op, kparams); + case HTP_OP_DIAG: return htp_kparams_invoke(sess, op, kparams); case HTP_OP_SOLVE_TRI: return htp_kparams_invoke(sess, op, kparams); + case HTP_OP_DSV4_HC_PRE: + return htp_kparams_invoke(sess, op, kparams); + + case HTP_OP_DSV4_HC_POST: + return htp_kparams_invoke(sess, op, kparams); + default: if (htp_op_is_unary(opcode)) { return htp_kparams_invoke(sess, op, kparams); diff --git a/ggml/src/ggml-hexagon/htp-kparams.h b/ggml/src/ggml-hexagon/htp-kparams.h index b757f430b83c..653554be4572 100644 --- a/ggml/src/ggml-hexagon/htp-kparams.h +++ b/ggml/src/ggml-hexagon/htp-kparams.h @@ -11,12 +11,14 @@ #include "htp/cpy-ops.h" #include "htp/cumsum-ops.h" #include "htp/diag-ops.h" +#include "htp/dsv4-hc-ops.h" #include "htp/fill-ops.h" #include "htp/flash-attn-ops.h" #include "htp/gated-delta-net-ops.h" #include "htp/get-rows-ops.h" #include "htp/glu-ops.h" #include "htp/im2col-ops.h" +#include "htp/lightning-indexer-ops.h" #include "htp/matmul-ops.h" #include "htp/pad-ops.h" #include "htp/pool-ops.h" @@ -28,6 +30,7 @@ #include "htp/solve-tri-ops.h" #include "htp/ssm-conv.h" #include "htp/sum-rows-ops.h" +#include "htp/top-k-mask-ops.h" #include "htp/unary-ops.h" #include "htp/upscale-ops.h" @@ -92,6 +95,14 @@ bool htp_kparams_solve_rms_norm_mul( struct htp_unary_kernel_params * kparams ); +bool htp_kparams_solve_rms_norm_mul_add( + const struct ggml_hexagon_session * sess, + const struct ggml_tensor * src0, + const struct ggml_tensor * src1, + const struct ggml_tensor * dst, + struct htp_unary_kernel_params * kparams +); + bool htp_kparams_solve_matmul_add( const struct ggml_hexagon_session * sess, const struct ggml_tensor * src0, diff --git a/ggml/src/ggml-hexagon/htp-opnode.cpp b/ggml/src/ggml-hexagon/htp-opnode.cpp index d4e5150c0f28..8dffedc4bb6c 100644 --- a/ggml/src/ggml-hexagon/htp-opnode.cpp +++ b/ggml/src/ggml-hexagon/htp-opnode.cpp @@ -5,11 +5,13 @@ #include "htp/binary-ops.h" #include "htp/cumsum-ops.h" #include "htp/diag-ops.h" +#include "htp/dsv4-hc-ops.h" #include "htp/fill-ops.h" #include "htp/flash-attn-ops.h" #include "htp/gated-delta-net-ops.h" #include "htp/get-rows-ops.h" #include "htp/glu-ops.h" +#include "htp/lightning-indexer-ops.h" #include "htp/matmul-ops.h" #include "htp/pad-ops.h" #include "htp/repeat-ops.h" @@ -18,6 +20,7 @@ #include "htp/solve-tri-ops.h" #include "htp/ssm-conv.h" #include "htp/sum-rows-ops.h" +#include "htp/top-k-mask-ops.h" #include "htp/unary-ops.h" #include "htp/upscale-ops.h" @@ -154,7 +157,7 @@ static std::string format_kernel_params(const htp_opnode & node) { } else if (type == HTP_MM_KERNEL_HVX_QUANT_ROW || type == HTP_MM_KERNEL_HVX_QUANT_BLOCK) { path = "hvx-tiled"; } - snprintf(str, max_size, "%s vtcm %d", path, (int) kparams->vtcm_size); + snprintf(str, max_size, "%s nhvx %u vtcm %u", path, (unsigned int) kparams->n_hvx, (unsigned int) kparams->vtcm_size); } else if (node.opcode == HTP_OP_FLASH_ATTN_EXT) { const auto * kparams = node.kparams(); const char * path = "unknown"; @@ -164,10 +167,11 @@ static std::string format_kernel_params(const htp_opnode & node) { } else if (type == HTP_FA_KERNEL_HVX) { path = "hvx"; } - snprintf(str, max_size, "%s vtcm %d", path, (int) kparams->vtcm_size); + snprintf(str, max_size, "%s nhvx %u vtcm %u", path, (unsigned int) kparams->n_hvx, (unsigned int) kparams->vtcm_size); } else if (htp_op_is_unary(node.opcode)) { const auto * kparams = node.kparams(); - snprintf(str, max_size, "%s vtcm %d", kparams->col_tile ? "wide-row" : "row-block", (int) kparams->vtcm_size); + snprintf(str, max_size, "%s nhvx %u vtcm %u", kparams->col_tile ? "wide-row" : "row-block", + (unsigned int) kparams->n_hvx, (unsigned int) kparams->vtcm_size); } else if (node.opcode == HTP_OP_MDEV_GROUP && node.node) { snprintf(str, max_size, "idx %d count %d", (int) node.node->op_params[0], (int) node.dst()->ne[1]); } else if ((node.opcode == HTP_OP_FENCE || node.opcode == HTP_OP_CPY_FENCE) && node.node) { @@ -176,25 +180,43 @@ static std::string format_kernel_params(const htp_opnode & node) { snprintf(str, max_size, "seq 0x%x -> 0x%x", (uint32_t) node.node->op_params[0], (uint32_t) node.node->op_params[1]); } else if (node.opcode == HTP_OP_SSM_CONV) { const auto * kparams = node.kparams(); - snprintf(str, max_size, "%s vtcm %d", kparams->n_t == 1 ? "decode" : "prefill", (int) kparams->vtcm_size); + snprintf(str, max_size, "%s nhvx %u vtcm %u", kparams->n_t == 1 ? "decode" : "prefill", + (unsigned int) kparams->n_hvx, (unsigned int) kparams->vtcm_size); } else if (node.opcode == HTP_OP_SOFTMAX) { const auto * kparams = node.kparams(); - snprintf(str, max_size, "k%d nth %d vtcm %d", (int) kparams->kernel_type, (int) kparams->n_threads, (int) kparams->vtcm_size); + snprintf(str, max_size, "k%d nhvx %u vtcm %u", (int) kparams->kernel_type, + (unsigned int) kparams->n_hvx, (unsigned int) kparams->vtcm_size); } else if (node.opcode == HTP_OP_GATED_DELTA_NET) { const auto * kparams = node.kparams(); const char * path = (kparams->kernel_type == HTP_GDN_KERNEL_HMX_CHUNKED) ? "hmx-chunked" : "hvx-recurrent"; - snprintf(str, max_size, "%s-%s vtcm %u", path, kparams->kda ? "kda" : "scalar", - (unsigned int) kparams->vtcm_size); + snprintf(str, max_size, "%s-%s nhvx %u vtcm %u", path, kparams->kda ? "kda" : "scalar", + (unsigned int) kparams->n_hvx, (unsigned int) kparams->vtcm_size); + } else if (node.opcode == HTP_OP_DSV4_HC_PRE) { + const auto * kparams = node.kparams(); + const char * ktype_str = kparams->kernel_type == HTP_DSV4_HC_PRE_KERNEL_GATED ? "gated" : "plain"; + snprintf(str, max_size, "%s nhvx %u vtcm %u", ktype_str, + (unsigned int) kparams->n_hvx, (unsigned int) kparams->vtcm_size); + } else if (node.opcode == HTP_OP_DSV4_HC_POST) { + const auto * kparams = node.kparams(); + const char * ktype_str = kparams->kernel_type == HTP_DSV4_HC_POST_KERNEL_COMB ? "comb" : "identity"; + snprintf(str, max_size, "%s nhvx %u vtcm %u", ktype_str, + (unsigned int) kparams->n_hvx, (unsigned int) kparams->vtcm_size); + } else if (node.opcode == HTP_OP_LIGHTNING_INDEXER) { + const auto * kparams = node.kparams(); + snprintf(str, max_size, "%s nhvx %u tok %u bc %u vtcm %u", + kparams->kernel_type == HTP_LID_KERNEL_HMX ? "hmx" : "hvx", + (unsigned int) kparams->n_hvx, (unsigned int) kparams->n_tokens_tile, + (unsigned int) kparams->kv_block, (unsigned int) kparams->vtcm_size); } else if (node.opcode == HTP_OP_MUL || node.opcode == HTP_OP_ADD || node.opcode == HTP_OP_ADD_ID || node.opcode == HTP_OP_SUB || node.opcode == HTP_OP_DIV) { const auto * kparams = node.kparams(); - snprintf(str, max_size, "vtcm %u", (unsigned int) kparams->vtcm_size); + snprintf(str, max_size, "nhvx %u vtcm %u", (unsigned int) kparams->n_hvx, (unsigned int) kparams->vtcm_size); } else if (node.opcode == HTP_OP_ARGSORT || node.opcode == HTP_OP_TOP_K) { const auto * kparams = node.kparams(); - snprintf(str, max_size, "%s nth %d nchk %d chk %d vtcm %d", + snprintf(str, max_size, "%s nhvx %u nchk %u chk %u vtcm %u", node.opcode == HTP_OP_TOP_K ? "top_k" : "argsort", - (int) kparams->n_threads, (int) kparams->n_chunks, - (int) kparams->chunk_elems, (int) kparams->vtcm_size); + (unsigned int) kparams->n_hvx, (unsigned int) kparams->n_chunks, + (unsigned int) kparams->chunk_elems, (unsigned int) kparams->vtcm_size); } else if (node.opcode == HTP_OP_GET_ROWS) { const auto * kparams = node.kparams(); const char * ktype_str = "unknown"; @@ -203,28 +225,36 @@ static std::string format_kernel_params(const htp_opnode & node) { case HTP_GET_ROWS_KERNEL_TILED: ktype_str = "tiled"; break; case HTP_GET_ROWS_KERNEL_FLAT: ktype_str = "flat"; break; } - snprintf(str, max_size, "%s%s vtcm %u", ktype_str, kparams->n_threads > 1 ? "-multi" : "", (unsigned int) kparams->vtcm_size); + snprintf(str, max_size, "%s nhvx %u vtcm %u", ktype_str, (unsigned int) kparams->n_hvx, (unsigned int) kparams->vtcm_size); } else if (htp_op_is_glu(node.opcode)) { const auto * kparams = node.kparams(); - snprintf(str, max_size, "nth %d vtcm %u", (int) kparams->n_threads, (unsigned int) kparams->vtcm_size); + snprintf(str, max_size, "nhvx %u vtcm %u", (unsigned int) kparams->n_hvx, (unsigned int) kparams->vtcm_size); } else if (node.opcode == HTP_OP_PAD) { const auto * kparams = node.kparams(); - snprintf(str, max_size, "k%d nth %d vtcm %u", (int) kparams->kernel_type, (int) kparams->n_threads, (unsigned int) kparams->vtcm_size); + snprintf(str, max_size, "k%d nhvx %u vtcm %u", (int) kparams->kernel_type, (unsigned int) kparams->n_hvx, (unsigned int) kparams->vtcm_size); } else if (node.opcode == HTP_OP_CUMSUM) { const auto * kparams = node.kparams(); - snprintf(str, max_size, "%s nth %d vtcm %u", kparams->use_dma ? "dma" : "hvx", (int) kparams->n_threads, (unsigned int) kparams->vtcm_size); + snprintf(str, max_size, "%s nhvx %u vtcm %u", kparams->use_dma ? "dma" : "hvx", (unsigned int) kparams->n_hvx, (unsigned int) kparams->vtcm_size); + } else if (node.opcode == HTP_OP_TOP_K_MASK) { + const auto * kparams = node.kparams(); + const char * ktype_str = "?"; + switch (kparams->kernel_type) { + case HTP_TOP_K_MASK_KERNEL_ROWS: ktype_str = "rows"; break; + case HTP_TOP_K_MASK_KERNEL_PICKS: ktype_str = "picks"; break; + } + snprintf(str, max_size, "%s nhvx %u tile %u x%u vtcm %u", ktype_str, (unsigned int) kparams->n_hvx, (unsigned int) kparams->tile_cols, (unsigned int) kparams->n_col_tiles, (unsigned int) kparams->vtcm_size); } else if (node.opcode == HTP_OP_FILL) { const auto * kparams = node.kparams(); - snprintf(str, max_size, "%s nth %d", kparams->kernel_type == HTP_FILL_KERNEL_CONTIGUOUS ? "contig" : "strided", (int) kparams->n_threads); + snprintf(str, max_size, "%s nhvx %u", kparams->kernel_type == HTP_FILL_KERNEL_CONTIGUOUS ? "contig" : "strided", (unsigned int) kparams->n_hvx); } else if (node.opcode == HTP_OP_ROLL) { const auto * kparams = node.kparams(); - snprintf(str, max_size, "k%d nth %d", (int) kparams->kernel_type, (int) kparams->n_threads); + snprintf(str, max_size, "k%d nhvx %u", (int) kparams->kernel_type, (unsigned int) kparams->n_hvx); } else if (node.opcode == HTP_OP_DIAG) { const auto * kparams = node.kparams(); - snprintf(str, max_size, "%s nth %d vtcm %u", kparams->use_dma ? "dma" : "hvx", (int) kparams->n_threads, (unsigned int) kparams->vtcm_size); + snprintf(str, max_size, "%s nhvx %u vtcm %u", kparams->use_dma ? "dma" : "hvx", (unsigned int) kparams->n_hvx, (unsigned int) kparams->vtcm_size); } else if (node.opcode == HTP_OP_SOLVE_TRI) { const auto * kparams = node.kparams(); - snprintf(str, max_size, "%s nth %d", kparams->batched ? "batched" : "chunked", (int) kparams->n_threads); + snprintf(str, max_size, "%s nhvx %u", kparams->batched ? "batched" : "chunked", (unsigned int) kparams->n_hvx); } else if (node.opcode == HTP_OP_UPSCALE) { const auto * kparams = node.kparams(); const char * ktype_str = "unknown"; @@ -237,7 +267,7 @@ static std::string format_kernel_params(const htp_opnode & node) { case HTP_UPSCALE_KERNEL_BILINEAR_AA: ktype_str = "bilinear-aa"; break; case HTP_UPSCALE_KERNEL_BICUBIC: ktype_str = "bicubic"; break; } - snprintf(str, max_size, "%s nth %d chk %d vtcm %u", ktype_str, (int) kparams->n_threads, (int) kparams->dst_nrows_per_chunk, (unsigned int) kparams->vtcm_size); + snprintf(str, max_size, "%s nhvx %u chk %u vtcm %u", ktype_str, (unsigned int) kparams->n_hvx, (unsigned int) kparams->dst_nrows_per_chunk, (unsigned int) kparams->vtcm_size); } else { snprintf(str, max_size, "----"); } diff --git a/ggml/src/ggml-hexagon/htp-repack.cpp b/ggml/src/ggml-hexagon/htp-repack.cpp index d2e3df2cf4b4..b5413791c183 100644 --- a/ggml/src/ggml-hexagon/htp-repack.cpp +++ b/ggml/src/ggml-hexagon/htp-repack.cpp @@ -15,7 +15,7 @@ # include #endif -// --- 4-bit block-32 nibble helpers (shared by Q4_0, IQ4_NL, Q4_1, MXFP4) --- +// --- 4-bit block-32 nibble helpers (shared by Q4_0, IQ4_NL, Q4_1, Q5_0, Q5_1, MXFP4) --- static inline void unpack_nibbles_32(uint8_t * dst, const uint8_t * qs) { for (unsigned int i = 0; i < 16; ++i) { @@ -48,6 +48,28 @@ static inline void transpose_tile_to_nibbles(uint8_t tile_quants[32][32], const } } +// 5th-bit plane of one row (qh bit k = quant k), same bit layout as the Q5_K plane +static inline void pack_q5_plane(uint8_t * plane, int row, uint32_t qh) { + for (int c = 0; c < 4; c++) { + uint32_t v = 0; + for (int i = 0; i < 4; i++) { + v |= ((qh >> (8 * i + 2 * c)) & 3) << (2 * i); + } + plane[c * 32 + row] = (uint8_t) v; + } +} + +static inline uint32_t unpack_q5_plane(const uint8_t * plane, int row) { + uint32_t qh = 0; + for (int c = 0; c < 4; c++) { + const uint32_t v = plane[c * 32 + row]; + for (int i = 0; i < 4; i++) { + qh |= ((v >> (2 * i)) & 3) << (8 * i + 2 * c); + } + } + return qh; +} + #if defined(__ARM_NEON) || defined(__aarch64__) static void repack_neon_transpose8x8(const uint8x8_t rows[8], uint8x8_t columns[8]) { const uint8x8x2_t t0 = vtrn_u8(rows[0], rows[1]); @@ -433,7 +455,7 @@ static void unpack_from_tiled_grid_32(void * data, const ggml_tensor * t, size_t } } -// --- Block-32 Quantized Types (Q4_0, IQ4_NL, Q4_1, Q8_0, MXFP4) --- +// --- Block-32 Quantized Types (Q4_0, IQ4_NL, Q4_1, Q5_0, Q5_1, Q8_0, MXFP4) --- static void repack_q4_0_tiled(ggml_tensor * t, const void * data, size_t offset, size_t size) { repack_to_tiled_grid_32(t, data, offset, size, HTP_TENSOR_TILE_SIZE_Q4_0, @@ -546,6 +568,141 @@ static void repack_tiled_q4_1(void * data, const ggml_tensor * t, size_t offset, }); } +static void repack_q5_0_tiled(ggml_tensor * t, const void * data, size_t offset, size_t size) { + repack_to_tiled_grid_32(t, data, offset, size, HTP_TENSOR_TILE_SIZE_Q5_0, + [](uint8_t * tile_dst, const block_q5_0 * src_slice, int64_t ne0, int64_t ne1, int64_t row_stride, int ct, int kt) { + uint8_t * plane_dst = tile_dst + 512; + ggml_half * scale_dst = (ggml_half *) (tile_dst + 640); +#if defined(__ARM_NEON) || defined(__aarch64__) + if (ct * 32 + 32 <= ne1 && kt < ne0 / 32) { + for (int row0 = 0; row0 < 32; row0 += 8) { + const uint8_t * qs[8]; + for (int rr = 0; rr < 8; ++rr) { + qs[rr] = src_slice[(ct * 32 + row0 + rr) * row_stride + kt].qs; + } + repack_neon_4bit_rows(qs, tile_dst, row0); + } + + for (int row = 0; row < 32; ++row) { + const block_q5_0 & src_block = src_slice[(ct * 32 + row) * row_stride + kt]; + uint32_t qh; + memcpy(&qh, src_block.qh, sizeof(qh)); + pack_q5_plane(plane_dst, row, ~qh); + scale_dst[row] = src_block.d; + } + return; + } +#endif + uint8_t tile_quants[32][32]; + for (int row = 0; row < 32; row++) { + const int64_t r = ct * 32 + row; + if (r < ne1 && kt < ne0 / 32) { + const block_q5_0 & b = src_slice[r * row_stride + kt]; + unpack_nibbles_32(tile_quants[row], b.qs); + uint32_t qh; + memcpy(&qh, b.qh, sizeof(qh)); + pack_q5_plane(plane_dst, row, ~qh); + scale_dst[row] = b.d; + } else { + memset(tile_quants[row], 0, 32); + pack_q5_plane(plane_dst, row, 0); + scale_dst[row] = 0; + } + } + transpose_nibbles_to_tile(tile_dst, tile_quants); + }); +} + +static void repack_tiled_q5_0(void * data, const ggml_tensor * t, size_t offset, size_t size) { + unpack_from_tiled_grid_32(data, t, offset, size, HTP_TENSOR_TILE_SIZE_Q5_0, + [](block_q5_0 * dst_slice, const uint8_t * tile_src, int64_t ne0, int64_t start_row, int64_t end_row, int64_t row_stride, int ct, int kt) { + uint8_t tile_quants[32][32]; + transpose_tile_to_nibbles(tile_quants, tile_src); + const uint8_t * plane_src = tile_src + 512; + const ggml_half * scale_src = (const ggml_half *) (tile_src + 640); + for (int row = 0; row < 32; row++) { + const int64_t r = ct * 32 + row; + if (r >= start_row && r < end_row && kt < ne0 / 32) { + block_q5_0 & b = dst_slice[(r - start_row) * row_stride + kt]; + pack_nibbles_32(b.qs, tile_quants[row]); + const uint32_t qh = ~unpack_q5_plane(plane_src, row); + memcpy(b.qh, &qh, sizeof(qh)); + b.d = scale_src[row]; + } + } + }); +} + +// Q5_1 uses the Q5_K tile layout +static void repack_q5_1_tiled(ggml_tensor * t, const void * data, size_t offset, size_t size) { + repack_to_tiled_grid_32(t, data, offset, size, HTP_TENSOR_TILE_SIZE_Q5_1, + [](uint8_t * tile_dst, const block_q5_1 * src_slice, int64_t ne0, int64_t ne1, int64_t row_stride, int ct, int kt) { + ggml_half * scale_dst = (ggml_half *) (tile_dst + 512); + uint8_t * plane_dst = tile_dst + 640; +#if defined(__ARM_NEON) || defined(__aarch64__) + if (ct * 32 + 32 <= ne1 && kt < ne0 / 32) { + for (int row0 = 0; row0 < 32; row0 += 8) { + const uint8_t * qs[8]; + for (int rr = 0; rr < 8; ++rr) { + qs[rr] = src_slice[(ct * 32 + row0 + rr) * row_stride + kt].qs; + } + repack_neon_4bit_rows(qs, tile_dst, row0); + } + + for (int row = 0; row < 32; ++row) { + const block_q5_1 & src_block = src_slice[(ct * 32 + row) * row_stride + kt]; + uint32_t qh; + memcpy(&qh, src_block.qh, sizeof(qh)); + pack_q5_plane(plane_dst, row, qh); + scale_dst[2 * row + 0] = src_block.d; + scale_dst[2 * row + 1] = src_block.m; + } + return; + } +#endif + uint8_t tile_quants[32][32]; + for (int row = 0; row < 32; row++) { + const int64_t r = ct * 32 + row; + if (r < ne1 && kt < ne0 / 32) { + const block_q5_1 & b = src_slice[r * row_stride + kt]; + unpack_nibbles_32(tile_quants[row], b.qs); + uint32_t qh; + memcpy(&qh, b.qh, sizeof(qh)); + pack_q5_plane(plane_dst, row, qh); + scale_dst[2 * row + 0] = b.d; + scale_dst[2 * row + 1] = b.m; + } else { + memset(tile_quants[row], 0, 32); + pack_q5_plane(plane_dst, row, 0); + scale_dst[2 * row + 0] = 0; + scale_dst[2 * row + 1] = 0; + } + } + transpose_nibbles_to_tile(tile_dst, tile_quants); + }); +} + +static void repack_tiled_q5_1(void * data, const ggml_tensor * t, size_t offset, size_t size) { + unpack_from_tiled_grid_32(data, t, offset, size, HTP_TENSOR_TILE_SIZE_Q5_1, + [](block_q5_1 * dst_slice, const uint8_t * tile_src, int64_t ne0, int64_t start_row, int64_t end_row, int64_t row_stride, int ct, int kt) { + uint8_t tile_quants[32][32]; + transpose_tile_to_nibbles(tile_quants, tile_src); + const ggml_half * scale_src = (const ggml_half *) (tile_src + 512); + const uint8_t * plane_src = tile_src + 640; + for (int row = 0; row < 32; row++) { + const int64_t r = ct * 32 + row; + if (r >= start_row && r < end_row && kt < ne0 / 32) { + block_q5_1 & b = dst_slice[(r - start_row) * row_stride + kt]; + pack_nibbles_32(b.qs, tile_quants[row]); + const uint32_t qh = unpack_q5_plane(plane_src, row); + memcpy(b.qh, &qh, sizeof(qh)); + b.d = scale_src[2 * row + 0]; + b.m = scale_src[2 * row + 1]; + } + } + }); +} + static void repack_mxfp4_tiled(ggml_tensor * t, const void * data, size_t offset, size_t size) { repack_to_tiled_grid_32(t, data, offset, size, HTP_TENSOR_TILE_SIZE_MXFP4, [](uint8_t * tile_dst, const block_mxfp4 * src_slice, int64_t ne0, int64_t ne1, int64_t row_stride, int ct, int kt) { @@ -1538,6 +1695,12 @@ void htp_repack_tensor_tiled(ggml_tensor * tensor, const void * data, size_t siz case GGML_TYPE_Q4_1: repack_q4_1_tiled(tensor, data, 0, size); break; + case GGML_TYPE_Q5_0: + repack_q5_0_tiled(tensor, data, 0, size); + break; + case GGML_TYPE_Q5_1: + repack_q5_1_tiled(tensor, data, 0, size); + break; case GGML_TYPE_Q8_0: repack_q8_0_tiled(tensor, data, 0, size); break; @@ -1593,6 +1756,12 @@ void htp_unpack_tensor_tiled(void * data, const ggml_tensor * tensor, size_t off case GGML_TYPE_Q4_1: repack_tiled_q4_1(data, tensor, offset, size); break; + case GGML_TYPE_Q5_0: + repack_tiled_q5_0(data, tensor, offset, size); + break; + case GGML_TYPE_Q5_1: + repack_tiled_q5_1(data, tensor, offset, size); + break; case GGML_TYPE_Q8_0: repack_tiled_q8_0(data, tensor, offset, size); break; diff --git a/ggml/src/ggml-hexagon/htp-repack.h b/ggml/src/ggml-hexagon/htp-repack.h index f656e9b8ddbe..5b71f9c5aae1 100644 --- a/ggml/src/ggml-hexagon/htp-repack.h +++ b/ggml/src/ggml-hexagon/htp-repack.h @@ -8,31 +8,31 @@ #include "htp/htp-tensor.h" static inline bool htp_is_tiled_weight_type(enum ggml_type type) { - return type == GGML_TYPE_Q4_0 || type == GGML_TYPE_Q4_1 || - type == GGML_TYPE_Q8_0 || type == GGML_TYPE_IQ4_NL || - type == GGML_TYPE_MXFP4 || type == GGML_TYPE_Q1_0 || - type == GGML_TYPE_Q6_K || + return type == GGML_TYPE_Q1_0 || type == GGML_TYPE_Q2_0 || + type == GGML_TYPE_Q4_0 || type == GGML_TYPE_Q4_1 || + type == GGML_TYPE_Q5_0 || type == GGML_TYPE_Q5_1 || + type == GGML_TYPE_MXFP4 || type == GGML_TYPE_IQ4_NL || + type == GGML_TYPE_Q8_0 || type == GGML_TYPE_Q6_K || + type == GGML_TYPE_Q2_K || type == GGML_TYPE_Q3_K || type == GGML_TYPE_Q4_K || type == GGML_TYPE_Q5_K || - type == GGML_TYPE_Q3_K || type == GGML_TYPE_Q2_K || - type == GGML_TYPE_Q2_0 || type == GGML_TYPE_F16 || type == GGML_TYPE_F32 || type == GGML_TYPE_BF16; } static inline bool htp_is_tiled_weight_tensor(const struct ggml_tensor * t) { - if (!ggml_is_contiguous(t) || ggml_is_view(t)) { + if (!htp_is_tiled_weight_type(t->type)) { return false; } - if (!htp_is_tiled_weight_type(t->type)) { + if (!ggml_is_contiguous(t) || ggml_is_view(t)) { return false; } - if (t->ne[0] < 32) { + if (t->ne[0] < 32 || (t->ne[0] % 32 != 0) || t->ne[3] != 1) { return false; } if (!ggml_is_quantized(t->type)) { - return (t->ne[0] % 32 == 0) && (t->ne[1] % 32 == 0) && (t->ne[2] == 1) && (t->ne[3] == 1); + return t->ne[1] > 1; } - return t->ne[3] == 1; + return true; } // Size of one tiled row in the DSP tiled layout. The Q3_K..Q6_K tiles store uncompressed scales/mins, @@ -59,10 +59,6 @@ static inline size_t htp_tiled_row_size(enum ggml_type type, int64_t ne0) { return ggml_row_size(type, ne0); } -static inline bool htp_is_hmx_weight_type(enum ggml_type type) { - return htp_is_tiled_weight_type(type); -} - void htp_repack_tensor_tiled(struct ggml_tensor * t, const void * data, size_t size); void htp_unpack_tensor_tiled(void * data, const struct ggml_tensor * t, size_t offset, size_t size); diff --git a/ggml/src/ggml-hexagon/htp/CMakeLists.txt b/ggml/src/ggml-hexagon/htp/CMakeLists.txt index b5d6d55daf41..cf7c2f297f80 100644 --- a/ggml/src/ggml-hexagon/htp/CMakeLists.txt +++ b/ggml/src/ggml-hexagon/htp/CMakeLists.txt @@ -37,6 +37,7 @@ add_library(${HTP_LIB} SHARED repeat-ops.c ssm-conv.c cumsum-ops.c + top-k-mask-ops.c fill-ops.c concat-ops.c diag-ops.c @@ -49,6 +50,8 @@ add_library(${HTP_LIB} SHARED pool-ops.c upscale-ops.c allreduce-ops.c + dsv4-hc-ops.c + lightning-indexer-ops.c ) target_compile_definitions(${HTP_LIB} PRIVATE diff --git a/ggml/src/ggml-hexagon/htp/allreduce-ops.c b/ggml/src/ggml-hexagon/htp/allreduce-ops.c index 7fcc4e866b6d..1b95917ae0e7 100644 --- a/ggml/src/ggml-hexagon/htp/allreduce-ops.c +++ b/ggml/src/ggml-hexagon/htp/allreduce-ops.c @@ -15,7 +15,7 @@ #include "hvx-utils.h" #include "htp-tensor.h" #include "dma-queue.h" -#include "hex-profile.h" +#include "htp-trace.h" #include "allreduce-ops.h" #include "htp-fence.h" @@ -29,9 +29,9 @@ struct htp_allreduce_context { uint32_t row_size_aligned; uint32_t rank_elem_start; uint32_t rank_nelem; - uint32_t elems_per_thread; + uint32_t elems_per_hvx; uint32_t block_elems; - uint32_t vtcm_size_per_thread; + uint32_t vtcm_size_per_hvx; bool is_row_bcast; uint8_t * src_spad_base[HTP_ALLREDUCE_MAX_RANKS]; uint8_t * dst_spad_base; @@ -39,7 +39,8 @@ struct htp_allreduce_context { }; #define DEFINE_ALLREDUCE_THREAD_DMA_1D(SUFFIX, TYPE, HVX_ADD_FN, HAS_ADD) \ -static void allreduce_thread_dma_1d_##SUFFIX(unsigned int nth, unsigned int ith, void * data) { \ +static void allreduce_thread_dma_1d_##SUFFIX(const htp_thread_t * th, unsigned int nth, unsigned int ith, \ + void * data) { \ struct htp_allreduce_context * actx = (struct htp_allreduce_context *) data; \ struct htp_ops_context * octx = actx->octx; \ \ @@ -47,19 +48,19 @@ static void allreduce_thread_dma_1d_##SUFFIX(unsigned int nth, unsigned int ith, const uint32_t n_dsts = actx->n_dsts; \ const uint32_t block_elems = actx->block_elems; \ \ - const uint32_t dr = actx->elems_per_thread; \ + const uint32_t dr = actx->elems_per_hvx; \ const uint32_t ir0 = actx->rank_elem_start + dr * ith; \ const uint32_t ir1 = MIN(ir0 + dr, actx->rank_elem_start + actx->rank_nelem); \ if (ir0 >= ir1) return; \ \ - struct htp_thread_trace * tr = &octx->ctx->trace[ith]; \ - dma_queue * dma_q = octx->ctx->dma[ith]; \ + struct htp_thread_trace * tr = th->trace; \ + dma_queue * dma_q = th->dma_q; \ \ - const size_t vtcm_thread_offset = ith * actx->vtcm_size_per_thread; \ - uint8_t * dst_spad_base = actx->dst_spad_base + vtcm_thread_offset; \ - uint8_t * res_spad_base = HAS_ADD ? (actx->res_spad_base + vtcm_thread_offset) : NULL; \ + const size_t vtcm_hvx_offset = ith * actx->vtcm_size_per_hvx; \ + uint8_t * dst_spad_base = actx->dst_spad_base + vtcm_hvx_offset; \ + uint8_t * res_spad_base = HAS_ADD ? (actx->res_spad_base + vtcm_hvx_offset) : NULL; \ \ - const size_t spad_half = actx->vtcm_size_per_thread / 2; \ + const size_t spad_half = actx->vtcm_size_per_hvx / 2; \ uint32_t ir_prefetch = ir0; \ int spad_idx = 0; \ \ @@ -72,7 +73,7 @@ static void allreduce_thread_dma_1d_##SUFFIX(unsigned int nth, unsigned int ith, dma_queue_push(dma_q, dma_make_data(d_ddr, d_spad), cur_bytes, cur_bytes, cur_bytes, 0); \ } \ for (uint32_t s = 0; s < n_ranks; s++) { \ - uint8_t * s_spad = actx->src_spad_base[s] + vtcm_thread_offset + spad_idx * spad_half; \ + uint8_t * s_spad = actx->src_spad_base[s] + vtcm_hvx_offset + spad_idx * spad_half; \ const dma_addr_t s_ddr = octx->src[s]->data + ir_prefetch * sizeof(TYPE); \ dma_queue_push(dma_q, dma_make_data(s_spad, s_ddr), cur_bytes, cur_bytes, cur_bytes, 1); \ } \ @@ -99,13 +100,13 @@ static void allreduce_thread_dma_1d_##SUFFIX(unsigned int nth, unsigned int ith, dma_queue_pop(dma_q); \ } \ uint8_t * d_spad = dst_spad_base + comp_spad_idx * spad_half; \ - uint8_t * s0_spad = actx->src_spad_base[0] + vtcm_thread_offset + comp_spad_idx * spad_half; \ - uint8_t * s1_spad = actx->src_spad_base[1] + vtcm_thread_offset + comp_spad_idx * spad_half; \ + uint8_t * s0_spad = actx->src_spad_base[0] + vtcm_hvx_offset + comp_spad_idx * spad_half; \ + uint8_t * s1_spad = actx->src_spad_base[1] + vtcm_hvx_offset + comp_spad_idx * spad_half; \ uint8_t * r_spad = HAS_ADD ? (res_spad_base + comp_spad_idx * spad_half) : NULL; \ htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir); \ HVX_ADD_FN(d_spad, s0_spad, s1_spad, cur_elems); \ for (uint32_t s = 2; s < n_ranks; s++) { \ - uint8_t * ss_spad = actx->src_spad_base[s] + vtcm_thread_offset + comp_spad_idx * spad_half; \ + uint8_t * ss_spad = actx->src_spad_base[s] + vtcm_hvx_offset + comp_spad_idx * spad_half; \ HVX_ADD_FN(d_spad, d_spad, ss_spad, cur_elems); \ } \ if (HAS_ADD) { \ @@ -120,7 +121,7 @@ static void allreduce_thread_dma_1d_##SUFFIX(unsigned int nth, unsigned int ith, uint32_t next_elems = MIN(block_elems, ir1 - ir_prefetch); \ size_t next_bytes = next_elems * sizeof(TYPE); \ for (uint32_t s = 0; s < n_ranks; s++) { \ - uint8_t * s_spad = actx->src_spad_base[s] + vtcm_thread_offset + comp_spad_idx * spad_half; \ + uint8_t * s_spad = actx->src_spad_base[s] + vtcm_hvx_offset + comp_spad_idx * spad_half; \ const dma_addr_t s_next = octx->src[s]->data + ir_prefetch * sizeof(TYPE); \ dma_queue_push(dma_q, dma_make_data(s_spad, s_next), next_bytes, next_bytes, next_bytes, 1); \ } \ @@ -143,7 +144,7 @@ DEFINE_ALLREDUCE_THREAD_DMA_1D(add_f16, __fp16, hvx_add_f16_aaa, 1) DEFINE_ALLREDUCE_THREAD_DMA_1D(add_f32, float, hvx_add_f32_aaa, 1) #define DEFINE_ALLREDUCE_THREAD_DMA_2D(SUFFIX, TYPE, HVX_ADD_FN, HAS_ADD, IS_ROW_BCAST) \ -static void allreduce_thread_dma_2d_##SUFFIX(unsigned int nth, unsigned int ith, void * data) { \ +static void allreduce_thread_dma_2d_##SUFFIX(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { \ struct htp_allreduce_context * actx = (struct htp_allreduce_context *) data; \ struct htp_ops_context * octx = actx->octx; \ \ @@ -154,19 +155,19 @@ static void allreduce_thread_dma_2d_##SUFFIX(unsigned int nth, unsigned int ith, const uint32_t row_size_aligned = actx->row_size_aligned; \ const uint32_t row_bytes = ne0 * sizeof(TYPE); \ \ - const uint32_t dr = actx->elems_per_thread; \ + const uint32_t dr = actx->elems_per_hvx; \ const uint32_t r0 = actx->rank_elem_start + dr * ith; \ const uint32_t r1 = MIN(r0 + dr, actx->rank_elem_start + actx->rank_nelem); \ if (r0 >= r1) return; \ \ - struct htp_thread_trace * tr = &octx->ctx->trace[ith]; \ - dma_queue * dma_q = octx->ctx->dma[ith]; \ + struct htp_thread_trace * tr = th->trace; \ + dma_queue * dma_q = th->dma_q; \ \ - const size_t vtcm_thread_offset = ith * actx->vtcm_size_per_thread; \ - uint8_t * dst_spad_base = actx->dst_spad_base + vtcm_thread_offset; \ - uint8_t * res_spad_base = HAS_ADD ? (IS_ROW_BCAST ? actx->res_spad_base : (actx->res_spad_base + vtcm_thread_offset)) : NULL; \ + const size_t vtcm_hvx_offset = ith * actx->vtcm_size_per_hvx; \ + uint8_t * dst_spad_base = actx->dst_spad_base + vtcm_hvx_offset; \ + uint8_t * res_spad_base = HAS_ADD ? (IS_ROW_BCAST ? actx->res_spad_base : (actx->res_spad_base + vtcm_hvx_offset)) : NULL; \ \ - const size_t spad_half = actx->vtcm_size_per_thread / 2; \ + const size_t spad_half = actx->vtcm_size_per_hvx / 2; \ uint32_t r_prefetch = r0; \ int spad_idx = 0; \ \ @@ -178,7 +179,7 @@ static void allreduce_thread_dma_2d_##SUFFIX(unsigned int nth, unsigned int ith, dma_queue_push(dma_q, dma_make_data(d_ddr, d_spad), octx->dsts[d]->nb[1], row_size_aligned, row_bytes, 0); \ } \ for (uint32_t s = 0; s < n_ranks; s++) { \ - uint8_t * s_spad = actx->src_spad_base[s] + vtcm_thread_offset + spad_idx * spad_half; \ + uint8_t * s_spad = actx->src_spad_base[s] + vtcm_hvx_offset + spad_idx * spad_half; \ const dma_addr_t s_ddr = octx->src[s]->data + r_prefetch * octx->src[s]->nb[1]; \ dma_queue_push(dma_q, dma_make_data(s_spad, s_ddr), row_size_aligned, octx->src[s]->nb[1], row_bytes, cur_rows); \ } \ @@ -204,8 +205,8 @@ static void allreduce_thread_dma_2d_##SUFFIX(unsigned int nth, unsigned int ith, dma_queue_pop(dma_q); \ } \ uint8_t * d_spad = dst_spad_base + comp_spad_idx * spad_half; \ - uint8_t * s0_spad = actx->src_spad_base[0] + vtcm_thread_offset + comp_spad_idx * spad_half; \ - uint8_t * s1_spad = actx->src_spad_base[1] + vtcm_thread_offset + comp_spad_idx * spad_half; \ + uint8_t * s0_spad = actx->src_spad_base[0] + vtcm_hvx_offset + comp_spad_idx * spad_half; \ + uint8_t * s1_spad = actx->src_spad_base[1] + vtcm_hvx_offset + comp_spad_idx * spad_half; \ uint8_t * r_spad = (HAS_ADD && !IS_ROW_BCAST) ? (res_spad_base + comp_spad_idx * spad_half) : NULL; \ htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) r); \ for (uint32_t row = 0; row < cur_rows; row++) { \ @@ -214,7 +215,7 @@ static void allreduce_thread_dma_2d_##SUFFIX(unsigned int nth, unsigned int ith, const uint8_t * s1_row = s1_spad + row * row_size_aligned; \ HVX_ADD_FN(d_row, s0_row, s1_row, ne0); \ for (uint32_t s = 2; s < n_ranks; s++) { \ - const uint8_t * ss_row = actx->src_spad_base[s] + vtcm_thread_offset + comp_spad_idx * spad_half + row * row_size_aligned; \ + const uint8_t * ss_row = actx->src_spad_base[s] + vtcm_hvx_offset + comp_spad_idx * spad_half + row * row_size_aligned; \ HVX_ADD_FN(d_row, d_row, ss_row, ne0); \ } \ if (HAS_ADD) { \ @@ -230,7 +231,7 @@ static void allreduce_thread_dma_2d_##SUFFIX(unsigned int nth, unsigned int ith, if (r_prefetch < r1) { \ uint32_t next_rows = MIN(block_rows, r1 - r_prefetch); \ for (uint32_t s = 0; s < n_ranks; s++) { \ - uint8_t * s_spad = actx->src_spad_base[s] + vtcm_thread_offset + comp_spad_idx * spad_half; \ + uint8_t * s_spad = actx->src_spad_base[s] + vtcm_hvx_offset + comp_spad_idx * spad_half; \ const dma_addr_t s_next = octx->src[s]->data + r_prefetch * octx->src[s]->nb[1]; \ dma_queue_push(dma_q, dma_make_data(s_spad, s_next), row_size_aligned, octx->src[s]->nb[1], row_bytes, next_rows); \ } \ @@ -259,19 +260,19 @@ static int validate_allreduce( const struct htp_allreduce_kernel_params * kparams, uint32_t n_ranks ) { - if (kparams->n_threads == 0 || (uint32_t) kparams->n_threads > octx->ctx->n_threads) { + if (kparams->n_hvx == 0 || (uint32_t) kparams->n_hvx > octx->ctx->n_hvx) { return HTP_STATUS_INVAL_PARAMS; } - octx->n_threads = (uint32_t) kparams->n_threads; + octx->n_hvx = (uint32_t) kparams->n_hvx; - if (kparams->vtcm_size_per_thread <= 0 || kparams->vtcm_size <= 0) { + if (kparams->vtcm_size_per_hvx <= 0 || kparams->vtcm_size <= 0) { return HTP_STATUS_INVAL_PARAMS; } const bool has_add = (octx->op == HTP_OP_ALLREDUCE_ADD); const size_t n_vtcm_buffers = htp_allreduce_vtcm_buffer_count( - n_ranks, octx->n_threads, has_add, kparams->is_row_bcast != 0); - const size_t vtcm_size = n_vtcm_buffers * (size_t) kparams->vtcm_size_per_thread; + n_ranks, octx->n_hvx, has_add, kparams->is_row_bcast != 0); + const size_t vtcm_size = n_vtcm_buffers * (size_t) kparams->vtcm_size_per_hvx; if (vtcm_size != (size_t) kparams->vtcm_size) { return HTP_STATUS_INVAL_PARAMS; } @@ -360,10 +361,10 @@ int op_allreduce(struct htp_ops_context * octx) { // 2. Multi-threaded Reduction across assigned rank chunk if (nelem > 0) { - const uint32_t n_threads = (uint32_t) kparams->n_threads; + const uint32_t n_hvx = (uint32_t) kparams->n_hvx; const uint32_t block_elems = (uint32_t) kparams->block_elems; - const uint32_t elems_per_thread = (uint32_t) kparams->elems_per_thread; - const uint32_t vtcm_size_per_thread = (uint32_t) kparams->vtcm_size_per_thread; + const uint32_t elems_per_hvx = (uint32_t) kparams->elems_per_hvx; + const uint32_t vtcm_size_per_hvx = (uint32_t) kparams->vtcm_size_per_hvx; struct htp_allreduce_context actx; actx.octx = octx; @@ -375,9 +376,9 @@ int op_allreduce(struct htp_ops_context * octx) { actx.row_size_aligned = (uint32_t) kparams->row_size_aligned; actx.rank_elem_start = (uint32_t) kparams->rank_elem_start; actx.rank_nelem = (uint32_t) kparams->rank_nelem; - actx.elems_per_thread = elems_per_thread; + actx.elems_per_hvx = elems_per_hvx; actx.block_elems = block_elems; - actx.vtcm_size_per_thread = vtcm_size_per_thread; + actx.vtcm_size_per_hvx = vtcm_size_per_hvx; actx.is_row_bcast = (kparams->is_row_bcast != 0); work_queue_func_t reduce_fun = NULL; @@ -409,13 +410,13 @@ int op_allreduce(struct htp_ops_context * octx) { uint8_t * vtcm_ptr = (uint8_t *) octx->ctx->vtcm_base; for (uint32_t s = 0; s < n_ranks; s++) { actx.src_spad_base[s] = vtcm_ptr; - vtcm_ptr += n_threads * vtcm_size_per_thread; + vtcm_ptr += n_hvx * vtcm_size_per_hvx; } actx.dst_spad_base = vtcm_ptr; - vtcm_ptr += n_threads * vtcm_size_per_thread; + vtcm_ptr += n_hvx * vtcm_size_per_hvx; if (has_add) { actx.res_spad_base = vtcm_ptr; - vtcm_ptr += (actx.is_row_bcast ? 1 : n_threads) * vtcm_size_per_thread; + vtcm_ptr += (actx.is_row_bcast ? 1 : n_hvx) * vtcm_size_per_hvx; } if (has_add && actx.is_row_bcast) { @@ -426,7 +427,7 @@ int op_allreduce(struct htp_ops_context * octx) { dma_queue_pop(dma_q); } - work_queue_run(octx->ctx->work_queue, reduce_fun, &actx, n_threads); + work_queue_run(octx->ctx->hvx_queue, reduce_fun, &actx, n_hvx); } // 4. Exit Barrier: Synchronize all ranks after writing diff --git a/ggml/src/ggml-hexagon/htp/allreduce-ops.h b/ggml/src/ggml-hexagon/htp/allreduce-ops.h index 1264c1cbbcc3..1fb486cb6d71 100644 --- a/ggml/src/ggml-hexagon/htp/allreduce-ops.h +++ b/ggml/src/ggml-hexagon/htp/allreduce-ops.h @@ -24,21 +24,21 @@ enum htp_allreduce_mode { static inline size_t htp_allreduce_vtcm_buffer_count( uint32_t n_ranks, - uint32_t n_threads, + uint32_t n_hvx, bool has_add, bool is_row_bcast ) { - return (size_t) (n_ranks + 1) * n_threads + (has_add ? (is_row_bcast ? 1 : n_threads) : 0); + return (size_t) (n_ranks + 1) * n_hvx + (has_add ? (is_row_bcast ? 1 : n_hvx) : 0); } struct htp_allreduce_kernel_params { uint8_t kernel_type; int32_t rank; int32_t n_ranks; - int32_t n_threads; + int32_t n_hvx; int32_t block_elems; // 1D: block_elems, 2D: block_rows - int32_t elems_per_thread; // 1D: nelem_per_thread, 2D: nrows_per_thread - int32_t vtcm_size_per_thread; + int32_t elems_per_hvx; // 1D: nelem_per_hvx, 2D: nrows_per_hvx + int32_t vtcm_size_per_hvx; int32_t vtcm_size; int32_t ne0; int32_t ne1; diff --git a/ggml/src/ggml-hexagon/htp/argsort-ops.c b/ggml/src/ggml-hexagon/htp/argsort-ops.c index 7aa7ca8e41c9..af260595494b 100644 --- a/ggml/src/ggml-hexagon/htp/argsort-ops.c +++ b/ggml/src/ggml-hexagon/htp/argsort-ops.c @@ -315,55 +315,61 @@ static void bitonic_sort_vtcm_desc(uint8_t * values, uint8_t * indices, uint32_t } } -static void top_k_select_tiled(dma_queue * dma_q, struct htp_thread_trace * tr, - dma_addr_t src_addr, uint32_t n, uint32_t k, - float * values_buf, int32_t * indices_buf, - float * out_values, int32_t * out_indices, - uint32_t chunk_base) { - const uint32_t tile_elems = 1024; - uint32_t n_tiles = (n + tile_elems - 1) / tile_elems; - uint32_t candidate_count = n_tiles * k; - uint32_t merge_n_vec = hmx_ceil_div(candidate_count, 32); - uint32_t merge_n_vec_pow2 = 1; - while (merge_n_vec_pow2 < merge_n_vec) merge_n_vec_pow2 <<= 1; - uint32_t merge_elems = merge_n_vec_pow2 * 32; - float * candidate_values = values_buf + tile_elems; - int32_t * candidate_indices = indices_buf + tile_elems; - uint32_t candidate_pos = 0; - - for (uint32_t offset = 0; offset < n; offset += tile_elems) { - uint32_t tile_count = MIN(tile_elems, n - offset); - dma_sync_read(dma_q, values_buf, src_addr + (size_t) offset * sizeof(float), tile_count * sizeof(float)); - - htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) offset); - if (tile_count < tile_elems) { - hvx_splat_f32_u((uint8_t *) (values_buf + tile_count), -INFINITY, tile_elems - tile_count); +// top-k of one row held in VTCM, k <= 64. Keeps a sorted list of the k best, rejects 256-float blocks with one max-reduce. +// Writes k values and k indices (ties: first seen wins). Slots with no finite value get -INF and index 0. +static void top_k_scan(const float * vals, uint32_t n, uint32_t k, float * out_v, int32_t * out_i) { + float best_v[64]; + int32_t best_i[64]; + uint32_t cnt = 0; + float thr = -INFINITY; + + const uint32_t blk = 256; + for (uint32_t off = 0; off < n; off += blk) { + const uint32_t bn = MIN(blk, n - off); + if (hvx_reduce_max_f32((const uint8_t *) (vals + off), bn) <= thr) { + continue; } - - bitonic_sort_vtcm_desc((uint8_t *) values_buf, (uint8_t *) indices_buf, tile_elems / 32, true); - uint32_t tile_k = MIN(k, tile_count); - for (uint32_t j = 0; j < tile_k; j++) { - candidate_values[candidate_pos] = values_buf[j]; - candidate_indices[candidate_pos] = indices_buf[j] + (int32_t) offset; - candidate_pos++; + for (uint32_t vi = 0; vi < bn; vi += 32) { + const uint32_t vn = MIN(32, bn - vi); + HVX_VectorAlias v_local; + v_local.v = hvx_vmem(vals + off + vi); + for (uint32_t i = 0; i < vn; i++) { + const float v = v_local.fp32[i]; + if (!(v > thr)) continue; + uint32_t pos = cnt < k ? cnt++ : k - 1; + while (pos > 0 && best_v[pos - 1] < v) { + best_v[pos] = best_v[pos - 1]; + best_i[pos] = best_i[pos - 1]; + pos--; + } + best_v[pos] = v; + best_i[pos] = (int32_t) (off + vi + i); + if (cnt == k) thr = best_v[k - 1]; + } } - htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) offset); } - htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, 0); - if (merge_elems > candidate_pos) { - hvx_splat_f32_u((uint8_t *) (candidate_values + candidate_pos), -INFINITY, merge_elems - candidate_pos); - for (uint32_t j = candidate_pos; j < merge_elems; j++) { - candidate_indices[j] = 0; + // fewer than k values above -INF: take -INF or NaN elements, then pad with index 0 + for (uint32_t off = 0; off < n && cnt < k; off += 32) { + const uint32_t vn = MIN(32, n - off); + HVX_VectorAlias v_local; + v_local.v = hvx_vmem(vals + off); + for (uint32_t j = 0; j < vn && cnt < k; j++) { + if (!(v_local.fp32[j] > -INFINITY)) { + best_v[cnt] = -INFINITY; + best_i[cnt] = (int32_t) (off + j); + cnt++; + } } } - - bitonic_sort_vtcm_desc((uint8_t *) candidate_values, (uint8_t *) candidate_indices, merge_n_vec_pow2, false); - htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, 0); + for (; cnt < k; cnt++) { + best_v[cnt] = -INFINITY; + best_i[cnt] = 0; + } for (uint32_t j = 0; j < k; j++) { - out_values[j] = candidate_values[j]; - out_indices[j] = candidate_indices[j] + (int32_t) chunk_base; + out_v[j] = best_v[j]; + out_i[j] = best_i[j]; } } @@ -457,10 +463,10 @@ struct htp_sort_chunk_ctx { size_t phase1_slot_size; float * merge_values; int32_t * merge_indices; - uint32_t real_count[HTP_MAX_NTHREADS]; + uint32_t real_count[HTP_OP_MAX_NHVX]; }; -static void htp_sort_chunk_job(unsigned int n, unsigned int i, void * data) { +static void htp_sort_chunk_job(const htp_thread_t * th, unsigned int n, unsigned int i, void * data) { struct htp_sort_chunk_ctx * cctx = (struct htp_sort_chunk_ctx *) data; struct htp_ops_context * octx = cctx->octx; const struct htp_sort_kernel_params * kparams = cctx->kparams; @@ -481,7 +487,7 @@ static void htp_sort_chunk_job(unsigned int n, unsigned int i, void * data) { float * merge_values = cctx->merge_values; int32_t * merge_indices = cctx->merge_indices; - struct htp_thread_trace * tr = &octx->ctx->trace[i]; + struct htp_thread_trace * tr = th->trace; const dma_addr_t src_addr = src0->data + (size_t) chunk_base * sizeof(float); if (kparams->is_top_k) { @@ -497,7 +503,7 @@ static void htp_sort_chunk_job(unsigned int n, unsigned int i, void * data) { } if (local_k == 1 && ne00 >= 128*1024) { - dma_sync_read(octx->ctx->dma[i], values_buf, src_addr, real_count * sizeof(float)); + dma_sync_read(th->dma_q, values_buf, src_addr, real_count * sizeof(float)); htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) i); float max_value; @@ -509,14 +515,18 @@ static void htp_sort_chunk_job(unsigned int n, unsigned int i, void * data) { } if (local_k > 1 && local_k <= 64 && chunk_elems > 1024) { - top_k_select_tiled(octx->ctx->dma[i], tr, src_addr, real_count, local_k, - values_buf, indices_buf, - &merge_values[i * local_k], &merge_indices[i * local_k], - chunk_base); + dma_sync_read(th->dma_q, values_buf, src_addr, real_count * sizeof(float)); + + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) i); + top_k_scan(values_buf, real_count, local_k, &merge_values[i * local_k], &merge_indices[i * local_k]); + for (uint32_t j = 0; j < local_k; j++) { + merge_indices[i * local_k + j] += (int32_t) chunk_base; + } + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) i); return; } - dma_sync_read(octx->ctx->dma[i], values_buf, src_addr, real_count * sizeof(float)); + dma_sync_read(th->dma_q, values_buf, src_addr, real_count * sizeof(float)); htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) i); if (chunk_elems > real_count) { @@ -532,7 +542,7 @@ static void htp_sort_chunk_job(unsigned int n, unsigned int i, void * data) { htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) i); } else { if (real_count > 0) { - dma_sync_read(octx->ctx->dma[i], values_buf, src_addr, real_count * sizeof(float)); + dma_sync_read(th->dma_q, values_buf, src_addr, real_count * sizeof(float)); htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) i); init_indices_ramp(indices_buf, real_count, chunk_base); @@ -552,7 +562,7 @@ struct htp_argsort_merge_l1_ctx { bool asc; }; -static void htp_argsort_merge_l1_job(unsigned int n, unsigned int i, void * data) { +static void htp_argsort_merge_l1_job(const htp_thread_t * th, unsigned int n, unsigned int i, void * data) { struct htp_argsort_merge_l1_ctx * mctx = (struct htp_argsort_merge_l1_ctx *) data; struct htp_sort_chunk_ctx * cctx = mctx->cctx; const struct htp_sort_kernel_params * kparams = cctx->kparams; @@ -575,7 +585,7 @@ static void htp_argsort_merge_l1_job(unsigned int n, unsigned int i, void * data float * out_val = cctx->merge_values + out_offset; int32_t * out_idx = cctx->merge_indices + out_offset; - struct htp_thread_trace * tr = &cctx->octx->ctx->trace[i]; + struct htp_thread_trace * tr = th->trace; htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) i); merge_runs(val0, idx0, count0, val1, idx1, count1, out_val, out_idx, mctx->asc); htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) i); @@ -586,7 +596,7 @@ struct htp_argsort_merge_l2_ctx { bool asc; }; -static void htp_argsort_merge_l2_job(unsigned int n, unsigned int i, void * data) { +static void htp_argsort_merge_l2_job(const htp_thread_t * th, unsigned int n, unsigned int i, void * data) { struct htp_argsort_merge_l2_ctx * mctx = (struct htp_argsort_merge_l2_ctx *) data; struct htp_sort_chunk_ctx * cctx = mctx->cctx; const struct htp_sort_kernel_params * kparams = cctx->kparams; @@ -609,7 +619,7 @@ static void htp_argsort_merge_l2_job(unsigned int n, unsigned int i, void * data float * out_val = (float *) slot_base; int32_t * out_idx = (int32_t *) (slot_base + 4 * chunk_elems * sizeof(float)); - struct htp_thread_trace * tr = &cctx->octx->ctx->trace[i]; + struct htp_thread_trace * tr = th->trace; htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) i); merge_runs(val0, idx0, count0, val1, idx1, count1, out_val, out_idx, mctx->asc); htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) i); @@ -622,7 +632,7 @@ struct htp_sort_multi_row_ctx { size_t phase1_slot_size; uint32_t row_start; uint32_t row_end; - uint32_t nrows_per_thread; + uint32_t nrows_per_hvx; }; struct htp_tensor_row_coord { @@ -668,10 +678,14 @@ static inline void compute_row_sort( uint32_t ne00, uint32_t chunk_elems, bool is_top_k, - enum ggml_sort_order order + enum ggml_sort_order order, + uint32_t k ) { if (is_top_k) { - if (ne00 == 32 || ne00 == 64 || ne00 == 128 || ne00 == 256 || ne00 == 512 || ne00 == 1024) { + if (k > 1 && k <= 64 && ne00 > 1024) { + float out_v[64]; + top_k_scan(cur_values, ne00, k, out_v, cur_indices); + } else if (ne00 == 32 || ne00 == 64 || ne00 == 128 || ne00 == 256 || ne00 == 512 || ne00 == 1024) { switch (ne00) { case 32: sort32_f32_hvx((uint8_t *) cur_values, (uint8_t *) cur_indices, GGML_SORT_ORDER_DESC); break; case 64: sort64_f32_hvx((uint8_t *) cur_values, (uint8_t *) cur_indices, GGML_SORT_ORDER_DESC); break; @@ -707,7 +721,7 @@ static inline void compute_row_sort( } } -static void htp_sort_multi_row_job(unsigned int n, unsigned int ith, void * data) { +static void htp_sort_multi_row_job(const htp_thread_t * th, unsigned int n, unsigned int ith, void * data) { struct htp_sort_multi_row_ctx * mctx = (struct htp_sort_multi_row_ctx *) data; struct htp_ops_context * octx = mctx->octx; const struct htp_sort_kernel_params * kparams = mctx->kparams; @@ -728,21 +742,21 @@ static void htp_sort_multi_row_job(unsigned int n, unsigned int ith, void * data uint32_t row_start = mctx->row_start; uint32_t row_end = mctx->row_end; - uint32_t r_start = row_start + ith * mctx->nrows_per_thread; - uint32_t r_end = MIN(r_start + mctx->nrows_per_thread, row_end); + uint32_t r_start = row_start + ith * mctx->nrows_per_hvx; + uint32_t r_end = MIN(r_start + mctx->nrows_per_hvx, row_end); if (r_start >= r_end) return; size_t slot_size = mctx->phase1_slot_size; size_t spad_offset = ith * slot_size * (size_t) kparams->n_slots; - uint8_t * thread_spad = mctx->phase1_base + spad_offset; + uint8_t * hvx_spad = mctx->phase1_base + spad_offset; uint32_t chunk_elems = (uint32_t) kparams->chunk_elems; - float * values_buf = (float *) thread_spad; - int32_t * indices_buf = (int32_t *) (thread_spad + chunk_elems * sizeof(float)); + float * values_buf = (float *) hvx_spad; + int32_t * indices_buf = (int32_t *) (hvx_spad + chunk_elems * sizeof(float)); - dma_queue * dma_q = octx->ctx->dma[ith]; - struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + dma_queue * dma_q = th->dma_q; + struct htp_thread_trace * tr = th->trace; bool is_top_k = kparams->is_top_k != 0; uint32_t k = dst->ne[0]; @@ -754,7 +768,7 @@ static void htp_sort_multi_row_job(unsigned int n, unsigned int ith, void * data if (kparams->n_slots == 2) { struct htp_tensor_row_coord prime_coord = init_row_coord(r_start, ne01, ne02, &kparams->div_ne01, &kparams->div_ne01_ne02); for (uint32_t r = r_start, spad_idx = 0; r < r_end && spad_idx < 2; r++, spad_idx++) { - uint8_t * cur_spad = thread_spad + spad_idx * slot_size; + uint8_t * cur_spad = hvx_spad + spad_idx * slot_size; float * cur_values = (float *) cur_spad; int32_t * cur_indices = (int32_t *) (cur_spad + chunk_elems * sizeof(float)); @@ -777,7 +791,7 @@ static void htp_sort_multi_row_job(unsigned int n, unsigned int ith, void * data for (uint32_t r = r_start; r < r_end; r++) { uint32_t cur_slot = (r - r_start) & 1; - uint8_t * cur_spad = thread_spad + cur_slot * slot_size; + uint8_t * cur_spad = hvx_spad + cur_slot * slot_size; float * cur_values = (float *) cur_spad; int32_t * cur_indices = (int32_t *) (cur_spad + chunk_elems * sizeof(float)); @@ -788,7 +802,7 @@ static void htp_sort_multi_row_job(unsigned int n, unsigned int ith, void * data dma_queue_pop(dma_q); htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) r); - compute_row_sort(cur_values, cur_indices, ne00, chunk_elems, is_top_k, order); + compute_row_sort(cur_values, cur_indices, ne00, chunk_elems, is_top_k, order, k); htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) r); // Push writeback of current slot @@ -817,7 +831,7 @@ static void htp_sort_multi_row_job(unsigned int n, unsigned int ith, void * data dma_sync_read(dma_q, values_buf, src_addr, src_bytes); htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) r); - compute_row_sort(values_buf, indices_buf, ne00, chunk_elems, is_top_k, order); + compute_row_sort(values_buf, indices_buf, ne00, chunk_elems, is_top_k, order, k); htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) r); dma_sync_write(dma_q, dst_addr, indices_buf, dst_bytes); @@ -862,7 +876,7 @@ static int op_sort_common(struct htp_ops_context * octx, bool is_top_k) { const struct htp_sort_kernel_params * kparams = (const struct htp_sort_kernel_params *) octx->kernel_params; - if (!htp_ops_context_set_n_threads(octx, kparams->n_threads)) { + if (!htp_ops_context_set_n_hvx(octx, kparams->n_hvx)) { return HTP_STATUS_INVAL_PARAMS; } @@ -872,7 +886,7 @@ static int op_sort_common(struct htp_ops_context * octx, bool is_top_k) { (uint32_t) kparams->chunk_elems, (uint32_t) kparams->n_chunks, (uint32_t) kparams->merge_elems, - (uint32_t) kparams->n_threads, + (uint32_t) kparams->n_hvx, (uint32_t) kparams->n_slots); if (octx->ctx->vtcm_size < layout.total_bytes) { @@ -895,7 +909,7 @@ static int op_sort_common(struct htp_ops_context * octx, bool is_top_k) { cctx.merge_indices = merge_indices; memset(cctx.real_count, 0, sizeof(cctx.real_count)); - work_queue_run(octx->ctx->work_queue, htp_sort_chunk_job, &cctx, (uint32_t) kparams->n_chunks); + work_queue_run(octx->ctx->hvx_queue, htp_sort_chunk_job, &cctx, (uint32_t) kparams->n_chunks); if (is_top_k) { uint32_t chunk_elems = (uint32_t) kparams->chunk_elems; @@ -938,10 +952,10 @@ static int op_sort_common(struct htp_ops_context * octx, bool is_top_k) { bool asc = kparams->order == GGML_SORT_ORDER_ASC; if (kparams->n_chunks == 8) { struct htp_argsort_merge_l1_ctx l1ctx = { &cctx, asc }; - work_queue_run(octx->ctx->work_queue, htp_argsort_merge_l1_job, &l1ctx, 4); + work_queue_run(octx->ctx->hvx_queue, htp_argsort_merge_l1_job, &l1ctx, 4); struct htp_argsort_merge_l2_ctx l2ctx = { &cctx, asc }; - work_queue_run(octx->ctx->work_queue, htp_argsort_merge_l2_job, &l2ctx, 2); + work_queue_run(octx->ctx->hvx_queue, htp_argsort_merge_l2_job, &l2ctx, 2); size_t slot_size = layout.phase1_slot_size; uint32_t chunk_elems = (uint32_t) kparams->chunk_elems; @@ -962,7 +976,7 @@ static int op_sort_common(struct htp_ops_context * octx, bool is_top_k) { dma_sync_write(octx->ctx->dma[0], dst->data, final_idx, ne00 * sizeof(int32_t)); } else if (kparams->n_chunks == 4) { struct htp_argsort_merge_l1_ctx l1ctx = { &cctx, asc }; - work_queue_run(octx->ctx->work_queue, htp_argsort_merge_l1_job, &l1ctx, 2); + work_queue_run(octx->ctx->hvx_queue, htp_argsort_merge_l1_job, &l1ctx, 2); uint32_t chunk_elems = (uint32_t) kparams->chunk_elems; uint32_t n01 = cctx.real_count[0] + cctx.real_count[1]; @@ -1017,13 +1031,13 @@ static int op_sort_common(struct htp_ops_context * octx, bool is_top_k) { if (rows_per_chunk == 0) { rows_per_chunk = 1; } - uint32_t dr = fastdiv(nrows + octx->n_threads - 1, &octx->n_threads_div); + uint32_t dr = fastdiv(nrows + octx->n_hvx - 1, &octx->n_hvx_div); if (rows_per_chunk > 1) { dr = hex_round_up(dr, rows_per_chunk); } - mctx.nrows_per_thread = dr; + mctx.nrows_per_hvx = dr; - work_queue_run(octx->ctx->work_queue, htp_sort_multi_row_job, &mctx, octx->n_threads); + work_queue_run(octx->ctx->hvx_queue, htp_sort_multi_row_job, &mctx, octx->n_hvx); return HTP_STATUS_OK; } diff --git a/ggml/src/ggml-hexagon/htp/argsort-ops.h b/ggml/src/ggml-hexagon/htp/argsort-ops.h index f74189fba5d4..d1cb8087b584 100644 --- a/ggml/src/ggml-hexagon/htp/argsort-ops.h +++ b/ggml/src/ggml-hexagon/htp/argsort-ops.h @@ -10,7 +10,7 @@ #include "htp-vtcm.h" struct htp_sort_kernel_params { - uint32_t n_threads; + uint32_t n_hvx; uint32_t order; // GGML_SORT_ORDER_ASC (0) or GGML_SORT_ORDER_DESC (1) uint32_t is_top_k; // 1 if TOP_K, 0 if ARGSORT uint32_t chunk_elems; @@ -31,7 +31,7 @@ struct htp_sort_vtcm_layout { uint32_t chunk_elems; uint32_t n_chunks; uint32_t merge_elems; - uint32_t n_threads; + uint32_t n_hvx; uint32_t n_slots; }; @@ -41,14 +41,14 @@ static inline void htp_sort_vtcm_layout_build( uint32_t chunk_elems, uint32_t n_chunks, uint32_t merge_elems, - uint32_t n_threads, + uint32_t n_hvx, uint32_t n_slots) { memset(L, 0, sizeof(*L)); L->chunk_elems = chunk_elems; L->n_chunks = n_chunks; L->merge_elems = merge_elems; - L->n_threads = n_threads; + L->n_hvx = n_hvx; L->n_slots = n_slots; const size_t values_size = ((chunk_elems * sizeof(float)) + 127) & ~127; @@ -57,7 +57,7 @@ static inline void htp_sort_vtcm_layout_build( size_t off = 0; if (total_rows > 1) { - VTCM_LAYOUT_ALLOC(off, off_phase1, L->phase1_slot_size * n_slots * n_threads); + VTCM_LAYOUT_ALLOC(off, off_phase1, L->phase1_slot_size * n_slots * n_hvx); } else { VTCM_LAYOUT_ALLOC(off, off_phase1, L->phase1_slot_size * n_chunks * n_slots); const size_t merge_values_size = ((merge_elems * sizeof(float)) + 127) & ~127; @@ -73,14 +73,14 @@ static inline bool htp_sort_solve_layout( uint32_t ne00, uint32_t total_rows, uint32_t k, - uint32_t n_threads_max, + uint32_t n_hvx_max, size_t vtcm_budget, bool is_top_k) { memset(layout, 0, sizeof(*layout)); if (total_rows > 1) { - uint32_t n_threads = total_rows < n_threads_max ? total_rows : n_threads_max; + uint32_t n_hvx = total_rows < n_hvx_max ? total_rows : n_hvx_max; uint32_t n_vec = (ne00 + 31) / 32; uint32_t n_vec_pow2 = 1; @@ -96,13 +96,13 @@ static inline bool htp_sort_solve_layout( n_slots = 1; } - size_t spad_per_thread = spad_per_slot * n_slots; + size_t spad_per_hvx = spad_per_slot * n_slots; - while (n_threads > 1 && (spad_per_thread * n_threads) > vtcm_budget) { - n_threads--; + while (n_hvx > 1 && (spad_per_hvx * n_hvx) > vtcm_budget) { + n_hvx--; } - htp_sort_vtcm_layout_build(layout, total_rows, ne00_padded, 1, 0, n_threads, n_slots); + htp_sort_vtcm_layout_build(layout, total_rows, ne00_padded, 1, 0, n_hvx, n_slots); return layout->total_bytes <= vtcm_budget; } @@ -112,7 +112,7 @@ static inline bool htp_sort_solve_layout( uint32_t n_chunks = 1; if (ne00 > 1024) { - while (n_chunks * 2 <= n_threads_max && n_chunks * 2 <= n_vec_pow2) { + while (n_chunks * 2 <= n_hvx_max && n_chunks * 2 <= n_vec_pow2) { n_chunks *= 2; } } diff --git a/ggml/src/ggml-hexagon/htp/binary-ops.c b/ggml/src/ggml-hexagon/htp/binary-ops.c index 48f461e46673..a5951556b131 100644 --- a/ggml/src/ggml-hexagon/htp/binary-ops.c +++ b/ggml/src/ggml-hexagon/htp/binary-ops.c @@ -14,7 +14,7 @@ #define GGML_COMMON_DECL_C #include "ggml-common.h" #include "hex-common.h" -#include "hex-profile.h" +#include "htp-trace.h" #include "binary-ops.h" #include "htp-ctx.h" #include "htp-ops.h" @@ -39,7 +39,7 @@ struct htp_binary_context { struct fastdiv_values src1_dim3_div; // ne13 uint32_t block_max; - uint32_t nrows_per_thread; + uint32_t nrows_per_hvx; uint32_t total_rows; uint32_t row_start; uint32_t n_srcs; @@ -260,7 +260,7 @@ static void compute_add_id_f32( } // 1. Scalar src1 in VTCM via DMA (ne10 == 1) -static void binary_thread_scalar(unsigned int nth, unsigned int ith, void * data) { +static void binary_thread_scalar(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { struct htp_binary_context * bctx = (struct htp_binary_context *) data; struct htp_ops_context * octx = bctx->octx; htp_binary_preamble; @@ -270,21 +270,21 @@ static void binary_thread_scalar(unsigned int nth, unsigned int ith, void * data const size_t row_stride = (size_t) ne11 * elem_size; const uint32_t row_size_bytes = bctx->row_size_bytes; - const uint32_t start_row = bctx->row_start + bctx->nrows_per_thread * ith; - const uint32_t end_row = MIN(start_row + bctx->nrows_per_thread, bctx->row_start + bctx->total_rows); + const uint32_t start_row = bctx->row_start + bctx->nrows_per_hvx * ith; + const uint32_t end_row = MIN(start_row + bctx->nrows_per_hvx, bctx->row_start + bctx->total_rows); if (start_row >= end_row) return; FARF(HIGH, "binary-scalar: %d/%d (%u:%u) row-size %u (%u)", ith, nth, start_row, end_row, nb01, bctx->dst_row_size_aligned); const struct htp_binary_vtcm_layout * layout = &bctx->vtcm_layout; - uint8_t * src0_spad_base = VTCM_LAYOUT_PTR(uint8_t, bctx->vtcm_base, layout->off_src0) + (ith * layout->src0_bytes_per_thread); - uint8_t * dst_spad_base = VTCM_LAYOUT_PTR(uint8_t, bctx->vtcm_base, layout->off_dst) + (ith * layout->dst_bytes_per_thread); + uint8_t * src0_spad_base = VTCM_LAYOUT_PTR(uint8_t, bctx->vtcm_base, layout->off_src0) + (ith * layout->src0_bytes_per_hvx); + uint8_t * dst_spad_base = VTCM_LAYOUT_PTR(uint8_t, bctx->vtcm_base, layout->off_dst) + (ith * layout->dst_bytes_per_hvx); size_t src0_spad_half = layout->src0_spad_half_size; size_t dst_spad_half = layout->dst_spad_half_size; const void * s1_table = VTCM_LAYOUT_PTR(const void, bctx->vtcm_base, layout->off_src1); - dma_queue * dma_q = octx->ctx->dma[ith]; + dma_queue * dma_q = th->dma_q; uint32_t ir_prefetch = start_row; int spad_idx = 0; @@ -309,7 +309,7 @@ static void binary_thread_scalar(unsigned int nth, unsigned int ith, void * data spad_idx ^= 1; } - struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + struct htp_thread_trace * tr = th->trace; compute_scalar_t compute = (compute_scalar_t) bctx->compute; for (uint32_t ir = start_row; ir < end_row; ) { @@ -358,29 +358,29 @@ static void binary_thread_scalar(unsigned int nth, unsigned int ith, void * data } // 2. Vector Same Shape (ne1x == ne0x) or Simple Broadcast -static void binary_thread_vector_same_shape(unsigned int nth, unsigned int ith, void * data) { +static void binary_thread_vector_same_shape(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { struct htp_binary_context * bctx = (struct htp_binary_context *) data; struct htp_ops_context * octx = bctx->octx; htp_binary_preamble; const uint32_t row_size_bytes = bctx->row_size_bytes; - const uint32_t start_row = bctx->row_start + bctx->nrows_per_thread * ith; - const uint32_t end_row = MIN(start_row + bctx->nrows_per_thread, bctx->row_start + bctx->total_rows); + const uint32_t start_row = bctx->row_start + bctx->nrows_per_hvx * ith; + const uint32_t end_row = MIN(start_row + bctx->nrows_per_hvx, bctx->row_start + bctx->total_rows); if (start_row >= end_row) return; FARF(HIGH, "binary-same-shape: %d/%d (%u:%u) row-size %u (%u)", ith, nth, start_row, end_row, nb01, bctx->dst_row_size_aligned); const struct htp_binary_vtcm_layout * layout = &bctx->vtcm_layout; - uint8_t * src0_spad_base = VTCM_LAYOUT_PTR(uint8_t, bctx->vtcm_base, layout->off_src0) + (ith * layout->src0_bytes_per_thread); - uint8_t * src1_spad_base = VTCM_LAYOUT_PTR(uint8_t, bctx->vtcm_base, layout->off_src1) + (ith * layout->src1_bytes_per_thread); - uint8_t * dst_spad_base = VTCM_LAYOUT_PTR(uint8_t, bctx->vtcm_base, layout->off_dst) + (ith * layout->dst_bytes_per_thread); + uint8_t * src0_spad_base = VTCM_LAYOUT_PTR(uint8_t, bctx->vtcm_base, layout->off_src0) + (ith * layout->src0_bytes_per_hvx); + uint8_t * src1_spad_base = VTCM_LAYOUT_PTR(uint8_t, bctx->vtcm_base, layout->off_src1) + (ith * layout->src1_bytes_per_hvx); + uint8_t * dst_spad_base = VTCM_LAYOUT_PTR(uint8_t, bctx->vtcm_base, layout->off_dst) + (ith * layout->dst_bytes_per_hvx); size_t src0_spad_half = layout->src0_spad_half_size; size_t src1_spad_half = layout->src1_spad_half_size; size_t dst_spad_half = layout->dst_spad_half_size; - dma_queue * dma_q = octx->ctx->dma[ith]; + dma_queue * dma_q = th->dma_q; const uint32_t n_srcs = bctx->n_srcs; uint32_t ir_prefetch = start_row; int spad_idx = 0; @@ -415,7 +415,7 @@ static void binary_thread_vector_same_shape(unsigned int nth, unsigned int ith, spad_idx ^= 1; } - struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + struct htp_thread_trace * tr = th->trace; compute_same_shape_t compute = (compute_same_shape_t) bctx->compute; for (uint32_t ir = start_row; ir < end_row; ) { @@ -472,27 +472,27 @@ static void binary_thread_vector_same_shape(unsigned int nth, unsigned int ith, } // 3. Row Broadcast (ne11 == 1, ne12 == 1, single row src1) -static void binary_thread_vector_row_broadcast(unsigned int nth, unsigned int ith, void * data) { +static void binary_thread_vector_row_broadcast(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { struct htp_binary_context * bctx = (struct htp_binary_context *) data; struct htp_ops_context * octx = bctx->octx; htp_binary_preamble; const uint32_t row_size_bytes = bctx->row_size_bytes; - const uint32_t start_row = bctx->row_start + bctx->nrows_per_thread * ith; - const uint32_t end_row = MIN(start_row + bctx->nrows_per_thread, bctx->row_start + bctx->total_rows); + const uint32_t start_row = bctx->row_start + bctx->nrows_per_hvx * ith; + const uint32_t end_row = MIN(start_row + bctx->nrows_per_hvx, bctx->row_start + bctx->total_rows); if (start_row >= end_row) return; FARF(HIGH, "binary-row-bcast: %d/%d (%u:%u) row-size %u (%u)", ith, nth, start_row, end_row, nb01, bctx->dst_row_size_aligned); const struct htp_binary_vtcm_layout * layout = &bctx->vtcm_layout; - uint8_t * src0_spad_base = VTCM_LAYOUT_PTR(uint8_t, bctx->vtcm_base, layout->off_src0) + (ith * layout->src0_bytes_per_thread); - uint8_t * dst_spad_base = VTCM_LAYOUT_PTR(uint8_t, bctx->vtcm_base, layout->off_dst) + (ith * layout->dst_bytes_per_thread); + uint8_t * src0_spad_base = VTCM_LAYOUT_PTR(uint8_t, bctx->vtcm_base, layout->off_src0) + (ith * layout->src0_bytes_per_hvx); + uint8_t * dst_spad_base = VTCM_LAYOUT_PTR(uint8_t, bctx->vtcm_base, layout->off_dst) + (ith * layout->dst_bytes_per_hvx); size_t src0_spad_half = layout->src0_spad_half_size; size_t dst_spad_half = layout->dst_spad_half_size; - dma_queue * dma_q = octx->ctx->dma[ith]; + dma_queue * dma_q = th->dma_q; uint32_t ir_prefetch = start_row; int spad_idx = 0; @@ -517,7 +517,7 @@ static void binary_thread_vector_row_broadcast(unsigned int nth, unsigned int it spad_idx ^= 1; } - struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + struct htp_thread_trace * tr = th->trace; compute_row_bcast_t compute = (compute_row_bcast_t) bctx->compute; for (uint32_t ir = start_row; ir < end_row; ) { @@ -555,22 +555,22 @@ static void binary_thread_vector_row_broadcast(unsigned int nth, unsigned int it } // 4. Vector Complex (ne10 == ne00, complex broadcast) -static void binary_thread_vector_complex(unsigned int nth, unsigned int ith, void * data) { +static void binary_thread_vector_complex(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { struct htp_binary_context * bctx = (struct htp_binary_context *) data; struct htp_ops_context * octx = bctx->octx; htp_binary_preamble; const uint32_t row_size_bytes = bctx->row_size_bytes; - const uint32_t start_row = bctx->row_start + bctx->nrows_per_thread * ith; - const uint32_t end_row = MIN(start_row + bctx->nrows_per_thread, bctx->row_start + bctx->total_rows); + const uint32_t start_row = bctx->row_start + bctx->nrows_per_hvx * ith; + const uint32_t end_row = MIN(start_row + bctx->nrows_per_hvx, bctx->row_start + bctx->total_rows); if (start_row >= end_row) return; FARF(HIGH, "binary-complex: %d/%d (%u:%u) row-size %u (%u)", ith, nth, start_row, end_row, nb01, bctx->dst_row_size_aligned); const struct htp_binary_vtcm_layout * layout = &bctx->vtcm_layout; - uint8_t * src0_spad_base = VTCM_LAYOUT_PTR(uint8_t, bctx->vtcm_base, layout->off_src0) + (ith * layout->src0_bytes_per_thread); - uint8_t * dst_spad_base = VTCM_LAYOUT_PTR(uint8_t, bctx->vtcm_base, layout->off_dst) + (ith * layout->dst_bytes_per_thread); + uint8_t * src0_spad_base = VTCM_LAYOUT_PTR(uint8_t, bctx->vtcm_base, layout->off_src0) + (ith * layout->src0_bytes_per_hvx); + uint8_t * dst_spad_base = VTCM_LAYOUT_PTR(uint8_t, bctx->vtcm_base, layout->off_dst) + (ith * layout->dst_bytes_per_hvx); size_t src0_spad_half = layout->src0_spad_half_size; size_t dst_spad_half = layout->dst_spad_half_size; const uint8_t * vtcm_src1 = VTCM_LAYOUT_PTR(const uint8_t, bctx->vtcm_base, layout->off_src1); @@ -580,7 +580,7 @@ static void binary_thread_vector_complex(unsigned int nth, unsigned int ith, voi const size_t vtcm_plane_stride = (size_t) ne11 * vtcm_row_stride; const size_t vtcm_vol_stride = (size_t) ne12 * vtcm_plane_stride; - dma_queue * dma_q = octx->ctx->dma[ith]; + dma_queue * dma_q = th->dma_q; uint32_t ir_prefetch = start_row; int spad_idx = 0; @@ -603,7 +603,7 @@ static void binary_thread_vector_complex(unsigned int nth, unsigned int ith, voi spad_idx ^= 1; } - struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + struct htp_thread_trace * tr = th->trace; compute_complex_t compute = (compute_complex_t) bctx->compute; for (uint32_t ir = start_row; ir < end_row; ) { @@ -646,19 +646,19 @@ static void binary_thread_vector_complex(unsigned int nth, unsigned int ith, voi } // 5. Element Repeat (ne10 != ne00) -static void binary_thread_element_repeat(unsigned int nth, unsigned int ith, void * data) { +static void binary_thread_element_repeat(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { struct htp_binary_context * bctx = (struct htp_binary_context *) data; struct htp_ops_context * octx = bctx->octx; htp_binary_preamble; const uint32_t row_size_bytes = bctx->row_size_bytes; - const uint32_t start_row = bctx->row_start + bctx->nrows_per_thread * ith; - const uint32_t end_row = MIN(start_row + bctx->nrows_per_thread, bctx->row_start + bctx->total_rows); + const uint32_t start_row = bctx->row_start + bctx->nrows_per_hvx * ith; + const uint32_t end_row = MIN(start_row + bctx->nrows_per_hvx, bctx->row_start + bctx->total_rows); if (start_row >= end_row) return; const struct htp_binary_vtcm_layout * layout = &bctx->vtcm_layout; - uint8_t * src0_spad_base = VTCM_LAYOUT_PTR(uint8_t, bctx->vtcm_base, layout->off_src0) + (ith * layout->src0_bytes_per_thread); - uint8_t * dst_spad_base = VTCM_LAYOUT_PTR(uint8_t, bctx->vtcm_base, layout->off_dst) + (ith * layout->dst_bytes_per_thread); + uint8_t * src0_spad_base = VTCM_LAYOUT_PTR(uint8_t, bctx->vtcm_base, layout->off_src0) + (ith * layout->src0_bytes_per_hvx); + uint8_t * dst_spad_base = VTCM_LAYOUT_PTR(uint8_t, bctx->vtcm_base, layout->off_dst) + (ith * layout->dst_bytes_per_hvx); size_t src0_spad_half = layout->src0_spad_half_size; size_t dst_spad_half = layout->dst_spad_half_size; const uint8_t * vtcm_src1 = VTCM_LAYOUT_PTR(const uint8_t, bctx->vtcm_base, layout->off_src1); @@ -671,7 +671,7 @@ static void binary_thread_element_repeat(unsigned int nth, unsigned int ith, voi FARF(HIGH, "binary-repeat: %d/%d (%u:%u) row-size %u (%u)", ith, nth, start_row, end_row, nb01, bctx->dst_row_size_aligned); - dma_queue * dma_q = octx->ctx->dma[ith]; + dma_queue * dma_q = th->dma_q; uint32_t ir_prefetch = start_row; int spad_idx = 0; @@ -694,7 +694,7 @@ static void binary_thread_element_repeat(unsigned int nth, unsigned int ith, voi spad_idx ^= 1; } - struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + struct htp_thread_trace * tr = th->trace; compute_repeat_t compute = (compute_repeat_t) bctx->compute; for (uint32_t ir = start_row; ir < end_row; ) { @@ -737,7 +737,7 @@ static void binary_thread_element_repeat(unsigned int nth, unsigned int ith, voi } // 6. ADD_ID (src1 gathered via src2 indices) -static void binary_thread_add_id_f32(unsigned int nth, unsigned int ith, void * data) { +static void binary_thread_add_id_f32(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { struct htp_binary_context * bctx = (struct htp_binary_context *) data; struct htp_ops_context * octx = bctx->octx; @@ -760,19 +760,19 @@ static void binary_thread_add_id_f32(unsigned int nth, unsigned int ith, void * const uint32_t nb3 = dst->nb[3]; const uint32_t row_size_bytes = bctx->row_size_bytes; - const uint32_t start_row = bctx->row_start + bctx->nrows_per_thread * ith; - const uint32_t end_row = MIN(start_row + bctx->nrows_per_thread, bctx->row_start + bctx->total_rows); + const uint32_t start_row = bctx->row_start + bctx->nrows_per_hvx * ith; + const uint32_t end_row = MIN(start_row + bctx->nrows_per_hvx, bctx->row_start + bctx->total_rows); if (start_row >= end_row) return; const struct htp_binary_vtcm_layout * layout = &bctx->vtcm_layout; - uint8_t * src0_spad_base = VTCM_LAYOUT_PTR(uint8_t, bctx->vtcm_base, layout->off_src0) + (ith * layout->src0_bytes_per_thread); - uint8_t * dst_spad_base = VTCM_LAYOUT_PTR(uint8_t, bctx->vtcm_base, layout->off_dst) + (ith * layout->dst_bytes_per_thread); + uint8_t * src0_spad_base = VTCM_LAYOUT_PTR(uint8_t, bctx->vtcm_base, layout->off_src0) + (ith * layout->src0_bytes_per_hvx); + uint8_t * dst_spad_base = VTCM_LAYOUT_PTR(uint8_t, bctx->vtcm_base, layout->off_dst) + (ith * layout->dst_bytes_per_hvx); const uint8_t * vtcm_src1 = VTCM_LAYOUT_PTR(const uint8_t, bctx->vtcm_base, layout->off_src1); const int32_t * vtcm_src2 = VTCM_LAYOUT_PTR(const int32_t, bctx->vtcm_base, layout->off_src2); size_t src0_spad_half = layout->src0_spad_half_size; size_t dst_spad_half = layout->dst_spad_half_size; - dma_queue * dma_q = octx->ctx->dma[ith]; + dma_queue * dma_q = th->dma_q; uint32_t ir_prefetch = start_row; int spad_idx = 0; @@ -795,7 +795,7 @@ static void binary_thread_add_id_f32(unsigned int nth, unsigned int ith, void * spad_idx ^= 1; } - struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + struct htp_thread_trace * tr = th->trace; for (uint32_t ir = start_row; ir < end_row; ) { uint32_t current_block_size = calc_block_size(bctx, ir, end_row, ne01, ne02); @@ -870,7 +870,7 @@ struct binary_chunked_context { uint32_t elem_start; uint32_t nelem; uint32_t chunk_size; - uint32_t chunks_per_thread; + uint32_t chunks_per_hvx; uint32_t total_chunks; uint32_t n_srcs; bool is_scalar; @@ -881,15 +881,15 @@ struct binary_chunked_context { compute_binary_scalar_chunked_f16_t compute_scalar_f16; }; -static void binary_thread_chunked(unsigned int nth, unsigned int ith, void * data) { +static void binary_thread_chunked(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { (void) nth; struct binary_chunked_context * ctx = (struct binary_chunked_context *) data; struct htp_ops_context * octx = ctx->octx; const struct htp_tensor * src0 = octx->src[0]; const struct htp_tensor * dst = octx->dst; - const uint32_t start_chunk = ctx->chunks_per_thread * ith; - const uint32_t end_chunk = MIN(start_chunk + ctx->chunks_per_thread, ctx->total_chunks); + const uint32_t start_chunk = ctx->chunks_per_hvx * ith; + const uint32_t end_chunk = MIN(start_chunk + ctx->chunks_per_hvx, ctx->total_chunks); if (start_chunk >= end_chunk) { return; } @@ -903,11 +903,11 @@ static void binary_thread_chunked(unsigned int nth, unsigned int ith, void * dat ith, nth, start_chunk, end_chunk, ctx->total_chunks, ctx->nelem); const struct htp_binary_vtcm_layout * layout = &ctx->vtcm_layout; - uint8_t * src0_spad_base = VTCM_LAYOUT_PTR(uint8_t, ctx->vtcm_base, layout->off_src0) + (ith * layout->src0_bytes_per_thread); - uint8_t * src1_spad_base = ctx->is_scalar ? NULL : (VTCM_LAYOUT_PTR(uint8_t, ctx->vtcm_base, layout->off_src1) + (ith * layout->src1_bytes_per_thread)); - uint8_t * dst_spad_base = VTCM_LAYOUT_PTR(uint8_t, ctx->vtcm_base, layout->off_dst) + (ith * layout->dst_bytes_per_thread); + uint8_t * src0_spad_base = VTCM_LAYOUT_PTR(uint8_t, ctx->vtcm_base, layout->off_src0) + (ith * layout->src0_bytes_per_hvx); + uint8_t * src1_spad_base = ctx->is_scalar ? NULL : (VTCM_LAYOUT_PTR(uint8_t, ctx->vtcm_base, layout->off_src1) + (ith * layout->src1_bytes_per_hvx)); + uint8_t * dst_spad_base = VTCM_LAYOUT_PTR(uint8_t, ctx->vtcm_base, layout->off_dst) + (ith * layout->dst_bytes_per_hvx); - dma_queue * dma_q = octx->ctx->dma[ith]; + dma_queue * dma_q = th->dma_q; const uint32_t n_srcs = ctx->n_srcs; uint32_t prefetch_chunk = start_chunk; int spad_idx = 0; @@ -939,7 +939,7 @@ static void binary_thread_chunked(unsigned int nth, unsigned int ith, void * dat spad_idx ^= 1; } - struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + struct htp_thread_trace * tr = th->trace; for (uint32_t c = start_chunk; c < end_chunk; c++) { const uint32_t c_start = ctx->elem_start + c * chunk_size; @@ -1091,7 +1091,7 @@ static int execute_op_binary(struct htp_ops_context * octx) { return HTP_STATUS_OK; } - if (!htp_ops_context_set_n_threads(octx, kparams->n_threads)) { + if (!htp_ops_context_set_n_hvx(octx, kparams->n_hvx)) { return HTP_STATUS_INVAL_PARAMS; } @@ -1103,8 +1103,8 @@ static int execute_op_binary(struct htp_ops_context * octx) { const uint32_t chunk_size = kparams->chunk_size; const uint32_t total_chunks = (nelem + chunk_size - 1) / chunk_size; - const uint32_t n_threads = (total_chunks >= 2) ? MIN(octx->n_threads, total_chunks) : 1; - const uint32_t chunks_per_thread = (total_chunks + n_threads - 1) / n_threads; + const uint32_t n_hvx = (total_chunks >= 2) ? MIN(octx->n_hvx, total_chunks) : 1; + const uint32_t chunks_per_hvx = (total_chunks + n_hvx - 1) / n_hvx; float scalar_f32 = 0.0f; _Float16 scalar_f16 = 0; @@ -1128,7 +1128,7 @@ static int execute_op_binary(struct htp_ops_context * octx) { .elem_start = elem_start, .nelem = nelem, .chunk_size = chunk_size, - .chunks_per_thread = chunks_per_thread, + .chunks_per_hvx = chunks_per_hvx, .total_chunks = total_chunks, .n_srcs = n_srcs, .is_scalar = is_scalar, @@ -1139,7 +1139,7 @@ static int execute_op_binary(struct htp_ops_context * octx) { .compute_scalar_f16 = compute_scalar_f16, }; - work_queue_run(octx->ctx->work_queue, binary_thread_chunked, &cctx, n_threads); + work_queue_run(octx->ctx->hvx_queue, binary_thread_chunked, &cctx, n_hvx); return HTP_STATUS_OK; } @@ -1159,11 +1159,11 @@ static int execute_op_binary(struct htp_ops_context * octx) { return HTP_STATUS_OK; } - if (!htp_ops_context_set_n_threads(octx, kparams->n_threads)) { + if (!htp_ops_context_set_n_hvx(octx, kparams->n_hvx)) { return HTP_STATUS_INVAL_PARAMS; } - const uint32_t n_threads = octx->n_threads; + const uint32_t n_hvx = octx->n_hvx; const size_t src0_row_size_aligned = kparams->src0_row_size_aligned; const size_t src1_row_size_aligned = kparams->src1_row_size_aligned; const size_t dst_row_size_aligned = kparams->dst_row_size_aligned; @@ -1176,7 +1176,7 @@ static int execute_op_binary(struct htp_ops_context * octx) { return HTP_STATUS_VTCM_TOO_SMALL; } - worker_callback_t worker_func = NULL; + work_queue_func_t worker_func = NULL; void * compute_func = NULL; switch (kparams->kernel_type) { @@ -1336,7 +1336,7 @@ static int execute_op_binary(struct htp_ops_context * octx) { } bctx.octx = octx; - bctx.nrows_per_thread = fastdiv(nrows + n_threads - 1, &octx->n_threads_div); + bctx.nrows_per_hvx = fastdiv(nrows + n_hvx - 1, &octx->n_hvx_div); bctx.total_rows = nrows; bctx.row_start = row_start; bctx.n_srcs = n_srcs; @@ -1375,7 +1375,7 @@ static int execute_op_binary(struct htp_ops_context * octx) { } } - work_queue_run(octx->ctx->work_queue, worker_func, &bctx, n_threads); + work_queue_run(octx->ctx->hvx_queue, worker_func, &bctx, n_hvx); return HTP_STATUS_OK; } diff --git a/ggml/src/ggml-hexagon/htp/binary-ops.h b/ggml/src/ggml-hexagon/htp/binary-ops.h index 8c5e683e81b3..6f849396ce45 100644 --- a/ggml/src/ggml-hexagon/htp/binary-ops.h +++ b/ggml/src/ggml-hexagon/htp/binary-ops.h @@ -30,7 +30,7 @@ struct htp_binary_kernel_params { uint8_t is_scalar; uint8_t pad[2]; - uint32_t n_threads; + uint32_t n_hvx; uint32_t rows_per_buffer; uint32_t src0_row_size_aligned; @@ -86,9 +86,9 @@ struct htp_binary_vtcm_layout { size_t off_src2; size_t off_dst; - size_t src0_bytes_per_thread; - size_t src1_bytes_per_thread; - size_t dst_bytes_per_thread; + size_t src0_bytes_per_hvx; + size_t src1_bytes_per_hvx; + size_t dst_bytes_per_hvx; size_t src0_spad_half_size; size_t src1_spad_half_size; @@ -106,8 +106,8 @@ static inline void htp_binary_vtcm_layout_build( ) { memset(L, 0, sizeof(*L)); - const uint32_t n_threads = kparams->n_threads; - if (n_threads == 0) { + const uint32_t n_hvx = kparams->n_hvx; + if (n_hvx == 0) { return; } @@ -119,9 +119,9 @@ static inline void htp_binary_vtcm_layout_build( return; } - L->src0_bytes_per_thread = 2 * chunk_bytes; - L->src1_bytes_per_thread = kparams->is_scalar ? 0 : (2 * (n_srcs - 1) * chunk_bytes); - L->dst_bytes_per_thread = 2 * chunk_bytes; + L->src0_bytes_per_hvx = 2 * chunk_bytes; + L->src1_bytes_per_hvx = kparams->is_scalar ? 0 : (2 * (n_srcs - 1) * chunk_bytes); + L->dst_bytes_per_hvx = 2 * chunk_bytes; L->src0_spad_half_size = chunk_bytes; L->src1_spad_half_size = kparams->is_scalar ? 0 : chunk_bytes; @@ -130,9 +130,9 @@ static inline void htp_binary_vtcm_layout_build( L->rows_per_buffer = 1; L->src1_size = 0; - const size_t src0_total = n_threads * L->src0_bytes_per_thread; - const size_t src1_total = kparams->is_scalar ? 128 : (n_threads * L->src1_bytes_per_thread); - const size_t dst_total = n_threads * L->dst_bytes_per_thread; + const size_t src0_total = n_hvx * L->src0_bytes_per_hvx; + const size_t src1_total = kparams->is_scalar ? 128 : (n_hvx * L->src1_bytes_per_hvx); + const size_t dst_total = n_hvx * L->dst_bytes_per_hvx; size_t off = 0; VTCM_LAYOUT_ALLOC(off, off_src0, src0_total); @@ -158,7 +158,7 @@ static inline void htp_binary_vtcm_layout_build( const size_t rows_per_buffer = (kparams->rows_per_buffer > 0) ? kparams->rows_per_buffer - : (vtcm_size - static_vtcm_size) / (n_threads * spad_row_total); + : (vtcm_size - static_vtcm_size) / (n_hvx * spad_row_total); if (rows_per_buffer == 0) { return; } @@ -167,24 +167,24 @@ static inline void htp_binary_vtcm_layout_build( L->src1_size = kparams->src1_size; L->src2_size = kparams->src2_size; - L->src0_bytes_per_thread = rows_per_buffer * 2 * kparams->src0_row_size_aligned; - L->dst_bytes_per_thread = rows_per_buffer * 2 * kparams->dst_row_size_aligned; - L->src1_bytes_per_thread = (kparams->kernel_type == HTP_BINARY_KERNEL_SAME_SHAPE) + L->src0_bytes_per_hvx = rows_per_buffer * 2 * kparams->src0_row_size_aligned; + L->dst_bytes_per_hvx = rows_per_buffer * 2 * kparams->dst_row_size_aligned; + L->src1_bytes_per_hvx = (kparams->kernel_type == HTP_BINARY_KERNEL_SAME_SHAPE) ? rows_per_buffer * 2 * (n_srcs - 1) * kparams->src1_row_size_aligned : 0; - L->src0_spad_half_size = L->src0_bytes_per_thread / 2; + L->src0_spad_half_size = L->src0_bytes_per_hvx / 2; L->src1_spad_half_size = (kparams->kernel_type == HTP_BINARY_KERNEL_SAME_SHAPE) ? rows_per_buffer * kparams->src1_row_size_aligned : 0; - L->dst_spad_half_size = L->dst_bytes_per_thread / 2; + L->dst_spad_half_size = L->dst_bytes_per_hvx / 2; - const size_t src0_total = n_threads * L->src0_bytes_per_thread; + const size_t src0_total = n_hvx * L->src0_bytes_per_hvx; const size_t src1_total = (kparams->src1_size > 0) ? kparams->src1_size - : n_threads * L->src1_bytes_per_thread; + : n_hvx * L->src1_bytes_per_hvx; const size_t src2_total = kparams->src2_size; - const size_t dst_total = n_threads * L->dst_bytes_per_thread; + const size_t dst_total = n_hvx * L->dst_bytes_per_hvx; size_t off = 0; VTCM_LAYOUT_ALLOC(off, off_src0, src0_total); @@ -200,15 +200,15 @@ static inline bool htp_binary_solve_chunked( struct htp_binary_kernel_params * kparams, uint32_t total_elems, uint32_t elem_size, - uint32_t max_threads, + uint32_t max_hvx, uint32_t n_srcs, bool is_scalar, size_t vtcm_size ) { const size_t n_bufs = is_scalar ? 2 : (size_t) n_srcs + 1; const size_t scalar_bytes = is_scalar ? 128 : 0; - const size_t budget_per_buf = (max_threads > 0 && vtcm_size > scalar_bytes) - ? hex_align_down((vtcm_size - scalar_bytes) / (2 * max_threads * n_bufs), 128) + const size_t budget_per_buf = (max_hvx > 0 && vtcm_size > scalar_bytes) + ? hex_align_down((vtcm_size - scalar_bytes) / (2 * max_hvx * n_bufs), 128) : 0; const uint32_t max_chunk_bytes = (budget_per_buf > 0 && budget_per_buf < 32768) ? (uint32_t) budget_per_buf : 32768u; const uint32_t max_chunk_elems = max_chunk_bytes / elem_size; @@ -217,16 +217,16 @@ static inline bool htp_binary_solve_chunked( } const uint32_t min_chunk_elems = 256 < max_chunk_elems ? 256 : max_chunk_elems; - const uint32_t target_chunk_elems = hex_round_up((total_elems + (2 * max_threads) - 1) / (2 * max_threads), 32); + const uint32_t target_chunk_elems = hex_round_up((total_elems + (2 * max_hvx) - 1) / (2 * max_hvx), 32); const uint32_t chunk_size = (target_chunk_elems < min_chunk_elems) ? min_chunk_elems : (target_chunk_elems > max_chunk_elems) ? max_chunk_elems : target_chunk_elems; const uint32_t chunk_bytes = hex_round_up(chunk_size * elem_size, 128); const uint32_t total_chunks = (total_elems + chunk_size - 1) / chunk_size; - const uint32_t n_threads = (total_chunks >= 2) ? (max_threads < total_chunks ? max_threads : total_chunks) : 1; + const uint32_t n_hvx = (total_chunks >= 2) ? (max_hvx < total_chunks ? max_hvx : total_chunks) : 1; kparams->kernel_type = HTP_BINARY_KERNEL_CHUNKED; - kparams->n_threads = n_threads; + kparams->n_hvx = n_hvx; kparams->n_srcs = (uint8_t) n_srcs; kparams->rows_per_buffer = 1; kparams->src1_row_size_aligned = is_scalar ? 0 : kparams->src0_row_size_aligned; diff --git a/ggml/src/ggml-hexagon/htp/concat-ops.c b/ggml/src/ggml-hexagon/htp/concat-ops.c index 8f3e199c252e..a374a8fd6020 100644 --- a/ggml/src/ggml-hexagon/htp/concat-ops.c +++ b/ggml/src/ggml-hexagon/htp/concat-ops.c @@ -3,7 +3,7 @@ #include "hex-common.h" #include "dma-copy.h" #include "hex-fastdiv.h" -#include "hex-profile.h" +#include "htp-trace.h" #include "hexagon_protos.h" #include "hexagon_types.h" #include "htp-ctx.h" @@ -21,11 +21,11 @@ struct htp_concat_context { struct htp_ops_context * octx; uint8_t * spad0_base; uint8_t * spad1_base; - uint32_t spad0_size_per_thread; - uint32_t spad1_size_per_thread; + uint32_t spad0_size_per_hvx; + uint32_t spad1_size_per_hvx; uint32_t row_start; uint32_t nrows; - uint32_t nrows_per_thread; + uint32_t nrows_per_hvx; uint32_t nplanes; struct fastdiv_values div_ne2; }; @@ -36,7 +36,7 @@ static inline dma_addr_t concat_plane_addr(const struct htp_tensor * t, uint32_t return t->data + i2 * t->nb[2] + i3 * t->nb[3]; } -static void concat_2d_f32_transposed(unsigned int nth, unsigned int ith, void * data) { +static void concat_2d_f32_transposed(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { struct htp_concat_context * cctx = (struct htp_concat_context *) data; struct htp_ops_context * octx = cctx->octx; @@ -48,12 +48,12 @@ static void concat_2d_f32_transposed(unsigned int nth, unsigned int ith, void * const uint32_t src1_ne0 = src1->ne[0]; // dst rows [row_begin, row_end) of this thread; a dst row is a src0 row followed by a src1 column - const uint32_t row_begin = cctx->row_start + ith * cctx->nrows_per_thread; - const uint32_t row_end = MIN(row_begin + cctx->nrows_per_thread, cctx->row_start + cctx->nrows); + const uint32_t row_begin = cctx->row_start + ith * cctx->nrows_per_hvx; + const uint32_t row_end = MIN(row_begin + cctx->nrows_per_hvx, cctx->row_start + cctx->nrows); if (row_begin >= row_end) return; - dma_queue * dma_q = octx->ctx->dma[ith]; - struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + dma_queue * dma_q = th->dma_q; + struct htp_thread_trace * tr = th->trace; const struct fastdiv_values * div_ne2 = &cctx->div_ne2; const uint32_t ne2 = dst->ne[2]; @@ -72,8 +72,8 @@ static void concat_2d_f32_transposed(unsigned int nth, unsigned int ith, void * const uint32_t spad0_stride = src0_row_padded + block_cols_padded * sizeof(float); // two slots per scratchpad: tile b gathers in slot b&1 while dst[b-1] and src1[b+1] DMA in the other - uint8_t * spad0_base = cctx->spad0_base + ith * cctx->spad0_size_per_thread; - uint8_t * spad1_base = cctx->spad1_base + ith * cctx->spad1_size_per_thread; + uint8_t * spad0_base = cctx->spad0_base + ith * cctx->spad0_size_per_hvx; + uint8_t * spad1_base = cctx->spad1_base + ith * cctx->spad1_size_per_hvx; const uint32_t spad0_slot = block_rows * spad0_stride; const uint32_t spad1_slot = block_cols_padded * spad1_stride; @@ -153,7 +153,7 @@ static void concat_2d_f32_transposed(unsigned int nth, unsigned int ith, void * dma_queue_flush(dma_q); } -static void concat_2d_f16_transposed(unsigned int nth, unsigned int ith, void * data) { +static void concat_2d_f16_transposed(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { struct htp_concat_context * cctx = (struct htp_concat_context *) data; struct htp_ops_context * octx = cctx->octx; @@ -165,12 +165,12 @@ static void concat_2d_f16_transposed(unsigned int nth, unsigned int ith, void * const uint32_t src1_ne0 = src1->ne[0]; // dst rows [row_begin, row_end) of this thread; a dst row is a src0 row followed by a src1 column - const uint32_t row_begin = cctx->row_start + ith * cctx->nrows_per_thread; - const uint32_t row_end = MIN(row_begin + cctx->nrows_per_thread, cctx->row_start + cctx->nrows); + const uint32_t row_begin = cctx->row_start + ith * cctx->nrows_per_hvx; + const uint32_t row_end = MIN(row_begin + cctx->nrows_per_hvx, cctx->row_start + cctx->nrows); if (row_begin >= row_end) return; - dma_queue * dma_q = octx->ctx->dma[ith]; - struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + dma_queue * dma_q = th->dma_q; + struct htp_thread_trace * tr = th->trace; const struct fastdiv_values * div_ne2 = &cctx->div_ne2; const uint32_t ne2 = dst->ne[2]; @@ -189,8 +189,8 @@ static void concat_2d_f16_transposed(unsigned int nth, unsigned int ith, void * const uint32_t spad0_stride = src0_row_padded + block_cols_padded * sizeof(__fp16); // two slots per scratchpad: tile b gathers in slot b&1 while dst[b-1] and src1[b+1] DMA in the other - uint8_t * spad0_base = cctx->spad0_base + ith * cctx->spad0_size_per_thread; - uint8_t * spad1_base = cctx->spad1_base + ith * cctx->spad1_size_per_thread; + uint8_t * spad0_base = cctx->spad0_base + ith * cctx->spad0_size_per_hvx; + uint8_t * spad1_base = cctx->spad1_base + ith * cctx->spad1_size_per_hvx; const uint32_t spad0_slot = block_rows * spad0_stride; const uint32_t spad1_slot = block_cols_padded * spad1_stride; @@ -311,7 +311,7 @@ static int concat_regular(struct htp_ops_context * octx, int dim, uint32_t type_ } static int concat_transposed(struct htp_ops_context * octx, const struct htp_concat_kernel_params * kparams, uint32_t type_size) { - if (!htp_ops_context_set_n_threads(octx, kparams->n_threads)) { + if (!htp_ops_context_set_n_hvx(octx, kparams->n_hvx)) { return HTP_STATUS_INVAL_PARAMS; } @@ -334,10 +334,10 @@ static int concat_transposed(struct htp_ops_context * octx, const struct htp_con return HTP_STATUS_VTCM_TOO_SMALL; } - const uint32_t n_threads = octx->n_threads; + const uint32_t n_hvx = octx->n_hvx; struct htp_concat_transposed_vtcm_layout layout; - htp_concat_transposed_vtcm_layout_build(&layout, octx->src[0]->ne[0], octx->src[1]->ne[0], type_size, n_threads); + htp_concat_transposed_vtcm_layout_build(&layout, octx->src[0]->ne[0], octx->src[1]->ne[0], type_size, n_hvx); uint8_t * vtcm_base = (uint8_t *) octx->ctx->vtcm_base; @@ -345,16 +345,16 @@ static int concat_transposed(struct htp_ops_context * octx, const struct htp_con cctx.octx = octx; cctx.spad0_base = VTCM_LAYOUT_PTR(uint8_t, vtcm_base, layout.off_spad0); cctx.spad1_base = VTCM_LAYOUT_PTR(uint8_t, vtcm_base, layout.off_spad1); - cctx.spad0_size_per_thread = layout.src0_spad_size_per_thread; - cctx.spad1_size_per_thread = layout.src1_spad_size_per_thread; + cctx.spad0_size_per_hvx = layout.src0_spad_size_per_hvx; + cctx.spad1_size_per_hvx = layout.src1_spad_size_per_hvx; cctx.row_start = row_start; cctx.nrows = nrows; cctx.nplanes = dst->ne[2] * dst->ne[3]; cctx.div_ne2 = kparams->u.transposed.div_ne2; - cctx.nrows_per_thread = fastdiv(nrows + n_threads - 1, &octx->n_threads_div); + cctx.nrows_per_hvx = fastdiv(nrows + n_hvx - 1, &octx->n_hvx_div); work_queue_func_t worker_func = (type_size == 4) ? concat_2d_f32_transposed : concat_2d_f16_transposed; - work_queue_run(octx->ctx->work_queue, worker_func, &cctx, n_threads); + work_queue_run(octx->ctx->hvx_queue, worker_func, &cctx, n_hvx); return HTP_STATUS_OK; } @@ -377,7 +377,7 @@ int op_concat(struct htp_ops_context * octx) { break; case HTP_CONCAT_KERNEL_GATHER_ROWS: - status = htp_gather_rows(octx, src0, src1, &kparams->u.gather, kparams->n_threads); + status = htp_gather_rows(octx, src0, src1, &kparams->u.gather, kparams->n_hvx); break; default: diff --git a/ggml/src/ggml-hexagon/htp/concat-ops.h b/ggml/src/ggml-hexagon/htp/concat-ops.h index 1f07444349bf..580fe0e15c6b 100644 --- a/ggml/src/ggml-hexagon/htp/concat-ops.h +++ b/ggml/src/ggml-hexagon/htp/concat-ops.h @@ -21,7 +21,7 @@ struct htp_concat_transposed_params { struct htp_concat_kernel_params { uint8_t kernel_type; uint8_t dim; - uint8_t n_threads; + uint8_t n_hvx; uint8_t pad; uint32_t vtcm_size; @@ -39,8 +39,8 @@ _Static_assert(sizeof(struct htp_concat_kernel_params) <= 128, "htp_concat_kerne #endif struct htp_concat_transposed_vtcm_layout { - uint32_t src0_spad_size_per_thread; - uint32_t src1_spad_size_per_thread; + uint32_t src0_spad_size_per_hvx; + uint32_t src1_spad_size_per_hvx; size_t off_spad0; size_t off_spad1; uint32_t total_bytes; @@ -51,7 +51,7 @@ static inline void htp_concat_transposed_vtcm_layout_build( uint32_t src0_ne0, uint32_t src1_ne0, uint32_t type_size, - uint32_t n_threads) { + uint32_t n_hvx) { // src1_ne0 is processed in chunks of block_cols tokens, so VTCM does not grow with the batch uint32_t block_rows = 128; @@ -62,12 +62,12 @@ static inline void htp_concat_transposed_vtcm_layout_build( uint32_t spad0_row_bytes = hex_round_up(src0_ne0 * type_size, 128) + block_cols_padded * type_size; // double-buffered: two slots per thread - L->src0_spad_size_per_thread = 2 * block_rows * spad0_row_bytes; - L->src1_spad_size_per_thread = 2 * block_cols_padded * spad1_stride; + L->src0_spad_size_per_hvx = 2 * block_rows * spad0_row_bytes; + L->src1_spad_size_per_hvx = 2 * block_cols_padded * spad1_stride; size_t off = 0; - VTCM_LAYOUT_ALLOC(off, off_spad0, (size_t) n_threads * L->src0_spad_size_per_thread); - VTCM_LAYOUT_ALLOC(off, off_spad1, (size_t) n_threads * L->src1_spad_size_per_thread); + VTCM_LAYOUT_ALLOC(off, off_spad0, (size_t) n_hvx * L->src0_spad_size_per_hvx); + VTCM_LAYOUT_ALLOC(off, off_spad1, (size_t) n_hvx * L->src1_spad_size_per_hvx); L->total_bytes = (uint32_t) off; } diff --git a/ggml/src/ggml-hexagon/htp/conv2d-ops.c b/ggml/src/ggml-hexagon/htp/conv2d-ops.c index 42e70e089640..67e8ec67b70f 100644 --- a/ggml/src/ggml-hexagon/htp/conv2d-ops.c +++ b/ggml/src/ggml-hexagon/htp/conv2d-ops.c @@ -12,7 +12,7 @@ #include "dma-queue.h" #include "hex-common.h" #include "hex-fastdiv.h" -#include "hex-profile.h" +#include "htp-trace.h" #include "htp-ctx.h" #include "htp-ops.h" #include "htp-tensor.h" @@ -87,11 +87,11 @@ static inline HVX_Vector conv2d_hmx_rt_restore(const __fp16 * scales, uint32_t r return Q6_V_vsplat_R((127 + e) << 23); } -// Output rows [*row_base, *row_base + *n_rows) of N * OH for this device. +// Output rows [*row_base, *row_base + *n_rows) of total for this device. // The output is written by DMA only, so device ranges need no cache line alignment. -static void conv2d_mdev_rows(const struct htp_ops_context * octx, uint32_t * row_base, uint32_t * n_rows) { +static void conv2d_mdev_rows(const struct htp_ops_context * octx, uint32_t total, uint32_t * row_base, uint32_t * n_rows) { *row_base = 0; - *n_rows = octx->dst->ne[3] * octx->dst->ne[1]; + *n_rows = total; if (octx->ctx->mdev.count > 1) { const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition( *n_rows, 1, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div); @@ -100,7 +100,7 @@ static void conv2d_mdev_rows(const struct htp_ops_context * octx, uint32_t * row } } -static void conv2d_hmx_worker(void * data) { +static void conv2d_hmx_worker(const htp_thread_t * th, void * data) { struct htp_conv2d_hmx_task * task = (struct htp_conv2d_hmx_task *) data; const size_t dot_stride = task->n_dot_tiles * HMX_FP16_TILE_N_ELMS; @@ -134,9 +134,51 @@ static void conv2d_hmx_worker(void * data) { } } +// Tiled path values whose source differs between conv_2d and conv_3d. +// conv_3d runs as conv_2d with virtual KH = KD * KHr and OH = OD * OHr, only the input stage unfolds them. +struct htp_conv_geom { + uint32_t KH, KHr, Cin, Cout; + uint32_t OH, OHr, N, ID; + int32_t s0, s1, s2, p0, p1, p2, d0, d1, d2; + size_t x_nb_d, x_nb_c, x_nb_n; // input strides for d, cin, n + size_t y_nb_c, y_nb_n; // dst strides for cout, n + size_t w_nb_co; // kernel cout stride +}; + +// conv_3d tensors are [W, H, D, C * N] with the channel as the inner index of ne[3] +static void conv_geom_init(struct htp_conv_geom * g, const struct htp_ops_context * octx) { + const struct htp_tensor * kernel = octx->src[0]; + const struct htp_tensor * input = octx->src[1]; + const struct htp_tensor * dst = octx->dst; + const int32_t * op = octx->op_params; + + if (octx->op == HTP_OP_CONV_3D) { + const uint32_t IC = op[9]; + const uint32_t OC = op[11]; + *g = (struct htp_conv_geom) { + .KH = kernel->ne[2] * kernel->ne[1], .KHr = kernel->ne[1], .Cin = IC, .Cout = OC, + .OH = dst->ne[2] * dst->ne[1], .OHr = dst->ne[1], .N = op[10], .ID = input->ne[2], + .s0 = op[0], .s1 = op[1], .s2 = op[2], .p0 = op[3], .p1 = op[4], .p2 = op[5], .d0 = op[6], .d1 = op[7], .d2 = op[8], + .x_nb_d = input->nb[2], .x_nb_c = input->nb[3], .x_nb_n = (size_t) IC * input->nb[3], + .y_nb_c = dst->nb[3], .y_nb_n = (size_t) OC * dst->nb[3], + .w_nb_co = (size_t) IC * kernel->nb[3], + }; + } else { + *g = (struct htp_conv_geom) { + .KH = kernel->ne[1], .KHr = kernel->ne[1], .Cin = kernel->ne[2], .Cout = kernel->ne[3], + .OH = dst->ne[1], .OHr = dst->ne[1], .N = dst->ne[3], .ID = 1, + .s0 = op[0], .s1 = op[1], .s2 = 1, .p0 = op[2], .p1 = op[3], .p2 = 0, .d0 = op[4], .d1 = op[5], .d2 = 1, + .x_nb_d = 0, .x_nb_c = input->nb[2], .x_nb_n = input->nb[3], + .y_nb_c = dst->nb[2], .y_nb_n = dst->nb[3], + .w_nb_co = kernel->nb[3], + }; + } +} + struct htp_conv2d_hmx_context { struct htp_ops_context * octx; const struct htp_conv2d_kernel_params * kp; + const struct htp_conv_geom * g; struct htp_conv2d_vtcm_layout layout; __fp16 * scales; // [rt] HMX scales __fp16 * w_tiles; @@ -163,20 +205,20 @@ static inline uint32_t conv2d_hmx_chunk_range(const struct htp_conv2d_hmx_contex return cs; } -static void conv2d_hmx_wpack_thread(unsigned int nth, unsigned int ith, void * data) { +static void conv2d_hmx_wpack_thread(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { const struct htp_conv2d_hmx_context * hc = (const struct htp_conv2d_hmx_context *) data; struct htp_ops_context * octx = hc->octx; const struct htp_conv2d_kernel_params * kp = hc->kp; const struct htp_tensor * kernel = octx->src[0]; - const uint32_t Cout = kernel->ne[3]; + const uint32_t Cout = hc->g->Cout; const uint32_t K = kp->tiled.k_size; const uint32_t k_pad = kp->tiled.n_k_tiles * HMX_FP16_TILE_N_COLS; const uint32_t WK = kp->tiled.wk_chunk; const bool is_f16 = kernel->type == HTP_TYPE_F16; const uint32_t es = is_f16 ? sizeof(__fp16) : sizeof(float); - dma_queue * dmaq = octx->ctx->dma[ith]; + dma_queue * dmaq = th->dma_q; uint8_t * thr = hc->thr_base + (size_t) ith * kp->thr_size; const size_t w_buf_size = (size_t) WK * 32 * sizeof(__fp16); const size_t w_raw_size = (size_t) WK * 32 * sizeof(float); @@ -190,7 +232,7 @@ static void conv2d_hmx_wpack_thread(unsigned int nth, unsigned int ith, void * d const HVX_Vector zero = Q6_V_vzero(); - struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + struct htp_thread_trace * tr = th->trace; for (uint32_t rt = ith; rt < hc->n_rt; rt += nth) { const uint32_t co0 = (hc->rt0 + rt) * 32; @@ -200,8 +242,8 @@ static void conv2d_hmx_wpack_thread(unsigned int nth, unsigned int ith, void * d for (uint32_t i = 0; i < 2 && i * WK < k_pad; i++) { const uint32_t kc = i * WK; const uint32_t kv = MIN(WK, K - kc); - dma_queue_push(dmaq, dma_make_data(wdma[i], kernel->data + (size_t) co0 * kernel->nb[3] + (size_t) kc * es), - WK * es, kernel->nb[3], kv * es, nco); + dma_queue_push(dmaq, dma_make_data(wdma[i], kernel->data + (size_t) co0 * hc->g->w_nb_co + (size_t) kc * es), + WK * es, hc->g->w_nb_co, kv * es, nco); } for (uint32_t kc = 0; kc < k_pad; kc += WK) { @@ -234,42 +276,50 @@ static void conv2d_hmx_wpack_thread(unsigned int nth, unsigned int ith, void * d const uint32_t next_kc = kc + 2 * WK; if (next_kc < k_pad) { const uint32_t next_kv = MIN(WK, K - next_kc); - dma_queue_push(dmaq, dma_make_data(wdma[bi], kernel->data + (size_t) co0 * kernel->nb[3] + (size_t) next_kc * es), - WK * es, kernel->nb[3], next_kv * es, nco); + dma_queue_push(dmaq, dma_make_data(wdma[bi], kernel->data + (size_t) co0 * hc->g->w_nb_co + (size_t) next_kc * es), + WK * es, hc->g->w_nb_co, next_kv * es, nco); } bi ^= 1; } } } -// Stage Cin*KH input rows for G col tiles: stage[(cin*KH + kh)*SR + j] = x[n][cin][ih(kh)][ow0 - p0 + j], zero outside +// Stage Cin*KH input rows for G col tiles: stage[(cin*KH + kh)*SR + j] = x[n][cin][ih(kh)][ow0*s0 - p0 + j], zero outside +// For conv_3d, kh = kd * KHr + kh' and oh = od * OHr + oh' are unfolded here static void conv2d_hmx_stage(const struct htp_ops_context * octx, const struct htp_conv2d_kernel_params * kp, - dma_queue * dmaq, float * stage, uint32_t n, uint32_t oh, uint32_t ow0, uint32_t G) { + const struct htp_conv_geom * g, dma_queue * dmaq, float * stage, uint32_t n, uint32_t oh, + uint32_t ow0, uint32_t G) { const struct htp_tensor * kernel = octx->src[0]; const struct htp_tensor * input = octx->src[1]; const uint32_t KW = kernel->ne[0]; - const uint32_t KH = kernel->ne[1]; - const uint32_t Cin = kernel->ne[2]; + const uint32_t KH = g->KH; + const uint32_t Cin = g->Cin; const int32_t IW = input->ne[0]; const int32_t IH = input->ne[1]; - const int32_t s1 = octx->op_params[1]; - const int32_t p0 = octx->op_params[2]; - const int32_t p1 = octx->op_params[3]; - const int32_t d0 = octx->op_params[4]; - const int32_t d1 = octx->op_params[5]; + const int32_t s0 = g->s0; + const int32_t s1 = g->s1; + const int32_t p0 = g->p0; + const int32_t p1 = g->p1; + const int32_t d0 = g->d0; + const int32_t d1 = g->d1; const uint32_t SR = kp->tiled.stage_row; - const int32_t Wb = 32 * G + (KW - 1) * d0; - const int32_t iw0 = (int32_t) ow0 - p0; + const int32_t Wb = 32 * G * s0 + (KW - 1) * d0; + const int32_t iw0 = (int32_t) ow0 * s0 - p0; const int32_t lo = MAX(0, -iw0); const int32_t hi = MIN(Wb, IW - iw0); + const uint32_t od = fastdiv(oh, &kp->tiled.div_ohr); + oh -= od * g->OHr; + for (uint32_t kh = 0; kh < KH; kh++) { - const int32_t ih = (int32_t) oh * s1 + (int32_t) kh * d1 - p1; + const int32_t kd = fastdiv(kh, &kp->tiled.div_khr); + const int32_t id = (int32_t) od * g->s2 + kd * g->d2 - g->p2; + const int32_t ih = (int32_t) oh * s1 + (int32_t) (kh - kd * g->KHr) * d1 - p1; float * rows = stage + (size_t) kh * SR; - if (ih < 0 || ih >= IH || hi <= lo) { + if (id < 0 || id >= (int32_t) g->ID || ih < 0 || ih >= IH || hi <= lo) { for (uint32_t c = 0; c < Cin; c++) { hvx_splat_f32_u(rows + (size_t) c * KH * SR, 0.0f, Wb); } @@ -277,8 +327,8 @@ static void conv2d_hmx_stage(const struct htp_ops_context * octx, const struct h continue; } - const dma_addr_t src = input->data + (size_t) n * input->nb[3] + (size_t) ih * input->nb[1] + (size_t) (iw0 + lo) * sizeof(float); - dma_queue_push(dmaq, dma_make_data(rows + lo, src), (size_t) KH * SR * sizeof(float), input->nb[2], + const dma_addr_t src = input->data + (size_t) n * g->x_nb_n + (size_t) id * g->x_nb_d + (size_t) ih * input->nb[1] + (size_t) (iw0 + lo) * sizeof(float); + dma_queue_push(dmaq, dma_make_data(rows + lo, src), (size_t) KH * SR * sizeof(float), g->x_nb_c, (size_t) (hi - lo) * sizeof(float), Cin); if (lo > 0 || hi < Wb) { @@ -295,9 +345,24 @@ static void conv2d_hmx_stage(const struct htp_ops_context * octx, const struct h } } +static inline HVX_Vector conv2d_hmx_load_strided_f32(const float * src, uint32_t s0, HVX_Vector offsets, HVX_Vector * scratch) { + if (s0 == 1) { + return hvx_vmemu(src); + } + if (s0 == 2) { + const HVX_Vector lo = hvx_vmemu(src); + const HVX_Vector hi = hvx_vmemu(src + 32); + return Q6_V_lo_W(Q6_W_vdeal_VVR(hi, lo, -4)); + } + Q6_vgather_ARMVw(scratch, (size_t) src, 32 * s0 * sizeof(float) - 1, offsets); + hvx_gather_sync(scratch); + return *scratch; +} + // One weight-side tile column: vector i holds k rows 2i and 2i+1 interleaved over 32 pixels static inline void conv2d_hmx_pack_tile(__fp16 * restrict dst, const float * restrict src, uint32_t K, uint32_t KW, - uint32_t d0, uint32_t SR, uint32_t n_vec) { + uint32_t s0, uint32_t d0, uint32_t SR, uint32_t n_vec, HVX_Vector offsets, + HVX_Vector * scratch) { HVX_Vector * restrict out = (HVX_Vector *) dst; const HVX_Vector zero = Q6_V_vzero(); const uint32_t wrap = SR - KW * d0; @@ -307,13 +372,13 @@ static inline void conv2d_hmx_pack_tile(__fp16 * restrict dst, const float * res uint32_t i = 0; for (; i < K / 2; i++) { - const HVX_Vector a = hvx_vmemu(p); + const HVX_Vector a = conv2d_hmx_load_strided_f32(p, s0, offsets, scratch); p += d0; if (++kw == KW) { kw = 0; p += wrap; } - const HVX_Vector b = hvx_vmemu(p); + const HVX_Vector b = conv2d_hmx_load_strided_f32(p, s0, offsets, scratch); p += d0; if (++kw == KW) { kw = 0; @@ -322,14 +387,14 @@ static inline void conv2d_hmx_pack_tile(__fp16 * restrict dst, const float * res out[i] = conv2d_f32_to_f16_shuff(a, b); } if (K & 1) { - out[i++] = conv2d_f32_to_f16_shuff(hvx_vmemu(p), zero); + out[i++] = conv2d_f32_to_f16_shuff(conv2d_hmx_load_strided_f32(p, s0, offsets, scratch), zero); } for (; i < n_vec; i++) { out[i] = zero; } } -// CWHN: stage[kh*SR + j*Cin + cin] = x[n][ih(kh)][ow0 - p0 + j][cin], zero outside +// CWHN: stage[kh*SR + j*Cin + cin] = x[n][ih(kh)][ow0*s0 - p0 + j][cin], zero outside static void conv2d_hmx_stage_cwhn(const struct htp_ops_context * octx, const struct htp_conv2d_kernel_params * kp, dma_queue * dmaq, float * stage, uint32_t n, uint32_t oh, uint32_t ow0, uint32_t G) { const struct htp_tensor * kernel = octx->src[0]; @@ -340,6 +405,7 @@ static void conv2d_hmx_stage_cwhn(const struct htp_ops_context * octx, const str const uint32_t Cin = kernel->ne[2]; const int32_t IW = input->ne[0]; const int32_t IH = input->ne[1]; + const int32_t s0 = octx->op_params[0]; const int32_t s1 = octx->op_params[1]; const int32_t p0 = octx->op_params[2]; const int32_t p1 = octx->op_params[3]; @@ -347,8 +413,8 @@ static void conv2d_hmx_stage_cwhn(const struct htp_ops_context * octx, const str const int32_t d1 = octx->op_params[5]; const uint32_t SR = kp->tiled.stage_row; - const int32_t Wb = 32 * G + (KW - 1) * d0; - const int32_t iw0 = (int32_t) ow0 - p0; + const int32_t Wb = 32 * G * s0 + (KW - 1) * d0; + const int32_t iw0 = (int32_t) ow0 * s0 - p0; const int32_t lo = MAX(0, -iw0); const int32_t hi = MIN(Wb, IW - iw0); @@ -383,8 +449,8 @@ static void conv2d_hmx_cwhn_f16(__fp16 * restrict dst, const float * restrict sr // Stage the segment of col tiles starting at t, returns its number of col tiles static uint32_t conv2d_hmx_issue_seg(const struct htp_ops_context * octx, const struct htp_conv2d_kernel_params * kp, - dma_queue * dmaq, float * stage, uint32_t t, uint32_t t1) { - const uint32_t OH = octx->dst->ne[1]; + const struct htp_conv_geom * g, dma_queue * dmaq, float * stage, uint32_t t, uint32_t t1) { + const uint32_t OH = g->OH; const uint32_t n_owt = kp->tiled.n_ow_tiles; const uint32_t row = fastdiv(t, &kp->tiled.div_ow_tiles); const uint32_t owt = t - row * n_owt; @@ -395,7 +461,7 @@ static uint32_t conv2d_hmx_issue_seg(const struct htp_ops_context * octx, const if (kp->cwhn) { conv2d_hmx_stage_cwhn(octx, kp, dmaq, stage, n, oh, owt * 32, G); } else { - conv2d_hmx_stage(octx, kp, dmaq, stage, n, oh, owt * 32, G); + conv2d_hmx_stage(octx, kp, g, dmaq, stage, n, oh, owt * 32, G); } return G; } @@ -487,7 +553,7 @@ static inline void conv2d_transpose_strided(const __fp16 * restrict base, uint32 // the second one rotated into the upper half. static void conv2d_hmx_pack_tile_cwhn(__fp16 * restrict dst, const __fp16 * restrict src, HVX_Vector * restrict vin, HVX_Vector * restrict scratch, const struct htp_conv2d_kernel_params * kp, - uint32_t gpr, uint32_t Cin, uint32_t d0) { + uint32_t gpr, uint32_t Cin, uint32_t s0, uint32_t d0) { const uint32_t k_pad = kp->tiled.n_k_tiles * HMX_FP16_TILE_N_COLS; const uint32_t KG = kp->tiled.k_group; const uint32_t KGp = kp->tiled.k_group_pad; @@ -523,15 +589,15 @@ static void conv2d_hmx_pack_tile_cwhn(__fp16 * restrict dst, const __fp16 * rest } if (va[0] == 32 && va[1] == 32 && pa[1] == pa[0] + 32) { - conv2d_transpose_strided(pa[0], Cin, scratch, out + 32 * b, out + 32 * b + 16); + conv2d_transpose_strided(pa[0], Cin * s0, scratch, out + 32 * b, out + 32 * b + 16); continue; } const HVX_VectorPred q0 = Q6_Q_vsetq_R(va[0] * sizeof(__fp16)); const HVX_VectorPred q1 = Q6_Q_vsetq_R(va[1] * sizeof(__fp16)); for (uint32_t j = 0; j < 32; j++) { - const HVX_Vector v0 = Q6_V_vand_QV(q0, hvx_vmemu(pa[0] + j * Cin)); - const HVX_Vector v1 = Q6_V_vand_QV(q1, hvx_vmemu(pa[1] + j * Cin)); + const HVX_Vector v0 = Q6_V_vand_QV(q0, hvx_vmemu(pa[0] + j * Cin * s0)); + const HVX_Vector v1 = Q6_V_vand_QV(q1, hvx_vmemu(pa[1] + j * Cin * s0)); vin[j] = Q6_V_vor_VV(v0, Q6_V_vror_VR(v1, 64)); } @@ -638,7 +704,7 @@ static inline bool conv2d_wpack_kmajor_next_task(struct conv2d_wpack_kmajor_iter } // Kernel stored as [k][cout] (cout contiguous): DMA k rows of 64 cout, transpose into two row tiles -static void conv2d_hmx_wpack_kmajor_thread(unsigned int nth, unsigned int ith, void * data) { +static void conv2d_hmx_wpack_kmajor_thread(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { const struct htp_conv2d_hmx_context * hc = (const struct htp_conv2d_hmx_context *) data; struct htp_ops_context * octx = hc->octx; const struct htp_conv2d_kernel_params * kp = hc->kp; @@ -655,7 +721,7 @@ static void conv2d_hmx_wpack_kmajor_thread(unsigned int nth, unsigned int ith, v const uint32_t es = is_f16 ? sizeof(__fp16) : sizeof(float); const uint32_t n_cb = (hc->n_rt + 1) / 2; - dma_queue * dmaq = octx->ctx->dma[ith]; + dma_queue * dmaq = th->dma_q; const size_t w_raw_size = (size_t) WK * 64 * sizeof(float); uint8_t * wraws[2] = { hc->thr_base + (size_t) ith * kp->thr_size, hc->thr_base + (size_t) ith * kp->thr_size + w_raw_size }; @@ -663,7 +729,7 @@ static void conv2d_hmx_wpack_kmajor_thread(unsigned int nth, unsigned int ith, v HVX_Vector * scratch = vin + 32; HVX_Vector * sink = scratch + 32; - struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + struct htp_thread_trace * tr = th->trace; struct conv2d_wpack_kmajor_iter it = { .cb = 0, @@ -712,18 +778,19 @@ static void conv2d_hmx_wpack_kmajor_thread(unsigned int nth, unsigned int ith, v } } -static void conv2d_hmx_pack(const struct htp_conv2d_hmx_context * hc, uint32_t ith) { +static void conv2d_hmx_pack(const struct htp_conv2d_hmx_context * hc, const htp_thread_t * th, uint32_t ith) { struct htp_ops_context * octx = hc->octx; const struct htp_conv2d_kernel_params * kp = hc->kp; const struct htp_tensor * kernel = octx->src[0]; const uint32_t KW = kernel->ne[0]; - const uint32_t KH = kernel->ne[1]; - const uint32_t Cin = kernel->ne[2]; - const uint32_t d0 = octx->op_params[4]; + const uint32_t KH = hc->g->KH; + const uint32_t Cin = hc->g->Cin; + const uint32_t s0 = (uint32_t) hc->g->s0; + const uint32_t d0 = (uint32_t) hc->g->d0; const uint32_t n_vec = kp->tiled.n_k_tiles * (HMX_FP16_TILE_N_ELMS / 64); - dma_queue * dmaq = octx->ctx->dma[ith]; + dma_queue * dmaq = th->dma_q; uint8_t * thr = hc->thr_base + (size_t) ith * kp->thr_size; float * stages[2] = { (float *) thr, @@ -741,16 +808,20 @@ static void conv2d_hmx_pack(const struct htp_conv2d_hmx_context * hc, uint32_t i uint32_t t0, t1; const uint32_t cs = conv2d_hmx_chunk_range(hc, hc->pack_chunk, hc->pack_tpt, ith, &t0, &t1); - struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + struct htp_thread_trace * tr = th->trace; if (t0 >= t1) { return; } + // Reuse output staging after unpack has drained its DMA. + HVX_Vector * gather_tmp = (HVX_Vector *) (thr + (size_t) n_stage * kp->stage_size); + const HVX_Vector offsets = kp->cwhn || s0 <= 2 ? Q6_V_vzero() : hvx_vec_gather_offsets_w(s0 * sizeof(float)); + uint32_t ng[2] = { 0, 0 }; uint32_t tq = t0; for (uint32_t i = 0; i < n_stage && tq < t1; i++) { - ng[i] = conv2d_hmx_issue_seg(octx, kp, dmaq, stages[i], tq, t1); + ng[i] = conv2d_hmx_issue_seg(octx, kp, hc->g, dmaq, stages[i], tq, t1); tq += ng[i]; } @@ -765,22 +836,29 @@ static void conv2d_hmx_pack(const struct htp_conv2d_hmx_context * hc, uint32_t i htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_A_PREP, (uint16_t) t); if (kp->cwhn) { - const uint32_t Wb = 32 * G + (KW - 1) * d0; + const uint32_t Wb = 32 * G * s0 + (KW - 1) * d0; conv2d_hmx_cwhn_f16(stage16, stage, KH, kp->tiled.stage_row, Wb * Cin); for (uint32_t g = 0; g < G; g++) { __fp16 * tile = act + (size_t) (t - cs + g) * n_vec * 64; - conv2d_hmx_pack_tile_cwhn(tile, stage16 + (size_t) g * 32 * Cin, vin, scratch, kp, gpr, Cin, d0); + conv2d_hmx_pack_tile_cwhn(tile, stage16 + (size_t) g * 32 * Cin * s0, vin, scratch, kp, gpr, Cin, s0, d0); } } else { for (uint32_t g = 0; g < G; g++) { - __fp16 * tile = act + (size_t) (t - cs + g) * n_vec * 64; - conv2d_hmx_pack_tile(tile, stage + g * 32, kp->tiled.k_size, KW, d0, kp->tiled.stage_row, n_vec); + __fp16 * tile = act + (size_t) (t - cs + g) * n_vec * 64; + const float * pixels = stage + (size_t) g * 32 * s0; + if (s0 == 1) { + conv2d_hmx_pack_tile(tile, pixels, kp->tiled.k_size, KW, 1, d0, kp->tiled.stage_row, n_vec, offsets, gather_tmp); + } else if (s0 == 2) { + conv2d_hmx_pack_tile(tile, pixels, kp->tiled.k_size, KW, 2, d0, kp->tiled.stage_row, n_vec, offsets, gather_tmp); + } else { + conv2d_hmx_pack_tile(tile, pixels, kp->tiled.k_size, KW, s0, d0, kp->tiled.stage_row, n_vec, offsets, gather_tmp); + } } } htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_A_PREP, (uint16_t) t); if (tq < t1) { - ng[bi] = conv2d_hmx_issue_seg(octx, kp, dmaq, stage, tq, t1); + ng[bi] = conv2d_hmx_issue_seg(octx, kp, hc->g, dmaq, stage, tq, t1); tq += ng[bi]; } t += G; @@ -788,18 +866,18 @@ static void conv2d_hmx_pack(const struct htp_conv2d_hmx_context * hc, uint32_t i } } -static void conv2d_hmx_unpack(const struct htp_conv2d_hmx_context * hc, uint32_t ith) { +static void conv2d_hmx_unpack(const struct htp_conv2d_hmx_context * hc, const htp_thread_t * th, uint32_t ith) { struct htp_ops_context * octx = hc->octx; const struct htp_conv2d_kernel_params * kp = hc->kp; const struct htp_tensor * dst = octx->dst; const uint32_t OW = dst->ne[0]; - const uint32_t OH = dst->ne[1]; - const uint32_t Cout = dst->ne[2]; + const uint32_t OH = hc->g->OH; + const uint32_t Cout = hc->g->Cout; const uint32_t n_owt = kp->tiled.n_ow_tiles; const uint32_t OS = kp->tiled.seg_tiles * 32; - dma_queue * dmaq = octx->ctx->dma[ith]; + dma_queue * dmaq = th->dma_q; uint8_t * thr = hc->thr_base + (size_t) ith * kp->thr_size + (size_t) (kp->tiled.n_stage > 1 ? 2 : 1) * kp->stage_size; float * ostage[2] = { (float *) thr, @@ -811,7 +889,7 @@ static void conv2d_hmx_unpack(const struct htp_conv2d_hmx_context * hc, uint32_t const uint32_t cs = conv2d_hmx_chunk_range(hc, hc->unpack_chunk, hc->unpack_tpt, ith, &t0, &t1); const uint32_t nct = MIN(cs + kp->ct_chunk, hc->t_end) - cs; - struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + struct htp_thread_trace * tr = th->trace; if (t0 >= t1) { return; @@ -850,8 +928,8 @@ static void conv2d_hmx_unpack(const struct htp_conv2d_hmx_context * hc, uint32_t } htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_O_PROC, (uint16_t) t); - const dma_addr_t ddr = dst->data + (size_t) n * dst->nb[3] + (size_t) oh * dst->nb[1] + (size_t) co0 * dst->nb[2] + (size_t) ow0 * sizeof(float); - dma_queue_push(dmaq, dma_make_data(ddr, ob), dst->nb[2], (size_t) OS * sizeof(float), (size_t) nv * sizeof(float), nco); + const dma_addr_t ddr = dst->data + (size_t) n * hc->g->y_nb_n + (size_t) oh * dst->nb[1] + (size_t) co0 * hc->g->y_nb_c + (size_t) ow0 * sizeof(float); + dma_queue_push(dmaq, dma_make_data(ddr, ob), hc->g->y_nb_c, (size_t) OS * sizeof(float), (size_t) nv * sizeof(float), nco); n_pushed++; b ^= 1; } @@ -867,11 +945,11 @@ static void conv2d_hmx_unpack(const struct htp_conv2d_hmx_context * hc, uint32_t } } -static void conv2d_hmx_phase_thread(unsigned int nth, unsigned int ith, void * data) { +static void conv2d_hmx_phase_thread(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { const struct htp_conv2d_hmx_context * hc = (const struct htp_conv2d_hmx_context *) data; if (hc->unpack_chunk >= 0) { - conv2d_hmx_unpack(hc, ith); + conv2d_hmx_unpack(hc, th, ith); } // weights of this rt block are packed before its first chunk if (hc->pack_chunk == 0) { @@ -882,7 +960,7 @@ static void conv2d_hmx_phase_thread(unsigned int nth, unsigned int ith, void * d } } if (hc->pack_chunk >= 0) { - conv2d_hmx_pack(hc, ith); + conv2d_hmx_pack(hc, th, ith); } } @@ -900,12 +978,15 @@ static void conv2d_hmx_run_phase(struct htp_conv2d_hmx_context * hc, int32_t pac hc->unpack_chunk = unpack; hc->pack_tpt = conv2d_hmx_tpt(hc, pack, nth); hc->unpack_tpt = conv2d_hmx_tpt(hc, unpack, nth); - work_queue_run(hc->octx->ctx->work_queue, conv2d_hmx_phase_thread, hc, nth); + work_queue_run(hc->octx->ctx->hvx_queue, conv2d_hmx_phase_thread, hc, nth); } static int conv2d_hmx_tiled(struct htp_ops_context * octx, const struct htp_conv2d_kernel_params * kp) { + struct htp_conv_geom g; + conv_geom_init(&g, octx); + uint32_t row_base, n_rows; - conv2d_mdev_rows(octx, &row_base, &n_rows); + conv2d_mdev_rows(octx, g.N * g.OH, &row_base, &n_rows); if (n_rows == 0) { return HTP_STATUS_OK; } @@ -916,13 +997,14 @@ static int conv2d_hmx_tiled(struct htp_ops_context * octx, const struct htp_conv const size_t out_size = (size_t) kp->rt_block * kp->ct_chunk * tile; struct htp_conv2d_vtcm_layout layout; - htp_conv2d_vtcm_layout_build(&layout, kp->rt_block, w_size, act_size, out_size, kp->thr_size, kp->n_threads); + htp_conv2d_vtcm_layout_build(&layout, kp->rt_block, w_size, act_size, out_size, kp->thr_size, kp->n_hvx); uint8_t * base = (uint8_t *) octx->ctx->vtcm_base; struct htp_conv2d_hmx_context hc = { .octx = octx, .kp = kp, + .g = &g, .layout = layout, .scales = VTCM_LAYOUT_PTR(__fp16, base, layout.off_scales), .w_tiles = VTCM_LAYOUT_PTR(__fp16, base, layout.off_w), @@ -933,7 +1015,7 @@ static int conv2d_hmx_tiled(struct htp_ops_context * octx, const struct htp_conv .t_end = (row_base + n_rows) * kp->tiled.n_ow_tiles, }; - const uint32_t nth = kp->n_threads; + const uint32_t nth = kp->n_hvx; const int32_t n_chunks = (int32_t) ((hc.t_end - hc.t_base + kp->ct_chunk - 1) / kp->ct_chunk); struct htp_conv2d_hmx_task task; @@ -941,7 +1023,7 @@ static int conv2d_hmx_tiled(struct htp_ops_context * octx, const struct htp_conv for (uint32_t rt0 = 0; rt0 < kp->n_row_tiles; rt0 += kp->rt_block) { hc.rt0 = rt0; hc.n_rt = MIN(kp->rt_block, kp->n_row_tiles - rt0); - work_queue_run(octx->ctx->work_queue, kp->w_kmajor ? conv2d_hmx_wpack_kmajor_thread : conv2d_hmx_wpack_thread, &hc, nth); + work_queue_run(octx->ctx->hvx_queue, kp->w_kmajor ? conv2d_hmx_wpack_kmajor_thread : conv2d_hmx_wpack_thread, &hc, nth); conv2d_hmx_run_phase(&hc, 0, -1, nth); @@ -1028,7 +1110,7 @@ static inline uint32_t conv2d_hmx_win_act_off(uint32_t u) { return ((u >> 5) << 11) | ((u & 30) << 6) | ((u & 1) << 1); } -static void conv2d_hmx_win_worker(void * data) { +static void conv2d_hmx_win_worker(const htp_thread_t * th, void * data) { const struct htp_conv2d_win_task * j = (const struct htp_conv2d_win_task *) data; const uint32_t * taps = j->taps; @@ -1245,7 +1327,7 @@ static inline void conv2d_hmx_win_wpack_kmajor_compute(const struct htp_conv2d_w } // Cout-contiguous kernel: DMA w[t][c][32 cout] rows, tile vector p = (row 2p, row 2p+1) interleaved per cout -static void conv2d_hmx_win_wpack_kmajor(const struct htp_conv2d_win_context * wc, unsigned int nth, unsigned int ith) { +static void conv2d_hmx_win_wpack_kmajor(const struct htp_conv2d_win_context * wc, const htp_thread_t * th, unsigned int nth, unsigned int ith) { struct htp_ops_context * octx = wc->octx; const struct htp_conv2d_kernel_params * kp = wc->kp; const struct htp_tensor * kernel = octx->src[0]; @@ -1260,13 +1342,13 @@ static void conv2d_hmx_win_wpack_kmajor(const struct htp_conv2d_win_context * wc const bool is_f16 = kernel->type == HTP_TYPE_F16; const uint32_t es = is_f16 ? sizeof(__fp16) : sizeof(float); - dma_queue * dmaq = octx->ctx->dma[ith]; + dma_queue * dmaq = th->dma_q; const size_t w_raw_size = (size_t) TG * 32 * VLEN; uint8_t * raws[2] = { wc->thr_base + (size_t) ith * kp->thr_size, wc->thr_base + (size_t) ith * kp->thr_size + w_raw_size }; const HVX_Vector zero = Q6_V_vzero(); - struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + struct htp_thread_trace * tr = th->trace; const uint32_t n_tg = (T + TG - 1) / TG; const uint32_t n_units = wc->n_rt * n_cc * n_tg; @@ -1409,7 +1491,7 @@ static void conv2d_hmx_win_wrows_pack(const struct htp_conv2d_win_context * wc, } } -static void conv2d_hmx_win_wpack_rows(const struct htp_conv2d_win_context * wc, unsigned int nth, unsigned int ith) { +static void conv2d_hmx_win_wpack_rows(const struct htp_conv2d_win_context * wc, const htp_thread_t * th, unsigned int nth, unsigned int ith) { struct htp_ops_context * octx = wc->octx; const struct htp_conv2d_kernel_params * kp = wc->kp; const struct htp_tensor * kernel = octx->src[0]; @@ -1419,7 +1501,7 @@ static void conv2d_hmx_win_wpack_rows(const struct htp_conv2d_win_context * wc, const uint32_t kpw = kernel->type == HTP_TYPE_F16 ? 64 : 32; // k per vector const size_t wraw = (size_t) 32 * ((CS * T + kpw - 1) / kpw) * VLEN; - dma_queue * dmaq = octx->ctx->dma[ith]; + dma_queue * dmaq = th->dma_q; uint8_t * raws[2] = { wc->thr_base + (size_t) ith * kp->thr_size, wc->thr_base + (size_t) ith * kp->thr_size + wraw }; uint8_t * xr = raws[1] + wraw; HVX_Vector * vin = (HVX_Vector *) (xr + wraw); @@ -1428,7 +1510,7 @@ static void conv2d_hmx_win_wpack_rows(const struct htp_conv2d_win_context * wc, const uint32_t n_cs = (32 + CS - 1) / CS; const uint32_t n_units = wc->n_rt * kp->window.n_cc * n_cs; - struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + struct htp_thread_trace * tr = th->trace; // one DMA per unit: push(N), push(N + 1), pop(N), pack(N), push(N + 2), ... for (uint32_t k = 0, unit = ith; k < 2 && unit < n_units; k++, unit += nth) { @@ -1449,13 +1531,13 @@ static void conv2d_hmx_win_wpack_rows(const struct htp_conv2d_win_context * wc, } } -static void conv2d_hmx_win_wpack_thread(unsigned int nth, unsigned int ith, void * data) { +static void conv2d_hmx_win_wpack_thread(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { const struct htp_conv2d_win_context * wc = (const struct htp_conv2d_win_context *) data; if (wc->kp->w_kmajor) { - conv2d_hmx_win_wpack_kmajor(wc, nth, ith); + conv2d_hmx_win_wpack_kmajor(wc, th, nth, ith); } else { - conv2d_hmx_win_wpack_rows(wc, nth, ith); + conv2d_hmx_win_wpack_rows(wc, th, nth, ith); } } @@ -1901,7 +1983,7 @@ static void conv2d_hmx_win_croutons_fold_cwhn(const struct htp_conv2d_win_contex } } -static void conv2d_hmx_win_pack(const struct htp_conv2d_win_context * wc, unsigned int nth, unsigned int ith) { +static void conv2d_hmx_win_pack(const struct htp_conv2d_win_context * wc, const htp_thread_t * th, unsigned int nth, unsigned int ith) { struct htp_ops_context * octx = wc->octx; const struct htp_conv2d_kernel_params * kp = wc->kp; @@ -1912,7 +1994,7 @@ static void conv2d_hmx_win_pack(const struct htp_conv2d_win_context * wc, unsign const uint32_t b0 = MIN(ith * bpt, ncb); const uint32_t b1 = MIN(b0 + bpt, ncb); - dma_queue * dmaq = octx->ctx->dma[ith]; + dma_queue * dmaq = th->dma_q; uint8_t * thr = wc->thr_base + (size_t) ith * kp->thr_size; float * stages[2] = { (float *) thr, (float *) (thr + kp->stage_size) }; __fp16 * act = wc->act[wc->pack_chunk & 1]; @@ -1920,7 +2002,7 @@ static void conv2d_hmx_win_pack(const struct htp_conv2d_win_context * wc, unsign uint32_t (*stage_fn)(const struct htp_conv2d_win_context *, dma_queue *, float *, uint32_t, uint32_t) = kp->window.fold ? conv2d_hmx_win_stage_fold : conv2d_hmx_win_stage; - struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + struct htp_thread_trace * tr = th->trace; const uint32_t SB = kp->window.seg_b; @@ -1959,7 +2041,7 @@ static void conv2d_hmx_win_pack(const struct htp_conv2d_win_context * wc, unsign } } -static void conv2d_hmx_win_unpack(const struct htp_conv2d_win_context * wc, unsigned int nth, unsigned int ith) { +static void conv2d_hmx_win_unpack(const struct htp_conv2d_win_context * wc, const htp_thread_t * th, unsigned int nth, unsigned int ith) { struct htp_ops_context * octx = wc->octx; const struct htp_conv2d_kernel_params * kp = wc->kp; const struct htp_tensor * dst = octx->dst; @@ -1978,12 +2060,12 @@ static void conv2d_hmx_win_unpack(const struct htp_conv2d_win_context * wc, unsi const uint32_t i0 = MIN(ith * tpt, nt); const uint32_t i1 = MIN(i0 + tpt, nt); - dma_queue * dmaq = octx->ctx->dma[ith]; + dma_queue * dmaq = th->dma_q; uint8_t * thr = wc->thr_base + (size_t) ith * kp->thr_size + 2 * (size_t) kp->stage_size; float * obuf = (float *) thr; __fp16 * out = wc->out[wc->unpack_chunk & 1]; - struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + struct htp_thread_trace * tr = th->trace; // nd[bi] = DMAs pushed from obuf bi, popped before it is reused HTP_CONV2D_WIN_OBUF steps later uint32_t nd[HTP_CONV2D_WIN_OBUF] = { 0 }; @@ -2072,11 +2154,11 @@ static void conv2d_hmx_win_unpack(const struct htp_conv2d_win_context * wc, unsi } // per-thread callback for one pipeline phase -static void conv2d_hmx_win_phase_thread(unsigned int nth, unsigned int ith, void * data) { +static void conv2d_hmx_win_phase_thread(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { const struct htp_conv2d_win_context * wc = (const struct htp_conv2d_win_context *) data; if (wc->unpack_chunk >= 0) { - conv2d_hmx_win_unpack(wc, nth, ith); + conv2d_hmx_win_unpack(wc, th, nth, ith); } // weights of this rt block are packed before its first chunk if (wc->pack_chunk == 0) { @@ -2088,12 +2170,12 @@ static void conv2d_hmx_win_phase_thread(unsigned int nth, unsigned int ith, void } } if (wc->pack_chunk >= 0) { - conv2d_hmx_win_pack(wc, nth, ith); + conv2d_hmx_win_pack(wc, th, nth, ith); } } // zero out 2 act buffer regions in VTCM before packing -static void conv2d_hmx_win_zero_thread(unsigned int nth, unsigned int ith, void * data) { +static void conv2d_hmx_win_zero_thread(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { const struct htp_conv2d_win_context * wc = (const struct htp_conv2d_win_context *) data; const size_t n = 2 * wc->layout.act_size / sizeof(float); @@ -2101,7 +2183,7 @@ static void conv2d_hmx_win_zero_thread(unsigned int nth, unsigned int ith, void const size_t i0 = MIN((size_t) ith * per, n); const size_t i1 = MIN(i0 + per, n); - struct htp_thread_trace * tr = &wc->octx->ctx->trace[ith]; + struct htp_thread_trace * tr = th->trace; const uint16_t block = (uint16_t) (i0 / VLEN_FP32); htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_A_PREP, block); if (i1 > i0) { @@ -2113,7 +2195,7 @@ static void conv2d_hmx_win_zero_thread(unsigned int nth, unsigned int ith, void static void conv2d_hmx_win_run_phase(struct htp_conv2d_win_context * wc, int32_t pack, int32_t unpack) { wc->pack_chunk = pack; wc->unpack_chunk = unpack; - work_queue_run(wc->octx->ctx->work_queue, conv2d_hmx_win_phase_thread, wc, wc->kp->n_threads); + work_queue_run(wc->octx->ctx->hvx_queue, conv2d_hmx_win_phase_thread, wc, wc->kp->n_hvx); } @@ -2153,7 +2235,7 @@ static int conv2d_hmx_win(struct htp_ops_context * octx, const struct htp_conv2d const uint32_t d1 = octx->op_params[5]; uint32_t row_base, n_rows; - conv2d_mdev_rows(octx, &row_base, &n_rows); + conv2d_mdev_rows(octx, dst->ne[3] * dst->ne[1], &row_base, &n_rows); if (n_rows == 0) { return HTP_STATUS_OK; } @@ -2170,7 +2252,7 @@ static int conv2d_hmx_win(struct htp_ops_context * octx, const struct htp_conv2d const size_t out_size = (size_t) kp->rt_block * kp->ct_chunk * tile; struct htp_conv2d_vtcm_layout layout; - htp_conv2d_vtcm_layout_build(&layout, kp->rt_block, w_size, act_size, out_size, kp->thr_size, kp->n_threads); + htp_conv2d_vtcm_layout_build(&layout, kp->rt_block, w_size, act_size, out_size, kp->thr_size, kp->n_hvx); uint8_t * base = (uint8_t *) octx->ctx->vtcm_base; @@ -2193,7 +2275,7 @@ static int conv2d_hmx_win(struct htp_ops_context * octx, const struct htp_conv2d const uint32_t n_ch = octx->src[1]->ne[2] * (kp->window.fold ? KW : 1); if (!kp->cwhn && n_ch <= 2 * CONV2D_WIN_SCAT_MAX) { wctx.n_scat = (n_ch + 1) / 2; - work_queue_run(octx->ctx->work_queue, conv2d_hmx_win_zero_thread, &wctx, kp->n_threads); + work_queue_run(octx->ctx->hvx_queue, conv2d_hmx_win_zero_thread, &wctx, kp->n_hvx); } const int32_t n_chunks = (int32_t) ((wctx.t_end - wctx.t_base + kp->ct_chunk - 1) / kp->ct_chunk); @@ -2225,7 +2307,7 @@ static int conv2d_hmx_win(struct htp_ops_context * octx, const struct htp_conv2d for (uint32_t rt0 = 0; rt0 < kp->n_row_tiles; rt0 += kp->rt_block) { wctx.rt0 = rt0; wctx.n_rt = MIN(kp->rt_block, kp->n_row_tiles - rt0); - work_queue_run(octx->ctx->work_queue, conv2d_hmx_win_wpack_thread, &wctx, kp->n_threads); + work_queue_run(octx->ctx->hvx_queue, conv2d_hmx_win_wpack_thread, &wctx, kp->n_hvx); conv2d_hmx_win_run_phase(&wctx, 0, -1); @@ -3021,10 +3103,10 @@ struct conv2d_dw_dma_context { conv2d_dw_compute_fn_t compute_fn; uint32_t task_start; uint32_t task_count; - uint32_t tasks_per_thread; + uint32_t tasks_per_hvx; }; -static void conv2d_dw_dma_thread(unsigned int nth, unsigned int ith, void * data) { +static void conv2d_dw_dma_thread(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { const struct conv2d_dw_dma_context * ctx = (const struct conv2d_dw_dma_context *) data; const struct htp_ops_context * octx = ctx->octx; const struct htp_conv2d_dw_kernel_params * kp = ctx->kp; @@ -3033,24 +3115,24 @@ static void conv2d_dw_dma_thread(unsigned int nth, unsigned int ith, void * data const struct htp_tensor * dst = octx->dst; conv2d_dw_compute_fn_t compute_fn = ctx->compute_fn; - const uint32_t first = ctx->task_start + ith * ctx->tasks_per_thread; - const uint32_t last = MIN(first + ctx->tasks_per_thread, ctx->task_start + ctx->task_count); + const uint32_t first = ctx->task_start + ith * ctx->tasks_per_hvx; + const uint32_t last = MIN(first + ctx->tasks_per_hvx, ctx->task_start + ctx->task_count); if (first >= last) { return; } (void) nth; - struct htp_thread_trace * tr = &octx->ctx->trace[ith]; - dma_queue * dma_q = octx->ctx->dma[ith]; + struct htp_thread_trace * tr = th->trace; + dma_queue * dma_q = th->dma_q; const uint32_t src_spad_half = (uint32_t) L->src_spad_half_size; const uint32_t dst_spad_half = (uint32_t) L->dst_spad_half_size; const uint32_t pad_top_stride = kp->pad_top_stride; uint8_t * vtcm_base = (uint8_t *) octx->ctx->vtcm_base; - uint8_t * src_spad = VTCM_LAYOUT_PTR(uint8_t, vtcm_base, L->off_src) + ith * L->src_bytes_per_thread; - uint8_t * dst_spad = VTCM_LAYOUT_PTR(uint8_t, vtcm_base, L->off_dst) + ith * L->dst_bytes_per_thread; + uint8_t * src_spad = VTCM_LAYOUT_PTR(uint8_t, vtcm_base, L->off_src) + ith * L->src_bytes_per_hvx; + uint8_t * dst_spad = VTCM_LAYOUT_PTR(uint8_t, vtcm_base, L->off_dst) + ith * L->dst_bytes_per_hvx; float * srcb2[2] = { (float *) src_spad, (float *) (src_spad + src_spad_half) }; float * dstb2[2] = { (float *) dst_spad, (float *) (dst_spad + dst_spad_half) }; @@ -3131,7 +3213,7 @@ int op_conv_2d_dw(struct htp_ops_context * octx) { const uint32_t w_size = src0->size; struct htp_conv2d_dw_vtcm_layout layout; - htp_conv2d_dw_vtcm_layout_build(&layout, max_src_bytes, max_dst_bytes, w_size, kp->n_threads); + htp_conv2d_dw_vtcm_layout_build(&layout, max_src_bytes, max_dst_bytes, w_size, kp->n_hvx); uint8_t * vtcm_base = (uint8_t *) octx->ctx->vtcm_base; void * w_vtcm = VTCM_LAYOUT_PTR(void, vtcm_base, layout.off_w); @@ -3158,8 +3240,8 @@ int op_conv_2d_dw(struct htp_ops_context * octx) { return HTP_STATUS_OK; } - const uint32_t nth = MIN(kp->n_threads, task_count); - const uint32_t tasks_per_thread = (task_count + nth - 1) / nth; + const uint32_t nth = MIN(kp->n_hvx, task_count); + const uint32_t tasks_per_hvx = (task_count + nth - 1) / nth; struct conv2d_dw_dma_context ctx = { .octx = octx, @@ -3169,9 +3251,9 @@ int op_conv_2d_dw(struct htp_ops_context * octx) { .compute_fn = conv2d_dw_kernels[kp->kernel_type][kp->is_f16 ? 1 : 0], .task_start = task_start, .task_count = task_count, - .tasks_per_thread = tasks_per_thread, + .tasks_per_hvx = tasks_per_hvx, }; - work_queue_run(octx->ctx->work_queue, conv2d_dw_dma_thread, &ctx, nth); + work_queue_run(octx->ctx->hvx_queue, conv2d_dw_dma_thread, &ctx, nth); return HTP_STATUS_OK; } diff --git a/ggml/src/ggml-hexagon/htp/conv2d-ops.h b/ggml/src/ggml-hexagon/htp/conv2d-ops.h index a48bf3e83ffe..29dfa812fa54 100644 --- a/ggml/src/ggml-hexagon/htp/conv2d-ops.h +++ b/ggml/src/ggml-hexagon/htp/conv2d-ops.h @@ -35,6 +35,8 @@ struct htp_conv2d_tiled_params { uint32_t wk_chunk; struct fastdiv_values div_ow_tiles; struct fastdiv_values div_oh; + struct fastdiv_values div_khr; // kernel ne[1], unfolds conv_3d kh = kd * KHr + kh' + struct fastdiv_values div_ohr; // dst ne[1], unfolds conv_3d oh = od * OHr + oh' }; struct htp_conv2d_window_params { @@ -58,7 +60,7 @@ struct htp_conv2d_window_params { struct htp_conv2d_kernel_params { uint8_t kernel_type; - uint8_t n_threads; + uint8_t n_hvx; uint8_t cwhn; uint8_t w_kmajor; @@ -96,7 +98,7 @@ enum htp_conv2d_dw_kernel_type { struct htp_conv2d_dw_kernel_params { uint8_t kernel_type; - uint8_t n_threads; + uint8_t n_hvx; uint8_t cwhn; uint8_t is_f16; @@ -138,8 +140,8 @@ struct htp_conv2d_dw_vtcm_layout { size_t off_dst; size_t src_spad_half_size; size_t dst_spad_half_size; - size_t src_bytes_per_thread; - size_t dst_bytes_per_thread; + size_t src_bytes_per_hvx; + size_t dst_bytes_per_hvx; }; static inline void htp_conv2d_dw_vtcm_layout_build( @@ -147,17 +149,17 @@ static inline void htp_conv2d_dw_vtcm_layout_build( uint32_t max_src_bytes, uint32_t max_dst_bytes, uint32_t w_size, - uint32_t n_threads + uint32_t n_hvx ) { L->src_spad_half_size = hex_round_up(max_src_bytes + 256, 128); L->dst_spad_half_size = hex_round_up(max_dst_bytes + 128, 128); - L->src_bytes_per_thread = 2 * L->src_spad_half_size; - L->dst_bytes_per_thread = 2 * L->dst_spad_half_size; + L->src_bytes_per_hvx = 2 * L->src_spad_half_size; + L->dst_bytes_per_hvx = 2 * L->dst_spad_half_size; size_t off = 0; VTCM_LAYOUT_ALLOC(off, off_w, hex_round_up(w_size, 128)); - VTCM_LAYOUT_ALLOC(off, off_src, L->src_bytes_per_thread * n_threads); - VTCM_LAYOUT_ALLOC(off, off_dst, L->dst_bytes_per_thread * n_threads); + VTCM_LAYOUT_ALLOC(off, off_src, L->src_bytes_per_hvx * n_hvx); + VTCM_LAYOUT_ALLOC(off, off_dst, L->dst_bytes_per_hvx * n_hvx); L->total_bytes = off; } @@ -179,7 +181,7 @@ static inline void htp_conv2d_vtcm_layout_build( size_t act_size, size_t out_size, size_t thr_size, - uint32_t n_threads + uint32_t n_hvx ) { const size_t sc = hex_round_up((size_t) rt_block * HTP_CONV2D_HMX_SCALES_SIZE, 2048); size_t off = 0; @@ -189,7 +191,7 @@ static inline void htp_conv2d_vtcm_layout_build( VTCM_LAYOUT_ALLOC(off, off_act[1], act_size); VTCM_LAYOUT_ALLOC(off, off_out[0], out_size); VTCM_LAYOUT_ALLOC(off, off_out[1], out_size); - VTCM_LAYOUT_ALLOC(off, off_thr, (size_t) n_threads * thr_size); + VTCM_LAYOUT_ALLOC(off, off_thr, (size_t) n_hvx * thr_size); L->act_size = act_size; L->total_bytes = off; } diff --git a/ggml/src/ggml-hexagon/htp/cpy-ops.c b/ggml/src/ggml-hexagon/htp/cpy-ops.c index fce45c2d4736..cdf4d8ec62eb 100644 --- a/ggml/src/ggml-hexagon/htp/cpy-ops.c +++ b/ggml/src/ggml-hexagon/htp/cpy-ops.c @@ -26,15 +26,19 @@ struct htp_copy_context { uint32_t row_start; uint32_t nrows; - uint32_t src0_nrows_per_thread; + uint32_t src0_nrows_per_hvx; uint32_t elem_start; uint32_t nelem; - uint32_t elem_per_thread; + uint32_t elem_per_hvx; struct htp_copy_convert_vtcm_layout layout; uint8_t * vtcm_src0; uint8_t * vtcm_dst; + + uint32_t tile_start; + uint32_t ntiles; + uint32_t tiles_per_hvx; }; #define cpy_preamble \ @@ -62,16 +66,17 @@ struct htp_copy_context { const uint32_t nb3 = dst->nb[3]; #define DEFINE_CPY_RESHAPE(NAME, ELEM_TYPE, ELEM_SIZE) \ -static void cpy_thread_##NAME##_reshape(unsigned int nth, unsigned int ith, void * data) { \ +static void cpy_thread_##NAME##_reshape(const htp_thread_t * th, unsigned int nth, unsigned int ith, \ + void * data) { \ struct htp_copy_context * ct = (struct htp_copy_context *) data; \ struct htp_ops_context * octx = ct->octx; \ cpy_preamble; \ - const uint32_t th_nelem = ct->elem_per_thread; \ + const uint32_t th_nelem = ct->elem_per_hvx; \ const uint32_t th_start = ct->elem_start + ith * th_nelem; \ const uint32_t th_end = MIN(th_start + th_nelem, ct->elem_start + ct->nelem); \ if (th_start >= th_end) return; \ \ - dma_queue * dma_q = octx->ctx->dma[ith]; \ + dma_queue * dma_q = th->dma_q; \ \ const uint32_t ne01_ne00 = ne01 * ne00; \ const uint32_t ne02_ne01_ne00 = ne02 * ne01_ne00; \ @@ -162,19 +167,185 @@ static inline void cpy_cursor_advance(struct cpy_row_cursor * c, uint32_t n, uin } } +struct cpy_tile_cursor { + uint32_t ibk; + uint32_t ib0; + uint32_t i2; + uint32_t i3; +}; + +static inline void cpy_tile_cursor_advance(struct cpy_tile_cursor * c, const struct htp_copy_transpose_params * tp) { + if (++c->ibk == tp->nkb) { + c->ibk = 0; + if (++c->ib0 == tp->n0b) { + c->ib0 = 0; + if (++c->i2 == tp->ne[2]) { + c->i2 = 0; + c->i3++; + } + } + } +} + +// Gather offsets for element e = l >> log2_m at (e % r, e / r) +static inline HVX_Vector cpy_transpose_offsets(const struct htp_copy_transpose_params * tp) { + const uint32_t r_mask = (1u << tp->log2_r) - 1; + if (tp->elem_size != 2) { + const uint32_t m_mask = (1u << tp->log2_m) - 1; + int32_t offsets[32] __attribute__((aligned(128))); + for (uint32_t l = 0; l < 32; l++) { + const uint32_t e = l >> tp->log2_m; + offsets[l] = (e & r_mask) * tp->in_stride + (e >> tp->log2_r) * tp->elem_size + (l & m_mask) * 4; + } + return *(HVX_Vector *) offsets; + } + int16_t offsets[64] __attribute__((aligned(128))); + for (uint32_t l = 0; l < 64; l++) { + offsets[l] = (l & r_mask) * tp->in_stride + (l >> tp->log2_r) * 2; + } + return *(HVX_Vector *) offsets; +} + +static inline void cpy_transpose_push_in(dma_queue * q, const struct htp_copy_transpose_params * tp, dma_addr_t src, + const struct cpy_tile_cursor * c, uint8_t * buf) { + const uint32_t a0 = c->ib0 * tp->blk0; + const uint32_t ak = c->ibk * tp->blkk; + const uint32_t off = a0 * tp->src_nb[0] + ak * tp->elem_size + c->i2 * tp->src_nb[1] + c->i3 * tp->src_nb[2]; + dma_queue_push(q, dma_make_data(buf, src + off), tp->in_stride, tp->src_nb[0], + MIN(tp->blkk, tp->ne[1] - ak) * tp->elem_size, MIN(tp->blk0, tp->ne[0] - a0)); +} + +// Pipelined tile worker: DMA in -> vgather transpose -> DMA out +static void cpy_thread_transpose(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { + struct htp_copy_context * ct = (struct htp_copy_context *) data; + struct htp_ops_context * octx = ct->octx; + const struct htp_copy_transpose_params * tp = &ct->kparams->u.transpose; + + const uint32_t t0 = ct->tile_start + ith * ct->tiles_per_hvx; + const uint32_t t1 = MIN(t0 + ct->tiles_per_hvx, ct->tile_start + ct->ntiles); + if (t0 >= t1) { + return; + } + + dma_queue * dma_q = th->dma_q; + struct htp_thread_trace * tr = th->trace; + + const dma_addr_t src = octx->src[0]->data; + const dma_addr_t dst = octx->dst->data; + + const uint32_t es = tp->elem_size; + const uint32_t ne0 = tp->ne[0]; + const uint32_t nek = tp->ne[1]; + const uint32_t in_stride = tp->in_stride; + const uint32_t out_stride = tp->out_stride; + + const HVX_Vector offsets = cpy_transpose_offsets(tp); + const uint32_t packed_mu = (ne0 - 1) * in_stride + 127; + const uint32_t packed_step = 128 >> tp->log2_r; + const uint32_t epv = 1u << tp->log2_r; + + uint8_t * in_buf[2]; + uint8_t * out_buf[2]; + in_buf[0] = ct->vtcm_src0 + ith * 2 * tp->in_buf_size; + in_buf[1] = in_buf[0] + tp->in_buf_size; + out_buf[0] = ct->vtcm_dst + ith * 2 * tp->out_buf_size; + out_buf[1] = out_buf[0] + tp->out_buf_size; + + struct cpy_tile_cursor cur; + { + const uint32_t q0 = fastdiv(t0, &tp->div_nkb); + const uint32_t q1 = fastdiv(q0, &tp->div_n0b); + cur.ibk = t0 - q0 * tp->nkb; + cur.ib0 = q0 - q1 * tp->n0b; + cur.i3 = fastdiv(q1, &tp->div_ne2); + cur.i2 = q1 - cur.i3 * tp->ne[2]; + } + + struct cpy_tile_cursor nxt = cur; + + cpy_transpose_push_in(dma_q, tp, src, &cur, in_buf[0]); + + for (uint32_t t = t0, i = 0; t < t1; t++, i++) { + while (dma_queue_depth(dma_q) > (i > 0 ? 1u : 0u)) { + dma_queue_pop(dma_q); + } + + uint8_t * in = in_buf[i & 1]; + uint8_t * out = out_buf[i & 1]; + + const uint32_t a0 = cur.ib0 * tp->blk0; + const uint32_t ak = cur.ibk * tp->blkk; + const uint32_t n0 = MIN(tp->blk0, ne0 - a0); + const uint32_t nk = MIN(tp->blkk, nek - ak); + + if (t + 1 < t1) { + cpy_tile_cursor_advance(&nxt, tp); + cpy_transpose_push_in(dma_q, tp, src, &nxt, in_buf[(i + 1) & 1]); + } + + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) t); + if (tp->packed) { + const uint32_t nvec = (nk * ne0 * es + 127) >> 7; + if (es != 2) { + #pragma unroll(4) + for (uint32_t v = 0; v < nvec; v++) { + Q6_vgather_ARMVw((HVX_Vector *) (out + v * 128), (size_t) (in + v * packed_step), packed_mu, offsets); + } + } else { + #pragma unroll(4) + for (uint32_t v = 0; v < nvec; v++) { + Q6_vgather_ARMVh((HVX_Vector *) (out + v * 128), (size_t) (in + v * packed_step), packed_mu, offsets); + } + } + } else { + for (uint32_t rb = 0; rb < n0; rb += epv) { + const uint32_t mu = (MIN(epv, n0 - rb) - 1) * in_stride + es - 1; + uint8_t * d = out + rb * es; + const uint8_t * s = in + rb * in_stride; + if (es != 2) { + #pragma unroll(4) + for (uint32_t ik = 0; ik < nk; ik++) { + Q6_vgather_ARMVw((HVX_Vector *) (d + ik * out_stride), (size_t) (s + ik * es), mu, offsets); + } + } else { + #pragma unroll(4) + for (uint32_t ik = 0; ik < nk; ik++) { + Q6_vgather_ARMVh((HVX_Vector *) (d + ik * out_stride), (size_t) (s + ik * 2), mu, offsets); + } + } + } + } + hvx_gather_sync(out); + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) t); + + const uint32_t doff = a0 * es + ak * tp->dst_nb[0] + cur.i2 * tp->dst_nb[1] + cur.i3 * tp->dst_nb[2]; + if (tp->packed) { + const uint32_t n = nk * ne0 * es; + dma_queue_push(dma_q, dma_make_data(dst + doff, out), n, n, n, 1); + } else { + dma_queue_push(dma_q, dma_make_data(dst + doff, out), tp->dst_nb[0], out_stride, n0 * es, nk); + } + + cur = nxt; + } + + dma_queue_flush(dma_q); +} + #define DEFINE_CPY_CONVERT_SAMESHAPE(NAME, CONV_FUNC) \ -static void cpy_thread_##NAME##_sameshape(unsigned int nth, unsigned int ith, void * data) { \ +static void cpy_thread_##NAME##_sameshape(const htp_thread_t * th, unsigned int nth, \ + unsigned int ith, void * data) { \ struct htp_copy_context * ct = (struct htp_copy_context *) data; \ struct htp_ops_context * octx = ct->octx; \ cpy_preamble; \ \ - const uint32_t dr = ct->src0_nrows_per_thread; \ + const uint32_t dr = ct->src0_nrows_per_hvx; \ const uint32_t ir0 = ct->row_start + dr * ith; \ const uint32_t ir1 = MIN(ir0 + dr, ct->row_start + ct->nrows); \ if (ir0 >= ir1) return; \ \ - dma_queue * dma_q = octx->ctx->dma[ith]; \ - struct htp_thread_trace * tr = &octx->ctx->trace[ith]; \ + dma_queue * dma_q = th->dma_q; \ + struct htp_thread_trace * tr = th->trace; \ \ const struct htp_copy_convert_params * cvt = &ct->kparams->u.convert; \ const struct htp_copy_convert_vtcm_layout * L = &ct->layout; \ @@ -183,8 +354,8 @@ static void cpy_thread_##NAME##_sameshape(unsigned int nth, unsigned int ith, vo const uint32_t src0_row_stride = L->src0_row_stride; \ const uint32_t dst_row_stride = L->dst_row_stride; \ const uint32_t blk_rows = L->blk_rows; \ - uint8_t * vtcm_src0_base = ct->vtcm_src0 + ith * L->spad0_size_per_thread; \ - uint8_t * vtcm_dst_base = ct->vtcm_dst + ith * L->spad1_size_per_thread; \ + uint8_t * vtcm_src0_base = ct->vtcm_src0 + ith * L->spad0_size_per_hvx; \ + uint8_t * vtcm_dst_base = ct->vtcm_dst + ith * L->spad1_size_per_hvx; \ const uint32_t src0_row_size = ne00 * ct->kparams->src0_type_size; \ const uint32_t dst_row_size = ne00 * ct->kparams->dst_type_size; \ \ @@ -341,7 +512,7 @@ static int cpy_sameshape_convert(struct htp_ops_context * octx, const struct htp const struct htp_tensor * src0 = octx->src[0]; const struct htp_tensor * dst = octx->dst; - if (!htp_ops_context_set_n_threads(octx, kparams->n_threads)) { + if (!htp_ops_context_set_n_hvx(octx, kparams->n_hvx)) { return HTP_STATUS_INVAL_PARAMS; } @@ -363,17 +534,17 @@ static int cpy_sameshape_convert(struct htp_ops_context * octx, const struct htp return HTP_STATUS_VTCM_TOO_SMALL; } - const uint32_t n_threads = octx->n_threads; + const uint32_t n_hvx = octx->n_hvx; const struct htp_copy_convert_params * cvt = &kparams->u.convert; struct htp_copy_context ct; - htp_copy_convert_vtcm_layout_build(&ct.layout, src0->ne[0], kparams->src0_type_size, kparams->dst_type_size, cvt->blk_rows, n_threads); + htp_copy_convert_vtcm_layout_build(&ct.layout, src0->ne[0], kparams->src0_type_size, kparams->dst_type_size, cvt->blk_rows, n_hvx); ct.octx = octx; ct.kparams = kparams; ct.row_start = row_start; ct.nrows = nrows; - ct.src0_nrows_per_thread = fastdiv(nrows + n_threads - 1, &octx->n_threads_div); + ct.src0_nrows_per_hvx = fastdiv(nrows + n_hvx - 1, &octx->n_hvx_div); uint8_t * vtcm_base = (uint8_t *) octx->ctx->vtcm_base; ct.vtcm_src0 = VTCM_LAYOUT_PTR(uint8_t, vtcm_base, ct.layout.off_src0); @@ -392,7 +563,7 @@ static int cpy_sameshape_convert(struct htp_ops_context * octx, const struct htp return HTP_STATUS_NO_SUPPORT; } - work_queue_run(octx->ctx->work_queue, copy_fun, &ct, n_threads); + work_queue_run(octx->ctx->hvx_queue, copy_fun, &ct, n_hvx); return HTP_STATUS_OK; } @@ -400,7 +571,7 @@ static int cpy_reshape(struct htp_ops_context * octx, const struct htp_copy_kern const struct htp_tensor * src0 = octx->src[0]; const struct htp_tensor * dst = octx->dst; - if (!htp_ops_context_set_n_threads(octx, kparams->n_threads)) { + if (!htp_ops_context_set_n_hvx(octx, kparams->n_hvx)) { return HTP_STATUS_INVAL_PARAMS; } @@ -418,14 +589,14 @@ static int cpy_reshape(struct htp_ops_context * octx, const struct htp_copy_kern return HTP_STATUS_OK; } - const uint32_t n_threads = octx->n_threads; + const uint32_t n_hvx = octx->n_hvx; struct htp_copy_context ct; ct.octx = octx; ct.kparams = kparams; ct.elem_start = elem_start; ct.nelem = nelem; - ct.elem_per_thread = fastdiv(nelem + n_threads - 1, &octx->n_threads_div); + ct.elem_per_hvx = fastdiv(nelem + n_hvx - 1, &octx->n_hvx_div); work_queue_func_t copy_fun = NULL; switch (src0->type) { @@ -435,7 +606,49 @@ static int cpy_reshape(struct htp_ops_context * octx, const struct htp_copy_kern default: return HTP_STATUS_NO_SUPPORT; } - work_queue_run(octx->ctx->work_queue, copy_fun, &ct, n_threads); + work_queue_run(octx->ctx->hvx_queue, copy_fun, &ct, n_hvx); + return HTP_STATUS_OK; +} + +static int cpy_transpose(struct htp_ops_context * octx, const struct htp_copy_kernel_params * kparams) { + if (!htp_ops_context_set_n_hvx(octx, kparams->n_hvx)) { + return HTP_STATUS_INVAL_PARAMS; + } + + const struct htp_copy_transpose_params * tp = &kparams->u.transpose; + + uint32_t tile_start = 0; + uint32_t ntiles = tp->total_tiles; + + if (octx->ctx->mdev.count > 1) { + const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition( + tp->total_tiles, 1, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div); + tile_start = range.start; + ntiles = range.count; + } + + if (ntiles == 0) { + return HTP_STATUS_OK; + } + + if (kparams->vtcm_size > octx->ctx->vtcm_size) { + return HTP_STATUS_VTCM_TOO_SMALL; + } + + const uint32_t n_hvx = octx->n_hvx; + + struct htp_copy_context ct; + ct.octx = octx; + ct.kparams = kparams; + ct.tile_start = tile_start; + ct.ntiles = ntiles; + ct.tiles_per_hvx = fastdiv(ntiles + n_hvx - 1, &octx->n_hvx_div); + + uint8_t * vtcm_base = (uint8_t *) octx->ctx->vtcm_base; + ct.vtcm_src0 = vtcm_base; + ct.vtcm_dst = vtcm_base + (size_t) n_hvx * 2 * tp->in_buf_size; + + work_queue_run(octx->ctx->hvx_queue, cpy_thread_transpose, &ct, n_hvx); return HTP_STATUS_OK; } @@ -459,8 +672,11 @@ int op_cpy(struct htp_ops_context * octx) { case HTP_COPY_KERNEL_RESHAPE: status = cpy_reshape(octx, kparams); break; + case HTP_COPY_KERNEL_TRANSPOSE: + status = cpy_transpose(octx, kparams); + break; case HTP_COPY_KERNEL_GATHER_ROWS: - status = htp_gather_rows(octx, octx->src[0], NULL, &kparams->u.gather, kparams->n_threads); + status = htp_gather_rows(octx, octx->src[0], NULL, &kparams->u.gather, kparams->n_hvx); break; default: status = HTP_STATUS_NO_SUPPORT; diff --git a/ggml/src/ggml-hexagon/htp/cpy-ops.h b/ggml/src/ggml-hexagon/htp/cpy-ops.h index c3b97c0cab4b..1ea50ca1d452 100644 --- a/ggml/src/ggml-hexagon/htp/cpy-ops.h +++ b/ggml/src/ggml-hexagon/htp/cpy-ops.h @@ -15,6 +15,7 @@ enum htp_copy_kernel_type { HTP_COPY_KERNEL_RESHAPE = 4, HTP_COPY_KERNEL_SCALAR = 5, HTP_COPY_KERNEL_GATHER_ROWS = 6, + HTP_COPY_KERNEL_TRANSPOSE = 7, }; struct htp_copy_convert_params { @@ -32,11 +33,34 @@ struct htp_copy_reshape_params { struct fastdiv_values div_ne02_ne01_ne00; }; +// Batched 2D transpose of elem_size elements (dim 0 dense in dst, dim 1 dense in src) +struct htp_copy_transpose_params { + uint32_t ne[3]; // dims 0..2 + uint32_t src_nb[3]; // src strides for dims 0, 2, 3 + uint32_t dst_nb[3]; // dst strides for dims 1, 2, 3 + struct fastdiv_values div_nkb; + struct fastdiv_values div_n0b; + struct fastdiv_values div_ne2; + uint32_t blk0; // dim-0 elements per tile + uint32_t blkk; // dim-1 elements per tile + uint32_t n0b; // tiles along dim 0 + uint32_t nkb; // tiles along dim 1 + uint32_t in_stride; // VTCM row pitch of staged src tile + uint32_t out_stride; // VTCM row pitch of transposed tile + uint32_t in_buf_size; + uint32_t out_buf_size; + uint32_t total_tiles; + uint8_t elem_size; + uint8_t packed; // gather straight to dst order + uint8_t log2_m; // log2(gather lanes per element) + uint8_t log2_r; // log2(dim-0 elements per vector) +}; + struct htp_copy_kernel_params { uint8_t kernel_type; uint8_t src0_type_size; uint8_t dst_type_size; - uint8_t n_threads; + uint8_t n_hvx; uint32_t total_elems; uint32_t total_rows; @@ -46,6 +70,7 @@ struct htp_copy_kernel_params { struct htp_copy_convert_params convert; struct htp_copy_reshape_params reshape; struct htp_gather_rows_params gather; + struct htp_copy_transpose_params transpose; } u; }; @@ -63,8 +88,8 @@ struct htp_copy_convert_vtcm_layout { uint32_t blk_rows; uint32_t src0_buf_size; uint32_t dst_buf_size; - uint32_t spad0_size_per_thread; - uint32_t spad1_size_per_thread; + uint32_t spad0_size_per_hvx; + uint32_t spad1_size_per_hvx; }; static inline void htp_copy_convert_vtcm_layout_build( @@ -73,7 +98,7 @@ static inline void htp_copy_convert_vtcm_layout_build( uint32_t src_type_size, uint32_t dst_type_size, uint32_t blk_rows, - uint32_t n_threads) { + uint32_t n_hvx) { const uint32_t src0_row_stride = hex_round_up(ne00 * src_type_size, 256); const uint32_t dst_row_stride = hex_round_up(ne00 * dst_type_size, 256); @@ -83,12 +108,12 @@ static inline void htp_copy_convert_vtcm_layout_build( L->blk_rows = blk_rows; L->src0_buf_size = blk_rows * src0_row_stride; L->dst_buf_size = blk_rows * dst_row_stride; - L->spad0_size_per_thread = 2 * L->src0_buf_size; - L->spad1_size_per_thread = 2 * L->dst_buf_size; + L->spad0_size_per_hvx = 2 * L->src0_buf_size; + L->spad1_size_per_hvx = 2 * L->dst_buf_size; size_t off = 0; - VTCM_LAYOUT_ALLOC(off, off_src0, (size_t) n_threads * L->spad0_size_per_thread); - VTCM_LAYOUT_ALLOC(off, off_dst, (size_t) n_threads * L->spad1_size_per_thread); + VTCM_LAYOUT_ALLOC(off, off_src0, (size_t) n_hvx * L->spad0_size_per_hvx); + VTCM_LAYOUT_ALLOC(off, off_dst, (size_t) n_hvx * L->spad1_size_per_hvx); L->total_bytes = off; } @@ -99,7 +124,7 @@ static inline void htp_copy_convert_solve_layout( uint32_t ne01, uint32_t src_type_size, uint32_t dst_type_size, - uint32_t n_threads) { + uint32_t n_hvx) { const uint32_t src0_row_stride = hex_round_up(ne00 * src_type_size, 256); const uint32_t dst_row_stride = hex_round_up(ne00 * dst_type_size, 256); @@ -113,7 +138,77 @@ static inline void htp_copy_convert_solve_layout( blk_rows = 1; } - htp_copy_convert_vtcm_layout_build(layout, ne00, src_type_size, dst_type_size, blk_rows, n_threads); + htp_copy_convert_vtcm_layout_build(layout, ne00, src_type_size, dst_type_size, blk_rows, n_hvx); +} + +#define HTP_COPY_TRANSPOSE_BLK_BYTES (64 * 1024) + +static inline bool htp_copy_transpose_layout_build(struct htp_copy_transpose_params * tp, uint32_t blk_bytes) { + const uint32_t ne0 = tp->ne[0]; + const uint32_t nek = tp->ne[1]; + const uint32_t es = tp->elem_size; + const uint32_t epv = 128 / es; + + uint32_t blk0; + uint32_t blkk; + if (tp->packed) { + // Packed: vector holds epv / ne0 whole dst rows + blk0 = ne0; + blkk = blk_bytes / (ne0 * es); + if (es == 2 && blkk > 16384 / ne0 - 64) { + blkk = 16384 / ne0 - 64; + } + if (blkk > nek) { + blkk = nek; + } + if (blkk == 0) { + return false; + } + } else if (nek <= epv) { + // Stage short contiguous dim and expand dim 0 + blkk = nek; + blk0 = blk_bytes / (nek * es); + blk0 -= blk0 % epv; + if (blk0 > hex_round_up(ne0, epv)) { + blk0 = hex_round_up(ne0, epv); + } + if (blk0 < epv) { + return false; + } + } else { + blk0 = hex_round_up(ne0, epv); + if (blk0 > 256) { + blk0 = 256; + } + blkk = blk_bytes / (blk0 * es); + blkk -= blkk % epv; + // Cap f16 blkk to 256 for 16-bit gather offsets + const uint32_t max_blkk = (es == 2 ? 512 : 4096) / es; + if (blkk > max_blkk) { + blkk = max_blkk; + } + if (blkk > hex_round_up(nek, epv)) { + blkk = hex_round_up(nek, epv); + } + if (blkk < epv) { + return false; + } + } + + tp->blk0 = blk0; + tp->blkk = blkk; + tp->n0b = (ne0 + blk0 - 1) / blk0; + tp->nkb = (nek + blkk - 1) / blkk; + // Pad row pitch by 1 element to avoid VTCM bank conflicts + const uint32_t pad = (es == 2) ? 2 : 4; + tp->in_stride = blkk * es + pad; + tp->out_stride = blk0 * es; + // Extra vector for packed gather overhang + tp->in_buf_size = hex_round_up(blk0 * tp->in_stride + (tp->packed ? 128 : 0), 128); + tp->out_buf_size = hex_round_up(blkk * tp->out_stride, 128); + tp->log2_m = es == 2 ? 0 : __builtin_ctz(es) - 2; + tp->log2_r = __builtin_ctz(tp->packed ? ne0 : epv); + return true; } #if defined(__cplusplus) diff --git a/ggml/src/ggml-hexagon/htp/cumsum-ops.c b/ggml/src/ggml-hexagon/htp/cumsum-ops.c index 36fc81276099..292a054550ae 100644 --- a/ggml/src/ggml-hexagon/htp/cumsum-ops.c +++ b/ggml/src/ggml-hexagon/htp/cumsum-ops.c @@ -8,7 +8,7 @@ #define GGML_COMMON_DECL_C #include "ggml-common.h" #include "hex-common.h" -#include "hex-profile.h" +#include "htp-trace.h" #include "htp-ctx.h" #include "htp-ops.h" #include "htp-tensor.h" @@ -48,7 +48,7 @@ struct htp_cumsum_context { size_t dst_row_size; size_t src_row_size_aligned; size_t dst_row_size_aligned; - uint32_t rows_per_thread; + uint32_t rows_per_hvx; uint32_t total_rows; uint32_t row_start; }; @@ -57,7 +57,7 @@ struct htp_cumsum_context { struct htp_cumsum_context * cctx = (struct htp_cumsum_context *) data; \ struct htp_ops_context * octx = cctx->octx; \ htp_cumsum_tensors_preamble; \ - dma_queue * dma_q = octx->ctx->dma[ith]; + dma_queue * dma_q = th->dma_q; // --------------------------------------------------------------------------- // HVX prefix scan helpers @@ -118,11 +118,11 @@ static inline void hvx_cumsum_row_f32(const float * restrict src, float * restri // Per thread worker: Double-buffered DMA // --------------------------------------------------------------------------- -static void cumsum_thread_f32_dma(unsigned int nth, unsigned int ith, void * data) { +static void cumsum_thread_f32_dma(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { htp_cumsum_preamble; - const uint32_t ir0 = cctx->row_start + cctx->rows_per_thread * ith; - const uint32_t ir1 = MIN(ir0 + cctx->rows_per_thread, cctx->row_start + cctx->total_rows); + const uint32_t ir0 = cctx->row_start + cctx->rows_per_hvx * ith; + const uint32_t ir1 = MIN(ir0 + cctx->rows_per_hvx, cctx->row_start + cctx->total_rows); if (ir0 >= ir1) { return; @@ -151,7 +151,7 @@ static void cumsum_thread_f32_dma(unsigned int nth, unsigned int ith, void * dat src_row_size_aligned, src_row_size, src_row_size, 1); } - struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + struct htp_thread_trace * tr = th->trace; for (uint32_t ir = ir0; ir < ir1; ir++) { float * dst_spad_row = (float *) dma_queue_pop(dma_q).src; @@ -184,16 +184,16 @@ static void cumsum_thread_f32_dma(unsigned int nth, unsigned int ith, void * dat // Per thread worker: Direct HVX (no DMA) // --------------------------------------------------------------------------- -static void cumsum_thread_f32(unsigned int nth, unsigned int ith, void * data) { +static void cumsum_thread_f32(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { htp_cumsum_preamble; const uint8_t * src_data = (const uint8_t *) src0->data; uint8_t * dst_data = (uint8_t *) dst->data; - const uint32_t ir0 = cctx->row_start + cctx->rows_per_thread * ith; - const uint32_t ir1 = MIN(ir0 + cctx->rows_per_thread, cctx->row_start + cctx->total_rows); + const uint32_t ir0 = cctx->row_start + cctx->rows_per_hvx * ith; + const uint32_t ir1 = MIN(ir0 + cctx->rows_per_hvx, cctx->row_start + cctx->total_rows); - struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + struct htp_thread_trace * tr = th->trace; htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir0); for (uint32_t ir = ir0; ir < ir1; ir++) { @@ -234,11 +234,11 @@ int op_cumsum_f32(struct htp_ops_context * octx) { return HTP_STATUS_OK; } - if (!htp_ops_context_set_n_threads(octx, MIN(kparams->n_threads, nrows))) { + if (!htp_ops_context_set_n_hvx(octx, MIN(kparams->n_hvx, nrows))) { return HTP_STATUS_INVAL_PARAMS; } - const uint32_t n_threads = octx->n_threads; + const uint32_t n_hvx = octx->n_hvx; if (!kparams->use_dma && (htp_tensor_is_extended(src0) || htp_tensor_is_extended(dst))) { return HTP_STATUS_NO_SUPPORT; @@ -246,12 +246,12 @@ int op_cumsum_f32(struct htp_ops_context * octx) { struct htp_cumsum_vtcm_layout layout = { 0 }; if (kparams->use_dma) { - htp_cumsum_vtcm_layout_build(&layout, src_row_size, dst_row_size, n_threads); + htp_cumsum_vtcm_layout_build(&layout, src_row_size, dst_row_size, n_hvx); - octx->src0_spad.size_per_thread = layout.src0_bytes_per_thread; - octx->dst_spad.size_per_thread = layout.dst_bytes_per_thread; - octx->src0_spad.size = n_threads * layout.src0_bytes_per_thread; - octx->dst_spad.size = n_threads * layout.dst_bytes_per_thread; + octx->src0_spad.size_per_hvx = layout.src0_bytes_per_hvx; + octx->dst_spad.size_per_hvx = layout.dst_bytes_per_hvx; + octx->src0_spad.size = n_hvx * layout.src0_bytes_per_hvx; + octx->dst_spad.size = n_hvx * layout.dst_bytes_per_hvx; octx->src0_spad.data = VTCM_LAYOUT_PTR(uint8_t, octx->ctx->vtcm_base, layout.off_src0); octx->dst_spad.data = VTCM_LAYOUT_PTR(uint8_t, octx->ctx->vtcm_base, layout.off_dst); octx->src0_spad.src = NULL; @@ -264,15 +264,15 @@ int op_cumsum_f32(struct htp_ops_context * octx) { .dst_row_size = dst_row_size, .src_row_size_aligned = layout.src_row_size_aligned, .dst_row_size_aligned = layout.dst_row_size_aligned, - .rows_per_thread = fastdiv(nrows + n_threads - 1, &octx->n_threads_div), + .rows_per_hvx = fastdiv(nrows + n_hvx - 1, &octx->n_hvx_div), .total_rows = nrows, .row_start = row_start, }; if (kparams->use_dma) { - work_queue_run(octx->ctx->work_queue, cumsum_thread_f32_dma, &cctx, n_threads); + work_queue_run(octx->ctx->hvx_queue, cumsum_thread_f32_dma, &cctx, n_hvx); } else { - work_queue_run(octx->ctx->work_queue, cumsum_thread_f32, &cctx, n_threads); + work_queue_run(octx->ctx->hvx_queue, cumsum_thread_f32, &cctx, n_hvx); } return HTP_STATUS_OK; diff --git a/ggml/src/ggml-hexagon/htp/cumsum-ops.h b/ggml/src/ggml-hexagon/htp/cumsum-ops.h index 6e1ac316fedd..48f4053ed49c 100644 --- a/ggml/src/ggml-hexagon/htp/cumsum-ops.h +++ b/ggml/src/ggml-hexagon/htp/cumsum-ops.h @@ -11,30 +11,30 @@ struct htp_cumsum_vtcm_layout { size_t off_dst; size_t src_row_size_aligned; size_t dst_row_size_aligned; - size_t src0_bytes_per_thread; - size_t dst_bytes_per_thread; + size_t src0_bytes_per_hvx; + size_t dst_bytes_per_hvx; }; static inline void htp_cumsum_vtcm_layout_build( struct htp_cumsum_vtcm_layout * L, size_t src_row_size, size_t dst_row_size, - uint32_t n_threads + uint32_t n_hvx ) { L->src_row_size_aligned = hex_round_up(src_row_size, 128); L->dst_row_size_aligned = hex_round_up(dst_row_size, 128); - L->src0_bytes_per_thread = 2 * L->src_row_size_aligned; - L->dst_bytes_per_thread = 2 * L->dst_row_size_aligned; + L->src0_bytes_per_hvx = 2 * L->src_row_size_aligned; + L->dst_bytes_per_hvx = 2 * L->dst_row_size_aligned; size_t off = 0; - VTCM_LAYOUT_ALLOC(off, off_src0, L->src0_bytes_per_thread * n_threads); - VTCM_LAYOUT_ALLOC(off, off_dst, L->dst_bytes_per_thread * n_threads); + VTCM_LAYOUT_ALLOC(off, off_src0, L->src0_bytes_per_hvx * n_hvx); + VTCM_LAYOUT_ALLOC(off, off_dst, L->dst_bytes_per_hvx * n_hvx); L->total_bytes = off; } struct htp_cumsum_kernel_params { - uint32_t n_threads; + uint32_t n_hvx; uint32_t total_rows; uint32_t use_dma; uint32_t vtcm_size; diff --git a/ggml/src/ggml-hexagon/htp/diag-ops.c b/ggml/src/ggml-hexagon/htp/diag-ops.c index 7f03e855013f..ea1a50316d66 100644 --- a/ggml/src/ggml-hexagon/htp/diag-ops.c +++ b/ggml/src/ggml-hexagon/htp/diag-ops.c @@ -6,7 +6,7 @@ #define GGML_COMMON_DECL_C #include "ggml-common.h" #include "hex-common.h" -#include "hex-profile.h" +#include "htp-trace.h" #include "htp-ctx.h" #include "htp-ops.h" #include "htp-tensor.h" @@ -40,7 +40,7 @@ struct htp_diag_context { size_t dst_row_size; size_t src_batch_size_aligned; size_t dst_row_size_aligned; - uint32_t batches_per_thread; + uint32_t batches_per_hvx; uint32_t total_batches; uint32_t batch_start; }; @@ -60,12 +60,12 @@ static inline void hvx_diag_row_f32(const float * restrict src, float * restrict // Per thread worker: DMA src fetch, compute in VTCM, DMA dst writeback // --------------------------------------------------------------------------- -static void diag_thread_f32_dma(unsigned int nth, unsigned int ith, void * data) { +static void diag_thread_f32_dma(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { htp_diag_preamble; - dma_queue * dma_q = octx->ctx->dma[ith]; + dma_queue * dma_q = th->dma_q; - const uint32_t ib0 = dctx->batch_start + dctx->batches_per_thread * ith; - const uint32_t ib1 = MIN(ib0 + dctx->batches_per_thread, dctx->batch_start + dctx->total_batches); + const uint32_t ib0 = dctx->batch_start + dctx->batches_per_hvx * ith; + const uint32_t ib1 = MIN(ib0 + dctx->batches_per_hvx, dctx->batch_start + dctx->total_batches); if (ib0 >= ib1) { return; @@ -83,7 +83,7 @@ static void diag_thread_f32_dma(unsigned int nth, unsigned int ith, void * data) uint8_t * src_spad = octx->src0_spad.data + (ith * src_batch_size_aligned); uint8_t * dst_spad = octx->dst_spad.data + (ith * dst_row_size_aligned); - struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + struct htp_thread_trace * tr = th->trace; for (uint32_t ib = ib0; ib < ib1; ib++) { const uint32_t i3 = fastdiv(ib, &dctx->kparams->div_ne02); @@ -124,16 +124,16 @@ static void diag_thread_f32_dma(unsigned int nth, unsigned int ith, void * data) // Per thread worker: Direct HVX (no DMA) // --------------------------------------------------------------------------- -static void diag_thread_f32(unsigned int nth, unsigned int ith, void * data) { +static void diag_thread_f32(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { htp_diag_preamble; const uint8_t * src_data = (const uint8_t *) src0->data; uint8_t * dst_data = (uint8_t *) dst->data; - const uint32_t ib0 = dctx->batch_start + dctx->batches_per_thread * ith; - const uint32_t ib1 = MIN(ib0 + dctx->batches_per_thread, dctx->batch_start + dctx->total_batches); + const uint32_t ib0 = dctx->batch_start + dctx->batches_per_hvx * ith; + const uint32_t ib1 = MIN(ib0 + dctx->batches_per_hvx, dctx->batch_start + dctx->total_batches); - struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + struct htp_thread_trace * tr = th->trace; htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ib0); for (uint32_t ib = ib0; ib < ib1; ib++) { @@ -177,11 +177,11 @@ int op_diag_f32(struct htp_ops_context * octx) { return HTP_STATUS_OK; } - if (!htp_ops_context_set_n_threads(octx, MIN(kparams->n_threads, nbatches))) { + if (!htp_ops_context_set_n_hvx(octx, MIN(kparams->n_hvx, nbatches))) { return HTP_STATUS_INVAL_PARAMS; } - const uint32_t n_threads = octx->n_threads; + const uint32_t n_hvx = octx->n_hvx; if (!kparams->use_dma && (htp_tensor_is_extended(src0) || htp_tensor_is_extended(dst))) { return HTP_STATUS_NO_SUPPORT; @@ -192,12 +192,12 @@ int op_diag_f32(struct htp_ops_context * octx) { struct htp_diag_vtcm_layout layout = { 0 }; if (kparams->use_dma) { - htp_diag_vtcm_layout_build(&layout, src_batch_size, dst_row_size, n_threads); + htp_diag_vtcm_layout_build(&layout, src_batch_size, dst_row_size, n_hvx); - octx->src0_spad.size_per_thread = layout.src0_bytes_per_thread; - octx->dst_spad.size_per_thread = layout.dst_bytes_per_thread; - octx->src0_spad.size = n_threads * layout.src0_bytes_per_thread; - octx->dst_spad.size = n_threads * layout.dst_bytes_per_thread; + octx->src0_spad.size_per_hvx = layout.src0_bytes_per_hvx; + octx->dst_spad.size_per_hvx = layout.dst_bytes_per_hvx; + octx->src0_spad.size = n_hvx * layout.src0_bytes_per_hvx; + octx->dst_spad.size = n_hvx * layout.dst_bytes_per_hvx; octx->src0_spad.data = VTCM_LAYOUT_PTR(uint8_t, octx->ctx->vtcm_base, layout.off_src0); octx->dst_spad.data = VTCM_LAYOUT_PTR(uint8_t, octx->ctx->vtcm_base, layout.off_dst); octx->src0_spad.src = NULL; @@ -211,15 +211,15 @@ int op_diag_f32(struct htp_ops_context * octx) { .dst_row_size = dst_row_size, .src_batch_size_aligned = layout.src_batch_size_aligned, .dst_row_size_aligned = layout.dst_row_size_aligned, - .batches_per_thread = fastdiv(nbatches + n_threads - 1, &octx->n_threads_div), + .batches_per_hvx = fastdiv(nbatches + n_hvx - 1, &octx->n_hvx_div), .total_batches = nbatches, .batch_start = batch_start, }; if (kparams->use_dma) { - work_queue_run(octx->ctx->work_queue, diag_thread_f32_dma, &dctx, n_threads); + work_queue_run(octx->ctx->hvx_queue, diag_thread_f32_dma, &dctx, n_hvx); } else { - work_queue_run(octx->ctx->work_queue, diag_thread_f32, &dctx, n_threads); + work_queue_run(octx->ctx->hvx_queue, diag_thread_f32, &dctx, n_hvx); } return HTP_STATUS_OK; diff --git a/ggml/src/ggml-hexagon/htp/diag-ops.h b/ggml/src/ggml-hexagon/htp/diag-ops.h index 983bc02423f2..609a061c5108 100644 --- a/ggml/src/ggml-hexagon/htp/diag-ops.h +++ b/ggml/src/ggml-hexagon/htp/diag-ops.h @@ -12,30 +12,30 @@ struct htp_diag_vtcm_layout { size_t off_dst; size_t src_batch_size_aligned; size_t dst_row_size_aligned; - size_t src0_bytes_per_thread; - size_t dst_bytes_per_thread; + size_t src0_bytes_per_hvx; + size_t dst_bytes_per_hvx; }; static inline void htp_diag_vtcm_layout_build( struct htp_diag_vtcm_layout * L, size_t src_batch_size, size_t dst_row_size, - uint32_t n_threads + uint32_t n_hvx ) { L->src_batch_size_aligned = hex_round_up(src_batch_size, 128); L->dst_row_size_aligned = hex_round_up(dst_row_size, 128); - L->src0_bytes_per_thread = L->src_batch_size_aligned; - L->dst_bytes_per_thread = L->dst_row_size_aligned; + L->src0_bytes_per_hvx = L->src_batch_size_aligned; + L->dst_bytes_per_hvx = L->dst_row_size_aligned; size_t off = 0; - VTCM_LAYOUT_ALLOC(off, off_src0, L->src0_bytes_per_thread * n_threads); - VTCM_LAYOUT_ALLOC(off, off_dst, L->dst_bytes_per_thread * n_threads); + VTCM_LAYOUT_ALLOC(off, off_src0, L->src0_bytes_per_hvx * n_hvx); + VTCM_LAYOUT_ALLOC(off, off_dst, L->dst_bytes_per_hvx * n_hvx); L->total_bytes = off; } struct htp_diag_kernel_params { - uint32_t n_threads; + uint32_t n_hvx; uint32_t total_batches; uint32_t use_dma; uint32_t can_split; diff --git a/ggml/src/ggml-hexagon/htp/dma-queue.h b/ggml/src/ggml-hexagon/htp/dma-queue.h index 678fc61d7e45..28d06119b75c 100644 --- a/ggml/src/ggml-hexagon/htp/dma-queue.h +++ b/ggml/src/ggml-hexagon/htp/dma-queue.h @@ -9,7 +9,7 @@ #include #include "hex-utils.h" -#include "hex-profile.h" +#include "htp-trace.h" #ifdef __cplusplus extern "C" { diff --git a/ggml/src/ggml-hexagon/htp/dsv4-hc-ops.c b/ggml/src/ggml-hexagon/htp/dsv4-hc-ops.c new file mode 100644 index 000000000000..b7123923caa7 --- /dev/null +++ b/ggml/src/ggml-hexagon/htp/dsv4-hc-ops.c @@ -0,0 +1,675 @@ +#pragma clang diagnostic ignored "-Wunused-variable" +#pragma clang diagnostic ignored "-Wunused-function" + +#include + +#include +#include +#include + +#include "dma-queue.h" +#include "hex-common.h" +#include "htp-trace.h" +#include "htp-ctx.h" +#include "htp-tensor.h" +#include "hvx-utils.h" +#include "dsv4-hc-ops.h" + +// --------------------------------------------------------------------------- +// DSV4_HC_PRE +// --------------------------------------------------------------------------- + +// hvx_vec_fast_sigmoid_f32_guard_2it with fewer live constants and predicates +static inline HVX_Vector dsv4_hc_sigmoid_f32(HVX_Vector v, HVX_Vector max_exp, HVX_Vector min_exp) { + v = Q6_Vsf_vmin_VsfVsf(Q6_Vsf_vmax_VsfVsf(v, min_exp), max_exp); + v = Q6_Vqf32_vmpy_VsfVsf(v, hvx_vec_splat_f32(0.5f * 1.442695022f)); + + HVX_Vector in_int = Q6_Vw_equals_Vsf(Q6_Vsf_equals_Vqf32(v)); + HVX_Vector x = Q6_Vqf32_vsub_Vqf32Vsf(v, Q6_Vsf_equals_Vw(in_int)); + HVX_Vector xx = Q6_Vqf32_vmpy_Vqf32Vqf32(x, x); + + HVX_Vector v1 = Q6_Vqf32_vmpy_VsfVsf(Q6_Vsf_equals_Vqf32(xx), Q6_V_vsplat_R(FAST_SIGMOID_C2)); + v1 = Q6_Vqf32_vadd_Vqf32Vsf(v1, Q6_V_vsplat_R(FAST_SIGMOID_LOG2F)); + + HVX_Vector v2 = Q6_Vqf32_vmpy_VsfVsf(Q6_Vsf_equals_Vqf32(x), Q6_V_vsplat_R(FAST_SIGMOID_C1)); + v2 = Q6_Vqf32_vmpy_Vqf32Vqf32(v2, xx); + v2 = Q6_Vqf32_vadd_Vqf32Vqf32(v2, x); + + HVX_Vector v3 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_Vqf32Vqf32(v2, v1)); + v3 = Q6_Vw_vaslacc_VwVwR(v3, in_int, 24); + + HVX_Vector v4 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vsub_Vqf32Vqf32(v2, v1)); + HVX_Vector v5 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vsub_VsfVsf(v3, v4)); + + // Newton-Raphson with 2 iterations + HVX_Vector r = Q6_Vw_vsub_VwVw(Q6_V_vsplat_R(0x7EEEEBB3), v5); + for (int i = 0; i < 2; ++i) { + const HVX_Vector rv = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(r, v5)); + r = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vsub_Vqf32Vqf32(Q6_Vqf32_vadd_VsfVsf(r, r), Q6_Vqf32_vmpy_VsfVsf(r, rv))); + } + + return Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(v3, r)); +} + +struct dsv4_hc_pre_context { + struct htp_ops_context * octx; + const struct htp_dsv4_hc_pre_kernel_params * kparams; + uint32_t token_start; + uint32_t n_chunks; + uint32_t total_jobs; + uint32_t jobs_per_hvx; + float scale; + uint8_t * vtcm; + uint32_t dma_block_chunks; + uint32_t dma_row_stride; + uint32_t dma_buffer_size; +}; + +struct dsv4_hc_block { + uint32_t it; + uint32_t chunk; + uint32_t count; +}; + +static inline struct dsv4_hc_block dsv4_hc_pre_block_make( + const struct dsv4_hc_pre_context * dctx, + uint32_t job, + uint32_t end) { + const uint32_t local_token = fastdiv(job, &dctx->kparams->div_n_chunks); + const uint32_t chunk = job - local_token*dctx->n_chunks; + const uint32_t count = hex_smin(dctx->dma_block_chunks, hex_smin(dctx->n_chunks - chunk, end - job)); + return (struct dsv4_hc_block) { + .it = dctx->token_start + local_token, + .chunk = chunk, + .count = count, + }; +} + +static inline void dsv4_hc_pre_push_plain( + dma_queue * dma_q, + const struct dsv4_hc_pre_context * dctx, + const struct dsv4_hc_block * block, + uint8_t * spad) { + const struct htp_tensor * x = dctx->octx->src[0]; + const struct htp_tensor * weights = dctx->octx->src[1]; + const uint32_t row_stride = dctx->dma_row_stride; + const uint32_t row_size = block->count * VLEN; + + dma_queue_push(dma_q, dma_make_data(spad, x->data + (block->it*x->nb[2] + block->chunk*VLEN)), row_stride, x->nb[1], row_size, 4); + dma_queue_push(dma_q, dma_make_data(spad + 5*row_stride, weights->data + (block->it*weights->nb[1])), 16, 16, 16, 1); +} + +static inline void dsv4_hc_pre_push_gated( + dma_queue * dma_q, + const struct dsv4_hc_pre_context * dctx, + const struct dsv4_hc_block * block, + uint8_t * spad) { + const struct htp_tensor * x = dctx->octx->src[0]; + const struct htp_tensor * g = dctx->octx->src[1]; + const uint32_t row_stride = dctx->dma_row_stride; + const uint32_t row_size = block->count * VLEN; + + dma_queue_push(dma_q, dma_make_data(spad, x->data + (block->it*x->nb[2] + block->chunk*VLEN)), row_stride, x->nb[1], row_size, 4); + dma_queue_push(dma_q, dma_make_data(spad + 4*row_stride, g->data + (block->it*g->nb[2] + block->chunk*VLEN)), row_stride, g->nb[1], row_size, 4); +} + +static __attribute__((noinline)) void dsv4_hc_pre_plain_block_f32( + const struct dsv4_hc_pre_context * dctx, + const struct dsv4_hc_block * block, + const uint8_t * x_spad, + uint8_t * d_spad, + const float * wb) { + const uint32_t row_stride = dctx->dma_row_stride; + const HVX_Vector scale_v = hvx_vec_splat_f32(dctx->scale); + const HVX_Vector w0 = hvx_vec_splat_f32(wb[0]); + const HVX_Vector w1 = hvx_vec_splat_f32(wb[1]); + const HVX_Vector w2 = hvx_vec_splat_f32(wb[2]); + const HVX_Vector w3 = hvx_vec_splat_f32(wb[3]); + + for (uint32_t c = 0; c < block->count; ++c) { + const uint8_t * xb = x_spad + c*VLEN; + + HVX_Vector x0 = *(const HVX_Vector *) xb; + HVX_Vector x1 = *(const HVX_Vector *) (xb + row_stride); + HVX_Vector x2 = *(const HVX_Vector *) (xb + 2*row_stride); + HVX_Vector x3 = *(const HVX_Vector *) (xb + 3*row_stride); + + HVX_Vector acc = hvx_vec_mul_f32_f32(x0, w0); + acc = hvx_vec_add_f32_f32(acc, hvx_vec_mul_f32_f32(x1, w1)); + acc = hvx_vec_add_f32_f32(acc, hvx_vec_mul_f32_f32(x2, w2)); + acc = hvx_vec_add_f32_f32(acc, hvx_vec_mul_f32_f32(x3, w3)); + + *(HVX_Vector *) (d_spad + c*VLEN) = hvx_vec_mul_f32_f32(acc, scale_v); + } +} + +static __attribute__((noinline)) void dsv4_hc_pre_gated_block_f32( + const struct dsv4_hc_pre_context * dctx, + const struct dsv4_hc_block * block, + const uint8_t * x_spad, + const uint8_t * g_spad, + uint8_t * d_spad) { + const uint32_t row_stride = dctx->dma_row_stride; + const HVX_Vector scale_v = hvx_vec_splat_f32(dctx->scale); + const HVX_Vector max_exp = hvx_vec_splat_f32(87.0f); + const HVX_Vector min_exp = hvx_vec_splat_f32(-87.0f); + + for (uint32_t c = 0; c < block->count; ++c) { + const uint8_t * xb = x_spad + c*VLEN; + const uint8_t * gb = g_spad + c*VLEN; + + HVX_Vector xv = *(const HVX_Vector *) xb; + HVX_Vector gv = dsv4_hc_sigmoid_f32(*(const HVX_Vector *) gb, max_exp, min_exp); + HVX_Vector acc = hvx_vec_mul_f32_f32(xv, gv); + + xv = *(const HVX_Vector *) (xb + row_stride); + gv = dsv4_hc_sigmoid_f32(*(const HVX_Vector *) (gb + row_stride), max_exp, min_exp); + acc = hvx_vec_add_f32_f32(acc, hvx_vec_mul_f32_f32(xv, gv)); + + xv = *(const HVX_Vector *) (xb + 2*row_stride); + gv = dsv4_hc_sigmoid_f32(*(const HVX_Vector *) (gb + 2*row_stride), max_exp, min_exp); + acc = hvx_vec_add_f32_f32(acc, hvx_vec_mul_f32_f32(xv, gv)); + + xv = *(const HVX_Vector *) (xb + 3*row_stride); + gv = dsv4_hc_sigmoid_f32(*(const HVX_Vector *) (gb + 3*row_stride), max_exp, min_exp); + acc = hvx_vec_add_f32_f32(acc, hvx_vec_mul_f32_f32(xv, gv)); + + *(HVX_Vector *) (d_spad + c*VLEN) = hvx_vec_mul_f32_f32(acc, scale_v); + } +} + +static void dsv4_hc_pre_thread_plain_f32(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { + struct dsv4_hc_pre_context * dctx = (struct dsv4_hc_pre_context *) data; + struct htp_ops_context * octx = dctx->octx; + + const struct htp_tensor * dst = octx->dst; + const uint64_t d_data = dst->data; + + const uint32_t row_stride = dctx->dma_row_stride; + const uint32_t buffer_size = dctx->dma_buffer_size; + const uint32_t start = dctx->jobs_per_hvx * ith; + const uint32_t end = hex_smin(start + dctx->jobs_per_hvx, dctx->total_jobs); + if (start >= end) { + return; + } + + uint8_t * spad[2] = { + dctx->vtcm + 2*buffer_size*ith, + dctx->vtcm + 2*buffer_size*ith + buffer_size, + }; + + dma_queue * dma_q = th->dma_q; + uint32_t prefetch_job = start; + for (uint32_t i = 0; i < 2 && prefetch_job < end; ++i) { + const struct dsv4_hc_block block = dsv4_hc_pre_block_make(dctx, prefetch_job, end); + // empty dst writeback keeps pop order dst, x, weights in loop + dma_queue_push(dma_q, dma_make_data(d_data, spad[i] + 4*row_stride), block.count*VLEN, block.count*VLEN, 0, 0); + dsv4_hc_pre_push_plain(dma_q, dctx, &block, spad[i]); + prefetch_job += block.count; + } + + struct htp_thread_trace * tr = th->trace; + + uint32_t job = start; + uint32_t curr = 0; + while (job < end) { + const struct dsv4_hc_block block = dsv4_hc_pre_block_make(dctx, job, end); + + dma_queue_pop(dma_q); + dma_queue_pop(dma_q); + dma_queue_pop(dma_q); + + const float * wb = (const float *) (spad[curr] + 5*row_stride); + uint8_t * d_spad = spad[curr] + 4*row_stride; + + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) block.it); + dsv4_hc_pre_plain_block_f32(dctx, &block, spad[curr], d_spad, wb); + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) block.it); + + const uint64_t d_dst = d_data + (block.it*dst->nb[1] + block.chunk*VLEN); + dma_queue_push(dma_q, dma_make_data(d_dst, d_spad), block.count*VLEN, block.count*VLEN, block.count*VLEN, 1); + + if (prefetch_job < end) { + const struct dsv4_hc_block next = dsv4_hc_pre_block_make(dctx, prefetch_job, end); + dsv4_hc_pre_push_plain(dma_q, dctx, &next, spad[curr]); + prefetch_job += next.count; + } + + job += block.count; + curr ^= 1; + } + + dma_queue_flush(dma_q); + + (void) nth; +} + +static void dsv4_hc_pre_thread_gated_f32(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { + struct dsv4_hc_pre_context * dctx = (struct dsv4_hc_pre_context *) data; + struct htp_ops_context * octx = dctx->octx; + + const struct htp_tensor * dst = octx->dst; + const uint64_t d_data = dst->data; + + const uint32_t row_stride = dctx->dma_row_stride; + const uint32_t buffer_size = dctx->dma_buffer_size; + const uint32_t start = dctx->jobs_per_hvx * ith; + const uint32_t end = hex_smin(start + dctx->jobs_per_hvx, dctx->total_jobs); + if (start >= end) { + return; + } + + uint8_t * spad[2] = { + dctx->vtcm + 2*buffer_size*ith, + dctx->vtcm + 2*buffer_size*ith + buffer_size, + }; + + dma_queue * dma_q = th->dma_q; + uint32_t prefetch_job = start; + for (uint32_t i = 0; i < 2 && prefetch_job < end; ++i) { + const struct dsv4_hc_block block = dsv4_hc_pre_block_make(dctx, prefetch_job, end); + // empty dst writeback keeps pop order dst, x, g in loop + dma_queue_push(dma_q, dma_make_data(d_data, spad[i] + 8*row_stride), block.count*VLEN, block.count*VLEN, 0, 0); + dsv4_hc_pre_push_gated(dma_q, dctx, &block, spad[i]); + prefetch_job += block.count; + } + + struct htp_thread_trace * tr = th->trace; + + uint32_t job = start; + uint32_t curr = 0; + while (job < end) { + const struct dsv4_hc_block block = dsv4_hc_pre_block_make(dctx, job, end); + + dma_queue_pop(dma_q); + dma_queue_pop(dma_q); + dma_queue_pop(dma_q); + + uint8_t * d_spad = spad[curr] + 8*row_stride; + + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) block.it); + dsv4_hc_pre_gated_block_f32(dctx, &block, spad[curr], spad[curr] + 4*row_stride, d_spad); + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) block.it); + + const uint64_t d_dst = d_data + (block.it*dst->nb[1] + block.chunk*VLEN); + dma_queue_push(dma_q, dma_make_data(d_dst, d_spad), block.count*VLEN, block.count*VLEN, block.count*VLEN, 1); + + if (prefetch_job < end) { + const struct dsv4_hc_block next = dsv4_hc_pre_block_make(dctx, prefetch_job, end); + dsv4_hc_pre_push_gated(dma_q, dctx, &next, spad[curr]); + prefetch_job += next.count; + } + + job += block.count; + curr ^= 1; + } + + dma_queue_flush(dma_q); + + (void) nth; +} + +int op_dsv4_hc_pre(struct htp_ops_context * octx) { + const struct htp_tensor * x = octx->src[0]; + const struct htp_tensor * weights = octx->src[1]; + const struct htp_tensor * dst = octx->dst; + + const struct htp_dsv4_hc_pre_kernel_params * kparams = + (const struct htp_dsv4_hc_pre_kernel_params *) octx->kernel_params; + + const uint32_t n_hvx = kparams->n_hvx; + if (!htp_ops_context_set_n_hvx_div(octx, n_hvx, kparams->div_n_hvx)) { + return HTP_STATUS_INVAL_PARAMS; + } + + uint32_t token_start = 0; + uint32_t n_tokens = x->ne[2]; + uint32_t total_jobs = kparams->total_jobs; + uint32_t jobs_per_hvx = kparams->jobs_per_hvx; + if (octx->ctx->mdev.count > 1) { + const bool can_split = htp_tensor_mdev_data_aligned(dst) && ((dst->nb[1] & (HTP_TENSOR_MDEV_LINE_SIZE - 1)) == 0); + const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(n_tokens, can_split ? 1 : 0, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div); + token_start = range.start; + n_tokens = range.count; + total_jobs = n_tokens * kparams->n_chunks; + jobs_per_hvx = fastdiv(total_jobs + n_hvx - 1, &kparams->div_n_hvx); + } + + if (n_tokens == 0) { + return HTP_STATUS_OK; + } + + float scale; + memcpy(&scale, octx->op_params, sizeof(scale)); + + struct dsv4_hc_pre_context dctx = { + .octx = octx, + .kparams = kparams, + .token_start = token_start, + .n_chunks = kparams->n_chunks, + .total_jobs = total_jobs, + .jobs_per_hvx = jobs_per_hvx, + .scale = scale, + .vtcm = (uint8_t *) octx->ctx->vtcm_base, + .dma_block_chunks = kparams->dma_block_chunks, + .dma_row_stride = kparams->dma_row_stride, + .dma_buffer_size = kparams->dma_buffer_size, + }; + + work_queue_func_t worker = NULL; + switch (kparams->kernel_type) { + case HTP_DSV4_HC_PRE_KERNEL_PLAIN: worker = dsv4_hc_pre_thread_plain_f32; break; + case HTP_DSV4_HC_PRE_KERNEL_GATED: worker = dsv4_hc_pre_thread_gated_f32; break; + default: return HTP_STATUS_INVAL_PARAMS; + } + + FARF(HIGH, "dsv4-hc-pre: (%ux%ux%u) -> (%ux%u) : ktype %u tokens %u:%u nhvx %u\n", + x->ne[0], x->ne[1], x->ne[2], dst->ne[0], dst->ne[1], kparams->kernel_type, + token_start, token_start + n_tokens, n_hvx); + + work_queue_run(octx->ctx->hvx_queue, worker, &dctx, n_hvx); + return HTP_STATUS_OK; +} + +// --------------------------------------------------------------------------- +// DSV4_HC_POST +// --------------------------------------------------------------------------- + +struct dsv4_hc_post_context { + struct htp_ops_context * octx; + const struct htp_dsv4_hc_post_kernel_params * kparams; + uint32_t token_start; + uint32_t total_jobs; + uint32_t jobs_per_hvx; + uint8_t * vtcm; +}; + +static inline struct dsv4_hc_block dsv4_hc_post_block_make( + const struct dsv4_hc_post_context * dctx, + uint32_t job, + uint32_t end, + uint32_t block_chunks) { + const uint32_t local_token = fastdiv(job, &dctx->kparams->div_n_chunks); + const uint32_t chunk = job - local_token*dctx->kparams->n_chunks; + const uint32_t count = hex_smin(block_chunks, hex_smin(dctx->kparams->n_chunks - chunk, end - job)); + return (struct dsv4_hc_block) { + .it = dctx->token_start + local_token, + .chunk = chunk, + .count = count, + }; +} + +static __attribute__((noinline)) void dsv4_hc_post_identity_block_f32( + const uint8_t * xb, + const uint8_t * rb, + size_t r_stride, + uint8_t * db, + size_t d_stride, + uint32_t count, + const float * p) { + for (uint32_t id = 0; id < 4; ++id) { + const HVX_Vector pv = hvx_vec_splat_f32(p[id]); + const uint8_t * rr = rb + id*r_stride; + uint8_t * dd = db + id*d_stride; + + for (uint32_t i = 0; i < count; ++i) { + const HVX_Vector xv = *(const HVX_Vector *) (xb + i*VLEN); + const HVX_Vector rv = *(const HVX_Vector *) (rr + i*VLEN); + *(HVX_Vector *) (dd + i*VLEN) = hvx_vec_add_f32_f32(hvx_vec_mul_f32_f32(xv, pv), rv); + } + } +} + +static __attribute__((noinline)) void dsv4_hc_post_comb_block_f32( + const uint8_t * xb, + const uint8_t * rb, + size_t r_stride, + uint8_t * db, + size_t d_stride, + uint32_t count, + const float * p, + const float * c) { + for (uint32_t id = 0; id < 4; ++id) { + const HVX_Vector pv = hvx_vec_splat_f32(p[id]); + const HVX_Vector c0 = hvx_vec_splat_f32(c[id]); + const HVX_Vector c1 = hvx_vec_splat_f32(c[4 + id]); + const HVX_Vector c2 = hvx_vec_splat_f32(c[8 + id]); + const HVX_Vector c3 = hvx_vec_splat_f32(c[12 + id]); + uint8_t * dd = db + id*d_stride; + + for (uint32_t i = 0; i < count; ++i) { + const uint8_t * r = rb + i*VLEN; + HVX_Vector acc = hvx_vec_mul_f32_f32(*(const HVX_Vector *) (xb + i*VLEN), pv); + acc = hvx_vec_add_f32_f32(acc, hvx_vec_mul_f32_f32(*(const HVX_Vector *) r, c0)); + acc = hvx_vec_add_f32_f32(acc, hvx_vec_mul_f32_f32(*(const HVX_Vector *) (r + r_stride), c1)); + acc = hvx_vec_add_f32_f32(acc, hvx_vec_mul_f32_f32(*(const HVX_Vector *) (r + 2*r_stride), c2)); + acc = hvx_vec_add_f32_f32(acc, hvx_vec_mul_f32_f32(*(const HVX_Vector *) (r + 3*r_stride), c3)); + *(HVX_Vector *) (dd + i*VLEN) = acc; + } + } +} + +static inline void dsv4_hc_post_push_identity( + dma_queue * dma_q, + const struct dsv4_hc_post_context * dctx, + const struct dsv4_hc_block * block, + uint8_t * spad) { + const struct htp_tensor * x = dctx->octx->src[0]; + const struct htp_tensor * residual = dctx->octx->src[1]; + const struct htp_tensor * post = dctx->octx->src[2]; + const uint32_t row_stride = dctx->kparams->dma_row_stride; + const uint32_t row_size = block->count * VLEN; + + dma_queue_push(dma_q, dma_make_data(spad, residual->data + (block->it*residual->nb[2] + block->chunk*VLEN)), + row_stride, residual->nb[1], row_size, 4); + dma_queue_push(dma_q, dma_make_data(spad + 4*row_stride, x->data + (block->it*x->nb[1] + block->chunk*VLEN)), + row_stride, row_size, row_size, 1); + dma_queue_push(dma_q, dma_make_data(spad + 9*row_stride, post->data + (block->it*post->nb[1])), + 16, 16, 16, 1); +} + +static inline void dsv4_hc_post_push_comb( + dma_queue * dma_q, + const struct dsv4_hc_post_context * dctx, + const struct dsv4_hc_block * block, + uint8_t * spad) { + const struct htp_tensor * comb = dctx->octx->src[3]; + const uint32_t row_stride = dctx->kparams->dma_row_stride; + + dsv4_hc_post_push_identity(dma_q, dctx, block, spad); + dma_queue_push(dma_q, dma_make_data(spad + 9*row_stride + 64, comb->data + (block->it*comb->nb[2])), + 16, comb->nb[1], 16, 4); +} + +static void dsv4_hc_post_thread_identity_f32(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { + struct dsv4_hc_post_context * dctx = (struct dsv4_hc_post_context *) data; + struct htp_ops_context * octx = dctx->octx; + + const struct htp_tensor * dst = octx->dst; + const uint64_t d_data = dst->data; + + const uint32_t block_chunks = dctx->kparams->dma_block_chunks; + const uint32_t row_stride = dctx->kparams->dma_row_stride; + const uint32_t buffer_size = dctx->kparams->dma_buffer_size; + const uint32_t start = dctx->jobs_per_hvx * ith; + const uint32_t end = hex_smin(start + dctx->jobs_per_hvx, dctx->total_jobs); + if (start >= end) { + return; + } + + uint8_t * spad[2] = { + dctx->vtcm + 2*buffer_size*ith, + dctx->vtcm + 2*buffer_size*ith + buffer_size, + }; + + dma_queue * dma_q = th->dma_q; + uint32_t prefetch_job = start; + for (uint32_t i = 0; i < 2 && prefetch_job < end; ++i) { + const struct dsv4_hc_block block = dsv4_hc_post_block_make(dctx, prefetch_job, end, block_chunks); + // empty dst writeback keeps pop order dst, residual, x, post in loop + dma_queue_push(dma_q, dma_make_data(d_data, spad[i] + 5*row_stride), dst->nb[1], row_stride, 0, 0); + dsv4_hc_post_push_identity(dma_q, dctx, &block, spad[i]); + prefetch_job += block.count; + } + + struct htp_thread_trace * tr = th->trace; + + uint32_t job = start; + uint32_t curr = 0; + while (job < end) { + const struct dsv4_hc_block block = dsv4_hc_post_block_make(dctx, job, end, block_chunks); + + dma_queue_pop(dma_q); + dma_queue_pop(dma_q); + dma_queue_pop(dma_q); + dma_queue_pop(dma_q); + + const float * p = (const float *) (spad[curr] + 9*row_stride); + uint8_t * d_spad = spad[curr] + 5*row_stride; + + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) block.it); + dsv4_hc_post_identity_block_f32(spad[curr] + 4*row_stride, spad[curr], row_stride, d_spad, row_stride, block.count, p); + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) block.it); + + const uint64_t d_dst = d_data + (block.it*dst->nb[2] + block.chunk*VLEN); + dma_queue_push(dma_q, dma_make_data(d_dst, d_spad), dst->nb[1], row_stride, block.count*VLEN, 4); + + if (prefetch_job < end) { + const struct dsv4_hc_block next = dsv4_hc_post_block_make(dctx, prefetch_job, end, block_chunks); + dsv4_hc_post_push_identity(dma_q, dctx, &next, spad[curr]); + prefetch_job += next.count; + } + + job += block.count; + curr ^= 1; + } + + dma_queue_flush(dma_q); + + (void) nth; +} + +static void dsv4_hc_post_thread_comb_f32(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { + struct dsv4_hc_post_context * dctx = (struct dsv4_hc_post_context *) data; + struct htp_ops_context * octx = dctx->octx; + + const struct htp_tensor * dst = octx->dst; + const uint64_t d_data = dst->data; + + const uint32_t block_chunks = dctx->kparams->dma_block_chunks; + const uint32_t row_stride = dctx->kparams->dma_row_stride; + const uint32_t buffer_size = dctx->kparams->dma_buffer_size; + const uint32_t start = dctx->jobs_per_hvx * ith; + const uint32_t end = hex_smin(start + dctx->jobs_per_hvx, dctx->total_jobs); + if (start >= end) { + return; + } + + uint8_t * spad[2] = { + dctx->vtcm + 2*buffer_size*ith, + dctx->vtcm + 2*buffer_size*ith + buffer_size, + }; + + dma_queue * dma_q = th->dma_q; + uint32_t prefetch_job = start; + for (uint32_t i = 0; i < 2 && prefetch_job < end; ++i) { + const struct dsv4_hc_block block = dsv4_hc_post_block_make(dctx, prefetch_job, end, block_chunks); + // empty dst writeback keeps pop order dst, residual, x, post, comb in loop + dma_queue_push(dma_q, dma_make_data(d_data, spad[i] + 5*row_stride), dst->nb[1], row_stride, 0, 0); + dsv4_hc_post_push_comb(dma_q, dctx, &block, spad[i]); + prefetch_job += block.count; + } + + struct htp_thread_trace * tr = th->trace; + + uint32_t job = start; + uint32_t curr = 0; + while (job < end) { + const struct dsv4_hc_block block = dsv4_hc_post_block_make(dctx, job, end, block_chunks); + + dma_queue_pop(dma_q); + dma_queue_pop(dma_q); + dma_queue_pop(dma_q); + dma_queue_pop(dma_q); + dma_queue_pop(dma_q); + + const float * p = (const float *) (spad[curr] + 9*row_stride); + const float * c = (const float *) (spad[curr] + 9*row_stride + 64); + uint8_t * d_spad = spad[curr] + 5*row_stride; + + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) block.it); + dsv4_hc_post_comb_block_f32(spad[curr] + 4*row_stride, spad[curr], row_stride, d_spad, row_stride, block.count, p, c); + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) block.it); + + const uint64_t d_dst = d_data + (block.it*dst->nb[2] + block.chunk*VLEN); + dma_queue_push(dma_q, dma_make_data(d_dst, d_spad), dst->nb[1], row_stride, block.count*VLEN, 4); + + if (prefetch_job < end) { + const struct dsv4_hc_block next = dsv4_hc_post_block_make(dctx, prefetch_job, end, block_chunks); + dsv4_hc_post_push_comb(dma_q, dctx, &next, spad[curr]); + prefetch_job += next.count; + } + + job += block.count; + curr ^= 1; + } + + dma_queue_flush(dma_q); + + (void) nth; +} + +int op_dsv4_hc_post(struct htp_ops_context * octx) { + const struct htp_tensor * x = octx->src[0]; + const struct htp_tensor * residual = octx->src[1]; + const struct htp_tensor * dst = octx->dst; + + const struct htp_dsv4_hc_post_kernel_params * kparams = + (const struct htp_dsv4_hc_post_kernel_params *) octx->kernel_params; + + const uint32_t n_hvx = kparams->n_hvx; + if (!htp_ops_context_set_n_hvx_div(octx, n_hvx, kparams->div_n_hvx)) { + return HTP_STATUS_INVAL_PARAMS; + } + + uint32_t token_start = 0; + uint32_t n_tokens = x->ne[1]; + uint32_t total_jobs = kparams->total_jobs; + uint32_t jobs_per_hvx = kparams->jobs_per_hvx; + if (octx->ctx->mdev.count > 1) { + const bool can_split = htp_tensor_mdev_data_aligned(dst) && ((dst->nb[2] & (HTP_TENSOR_MDEV_LINE_SIZE - 1)) == 0); + const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(n_tokens, can_split ? 1 : 0, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div); + token_start = range.start; + n_tokens = range.count; + total_jobs = n_tokens * kparams->n_chunks; + jobs_per_hvx = fastdiv(total_jobs + n_hvx - 1, &kparams->div_n_hvx); + } + + if (n_tokens == 0) { + return HTP_STATUS_OK; + } + + struct dsv4_hc_post_context dctx = { + .octx = octx, + .kparams = kparams, + .token_start = token_start, + .total_jobs = total_jobs, + .jobs_per_hvx = jobs_per_hvx, + .vtcm = (uint8_t *) octx->ctx->vtcm_base, + }; + + work_queue_func_t worker = NULL; + switch (kparams->kernel_type) { + case HTP_DSV4_HC_POST_KERNEL_IDENTITY: worker = dsv4_hc_post_thread_identity_f32; break; + case HTP_DSV4_HC_POST_KERNEL_COMB: worker = dsv4_hc_post_thread_comb_f32; break; + default: return HTP_STATUS_INVAL_PARAMS; + } + + FARF(HIGH, "dsv4-hc-post: (%ux%u) (%ux%ux%u) -> (%ux%ux%u) : ktype %u tokens %u:%u nhvx %u\n", + x->ne[0], x->ne[1], residual->ne[0], residual->ne[1], residual->ne[2], + dst->ne[0], dst->ne[1], dst->ne[2], kparams->kernel_type, + token_start, token_start + n_tokens, n_hvx); + + work_queue_run(octx->ctx->hvx_queue, worker, &dctx, n_hvx); + return HTP_STATUS_OK; +} diff --git a/ggml/src/ggml-hexagon/htp/dsv4-hc-ops.h b/ggml/src/ggml-hexagon/htp/dsv4-hc-ops.h new file mode 100644 index 000000000000..58a3d0d3c81f --- /dev/null +++ b/ggml/src/ggml-hexagon/htp/dsv4-hc-ops.h @@ -0,0 +1,77 @@ +#ifndef HTP_DSV4_HC_OPS_H +#define HTP_DSV4_HC_OPS_H + +#include + +#include "hex-fastdiv.h" +#include "htp-ops.h" + +#define HTP_DSV4_HC_CHUNK_ELEMS 32 +#define HTP_DSV4_HC_MAX_CHUNKS 128 +#define HTP_DSV4_HC_PRE_GATED_ROWS 9 +#define HTP_DSV4_HC_PRE_PLAIN_ROWS 6 + +enum htp_dsv4_hc_pre_kernel_type { + HTP_DSV4_HC_PRE_KERNEL_PLAIN = 0, + HTP_DSV4_HC_PRE_KERNEL_GATED = 1, +}; + +struct htp_dsv4_hc_pre_kernel_params { + uint32_t kernel_type; + uint32_t n_hvx; + uint32_t n_chunks; + uint32_t total_jobs; + uint32_t jobs_per_hvx; + + uint32_t dma_block_chunks; + uint32_t dma_row_stride; + uint32_t dma_buffer_size; + uint32_t vtcm_size; + + struct fastdiv_values div_n_chunks; + struct fastdiv_values div_n_hvx; +}; + +#if defined(__cplusplus) +static_assert(sizeof(struct htp_dsv4_hc_pre_kernel_params) <= 128, "htp_dsv4_hc_pre_kernel_params is too large for kernel_params blob"); +#else +_Static_assert(sizeof(struct htp_dsv4_hc_pre_kernel_params) <= 128, "htp_dsv4_hc_pre_kernel_params is too large for kernel_params blob"); +#endif + +// dma buffer holds residual rows 0..3, x at row 4, dst rows 5..8, post/comb at row 9 +#define HTP_DSV4_HC_POST_ROWS 10 + +enum htp_dsv4_hc_post_kernel_type { + HTP_DSV4_HC_POST_KERNEL_IDENTITY = 0, + HTP_DSV4_HC_POST_KERNEL_COMB = 1, +}; + +struct htp_dsv4_hc_post_kernel_params { + uint32_t kernel_type; + uint32_t n_hvx; + uint32_t n_chunks; + uint32_t total_jobs; + uint32_t jobs_per_hvx; + uint32_t has_comb; + + uint32_t dma_block_chunks; + uint32_t dma_row_stride; + uint32_t dma_buffer_size; + uint32_t vtcm_size; + + struct fastdiv_values div_n_chunks; + struct fastdiv_values div_n_hvx; +}; + +#if defined(__cplusplus) +static_assert(sizeof(struct htp_dsv4_hc_post_kernel_params) <= 128, "htp_dsv4_hc_post_kernel_params is too large for kernel_params blob"); +#else +_Static_assert(sizeof(struct htp_dsv4_hc_post_kernel_params) <= 128, "htp_dsv4_hc_post_kernel_params is too large for kernel_params blob"); +#endif + +struct htp_ops_context; + +int op_dsv4_hc_pre(struct htp_ops_context * octx); +int op_dsv4_hc_post(struct htp_ops_context * octx); + +#endif // HTP_DSV4_HC_OPS_H diff --git a/ggml/src/ggml-hexagon/htp/fill-ops.c b/ggml/src/ggml-hexagon/htp/fill-ops.c index 2cab31b5fd0a..62026306304f 100644 --- a/ggml/src/ggml-hexagon/htp/fill-ops.c +++ b/ggml/src/ggml-hexagon/htp/fill-ops.c @@ -6,7 +6,7 @@ #include #include "hex-common.h" -#include "hex-profile.h" +#include "htp-trace.h" #include "hvx-copy.h" #include "hvx-utils.h" @@ -36,27 +36,27 @@ struct htp_fill_context { struct htp_ops_context * octx; const struct htp_fill_kernel_params * kparams; - uint32_t nrows_per_thread; + uint32_t nrows_per_hvx; uint32_t total_rows; // ne1 * ne2 * ne3 uint32_t row_start; HVX_Vector splat_vec; uint32_t elem_size; }; -static void fill_thread_contiguous(unsigned int nth, unsigned int ith, void * data) { +static void fill_thread_contiguous(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { const struct htp_fill_context * fctx = (const struct htp_fill_context *) data; struct htp_ops_context * octx = fctx->octx; fill_preamble; // Parallelise over the flat row index spanning ne1*ne2*ne3 - const uint32_t ir0 = fctx->row_start + fctx->nrows_per_thread * ith; - const uint32_t ir1 = MIN(ir0 + fctx->nrows_per_thread, fctx->row_start + fctx->total_rows); + const uint32_t ir0 = fctx->row_start + fctx->nrows_per_hvx * ith; + const uint32_t ir1 = MIN(ir0 + fctx->nrows_per_hvx, fctx->row_start + fctx->total_rows); if (ir0 >= ir1) { return; } - struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + struct htp_thread_trace * tr = th->trace; htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir0); const uint32_t elem_start = ir0 * ne0; @@ -68,21 +68,21 @@ static void fill_thread_contiguous(unsigned int nth, unsigned int ith, void * da FARF(HIGH, "fill-contig %u/%u: rows %u:%u\n", ith, nth, ir0, ir1); } -static void fill_thread_strided(unsigned int nth, unsigned int ith, void * data) { +static void fill_thread_strided(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { const struct htp_fill_context * fctx = (const struct htp_fill_context *) data; struct htp_ops_context * octx = fctx->octx; const struct htp_fill_kernel_params * kparams = fctx->kparams; fill_preamble; // Parallelise over the flat row index spanning ne1*ne2*ne3 - const uint32_t ir0 = fctx->row_start + fctx->nrows_per_thread * ith; - const uint32_t ir1 = MIN(ir0 + fctx->nrows_per_thread, fctx->row_start + fctx->total_rows); + const uint32_t ir0 = fctx->row_start + fctx->nrows_per_hvx * ith; + const uint32_t ir1 = MIN(ir0 + fctx->nrows_per_hvx, fctx->row_start + fctx->total_rows); if (ir0 >= ir1) { return; } - struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + struct htp_thread_trace * tr = th->trace; htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir0); for (uint32_t ir = ir0; ir < ir1; ++ir) { @@ -120,11 +120,11 @@ int op_fill(struct htp_ops_context * octx) { return HTP_STATUS_OK; } - if (!htp_ops_context_set_n_threads(octx, MIN(kparams->n_threads, nrows))) { + if (!htp_ops_context_set_n_hvx(octx, MIN(kparams->n_hvx, nrows))) { return HTP_STATUS_INVAL_PARAMS; } - const uint32_t n_threads = octx->n_threads; + const uint32_t n_hvx = octx->n_hvx; FARF(HIGH, "fill: (%ux%ux%ux%u) type=%u ktype=%u\n", dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3], dst->type, kparams->kernel_type); @@ -135,7 +135,7 @@ int op_fill(struct htp_ops_context * octx) { struct htp_fill_context fctx = { .octx = octx, .kparams = kparams, - .nrows_per_thread = fastdiv(nrows + n_threads - 1, &octx->n_threads_div), + .nrows_per_hvx = fastdiv(nrows + n_hvx - 1, &octx->n_hvx_div), .total_rows = nrows, .row_start = row_start, .elem_size = kparams->elem_size, @@ -156,7 +156,7 @@ int op_fill(struct htp_ops_context * octx) { ? fill_thread_contiguous : fill_thread_strided; - work_queue_run(octx->ctx->work_queue, worker, &fctx, n_threads); + work_queue_run(octx->ctx->hvx_queue, worker, &fctx, n_hvx); return HTP_STATUS_OK; } diff --git a/ggml/src/ggml-hexagon/htp/fill-ops.h b/ggml/src/ggml-hexagon/htp/fill-ops.h index d62f997513d5..06fe082092b0 100644 --- a/ggml/src/ggml-hexagon/htp/fill-ops.h +++ b/ggml/src/ggml-hexagon/htp/fill-ops.h @@ -11,7 +11,7 @@ enum htp_fill_kernel_type { struct htp_fill_kernel_params { uint8_t kernel_type; - uint32_t n_threads; + uint32_t n_hvx; uint32_t total_rows; uint32_t elem_size; uint32_t rows_per_chunk; diff --git a/ggml/src/ggml-hexagon/htp/flash-attn-ops.c b/ggml/src/ggml-hexagon/htp/flash-attn-ops.c index ee258172b1ac..0b7fc2dae77e 100644 --- a/ggml/src/ggml-hexagon/htp/flash-attn-ops.c +++ b/ggml/src/ggml-hexagon/htp/flash-attn-ops.c @@ -14,7 +14,7 @@ #include "dma-queue.h" #include "hex-fastdiv.h" -#include "hex-profile.h" +#include "htp-trace.h" #include "hmx-queue.h" #include "hmx-utils.h" #include "hvx-utils.h" @@ -44,7 +44,7 @@ struct hvx_fa_ddr_spad { __fp16 slopes[512]; uint16_t active_blocks[FA_MAX_KV_BLOCKS]; - uint16_t thread_active_blocks[HTP_MAX_NTHREADS][FA_MAX_KV_BLOCKS] __attribute__((aligned(128))); + uint16_t thread_active_blocks[HTP_MAX_NHVX][FA_MAX_KV_BLOCKS] __attribute__((aligned(128))); }; struct hmx_fa_ddr_spad { @@ -95,7 +95,7 @@ struct htp_fa_context { size_t size_m_block; uint32_t qrows; - uint32_t qrows_per_thread; + uint32_t qrows_per_hvx; uint32_t qrow_start; bool is_q_fp32; @@ -111,7 +111,7 @@ struct htp_fa_context { float * spad_sinks; struct hvx_fa_ddr_spad * ddr; - dma_cache_dm * m_cache; // [n_threads] + dma_cache_dm * m_cache; // [n_hvx] const struct htp_tensor * k; const struct htp_tensor * v; @@ -122,8 +122,8 @@ struct hmx_fa_context { const struct htp_tensor * sinks; // attention sinks (src[4]), NULL if absent const struct htp_tensor * k; const struct htp_tensor * v; - bool pipeline; // true when n_kv_blocks >= FA_MIN_KV_BLOCKS && n_threads >= 2 - uint32_t n_threads; + bool pipeline; // true when n_kv_blocks >= FA_MIN_KV_BLOCKS && n_hvx >= 2 + uint32_t n_hvx; // Op parameters __fp16 scale; @@ -171,7 +171,7 @@ struct hmx_fa_context { HVX_Vector * vtcm_l_vec; // Row sum [g_br] HVX_Vector * vtcm_s_rowmax; // Softmax intermediate [g_br] HVX_Vector * vtcm_p_rowsum; // Softmax intermediate [g_br] - HVX_Vector * vtcm_row_bufs; // Per-thread softmax row scratch [n_threads][2][Bc/64] + HVX_Vector * vtcm_row_bufs; // Per-HVX softmax row scratch [n_hvx][2][Bc/64] uint8_t * vtcm_hmx_scales_id; // HMX output scales (identity) uint8_t * vtcm_hmx_scales_qk; // HMX output scales (qk_scale) __fp16 * vtcm_mask_buf; // VTCM mask buffer [Br * m_line], DMA'd per KV block @@ -260,7 +260,7 @@ static uint32_t fa_scan_active_blocks( return n_active; } -static void flash_attn_ext_f16_thread(unsigned int nth, unsigned int ith, void * data) { +static void flash_attn_ext_f16_thread(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { struct htp_fa_context * factx = (struct htp_fa_context *) data; const struct htp_ops_context * octx = factx->octx; const struct htp_tensor * q = octx->src[0]; @@ -306,15 +306,15 @@ static void flash_attn_ext_f16_thread(unsigned int nth, unsigned int ith, void * const uint32_t nb3 = dst->nb[3]; // total rows in q - const uint32_t dr = factx->qrows_per_thread; + const uint32_t dr = factx->qrows_per_hvx; const uint32_t ir0 = factx->qrow_start + dr * ith; const uint32_t ir1 = MIN(ir0 + dr, factx->qrow_start + factx->qrows); if (ir0 >= ir1) return; - struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + struct htp_thread_trace * tr = th->trace; - dma_queue * dma_q = octx->ctx->dma[ith]; + dma_queue * dma_q = th->dma_q; const uint32_t DK = nek0; const uint32_t DV = nev0; @@ -774,7 +774,7 @@ typedef struct { size_t buf_idx; } fa_k_int_args_t; -static void fa_k_interleave_thread(unsigned int n, unsigned int i, void * data) { +static void fa_k_interleave_thread(const htp_thread_t * th, unsigned int n, unsigned int i, void * data) { fa_k_int_args_t * args = (fa_k_int_args_t *) data; struct hmx_fa_context * factx = args->factx; @@ -787,7 +787,7 @@ static void fa_k_interleave_thread(unsigned int n, unsigned int i, void * data) return; } - struct htp_thread_trace * tr = &factx->octx->ctx->trace[i]; + struct htp_thread_trace * tr = th->trace; htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_FA_K_PREP, (uint16_t) (args->kv_start + start)); if (factx->k->type == HTP_TYPE_Q8_0) { for (uint32_t r = start; r < end; ++r) { @@ -801,18 +801,14 @@ static void fa_k_interleave_thread(unsigned int n, unsigned int i, void * data) } static void fa_phase_k_interleave(struct hmx_fa_context * factx, uint32_t kv_rows, size_t src_stride, void * curr_k, uint32_t kv_start, size_t buf_idx) { - work_queue_t wp = factx->octx->ctx->work_queue; + work_queue_t wp = factx->octx->ctx->hvx_queue; uint32_t n = 1; - if (factx->n_threads > 1 && kv_rows >= factx->n_threads * 2) { - n = factx->n_threads; + if (factx->n_hvx > 1 && kv_rows >= factx->n_hvx * 2) { + n = factx->n_hvx; } uint32_t rows_per_t = hex_align_up(hmx_ceil_div(kv_rows, n), 2); fa_k_int_args_t args = { factx, kv_rows, src_stride, curr_k, kv_start, rows_per_t, buf_idx }; - if (n > 1) { - work_queue_run(wp, fa_k_interleave_thread, &args, n); - } else { - fa_k_interleave_thread(1, 0, &args); - } + work_queue_run(wp, fa_k_interleave_thread, &args, n); } typedef struct { @@ -826,7 +822,7 @@ typedef struct { uint32_t rows_per_t; } fa_v_int_args_t; -static void fa_v_interleave_thread(unsigned int n, unsigned int i, void * data) { +static void fa_v_interleave_thread(const htp_thread_t * th, unsigned int n, unsigned int i, void * data) { fa_v_int_args_t * args = (fa_v_int_args_t *) data; struct hmx_fa_context * factx = args->factx; @@ -841,7 +837,7 @@ static void fa_v_interleave_thread(unsigned int n, unsigned int i, void * data) __fp16 * v_tiles_dst = (__fp16 *) args->v_tiles_dst; - struct htp_thread_trace * tr = &factx->octx->ctx->trace[i]; + struct htp_thread_trace * tr = th->trace; htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_FA_V_PREP, (uint16_t) (args->kv_start + start)); if (factx->v->type == HTP_TYPE_Q8_0) { for (uint32_t r = start; r < end; ++r) { @@ -861,18 +857,14 @@ static void fa_phase_v_interleave(struct hmx_fa_context * factx, void * v_tiles_dst, size_t n_col_tiles, uint32_t kv_start) { - work_queue_t wp = factx->octx->ctx->work_queue; + work_queue_t wp = factx->octx->ctx->hvx_queue; uint32_t n = 1; - if (factx->n_threads > 1 && kv_rows >= factx->n_threads * 2) { - n = factx->n_threads; + if (factx->n_hvx > 1 && kv_rows >= factx->n_hvx * 2) { + n = factx->n_hvx; } uint32_t rows_per_t = hex_align_up(hmx_ceil_div(kv_rows, n), 2); fa_v_int_args_t args = { factx, kv_rows, src_stride, v_src, v_tiles_dst, n_col_tiles, kv_start, rows_per_t }; - if (n > 1) { - work_queue_run(wp, fa_v_interleave_thread, &args, n); - } else { - fa_v_interleave_thread(1, 0, &args); - } + work_queue_run(wp, fa_v_interleave_thread, &args, n); } typedef struct { @@ -888,7 +880,7 @@ typedef struct { atomic_uint barrier; } fa_q_load_args_t; -static void fa_q_load_thread(unsigned int n, unsigned int i, void * data) { +static void fa_q_load_thread(const htp_thread_t * th, unsigned int n, unsigned int i, void * data) { fa_q_load_args_t * args = (fa_q_load_args_t *) data; struct hmx_fa_context * factx = args->factx; @@ -902,7 +894,7 @@ static void fa_q_load_thread(unsigned int n, unsigned int i, void * data) { const size_t start = (size_t) i * rows_per_t; const size_t end = hex_smin(start + rows_per_t, factx->g_br); - struct htp_thread_trace * tr = &factx->octx->ctx->trace[i]; + struct htp_thread_trace * tr = th->trace; htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_FA_Q_PREP, (uint16_t) (args->q_start * G + start)); // Parallel initialization of per-block state @@ -1010,10 +1002,10 @@ static void fa_phase_q_load(struct hmx_fa_context * factx, uint32_t kv_head, uint32_t ib3, size_t n_rows_g) { - work_queue_t wp = factx->octx->ctx->work_queue; + work_queue_t wp = factx->octx->ctx->hvx_queue; uint32_t n = 1; - if (factx->n_threads > 1 && n_rows_g >= (size_t) (factx->n_threads * 2)) { - n = factx->n_threads; + if (factx->n_hvx > 1 && n_rows_g >= (size_t) (factx->n_hvx * 2)) { + n = factx->n_hvx; } size_t rows_per_t = hex_align_up(hmx_ceil_div(factx->g_br, n), 2); const uint32_t n_rows_q = hex_smin(factx->Br, factx->neq1 - q_start); @@ -1028,11 +1020,7 @@ static void fa_phase_q_load(struct hmx_fa_context * factx, args.n_rows_q = n_rows_q; args.q_transposed = q->nb[1] < q->nb[2]; atomic_init(&args.barrier, n); - if (n > 1) { - work_queue_run(wp, fa_q_load_thread, &args, n); - } else { - fa_q_load_thread(1, 0, &args); - } + work_queue_run(wp, fa_q_load_thread, &args, n); } typedef struct { @@ -1046,7 +1034,7 @@ typedef struct { size_t rows_per_t; } fa_o_store_args_t; -static void fa_o_store_thread_f32(unsigned int n, unsigned int i, void * data) { +static void fa_o_store_thread_f32(const htp_thread_t * th, unsigned int n, unsigned int i, void * data) { fa_o_store_args_t * args = (fa_o_store_args_t *) data; struct hmx_fa_context * factx = args->factx; @@ -1062,7 +1050,7 @@ static void fa_o_store_thread_f32(unsigned int n, unsigned int i, void * data) { return; } - struct htp_thread_trace * tr = &factx->octx->ctx->trace[i]; + struct htp_thread_trace * tr = th->trace; htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_O_PROC, (uint16_t) (args->q_start * G + start)); const struct htp_tensor * dst = args->dst; @@ -1113,7 +1101,7 @@ static void fa_o_store_thread_f32(unsigned int n, unsigned int i, void * data) { htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_O_PROC, (uint16_t) (args->q_start * G + start)); } -static void fa_o_store_thread_f16(unsigned int n, unsigned int i, void * data) { +static void fa_o_store_thread_f16(const htp_thread_t * th, unsigned int n, unsigned int i, void * data) { fa_o_store_args_t * args = (fa_o_store_args_t *) data; struct hmx_fa_context * factx = args->factx; @@ -1128,7 +1116,7 @@ static void fa_o_store_thread_f16(unsigned int n, unsigned int i, void * data) { return; } - struct htp_thread_trace * tr = &factx->octx->ctx->trace[i]; + struct htp_thread_trace * tr = th->trace; htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_O_PROC, (uint16_t) (args->q_start * G + start)); const struct htp_tensor * dst = args->dst; @@ -1190,19 +1178,15 @@ static void fa_phase_o_store(struct hmx_fa_context * factx, uint32_t kv_head, uint32_t ib3, size_t n_rows_g) { - work_queue_t wp = factx->octx->ctx->work_queue; + work_queue_t wp = factx->octx->ctx->hvx_queue; uint32_t n = 1; - if (factx->n_threads > 1 && n_rows_g >= (size_t) (factx->n_threads * 2)) { - n = factx->n_threads; + if (factx->n_hvx > 1 && n_rows_g >= (size_t) (factx->n_hvx * 2)) { + n = factx->n_hvx; } size_t rows_per_t = hmx_ceil_div(n_rows_g, n); fa_o_store_args_t args = { factx, dst, o_tile_src, q_start, kv_head, ib3, n_rows_g, rows_per_t }; - worker_callback_t store_fn = factx->is_dst_fp32 ? fa_o_store_thread_f32 : fa_o_store_thread_f16; - if (n > 1) { - work_queue_run(wp, store_fn, &args, n); - } else { - store_fn(1, 0, &args); - } + work_queue_func_t store_fn = factx->is_dst_fp32 ? fa_o_store_thread_f32 : fa_o_store_thread_f16; + work_queue_run(wp, store_fn, &args, n); } typedef struct { @@ -1229,7 +1213,7 @@ typedef struct { } fa_softmax_args_t; static inline void fa_softmax_impl( - unsigned int n, unsigned int i, void * data, + const htp_thread_t * th, unsigned int n, unsigned int i, void * data, const bool has_mask, const bool mask_broadcast, const bool is_g1, @@ -1259,7 +1243,7 @@ static inline void fa_softmax_impl( return; } - struct htp_thread_trace * tr = &factx->octx->ctx->trace[i]; + struct htp_thread_trace * tr = th->trace; htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_FA_SFM, (uint16_t) (args->q_start * G + vec_start * 64)); // Per-thread row scratch: thread i uses bufs at offset i * 2 * stride @@ -1619,8 +1603,8 @@ static inline void fa_softmax_impl( htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_FA_SFM, (uint16_t) (args->q_start * G + vec_start * 64)); } -static void fa_softmax_thread_nomask(unsigned int n, unsigned int i, void * data) { - fa_softmax_impl(n, i, data, +static void fa_softmax_thread_nomask(const htp_thread_t * th, unsigned int n, unsigned int i, void * data) { + fa_softmax_impl(th, n, i, data, /*has_mask=*/false, /*mask_broadcast=*/false, /*is_g1=*/false, @@ -1628,8 +1612,8 @@ static void fa_softmax_thread_nomask(unsigned int n, unsigned int i, void * data /*has_softcap=*/false); } -static void fa_softmax_thread_mask_broadcast_g1(unsigned int n, unsigned int i, void * data) { - fa_softmax_impl(n, i, data, +static void fa_softmax_thread_mask_broadcast_g1(const htp_thread_t * th, unsigned int n, unsigned int i, void * data) { + fa_softmax_impl(th, n, i, data, /*has_mask=*/true, /*mask_broadcast=*/true, /*is_g1=*/true, @@ -1637,8 +1621,8 @@ static void fa_softmax_thread_mask_broadcast_g1(unsigned int n, unsigned int i, /*has_softcap=*/false); } -static void fa_softmax_thread_mask_broadcast_gn(unsigned int n, unsigned int i, void * data) { - fa_softmax_impl(n, i, data, +static void fa_softmax_thread_mask_broadcast_gn(const htp_thread_t * th, unsigned int n, unsigned int i, void * data) { + fa_softmax_impl(th, n, i, data, /*has_mask=*/true, /*mask_broadcast=*/true, /*is_g1=*/false, @@ -1646,7 +1630,7 @@ static void fa_softmax_thread_mask_broadcast_gn(unsigned int n, unsigned int i, /*has_softcap=*/false); } -static void fa_softmax_thread(unsigned int n, unsigned int i, void * data) { +static void fa_softmax_thread(const htp_thread_t * th, unsigned int n, unsigned int i, void * data) { fa_softmax_args_t * args = (fa_softmax_args_t *) data; struct hmx_fa_context * factx = args->factx; @@ -1656,7 +1640,7 @@ static void fa_softmax_thread(unsigned int n, unsigned int i, void * data) { const bool has_alibi = args->has_alibi; const bool has_softcap = factx->has_softcap; - fa_softmax_impl(n, i, data, has_mask, mask_broadcast, is_g1, has_alibi, has_softcap); + fa_softmax_impl(th, n, i, data, has_mask, mask_broadcast, is_g1, has_alibi, has_softcap); } static inline HVX_Vector hvx_fa_load_sinks(const float * sinks_data, size_t idx, size_t count) { @@ -1742,10 +1726,10 @@ static void fa_phase_softmax_and_build_d(struct hmx_fa_context * factx, fa_softmax_args_t * sargs, size_t n_row_tiles, size_t n_row_tiles_g_br) { - work_queue_t wp = factx->octx->ctx->work_queue; + work_queue_t wp = factx->octx->ctx->hvx_queue; const size_t n_row_vec_cnt = hmx_ceil_div(sargs->n_rows_g, 64); - worker_callback_t softmax_fn = fa_softmax_thread; + work_queue_func_t softmax_fn = fa_softmax_thread; if (sargs->mask == NULL && !factx->has_softcap && !sargs->has_alibi) { softmax_fn = fa_softmax_thread_nomask; } else if (sargs->mask != NULL && factx->mask_broadcast && !factx->has_softcap && !sargs->has_alibi) { @@ -1756,13 +1740,12 @@ static void fa_phase_softmax_and_build_d(struct hmx_fa_context * factx, } } - if (factx->n_threads > 1 && n_row_vec_cnt >= 2) { - uint32_t n_use = (uint32_t) hex_smin((size_t) factx->n_threads, n_row_vec_cnt); + uint32_t n_use = 1; + if (factx->n_hvx > 1 && n_row_vec_cnt >= 2) { + n_use = (uint32_t) hex_smin((size_t) factx->n_hvx, n_row_vec_cnt); sargs->thread_div = init_fastdiv_values(n_use); - work_queue_run(wp, softmax_fn, sargs, n_use); - } else { - softmax_fn(1, 0, sargs); } + work_queue_run(wp, softmax_fn, sargs, n_use); } // ============================================================================ @@ -1780,7 +1763,7 @@ typedef struct { uint8_t * hmx_scales; } hmx_fa_qk_job_t; -static void hmx_fa_qk_dot_worker(void * data) { +static void hmx_fa_qk_dot_worker(const htp_thread_t * th, void * data) { hmx_fa_qk_job_t * job = (hmx_fa_qk_job_t *) data; const size_t n_row_tiles = job->n_row_tiles; const size_t n_col_tiles = job->n_col_tiles; @@ -1822,7 +1805,7 @@ typedef struct { size_t DV; } hmx_fa_o_update_job_t; -static void hmx_fa_o_update_worker(void * data) { +static void hmx_fa_o_update_worker(const htp_thread_t * th, void * data) { hmx_fa_o_update_job_t * job = (hmx_fa_o_update_job_t *) data; const size_t n_row_tiles = job->n_row_tiles; const size_t n_col_tiles = job->n_col_tiles; @@ -1867,7 +1850,7 @@ typedef struct { size_t DV; } hmx_fa_o_norm_job_t; -static void hmx_fa_o_norm_worker(void * data) { +static void hmx_fa_o_norm_worker(const htp_thread_t * th, void * data) { hmx_fa_o_norm_job_t * job = (hmx_fa_o_norm_job_t *) data; const size_t n_row_tiles = job->n_row_tiles; const size_t n_row_tiles_g_br = job->n_row_tiles_g_br; @@ -1992,7 +1975,6 @@ static inline void fa_prefetch_block(dma_queue * dma_q, const struct htp_tensor int hmx_flash_attn_ext(struct htp_ops_context * octx) { struct htp_thread_trace * tr_hvx = &octx->ctx->trace[0]; - struct htp_thread_trace * tr_hmx = &octx->ctx->trace[HTP_MAX_NTHREADS]; const struct htp_tensor * q = octx->src[0]; const struct htp_tensor * k = octx->src[1]; const struct htp_tensor * v = octx->src[2]; @@ -2037,7 +2019,7 @@ int hmx_flash_attn_ext(struct htp_ops_context * octx) { factx.sinks = octx->src[4]; // NULL if this op has no attention sinks factx.k = k; factx.v = v; - factx.n_threads = kparams->n_threads; + factx.n_hvx = kparams->n_hvx; factx.DK = DK; factx.DV = DV; factx.DK_pad = DK_pad; @@ -2077,7 +2059,7 @@ int hmx_flash_attn_ext(struct htp_ops_context * octx) { const uint32_t Bc = factx.Bc; const uint32_t g_br = factx.g_br; const bool pipeline = factx.pipeline; - const uint32_t n_threads = factx.n_threads; + const uint32_t n_hvx = factx.n_hvx; const uint32_t G = factx.G; // Multi-device: prefer head-parallel partitioning (each core owns a disjoint head @@ -2147,7 +2129,7 @@ int hmx_flash_attn_ext(struct htp_ops_context * octx) { // Build the VTCM layout once (shared with the host estimator) and place every // scratch buffer at its computed offset. Padded head dims size the HMX tiles. struct hmx_fa_vtcm_layout L; - hmx_fa_vtcm_layout_build(&L, G, DK_pad, DV_pad, Br, Bc, n_threads, pipeline, factx.is_q_fp32, factx.sinks != NULL, factx.n_heads); + hmx_fa_vtcm_layout_build(&L, G, DK_pad, DV_pad, Br, Bc, n_hvx, pipeline, factx.is_q_fp32, factx.sinks != NULL, factx.n_heads); if (L.total_bytes > ctx->vtcm_size) { return HTP_STATUS_VTCM_TOO_SMALL; @@ -2681,7 +2663,7 @@ int op_flash_attn_ext(struct htp_ops_context * octx) { return HTP_STATUS_NO_SUPPORT; } - if (!htp_ops_context_set_n_threads(octx, kparams->n_threads)) { + if (!htp_ops_context_set_n_hvx(octx, kparams->n_hvx)) { return HTP_STATUS_INVAL_PARAMS; } @@ -2776,16 +2758,16 @@ int op_flash_attn_ext(struct htp_ops_context * octx) { return HTP_STATUS_OK; } - const uint32_t n_threads = octx->n_threads; + const uint32_t n_hvx = octx->n_hvx; factx.qrows = qrows; factx.qrow_start = qrow_start; - factx.qrows_per_thread = fastdiv(qrows + n_threads - 1, &octx->n_threads_div); + factx.qrows_per_hvx = fastdiv(qrows + n_hvx - 1, &octx->n_hvx_div); const bool has_mask = (mask != NULL); const bool has_sinks = (octx->src[4] != NULL); struct hvx_fa_vtcm_layout L; - hvx_fa_vtcm_layout_build(&L, k->ne[0], v->ne[0], factx.is_q_fp32, has_mask, has_sinks, n_head, n_threads); + hvx_fa_vtcm_layout_build(&L, k->ne[0], v->ne[0], factx.is_q_fp32, has_mask, has_sinks, n_head, n_hvx); if (L.total_bytes > octx->ctx->vtcm_size) { return HTP_STATUS_VTCM_TOO_SMALL; @@ -2822,7 +2804,7 @@ int op_flash_attn_ext(struct htp_ops_context * octx) { dma_queue_pop(dma_q); } - work_queue_run(octx->ctx->work_queue, flash_attn_ext_f16_thread, &factx, n_threads); + work_queue_run(octx->ctx->hvx_queue, flash_attn_ext_f16_thread, &factx, n_hvx); return HTP_STATUS_OK; } diff --git a/ggml/src/ggml-hexagon/htp/flash-attn-ops.h b/ggml/src/ggml-hexagon/htp/flash-attn-ops.h index f2654f0a83dc..70cf3e41aa23 100644 --- a/ggml/src/ggml-hexagon/htp/flash-attn-ops.h +++ b/ggml/src/ggml-hexagon/htp/flash-attn-ops.h @@ -46,7 +46,7 @@ struct htp_fa_kernel_params { uint8_t kernel_type; // enum htp_fa_kernel_type uint8_t head_split; // 1 = partition by KV heads in multicore, 0 = token partition uint8_t flags; // HTP_FA_FLAG_* - uint8_t n_threads; // Number of threads to run + uint8_t n_hvx; // Number of threads to run // Common parameters uint16_t Br; @@ -155,7 +155,7 @@ struct hmx_fa_vtcm_layout { static inline void hmx_fa_vtcm_layout_build(struct hmx_fa_vtcm_layout * L, size_t gqa_factor, size_t DK, size_t DV, - size_t Br, size_t Bc, size_t n_threads, + size_t Br, size_t Bc, size_t n_hvx, bool pipeline, bool is_q_fp32, bool has_sinks, size_t n_heads) { const size_t g_br = hex_align_up(gqa_factor * Br, HMX_FP16_TILE_N_ROWS); @@ -207,7 +207,7 @@ static inline void hmx_fa_vtcm_layout_build(struct hmx_fa_vtcm_layout * L, VTCM_LAYOUT_ALLOC_OPTIONAL(off_group_b, off_p_tiles[1], s_tile_size, pipeline); VTCM_LAYOUT_ALLOC(off_group_b, off_s_rowmax, col_vec_size); VTCM_LAYOUT_ALLOC(off_group_b, off_p_rowsum, col_vec_size); - VTCM_LAYOUT_ALLOC(off_group_b, off_row_bufs, row_vec_size * 2 * n_threads); + VTCM_LAYOUT_ALLOC(off_group_b, off_row_bufs, row_vec_size * 2 * n_hvx); const size_t group_b_size = off_group_b - off_group_b_c; @@ -251,9 +251,9 @@ static inline void hmx_fa_vtcm_layout_build(struct hmx_fa_vtcm_layout * L, } // Exact VTCM usage for a given (gqa_factor, DK, DV, Br, Bc) configuration. -static inline size_t hmx_fa_compute_vtcm_usage(size_t gqa_factor, size_t DK, size_t DV, size_t Br, size_t Bc, size_t n_threads, bool pipeline, bool is_q_fp32, bool has_sinks, size_t n_heads) { +static inline size_t hmx_fa_compute_vtcm_usage(size_t gqa_factor, size_t DK, size_t DV, size_t Br, size_t Bc, size_t n_hvx, bool pipeline, bool is_q_fp32, bool has_sinks, size_t n_heads) { struct hmx_fa_vtcm_layout L; - hmx_fa_vtcm_layout_build(&L, gqa_factor, DK, DV, Br, Bc, n_threads, pipeline, is_q_fp32, has_sinks, n_heads); + hmx_fa_vtcm_layout_build(&L, gqa_factor, DK, DV, Br, Bc, n_hvx, pipeline, is_q_fp32, has_sinks, n_heads); return L.total_bytes; } @@ -282,7 +282,7 @@ static inline void hvx_fa_vtcm_layout_build(struct hvx_fa_vtcm_layout * L, size_t DK, size_t DV, bool is_q_fp32, bool has_mask, bool has_sinks, size_t n_heads, - size_t n_threads) { + size_t n_hvx) { const size_t size_q_row_padded = hex_round_up(DK * (is_q_fp32 ? 4 : 2), 128); const size_t size_k_row_padded = hex_round_up(DK * sizeof(__fp16), 128); const size_t size_v_row_padded = hex_round_up(DV * sizeof(__fp16), 128); @@ -296,11 +296,11 @@ static inline void hvx_fa_vtcm_layout_build(struct hvx_fa_vtcm_layout * L, size_t off = 0; - VTCM_LAYOUT_ALLOC(off, off_q, size_q_block * n_threads); - VTCM_LAYOUT_ALLOC(off, off_k, size_k_block * 2 * n_threads); - VTCM_LAYOUT_ALLOC(off, off_v, size_v_block * 2 * n_threads); - VTCM_LAYOUT_ALLOC_OPTIONAL(off, off_m, size_m_block * HVX_FA_DMA_CACHE_SIZE * n_threads, has_mask); - VTCM_LAYOUT_ALLOC(off, off_a, size_vkq_acc * n_threads); + VTCM_LAYOUT_ALLOC(off, off_q, size_q_block * n_hvx); + VTCM_LAYOUT_ALLOC(off, off_k, size_k_block * 2 * n_hvx); + VTCM_LAYOUT_ALLOC(off, off_v, size_v_block * 2 * n_hvx); + VTCM_LAYOUT_ALLOC_OPTIONAL(off, off_m, size_m_block * HVX_FA_DMA_CACHE_SIZE * n_hvx, has_mask); + VTCM_LAYOUT_ALLOC(off, off_a, size_vkq_acc * n_hvx); VTCM_LAYOUT_ALLOC_OPTIONAL(off, off_sinks, size_sinks, has_sinks); L->size_q_block = size_q_block; @@ -312,9 +312,9 @@ static inline void hvx_fa_vtcm_layout_build(struct hvx_fa_vtcm_layout * L, L->total_bytes = off; } -static inline size_t hvx_fa_compute_vtcm_usage(size_t DK, size_t DV, bool is_q_fp32, bool has_mask, bool has_sinks, size_t n_heads, size_t n_threads) { +static inline size_t hvx_fa_compute_vtcm_usage(size_t DK, size_t DV, bool is_q_fp32, bool has_mask, bool has_sinks, size_t n_heads, size_t n_hvx) { struct hvx_fa_vtcm_layout L; - hvx_fa_vtcm_layout_build(&L, DK, DV, is_q_fp32, has_mask, has_sinks, n_heads, n_threads); + hvx_fa_vtcm_layout_build(&L, DK, DV, is_q_fp32, has_mask, has_sinks, n_heads, n_hvx); return L.total_bytes; } @@ -329,7 +329,7 @@ static inline int hmx_fa_find_chunk_size(size_t * Br_out, size_t qo_len, size_t kv_len, size_t vtcm_budget, - size_t n_threads, + size_t n_hvx, bool is_q_fp32, bool has_sinks, size_t n_heads, @@ -337,7 +337,7 @@ static inline int hmx_fa_find_chunk_size(size_t * Br_out, const size_t T = HMX_FP16_TILE_N_ROWS; // 32 const size_t br_unit = hmx_ceil_div(T, gqa_factor); const size_t bc_unit = HMX_FP16_TILE_N_COLS * 2; // 64 - const bool can_pipeline = (kv_len >= FA_MIN_KV_BLOCKS * bc_unit && n_threads >= 2); + const bool can_pipeline = (kv_len >= FA_MIN_KV_BLOCKS * bc_unit && n_hvx >= 2); // Br_max: largest Br aligned to br_unit that does not exceed qo_len. const size_t Br_max = qo_len >= br_unit ? hex_align_down(qo_len, br_unit) : br_unit; @@ -361,15 +361,15 @@ static inline int hmx_fa_find_chunk_size(size_t * Br_out, for (size_t Br = Br_max; Br >= br_unit; Br -= br_unit) { // Try all Bc candidates from Bc_limit down to bc_unit for (size_t Bc = Bc_limit; Bc >= bc_unit; Bc -= bc_unit) { - size_t vtcm_needed = hmx_fa_compute_vtcm_usage(gqa_factor, DK, DV, Br, Bc, n_threads, can_pipeline, is_q_fp32, has_sinks, n_heads); + size_t vtcm_needed = hmx_fa_compute_vtcm_usage(gqa_factor, DK, DV, Br, Bc, n_hvx, can_pipeline, is_q_fp32, has_sinks, n_heads); if (vtcm_needed <= vtcm_budget) { // This Bc fits for this Br! const size_t q_blocks = (qo_len + Br - 1) / Br; const size_t kv_blocks = (kv_len + Bc - 1) / Bc; - const size_t actual_threads = (kv_blocks >= 3 && n_threads >= 2) ? n_threads : 1; + const size_t actual_hvx = (kv_blocks >= 3 && n_hvx >= 2) ? n_hvx : 1; const size_t n_rows_g = Br * gqa_factor; const size_t n_row_vec_cnt = (n_rows_g + 63) / 64; - const size_t n_use = n_row_vec_cnt < actual_threads ? n_row_vec_cnt : actual_threads; + const size_t n_use = n_row_vec_cnt < actual_hvx ? n_row_vec_cnt : actual_hvx; const size_t vecs_per_t = n_use > 0 ? (n_row_vec_cnt + n_use - 1) / n_use : 1; const size_t c_iter_actual = c_iter_base + c_softmax * vecs_per_t; diff --git a/ggml/src/ggml-hexagon/htp/gated-delta-net-ops.c b/ggml/src/ggml-hexagon/htp/gated-delta-net-ops.c index a70f411858c6..0280f99a4af4 100644 --- a/ggml/src/ggml-hexagon/htp/gated-delta-net-ops.c +++ b/ggml/src/ggml-hexagon/htp/gated-delta-net-ops.c @@ -748,7 +748,7 @@ static inline void gdn_step_scalar_f32( } } -static void gated_delta_net_f32_pp_thread(unsigned int nth, unsigned int ith, void * data) { +static void gated_delta_net_f32_pp_thread(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { struct htp_gdn_context * gctx = (struct htp_gdn_context *) data; struct htp_ops_context * octx = gctx->octx; const struct htp_gdn_kernel_params * kparams = gctx->kparams; @@ -777,10 +777,10 @@ static void gated_delta_net_f32_pp_thread(unsigned int nth, unsigned int ith, vo float * dst_base = (float *) (uintptr_t) dst->data; float * state_out_base = dst_cache ? (float *) (uintptr_t) dst_cache->data : (dst_base + S_v * H * n_tokens * n_seqs); - dma_queue * dma_q = octx->ctx->dma[ith]; + dma_queue * dma_q = th->dma_q; const struct htp_gdn_vtcm_layout * layout = &gctx->layout; float * s_work[2]; - s_work[0] = (float *) (gctx->vtcm_base + layout->bytes_per_thread * ith); + s_work[0] = (float *) (gctx->vtcm_base + layout->bytes_per_hvx * ith); s_work[1] = s_work[0] + layout->state_aligned / sizeof(float); const struct fastdiv_values * fd_H = &kparams->div_H; @@ -817,7 +817,7 @@ static void gated_delta_net_f32_pp_thread(unsigned int nth, unsigned int ith, vo spad_idx ^= 1; } - struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + struct htp_thread_trace * tr = th->trace; int curr_spad_idx = 0; for (uint32_t ir = gctx->row_start + ith; ir < row_end; ir += nth) { @@ -892,7 +892,7 @@ static void gated_delta_net_f32_pp_thread(unsigned int nth, unsigned int ith, vo dma_queue_flush(dma_q); } -static void gated_delta_net_f32_tg_thread(unsigned int nth, unsigned int ith, void * data) { +static void gated_delta_net_f32_tg_thread(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { struct htp_gdn_context * gctx = (struct htp_gdn_context *) data; struct htp_ops_context * octx = gctx->octx; const struct htp_gdn_kernel_params * kparams = gctx->kparams; @@ -918,10 +918,10 @@ static void gated_delta_net_f32_tg_thread(unsigned int nth, unsigned int ith, vo const float scale = kparams->scale; float * dst_base = (float *) (uintptr_t) dst->data; - dma_queue * dma_q = octx->ctx->dma[ith]; + dma_queue * dma_q = th->dma_q; const struct htp_gdn_vtcm_layout * layout = &gctx->layout; float * s_work[2]; - s_work[0] = (float *) (gctx->vtcm_base + layout->bytes_per_thread * ith); + s_work[0] = (float *) (gctx->vtcm_base + layout->bytes_per_hvx * ith); s_work[1] = s_work[0] + layout->state_aligned / sizeof(float); const struct fastdiv_values * fd_H = &kparams->div_H; @@ -957,7 +957,7 @@ static void gated_delta_net_f32_tg_thread(unsigned int nth, unsigned int ith, vo spad_idx ^= 1; } - struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + struct htp_thread_trace * tr = th->trace; int curr_spad_idx = 0; for (uint32_t ir = gctx->row_start + ith; ir < row_end; ir += nth) { @@ -1031,7 +1031,7 @@ struct htp_gdn_hmx_gemm_task { uint8_t * hmx_scales; }; -static void htp_gdn_hmx_gemm_worker(void * data) { +static void htp_gdn_hmx_gemm_worker(const htp_thread_t * th, void * data) { struct htp_gdn_hmx_gemm_task * task = (struct htp_gdn_hmx_gemm_task *) data; asm volatile(HMX_SET_BIAS("%0") :: "r"((unsigned int)task->hmx_scales)); @@ -1623,7 +1623,7 @@ static inline void gdn_init_head_ptrs( head->rows_inv = VTCM_LAYOUT_PTR(HVX_Vector, vtcm_base, L->off_rows_inv + h * row_vecs_sz); head->rows_a = VTCM_LAYOUT_PTR(HVX_Vector, vtcm_base, L->off_rows_a + h * row_vecs_sz); - head->vtcm_m = VTCM_LAYOUT_PTR(HVX_Vector, vtcm_base, L->off_thread_scratch + h * (64 * 128)); + head->vtcm_m = VTCM_LAYOUT_PTR(HVX_Vector, vtcm_base, L->off_hvx_scratch + h * (64 * 128)); head->vtcm_tmp = head->vtcm_m + 32; head->iv1 = base_iv1 + h; @@ -1654,10 +1654,10 @@ struct htp_gdn_batch_context { const struct htp_gdn_kernel_params * kparams; }; -static void gdn_hvx_init_state_worker(unsigned int n, unsigned int i, void * data) { +static void gdn_hvx_init_state_worker(const htp_thread_t * th, unsigned int n, unsigned int i, void * data) { (void) n; struct htp_gdn_batch_context * bctx = (struct htp_gdn_batch_context *) data; - struct htp_thread_trace * tr = &bctx->octx->ctx->trace[i]; + struct htp_thread_trace * tr = th->trace; htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, 0); struct htp_gdn_head_ptrs * head = &bctx->heads[i]; @@ -1690,10 +1690,10 @@ static inline HVX_Vector hvx_splat_last_f32(HVX_Vector v) { return hvx_vec_repl4(Q6_V_vror_VR(v, 124)); } -static void gdn_hvx_phase1a_worker(unsigned int n, unsigned int i, void * data) { +static void gdn_hvx_phase1a_worker(const htp_thread_t * th, unsigned int n, unsigned int i, void * data) { (void) n; struct htp_gdn_batch_context * bctx = (struct htp_gdn_batch_context *) data; - struct htp_thread_trace * tr = &bctx->octx->ctx->trace[i]; + struct htp_thread_trace * tr = th->trace; const uint16_t info = (uint16_t) bctx->c; htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_GDN_PREP, info); @@ -1763,10 +1763,10 @@ static void gdn_hvx_phase1a_worker(unsigned int n, unsigned int i, void * data) htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_GDN_PREP, info); } -static void gdn_hvx_phase1b_worker(unsigned int n, unsigned int i, void * data) { +static void gdn_hvx_phase1b_worker(const htp_thread_t * th, unsigned int n, unsigned int i, void * data) { (void) n; struct htp_gdn_batch_context * bctx = (struct htp_gdn_batch_context *) data; - struct htp_thread_trace * tr = &bctx->octx->ctx->trace[i]; + struct htp_thread_trace * tr = th->trace; const uint16_t info = (uint16_t) bctx->c; htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_GDN_PREP, info); @@ -1860,10 +1860,10 @@ static void gdn_hvx_phase1b_worker(unsigned int n, unsigned int i, void * data) htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_GDN_PREP, info); } -static void gdn_hvx_phase2_worker(unsigned int n, unsigned int i, void * data) { +static void gdn_hvx_phase2_worker(const htp_thread_t * th, unsigned int n, unsigned int i, void * data) { (void) n; struct htp_gdn_batch_context * bctx = (struct htp_gdn_batch_context *) data; - struct htp_thread_trace * tr = &bctx->octx->ctx->trace[i]; + struct htp_thread_trace * tr = th->trace; const uint16_t info = (uint16_t) bctx->c; htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_GDN_SOLVE, info); @@ -1884,10 +1884,10 @@ static void gdn_hvx_phase2_worker(unsigned int n, unsigned int i, void * data) { htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_GDN_SOLVE, info); } -static void gdn_hvx_phase3_worker(unsigned int n, unsigned int i, void * data) { +static void gdn_hvx_phase3_worker(const htp_thread_t * th, unsigned int n, unsigned int i, void * data) { (void) n; struct htp_gdn_batch_context * bctx = (struct htp_gdn_batch_context *) data; - struct htp_thread_trace * tr = &bctx->octx->ctx->trace[i]; + struct htp_thread_trace * tr = th->trace; const uint16_t info = (uint16_t) bctx->c; htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_GDN_V_PREP, info); @@ -1928,10 +1928,10 @@ static void gdn_hvx_phase3_worker(unsigned int n, unsigned int i, void * data) { htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_GDN_V_PREP, info); } -static void gdn_hvx_phase4_worker(unsigned int n, unsigned int i, void * data) { +static void gdn_hvx_phase4_worker(const htp_thread_t * th, unsigned int n, unsigned int i, void * data) { (void) n; struct htp_gdn_batch_context * bctx = (struct htp_gdn_batch_context *) data; - struct htp_thread_trace * tr = &bctx->octx->ctx->trace[i]; + struct htp_thread_trace * tr = th->trace; const uint16_t info = (uint16_t) bctx->c; htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_GDN_D_PREP, info); @@ -1959,10 +1959,10 @@ static void gdn_hvx_phase4_worker(unsigned int n, unsigned int i, void * data) { htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_GDN_D_PREP, info); } -static void gdn_hvx_phase5_worker(unsigned int n, unsigned int i, void * data) { +static void gdn_hvx_phase5_worker(const htp_thread_t * th, unsigned int n, unsigned int i, void * data) { (void) n; struct htp_gdn_batch_context * bctx = (struct htp_gdn_batch_context *) data; - struct htp_thread_trace * tr = &bctx->octx->ctx->trace[i]; + struct htp_thread_trace * tr = th->trace; const uint16_t info = (uint16_t) bctx->c; htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_GDN_OUT, info); @@ -1984,10 +1984,10 @@ static void gdn_hvx_phase5_worker(unsigned int n, unsigned int i, void * data) { htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_GDN_OUT, info); } -static void gdn_hvx_phase6_worker(unsigned int n, unsigned int i, void * data) { +static void gdn_hvx_phase6_worker(const htp_thread_t * th, unsigned int n, unsigned int i, void * data) { (void) n; struct htp_gdn_batch_context * bctx = (struct htp_gdn_batch_context *) data; - struct htp_thread_trace * tr = &bctx->octx->ctx->trace[i]; + struct htp_thread_trace * tr = th->trace; const uint16_t info = (uint16_t) bctx->c; htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_GDN_STATE, info); @@ -2040,7 +2040,7 @@ static int gated_delta_net_f32_hmx_chunked( const uint32_t n_sv_tiles = S_v / 32; struct htp_gdn_hmx_vtcm_layout L; - htp_gdn_hmx_vtcm_layout_build(&L, S_v, chunk_size, kparams->n_heads_batch, kparams->n_threads, kparams->pipeline != 0); + htp_gdn_hmx_vtcm_layout_build(&L, S_v, chunk_size, kparams->n_heads_batch, kparams->n_hvx, kparams->pipeline != 0); if (L.total_bytes > octx->ctx->vtcm_size) { return HTP_STATUS_VTCM_TOO_SMALL; @@ -2062,7 +2062,7 @@ static int gated_delta_net_f32_hmx_chunked( hmx_queue_t hmx_q = octx->ctx->hmx_queue; dma_queue * dma_q = octx->ctx->dma[0]; - work_queue_t wp = octx->ctx->work_queue; + work_queue_t wp = octx->ctx->hvx_queue; struct htp_gdn_head_ptrs heads[8]; struct htp_gdn_hmx_gemm_task gemm_tasks[8][9]; @@ -2286,7 +2286,7 @@ int op_gated_delta_net(struct htp_ops_context * octx) { } const struct htp_gdn_kernel_params * kparams = (const struct htp_gdn_kernel_params *) octx->kernel_params; - if (!htp_ops_context_set_n_threads(octx, kparams->n_threads)) { + if (!htp_ops_context_set_n_hvx(octx, kparams->n_hvx)) { return HTP_STATUS_INVAL_PARAMS; } @@ -2321,7 +2321,7 @@ int op_gated_delta_net(struct htp_ops_context * octx) { return gated_delta_net_f32_hmx_chunked(octx, kparams, row_start, nrows); } - const uint32_t n_threads = (nrows < octx->n_threads) ? nrows : octx->n_threads; + const uint32_t n_hvx = (nrows < octx->n_hvx) ? nrows : octx->n_hvx; struct htp_gdn_context gctx; gctx.octx = octx; @@ -2329,7 +2329,7 @@ int op_gated_delta_net(struct htp_ops_context * octx) { gctx.row_start = row_start; gctx.nrows = nrows; - htp_gdn_vtcm_layout_build(&gctx.layout, S_v, n_threads); + htp_gdn_vtcm_layout_build(&gctx.layout, S_v, n_hvx); gctx.vtcm_base = VTCM_LAYOUT_PTR(uint8_t, octx->ctx->vtcm_base, gctx.layout.off_state); if (gctx.layout.total_bytes > octx->ctx->vtcm_size) { @@ -2337,18 +2337,18 @@ int op_gated_delta_net(struct htp_ops_context * octx) { } FARF(HIGH, "gated-delta-net-f32: q(%ux%ux%ux%u) k(%ux%ux%ux%u) v(%ux%ux%ux%u) state(%ux%ux%ux%u) -> (%ux%ux%ux%u) : " - "vtcm-size %zu n_threads %u\n", + "vtcm-size %zu n_hvx %u\n", octx->src[0]->ne[0], octx->src[0]->ne[1], octx->src[0]->ne[2], octx->src[0]->ne[3], octx->src[1]->ne[0], octx->src[1]->ne[1], octx->src[1]->ne[2], octx->src[1]->ne[3], octx->src[2]->ne[0], octx->src[2]->ne[1], octx->src[2]->ne[2], octx->src[2]->ne[3], octx->src[5]->ne[0], octx->src[5]->ne[1], octx->src[5]->ne[2], octx->src[5]->ne[3], dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3], - gctx.layout.total_bytes, n_threads); + gctx.layout.total_bytes, n_hvx); if (n_tokens == 1) { - work_queue_run(octx->ctx->work_queue, gated_delta_net_f32_tg_thread, &gctx, n_threads); + work_queue_run(octx->ctx->hvx_queue, gated_delta_net_f32_tg_thread, &gctx, n_hvx); } else { - work_queue_run(octx->ctx->work_queue, gated_delta_net_f32_pp_thread, &gctx, n_threads); + work_queue_run(octx->ctx->hvx_queue, gated_delta_net_f32_pp_thread, &gctx, n_hvx); } return HTP_STATUS_OK; diff --git a/ggml/src/ggml-hexagon/htp/gated-delta-net-ops.h b/ggml/src/ggml-hexagon/htp/gated-delta-net-ops.h index 0d88185880fe..aff1eb645b5b 100644 --- a/ggml/src/ggml-hexagon/htp/gated-delta-net-ops.h +++ b/ggml/src/ggml-hexagon/htp/gated-delta-net-ops.h @@ -29,7 +29,7 @@ struct htp_gdn_kernel_params { uint16_t n_chunks; uint16_t n_heads_batch; - uint32_t n_threads; + uint32_t n_hvx; uint32_t S_v; uint32_t H; uint32_t n_tokens; @@ -59,7 +59,7 @@ _Static_assert(sizeof(struct htp_gdn_kernel_params) <= 128, "htp_gdn_kernel_para struct htp_gdn_vtcm_layout { size_t state_aligned; - size_t bytes_per_thread; + size_t bytes_per_hvx; size_t off_state; size_t total_bytes; }; @@ -67,13 +67,13 @@ struct htp_gdn_vtcm_layout { static inline void htp_gdn_vtcm_layout_build( struct htp_gdn_vtcm_layout * L, uint32_t S_v, - uint32_t n_threads + uint32_t n_hvx ) { size_t state_bytes = (size_t) S_v * S_v * sizeof(float); L->state_aligned = hex_round_up(state_bytes, 128); - L->bytes_per_thread = 2 * L->state_aligned; + L->bytes_per_hvx = 2 * L->state_aligned; size_t off = 0; - VTCM_LAYOUT_ALLOC(off, off_state, L->bytes_per_thread * n_threads); + VTCM_LAYOUT_ALLOC(off, off_state, L->bytes_per_hvx * n_hvx); L->total_bytes = off; } @@ -128,7 +128,7 @@ struct htp_gdn_hmx_vtcm_layout { size_t off_rows_inv; size_t off_rows_a; - size_t off_thread_scratch; + size_t off_hvx_scratch; size_t off_scales_1; size_t state_f32_bytes; @@ -140,7 +140,7 @@ struct htp_gdn_hmx_vtcm_layout { size_t tile_64x64_bytes; uint32_t n_heads_batch; - uint32_t n_threads; + uint32_t n_hvx; bool pipeline; size_t total_bytes; }; @@ -150,16 +150,16 @@ static inline void htp_gdn_hmx_vtcm_layout_build( uint32_t S_v, uint32_t chunk_size, uint32_t n_heads_batch, - uint32_t n_threads, + uint32_t n_hvx, bool pipeline ) { memset(L, 0, sizeof(*L)); L->n_heads_batch = n_heads_batch; - L->n_threads = n_threads; + L->n_hvx = n_hvx; L->pipeline = pipeline; const size_t bh = (size_t) n_heads_batch; - const size_t nth = (size_t) (n_threads > 0 ? n_threads : 1); + const size_t nth = (size_t) (n_hvx > 0 ? n_hvx : 1); const size_t state_f32_sz = hex_round_up(S_v * S_v * sizeof(float), 2048); const size_t state_f16_sz = hex_round_up(S_v * S_v * sizeof(__fp16), 2048); @@ -248,9 +248,9 @@ static inline void htp_gdn_hmx_vtcm_layout_build( VTCM_LAYOUT_ALLOC(off, off_rows_inv, bh * row_vecs_sz); VTCM_LAYOUT_ALLOC(off, off_rows_a, bh * row_vecs_sz); - const size_t thread_scratch_sz = 64 * 128; + const size_t hvx_scratch_sz = 64 * 128; off = hex_align_up(off, HMX_FP16_TILE_SIZE); - VTCM_LAYOUT_ALLOC(off, off_thread_scratch, nth * thread_scratch_sz); + VTCM_LAYOUT_ALLOC(off, off_hvx_scratch, nth * hvx_scratch_sz); off = hex_align_up(off, HMX_FP16_TILE_SIZE); VTCM_LAYOUT_ALLOC(off, off_scales_1, HMX_FP16_TILE_SIZE); @@ -263,7 +263,7 @@ static inline bool htp_gdn_hmx_solve_layout( uint32_t chunk_size, uint32_t total_rows, size_t vtcm_budget, - uint32_t n_threads, + uint32_t n_hvx, bool pipeline, uint32_t * n_heads_batch_out ) { @@ -271,8 +271,8 @@ static inline bool htp_gdn_hmx_solve_layout( if (max_batch > total_rows) { max_batch = total_rows; } - if (max_batch > n_threads) { - max_batch = n_threads; + if (max_batch > n_hvx) { + max_batch = n_hvx; } static const uint32_t candidates[] = { 8, 6, 4, 2, 1 }; for (size_t i = 0; i < sizeof(candidates) / sizeof(candidates[0]); ++i) { @@ -281,7 +281,7 @@ static inline bool htp_gdn_hmx_solve_layout( continue; } struct htp_gdn_hmx_vtcm_layout L; - htp_gdn_hmx_vtcm_layout_build(&L, S_v, chunk_size, bh, n_threads, pipeline); + htp_gdn_hmx_vtcm_layout_build(&L, S_v, chunk_size, bh, n_hvx, pipeline); if (L.total_bytes <= vtcm_budget) { *layout_out = L; *n_heads_batch_out = bh; @@ -289,7 +289,7 @@ static inline bool htp_gdn_hmx_solve_layout( } } if (pipeline) { - return htp_gdn_hmx_solve_layout(layout_out, S_v, chunk_size, total_rows, vtcm_budget, n_threads, false, n_heads_batch_out); + return htp_gdn_hmx_solve_layout(layout_out, S_v, chunk_size, total_rows, vtcm_budget, n_hvx, false, n_heads_batch_out); } return false; } diff --git a/ggml/src/ggml-hexagon/htp/gather-rows.c b/ggml/src/ggml-hexagon/htp/gather-rows.c index 1e88445f1cb0..02804db27034 100644 --- a/ggml/src/ggml-hexagon/htp/gather-rows.c +++ b/ggml/src/ggml-hexagon/htp/gather-rows.c @@ -3,7 +3,7 @@ #include #include "hex-common.h" -#include "hex-profile.h" +#include "htp-trace.h" #include "hvx-utils.h" #include "dma-queue.h" #include "htp-ctx.h" @@ -23,7 +23,7 @@ struct gather_rows_task { dma_addr_t dst; uint32_t r_start; uint32_t rows; - uint32_t rows_per_thread; + uint32_t rows_per_hvx; const struct htp_gather_rows_params * kparams; gather_rows_compute_fn_t compute; }; @@ -190,21 +190,21 @@ static void gather_rows_compute_f16_vgn( } } -static void gather_rows_thread_f32(unsigned int nth, unsigned int ith, void * data) { +static void gather_rows_thread_f32(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { (void) nth; const struct gather_rows_task * task = (const struct gather_rows_task *) data; const struct htp_gather_rows_params * kparams = task->kparams; const struct htp_tensor * a = task->a; const struct htp_tensor * b = task->b; - const uint32_t r_offset = MIN(ith * task->rows_per_thread, task->rows); + const uint32_t r_offset = MIN(ith * task->rows_per_hvx, task->rows); const uint32_t r_beg = task->r_start + r_offset; - const uint32_t r_end = task->r_start + MIN(r_offset + task->rows_per_thread, task->rows); + const uint32_t r_end = task->r_start + MIN(r_offset + task->rows_per_hvx, task->rows); if (r_beg >= r_end) { return; } - struct htp_thread_trace * tr = &task->octx->ctx->trace[ith]; + struct htp_thread_trace * tr = th->trace; struct htp_gather_rows_vtcm_layout layout; htp_gather_rows_vtcm_layout_build(&layout, kparams->vg, kparams->region_size, kparams->nr_max, kparams->ne, 4); @@ -227,7 +227,7 @@ static void gather_rows_thread_f32(unsigned int nth, unsigned int ith, void * da gather_rows_init_tables_f32(tab, inc, kparams, a, b); htp_trace_event_stop(tr, HTP_TRACE_EVT_INIT, (uint16_t) r_beg); - dma_queue * q = task->octx->ctx->dma[ith]; + dma_queue * q = th->dma_q; const uint32_t n_in = 1 + (task->b ? (kparams->b_dense ? 1 : kparams->nb) : 0); const uint32_t nr_max = kparams->nr_max; const uint32_t nr0 = MIN(nr_max, r_end - r_beg); @@ -271,21 +271,21 @@ static void gather_rows_thread_f32(unsigned int nth, unsigned int ith, void * da dma_queue_flush(q); } -static void gather_rows_thread_f16(unsigned int nth, unsigned int ith, void * data) { +static void gather_rows_thread_f16(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { (void) nth; const struct gather_rows_task * task = (const struct gather_rows_task *) data; const struct htp_gather_rows_params * kparams = task->kparams; const struct htp_tensor * a = task->a; const struct htp_tensor * b = task->b; - const uint32_t r_offset = MIN(ith * task->rows_per_thread, task->rows); + const uint32_t r_offset = MIN(ith * task->rows_per_hvx, task->rows); const uint32_t r_beg = task->r_start + r_offset; - const uint32_t r_end = task->r_start + MIN(r_offset + task->rows_per_thread, task->rows); + const uint32_t r_end = task->r_start + MIN(r_offset + task->rows_per_hvx, task->rows); if (r_beg >= r_end) { return; } - struct htp_thread_trace * tr = &task->octx->ctx->trace[ith]; + struct htp_thread_trace * tr = th->trace; struct htp_gather_rows_vtcm_layout layout; htp_gather_rows_vtcm_layout_build(&layout, kparams->vg, kparams->region_size, kparams->nr_max, kparams->ne, 2); @@ -308,7 +308,7 @@ static void gather_rows_thread_f16(unsigned int nth, unsigned int ith, void * da gather_rows_init_tables_f16(tab, inc, kparams, a, b); htp_trace_event_stop(tr, HTP_TRACE_EVT_INIT, (uint16_t) r_beg); - dma_queue * q = task->octx->ctx->dma[ith]; + dma_queue * q = th->dma_q; const uint32_t n_in = 1 + (task->b ? (kparams->b_dense ? 1 : kparams->nb) : 0); const uint32_t nr_max = kparams->nr_max; @@ -357,13 +357,13 @@ int htp_gather_rows( const struct htp_tensor * a, const struct htp_tensor * b, const struct htp_gather_rows_params * kparams, - uint32_t n_threads) { + uint32_t n_hvx) { - if (!htp_ops_context_set_n_threads(octx, n_threads)) { + if (!htp_ops_context_set_n_hvx(octx, n_hvx)) { return HTP_STATUS_INVAL_PARAMS; } - assert(kparams->slice * octx->n_threads <= octx->ctx->vtcm_size); + assert(kparams->slice * octx->n_hvx <= octx->ctx->vtcm_size); const uint32_t total_rows = kparams->total_rows; uint32_t r_start = 0; @@ -389,7 +389,7 @@ int htp_gather_rows( task.r_start = r_start; task.rows = r_count; task.kparams = kparams; - task.rows_per_thread = kparams->rows_per_thread; + task.rows_per_hvx = kparams->rows_per_hvx; if (elem_size == 2) { task.compute = (kparams->vg == 1) ? gather_rows_compute_f16_vg1 : gather_rows_compute_f16_vgn; @@ -398,6 +398,6 @@ int htp_gather_rows( } work_queue_func_t worker_func = (elem_size == 2) ? gather_rows_thread_f16 : gather_rows_thread_f32; - work_queue_run(octx->ctx->work_queue, worker_func, &task, octx->n_threads); + work_queue_run(octx->ctx->hvx_queue, worker_func, &task, octx->n_hvx); return HTP_STATUS_OK; } diff --git a/ggml/src/ggml-hexagon/htp/gather-rows.h b/ggml/src/ggml-hexagon/htp/gather-rows.h index c08ef4d38fce..9ca215ac3864 100644 --- a/ggml/src/ggml-hexagon/htp/gather-rows.h +++ b/ggml/src/ggml-hexagon/htp/gather-rows.h @@ -22,7 +22,7 @@ struct htp_gather_rows_params { uint32_t region_size; uint32_t slice; uint32_t total_rows; - uint32_t rows_per_thread; + uint32_t rows_per_hvx; uint8_t b_dense; uint8_t pad[3]; }; @@ -35,7 +35,7 @@ struct htp_gather_rows_vtcm_layout { size_t off_out0; size_t off_region1; size_t off_out1; - size_t total_per_thread; + size_t total_per_hvx; }; static inline void htp_gather_rows_vtcm_layout_build( @@ -58,7 +58,7 @@ static inline void htp_gather_rows_vtcm_layout_build( VTCM_LAYOUT_ALLOC(off, off_out0, out_aligned); VTCM_LAYOUT_ALLOC(off, off_region1, region_aligned); VTCM_LAYOUT_ALLOC(off, off_out1, out_aligned); - L->total_per_thread = off; + L->total_per_hvx = off; } static inline bool htp_gather_rows_solve_layout( @@ -67,7 +67,7 @@ static inline bool htp_gather_rows_solve_layout( uint32_t nb, uint32_t b_nb1, bool b_dense, uint32_t elem_size, uint32_t total_rows, - uint32_t n_threads, + uint32_t n_hvx, uint32_t mdev_count, uint32_t vtcm_size) { @@ -86,13 +86,13 @@ static inline bool htp_gather_rows_solve_layout( const uint32_t rg = v_lanes / g; const uint32_t vg = ne / g; - const uint32_t slice = (vtcm_size / n_threads) & ~127u; + const uint32_t slice = (vtcm_size / n_hvx) & ~127u; const uint32_t total_chunks = total_rows / rg; const uint32_t chunks_per_dev = (total_chunks + mdev_count - 1) / mdev_count; const uint32_t max_rows_per_dev = (mdev_count > 1 && total_chunks >= mdev_count) ? (chunks_per_dev * rg + (total_rows % rg)) : total_rows; - const uint32_t rows_per_thread = hex_round_up((max_rows_per_dev + n_threads - 1) / n_threads, rg); + const uint32_t rows_per_hvx = hex_round_up((max_rows_per_dev + n_hvx - 1) / n_hvx, rg); const uint32_t fixed = 3 * vg * 128; const uint32_t per_row = a_nb1 + (nb > 0 ? (b_dense ? b_nb1 : nb * b_nb1) : 0) + ne * elem_size; @@ -101,7 +101,7 @@ static inline bool htp_gather_rows_solve_layout( } const uint32_t avail_per_buf = (slice - fixed) / 2; - uint32_t nr_max = MIN(avail_per_buf / per_row, rows_per_thread); + uint32_t nr_max = MIN(avail_per_buf / per_row, rows_per_hvx); if (elem_size == 2) { const uint32_t per_row_region = a_nb1 + (nb > 0 ? (b_dense ? b_nb1 : nb * b_nb1) : 0); if (per_row_region > 0) { @@ -132,7 +132,7 @@ static inline bool htp_gather_rows_solve_layout( struct htp_gather_rows_vtcm_layout layout; htp_gather_rows_vtcm_layout_build(&layout, vg, region_size, nr_max, ne, elem_size); - if ((elem_size != 2 || region_size <= 32768) && layout.total_per_thread <= slice) { + if ((elem_size != 2 || region_size <= 32768) && layout.total_per_hvx <= slice) { break; } @@ -154,7 +154,7 @@ static inline bool htp_gather_rows_solve_layout( kparams->region_size = region_size; kparams->slice = slice; kparams->total_rows = total_rows; - kparams->rows_per_thread = rows_per_thread; + kparams->rows_per_hvx = rows_per_hvx; kparams->b_dense = b_dense ? 1 : 0; kparams->pad[0] = 0; kparams->pad[1] = 0; @@ -183,7 +183,7 @@ int htp_gather_rows( const struct htp_tensor * a, const struct htp_tensor * b, const struct htp_gather_rows_params * kparams, - uint32_t n_threads); + uint32_t n_hvx); #endif #endif // HTP_GATHER_ROWS_H diff --git a/ggml/src/ggml-hexagon/htp/get-rows-ops.c b/ggml/src/ggml-hexagon/htp/get-rows-ops.c index 39acdfda667a..83e85f005243 100644 --- a/ggml/src/ggml-hexagon/htp/get-rows-ops.c +++ b/ggml/src/ggml-hexagon/htp/get-rows-ops.c @@ -37,7 +37,7 @@ struct get_rows_context { get_rows_tiled_compute_fn_t compute_tiled; uint32_t task_start; uint32_t tasks; - uint32_t tasks_per_thread; + uint32_t tasks_per_hvx; uint32_t tile_size; uint32_t tile_stride; uint32_t src0_row_size; @@ -95,20 +95,20 @@ static __attribute__((noinline)) uint32_t get_rows_read_index_vtcm( return (uint32_t) Q6_R_vextract_VR(v, off); } -static void get_rows_thread_sametype(unsigned int nth, unsigned int ith, void * data) { +static void get_rows_thread_sametype(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { struct get_rows_context * grctx = (struct get_rows_context *) data; struct htp_ops_context * octx = grctx->octx; const struct htp_get_rows_kernel_params * kparams = grctx->kparams; get_rows_preamble; - const uint32_t dr = grctx->tasks_per_thread; + const uint32_t dr = grctx->tasks_per_hvx; const uint32_t ir0 = grctx->task_start + dr * ith; if (ir0 >= grctx->task_start + grctx->tasks) { return; } const uint32_t ir1 = MIN(ir0 + dr, grctx->task_start + grctx->tasks); const uint32_t row_size_bytes = grctx->src0_row_size; - dma_queue * dma_q = octx->ctx->dma[ith]; + dma_queue * dma_q = th->dma_q; for (uint32_t i = ir0; i < ir1; ++i) { const uint32_t i12 = fastdiv(i, &kparams->div_ne10_ne11); @@ -174,23 +174,23 @@ static inline struct get_rows_flat_task get_rows_flat_calc_task( return task; } -static void get_rows_thread_flat(unsigned int nth, unsigned int ith, void * data) { +static void get_rows_thread_flat(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { struct get_rows_context * grctx = (struct get_rows_context *) data; struct htp_ops_context * octx = grctx->octx; get_rows_preamble; - const uint32_t dr = grctx->tasks_per_thread; + const uint32_t dr = grctx->tasks_per_hvx; const uint32_t ir0 = grctx->task_start + dr * ith; if (ir0 >= grctx->task_start + grctx->tasks) { return; } const uint32_t ir1 = MIN(ir0 + dr, grctx->task_start + grctx->tasks); - dma_queue * dma_q = octx->ctx->dma[ith]; - struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + dma_queue * dma_q = th->dma_q; + struct htp_thread_trace * tr = th->trace; const struct htp_get_rows_vtcm_layout * vtcm_layout = &grctx->vtcm_layout; - uint8_t * vtcm_src0 = VTCM_LAYOUT_PTR(uint8_t, grctx->vtcm_base, vtcm_layout->off_src0) + ith * vtcm_layout->src0_bytes_per_thread; - uint8_t * vtcm_dst = VTCM_LAYOUT_PTR(uint8_t, grctx->vtcm_base, vtcm_layout->off_dst) + ith * vtcm_layout->dst_bytes_per_thread; + uint8_t * vtcm_src0 = VTCM_LAYOUT_PTR(uint8_t, grctx->vtcm_base, vtcm_layout->off_src0) + ith * vtcm_layout->src0_bytes_per_hvx; + uint8_t * vtcm_dst = VTCM_LAYOUT_PTR(uint8_t, grctx->vtcm_base, vtcm_layout->off_dst) + ith * vtcm_layout->dst_bytes_per_hvx; const uint32_t src0_bytes = grctx->src0_row_size; const uint32_t dst_bytes = grctx->dst_row_size; get_rows_flat_compute_fn_t compute = grctx->compute_flat; @@ -276,12 +276,12 @@ static inline struct get_rows_tiled_task get_rows_tiled_calc_task( return task; } -static void get_rows_thread_tiled(unsigned int nth, unsigned int ith, void * data) { +static void get_rows_thread_tiled(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { struct get_rows_context * grctx = (struct get_rows_context *) data; struct htp_ops_context * octx = grctx->octx; get_rows_preamble; - const uint32_t dr = grctx->tasks_per_thread; + const uint32_t dr = grctx->tasks_per_hvx; const uint32_t ir0 = grctx->task_start + dr * ith; if (ir0 >= grctx->task_start + grctx->tasks) { return; @@ -290,10 +290,10 @@ static void get_rows_thread_tiled(unsigned int nth, unsigned int ith, void * dat const uint32_t ir1 = MIN(ir0 + dr, grctx->task_start + grctx->tasks); const uint32_t n_k_tiles = ne00 / htp_tensor_get_tile_k(octx->src[0]->type); const struct htp_get_rows_vtcm_layout * vtcm_layout = &grctx->vtcm_layout; - uint8_t * src_spad_base = VTCM_LAYOUT_PTR(uint8_t, grctx->vtcm_base, vtcm_layout->off_src0) + ith * vtcm_layout->src0_bytes_per_thread; - uint8_t * dst_spad_base = VTCM_LAYOUT_PTR(uint8_t, grctx->vtcm_base, vtcm_layout->off_dst) + ith * vtcm_layout->dst_bytes_per_thread; - dma_queue * dma_q = octx->ctx->dma[ith]; - struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + uint8_t * src_spad_base = VTCM_LAYOUT_PTR(uint8_t, grctx->vtcm_base, vtcm_layout->off_src0) + ith * vtcm_layout->src0_bytes_per_hvx; + uint8_t * dst_spad_base = VTCM_LAYOUT_PTR(uint8_t, grctx->vtcm_base, vtcm_layout->off_dst) + ith * vtcm_layout->dst_bytes_per_hvx; + dma_queue * dma_q = th->dma_q; + struct htp_thread_trace * tr = th->trace; const uint32_t tile_size = grctx->tile_size; const uint32_t tile_stride = grctx->tile_stride; @@ -414,11 +414,11 @@ int op_get_rows(struct htp_ops_context * octx) { return HTP_STATUS_OK; } - if (!htp_ops_context_set_n_threads(octx, kparams->n_threads)) { + if (!htp_ops_context_set_n_hvx(octx, kparams->n_hvx)) { return HTP_STATUS_INVAL_PARAMS; } - const uint32_t n_threads = octx->n_threads; + const uint32_t n_hvx = octx->n_hvx; struct get_rows_context grctx; grctx.octx = octx; @@ -428,14 +428,14 @@ int op_get_rows(struct htp_ops_context * octx) { grctx.compute_tiled = NULL; grctx.task_start = task_start; grctx.tasks = tasks; - grctx.tasks_per_thread = fastdiv(tasks + n_threads - 1, &octx->n_threads_div); + grctx.tasks_per_hvx = fastdiv(tasks + n_hvx - 1, &octx->n_hvx_div); grctx.tile_size = 0; grctx.tile_stride = 0; grctx.src0_row_size = src0_row_size; grctx.dst_row_size = dst_row_size; grctx.idx_type = octx->src[1]->type; - htp_get_rows_vtcm_layout_build(&grctx.vtcm_layout, kparams->kernel_type, octx->src[0]->type, ne00, total_tasks, grctx.idx_type, n_threads); + htp_get_rows_vtcm_layout_build(&grctx.vtcm_layout, kparams->kernel_type, octx->src[0]->type, ne00, total_tasks, grctx.idx_type, n_hvx); if (grctx.vtcm_layout.total_bytes > octx->ctx->vtcm_size) { FARF(ERROR, "get-rows: VTCM reservation %zu is too small, needed %zu\n", @@ -497,14 +497,14 @@ int op_get_rows(struct htp_ops_context * octx) { grctx.read_index = get_rows_read_index_vtcm; } - FARF(HIGH, "get-rows: (%ux%ux%ux%u) x (%ux%ux%ux%u) -> (%ux%ux%ux%u) : src0-vtcm-size %zu dst-vtcm-size %zu kernel-type %d n-threads %d\n", + FARF(HIGH, "get-rows: (%ux%ux%ux%u) x (%ux%ux%ux%u) -> (%ux%ux%ux%u) : src0-vtcm-size %zu dst-vtcm-size %zu kernel-type %d n-hvx %d\n", octx->src[0]->ne[0], octx->src[0]->ne[1], octx->src[0]->ne[2], octx->src[0]->ne[3], octx->src[1]->ne[0], octx->src[1]->ne[1], octx->src[1]->ne[2], octx->src[1]->ne[3], octx->dst->ne[0], octx->dst->ne[1], octx->dst->ne[2], octx->dst->ne[3], - grctx.vtcm_layout.src0_bytes_per_thread * n_threads, - grctx.vtcm_layout.dst_bytes_per_thread * n_threads, - kparams->kernel_type, n_threads); + grctx.vtcm_layout.src0_bytes_per_hvx * n_hvx, + grctx.vtcm_layout.dst_bytes_per_hvx * n_hvx, + kparams->kernel_type, n_hvx); - work_queue_run(octx->ctx->work_queue, q_func, &grctx, n_threads); + work_queue_run(octx->ctx->hvx_queue, q_func, &grctx, n_hvx); return HTP_STATUS_OK; } diff --git a/ggml/src/ggml-hexagon/htp/get-rows-ops.h b/ggml/src/ggml-hexagon/htp/get-rows-ops.h index 4634be2e2ed2..986966f11f11 100644 --- a/ggml/src/ggml-hexagon/htp/get-rows-ops.h +++ b/ggml/src/ggml-hexagon/htp/get-rows-ops.h @@ -16,7 +16,7 @@ enum htp_get_rows_kernel_type { struct htp_get_rows_kernel_params { uint8_t kernel_type; - uint32_t n_threads; + uint32_t n_hvx; uint32_t total_tasks; uint32_t vtcm_size; @@ -34,8 +34,8 @@ struct htp_get_rows_vtcm_layout { size_t off_src0; size_t off_dst; - size_t src0_bytes_per_thread; - size_t dst_bytes_per_thread; + size_t src0_bytes_per_hvx; + size_t dst_bytes_per_hvx; size_t src0_spad_half_size; size_t dst_spad_half_size; @@ -48,7 +48,7 @@ static inline void htp_get_rows_vtcm_layout_build( uint32_t ne00, uint32_t total_tasks, int idx_type, - uint32_t n_threads) { + uint32_t n_hvx) { memset(L, 0, sizeof(*L)); @@ -71,14 +71,14 @@ static inline void htp_get_rows_vtcm_layout_build( L->src0_spad_half_size = (src0_row_size + 255) & ~255; L->dst_spad_half_size = (ne00 * sizeof(float) + 255) & ~255; - L->src0_bytes_per_thread = 2 * L->src0_spad_half_size; - L->dst_bytes_per_thread = 2 * L->dst_spad_half_size; + L->src0_bytes_per_hvx = 2 * L->src0_spad_half_size; + L->dst_bytes_per_hvx = 2 * L->dst_spad_half_size; } size_t off = 0; VTCM_LAYOUT_ALLOC(off, off_src1, src1_alloc_bytes); - VTCM_LAYOUT_ALLOC(off, off_src0, L->src0_bytes_per_thread * n_threads); - VTCM_LAYOUT_ALLOC(off, off_dst, L->dst_bytes_per_thread * n_threads); + VTCM_LAYOUT_ALLOC(off, off_src0, L->src0_bytes_per_hvx * n_hvx); + VTCM_LAYOUT_ALLOC(off, off_dst, L->dst_bytes_per_hvx * n_hvx); L->total_bytes = off; } diff --git a/ggml/src/ggml-hexagon/htp/glu-ops.c b/ggml/src/ggml-hexagon/htp/glu-ops.c index b4edddf3fe0a..223c1829f232 100644 --- a/ggml/src/ggml-hexagon/htp/glu-ops.c +++ b/ggml/src/ggml-hexagon/htp/glu-ops.c @@ -96,7 +96,7 @@ struct htp_glu_context { uint32_t block; uint32_t src0_nrows; - uint32_t src0_nrows_per_thread; + uint32_t src0_nrows_per_hvx; uint32_t row_start; int nc; @@ -104,9 +104,9 @@ struct htp_glu_context { uint8_t * vtcm_src1; uint8_t * vtcm_dst; - size_t vtcm_src0_size_per_thread; - size_t vtcm_src1_size_per_thread; - size_t vtcm_dst_size_per_thread; + size_t vtcm_src0_size_per_hvx; + size_t vtcm_src1_size_per_hvx; + size_t vtcm_dst_size_per_hvx; }; #define htp_glu_op_preamble \ @@ -186,53 +186,6 @@ static void swiglu_clamp_f32(const float * restrict src0, static const float GELU_COEF_A = 0.044715f; static const float SQRT_2_OVER_PI = 0.79788456080286535587989211986876f; -static inline HVX_Vector hvx_vec_fast_sigmoid_f32_2it(HVX_Vector v) { - v = Q6_Vqf32_vmpy_VsfVsf(v, Q6_V_vsplat_R(FAST_SIGMOID_LOG2F)); - v = Q6_Vqf32_vmpy_VsfVsf(Q6_Vsf_equals_Vqf32(v), Q6_V_vsplat_R(FAST_SIGMOID_C3)); - - HVX_Vector in_int = hvx_vec_truncate_f32(Q6_Vsf_equals_Vqf32(v)); - HVX_Vector x = Q6_Vqf32_vsub_Vqf32Vsf(v, Q6_Vsf_equals_Vw(in_int)); - HVX_Vector xx = Q6_Vqf32_vmpy_Vqf32Vqf32(x, x); - - HVX_Vector v1 = Q6_Vqf32_vmpy_VsfVsf(Q6_Vsf_equals_Vqf32(xx), Q6_V_vsplat_R(FAST_SIGMOID_C2)); - v1 = Q6_Vqf32_vadd_Vqf32Vsf(v1, Q6_V_vsplat_R(FAST_SIGMOID_LOG2F)); - - HVX_Vector v2 = Q6_Vqf32_vmpy_VsfVsf(Q6_Vsf_equals_Vqf32(x), Q6_V_vsplat_R(FAST_SIGMOID_C1)); - v2 = Q6_Vqf32_vmpy_Vqf32Vqf32(v2, xx); - v2 = Q6_Vqf32_vadd_Vqf32Vqf32(v2, x); - - HVX_Vector v3 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_Vqf32Vqf32(v2, v1)); - v3 = Q6_Vw_vaslacc_VwVwR(v3, in_int, 24); - - HVX_Vector v4 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vsub_Vqf32Vqf32(v2, v1)); - HVX_Vector v5 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vsub_VsfVsf(v3, v4)); - - // Newton-Raphson with 2 iterations - HVX_Vector two_sf = hvx_vec_splat_f32(2.0f); - HVX_Vector i_sf = Q6_Vw_vsub_VwVw(Q6_V_vsplat_R(0x7EEEEBB3), v5); - HVX_Vector r_qf = Q6_Vqf32_vmpy_VsfVsf( - i_sf, Q6_Vsf_equals_Vqf32(Q6_Vqf32_vsub_VsfVsf(two_sf, Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(i_sf, v5))))); - r_qf = Q6_Vqf32_vmpy_Vqf32Vqf32( - r_qf, Q6_Vqf32_vsub_VsfVsf(two_sf, Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(Q6_Vsf_equals_Vqf32(r_qf), v5)))); - HVX_Vector res = Q6_Vsf_equals_Vqf32(r_qf); - - res = Q6_Vqf32_vmpy_VsfVsf(v3, res); - - return Q6_Vsf_equals_Vqf32(res); -} - -static inline HVX_Vector hvx_vec_fast_sigmoid_f32_guard_2it(HVX_Vector v, - HVX_Vector one, - HVX_Vector max_exp, - HVX_Vector min_exp) { - const HVX_VectorPred pred_max = Q6_Q_vcmp_gt_VsfVsf(max_exp, v); - const HVX_VectorPred pred_min = Q6_Q_vcmp_gt_VsfVsf(v, min_exp); - - HVX_Vector out = hvx_vec_fast_sigmoid_f32_2it(v); - out = Q6_V_vmux_QVV(pred_max, out, one); - return Q6_V_vmux_QVV(pred_min, out, Q6_V_vzero()); -} - static inline void hvx_geglu_f32_aa(uint8_t * restrict dst, const uint8_t * restrict src0, const uint8_t * restrict src1, uint32_t n) { assert((unsigned long) dst % 128 == 0); assert((unsigned long) src0 % 128 == 0); @@ -769,11 +722,11 @@ static void sigglu_f16(const _Float16 * restrict src0, } #define DEFINE_GLU_PER_THREAD(TYPE, SUFFIX) \ - static void glu_##SUFFIX##_per_thread(unsigned int nth, unsigned int ith, void * data) { \ + static void glu_##SUFFIX##_per_hvx(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { \ struct htp_glu_context * gctx = (struct htp_glu_context *) data; \ htp_glu_preamble; \ \ - struct htp_thread_trace * tr = gctx->octx->ctx ? &gctx->octx->ctx->trace[ith] : NULL; \ + struct htp_thread_trace * tr = th->trace; \ \ size_t src0_row_size = gctx->src0_row_size; \ size_t src1_row_size = gctx->src1_row_size; \ @@ -783,10 +736,10 @@ static void sigglu_f16(const _Float16 * restrict src0, size_t src1_row_stride = gctx->src1_row_stride; \ \ const uint32_t src0_nrows = gctx->src0_nrows; \ - const uint32_t src0_nrows_per_thread = gctx->src0_nrows_per_thread; \ + const uint32_t src0_nrows_per_hvx = gctx->src0_nrows_per_hvx; \ \ - const uint32_t src0_start_row = gctx->row_start + src0_nrows_per_thread * ith; \ - const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, gctx->row_start + src0_nrows); \ + const uint32_t src0_start_row = gctx->row_start + src0_nrows_per_hvx * ith; \ + const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_hvx, gctx->row_start + src0_nrows); \ \ /* no work for this thread */ \ if (src0_start_row >= src0_end_row) { \ @@ -801,9 +754,9 @@ static void sigglu_f16(const _Float16 * restrict src0, const size_t src1_row_size_aligned = gctx->src1_row_size_aligned; \ const size_t dst_row_size_aligned = gctx->dst_row_size_aligned; \ \ - uint8_t * restrict src0_spad_data = gctx->vtcm_src0 + (ith * gctx->vtcm_src0_size_per_thread); \ - uint8_t * restrict src1_spad_data = gctx->vtcm_src1 + (ith * gctx->vtcm_src1_size_per_thread); \ - uint8_t * restrict dst_spad_data = gctx->vtcm_dst + (ith * gctx->vtcm_dst_size_per_thread); \ + uint8_t * restrict src0_spad_data = gctx->vtcm_src0 + (ith * gctx->vtcm_src0_size_per_hvx); \ + uint8_t * restrict src1_spad_data = gctx->vtcm_src1 + (ith * gctx->vtcm_src1_size_per_hvx); \ + uint8_t * restrict dst_spad_data = gctx->vtcm_dst + (ith * gctx->vtcm_dst_size_per_hvx); \ \ size_t src0_spad_half_size = gctx->src0_spad_half_size; \ size_t src1_spad_half_size = gctx->src1_spad_half_size; \ @@ -811,7 +764,7 @@ static void sigglu_f16(const _Float16 * restrict src0, \ const int BLOCK = gctx->block; \ \ - dma_queue * dma_q = gctx->octx->ctx->dma[ith]; \ + dma_queue * dma_q = th->dma_q; \ glu_compute_##SUFFIX##_fn_t compute = (glu_compute_##SUFFIX##_fn_t) gctx->compute; \ \ for (uint32_t ir = src0_start_row, spad_idx = 0; ir < src0_end_row && spad_idx < 2; ir += BLOCK, spad_idx++) { \ @@ -918,7 +871,7 @@ static int execute_op_glu_f32(struct htp_ops_context * octx) { const struct htp_glu_kernel_params * kparams = (const struct htp_glu_kernel_params *) octx->kernel_params; - if (!htp_ops_context_set_n_threads(octx, kparams->n_threads)) { + if (!htp_ops_context_set_n_hvx(octx, kparams->n_hvx)) { return HTP_STATUS_INVAL_PARAMS; } @@ -940,7 +893,7 @@ static int execute_op_glu_f32(struct htp_ops_context * octx) { return HTP_STATUS_OK; } - const uint32_t n_threads = octx->n_threads; + const uint32_t n_hvx = octx->n_hvx; // row_size = bytes of useful data per row (what the kernel touches / what DMA copies). // row_stride = bytes between successive rows in DDR (may exceed row_size for non-contig src). @@ -951,28 +904,28 @@ static int execute_op_glu_f32(struct htp_ops_context * octx) { const size_t src1_row_stride = src1 ? src1->nb[1] : src0->nb[1]; struct htp_glu_vtcm_layout L; - htp_glu_vtcm_layout_build(&L, dst_row_size, n_threads, kparams->vtcm_row_per_thread); + htp_glu_vtcm_layout_build(&L, dst_row_size, n_hvx, kparams->vtcm_row_per_hvx); const size_t src0_row_size_aligned = L.row_size_aligned; const size_t src1_row_size_aligned = L.row_size_aligned; const size_t dst_row_size_aligned = L.row_size_aligned; // Make sure the reserved vtcm size is sufficient - if (kparams->vtcm_row_per_thread == 0 || octx->ctx->vtcm_size < kparams->vtcm_size) { + if (kparams->vtcm_row_per_hvx == 0 || octx->ctx->vtcm_size < kparams->vtcm_size) { FARF(ERROR, "glu-%s : current VTCM reservation %zu is too small for even 1 row per thread, needed at least %zu\n", op_type, octx->ctx->vtcm_size, - (src0_row_size_aligned + src1_row_size_aligned + dst_row_size_aligned) * n_threads); + (src0_row_size_aligned + src1_row_size_aligned + dst_row_size_aligned) * n_hvx); return HTP_STATUS_VTCM_TOO_SMALL; } if (src1) { FARF(HIGH, "%s: %ux%ux%ux%u x %ux%ux%ux%u -> %ux%ux%ux%u : src0-vtcm-size %zu src1-vtcm-size %zu dst-vtcm-size %zu\n", op_type, src0->ne[0], src0->ne[1], src0->ne[2], src0->ne[3], src1->ne[0], src1->ne[1], src1->ne[2], - src1->ne[3], dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3], L.src0_bytes_per_thread * n_threads, - L.src1_bytes_per_thread * n_threads, L.dst_bytes_per_thread * n_threads); + src1->ne[3], dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3], L.src0_bytes_per_hvx * n_hvx, + L.src1_bytes_per_hvx * n_hvx, L.dst_bytes_per_hvx * n_hvx); } else { FARF(HIGH, "%s: %ux%ux%ux%u -> %ux%ux%ux%u : src0-vtcm-size %zu src1-vtcm-size %zu dst-vtcm-size %zu\n", op_type, src0->ne[0], src0->ne[1], src0->ne[2], src0->ne[3], dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3], - L.src0_bytes_per_thread * n_threads, L.src1_bytes_per_thread * n_threads, L.dst_bytes_per_thread * n_threads); + L.src0_bytes_per_hvx * n_hvx, L.src1_bytes_per_hvx * n_hvx, L.dst_bytes_per_hvx * n_hvx); } // Prepare context @@ -981,7 +934,7 @@ static int execute_op_glu_f32(struct htp_ops_context * octx) { gctx.compute = (void *) compute_fn; gctx.op_str = op_type; - gctx.src0_nrows_per_thread = fastdiv(nrows + n_threads - 1, &octx->n_threads_div); + gctx.src0_nrows_per_hvx = fastdiv(nrows + n_hvx - 1, &octx->n_hvx_div); gctx.src0_row_size = src0_row_size; gctx.src1_row_size = src1_row_size; @@ -999,13 +952,13 @@ static int execute_op_glu_f32(struct htp_ops_context * octx) { gctx.vtcm_src1 = VTCM_LAYOUT_PTR(uint8_t, base, L.off_src1); gctx.vtcm_dst = VTCM_LAYOUT_PTR(uint8_t, base, L.off_dst); - gctx.vtcm_src0_size_per_thread = (uint32_t) L.src0_bytes_per_thread; - gctx.vtcm_src1_size_per_thread = (uint32_t) L.src1_bytes_per_thread; - gctx.vtcm_dst_size_per_thread = (uint32_t) L.dst_bytes_per_thread; + gctx.vtcm_src0_size_per_hvx = (uint32_t) L.src0_bytes_per_hvx; + gctx.vtcm_src1_size_per_hvx = (uint32_t) L.src1_bytes_per_hvx; + gctx.vtcm_dst_size_per_hvx = (uint32_t) L.dst_bytes_per_hvx; - gctx.src0_spad_half_size = gctx.vtcm_src0_size_per_thread >> 1; - gctx.src1_spad_half_size = gctx.vtcm_src1_size_per_thread >> 1; - gctx.dst_spad_half_size = gctx.vtcm_dst_size_per_thread >> 1; + gctx.src0_spad_half_size = gctx.vtcm_src0_size_per_hvx >> 1; + gctx.src1_spad_half_size = gctx.vtcm_src1_size_per_hvx >> 1; + gctx.dst_spad_half_size = gctx.vtcm_dst_size_per_hvx >> 1; gctx.block = L.block; gctx.src0_nrows = nrows; @@ -1034,7 +987,7 @@ static int execute_op_glu_f32(struct htp_ops_context * octx) { gctx.data_src1 = data_src1; gctx.data_dst = dst->data; - work_queue_run(octx->ctx->work_queue, (worker_callback_t)glu_f32_per_thread, &gctx, n_threads); + work_queue_run(octx->ctx->hvx_queue, (work_queue_func_t)glu_f32_per_hvx, &gctx, n_hvx); return HTP_STATUS_OK; } @@ -1084,7 +1037,7 @@ static int execute_op_glu_f16(struct htp_ops_context * octx) { const struct htp_glu_kernel_params * kparams = (const struct htp_glu_kernel_params *) octx->kernel_params; - if (!htp_ops_context_set_n_threads(octx, kparams->n_threads)) { + if (!htp_ops_context_set_n_hvx(octx, kparams->n_hvx)) { return HTP_STATUS_INVAL_PARAMS; } @@ -1106,7 +1059,7 @@ static int execute_op_glu_f16(struct htp_ops_context * octx) { return HTP_STATUS_OK; } - const uint32_t n_threads = octx->n_threads; + const uint32_t n_hvx = octx->n_hvx; // row_size = bytes of useful data per row (what the kernel touches / what DMA copies). // row_stride = bytes between successive rows in DDR (may exceed row_size for non-contig src). @@ -1117,28 +1070,28 @@ static int execute_op_glu_f16(struct htp_ops_context * octx) { const size_t src1_row_stride = src1 ? src1->nb[1] : src0->nb[1]; struct htp_glu_vtcm_layout L; - htp_glu_vtcm_layout_build(&L, dst_row_size, n_threads, kparams->vtcm_row_per_thread); + htp_glu_vtcm_layout_build(&L, dst_row_size, n_hvx, kparams->vtcm_row_per_hvx); const size_t src0_row_size_aligned = L.row_size_aligned; const size_t src1_row_size_aligned = L.row_size_aligned; const size_t dst_row_size_aligned = L.row_size_aligned; // Make sure the reserved vtcm size is sufficient - if (kparams->vtcm_row_per_thread == 0 || octx->ctx->vtcm_size < kparams->vtcm_size) { + if (kparams->vtcm_row_per_hvx == 0 || octx->ctx->vtcm_size < kparams->vtcm_size) { FARF(ERROR, "glu-%s : current VTCM reservation %zu is too small for even 1 row per thread, needed at least %zu\n", op_type, octx->ctx->vtcm_size, - (src0_row_size_aligned + src1_row_size_aligned + dst_row_size_aligned) * n_threads); + (src0_row_size_aligned + src1_row_size_aligned + dst_row_size_aligned) * n_hvx); return HTP_STATUS_VTCM_TOO_SMALL; } if (src1) { FARF(HIGH, "%s: %ux%ux%ux%u x %ux%ux%ux%u -> %ux%ux%ux%u : src0-vtcm-size %zu src1-vtcm-size %zu dst-vtcm-size %zu\n", op_type, src0->ne[0], src0->ne[1], src0->ne[2], src0->ne[3], src1->ne[0], src1->ne[1], src1->ne[2], - src1->ne[3], dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3], L.src0_bytes_per_thread * n_threads, - L.src1_bytes_per_thread * n_threads, L.dst_bytes_per_thread * n_threads); + src1->ne[3], dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3], L.src0_bytes_per_hvx * n_hvx, + L.src1_bytes_per_hvx * n_hvx, L.dst_bytes_per_hvx * n_hvx); } else { FARF(HIGH, "%s: %ux%ux%ux%u -> %ux%ux%ux%u : src0-vtcm-size %zu src1-vtcm-size %zu dst-vtcm-size %zu\n", op_type, src0->ne[0], src0->ne[1], src0->ne[2], src0->ne[3], dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3], - L.src0_bytes_per_thread * n_threads, L.src1_bytes_per_thread * n_threads, L.dst_bytes_per_thread * n_threads); + L.src0_bytes_per_hvx * n_hvx, L.src1_bytes_per_hvx * n_hvx, L.dst_bytes_per_hvx * n_hvx); } // Prepare context @@ -1147,7 +1100,7 @@ static int execute_op_glu_f16(struct htp_ops_context * octx) { gctx.compute = (void *) compute_fn; gctx.op_str = op_type; - gctx.src0_nrows_per_thread = fastdiv(nrows + n_threads - 1, &octx->n_threads_div); + gctx.src0_nrows_per_hvx = fastdiv(nrows + n_hvx - 1, &octx->n_hvx_div); gctx.src0_row_size = src0_row_size; gctx.src1_row_size = src1_row_size; @@ -1165,13 +1118,13 @@ static int execute_op_glu_f16(struct htp_ops_context * octx) { gctx.vtcm_src1 = VTCM_LAYOUT_PTR(uint8_t, base, L.off_src1); gctx.vtcm_dst = VTCM_LAYOUT_PTR(uint8_t, base, L.off_dst); - gctx.vtcm_src0_size_per_thread = (uint32_t) L.src0_bytes_per_thread; - gctx.vtcm_src1_size_per_thread = (uint32_t) L.src1_bytes_per_thread; - gctx.vtcm_dst_size_per_thread = (uint32_t) L.dst_bytes_per_thread; + gctx.vtcm_src0_size_per_hvx = (uint32_t) L.src0_bytes_per_hvx; + gctx.vtcm_src1_size_per_hvx = (uint32_t) L.src1_bytes_per_hvx; + gctx.vtcm_dst_size_per_hvx = (uint32_t) L.dst_bytes_per_hvx; - gctx.src0_spad_half_size = gctx.vtcm_src0_size_per_thread >> 1; - gctx.src1_spad_half_size = gctx.vtcm_src1_size_per_thread >> 1; - gctx.dst_spad_half_size = gctx.vtcm_dst_size_per_thread >> 1; + gctx.src0_spad_half_size = gctx.vtcm_src0_size_per_hvx >> 1; + gctx.src1_spad_half_size = gctx.vtcm_src1_size_per_hvx >> 1; + gctx.dst_spad_half_size = gctx.vtcm_dst_size_per_hvx >> 1; gctx.block = L.block; gctx.src0_nrows = nrows; @@ -1200,7 +1153,7 @@ static int execute_op_glu_f16(struct htp_ops_context * octx) { gctx.data_src1 = data_src1; gctx.data_dst = dst->data; - work_queue_run(octx->ctx->work_queue, (worker_callback_t)glu_f16_per_thread, &gctx, n_threads); + work_queue_run(octx->ctx->hvx_queue, (work_queue_func_t)glu_f16_per_hvx, &gctx, n_hvx); return HTP_STATUS_OK; } diff --git a/ggml/src/ggml-hexagon/htp/glu-ops.h b/ggml/src/ggml-hexagon/htp/glu-ops.h index 41b3857b1281..5e248f7d4ab0 100644 --- a/ggml/src/ggml-hexagon/htp/glu-ops.h +++ b/ggml/src/ggml-hexagon/htp/glu-ops.h @@ -7,8 +7,8 @@ #include struct htp_glu_kernel_params { - uint32_t n_threads; - uint32_t vtcm_row_per_thread; + uint32_t n_hvx; + uint32_t vtcm_row_per_hvx; uint32_t vtcm_size; }; @@ -41,34 +41,34 @@ struct htp_glu_vtcm_layout { size_t off_dst; size_t row_size_aligned; - size_t src0_bytes_per_thread; - size_t src1_bytes_per_thread; - size_t dst_bytes_per_thread; + size_t src0_bytes_per_hvx; + size_t src1_bytes_per_hvx; + size_t dst_bytes_per_hvx; - uint32_t vtcm_row_per_thread; + uint32_t vtcm_row_per_hvx; uint32_t block; }; static inline void htp_glu_vtcm_layout_build( struct htp_glu_vtcm_layout * L, size_t dst_row_size, - uint32_t n_threads, - uint32_t vtcm_row_per_thread + uint32_t n_hvx, + uint32_t vtcm_row_per_hvx ) { const size_t row_size_aligned = hex_round_up(dst_row_size, 128); L->row_size_aligned = row_size_aligned; - L->vtcm_row_per_thread = vtcm_row_per_thread; - L->block = vtcm_row_per_thread >> 1; + L->vtcm_row_per_hvx = vtcm_row_per_hvx; + L->block = vtcm_row_per_hvx >> 1; - L->src0_bytes_per_thread = row_size_aligned * vtcm_row_per_thread; - L->src1_bytes_per_thread = row_size_aligned * vtcm_row_per_thread; - L->dst_bytes_per_thread = row_size_aligned * vtcm_row_per_thread; + L->src0_bytes_per_hvx = row_size_aligned * vtcm_row_per_hvx; + L->src1_bytes_per_hvx = row_size_aligned * vtcm_row_per_hvx; + L->dst_bytes_per_hvx = row_size_aligned * vtcm_row_per_hvx; size_t off = 0; - VTCM_LAYOUT_ALLOC(off, off_src0, L->src0_bytes_per_thread * n_threads); - VTCM_LAYOUT_ALLOC(off, off_src1, L->src1_bytes_per_thread * n_threads); - VTCM_LAYOUT_ALLOC(off, off_dst, L->dst_bytes_per_thread * n_threads); + VTCM_LAYOUT_ALLOC(off, off_src0, L->src0_bytes_per_hvx * n_hvx); + VTCM_LAYOUT_ALLOC(off, off_src1, L->src1_bytes_per_hvx * n_hvx); + VTCM_LAYOUT_ALLOC(off, off_dst, L->dst_bytes_per_hvx * n_hvx); L->total_bytes = off; } diff --git a/ggml/src/ggml-hexagon/htp/hex-profile.h b/ggml/src/ggml-hexagon/htp/hex-profile.h deleted file mode 100644 index 5c7a1a488831..000000000000 --- a/ggml/src/ggml-hexagon/htp/hex-profile.h +++ /dev/null @@ -1,66 +0,0 @@ -#ifndef HEX_PROFILE_H -#define HEX_PROFILE_H - -#include -#include -#include - -#include "hex-utils.h" -#include "htp-ops.h" - -#define HTP_TRACE_EVT_START 0 -#define HTP_TRACE_EVT_STOP 1 - -#ifndef HEX_NUM_PMU_COUNTERS -#define HEX_NUM_PMU_COUNTERS 8 -#endif - -static inline void hex_get_pmu(uint32_t counters[]) { -#if __HVX_ARCH__ >= 79 - asm volatile("%0 = upmucnt0" : "=r"(counters[0])); - asm volatile("%0 = upmucnt1" : "=r"(counters[1])); - asm volatile("%0 = upmucnt2" : "=r"(counters[2])); - asm volatile("%0 = upmucnt3" : "=r"(counters[3])); - asm volatile("%0 = upmucnt4" : "=r"(counters[4])); - asm volatile("%0 = upmucnt5" : "=r"(counters[5])); - asm volatile("%0 = upmucnt6" : "=r"(counters[6])); - asm volatile("%0 = upmucnt7" : "=r"(counters[7])); -#else - counters[0] = qurt_pmu_get(QURT_PMUCNT0); - counters[1] = qurt_pmu_get(QURT_PMUCNT1); - counters[2] = qurt_pmu_get(QURT_PMUCNT2); - counters[3] = qurt_pmu_get(QURT_PMUCNT3); - counters[4] = qurt_pmu_get(QURT_PMUCNT4); - counters[5] = qurt_pmu_get(QURT_PMUCNT5); - counters[6] = qurt_pmu_get(QURT_PMUCNT6); - counters[7] = qurt_pmu_get(QURT_PMUCNT7); -#endif -} - -struct htp_thread_trace { - uint32_t count; - uint32_t max_events; - struct htp_trace_desc * events; -}; - -static inline void htp_trace_event(struct htp_thread_trace * tr, uint16_t id, uint16_t info, uint32_t type) { - if (!tr->max_events) { - return; - } - uint32_t i = tr->count++; - if (i < tr->max_events) { - tr->events[i].id = id; - tr->events[i].info = info | (type == HTP_TRACE_EVT_STOP ? 0x8000 : 0); - tr->events[i].cycles = (uint32_t) hex_get_cycles(); - } -} - -static inline void htp_trace_event_start(struct htp_thread_trace * tr, uint16_t id, uint16_t info) { - htp_trace_event(tr, id, info, HTP_TRACE_EVT_START); -} - -static inline void htp_trace_event_stop(struct htp_thread_trace * tr, uint16_t id, uint16_t info) { - htp_trace_event(tr, id, info, HTP_TRACE_EVT_STOP); -} - -#endif /* HEX_PROFILE_H */ diff --git a/ggml/src/ggml-hexagon/htp/hex-utils.h b/ggml/src/ggml-hexagon/htp/hex-utils.h index abb44da25271..cb3093dece48 100644 --- a/ggml/src/ggml-hexagon/htp/hex-utils.h +++ b/ggml/src/ggml-hexagon/htp/hex-utils.h @@ -25,6 +25,32 @@ static inline uint64_t hex_get_pktcnt() { return pktcnt; } +#ifndef HEX_NUM_PMU_COUNTERS +#define HEX_NUM_PMU_COUNTERS 8 +#endif + +static inline void hex_get_pmu(uint32_t counters[]) { +#if __HVX_ARCH__ >= 79 + asm volatile("%0 = upmucnt0" : "=r"(counters[0])); + asm volatile("%0 = upmucnt1" : "=r"(counters[1])); + asm volatile("%0 = upmucnt2" : "=r"(counters[2])); + asm volatile("%0 = upmucnt3" : "=r"(counters[3])); + asm volatile("%0 = upmucnt4" : "=r"(counters[4])); + asm volatile("%0 = upmucnt5" : "=r"(counters[5])); + asm volatile("%0 = upmucnt6" : "=r"(counters[6])); + asm volatile("%0 = upmucnt7" : "=r"(counters[7])); +#else + counters[0] = qurt_pmu_get(QURT_PMUCNT0); + counters[1] = qurt_pmu_get(QURT_PMUCNT1); + counters[2] = qurt_pmu_get(QURT_PMUCNT2); + counters[3] = qurt_pmu_get(QURT_PMUCNT3); + counters[4] = qurt_pmu_get(QURT_PMUCNT4); + counters[5] = qurt_pmu_get(QURT_PMUCNT5); + counters[6] = qurt_pmu_get(QURT_PMUCNT6); + counters[7] = qurt_pmu_get(QURT_PMUCNT7); +#endif +} + static inline void hex_l2fetch(const void * p, uint32_t width, uint32_t stride, uint32_t height) { const uint64_t control = Q6_P_combine_RR(stride, Q6_R_combine_RlRl(width, height)); Q6_l2fetch_AP((void *) p, control); @@ -37,11 +63,6 @@ static inline void hex_l2fetch_block(const void * addr, size_t size) { hex_l2fetch(addr, width, width, height); } -#define HEX_L2_LINE_SIZE 128 -#define HEX_L2_BLOCK_SIZE (HEX_L2_LINE_SIZE * 4) // flush granularity (chunks per thread) -#define HEX_L2_FLUSH_WQ_THRESHOLD (4 * 1024) -#define HEX_L2_FLUSH_ALL_THRESHOLD (4 * 1024 * 1024) - static inline void hex_l2flush(void * addr, size_t size) { if (size == 0) return; const uint32_t s = ((uint32_t) addr) & ~(HEX_L2_LINE_SIZE - 1); diff --git a/ggml/src/ggml-hexagon/htp/hmx-mm-kernels-tiled.h b/ggml/src/ggml-hexagon/htp/hmx-mm-kernels-tiled.h index 65f90c17aad0..a828012dd7fc 100644 --- a/ggml/src/ggml-hexagon/htp/hmx-mm-kernels-tiled.h +++ b/ggml/src/ggml-hexagon/htp/hmx-mm-kernels-tiled.h @@ -18,8 +18,6 @@ typedef struct tiled_dequantize_state tiled_dequantize_state_t; typedef void (*tiled_dequant_task_fn_t)(const tiled_dequantize_state_t * state, uint32_t start_tile, uint32_t end_tile); struct tiled_dequantize_state { - struct htp_context * ctx; - struct htp_thread_trace * traces; __fp16 * dst; const uint8_t * src; tiled_dequant_task_fn_t task_fn; @@ -649,6 +647,106 @@ static void dequantize_tiled_weight_to_fp16_task_q5_k( } } +static void dequantize_tiled_weight_to_fp16_task_q5_0( + const tiled_dequantize_state_t *state, + uint32_t start_tile, uint32_t end_tile) { + + const HVX_Vector mask_h4 = Q6_Vb_vsplat_R(0x0F); + + for (uint32_t t = start_tile; t < end_tile; t++) { + const uint8_t * tile_src = state->src + t * state->aligned_tile_size; + __fp16 * dst_ptr = state->dst + t * HTP_MM_HMX_TILE_N_ELMS; + + HVX_Vector v_sc = hvx_vmem(tile_src + 640); + HVX_Vector v_scale_duplicated = Q6_V_lo_W(Q6_W_vshuff_VVR(v_sc, v_sc, -2)); + + // Load all 4 groups in parallel + HVX_Vector vq0 = hvx_vmem(tile_src + 0 * 128); + HVX_Vector vq1 = hvx_vmem(tile_src + 1 * 128); + HVX_Vector vq2 = hvx_vmem(tile_src + 2 * 128); + HVX_Vector vq3 = hvx_vmem(tile_src + 3 * 128); + + // Nibble extraction + HVX_Vector v_lo0 = Q6_V_vand_VV(vq0, mask_h4); + HVX_Vector v_hi0 = Q6_Vub_vlsr_VubR(vq0, 4); + HVX_Vector v_lo1 = Q6_V_vand_VV(vq1, mask_h4); + HVX_Vector v_hi1 = Q6_Vub_vlsr_VubR(vq1, 4); + HVX_Vector v_lo2 = Q6_V_vand_VV(vq2, mask_h4); + HVX_Vector v_hi2 = Q6_Vub_vlsr_VubR(vq2, 4); + HVX_Vector v_lo3 = Q6_V_vand_VV(vq3, mask_h4); + HVX_Vector v_hi3 = Q6_Vub_vlsr_VubR(vq3, 4); + + // Q5_0: OR 0xF0 where the plane flags a clear 5th bit (q - 16) + HVX_Vector v_plane = hvx_vmem(tile_src + 512); + v_lo0 = hvx_q5_0_or_neg(v_lo0, v_plane, 0); + v_hi0 = hvx_q5_0_or_neg(v_hi0, v_plane, 1); + v_lo1 = hvx_q5_0_or_neg(v_lo1, v_plane, 2); + v_hi1 = hvx_q5_0_or_neg(v_hi1, v_plane, 3); + v_lo2 = hvx_q5_0_or_neg(v_lo2, v_plane, 4); + v_hi2 = hvx_q5_0_or_neg(v_hi2, v_plane, 5); + v_lo3 = hvx_q5_0_or_neg(v_lo3, v_plane, 6); + v_hi3 = hvx_q5_0_or_neg(v_hi3, v_plane, 7); + + // Shuffling + HVX_VectorPair vp_shuf0 = Q6_W_vshuff_VVR(v_hi0, v_lo0, -1); + HVX_VectorPair vp_shuf1 = Q6_W_vshuff_VVR(v_hi1, v_lo1, -1); + HVX_VectorPair vp_shuf2 = Q6_W_vshuff_VVR(v_hi2, v_lo2, -1); + HVX_VectorPair vp_shuf3 = Q6_W_vshuff_VVR(v_hi3, v_lo3, -1); + + // Unpack to 16-bit + HVX_VectorPair vp_int16_lo0 = Q6_Wh_vunpack_Vb(Q6_V_lo_W(vp_shuf0)); + HVX_VectorPair vp_int16_hi0 = Q6_Wh_vunpack_Vb(Q6_V_hi_W(vp_shuf0)); + HVX_VectorPair vp_int16_lo1 = Q6_Wh_vunpack_Vb(Q6_V_lo_W(vp_shuf1)); + HVX_VectorPair vp_int16_hi1 = Q6_Wh_vunpack_Vb(Q6_V_hi_W(vp_shuf1)); + HVX_VectorPair vp_int16_lo2 = Q6_Wh_vunpack_Vb(Q6_V_lo_W(vp_shuf2)); + HVX_VectorPair vp_int16_hi2 = Q6_Wh_vunpack_Vb(Q6_V_hi_W(vp_shuf2)); + HVX_VectorPair vp_int16_lo3 = Q6_Wh_vunpack_Vb(Q6_V_lo_W(vp_shuf3)); + HVX_VectorPair vp_int16_hi3 = Q6_Wh_vunpack_Vb(Q6_V_hi_W(vp_shuf3)); + + // Convert and scale multiplication + HVX_Vector v_grp0_0 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(Q6_Vhf_equals_Vh(Q6_V_lo_W(vp_int16_lo0)), v_scale_duplicated)); + HVX_Vector v_grp0_1 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(Q6_Vhf_equals_Vh(Q6_V_hi_W(vp_int16_lo0)), v_scale_duplicated)); + HVX_Vector v_grp0_2 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(Q6_Vhf_equals_Vh(Q6_V_lo_W(vp_int16_hi0)), v_scale_duplicated)); + HVX_Vector v_grp0_3 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(Q6_Vhf_equals_Vh(Q6_V_hi_W(vp_int16_hi0)), v_scale_duplicated)); + + HVX_Vector v_grp1_0 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(Q6_Vhf_equals_Vh(Q6_V_lo_W(vp_int16_lo1)), v_scale_duplicated)); + HVX_Vector v_grp1_1 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(Q6_Vhf_equals_Vh(Q6_V_hi_W(vp_int16_lo1)), v_scale_duplicated)); + HVX_Vector v_grp1_2 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(Q6_Vhf_equals_Vh(Q6_V_lo_W(vp_int16_hi1)), v_scale_duplicated)); + HVX_Vector v_grp1_3 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(Q6_Vhf_equals_Vh(Q6_V_hi_W(vp_int16_hi1)), v_scale_duplicated)); + + HVX_Vector v_grp2_0 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(Q6_Vhf_equals_Vh(Q6_V_lo_W(vp_int16_lo2)), v_scale_duplicated)); + HVX_Vector v_grp2_1 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(Q6_Vhf_equals_Vh(Q6_V_hi_W(vp_int16_lo2)), v_scale_duplicated)); + HVX_Vector v_grp2_2 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(Q6_Vhf_equals_Vh(Q6_V_lo_W(vp_int16_hi2)), v_scale_duplicated)); + HVX_Vector v_grp2_3 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(Q6_Vhf_equals_Vh(Q6_V_hi_W(vp_int16_hi2)), v_scale_duplicated)); + + HVX_Vector v_grp3_0 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(Q6_Vhf_equals_Vh(Q6_V_lo_W(vp_int16_lo3)), v_scale_duplicated)); + HVX_Vector v_grp3_1 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(Q6_Vhf_equals_Vh(Q6_V_hi_W(vp_int16_lo3)), v_scale_duplicated)); + HVX_Vector v_grp3_2 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(Q6_Vhf_equals_Vh(Q6_V_lo_W(vp_int16_hi3)), v_scale_duplicated)); + HVX_Vector v_grp3_3 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(Q6_Vhf_equals_Vh(Q6_V_hi_W(vp_int16_hi3)), v_scale_duplicated)); + + // Parallel Stores + hvx_vmem(dst_ptr + 0 * 64) = v_grp0_0; + hvx_vmem(dst_ptr + 1 * 64) = v_grp0_1; + hvx_vmem(dst_ptr + 2 * 64) = v_grp0_2; + hvx_vmem(dst_ptr + 3 * 64) = v_grp0_3; + + hvx_vmem(dst_ptr + 4 * 64) = v_grp1_0; + hvx_vmem(dst_ptr + 5 * 64) = v_grp1_1; + hvx_vmem(dst_ptr + 6 * 64) = v_grp1_2; + hvx_vmem(dst_ptr + 7 * 64) = v_grp1_3; + + hvx_vmem(dst_ptr + 8 * 64) = v_grp2_0; + hvx_vmem(dst_ptr + 9 * 64) = v_grp2_1; + hvx_vmem(dst_ptr + 10 * 64) = v_grp2_2; + hvx_vmem(dst_ptr + 11 * 64) = v_grp2_3; + + hvx_vmem(dst_ptr + 12 * 64) = v_grp3_0; + hvx_vmem(dst_ptr + 13 * 64) = v_grp3_1; + hvx_vmem(dst_ptr + 14 * 64) = v_grp3_2; + hvx_vmem(dst_ptr + 15 * 64) = v_grp3_3; + } +} + // Q6_K stores 6-bit weights and one fp16 scale per 16 k, see HTP_TENSOR_TILE_SIZE_Q6_K. // A k-group holds 4 k per row, the HMX tile holds 2, so each group is dealt into two tiles. static void dequantize_tiled_weight_to_fp16_task_q6_k( @@ -1235,7 +1333,6 @@ typedef struct { uint32_t dst_stride; // DDR row stride uint32_t src2_stride; // DDR row stride for residual uint32_t dst_cols; // Actual output columns - struct htp_thread_trace * traces; } output_transfer_task_state_t; // activations : fp32 -> fp16 diff --git a/ggml/src/ggml-hexagon/htp/hmx-queue.c b/ggml/src/ggml-hexagon/htp/hmx-queue.c index c369d3dd23f0..e40a28bed74e 100644 --- a/ggml/src/ggml-hexagon/htp/hmx-queue.c +++ b/ggml/src/ggml-hexagon/htp/hmx-queue.c @@ -46,9 +46,9 @@ static inline void hmx_queue_process(hmx_queue_t q, bool* killed) { case HMX_QUEUE_WAKEUP: hmx_lock(q); break; default: hmx_lock(q); - htp_trace_event_start(q->trace, HTP_TRACE_EVT_HMX_COMP, ir); - d->func(d->data); - htp_trace_event_stop(q->trace, HTP_TRACE_EVT_HMX_COMP, ir); + htp_trace_event_start(q->th.trace, HTP_TRACE_EVT_HMX_COMP, ir); + d->func(&q->th, d->data); + htp_trace_event_stop(q->th.trace, HTP_TRACE_EVT_HMX_COMP, ir); break; } @@ -63,7 +63,7 @@ static inline void hmx_queue_process(hmx_queue_t q, bool* killed) { static void hmx_queue_thread(void * arg) { hmx_queue_t q = (hmx_queue_t) arg; - FARF(HIGH, "hmx-queue-thread: started"); + FARF(HIGH, "hmx-queue-thread (tid %u): started", q->th.tid); bool killed = false; @@ -90,7 +90,7 @@ static void hmx_queue_thread(void * arg) { hmx_queue_process(q, &killed); } - FARF(HIGH, "hmx-queue-thread: stopped"); + FARF(HIGH, "hmx-queue-thread (tid %u): stopped", q->th.tid); } size_t hmx_queue_sizeof(size_t capacity, uint32_t stack_size) { @@ -105,7 +105,7 @@ size_t hmx_queue_alignof(void) { return HEX_L2_LINE_SIZE; } -hmx_queue_t hmx_queue_init(void * ptr, size_t capacity, uint32_t stack_size, uint32_t hap_rctx, struct htp_thread_trace * trace) { +hmx_queue_t hmx_queue_init(void * ptr, size_t capacity, uint32_t stack_size, uint32_t hap_rctx, dma_queue_t * dma, struct htp_thread_trace * trace) { capacity = hex_ceil_pow2(capacity); size_t size_q = hex_align_up(sizeof(struct hmx_queue_s), HEX_L2_LINE_SIZE); size_t size_desc = hex_align_up(capacity * sizeof(struct hmx_queue_desc), HEX_L2_LINE_SIZE); @@ -126,7 +126,7 @@ hmx_queue_t hmx_queue_init(void * ptr, size_t capacity, uint32_t stack_size, uin q->stack = block; memset(q->stack, 0, stack_size); - q->trace = trace; + htp_thread_init(&q->th, dma, trace); // Match caller thread priority (same pattern as worker-pool.c). int prio = qurt_thread_get_priority(qurt_thread_get_id()); diff --git a/ggml/src/ggml-hexagon/htp/hmx-queue.h b/ggml/src/ggml-hexagon/htp/hmx-queue.h index c2b1859a2813..efefb1455b48 100644 --- a/ggml/src/ggml-hexagon/htp/hmx-queue.h +++ b/ggml/src/ggml-hexagon/htp/hmx-queue.h @@ -11,7 +11,8 @@ #include #include "hex-utils.h" -#include "hex-profile.h" +#include "htp-trace.h" +#include "htp-thread.h" #ifdef __cplusplus extern "C" { @@ -23,7 +24,7 @@ extern "C" { #define HMX_QUEUE_POLL_COUNT 1 #endif -typedef void (*hmx_queue_func)(void *); +typedef void (*hmx_queue_func)(const htp_thread_t * th, void *); // Dummy funcs used as signals enum hmx_queue_signal { @@ -49,10 +50,10 @@ struct hmx_queue_s { atomic_uint seqn; // incremented for all pushes, used with futex qurt_thread_t thread; + htp_thread_t th; void * stack; uint32_t hap_rctx; bool hmx_locked; - struct htp_thread_trace * trace; bool external_mem; // memory owned externally }; @@ -60,7 +61,7 @@ typedef struct hmx_queue_s * hmx_queue_t; size_t hmx_queue_sizeof(size_t capacity, uint32_t stack_size); size_t hmx_queue_alignof(void); -hmx_queue_t hmx_queue_init(void * ptr, size_t capacity, uint32_t stack_size, uint32_t hap_rctx, struct htp_thread_trace * trace); +hmx_queue_t hmx_queue_init(void * ptr, size_t capacity, uint32_t stack_size, uint32_t hap_rctx, dma_queue_t * dma, struct htp_thread_trace * trace); void hmx_queue_free(hmx_queue_t q); static inline struct hmx_queue_desc hmx_queue_make_desc(hmx_queue_func func, void * data) { diff --git a/ggml/src/ggml-hexagon/htp/htp-ctx.h b/ggml/src/ggml-hexagon/htp/htp-ctx.h index 0c455141d384..721fa2162d20 100644 --- a/ggml/src/ggml-hexagon/htp/htp-ctx.h +++ b/ggml/src/ggml-hexagon/htp/htp-ctx.h @@ -5,7 +5,7 @@ #include "hmx-queue.h" #include "htp-dcache.h" #include "htp-ops.h" -#include "hex-profile.h" +#include "htp-trace.h" #include "work-queue.h" #include "hex-fastdiv.h" @@ -17,10 +17,6 @@ #include #include -#ifndef HTP_MAX_NTHREADS -#define HTP_MAX_NTHREADS 10 -#endif - #define HTP_MAX_MMAPS 64 // Memory mapping @@ -37,7 +33,7 @@ struct htp_spad { uint8_t * data; // pointer to an area in vtcm uint32_t stride; // stride used inside this spad uint32_t size; // total size - uint32_t size_per_thread; // size per thread + uint32_t size_per_hvx; // size per hvx }; struct htp_mdev_group { @@ -71,14 +67,14 @@ struct htp_ops_context { struct htp_spad dst_spad; uint32_t flags; - uint32_t n_threads; - struct fastdiv_values n_threads_div; + uint32_t n_hvx; + struct fastdiv_values n_hvx_div; int status; // Must stay last: not cleared by htp_ops_context_reset(), ops init the caches they use union { - dma_cache_dm dma_cache_dm[HTP_MAX_NTHREADS]; - dma_cache_fa dma_cache_fa[HTP_MAX_NTHREADS]; + dma_cache_dm dma_cache_dm[HTP_MAX_NHVX]; + dma_cache_fa dma_cache_fa[HTP_MAX_NHVX]; } __attribute__((aligned(128))); }; @@ -88,12 +84,13 @@ struct htp_context { struct htp_mmap mmap[HTP_MAX_MMAPS]; dma_queue_t dma[HTP_MAX_NTHREADS]; - struct htp_thread_trace trace[HTP_MAX_NTHREADS + 1]; - work_queue_t work_queue; + struct htp_thread_trace trace[HTP_MAX_NTHREADS]; + work_queue_t hvx_queue; hmx_queue_t hmx_queue; uint32_t n_threads; - struct fastdiv_values n_threads_div; + uint32_t n_hvx; + struct fastdiv_values n_hvx_div; int thread_id; int thread_prio; @@ -126,26 +123,39 @@ struct htp_context { static inline void htp_ops_context_reset(struct htp_ops_context * octx, struct htp_context * ctx) { memset(octx, 0, offsetof(struct htp_ops_context, dma_cache_dm)); - octx->ctx = ctx; - octx->n_threads = ctx->n_threads; - octx->n_threads_div = ctx->n_threads_div; + octx->ctx = ctx; + octx->n_hvx = ctx->n_hvx; + octx->n_hvx_div = ctx->n_hvx_div; } -static inline bool htp_ops_context_set_n_threads(struct htp_ops_context * octx, uint32_t n_threads) { - if (n_threads == 0 || n_threads > octx->ctx->n_threads) { +static inline bool htp_ops_context_set_n_hvx_div( + struct htp_ops_context * octx, + uint32_t n_hvx, + struct fastdiv_values n_hvx_div) { + if (n_hvx == 0 || n_hvx > octx->ctx->n_hvx) { return false; } - if (n_threads != octx->n_threads) { - octx->n_threads = n_threads; - octx->n_threads_div = n_threads == octx->ctx->n_threads - ? octx->ctx->n_threads_div - : init_fastdiv_values(n_threads); - } + octx->n_hvx = n_hvx; + octx->n_hvx_div = n_hvx_div; return true; } +static inline bool htp_ops_context_set_n_hvx(struct htp_ops_context * octx, uint32_t n_hvx) { + if (n_hvx == 0 || n_hvx > octx->ctx->n_hvx) { + return false; + } + if (n_hvx == octx->n_hvx) { + return true; + } + + const struct fastdiv_values n_hvx_div = n_hvx == octx->ctx->n_hvx + ? octx->ctx->n_hvx_div + : init_fastdiv_values(n_hvx); + return htp_ops_context_set_n_hvx_div(octx, n_hvx, n_hvx_div); +} + static inline void htp_ops_context_set_status(struct htp_ops_context * octx, int status) { if (status > HTP_STATUS_OK && octx->status == HTP_STATUS_OK) { octx->status = status; @@ -174,6 +184,7 @@ int op_argsort(struct htp_ops_context * octx); int op_top_k(struct htp_ops_context * octx); int op_ssm_conv(struct htp_ops_context * octx); int op_cumsum(struct htp_ops_context * octx); +int op_top_k_mask(struct htp_ops_context * octx); int op_fill(struct htp_ops_context * octx); int op_concat(struct htp_ops_context * octx); int op_diag(struct htp_ops_context * octx); @@ -189,5 +200,6 @@ int op_pool_1d(struct htp_ops_context * octx); int op_allreduce(struct htp_ops_context * octx); int op_roll(struct htp_ops_context * octx); int op_upscale(struct htp_ops_context * octx); +int op_lightning_indexer(struct htp_ops_context * octx); #endif /* HTP_CTX_H */ diff --git a/ggml/src/ggml-hexagon/htp/htp-dcache.c b/ggml/src/ggml-hexagon/htp/htp-dcache.c index 840b1edb7a7c..52fc8e403864 100644 --- a/ggml/src/ggml-hexagon/htp/htp-dcache.c +++ b/ggml/src/ggml-hexagon/htp/htp-dcache.c @@ -7,7 +7,7 @@ #include "hex-common.h" #include "hex-utils.h" #include "hex-fastdiv.h" -#include "hex-profile.h" +#include "htp-trace.h" #include "htp-ctx.h" #include "work-queue.h" @@ -15,15 +15,14 @@ struct l2flush_range { uint32_t start; // line-aligned start address uint32_t end; // line-aligned end address uint32_t block_first; // global block index of this range's first block - uint32_t n_blocks; // number of HEX_L2_BLOCK_SIZE chunks (last may be partial) + uint32_t n_blocks; // number of HTP_DCACHE_BLOCK_SIZE chunks (last may be partial) }; struct l2flush_multi_task { - struct htp_thread_trace * trace; struct l2flush_range ranges[HTP_MAX_DIRTY_RANGES]; uint32_t n_ranges; uint32_t total_blocks; - uint32_t blocks_per_thread; + uint32_t blocks_per_hvx; }; static void flush_all_dcache(struct htp_context * ctx) { @@ -35,12 +34,12 @@ static void flush_all_dcache(struct htp_context * ctx) { memset(ctx->dirty_ranges, 0, sizeof(ctx->dirty_ranges)); } -static void l2flush_multi_worker(unsigned int n, unsigned int i, void * data) { +static void l2flush_multi_worker(const htp_thread_t * th, unsigned int n, unsigned int i, void * data) { struct l2flush_multi_task * task = (struct l2flush_multi_task *) data; (void) n; - const uint32_t gb_first = i * task->blocks_per_thread; - uint32_t gb_last = gb_first + task->blocks_per_thread; + const uint32_t gb_first = i * task->blocks_per_hvx; + uint32_t gb_last = gb_first + task->blocks_per_hvx; if (gb_last > task->total_blocks) { gb_last = task->total_blocks; } @@ -48,7 +47,7 @@ static void l2flush_multi_worker(unsigned int n, unsigned int i, void * data) { return; } - struct htp_thread_trace * tr = &task->trace[i]; + struct htp_thread_trace * tr = th->trace; htp_trace_event_start(tr, HTP_TRACE_EVT_L2FLUSH, gb_first); for (uint32_t r = 0; r < task->n_ranges; r++) { @@ -62,8 +61,8 @@ static void l2flush_multi_worker(unsigned int n, unsigned int i, void * data) { continue; } - const uint32_t s = rg->start + (lo - rb_first) * HEX_L2_BLOCK_SIZE; - uint32_t e = rg->start + (hi - rb_first) * HEX_L2_BLOCK_SIZE; + const uint32_t s = rg->start + (lo - rb_first) * HTP_DCACHE_BLOCK_SIZE; + uint32_t e = rg->start + (hi - rb_first) * HTP_DCACHE_BLOCK_SIZE; if (e > rg->end) { e = rg->end; } @@ -163,7 +162,7 @@ void htp_dcache_mark_dirty_tensors(struct htp_context * ctx, const struct htp_te total_evict_size += r->end - r->start; } - if (total_evict_size > HEX_L2_FLUSH_ALL_THRESHOLD) { + if (total_evict_size > HTP_DCACHE_FLUSH_ALL_THRESHOLD) { flush_all_dcache(ctx); for (uint32_t i = 0; i < n_pending; i++) { struct htp_dirty_range * r = &ctx->dirty_ranges[i]; @@ -174,9 +173,8 @@ void htp_dcache_mark_dirty_tensors(struct htp_context * ctx, const struct htp_te return; } - if (total_evict_size > HEX_L2_FLUSH_WQ_THRESHOLD && ctx->n_threads > 1 && n_evict <= HTP_MAX_DIRTY_RANGES) { + if (total_evict_size > HTP_DCACHE_FLUSH_WQ_THRESHOLD && ctx->n_hvx > 1 && n_evict <= HTP_MAX_DIRTY_RANGES) { struct l2flush_multi_task task; - task.trace = ctx->trace; task.n_ranges = n_evict; uint32_t block_acc = 0; @@ -188,14 +186,14 @@ void htp_dcache_mark_dirty_tensors(struct htp_context * ctx, const struct htp_te rg->start = hex_align_down((size_t) r->start, HEX_L2_LINE_SIZE); rg->end = hex_align_up((size_t) r->end, HEX_L2_LINE_SIZE); rg->block_first = block_acc; - rg->n_blocks = (rg->end - rg->start + HEX_L2_BLOCK_SIZE - 1) / HEX_L2_BLOCK_SIZE; + rg->n_blocks = (rg->end - rg->start + HTP_DCACHE_BLOCK_SIZE - 1) / HTP_DCACHE_BLOCK_SIZE; block_acc += rg->n_blocks; } - task.total_blocks = block_acc; - task.blocks_per_thread = fastdiv(block_acc + ctx->n_threads - 1, &ctx->n_threads_div); + task.total_blocks = block_acc; + task.blocks_per_hvx = fastdiv(block_acc + ctx->n_hvx - 1, &ctx->n_hvx_div); - work_queue_run(ctx->work_queue, l2flush_multi_worker, &task, ctx->n_threads); + work_queue_run(ctx->hvx_queue, l2flush_multi_worker, &task, ctx->n_hvx); } else { struct htp_thread_trace * tr = &ctx->trace[0]; htp_trace_event_start(tr, HTP_TRACE_EVT_L2FLUSH, 0); @@ -270,9 +268,8 @@ static inline bool is_tensor_dirty(struct htp_context * ctx, const struct htp_te } static void flush_dirty_ranges(struct htp_context * ctx, const struct htp_dirty_range * ranges, uint32_t n_ranges, uint64_t total_dirty) { - if (total_dirty >= HEX_L2_FLUSH_WQ_THRESHOLD && ctx->n_threads > 1) { + if (total_dirty >= HTP_DCACHE_FLUSH_WQ_THRESHOLD && ctx->n_hvx > 1) { struct l2flush_multi_task task; - task.trace = ctx->trace; task.n_ranges = n_ranges; uint32_t block_acc = 0; @@ -282,14 +279,14 @@ static void flush_dirty_ranges(struct htp_context * ctx, const struct htp_dirty_ rg->start = hex_align_down((size_t) r->start, HEX_L2_LINE_SIZE); rg->end = hex_align_up((size_t) r->end, HEX_L2_LINE_SIZE); rg->block_first = block_acc; - rg->n_blocks = (rg->end - rg->start + HEX_L2_BLOCK_SIZE - 1) / HEX_L2_BLOCK_SIZE; + rg->n_blocks = (rg->end - rg->start + HTP_DCACHE_BLOCK_SIZE - 1) / HTP_DCACHE_BLOCK_SIZE; block_acc += rg->n_blocks; } - task.total_blocks = block_acc; - task.blocks_per_thread = fastdiv(block_acc + ctx->n_threads - 1, &ctx->n_threads_div); + task.total_blocks = block_acc; + task.blocks_per_hvx = fastdiv(block_acc + ctx->n_hvx - 1, &ctx->n_hvx_div); - work_queue_run(ctx->work_queue, l2flush_multi_worker, &task, ctx->n_threads); + work_queue_run(ctx->hvx_queue, l2flush_multi_worker, &task, ctx->n_hvx); } else { struct htp_thread_trace * tr = &ctx->trace[0]; htp_trace_event_start(tr, HTP_TRACE_EVT_L2FLUSH, 0); @@ -319,7 +316,7 @@ void htp_dcache_flush_dirty_ranges(struct htp_context * ctx) { return; } - if (total_dirty > HEX_L2_FLUSH_ALL_THRESHOLD) { + if (total_dirty > HTP_DCACHE_FLUSH_ALL_THRESHOLD) { flush_all_dcache(ctx); return; } @@ -348,7 +345,7 @@ void htp_dcache_flush_dirty_tensors(struct htp_context * ctx, const struct htp_t return; } - if (total_dirty > HEX_L2_FLUSH_ALL_THRESHOLD) { + if (total_dirty > HTP_DCACHE_FLUSH_ALL_THRESHOLD) { flush_all_dcache(ctx); return; } diff --git a/ggml/src/ggml-hexagon/htp/htp-dcache.h b/ggml/src/ggml-hexagon/htp/htp-dcache.h index 1996ab258d3e..c1c37573ceb0 100644 --- a/ggml/src/ggml-hexagon/htp/htp-dcache.h +++ b/ggml/src/ggml-hexagon/htp/htp-dcache.h @@ -2,9 +2,13 @@ #define HTP_DCACHE_H #include +#include "hex-common.h" #include "htp-ops.h" -#define HTP_MAX_DIRTY_RANGES 64 +#define HTP_MAX_DIRTY_RANGES 64 +#define HTP_DCACHE_BLOCK_SIZE (HEX_L2_LINE_SIZE * 4) // flush granularity (chunks per thread) +#define HTP_DCACHE_FLUSH_WQ_THRESHOLD (4 * 1024) +#define HTP_DCACHE_FLUSH_ALL_THRESHOLD (4 * 1024 * 1024) struct htp_dirty_range { uint32_t start; diff --git a/ggml/src/ggml-hexagon/htp/htp-ops.h b/ggml/src/ggml-hexagon/htp/htp-ops.h index b8df01830b84..c28b8d61cb3e 100644 --- a/ggml/src/ggml-hexagon/htp/htp-ops.h +++ b/ggml/src/ggml-hexagon/htp/htp-ops.h @@ -21,6 +21,8 @@ enum htp_data_type { HTP_TYPE_F16 = 1, HTP_TYPE_Q4_0 = 2, HTP_TYPE_Q4_1 = 3, + HTP_TYPE_Q5_0 = 6, + HTP_TYPE_Q5_1 = 7, HTP_TYPE_Q8_0 = 8, HTP_TYPE_Q2_K = 10, HTP_TYPE_Q3_K = 11, @@ -51,6 +53,7 @@ enum htp_op_code { HTP_OP_MUL_MAT_ADD, HTP_OP_RMS_NORM, HTP_OP_RMS_NORM_MUL, + HTP_OP_RMS_NORM_MUL_ADD, HTP_OP_UNARY_SILU, HTP_OP_UNARY_GELU, HTP_OP_UNARY_SIGMOID, @@ -117,18 +120,26 @@ enum htp_op_code { HTP_OP_UNARY_CEIL, HTP_OP_UNARY_SGN, HTP_OP_UPSCALE, + HTP_OP_CONV_3D, + HTP_OP_DSV4_HC_PRE, + HTP_OP_DSV4_HC_POST, + HTP_OP_TOP_K_MASK, + HTP_OP_LIGHTNING_INDEXER, HTP_OP_INVALID }; -#define HTP_OP_MAX_DIMS 4 // aka GGML_MAX_DIMS -#define HTP_OP_MAX_INPUTS 10 // aka GGML_MAX_SRCS -#define HTP_OP_MAX_OUTPUTS 4 -#define HTP_OP_MAX_PARAMS 16 // aka GGML_MAX_OP_PARAMS +#define HTP_OP_MAX_DIMS 4 // aka GGML_MAX_DIMS +#define HTP_OP_MAX_INPUTS 10 // aka GGML_MAX_SRCS +#define HTP_OP_MAX_OUTPUTS 4 +#define HTP_OP_MAX_PARAMS 16 // aka GGML_MAX_OP_PARAMS #define HTP_OP_MAX_KERN_PARAMS 32 -#define HTP_OP_MAX_BUFS 64 -#define HTP_OP_MAX_TENSORS 65534 // must stay under 64K (uint16) +#define HTP_OP_MAX_BUFS 64 +#define HTP_OP_MAX_TENSORS 65534 // must stay under 64K (uint16) + +#define HTP_OP_MAX_NTHREADS 10 +#define HTP_OP_MAX_NHVX 8 #define HTP_FENCE_TIMEOUT (1000000000ULL) #define HTP_FENCE_SLOT_SIZE 128 @@ -185,8 +196,20 @@ struct htp_op_desc { uint16_t pad[2]; // padding to align to 64 bits }; -#ifndef HTP_MAX_NTHREADS +#ifdef __HVX_ARCH__ +#if __HVX_ARCH__ <= 73 +#define HTP_MAX_NTHREADS 6 +#define HTP_MAX_NHVX 4 +#elif __HVX_ARCH__ <= 75 +#define HTP_MAX_NTHREADS 8 +#define HTP_MAX_NHVX 6 +#else #define HTP_MAX_NTHREADS 10 +#define HTP_MAX_NHVX 8 +#endif + +_Static_assert(HTP_MAX_NTHREADS <= HTP_OP_MAX_NTHREADS, "HTP_MAX_NTHREADS exceeds HTP_OP_MAX_NTHREADS"); +_Static_assert(HTP_MAX_NHVX <= HTP_OP_MAX_NHVX, "HTP_MAX_NHVX exceeds HTP_OP_MAX_NHVX"); #endif #define HTP_TRACE_MAX_EVENTS 256 @@ -279,7 +302,7 @@ struct htp_opbatch_rsp { uint32_t n_tensors; // Number of tensors uint32_t n_ops; // Number of op profile descriptors uint32_t usecs; // Number of usec - uint32_t n_traces[HTP_MAX_NTHREADS + 1]; + uint32_t n_traces[HTP_OP_MAX_NTHREADS]; // struct htp_prof_desc profs[]; -- dspqueue buf 0 }; diff --git a/ggml/src/ggml-hexagon/htp/htp-tensor.h b/ggml/src/ggml-hexagon/htp/htp-tensor.h index a5ea0d533984..beacc7e4db58 100644 --- a/ggml/src/ggml-hexagon/htp/htp-tensor.h +++ b/ggml/src/ggml-hexagon/htp/htp-tensor.h @@ -17,6 +17,8 @@ #define HTP_TENSOR_TILE_SIZE_F16 2048 #define HTP_TENSOR_TILE_SIZE_Q4_0 576 #define HTP_TENSOR_TILE_SIZE_Q4_1 640 +#define HTP_TENSOR_TILE_SIZE_Q5_0 704 +#define HTP_TENSOR_TILE_SIZE_Q5_1 768 #define HTP_TENSOR_TILE_SIZE_Q8_0 1088 #define HTP_TENSOR_TILE_SIZE_IQ4_NL 576 #define HTP_TENSOR_TILE_SIZE_MXFP4 544 @@ -30,7 +32,7 @@ #define HTP_TENSOR_TILE_K_Q2_0 64 #define HTP_TENSOR_TILE_SIZE_Q2_0 576 -// Tiled weight tensor layouts (Q4_0, Q8_0, IQ4_NL, MXFP4, Q1_0, Q2_0, K-quants): +// Tiled weight tensor layouts (Q4_0, Q5_0, Q5_1, Q8_0, IQ4_NL, MXFP4, Q1_0, Q2_0, K-quants): // See docs/backend/snapdragon/developer.md ("Tiled Weight Formats"). struct htp_tensor_mdev_range { @@ -143,6 +145,8 @@ static inline uint32_t htp_tensor_get_tile_size(int type) { case HTP_TYPE_Q4_0: return HTP_TENSOR_TILE_SIZE_Q4_0; case HTP_TYPE_IQ4_NL: return HTP_TENSOR_TILE_SIZE_IQ4_NL; case HTP_TYPE_Q4_1: return HTP_TENSOR_TILE_SIZE_Q4_1; + case HTP_TYPE_Q5_0: return HTP_TENSOR_TILE_SIZE_Q5_0; + case HTP_TYPE_Q5_1: return HTP_TENSOR_TILE_SIZE_Q5_1; case HTP_TYPE_Q4_K: return HTP_TENSOR_TILE_SIZE_Q4_K; case HTP_TYPE_Q8_0: return HTP_TENSOR_TILE_SIZE_Q8_0; case HTP_TYPE_Q5_K: return HTP_TENSOR_TILE_SIZE_Q5_K; @@ -182,6 +186,8 @@ static inline uint32_t htp_tensor_get_row_size(int type, uint32_t ne00) { case HTP_TYPE_Q4_0: case HTP_TYPE_IQ4_NL: return (ne00 / 32) * 18; case HTP_TYPE_Q4_1: return (ne00 / 32) * 20; + case HTP_TYPE_Q5_0: return (ne00 / 32) * 22; + case HTP_TYPE_Q5_1: return (ne00 / 32) * 24; case HTP_TYPE_MXFP4: return (ne00 / 32) * 17; case HTP_TYPE_Q2_K: return (ne00 / 256) * 84; case HTP_TYPE_Q2_0: return (ne00 / 64) * 18; diff --git a/ggml/src/ggml-hexagon/htp/htp-thread.h b/ggml/src/ggml-hexagon/htp/htp-thread.h new file mode 100644 index 000000000000..e4ca874de3f4 --- /dev/null +++ b/ggml/src/ggml-hexagon/htp/htp-thread.h @@ -0,0 +1,33 @@ +#ifndef HTP_THREAD_H +#define HTP_THREAD_H + +#include +#include + +#include "dma-queue.h" +#include "htp-trace.h" + +typedef struct { + unsigned int tid; + dma_queue * dma_q; + struct htp_thread_trace * trace; +} htp_thread_t; + +extern atomic_uint htp_thread_count; + +static inline void htp_thread_reset(void) { + atomic_store_explicit(&htp_thread_count, 0, memory_order_relaxed); +} + +static inline unsigned int htp_thread_alloc_id(void) { + return atomic_fetch_add_explicit(&htp_thread_count, 1, memory_order_relaxed); +} + +static inline void htp_thread_init(htp_thread_t * th, dma_queue_t * dma, struct htp_thread_trace * trace) { + unsigned int tid = htp_thread_alloc_id(); + th->tid = tid; + th->dma_q = dma ? dma[tid] : NULL; + th->trace = trace ? &trace[tid] : NULL; +} + +#endif /* HTP_THREAD_H */ diff --git a/ggml/src/ggml-hexagon/htp/htp-trace.h b/ggml/src/ggml-hexagon/htp/htp-trace.h new file mode 100644 index 000000000000..66bd9110d65c --- /dev/null +++ b/ggml/src/ggml-hexagon/htp/htp-trace.h @@ -0,0 +1,38 @@ +#ifndef HTP_TRACE_H +#define HTP_TRACE_H + +#include + +#include "hex-utils.h" +#include "htp-ops.h" + +#define HTP_TRACE_EVT_START 0 +#define HTP_TRACE_EVT_STOP 1 + +struct htp_thread_trace { + uint32_t count; + uint32_t max_events; + struct htp_trace_desc * events; +}; + +static inline void htp_trace_event(struct htp_thread_trace * tr, uint16_t id, uint16_t info, uint32_t type) { + if (!tr->max_events) { + return; + } + uint32_t i = tr->count++; + if (i < tr->max_events) { + tr->events[i].id = id; + tr->events[i].info = info | (type == HTP_TRACE_EVT_STOP ? 0x8000 : 0); + tr->events[i].cycles = (uint32_t) hex_get_cycles(); + } +} + +static inline void htp_trace_event_start(struct htp_thread_trace * tr, uint16_t id, uint16_t info) { + htp_trace_event(tr, id, info, HTP_TRACE_EVT_START); +} + +static inline void htp_trace_event_stop(struct htp_thread_trace * tr, uint16_t id, uint16_t info) { + htp_trace_event(tr, id, info, HTP_TRACE_EVT_STOP); +} + +#endif /* HTP_TRACE_H */ diff --git a/ggml/src/ggml-hexagon/htp/hvx-mm-kernels-tiled.h b/ggml/src/ggml-hexagon/htp/hvx-mm-kernels-tiled.h index 01c671679664..4a6b2b5ea216 100644 --- a/ggml/src/ggml-hexagon/htp/hvx-mm-kernels-tiled.h +++ b/ggml/src/ggml-hexagon/htp/hvx-mm-kernels-tiled.h @@ -639,6 +639,74 @@ static inline HVX_VectorPair accum_5bit_32x2( return Q6_W_vcombine_VV(v_sum1, v_sum0); } +// OR 0xF0 into every lane of v whose flag j is set in the plane, i.e. 4-bit value | 0xF0 = q - 16 where the 5th bit is clear (Q5_0) +static inline HVX_Vector hvx_q5_0_or_neg(HVX_Vector v, HVX_Vector v_plane, int j) { + HVX_VectorPred q = Q6_Q_vand_VR(v_plane, 0x01010101u << j); + return Q6_V_vandor_VQR(v, q, 0xF0F0F0F0); +} + +// Q5_0: signed bytes (q - 16) straight from the nibbles and the plane +static inline HVX_VectorPair unpack_and_interleave_q5_0_x2(HVX_Vector v_src, HVX_Vector v_plane, int i, HVX_Vector mask_h4) { + HVX_Vector v_lo = hvx_q5_0_or_neg(Q6_V_vand_VV(v_src, mask_h4), v_plane, 2 * i); + HVX_Vector v_hi = hvx_q5_0_or_neg(Q6_Vub_vlsr_VubR(v_src, 4), v_plane, 2 * i + 1); + HVX_VectorPair v01_pair = Q6_W_vshuff_VVR(v_hi, v_lo, -1); + HVX_Vector v01_lo = Q6_V_lo_W(v01_pair); + HVX_Vector v01_hi = Q6_V_hi_W(v01_pair); + + HVX_Vector v23_lo = Q6_V_valign_VVR(v01_hi, v01_lo, 64); + HVX_Vector v_W0 = Q6_V_lo_W(Q6_W_vshuff_VVR(v23_lo, v01_lo, -2)); + + HVX_Vector v67_lo = Q6_V_valign_VVR(v01_lo, v01_hi, 64); + HVX_Vector v_W1 = Q6_V_lo_W(Q6_W_vshuff_VVR(v67_lo, v01_hi, -2)); + + return Q6_W_vcombine_VV(v_W1, v_W0); +} + +static inline HVX_Vector accum_q5_0_32x1( + const HVX_Vector * restrict vptr, + const HVX_Vector * restrict v_act +) { + HVX_Vector v_sum0 = Q6_V_vzero(); + HVX_Vector v_sum1 = Q6_V_vzero(); + HVX_Vector mask_h4 = Q6_Vb_vsplat_R(0x0F); + HVX_Vector v_plane = vptr[4]; + + #pragma unroll + for (int i = 0; i < 4; i++) { + HVX_VectorPair v_W_pair = unpack_and_interleave_q5_0_x2(vptr[i], v_plane, i, mask_h4); + v_sum0 = Q6_Vw_vrmpyacc_VwVbVb(v_sum0, Q6_V_lo_W(v_W_pair), v_act[i * 2 + 0]); + v_sum1 = Q6_Vw_vrmpyacc_VwVbVb(v_sum1, Q6_V_hi_W(v_W_pair), v_act[i * 2 + 1]); + } + + return Q6_Vw_vadd_VwVw(v_sum0, v_sum1); +} + +static inline HVX_VectorPair accum_q5_0_32x2( + const HVX_Vector * restrict vptr, + const HVX_Vector * restrict v_act0, + const HVX_Vector * restrict v_act1 +) { + HVX_Vector v_sum0 = Q6_V_vzero(); + HVX_Vector v_sum1 = Q6_V_vzero(); + HVX_Vector mask_h4 = Q6_Vb_vsplat_R(0x0F); + HVX_Vector v_plane = vptr[4]; + + #pragma unroll + for (int i = 0; i < 4; i++) { + HVX_VectorPair v_W_pair = unpack_and_interleave_q5_0_x2(vptr[i], v_plane, i, mask_h4); + HVX_Vector v_W0 = Q6_V_lo_W(v_W_pair); + HVX_Vector v_W1 = Q6_V_hi_W(v_W_pair); + + v_sum0 = Q6_Vw_vrmpyacc_VwVbVb(v_sum0, v_W0, v_act0[i * 2 + 0]); + v_sum0 = Q6_Vw_vrmpyacc_VwVbVb(v_sum0, v_W1, v_act0[i * 2 + 1]); + + v_sum1 = Q6_Vw_vrmpyacc_VwVbVb(v_sum1, v_W0, v_act1[i * 2 + 0]); + v_sum1 = Q6_Vw_vrmpyacc_VwVbVb(v_sum1, v_W1, v_act1[i * 2 + 1]); + } + + return Q6_W_vcombine_VV(v_sum1, v_sum0); +} + // Q6_K weights are stored unsigned (0..63), see HTP_TENSOR_TILE_SIZE_Q6_K. Unpack k-group g of a tile to signed bytes (q - 32) static inline HVX_Vector unpack_q6_k_group(const HVX_Vector * restrict vptr, int g, HVX_Vector mask_0f, HVX_Vector mask_03, HVX_Vector i32) { HVX_Vector v_lo = (g & 1) ? Q6_Vub_vlsr_VubR(vptr[g >> 1], 4) : Q6_V_vand_VV(vptr[g >> 1], mask_0f); @@ -1251,6 +1319,82 @@ static void tiled_vec_dot_q8_0_32x2(const uint32_t n, float * restrict s0, float } } +static void tiled_vec_dot_q5_0_32x1(const uint32_t n, float * restrict s, const void * restrict vx, const void * restrict vy, uint32_t valid_rows, const float * restrict sz) { + const uint8_t * restrict tile_ptr = vx; + const uint8_t * restrict y_q = vy; + + HVX_Vector v_sum_float = Q6_V_vzero(); + + uint32_t n_k_tiles = n / 32; + for (uint32_t kt = 0; kt < n_k_tiles; kt++) { + const HVX_Vector * restrict vptr = (const HVX_Vector *) (tile_ptr + kt * 768); + const HVX_Vector * restrict v_act = (const HVX_Vector *) (y_q + kt * 1152); + + HVX_Vector v_sum = accum_q5_0_32x1(vptr, v_act); + HVX_Vector v_sum_sf = Q6_Vsf_equals_Vw(v_sum); + + HVX_Vector v_scale_w = vptr[5]; + HVX_Vector v_scale_a = v_act[8]; + HVX_Vector v_scale_comb = hvx_vec_mul_f16_f16_to_f32_lower32(v_scale_w, v_scale_a); + HVX_Vector v_sum_scaled = hvx_vec_mul_f32_f32(v_sum_sf, v_scale_comb); + + v_sum_float = hvx_vec_add_f32_f32(v_sum_float, v_sum_scaled); + } + + if (sz) { + hvx_vec_store_u(s, valid_rows * sizeof(float), hvx_vec_add_f32_f32(v_sum_float, hvx_vmemu(sz))); + } else { + hvx_vec_store_u(s, valid_rows * sizeof(float), v_sum_float); + } +} + +static void tiled_vec_dot_q5_0_32x2(const uint32_t n, float * restrict s0, float * restrict s1, const void * restrict vx, const void * restrict vy0, const void * restrict vy1, uint32_t valid_rows, const float * restrict sz0, const float * restrict sz1) { + const uint8_t * restrict tile_ptr = vx; + const uint8_t * restrict y0_q = vy0; + const uint8_t * restrict y1_q = vy1; + + HVX_Vector v_sum_float_c0 = Q6_V_vzero(); + HVX_Vector v_sum_float_c1 = Q6_V_vzero(); + + uint32_t n_k_tiles = n / 32; + for (uint32_t kt = 0; kt < n_k_tiles; kt++) { + const HVX_Vector * restrict vptr = (const HVX_Vector *) (tile_ptr + kt * 768); + const HVX_Vector * restrict v_act0 = (const HVX_Vector *) (y0_q + kt * 1152); + const HVX_Vector * restrict v_act1 = (const HVX_Vector *) (y1_q + kt * 1152); + + HVX_VectorPair v_sums = accum_q5_0_32x2(vptr, v_act0, v_act1); + HVX_Vector v_sum_c0 = Q6_V_lo_W(v_sums); + HVX_Vector v_sum_c1 = Q6_V_hi_W(v_sums); + + HVX_Vector v_sum_sf_c0 = Q6_Vsf_equals_Vw(v_sum_c0); + HVX_Vector v_sum_sf_c1 = Q6_Vsf_equals_Vw(v_sum_c1); + + HVX_Vector v_scale_w = vptr[5]; + HVX_Vector v_scale_a_c0 = v_act0[8]; + HVX_Vector v_scale_a_c1 = v_act1[8]; + + HVX_Vector v_scale_comb_c0 = hvx_vec_mul_f16_f16_to_f32_lower32(v_scale_w, v_scale_a_c0); + HVX_Vector v_scale_comb_c1 = hvx_vec_mul_f16_f16_to_f32_lower32(v_scale_w, v_scale_a_c1); + + HVX_Vector v_sum_scaled_c0 = hvx_vec_mul_f32_f32(v_sum_sf_c0, v_scale_comb_c0); + HVX_Vector v_sum_scaled_c1 = hvx_vec_mul_f32_f32(v_sum_sf_c1, v_scale_comb_c1); + + v_sum_float_c0 = hvx_vec_add_f32_f32(v_sum_float_c0, v_sum_scaled_c0); + v_sum_float_c1 = hvx_vec_add_f32_f32(v_sum_float_c1, v_sum_scaled_c1); + } + + if (sz0) { + hvx_vec_store_u(s0, valid_rows * sizeof(float), hvx_vec_add_f32_f32(v_sum_float_c0, hvx_vmemu(sz0))); + } else { + hvx_vec_store_u(s0, valid_rows * sizeof(float), v_sum_float_c0); + } + if (sz1) { + hvx_vec_store_u(s1, valid_rows * sizeof(float), hvx_vec_add_f32_f32(v_sum_float_c1, hvx_vmemu(sz1))); + } else { + hvx_vec_store_u(s1, valid_rows * sizeof(float), v_sum_float_c1); + } +} + static void tiled_vec_dot_q5_k_32x1(const uint32_t n, float * restrict s, const void * restrict vx, const void * restrict vy, uint32_t valid_rows, const float * restrict sz) { const uint8_t * restrict tile_ptr = vx; const uint8_t * restrict y_q = vy; diff --git a/ggml/src/ggml-hexagon/htp/hvx-norm.h b/ggml/src/ggml-hexagon/htp/hvx-norm.h index 145c3086e2fa..5d13acc872e5 100644 --- a/ggml/src/ggml-hexagon/htp/hvx-norm.h +++ b/ggml/src/ggml-hexagon/htp/hvx-norm.h @@ -141,6 +141,71 @@ static inline void hvx_fast_rms_norm_mul_f32(const uint8_t * restrict src, } } +static inline void hvx_fast_rms_norm_mul_add_f32(const uint8_t * restrict src, + const uint8_t * restrict weight, + const uint8_t * restrict add, + uint8_t * restrict dst, + const int num_elems, + float epsilon, + float scale) { + const HVX_Vector * restrict v_src = (const HVX_Vector *) src; + const HVX_Vector * restrict v_weight = (const HVX_Vector *) weight; + const HVX_Vector * restrict v_add = (const HVX_Vector *) add; + HVX_Vector * restrict v_dst = (HVX_Vector *) dst; + + const int nvec = num_elems / VLEN_FP32; + const int nloe = num_elems % VLEN_FP32; + + HVX_Vector sum_v = Q6_V_vsplat_R(0x00000000); + HVX_Vector epsilon_v = hvx_vec_splat_f32(epsilon); + + #pragma unroll(4) + for (int i = 0; i < nvec; i++) { + HVX_Vector v1 = v_src[i]; + HVX_Vector v2 = Q6_Vqf32_vmpy_VsfVsf(v1, v1); + sum_v = Q6_Vqf32_vadd_Vqf32Vqf32(sum_v, v2); + } + + if (nloe > 0) { + HVX_VectorPred bmask = Q6_Q_vsetq_R(nloe * 4); + HVX_Vector v1 = Q6_V_vand_QV(bmask, v_src[nvec]); + HVX_Vector v2 = Q6_Vqf32_vmpy_VsfVsf(v1, v1); + sum_v = Q6_Vqf32_vadd_Vqf32Vqf32(sum_v, v2); + } + + sum_v = hvx_vec_reduce_sum_f32(Q6_Vsf_equals_Vqf32(sum_v)); + + HVX_Vector t_v = hvx_vec_splat_f32((float) num_elems); + HVX_Vector denom_v = hvx_vec_inverse_f32(t_v); + HVX_Vector mean_v = Q6_Vqf32_vmpy_VsfVsf(sum_v, denom_v); + HVX_Vector mean_epsilon_v = Q6_Vqf32_vadd_Vqf32Vsf(mean_v, epsilon_v); + + HVX_Vector scale_v = hvx_vec_rsqrt_f32(Q6_Vsf_equals_Vqf32(mean_epsilon_v)); + if (scale != 0.0f) { + scale_v = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(scale_v, hvx_vec_splat_f32(scale))); + } + + #pragma unroll(4) + for (int i = 0; i < nvec; i++) { + HVX_Vector v1 = v_src[i]; + HVX_Vector v2 = Q6_Vqf32_vmpy_VsfVsf(v1, scale_v); + HVX_Vector v3 = Q6_Vsf_equals_Vqf32(v2); + HVX_Vector v4 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(v3, v_weight[i])); + v_dst[i] = hvx_vec_add_f32_f32(v4, v_add[i]); + } + + if (nloe > 0) { + HVX_VectorPred bmask = Q6_Q_vsetq_R(nloe * 4); + HVX_Vector v1 = Q6_V_vand_QV(bmask, v_src[nvec]); + HVX_Vector v2 = Q6_Vqf32_vmpy_VsfVsf(v1, scale_v); + HVX_Vector v3 = Q6_Vsf_equals_Vqf32(v2); + HVX_Vector v4 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(v3, v_weight[nvec])); + HVX_Vector res_v = hvx_vec_add_f32_f32(v4, v_add[nvec]); + + hvx_vec_store_a(&v_dst[nvec], nloe * 4, res_v); + } +} + static inline void hvx_fast_norm_f32(const uint8_t * restrict src, uint8_t * restrict dst, const int num_elems, diff --git a/ggml/src/ggml-hexagon/htp/hvx-sigmoid.h b/ggml/src/ggml-hexagon/htp/hvx-sigmoid.h index d4d67ff2f07d..3f253a0b0325 100644 --- a/ggml/src/ggml-hexagon/htp/hvx-sigmoid.h +++ b/ggml/src/ggml-hexagon/htp/hvx-sigmoid.h @@ -52,6 +52,53 @@ static inline HVX_Vector hvx_vec_fast_sigmoid_f32_guard(HVX_Vector v, return Q6_V_vmux_QVV(pred_min, out, Q6_V_vzero()); } +static inline HVX_Vector hvx_vec_fast_sigmoid_f32_2it(HVX_Vector v) { + v = Q6_Vqf32_vmpy_VsfVsf(v, Q6_V_vsplat_R(FAST_SIGMOID_LOG2F)); + v = Q6_Vqf32_vmpy_VsfVsf(Q6_Vsf_equals_Vqf32(v), Q6_V_vsplat_R(FAST_SIGMOID_C3)); + + HVX_Vector in_int = hvx_vec_truncate_f32(Q6_Vsf_equals_Vqf32(v)); + HVX_Vector x = Q6_Vqf32_vsub_Vqf32Vsf(v, Q6_Vsf_equals_Vw(in_int)); + HVX_Vector xx = Q6_Vqf32_vmpy_Vqf32Vqf32(x, x); + + HVX_Vector v1 = Q6_Vqf32_vmpy_VsfVsf(Q6_Vsf_equals_Vqf32(xx), Q6_V_vsplat_R(FAST_SIGMOID_C2)); + v1 = Q6_Vqf32_vadd_Vqf32Vsf(v1, Q6_V_vsplat_R(FAST_SIGMOID_LOG2F)); + + HVX_Vector v2 = Q6_Vqf32_vmpy_VsfVsf(Q6_Vsf_equals_Vqf32(x), Q6_V_vsplat_R(FAST_SIGMOID_C1)); + v2 = Q6_Vqf32_vmpy_Vqf32Vqf32(v2, xx); + v2 = Q6_Vqf32_vadd_Vqf32Vqf32(v2, x); + + HVX_Vector v3 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_Vqf32Vqf32(v2, v1)); + v3 = Q6_Vw_vaslacc_VwVwR(v3, in_int, 24); + + HVX_Vector v4 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vsub_Vqf32Vqf32(v2, v1)); + HVX_Vector v5 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vsub_VsfVsf(v3, v4)); + + // Newton-Raphson with 2 iterations + HVX_Vector two_sf = hvx_vec_splat_f32(2.0f); + HVX_Vector i_sf = Q6_Vw_vsub_VwVw(Q6_V_vsplat_R(0x7EEEEBB3), v5); + HVX_Vector r_qf = Q6_Vqf32_vmpy_VsfVsf( + i_sf, Q6_Vsf_equals_Vqf32(Q6_Vqf32_vsub_VsfVsf(two_sf, Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(i_sf, v5))))); + r_qf = Q6_Vqf32_vmpy_Vqf32Vqf32( + r_qf, Q6_Vqf32_vsub_VsfVsf(two_sf, Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(Q6_Vsf_equals_Vqf32(r_qf), v5)))); + HVX_Vector res = Q6_Vsf_equals_Vqf32(r_qf); + + res = Q6_Vqf32_vmpy_VsfVsf(v3, res); + + return Q6_Vsf_equals_Vqf32(res); +} + +static inline HVX_Vector hvx_vec_fast_sigmoid_f32_guard_2it(HVX_Vector v, + HVX_Vector one, + HVX_Vector max_exp, + HVX_Vector min_exp) { + const HVX_VectorPred pred_max = Q6_Q_vcmp_gt_VsfVsf(max_exp, v); + const HVX_VectorPred pred_min = Q6_Q_vcmp_gt_VsfVsf(v, min_exp); + + HVX_Vector out = hvx_vec_fast_sigmoid_f32_2it(v); + out = Q6_V_vmux_QVV(pred_max, out, one); + return Q6_V_vmux_QVV(pred_min, out, Q6_V_vzero()); +} + static inline HVX_Vector hvx_vec_tanh_f32(HVX_Vector x) { // tanh(x) = 2 * sigmoid(2x) - 1 HVX_Vector two = hvx_vec_splat_f32(2.0f); diff --git a/ggml/src/ggml-hexagon/htp/im2col-ops.c b/ggml/src/ggml-hexagon/htp/im2col-ops.c index 9e662694b4cf..602ee1598f87 100644 --- a/ggml/src/ggml-hexagon/htp/im2col-ops.c +++ b/ggml/src/ggml-hexagon/htp/im2col-ops.c @@ -15,7 +15,7 @@ #include "htp-ops.h" #include "hvx-utils.h" #include "dma-queue.h" -#include "hex-profile.h" +#include "htp-trace.h" #include "htp-vtcm.h" #include "htp-tensor.h" #include "im2col-ops.h" @@ -25,16 +25,16 @@ struct htp_im2col_context { uint32_t pe_row_base; // first output row index assigned to this dev (DMA path) uint32_t pe_nrows; // number of output rows assigned to this dev (DMA path) - uint32_t pe_rows_per_thread; // output rows per worker - uint32_t pe_blocks_per_thread; // output-col blocks per worker (blocked DMA path) + uint32_t pe_rows_per_hvx; // output rows per worker + uint32_t pe_blocks_per_hvx; // output-col blocks per worker (blocked DMA path) uint32_t pe_src_row_bytes; // one output row's source, rounded 256 uint32_t pe_dst_row_bytes; // one output row's dst: OW*patch_stride*2, rounded 256 // Patch-embed DMA path VTCM ping-pong. uint8_t * pe_vtcm_src; // base of the 2x src buffers region uint8_t * pe_vtcm_dst; // base of the 2x dst buffers region - uint32_t pe_src_size_per_thread; // 2 * pe_src_row_bytes - uint32_t pe_dst_size_per_thread; // 2 * pe_dst_row_bytes + uint32_t pe_src_size_per_hvx; // 2 * pe_src_row_bytes + uint32_t pe_dst_size_per_hvx; // 2 * pe_dst_row_bytes uint32_t pe_owb; // output-col block size uint32_t pe_wb; // staged source window width @@ -50,10 +50,10 @@ struct htp_im2col_context { uint32_t patch_stride; // Per-thread initial coordinates (removes division from thread prologue) - uint32_t thread_in[HTP_MAX_NTHREADS]; - uint32_t thread_iod[HTP_MAX_NTHREADS]; - uint32_t thread_ioh[HTP_MAX_NTHREADS]; - uint32_t thread_c0[HTP_MAX_NTHREADS]; + uint32_t thread_in[HTP_MAX_NHVX]; + uint32_t thread_iod[HTP_MAX_NHVX]; + uint32_t thread_ioh[HTP_MAX_NHVX]; + uint32_t thread_c0[HTP_MAX_NHVX]; void * compute; uint32_t dst_elem; @@ -321,11 +321,11 @@ static void FNAME( IM2COL_GATHER_COMPUTE_BODY(im2col_gather_compute_f16, uint16_t, hvx_copy_f16_f32_uu) IM2COL_GATHER_COMPUTE_BODY(im2col_gather_compute_f32, float, hvx_copy_f32_uu) -static void im2col_blocked_dma_thread(unsigned int nth, unsigned int ith, void * data) { +static void im2col_blocked_dma_thread(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { (void) nth; struct htp_im2col_context * ictx = (struct htp_im2col_context *) data; struct htp_ops_context * octx = ictx->octx; - struct htp_thread_trace * restrict tr = &octx->ctx->trace[ith]; + struct htp_thread_trace * restrict tr = th->trace; const uint32_t OW = ictx->OW; const uint32_t KH = ictx->KH; const uint32_t owb = ictx->pe_owb; @@ -334,16 +334,16 @@ static void im2col_blocked_dma_thread(unsigned int nth, unsigned int ith, void * const uint32_t dst_elem = ictx->dst_elem; im2col_blocked_compute_fn_t compute = (im2col_blocked_compute_fn_t) ictx->compute; const dma_addr_t dst_data = octx->dst->data; - dma_queue * dma_q = octx->ctx->dma[ith]; - uint8_t * srcb_base = ictx->pe_vtcm_src + ith * ictx->pe_src_size_per_thread; - uint8_t * dstb_base = ictx->pe_vtcm_dst + ith * ictx->pe_dst_size_per_thread; + dma_queue * dma_q = th->dma_q; + uint8_t * srcb_base = ictx->pe_vtcm_src + ith * ictx->pe_src_size_per_hvx; + uint8_t * dstb_base = ictx->pe_vtcm_dst + ith * ictx->pe_dst_size_per_hvx; float * srcb2[2] = { (float *) srcb_base, (float *) (srcb_base + ictx->pe_src_row_bytes) }; void * dstb2[2] = { dstb_base, dstb_base + ictx->pe_dst_row_bytes }; float * scratch = (float *) (srcb_base + 2 * ictx->pe_src_row_bytes); // work is split over output-col blocks, so a single output row (1D) still uses every thread const uint32_t blk_end_max = ictx->pe_nrows * nbpr; - const uint32_t blk_start = ictx->pe_blocks_per_thread * ith; - const uint32_t blk_end = MIN(blk_start + ictx->pe_blocks_per_thread, blk_end_max); + const uint32_t blk_start = ictx->pe_blocks_per_hvx * ith; + const uint32_t blk_end = MIN(blk_start + ictx->pe_blocks_per_hvx, blk_end_max); if (blk_start >= blk_end) { return; } @@ -510,24 +510,24 @@ static void im2col_patchembed_compute_f32( } } -static void im2col_patchembed_dma_thread(unsigned int nth, unsigned int ith, void * data) { +static void im2col_patchembed_dma_thread(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { (void) nth; struct htp_im2col_context * ictx = (struct htp_im2col_context *) data; struct htp_ops_context * octx = ictx->octx; - struct htp_thread_trace * restrict tr = &octx->ctx->trace[ith]; + struct htp_thread_trace * restrict tr = th->trace; const uint32_t OW = ictx->OW; const uint32_t patch_stride = ictx->patch_stride; const uint32_t dst_elem = ictx->dst_elem; im2col_patchembed_compute_fn_t compute = (im2col_patchembed_compute_fn_t) ictx->compute; const dma_addr_t dst_data = octx->dst->data; - dma_queue * dma_q = octx->ctx->dma[ith]; - uint8_t * srcb_base = ictx->pe_vtcm_src + ith * ictx->pe_src_size_per_thread; - uint8_t * dstb_base = ictx->pe_vtcm_dst + ith * ictx->pe_dst_size_per_thread; + dma_queue * dma_q = th->dma_q; + uint8_t * srcb_base = ictx->pe_vtcm_src + ith * ictx->pe_src_size_per_hvx; + uint8_t * dstb_base = ictx->pe_vtcm_dst + ith * ictx->pe_dst_size_per_hvx; float * srcb2[2] = { (float *) srcb_base, (float *) (srcb_base + ictx->pe_src_row_bytes) }; void * dstb2[2] = { dstb_base, dstb_base + ictx->pe_dst_row_bytes }; const uint32_t row_end_max = ictx->pe_row_base + ictx->pe_nrows; - const uint32_t row_start = ictx->pe_row_base + ictx->pe_rows_per_thread * ith; - const uint32_t row_end = MIN(row_start + ictx->pe_rows_per_thread, row_end_max); + const uint32_t row_start = ictx->pe_row_base + ictx->pe_rows_per_hvx * ith; + const uint32_t row_end = MIN(row_start + ictx->pe_rows_per_hvx, row_end_max); if (row_start >= row_end) { return; } @@ -718,11 +718,11 @@ static void FNAME(const struct htp_im2col_context * ictx, IM2COL_3D_BLOCKED_COMPUTE_BODY(im2col_3d_blocked_compute_f16, uint16_t, hvx_copy_f16_f32_uu, hvx_splat_u16_u) IM2COL_3D_BLOCKED_COMPUTE_BODY(im2col_3d_blocked_compute_f32, float, hvx_copy_f32_uu, hvx_splat_f32_u) -static void im2col_3d_blocked_dma_thread(unsigned int nth, unsigned int ith, void * data) { +static void im2col_3d_blocked_dma_thread(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { (void) nth; struct htp_im2col_context * ictx = (struct htp_im2col_context *) data; struct htp_ops_context * octx = ictx->octx; - struct htp_thread_trace * restrict tr = &octx->ctx->trace[ith]; + struct htp_thread_trace * restrict tr = th->trace; const struct htp_tensor * restrict dst = octx->dst; const uint32_t OW = ictx->OW; const uint32_t KD = ictx->KD, KH = ictx->KH; @@ -732,15 +732,15 @@ static void im2col_3d_blocked_dma_thread(unsigned int nth, unsigned int ith, voi const uint32_t nbpr = ictx->pe_nbpr; const uint32_t dst_elem = ictx->dst_elem; const dma_addr_t dst_data = dst->data; - dma_queue * dma_q = octx->ctx->dma[ith]; - uint8_t * srcb_base = ictx->pe_vtcm_src + ith * ictx->pe_src_size_per_thread; - uint8_t * dstb_base = ictx->pe_vtcm_dst + ith * ictx->pe_dst_size_per_thread; + dma_queue * dma_q = th->dma_q; + uint8_t * srcb_base = ictx->pe_vtcm_src + ith * ictx->pe_src_size_per_hvx; + uint8_t * dstb_base = ictx->pe_vtcm_dst + ith * ictx->pe_dst_size_per_hvx; float * srcb2[2] = { (float *) srcb_base, (float *) (srcb_base + ictx->pe_src_row_bytes) }; void * dstb2[2] = { dstb_base, dstb_base + ictx->pe_dst_row_bytes }; float * scratch = (float *) (srcb_base + 2 * ictx->pe_src_row_bytes); const uint32_t row_end_max = ictx->pe_row_base + ictx->pe_nrows; - const uint32_t row_start = ictx->pe_row_base + ictx->pe_rows_per_thread * ith; - const uint32_t row_end = MIN(row_start + ictx->pe_rows_per_thread, row_end_max); + const uint32_t row_start = ictx->pe_row_base + ictx->pe_rows_per_hvx * ith; + const uint32_t row_end = MIN(row_start + ictx->pe_rows_per_hvx, row_end_max); if (row_start >= row_end) { return; } @@ -860,28 +860,28 @@ static void im2col_3d_patchembed_compute_f32( } } -static void im2col_3d_patchembed_dma_thread(unsigned int nth, unsigned int ith, void * data) { +static void im2col_3d_patchembed_dma_thread(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { (void) nth; struct htp_im2col_context * ictx = (struct htp_im2col_context *) data; struct htp_ops_context * octx = ictx->octx; - struct htp_thread_trace * restrict tr = &octx->ctx->trace[ith]; + struct htp_thread_trace * restrict tr = th->trace; const struct htp_tensor * restrict dst = octx->dst; const uint32_t KD = ictx->KD; const uint32_t OW = ictx->OW; const uint32_t patch_stride = ictx->patch_stride; const uint32_t dst_elem = ictx->dst_elem; const dma_addr_t dst_data = dst->data; - dma_queue * dma_q = octx->ctx->dma[ith]; + dma_queue * dma_q = th->dma_q; const uint32_t row_end_max = ictx->pe_row_base + ictx->pe_nrows; - const uint32_t row_start = ictx->pe_row_base + ictx->pe_rows_per_thread * ith; - const uint32_t row_end = MIN(row_start + ictx->pe_rows_per_thread, row_end_max); + const uint32_t row_start = ictx->pe_row_base + ictx->pe_rows_per_hvx * ith; + const uint32_t row_end = MIN(row_start + ictx->pe_rows_per_hvx, row_end_max); if (row_start >= row_end) { return; } const uint32_t nrows_local = row_end - row_start; - uint8_t * src_base = ictx->pe_vtcm_src + ith * ictx->pe_src_size_per_thread; - uint8_t * dstb_base = ictx->pe_vtcm_dst + ith * ictx->pe_dst_size_per_thread; + uint8_t * src_base = ictx->pe_vtcm_src + ith * ictx->pe_src_size_per_hvx; + uint8_t * dstb_base = ictx->pe_vtcm_dst + ith * ictx->pe_dst_size_per_hvx; float * srcb2[2] = { (float *) src_base, (float *) (src_base + ictx->pe_src_row_bytes) }; void * dstb2[2] = { dstb_base, dstb_base + ictx->pe_dst_row_bytes }; struct im2col_cursor cur = { @@ -948,29 +948,29 @@ int op_im2col(struct htp_ops_context * octx) { // the blocked path splits work over output-col blocks, so a single output row (1D) can use every thread const bool blocked = kparams->kernel_type == HTP_IM2COL_KERNEL_BLOCKED_DMA; const uint32_t nwork = blocked ? nrows * kparams->nbpr : nrows; - if (!htp_ops_context_set_n_threads(octx, MIN((uint32_t) kparams->n_threads, nwork))) { + if (!htp_ops_context_set_n_hvx(octx, MIN((uint32_t) kparams->n_hvx, nwork))) { return HTP_STATUS_INVAL_PARAMS; } struct htp_im2col_vtcm_layout layout; - htp_im2col_vtcm_layout_build(&layout, kparams->src_row_bytes, kparams->dst_row_bytes, kparams->n_threads, + htp_im2col_vtcm_layout_build(&layout, kparams->src_row_bytes, kparams->dst_row_bytes, kparams->n_hvx, htp_im2col_scratch_bytes(kparams->kernel_type, kparams->KH, kparams->patch_stride)); struct htp_im2col_context ictx = { 0 }; uint8_t * const base = octx->ctx->vtcm_base; - const uint32_t pth = octx->n_threads; + const uint32_t pth = octx->n_hvx; ictx.octx = octx; ictx.pe_row_base = row_base; ictx.pe_nrows = nrows; - ictx.pe_rows_per_thread = fastdiv(nrows + pth - 1, &octx->n_threads_div); - ictx.pe_blocks_per_thread = fastdiv(nwork + pth - 1, &octx->n_threads_div); + ictx.pe_rows_per_hvx = fastdiv(nrows + pth - 1, &octx->n_hvx_div); + ictx.pe_blocks_per_hvx = fastdiv(nwork + pth - 1, &octx->n_hvx_div); ictx.pe_src_row_bytes = kparams->src_row_bytes; ictx.pe_dst_row_bytes = kparams->dst_row_bytes; ictx.pe_vtcm_src = VTCM_LAYOUT_PTR(uint8_t, base, layout.off_src); ictx.pe_vtcm_dst = VTCM_LAYOUT_PTR(uint8_t, base, layout.off_dst); - ictx.pe_src_size_per_thread = layout.src_bytes_per_thread; - ictx.pe_dst_size_per_thread = layout.dst_bytes_per_thread; + ictx.pe_src_size_per_hvx = layout.src_bytes_per_hvx; + ictx.pe_dst_size_per_hvx = layout.dst_bytes_per_hvx; ictx.pe_owb = kparams->owb; ictx.pe_wb = kparams->wb; ictx.pe_nbpr = kparams->nbpr; @@ -990,10 +990,10 @@ int op_im2col(struct htp_ops_context * octx) { // a thread starts at a row, or at an output-col block (possibly mid-row) on the blocked path for (uint32_t ith = 0; ith < pth; ith++) { - uint32_t r = row_base + ictx.pe_rows_per_thread * ith; + uint32_t r = row_base + ictx.pe_rows_per_hvx * ith; uint32_t c0 = 0; if (blocked) { - const uint32_t blk = ictx.pe_blocks_per_thread * ith; + const uint32_t blk = ictx.pe_blocks_per_hvx * ith; const uint32_t q = fastdiv(blk, &kparams->div_nbpr); r = row_base + q; c0 = (blk - q * kparams->nbpr) * kparams->owb; @@ -1009,7 +1009,7 @@ int op_im2col(struct htp_ops_context * octx) { if (kparams->kernel_type == HTP_IM2COL_KERNEL_PATCHEMBED_DMA) { ictx.compute = dst->type == HTP_TYPE_F16 ? (void *) im2col_patchembed_compute_f16 : (void *) im2col_patchembed_compute_f32; - work_queue_run(octx->ctx->work_queue, im2col_patchembed_dma_thread, &ictx, pth); + work_queue_run(octx->ctx->hvx_queue, im2col_patchembed_dma_thread, &ictx, pth); return HTP_STATUS_OK; } else if (blocked) { if (kparams->KH == 1) { @@ -1019,7 +1019,7 @@ int op_im2col(struct htp_ops_context * octx) { ictx.compute = dst->type == HTP_TYPE_F16 ? (void *) im2col_blocked_compute_f16 : (void *) im2col_blocked_compute_f32; } - work_queue_run(octx->ctx->work_queue, im2col_blocked_dma_thread, &ictx, pth); + work_queue_run(octx->ctx->hvx_queue, im2col_blocked_dma_thread, &ictx, pth); return HTP_STATUS_OK; } @@ -1049,27 +1049,27 @@ int op_im2col_3d(struct htp_ops_context * octx) { return HTP_STATUS_OK; } - if (!htp_ops_context_set_n_threads(octx, MIN((uint32_t) kparams->n_threads, nrows))) { + if (!htp_ops_context_set_n_hvx(octx, MIN((uint32_t) kparams->n_hvx, nrows))) { return HTP_STATUS_INVAL_PARAMS; } struct htp_im2col_vtcm_layout layout; - htp_im2col_vtcm_layout_build(&layout, kparams->src_row_bytes, kparams->dst_row_bytes, kparams->n_threads, 256); + htp_im2col_vtcm_layout_build(&layout, kparams->src_row_bytes, kparams->dst_row_bytes, kparams->n_hvx, 256); struct htp_im2col_context ictx = { 0 }; uint8_t * const base = octx->ctx->vtcm_base; - const uint32_t pth = octx->n_threads; + const uint32_t pth = octx->n_hvx; ictx.octx = octx; ictx.pe_row_base = row_base; ictx.pe_nrows = nrows; - ictx.pe_rows_per_thread = fastdiv(nrows + pth - 1, &octx->n_threads_div); + ictx.pe_rows_per_hvx = fastdiv(nrows + pth - 1, &octx->n_hvx_div); ictx.pe_src_row_bytes = kparams->src_row_bytes; ictx.pe_dst_row_bytes = kparams->dst_row_bytes; ictx.pe_vtcm_src = VTCM_LAYOUT_PTR(uint8_t, base, layout.off_src); ictx.pe_vtcm_dst = VTCM_LAYOUT_PTR(uint8_t, base, layout.off_dst); - ictx.pe_src_size_per_thread = layout.src_bytes_per_thread; - ictx.pe_dst_size_per_thread = layout.dst_bytes_per_thread; + ictx.pe_src_size_per_hvx = layout.src_bytes_per_hvx; + ictx.pe_dst_size_per_hvx = layout.dst_bytes_per_hvx; ictx.pe_owb = kparams->owb; ictx.pe_wb = kparams->wb; ictx.pe_nbpr = kparams->nbpr; @@ -1087,7 +1087,7 @@ int op_im2col_3d(struct htp_ops_context * octx) { ictx.patch_stride = kparams->patch_stride; for (uint32_t ith = 0; ith < pth; ith++) { - const uint32_t rem = row_base + ictx.pe_rows_per_thread * ith; + const uint32_t rem = row_base + ictx.pe_rows_per_hvx * ith; const uint32_t q_oh = fastdiv(rem, &kparams->div_OH); ictx.thread_ioh[ith] = rem - q_oh * kparams->OH; const uint32_t in = fastdiv(q_oh, &kparams->div_OD); @@ -1100,12 +1100,12 @@ int op_im2col_3d(struct htp_ops_context * octx) { if (kparams->kernel_type == HTP_IM2COL_KERNEL_PATCHEMBED_DMA) { ictx.compute = dst->type == HTP_TYPE_F16 ? (void *) im2col_3d_patchembed_compute_f16 : (void *) im2col_3d_patchembed_compute_f32; - work_queue_run(octx->ctx->work_queue, im2col_3d_patchembed_dma_thread, &ictx, pth); + work_queue_run(octx->ctx->hvx_queue, im2col_3d_patchembed_dma_thread, &ictx, pth); return HTP_STATUS_OK; } else if (kparams->kernel_type == HTP_IM2COL_KERNEL_BLOCKED_DMA) { ictx.compute = dst->type == HTP_TYPE_F16 ? (void *) im2col_3d_blocked_compute_f16 : (void *) im2col_3d_blocked_compute_f32; - work_queue_run(octx->ctx->work_queue, im2col_3d_blocked_dma_thread, &ictx, pth); + work_queue_run(octx->ctx->hvx_queue, im2col_3d_blocked_dma_thread, &ictx, pth); return HTP_STATUS_OK; } diff --git a/ggml/src/ggml-hexagon/htp/im2col-ops.h b/ggml/src/ggml-hexagon/htp/im2col-ops.h index b34e6a09dbc1..23497d01d28e 100644 --- a/ggml/src/ggml-hexagon/htp/im2col-ops.h +++ b/ggml/src/ggml-hexagon/htp/im2col-ops.h @@ -16,28 +16,28 @@ struct htp_im2col_vtcm_layout { size_t total_bytes; size_t off_src; size_t off_dst; - uint32_t src_bytes_per_thread; - uint32_t dst_bytes_per_thread; + uint32_t src_bytes_per_hvx; + uint32_t dst_bytes_per_hvx; }; -// scratch_bytes: per-thread VTCM after the two src buffers (at least 256) +// scratch_bytes: per-hvx VTCM after the two src buffers (at least 256) static inline void htp_im2col_vtcm_layout_build( struct htp_im2col_vtcm_layout * L, uint32_t src_row_bytes, uint32_t dst_row_bytes, - uint32_t n_threads, + uint32_t n_hvx, uint32_t scratch_bytes) { - L->src_bytes_per_thread = 2 * src_row_bytes + (scratch_bytes > 256 ? scratch_bytes : 256); - L->dst_bytes_per_thread = 2 * dst_row_bytes; + L->src_bytes_per_hvx = 2 * src_row_bytes + (scratch_bytes > 256 ? scratch_bytes : 256); + L->dst_bytes_per_hvx = 2 * dst_row_bytes; size_t off = 0; - VTCM_LAYOUT_ALLOC(off, off_src, (size_t) L->src_bytes_per_thread * n_threads); - VTCM_LAYOUT_ALLOC(off, off_dst, (size_t) L->dst_bytes_per_thread * n_threads); + VTCM_LAYOUT_ALLOC(off, off_src, (size_t) L->src_bytes_per_hvx * n_hvx); + VTCM_LAYOUT_ALLOC(off, off_dst, (size_t) L->dst_bytes_per_hvx * n_hvx); L->total_bytes = off; } -// per-thread scratch: the KH == 1 blocked path keeps its offset table and one gathered row there, +// per-hvx scratch: the KH == 1 blocked path keeps its offset table and one gathered row there, // both padded to whole vectors static inline uint32_t htp_im2col_scratch_bytes(uint32_t kernel_type, uint32_t KH, uint32_t patch_stride) { if (kernel_type == HTP_IM2COL_KERNEL_BLOCKED_DMA && KH == 1) { @@ -48,7 +48,7 @@ static inline uint32_t htp_im2col_scratch_bytes(uint32_t kernel_type, uint32_t K struct htp_im2col_kernel_params { uint8_t kernel_type; - uint8_t n_threads; + uint8_t n_hvx; uint32_t vtcm_size; uint32_t src_row_bytes; diff --git a/ggml/src/ggml-hexagon/htp/lightning-indexer-ops.c b/ggml/src/ggml-hexagon/htp/lightning-indexer-ops.c new file mode 100644 index 000000000000..5d60c506e5e0 --- /dev/null +++ b/ggml/src/ggml-hexagon/htp/lightning-indexer-ops.c @@ -0,0 +1,1722 @@ +#pragma clang diagnostic ignored "-Wunused-variable" +#pragma clang diagnostic ignored "-Wunused-function" +#pragma clang diagnostic ignored "-Wunused-but-set-variable" + +#include +#include + +#include "dma-queue.h" +#include "htp-trace.h" +#include "hvx-utils.h" +#include "hvx-copy.h" +#include "htp-vtcm.h" +#include "work-queue.h" + +#define GGML_COMMON_DECL_C +#include "ggml-common.h" +#include "htp-ctx.h" +#include "htp-ops.h" +#include "htp-tensor.h" +#include "hvx-quant.h" +#include "hvx-fa-kernels.h" +#include "hmx-utils.h" +#include "hmx-fa-kernels.h" +#include "hmx-queue.h" + +#include "lightning-indexer-ops.h" + +struct htp_lid_context { + const struct htp_ops_context * octx; + const struct htp_lid_kernel_params * kp; + + uint32_t n_blocks; + uint32_t blocks_per_hvx; + + // current stream and token tile + uint32_t is; + uint32_t im3; + uint32_t t0; + uint32_t nt; + + struct htp_lid_vtcm_layout L; + + uint8_t * spad_q; + uint8_t * spad_w; + uint8_t * spad_qs; + uint8_t * spad_k; + uint8_t * spad_kt; + uint8_t * spad_m; + uint8_t * spad_o; +}; + +static inline void lid_prep_step_info(uint32_t k, uint32_t K, uint32_t t_first, uint32_t i0, uint32_t i1, uint32_t n_head, + uint32_t * out_t, uint32_t * out_ha, uint32_t * out_hb) { + const uint32_t t = t_first + k; + const uint32_t ha = (k == 0) ? (i0 - t * n_head) : 0; + const uint32_t hb = (k == K - 1) ? (i1 - t * n_head) : n_head; + *out_t = t; + *out_ha = ha; + *out_hb = hb; +} + +static inline void lid_prep_push_step(dma_queue * dma_q, const struct htp_lid_context * lctx, + const struct htp_tensor * q, const struct htp_tensor * w, + uint8_t * qs, uint8_t * ws, uint32_t qs_row_size, uint32_t w_row_size, + uint32_t D, uint32_t n_head, uint32_t t, uint32_t ha, uint32_t hb) { + const uint32_t it = lctx->t0 + t; + dma_queue_push(dma_q, dma_make_data(qs, q->data + it * q->nb[2] + lctx->is * q->nb[3] + ha * q->nb[1]), + qs_row_size, q->nb[1], D * sizeof(float), hb - ha); + dma_queue_push(dma_q, dma_make_data(ws, w->data + it * w->nb[1] + lctx->is * w->nb[3]), + w_row_size, w_row_size, n_head * sizeof(float), 1); +} + +// Splat every Q element (f16) and every head weight (f32) into a full vector. +// Work is split over (token, head) pairs. +static void lid_prep_thread(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { + struct htp_lid_context * lctx = (struct htp_lid_context *) data; + const struct htp_ops_context * octx = lctx->octx; + const struct htp_tensor * q = octx->src[0]; + const struct htp_tensor * w = octx->src[2]; + + const struct htp_lid_vtcm_layout * L = &lctx->L; + + const uint32_t D = q->ne[0]; + const uint32_t n_head = q->ne[1]; + + // nth is min(n_hvx, n_items): the same chunk as a split over n_hvx + const uint32_t n_items = lctx->nt * n_head; + const uint32_t chunk = fastdiv(n_items + octx->n_hvx - 1, &octx->n_hvx_div); + const uint32_t i0 = ith * chunk; + const uint32_t i1 = MIN(i0 + chunk, n_items); + + if (i0 >= i1) { + return; + } + + struct htp_thread_trace * tr = th->trace; + dma_queue * dma_q = th->dma_q; + + const uint32_t w_row_size = hex_round_up(n_head * sizeof(float), 128); + const size_t qs_size = n_head * L->size_qs_row; + const size_t slot_size = qs_size + w_row_size; + + uint8_t * qs[2]; + uint8_t * ws[2]; + qs[0] = lctx->spad_qs + ith * L->size_qs; + ws[0] = qs[0] + qs_size; + qs[1] = qs[0] + slot_size; + ws[1] = qs[1] + qs_size; + uint8_t * qh = qs[0] + 2 * slot_size; + + const uint32_t t_first = fastdiv(i0, &lctx->kp->div_n_head); + const uint32_t t_last = fastdiv(i1 - 1, &lctx->kp->div_n_head); + const uint32_t n_steps = t_last - t_first + 1; + + // Prologue: prime both slots + uint32_t t, ha, hb; + lid_prep_step_info(0, n_steps, t_first, i0, i1, n_head, &t, &ha, &hb); + lid_prep_push_step(dma_q, lctx, q, w, qs[0], ws[0], L->size_qs_row, w_row_size, D, n_head, t, ha, hb); + + if (n_steps > 1) { + lid_prep_step_info(1, n_steps, t_first, i0, i1, n_head, &t, &ha, &hb); + lid_prep_push_step(dma_q, lctx, q, w, qs[1], ws[1], L->size_qs_row, w_row_size, D, n_head, t, ha, hb); + } + + // Loop: pop(slot) -> compute(slot) -> push(slot, k + 2) + for (uint32_t k = 0; k < n_steps; ++k) { + const uint32_t slot = k & 1; + + dma_queue_pop(dma_q); + dma_queue_pop(dma_q); + + lid_prep_step_info(k, n_steps, t_first, i0, i1, n_head, &t, &ha, &hb); + const uint32_t it = lctx->t0 + t; + + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_A_PREP, (uint16_t) it); + + HVX_Vector * restrict q_sp = (HVX_Vector *) lctx->spad_q + (t * n_head + ha) * D; + for (uint32_t h = ha; h < hb; ++h) { + hvx_copy_f16_f32_aa(qh, qs[slot] + (h - ha) * L->size_qs_row, D); + + const HVX_Vector * restrict qv = (const HVX_Vector *) qh; + for (uint32_t d0 = 0; d0 < D; d0 += VLEN_FP16) { + const HVX_Vector v = qv[d0 / VLEN_FP16]; + const uint32_t nd = MIN(VLEN_FP16, D - d0); + for (uint32_t j = 0; j < nd; ++j) { + *q_sp++ = hvx_vec_repl_f16(Q6_V_vror_VR(v, j * SIZEOF_FP16)); + } + } + } + + const HVX_Vector * restrict wv = (const HVX_Vector *) ws[slot]; + HVX_Vector * restrict w_dst = (HVX_Vector *) (lctx->spad_w + t * n_head * VLEN); + for (uint32_t h = ha; h < hb; ++h) { + w_dst[h] = hvx_vec_repl4(Q6_V_vror_VR(wv[h / VLEN_FP32], (h % VLEN_FP32) * SIZEOF_FP32)); + } + + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_A_PREP, (uint16_t) it); + + if (k + 2 < n_steps) { + uint32_t next_t, next_ha, next_hb; + lid_prep_step_info(k + 2, n_steps, t_first, i0, i1, n_head, &next_t, &next_ha, &next_hb); + lid_prep_push_step(dma_q, lctx, q, w, qs[slot], ws[slot], L->size_qs_row, w_row_size, D, n_head, next_t, next_ha, next_hb); + } + } +} + +#if __HVX_ARCH__ < 79 +#define LID_MUL_F16(a, b) Q6_Vhf_equals_Wqf32(Q6_Wqf32_vmpy_VhfVhf(a, b)) +#else +#define LID_MUL_F16(a, b) Q6_Vhf_vmpy_VhfVhf(a, b) +#endif + +// Q8_0 row to f16 in place, 4 blocks (128 values) per step. +// Go backwards: the f16 output of a group is wider than its input and covers the input of the next group. +static inline void lid_dequant_row_q8_0(uint8_t * restrict row, uint32_t D) { + const int32_t nb = D / QK8_0; + + const HVX_VectorPred q32 = Q6_Q_vsetq_R(32); + const HVX_VectorPred q64 = Q6_Q_vsetq_R(64); + const HVX_VectorPred q96 = Q6_Q_vsetq_R(96); + + for (int32_t g = (nb - 1) / 4; g >= 0; --g) { + const uint8_t * src = row + g * 4 * sizeof(block_q8_0); + const HVX_Vector v0 = *(const HVX_UVector *) src; + const HVX_Vector v1 = *(const HVX_UVector *) (src + VLEN); + + HVX_Vector qs = Q6_V_vmux_QVV(q96, Q6_V_valign_VVR(v1, v0, 6), Q6_V_valign_VVR(v1, v0, 8)); + qs = Q6_V_vmux_QVV(q64, Q6_V_valign_VVR(v1, v0, 4), qs); + qs = Q6_V_vmux_QVV(q32, Q6_V_valign_VVR(v1, v0, 2), qs); + + const HVX_Vector s01 = Q6_V_vmux_QVV(q64, hvx_vec_repl_f16(v0), hvx_vec_repl_f16(Q6_V_valign_VVR(v1, v0, 34))); + const HVX_Vector s23 = Q6_V_vmux_QVV(q64, hvx_vec_repl_f16(Q6_V_valign_VVR(v1, v0, 68)), hvx_vec_repl_f16(Q6_V_valign_VVR(v1, v0, 102))); + + const HVX_VectorPair qh = Q6_Wh_vunpack_Vb(qs); + + HVX_Vector * dst = (HVX_Vector *) (row + g * 2 * VLEN); + dst[0] = Q6_Vhf_equals_Wqf32(Q6_Wqf32_vmpy_VhfVhf(Q6_Vhf_equals_Vh(Q6_V_lo_W(qh)), s01)); + if (g * 4 + 2 < nb) { + dst[1] = Q6_Vhf_equals_Wqf32(Q6_Wqf32_vmpy_VhfVhf(Q6_Vhf_equals_Vh(Q6_V_hi_W(qh)), s23)); + } + } +} + +// Q4_0 blocks i, i + 1 (v loaded at block i) to signed quants: block i in bytes 0..31, block i + 1 in bytes 32..63. +// qs[j] holds value j (low nibble) and j + 16 (high nibble). lo16 selects bytes 0..15 and 32..47. +static inline HVX_Vector lid_q4_0_pair_quants(HVX_Vector v, HVX_VectorPred q32, HVX_VectorPred lo16) { + const HVX_Vector qs = Q6_V_vmux_QVV(q32, Q6_V_vror_VR(v, sizeof(ggml_half)), + Q6_V_vror_VR(v, VLEN - 32 + sizeof(block_q4_0) + sizeof(ggml_half))); + const HVX_Vector lo = Q6_Vb_vsub_VbVb(Q6_V_vand_VV(qs, Q6_Vb_vsplat_R(0x0F)), Q6_Vb_vsplat_R(8)); + const HVX_Vector hi = Q6_Vb_vsub_VbVb(Q6_Vub_vlsr_VubR(qs, 4), Q6_Vb_vsplat_R(8)); + return Q6_V_vmux_QVV(lo16, lo, Q6_V_vlalign_VVR(hi, Q6_V_vzero(), 16)); +} + +// Q4_0 row to f16 in place, 2 blocks (64 values) per step. Backwards, as for Q8_0. +static inline void lid_dequant_row_q4_0(uint8_t * restrict row, uint32_t D) { + const int32_t nb = D / QK4_0; + + const HVX_VectorPred q32 = Q6_Q_vsetq_R(32); + const HVX_VectorPred q64 = Q6_Q_vsetq_R(64); + const HVX_VectorPred lo16 = Q6_Q_or_QQ(Q6_Q_vsetq_R(16), Q6_Q_and_QQn(Q6_Q_vsetq_R(48), q32)); + + for (int32_t p = (nb - 1) / 2; p >= 0; --p) { + const HVX_Vector v = *(const HVX_UVector *) (row + p * 2 * sizeof(block_q4_0)); + const HVX_Vector q = lid_q4_0_pair_quants(v, q32, lo16); + const HVX_Vector d1 = 2 * p + 1 < nb ? hvx_vec_repl_f16(Q6_V_vror_VR(v, sizeof(block_q4_0))) : Q6_V_vzero(); + const HVX_Vector d = Q6_V_vmux_QVV(q64, hvx_vec_repl_f16(v), d1); + + ((HVX_Vector *) row)[p] = LID_MUL_F16(Q6_Vhf_equals_Vh(Q6_V_lo_W(Q6_Wh_vunpack_Vb(q))), d); + } +} + +// Quantized kv block to f16 rows in place. Not inlined: the dequant constants would stay live across the compute loop. +static __attribute__((noinline)) void lid_dequant_block(uint8_t * restrict k_buf, size_t row_stride, uint32_t nkv, uint32_t D, bool q4) { + if (q4) { + for (uint32_t r = 0; r < nkv; ++r) { + lid_dequant_row_q4_0(k_buf + r * row_stride, D); + } + } else { + for (uint32_t r = 0; r < nkv; ++r) { + lid_dequant_row_q8_0(k_buf + r * row_stride, D); + } + } +} + +static inline void lid_convert_row_f32_to_f16(uint8_t * row, uint32_t D) { + const HVX_Vector zero = Q6_V_vzero(); + const HVX_Vector * src = (const HVX_Vector *) row; + HVX_Vector * dst = (HVX_Vector *) row; + + for (uint32_t d = 0; d < D; d += 64) { + dst[d / 64] = hvx_vec_f32_to_f16(src[d / 32], d + 32 < D ? src[d / 32 + 1] : zero); + } +} + +// Q8_0 gather transpose limits: tmp (two 64x64 scratch tiles) holds D / 2 pairs, gather offsets are 16 bit +static inline bool lid_q8_0_gather_ok(size_t row_size, uint32_t D) { + return D <= 4 * HTP_LID_KV_BLOCK && (HTP_LID_KV_BLOCK - 1) * row_size + (D / QK8_0) * sizeof(block_q8_0) <= 32767; +} + +// Transpose a raw Q8_0 kv block: one halfword gather reads q[d], q[d + 1] of all 64 rows, then split, widen to f16 and scale. +// Lanes of missing rows hold stale data; they are not stored. +static inline void lid_transpose_q8_0_gather(HVX_Vector * restrict kt, + HVX_Vector * restrict tmp, + HVX_Vector * restrict sc, + const uint8_t * restrict k_buf, + size_t row_size, + HVX_Vector offsets, + uint32_t D) { + const uint32_t mu = HTP_LID_KV_BLOCK * row_size; + for (uint32_t b = 0; b < D / QK8_0; ++b) { + Q6_vgather_ARMVh(sc + b, (size_t) (k_buf + b * sizeof(block_q8_0)), mu, offsets); + } + for (uint32_t d = 0; d < D; d += 2) { + const uint32_t off = (d / QK8_0) * sizeof(block_q8_0) + sizeof(ggml_half) + d % QK8_0; + Q6_vgather_ARMVh(tmp + d / 2, (size_t) (k_buf + off), mu, offsets); + } + hvx_gather_sync(sc); + hvx_gather_sync(tmp); + + for (uint32_t d = 0; d < D; d += 2) { + const HVX_Vector s = sc[d / QK8_0]; + const HVX_VectorPair qh = Q6_Wh_vunpack_Vb(tmp[d / 2]); + const HVX_VectorPair qd = Q6_W_vdeal_VVR(Q6_V_hi_W(qh), Q6_V_lo_W(qh), -2); + kt[d + 0] = Q6_Vhf_equals_Wqf32(Q6_Wqf32_vmpy_VhfVhf(Q6_Vhf_equals_Vh(Q6_V_lo_W(qd)), s)); + kt[d + 1] = Q6_Vhf_equals_Wqf32(Q6_Wqf32_vmpy_VhfVhf(Q6_Vhf_equals_Vh(Q6_V_hi_W(qd)), s)); + } +} + +static inline void lid_shuff_pass(HVX_Vector * restrict dst, const HVX_Vector * restrict src) { + for (uint32_t j = 0; j < 32; ++j) { + HVX_VectorPair p = Q6_W_vshuff_VVR(src[j + 32], src[j], -SIZEOF_FP16); + dst[2 * j + 0] = Q6_V_lo_W(p); + dst[2 * j + 1] = Q6_V_hi_W(p); + } +} + +// 64x64 f16 transpose: each shuffle pass rotates the 12-bit (row, lane) index by one bit, six passes swap row and lane +static inline void lid_transpose_64x64_f16(HVX_Vector * restrict dst, + const uint8_t * restrict src, + size_t src_stride, + uint32_t nkv, + HVX_Vector * restrict t0, + HVX_Vector * restrict t1) { + if (nkv >= HTP_LID_KV_BLOCK) { + for (uint32_t j = 0; j < 32; ++j) { + const HVX_Vector r0 = *(const HVX_Vector *) (src + j * src_stride); + const HVX_Vector r1 = *(const HVX_Vector *) (src + (j + 32) * src_stride); + HVX_VectorPair p = Q6_W_vshuff_VVR(r1, r0, -SIZEOF_FP16); + t0[2 * j + 0] = Q6_V_lo_W(p); + t0[2 * j + 1] = Q6_V_hi_W(p); + } + } else { + const HVX_Vector zero = Q6_V_vzero(); + for (uint32_t j = 0; j < 32; ++j) { + const HVX_Vector r0 = (j < nkv) ? *(const HVX_Vector *) (src + j * src_stride) : zero; + const HVX_Vector r1 = (j + 32 < nkv) ? *(const HVX_Vector *) (src + (j + 32) * src_stride) : zero; + HVX_VectorPair p = Q6_W_vshuff_VVR(r1, r0, -SIZEOF_FP16); + t0[2 * j + 0] = Q6_V_lo_W(p); + t0[2 * j + 1] = Q6_V_hi_W(p); + } + } + lid_shuff_pass(t1, t0); + lid_shuff_pass(t0, t1); + lid_shuff_pass(t1, t0); + lid_shuff_pass(t0, t1); + lid_shuff_pass(dst, t0); +} + +#if __HVX_ARCH__ < 79 +// qf32 accumulators, converted to sf once per head +static inline HVX_VectorPair lid_mpyacc(HVX_VectorPair acc, HVX_Vector x, HVX_Vector y) { + HVX_VectorPair m = Q6_Wqf32_vmpy_VhfVhf(x, y); + return Q6_W_vcombine_VV(Q6_Vqf32_vadd_Vqf32Vqf32(Q6_V_hi_W(acc), Q6_V_hi_W(m)), + Q6_Vqf32_vadd_Vqf32Vqf32(Q6_V_lo_W(acc), Q6_V_lo_W(m))); +} +static inline HVX_Vector lid_acc_to_sf(HVX_Vector v) { + return Q6_Vsf_equals_Vqf32(v); +} +#else +static inline HVX_VectorPair lid_mpyacc(HVX_VectorPair acc, HVX_Vector x, HVX_Vector y) { + return Q6_Wsf_vmpyacc_WsfVhfVhf(acc, x, y); +} +static inline HVX_Vector lid_acc_to_sf(HVX_Vector v) { + return v; +} +#endif + +// out += relu(s) * w, for the even and odd kv lanes +static inline HVX_VectorPair lid_head_acc(HVX_VectorPair out, HVX_VectorPair s, HVX_Vector w) { + const HVX_Vector zero = Q6_V_vzero(); + HVX_Vector e = HVX_OP_ADD_F32(Q6_V_lo_W(out), HVX_OP_MUL_F32(Q6_Vsf_vmax_VsfVsf(lid_acc_to_sf(Q6_V_lo_W(s)), zero), w)); + HVX_Vector o = HVX_OP_ADD_F32(Q6_V_hi_W(out), HVX_OP_MUL_F32(Q6_Vsf_vmax_VsfVsf(lid_acc_to_sf(Q6_V_hi_W(s)), zero), w)); + return Q6_W_vcombine_VV(o, e); +} + +// Store one output row: (even, odd) lanes back to natural order, add the mask +static inline void lid_store_row(float * restrict o, HVX_VectorPair out, const __fp16 * restrict m) { + const HVX_Vector zero = Q6_V_vzero(); + + const HVX_VectorPair out_nat = Q6_W_vshuff_VVR(Q6_V_hi_W(out), Q6_V_lo_W(out), -SIZEOF_FP32); + + HVX_Vector acc0 = Q6_V_lo_W(out_nat); + HVX_Vector acc1 = Q6_V_hi_W(out_nat); + + // qf32 math does not keep inf: add a finite mask, then put -inf back + const HVX_Vector m_f16 = *(const HVX_Vector *) m; + const HVX_VectorPred m_inf = Q6_Q_vcmp_eq_VhVh(m_f16, Q6_Vh_vsplat_R(0xFC00)); + + const HVX_VectorPair m_f32 = hvx_vec_f16_to_f32(Q6_V_vmux_QVV(m_inf, zero, m_f16)); + const HVX_VectorPair f_f32 = hvx_vec_f16_to_f32(Q6_V_vmux_QVV(m_inf, Q6_Vh_vsplat_R(0x3C00), zero)); + + const HVX_Vector ninf = Q6_V_vsplat_R(0xFF800000); + + acc0 = HVX_OP_ADD_F32(acc0, Q6_V_lo_W(m_f32)); + acc1 = HVX_OP_ADD_F32(acc1, Q6_V_hi_W(m_f32)); + acc0 = Q6_V_vmux_QVV(Q6_Q_vcmp_gt_VwVw(Q6_V_lo_W(f_f32), zero), ninf, acc0); + acc1 = Q6_V_vmux_QVV(Q6_Q_vcmp_gt_VwVw(Q6_V_hi_W(f_f32), zero), ninf, acc1); + + HVX_Vector * restrict vo = (HVX_Vector *) o; + vo[0] = acc0; + vo[1] = acc1; +} + +// One output row (token) of a kv block: sum_h relu(q_h . k) * w_h + mask +// kt holds the K block transposed (one f16 vector of 64 kv per d), q_t holds the Q splats of the token. +static inline void lid_compute_row(float * restrict o, + const HVX_Vector * restrict q_t, + const HVX_Vector * restrict w_t, + const HVX_Vector * restrict kt, + const __fp16 * restrict m, + uint32_t D, + uint32_t n_head) { + // f16 x f16 products land in (even kv, odd kv) lane order + HVX_VectorPair out = Q6_W_vzero(); + + uint32_t h = 0; + for (; h + 4 <= n_head; h += 4) { + const HVX_Vector * restrict q0 = q_t + h * D; + const HVX_Vector * restrict q1 = q0 + D; + const HVX_Vector * restrict q2 = q1 + D; + const HVX_Vector * restrict q3 = q2 + D; + + HVX_VectorPair s0 = Q6_W_vzero(); + HVX_VectorPair s1 = Q6_W_vzero(); + HVX_VectorPair s2 = Q6_W_vzero(); + HVX_VectorPair s3 = Q6_W_vzero(); + + for (uint32_t d = 0; d < D; ++d) { + const HVX_Vector k = kt[d]; + s0 = lid_mpyacc(s0, k, q0[d]); + s1 = lid_mpyacc(s1, k, q1[d]); + s2 = lid_mpyacc(s2, k, q2[d]); + s3 = lid_mpyacc(s3, k, q3[d]); + } + + out = lid_head_acc(out, s0, w_t[h + 0]); + out = lid_head_acc(out, s1, w_t[h + 1]); + out = lid_head_acc(out, s2, w_t[h + 2]); + out = lid_head_acc(out, s3, w_t[h + 3]); + } + for (; h < n_head; ++h) { + const HVX_Vector * restrict q0 = q_t + h * D; + + HVX_VectorPair s0 = Q6_W_vzero(); + for (uint32_t d = 0; d < D; ++d) { + s0 = lid_mpyacc(s0, kt[d], q0[d]); + } + out = lid_head_acc(out, s0, w_t[h]); + } + + lid_store_row(o, out, m); +} + +// Same as lid_compute_row for two kv blocks at once: each Q splat load feeds both blocks +static inline void lid_compute_row2(float * restrict o0, + float * restrict o1, + const HVX_Vector * restrict q_t, + const HVX_Vector * restrict w_t, + const HVX_Vector * restrict kt0, + const HVX_Vector * restrict kt1, + const __fp16 * restrict m0, + const __fp16 * restrict m1, + uint32_t D, + uint32_t n_head) { + HVX_VectorPair out0 = Q6_W_vzero(); + HVX_VectorPair out1 = Q6_W_vzero(); + + uint32_t h = 0; + for (; h + 4 <= n_head; h += 4) { + const HVX_Vector * restrict q0 = q_t + h * D; + const HVX_Vector * restrict q1 = q0 + D; + const HVX_Vector * restrict q2 = q1 + D; + const HVX_Vector * restrict q3 = q2 + D; + + HVX_VectorPair a0 = Q6_W_vzero(); + HVX_VectorPair a1 = Q6_W_vzero(); + HVX_VectorPair a2 = Q6_W_vzero(); + HVX_VectorPair a3 = Q6_W_vzero(); + HVX_VectorPair b0 = Q6_W_vzero(); + HVX_VectorPair b1 = Q6_W_vzero(); + HVX_VectorPair b2 = Q6_W_vzero(); + HVX_VectorPair b3 = Q6_W_vzero(); + + for (uint32_t d = 0; d < D; ++d) { + const HVX_Vector k0 = kt0[d]; + const HVX_Vector k1 = kt1[d]; + const HVX_Vector v0 = q0[d]; + const HVX_Vector v1 = q1[d]; + const HVX_Vector v2 = q2[d]; + const HVX_Vector v3 = q3[d]; + a0 = lid_mpyacc(a0, k0, v0); + b0 = lid_mpyacc(b0, k1, v0); + a1 = lid_mpyacc(a1, k0, v1); + b1 = lid_mpyacc(b1, k1, v1); + a2 = lid_mpyacc(a2, k0, v2); + b2 = lid_mpyacc(b2, k1, v2); + a3 = lid_mpyacc(a3, k0, v3); + b3 = lid_mpyacc(b3, k1, v3); + } + + out0 = lid_head_acc(out0, a0, w_t[h + 0]); + out0 = lid_head_acc(out0, a1, w_t[h + 1]); + out0 = lid_head_acc(out0, a2, w_t[h + 2]); + out0 = lid_head_acc(out0, a3, w_t[h + 3]); + out1 = lid_head_acc(out1, b0, w_t[h + 0]); + out1 = lid_head_acc(out1, b1, w_t[h + 1]); + out1 = lid_head_acc(out1, b2, w_t[h + 2]); + out1 = lid_head_acc(out1, b3, w_t[h + 3]); + } + for (; h < n_head; ++h) { + const HVX_Vector * restrict q0 = q_t + h * D; + + HVX_VectorPair a0 = Q6_W_vzero(); + HVX_VectorPair b0 = Q6_W_vzero(); + for (uint32_t d = 0; d < D; ++d) { + const HVX_Vector v0 = q0[d]; + a0 = lid_mpyacc(a0, kt0[d], v0); + b0 = lid_mpyacc(b0, kt1[d], v0); + } + out0 = lid_head_acc(out0, a0, w_t[h]); + out1 = lid_head_acc(out1, b0, w_t[h]); + } + + lid_store_row(o0, out0, m0); + lid_store_row(o1, out1, m1); +} + +// Each thread owns a range of kv blocks and streams K / mask in, output out (HTP_LID_N_BUF buffers) +static void lid_compute_thread(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { + struct htp_lid_context * lctx = (struct htp_lid_context *) data; + const struct htp_ops_context * octx = lctx->octx; + const struct htp_tensor * q = octx->src[0]; + const struct htp_tensor * k = octx->src[1]; + const struct htp_tensor * m = octx->src[3]; + const struct htp_tensor * dst = octx->dst; + + const uint32_t b0 = lctx->blocks_per_hvx * ith; + const uint32_t b1 = MIN(b0 + lctx->blocks_per_hvx, lctx->n_blocks); + + if (b0 >= b1) { + return; + } + + const struct htp_lid_vtcm_layout * L = &lctx->L; + + const uint32_t D = q->ne[0]; + const uint32_t n_head = q->ne[1]; + const uint32_t n_kv = k->ne[2]; + const uint32_t nt = lctx->nt; + + const bool is_f32 = (k->type == HTP_TYPE_F32); + const bool is_q8_0 = (k->type == HTP_TYPE_Q8_0); + const bool is_q4_0 = (k->type == HTP_TYPE_Q4_0); + const uint32_t k_row_size = lctx->kp->k_row_size; + + const uint32_t nbk2 = k->nb[2]; + const uint32_t nbm1 = m->nb[1]; + const uint32_t nb1 = dst->nb[1]; + + const dma_addr_t k_base = k->data + lctx->is * k->nb[3]; + const dma_addr_t m_base = m->data + lctx->t0 * nbm1 + lctx->im3 * m->nb[3]; + const dma_addr_t d_base = dst->data + lctx->t0 * nb1 + lctx->is * dst->nb[3]; + + uint8_t * spad_k = lctx->spad_k + ith * HTP_LID_N_BUF * L->size_k_buf; + uint8_t * spad_m = lctx->spad_m + ith * HTP_LID_N_BUF * L->size_m_buf; + uint8_t * spad_o = lctx->spad_o + ith * HTP_LID_N_BUF * L->size_o_buf; + + HVX_Vector * kt[2]; + kt[0] = (HVX_Vector *) (lctx->spad_kt + ith * L->size_kt); + kt[1] = kt[0] + hex_round_up(D, HTP_LID_KV_BLOCK); + HVX_Vector * kt_t0 = kt[1] + hex_round_up(D, HTP_LID_KV_BLOCK); + HVX_Vector * kt_t1 = kt_t0 + HTP_LID_KV_BLOCK; + + // Q8_0: transpose by gather from the raw rows (sc: one vector of block scales per 32 d) + const bool q8_g = is_q8_0 && lid_q8_0_gather_ok(L->size_k_raw_row, D); + HVX_Vector * sc = kt_t1 + HTP_LID_KV_BLOCK; + const HVX_Vector g_offs = hvx_vec_gather_offsets_h(L->size_k_raw_row); + + struct htp_thread_trace * tr = th->trace; + dma_queue * dma_q = th->dma_q; + + for (uint32_t b = b0, i = 0; b < b1 && i < HTP_LID_N_BUF; ++b, ++i) { + const uint32_t kv0 = b * HTP_LID_KV_BLOCK; + const uint32_t nkv = MIN(HTP_LID_KV_BLOCK, n_kv - kv0); + + dma_queue_push(dma_q, dma_make_data(d_base, spad_o + i * L->size_o_buf), + nb1, L->size_o_row, nkv * sizeof(float), 0); + dma_queue_push(dma_q, dma_make_data(spad_k + i * L->size_k_buf, k_base + kv0 * nbk2), + L->size_k_raw_row, nbk2, k_row_size, nkv); + dma_queue_push(dma_q, dma_make_data(spad_m + i * L->size_m_buf, m_base + kv0 * sizeof(__fp16)), + L->size_m_row, nbm1, nkv * sizeof(__fp16), nt); + } + + // Two kv blocks per step, the next two are in flight + for (uint32_t b = b0; b < b1; b += 2) { + const uint32_t nb = MIN(2, b1 - b); + + uint8_t * o_buf[2]; + uint8_t * k_buf[2]; + uint8_t * m_buf[2]; + + for (uint32_t j = 0; j < nb; ++j) { + const uint32_t nkv = MIN(HTP_LID_KV_BLOCK, n_kv - (b + j) * HTP_LID_KV_BLOCK); + + o_buf[j] = (uint8_t *) (uintptr_t) dma_queue_pop(dma_q).src; + k_buf[j] = (uint8_t *) (uintptr_t) dma_queue_pop(dma_q).dst; + m_buf[j] = (uint8_t *) (uintptr_t) dma_queue_pop(dma_q).dst; + + if ((is_q8_0 && !q8_g) || is_q4_0) { + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_W_DEQUANT, (uint16_t) (b + j)); + lid_dequant_block(k_buf[j], L->size_k_raw_row, nkv, D, is_q4_0); + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_W_DEQUANT, (uint16_t) (b + j)); + } + + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_W_PREP, (uint16_t) (b + j)); + if (is_f32) { + for (uint32_t r = 0; r < nkv; ++r) { + lid_convert_row_f32_to_f16(k_buf[j] + r * L->size_k_raw_row, D); + } + } + if (q8_g) { + lid_transpose_q8_0_gather(kt[j], kt_t0, sc, k_buf[j], L->size_k_raw_row, g_offs, D); + } else { + for (uint32_t d0 = 0; d0 < D; d0 += HTP_LID_KV_BLOCK) { + lid_transpose_64x64_f16(kt[j] + d0, k_buf[j] + d0 * sizeof(__fp16), L->size_k_raw_row, nkv, kt_t0, kt_t1); + } + } + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_W_PREP, (uint16_t) (b + j)); + } + + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) b); + for (uint32_t t = 0; t < nt; ++t) { + const HVX_Vector * q_t = (const HVX_Vector *) lctx->spad_q + t * n_head * D; + const HVX_Vector * w_t = (const HVX_Vector *) (lctx->spad_w + t * n_head * VLEN); + if (nb == 2) { + lid_compute_row2((float *) (o_buf[0] + t * L->size_o_row), (float *) (o_buf[1] + t * L->size_o_row), + q_t, w_t, kt[0], kt[1], + (const __fp16 *) (m_buf[0] + t * L->size_m_row), (const __fp16 *) (m_buf[1] + t * L->size_m_row), + D, n_head); + } else { + lid_compute_row((float *) (o_buf[0] + t * L->size_o_row), q_t, w_t, kt[0], + (const __fp16 *) (m_buf[0] + t * L->size_m_row), D, n_head); + } + } + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) b); + + for (uint32_t j = 0; j < nb; ++j) { + const uint32_t kv0 = (b + j) * HTP_LID_KV_BLOCK; + const uint32_t nkv = MIN(HTP_LID_KV_BLOCK, n_kv - kv0); + + dma_queue_push(dma_q, dma_make_data(d_base + kv0 * sizeof(float), o_buf[j]), + nb1, L->size_o_row, nkv * sizeof(float), nt); + + if (b + j + HTP_LID_N_BUF < b1) { + const uint32_t kv2 = kv0 + HTP_LID_N_BUF * HTP_LID_KV_BLOCK; + const uint32_t nkv2 = MIN(HTP_LID_KV_BLOCK, n_kv - kv2); + + dma_queue_push(dma_q, dma_make_data(k_buf[j], k_base + kv2 * nbk2), + L->size_k_raw_row, nbk2, k_row_size, nkv2); + dma_queue_push(dma_q, dma_make_data(m_buf[j], m_base + kv2 * sizeof(__fp16)), + L->size_m_row, nbm1, nkv2 * sizeof(__fp16), nt); + } + } + } + + for (uint32_t i = 0; i < MIN(b1 - b0, HTP_LID_N_BUF); ++i) { + dma_queue_pop(dma_q); + } + + (void) nth; +} + +// ----- HMX path: K is the activation, Q the weights, head sum by a second HMX pass ----- +// +// Channel c = t * n_head + h of a token tile. Per kv row tile (32 kv) and token tile u (32 tokens): +// mm1: S[kv, c] = relu(k . q_c) * |w_c| (convert scale |w_c|, pos clamp) +// mm2: O[kv, t] = sum_c S[kv, c] * G[c, t], G[c, t] = sign(w_c) for c of token t +// O tiles hold f16 (kv pair, token) words; gather one token column, add the mask, store f32. +// +// A run preps the Q sets of up to n_qsets (stream, token tile) pairs, then pipelines all their kv rows. +// Row x of a run is kv x % kv_seg of set x / kv_seg (kv_seg = n_kv padded to 64), so a block of Bc rows can +// span sets: it is split in segments, one per set. HMX computes block g while HVX preps K of g + 1 and extracts O of g - 1. + +struct lid_hmx_set { + uint32_t is; + uint32_t im3; + uint32_t t0; + uint32_t nt; + uint32_t n_ch; + uint32_t n_u; +}; + +struct htp_lid_hmx_context { + const struct htp_ops_context * octx; + const struct htp_lid_kernel_params * kp; + struct htp_lid_hmx_vtcm_layout L; + + uint32_t Bc; + uint32_t kv_seg; // rows per set: n_kv padded to 64 + uint32_t n_blocks; // per run + uint32_t n_tile; // tokens per token tile + uint32_t n_tt; // token tiles per stream + + uint32_t n_qsets; // Q sets per buffer + uint32_t qbuf; // Q buffer of the run (two buffers: the next run's Q is prepped during this run) + + uint32_t tile0; // first (stream, token tile) of the run + uint32_t n_run; // sets in the run + uint32_t n_qthr; // threads with Q work in the run + uint32_t nt_run; // tokens of each set in the run + uint32_t n_u_run; // token tiles of 32 of each set in the run + int32_t g; // current stage + + struct lid_hmx_set sets[HTP_LID_HMX_MAX_QSETS]; + + // next run, Q prepped in the stages of this one + bool nx_on; + uint32_t nx_n_run; + struct fastdiv_values nx_stg_div; // stages of this run: n_blocks + 1 + struct lid_hmx_set nx_sets[HTP_LID_HMX_MAX_QSETS]; + + __fp16 * qw; + __fp16 * gw; + uint8_t * sc; + uint8_t * sc_id; + __fp16 * s; + float * w; + uint8_t * qs; + uint8_t * k_raw[2]; + __fp16 * k_t[2]; + __fp16 * o[2]; + uint8_t * m[4]; + uint8_t * out[2]; +}; + +#define LID_HMX_MAX_SEGS HTP_LID_HMX_MAX_QSETS + +// Rows of one set in a block: valid rows [x0, x0 + nv) of the block are kv [kv0, kv0 + nv) of the set +struct lid_hmx_seg { + uint32_t set; + uint32_t x0; + uint32_t kv0; + uint32_t nv; +}; + +static inline struct lid_hmx_set lid_hmx_set_init(const struct htp_lid_hmx_context * hctx, uint32_t set) { + const struct htp_ops_context * octx = hctx->octx; + const uint32_t ti = hctx->tile0 + set; + + struct lid_hmx_set S; + S.is = fastdiv(ti, &hctx->kp->div_n_tt); + S.im3 = fastmodulo(S.is, octx->src[3]->ne[3], &hctx->kp->div_m_ne3); + S.t0 = (ti - S.is * hctx->n_tt) * hctx->n_tile; + S.nt = MIN(hctx->n_tile, octx->src[0]->ne[2] - S.t0); + S.n_ch = (S.nt * octx->src[0]->ne[1] + 31) / 32; + S.n_u = (S.nt + 31) / 32; + return S; +} + +// Segments of block g with valid rows. Returns their count, n_rows gets the block rows (multiple of 64). +static inline uint32_t lid_hmx_segs(const struct htp_lid_hmx_context * hctx, uint32_t g, struct lid_hmx_seg * seg, uint32_t * n_rows) { + const uint32_t n_kv = hctx->octx->src[1]->ne[2]; + const uint32_t ks = hctx->kv_seg; + const uint32_t xa = g * hctx->Bc; + const uint32_t xb = MIN(xa + hctx->Bc, hctx->n_run * ks); + + uint32_t n = 0; + for (uint32_t x = xa; x < xb;) { + const uint32_t set = fastdiv(x, &hctx->kp->div_kv_seg); + const uint32_t kv = x - set * ks; + const uint32_t end = MIN(xb, (set + 1) * ks); + if (kv < n_kv) { + seg[n].set = set; + seg[n].x0 = x - xa; + seg[n].kv0 = kv; + seg[n].nv = MIN(n_kv, kv + (end - x)) - kv; + n++; + } + x = end; + } + *n_rows = xb - xa; + return n; +} + +typedef struct { + const __fp16 * qw; + const __fp16 * gw; + const uint8_t * sc; + uint32_t rt0; + uint32_t n_rt; +} lid_hmx_job_seg_t; + +typedef struct { + lid_hmx_job_seg_t seg[LID_HMX_MAX_SEGS]; + uint32_t n_seg; + + const __fp16 * k_t; + const uint8_t * sc_id; + __fp16 * s; + __fp16 * o; + uint32_t n_ch; + uint32_t n_u; + uint32_t n_dot; + uint32_t n_head; + uint32_t n_sg; +} lid_hmx_job_t; + +// acc += sum of n tile products, a and b advance one tile per step +static inline void lid_hmx_dot(const __fp16 * a, const __fp16 * b, uint32_t n) { + for (; n > 32; n -= 32) { + asm volatile(HMX_LOAD_MPY_DEEP_F16("%1", "%2", "%0") :: "r"(65535), "r"(a), "r"(b)); + a += 32 * HMX_FP16_TILE_N_ELMS; + b += 32 * HMX_FP16_TILE_N_ELMS; + } + asm volatile(HMX_LOAD_MPY_DEEP_F16("%1", "%2", "%0") :: "r"(n * HMX_FP16_TILE_SIZE - 1), "r"(a), "r"(b)); +} + +// Per segment: channel tile outer, row tile inner, one bias load per channel tile. S holds [n_rt][n_sg] tiles. +static void lid_hmx_worker(const htp_thread_t * th, void * data) { + const lid_hmx_job_t * job = (const lid_hmx_job_t *) data; + + const size_t dot_stride = (size_t) job->n_dot * HMX_FP16_TILE_N_ELMS; + const size_t s_stride = (size_t) job->n_sg * HMX_FP16_TILE_N_ELMS; + const uint32_t nh = job->n_head; + + for (uint32_t i = 0; i < job->n_seg; ++i) { + const lid_hmx_job_seg_t * sg = &job->seg[i]; + const uint32_t n_rt = sg->n_rt; + const __fp16 * kt = job->k_t + sg->rt0 * dot_stride; + __fp16 * ot = job->o + sg->rt0 * job->n_u * HMX_FP16_TILE_N_ELMS; + + for (uint32_t u = 0; u < job->n_u; ++u) { + const uint32_t j0 = u * nh; + const uint32_t j1 = MIN(j0 + nh, job->n_ch); + + for (uint32_t c = j0; c < j1; ++c) { + const __fp16 * qw = sg->qw + c * dot_stride; + __fp16 * st = job->s + (c - j0) * HMX_FP16_TILE_N_ELMS; + + asm volatile(HMX_SET_BIAS("%0") :: "r"(sg->sc + c * 256)); + for (uint32_t rt = 0; rt < n_rt; ++rt) { + asm volatile(HMX_CLRACC_F16()); + lid_hmx_dot(kt + rt * dot_stride, qw, job->n_dot); + asm volatile("mxmem(%0, %1):after:pos.hf = acc\n" :: "r"(st + rt * s_stride), "r"(0) : "memory"); + } + } + + asm volatile(HMX_SET_BIAS("%0") :: "r"(job->sc_id)); + for (uint32_t rt = 0; rt < n_rt; ++rt) { + asm volatile(HMX_CLRACC_F16()); + lid_hmx_dot(job->s + rt * s_stride, sg->gw + j0 * HMX_FP16_TILE_N_ELMS, j1 - j0); + asm volatile(HMX_STORE_AFTER_F16("%0", "%1") :: "r"(ot + (rt * job->n_u + u) * HMX_FP16_TILE_N_ELMS), "r"(0) : "memory"); + } + } + } +} + +// Thread slices: K rows by row pairs of the padded block, mask / output rows by token. +// Block 0 is prepped together with Q: its K rows go to the threads without Q work. +// K rows are split only in stages on all n_hvx threads. +static inline void lid_hmx_rows(const struct htp_lid_hmx_context * hctx, uint32_t ith, uint32_t nth, uint32_t g, uint32_t n_rows, + uint32_t * r0, uint32_t * r1) { + uint32_t per; + if (g == 0 && hctx->n_qthr < nth) { + if (ith < hctx->n_qthr) { + *r0 = *r1 = 0; + return; + } + ith -= hctx->n_qthr; + per = (n_rows + nth - hctx->n_qthr - 1) / (nth - hctx->n_qthr); + } else { + per = fastdiv(n_rows + hctx->octx->n_hvx - 1, &hctx->octx->n_hvx_div); + } + per = hex_round_up(per, 2); + *r0 = MIN(ith * per, n_rows); + *r1 = MIN(*r0 + per, n_rows); +} + +// Split over n_hvx; a stage on one thread has one token, which this split also gives to thread 0 +static inline void lid_hmx_toks(const struct htp_lid_hmx_context * hctx, uint32_t ith, uint32_t nt, uint32_t * t0, uint32_t * t1) { + const uint32_t per = fastdiv(nt + hctx->octx->n_hvx - 1, &hctx->octx->n_hvx_div); + *t0 = MIN(ith * per, nt); + *t1 = MIN((ith + 1) * per, nt); +} + +// Push (or only count, push == false) the K row and mask DMAs of block g for this thread, one per segment +static uint32_t lid_hmx_in(const struct htp_lid_hmx_context * hctx, const htp_thread_t * th, uint32_t ith, uint32_t nth, uint32_t g, bool push) { + const struct htp_ops_context * octx = hctx->octx; + const struct htp_tensor * k = octx->src[1]; + const struct htp_tensor * m = octx->src[3]; + const struct htp_lid_hmx_vtcm_layout * L = &hctx->L; + + struct lid_hmx_seg seg[LID_HMX_MAX_SEGS]; + uint32_t n_rows; + const uint32_t n_seg = lid_hmx_segs(hctx, g, seg, &n_rows); + + uint32_t r0, r1, ta, tb; + lid_hmx_rows(hctx, ith, nth, g, n_rows, &r0, &r1); + lid_hmx_toks(hctx, ith, hctx->nt_run, &ta, &tb); + + dma_queue * dma_q = th->dma_q; + const uint32_t k_row_size = hctx->kp->k_row_size; + + uint32_t n = 0; + for (uint32_t i = 0; i < n_seg; ++i) { + const struct lid_hmx_set S = hctx->sets[seg[i].set]; + + const uint32_t ra = MAX(r0, seg[i].x0); + const uint32_t rb = MIN(r1, seg[i].x0 + seg[i].nv); + if (ra < rb) { + if (push) { + dma_queue_push(dma_q, dma_make_data(hctx->k_raw[g % 2] + ra * L->size_k_raw_row, + k->data + S.is * k->nb[3] + (seg[i].kv0 + ra - seg[i].x0) * k->nb[2]), + L->size_k_raw_row, k->nb[2], k_row_size, rb - ra); + } + n++; + } + if (ta < tb) { + if (push) { + dma_queue_push(dma_q, dma_make_data(hctx->m[g % 4] + ta * L->size_m_row + seg[i].x0 * sizeof(__fp16), + m->data + S.im3 * m->nb[3] + (S.t0 + ta) * m->nb[1] + seg[i].kv0 * sizeof(__fp16)), + L->size_m_row, m->nb[1], seg[i].nv * sizeof(__fp16), tb - ta); + } + n++; + } + } + return n; +} + +static inline uint32_t lid_hmx_push_in(const struct htp_lid_hmx_context * hctx, const htp_thread_t * th, uint32_t ith, uint32_t nth, uint32_t g) { + return lid_hmx_in(hctx, th, ith, nth, g, true); +} + +// Row pair (a, b) quants of two blocks (64 values each, bytes 0..63) to the two tile vectors, scaled by s0, s1 +static inline void lid_q8_0_pair_store(HVX_Vector * restrict dst, HVX_Vector a, HVX_Vector b, HVX_Vector s0, HVX_Vector s1, bool two) { + const HVX_VectorPair qh = Q6_Wh_vunpack_Vb(Q6_V_lo_W(Q6_W_vshuff_VVR(b, a, -1))); + dst[0] = LID_MUL_F16(Q6_Vhf_equals_Vh(Q6_V_lo_W(qh)), s0); + if (two) { + dst[16] = LID_MUL_F16(Q6_Vhf_equals_Vh(Q6_V_hi_W(qh)), s1); + } +} + +// Raw Q8_0 rows [r0, r1) to activation tiles, dequant fused with the row pair interleave. Rows >= nv are zero. +// Per row pair and two blocks: interleave the quants of both rows as bytes, widen, then scale by the (d_r, d_r+1) word. +static inline void lid_hmx_prep_k_q8_0(__fp16 * restrict k_t, const uint8_t * restrict k_raw, size_t row_stride, + uint32_t r0, uint32_t r1, uint32_t nv, uint32_t n_dot) { + const HVX_Vector zero = Q6_V_vzero(); + const HVX_VectorPred q32 = Q6_Q_vsetq_R(32); + + for (uint32_t r = r0; r < r1; r += 2) { + const uint8_t * p0 = k_raw + r * row_stride; + const uint8_t * p1 = p0 + row_stride; + const bool v0 = r < nv; + const bool v1 = r + 1 < nv; + + HVX_Vector * dst = (HVX_Vector *) (k_t + (r / 32) * n_dot * HMX_FP16_TILE_N_ELMS) + (r % 32) / 2; + + if (n_dot == 4) { + // D = 128: the row is 136 bytes, two aligned loads; scales at halfwords 0, 17, 34, 51 + const HVX_Vector a0 = v0 ? ((const HVX_Vector *) p0)[0] : zero; + const HVX_Vector a1 = v0 ? ((const HVX_Vector *) p0)[1] : zero; + const HVX_Vector b0 = v1 ? ((const HVX_Vector *) p1)[0] : zero; + const HVX_Vector b1 = v1 ? ((const HVX_Vector *) p1)[1] : zero; + + const HVX_VectorPair ds = Q6_W_vshuff_VVR(b0, a0, -SIZEOF_FP16); + + lid_q8_0_pair_store(dst, + Q6_V_vmux_QVV(q32, Q6_V_valign_VVR(a0, a0, 2), Q6_V_valign_VVR(a0, a0, 4)), + Q6_V_vmux_QVV(q32, Q6_V_valign_VVR(b0, b0, 2), Q6_V_valign_VVR(b0, b0, 4)), + hvx_vec_repl4(Q6_V_lo_W(ds)), hvx_vec_repl4(Q6_V_vror_VR(Q6_V_lo_W(ds), 17 * 4)), true); + lid_q8_0_pair_store(dst + 32, + Q6_V_vmux_QVV(q32, Q6_V_valign_VVR(a1, a0, 70), Q6_V_valign_VVR(a1, a0, 72)), + Q6_V_vmux_QVV(q32, Q6_V_valign_VVR(b1, b0, 70), Q6_V_valign_VVR(b1, b0, 72)), + hvx_vec_repl4(Q6_V_vror_VR(Q6_V_hi_W(ds), 2 * 4)), hvx_vec_repl4(Q6_V_vror_VR(Q6_V_hi_W(ds), 19 * 4)), true); + continue; + } + + for (uint32_t i = 0; i < n_dot; i += 2) { + const uint8_t * pa = p0 + i * sizeof(block_q8_0); + const uint8_t * pb = p1 + i * sizeof(block_q8_0); + + // quants of blocks i, i + 1 in bytes 0..63, scales d_i at byte 0 and d_i+1 at byte 34 + const HVX_Vector a = v0 ? Q6_V_vmux_QVV(q32, *(const HVX_UVector *) (pa + 2), *(const HVX_UVector *) (pa + 4)) : zero; + const HVX_Vector b = v1 ? Q6_V_vmux_QVV(q32, *(const HVX_UVector *) (pb + 2), *(const HVX_UVector *) (pb + 4)) : zero; + const HVX_Vector da = v0 ? *(const HVX_UVector *) pa : zero; + const HVX_Vector db = v1 ? *(const HVX_UVector *) pb : zero; + const HVX_Vector ds = Q6_V_lo_W(Q6_W_vshuff_VVR(db, da, -SIZEOF_FP16)); + + lid_q8_0_pair_store(dst + i * 16, a, b, hvx_vec_repl4(ds), hvx_vec_repl4(Q6_V_vror_VR(ds, 2 * sizeof(block_q8_0))), i + 1 < n_dot); + } + } +} + +// Raw Q4_0 rows [r0, r1) to activation tiles, same as Q8_0: per row pair and two blocks, one load per row. +// Rows >= nv are zero. +static inline void lid_hmx_prep_k_q4_0(__fp16 * restrict k_t, const uint8_t * restrict k_raw, size_t row_stride, + uint32_t r0, uint32_t r1, uint32_t nv, uint32_t n_dot) { + const HVX_Vector zero = Q6_V_vzero(); + const HVX_VectorPred q32 = Q6_Q_vsetq_R(32); + const HVX_VectorPred lo16 = Q6_Q_or_QQ(Q6_Q_vsetq_R(16), Q6_Q_and_QQn(Q6_Q_vsetq_R(48), q32)); + + for (uint32_t r = r0; r < r1; r += 2) { + const uint8_t * p0 = k_raw + r * row_stride; + const uint8_t * p1 = p0 + row_stride; + const bool v0 = r < nv; + const bool v1 = r + 1 < nv; + + HVX_Vector * dst = (HVX_Vector *) (k_t + (r / 32) * n_dot * HMX_FP16_TILE_N_ELMS) + (r % 32) / 2; + + for (uint32_t i = 0; i < n_dot; i += 2) { + // blocks i, i + 1: scales d_i at byte 0 and d_i+1 at byte 18 + const HVX_Vector da = v0 ? *(const HVX_UVector *) (p0 + i * sizeof(block_q4_0)) : zero; + const HVX_Vector db = v1 ? *(const HVX_UVector *) (p1 + i * sizeof(block_q4_0)) : zero; + const HVX_Vector ds = Q6_V_lo_W(Q6_W_vshuff_VVR(db, da, -SIZEOF_FP16)); + + lid_q8_0_pair_store(dst + i * 16, lid_q4_0_pair_quants(da, q32, lo16), lid_q4_0_pair_quants(db, q32, lo16), + hvx_vec_repl4(ds), hvx_vec_repl4(Q6_V_vror_VR(ds, 2 * sizeof(block_q4_0))), i + 1 < n_dot); + } + } +} + +// F16 rows [r0, r1) to activation tiles: row pairs interleaved. Rows >= nv are zero. +static inline void lid_hmx_prep_k_f16(__fp16 * restrict k_t, const uint8_t * restrict k_raw, size_t row_stride, + uint32_t r0, uint32_t r1, uint32_t nv, uint32_t n_dot) { + const HVX_Vector zero = Q6_V_vzero(); + + for (uint32_t r = r0; r < r1; r += 2) { + const HVX_Vector * p0 = (const HVX_Vector *) (k_raw + r * row_stride); + const HVX_Vector * p1 = (const HVX_Vector *) (k_raw + (r + 1) * row_stride); + const bool v0 = r < nv; + const bool v1 = r + 1 < nv; + + HVX_Vector * dst = (HVX_Vector *) (k_t + (r / 32) * n_dot * HMX_FP16_TILE_N_ELMS) + (r % 32) / 2; + + for (uint32_t i = 0; i < n_dot; i += 2) { + const HVX_VectorPair p = Q6_W_vshuff_VVR(v1 ? p1[i / 2] : zero, v0 ? p0[i / 2] : zero, -SIZEOF_FP16); + dst[i * 16] = Q6_V_lo_W(p); + if (i + 1 < n_dot) { + dst[(i + 1) * 16] = Q6_V_hi_W(p); + } + } + } +} + +// K rows of block g (this thread's rows) to activation tiles, one piece per set; set padding rows are zero. +// Not inlined: keeps the stage's registers out of the prep loops. +static __attribute__((noinline)) void lid_hmx_prep_k(const struct htp_lid_hmx_context * hctx, uint32_t ith, uint32_t nth, uint32_t g) { + const struct htp_lid_hmx_vtcm_layout * L = &hctx->L; + const struct htp_tensor * k = hctx->octx->src[1]; + + const uint32_t n_kv = k->ne[2]; + const uint32_t ks = hctx->kv_seg; + const uint32_t xa = g * hctx->Bc; + const uint32_t n_rows = MIN(hctx->Bc, hctx->n_run * ks - xa); + + uint32_t r0, r1; + lid_hmx_rows(hctx, ith, nth, g, n_rows, &r0, &r1); + + uint8_t * k_raw = hctx->k_raw[g % 2]; + __fp16 * k_t = hctx->k_t[g % 2]; + const bool f32 = k->type == HTP_TYPE_F32; + const bool q8 = k->type == HTP_TYPE_Q8_0; + const bool q4 = k->type == HTP_TYPE_Q4_0; + + for (uint32_t r = r0; r < r1;) { + const uint32_t x = xa + r; + const uint32_t set = fastdiv(x, &hctx->kp->div_kv_seg); + const uint32_t kv = x - set * ks; + const uint32_t end = MIN(r1, (set + 1) * ks - xa); + const uint32_t nv = r + (kv < n_kv ? n_kv - kv : 0); + + if (q8) { + lid_hmx_prep_k_q8_0(k_t, k_raw, L->size_k_raw_row, r, end, nv, L->n_dot); + } else if (q4) { + lid_hmx_prep_k_q4_0(k_t, k_raw, L->size_k_raw_row, r, end, nv, L->n_dot); + } else { + if (f32) { + for (uint32_t i = r; i < end; ++i) { + lid_convert_row_f32_to_f16(k_raw + i * L->size_k_raw_row, k->ne[0]); + } + } + lid_hmx_prep_k_f16(k_t, k_raw, L->size_k_raw_row, r, end, nv, L->n_dot); + } + r = end; + } +} + +// O column of each token (64 kv per gather) + mask -> f32 rows, then one DMA out per segment +static __attribute__((noinline)) void lid_hmx_extract(const struct htp_lid_hmx_context * hctx, const htp_thread_t * th, uint32_t ith, uint32_t nth, uint32_t g) { + const struct htp_ops_context * octx = hctx->octx; + const struct htp_tensor * dst = octx->dst; + const struct htp_lid_hmx_vtcm_layout * L = &hctx->L; + + uint32_t ta, tb; + lid_hmx_toks(hctx, ith, hctx->nt_run, &ta, &tb); + if (ta >= tb) { + return; + } + + struct lid_hmx_seg seg[LID_HMX_MAX_SEGS]; + uint32_t n_rows; + const uint32_t n_seg = lid_hmx_segs(hctx, g, seg, &n_rows); + + struct htp_thread_trace * tr = th->trace; + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_O_PROC, (uint16_t) g); + + const uint32_t n_u = hctx->n_u_run; + const uint32_t mu = (n_u + 1) * HMX_FP16_TILE_SIZE - 1; + + int32_t offs[32] __attribute__((aligned(128))); + for (int l = 0; l < 32; l++) { + offs[l] = (l / 16) * n_u * HMX_FP16_TILE_SIZE + (l % 16) * 128; + } + const HVX_Vector v_offs = *(const HVX_Vector *) offs; + + HVX_Vector * gbuf = (HVX_Vector *) (hctx->qs + ith * L->size_qs + 2 * 32 * L->size_qs_row + 32 * L->size_qh_row); + + const HVX_Vector zero = Q6_V_vzero(); + const HVX_Vector ninf = Q6_V_vsplat_R(0xFF800000); + const HVX_Vector one = Q6_Vh_vsplat_R(0x3C00); + const HVX_Vector hinf = Q6_Vh_vsplat_R(0xFC00); + + const __fp16 * o_base = hctx->o[g % 2]; + const uint8_t * m_buf = hctx->m[g % 4]; + uint8_t * o_buf = hctx->out[g % 2]; + + for (uint32_t i = 0; i < n_seg; ++i) { + const uint32_t q0 = seg[i].x0 / 64; + const uint32_t q1 = (seg[i].x0 + seg[i].nv + 63) / 64; + + for (uint32_t t = ta; t < tb; ++t) { + const uint32_t u = t / 32; + const HVX_Vector off = Q6_Vw_vadd_VwVw(v_offs, Q6_V_vsplat_R((t % 32) * 4)); + + for (uint32_t q = q0; q < q1; ++q) { + const __fp16 * base = o_base + (2 * q * n_u + u) * HMX_FP16_TILE_N_ELMS; + Q6_vgather_ARMVw(gbuf + q - q0, (size_t) base, mu, off); + } + hvx_gather_sync(gbuf + q1 - q0 - 1); + + const HVX_Vector * mv = (const HVX_Vector *) (m_buf + t * L->size_m_row); + HVX_Vector * ov = (HVX_Vector *) (o_buf + t * L->size_o_row); + + for (uint32_t q = q0; q < q1; ++q) { + const HVX_VectorPair s = hvx_vec_f16_to_f32(gbuf[q - q0]); + + // qf32 math does not keep inf: add a finite mask, then put -inf back + const HVX_Vector m_f16 = mv[q]; + const HVX_VectorPred m_inf = Q6_Q_vcmp_eq_VhVh(m_f16, hinf); + const HVX_VectorPair m_f32 = hvx_vec_f16_to_f32(Q6_V_vmux_QVV(m_inf, zero, m_f16)); + const HVX_VectorPair f_f32 = hvx_vec_f16_to_f32(Q6_V_vmux_QVV(m_inf, one, zero)); + + const HVX_Vector acc0 = HVX_OP_ADD_F32(Q6_V_lo_W(s), Q6_V_lo_W(m_f32)); + const HVX_Vector acc1 = HVX_OP_ADD_F32(Q6_V_hi_W(s), Q6_V_hi_W(m_f32)); + ov[2 * q + 0] = Q6_V_vmux_QVV(Q6_Q_vcmp_gt_VwVw(Q6_V_lo_W(f_f32), zero), ninf, acc0); + ov[2 * q + 1] = Q6_V_vmux_QVV(Q6_Q_vcmp_gt_VwVw(Q6_V_hi_W(f_f32), zero), ninf, acc1); + } + } + + const struct lid_hmx_set S = hctx->sets[seg[i].set]; + dma_queue_push(th->dma_q, + dma_make_data(dst->data + S.is * dst->nb[3] + (S.t0 + ta) * dst->nb[1] + seg[i].kv0 * sizeof(float), + o_buf + ta * L->size_o_row + seg[i].x0 * sizeof(float)), + dst->nb[1], L->size_o_row, seg[i].nv * sizeof(float), tb - ta); + } + + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_O_PROC, (uint16_t) g); +} + +// Number of output DMAs of block g for this thread +static inline uint32_t lid_hmx_n_out(const struct htp_lid_hmx_context * hctx, uint32_t ith, uint32_t nth, uint32_t g) { + uint32_t ta, tb; + lid_hmx_toks(hctx, ith, hctx->nt_run, &ta, &tb); + if (ta >= tb) { + return 0; + } + struct lid_hmx_seg seg[LID_HMX_MAX_SEGS]; + uint32_t n_rows; + return lid_hmx_segs(hctx, g, seg, &n_rows); +} + +static __attribute__((noinline)) void lid_hmx_q_chunk(const struct htp_lid_hmx_context * hctx, const htp_thread_t * th, uint32_t ith, uint32_t nth); + +// Pipeline stage g (HMX runs block g meanwhile): prep block g + 1, extract block g - 1 +static void lid_hmx_stage(const struct htp_lid_hmx_context * hctx, const htp_thread_t * th, uint32_t ith, uint32_t nth) { + const int32_t g = hctx->g; + const int32_t n = (int32_t) hctx->n_blocks; + + struct htp_thread_trace * tr = th->trace; + dma_queue * dma_q = th->dma_q; + + // FIFO order per thread: OUT(g - 3), IN(g + 1), OUT(g - 2). When this stage preps next run Q, also pop + // OUT(g - 2) so the queue is empty for the Q DMAs; else leave it in flight for one more stage. + const int32_t lag = hctx->nx_on ? 2 : 3; + + uint32_t n_pop = 0; + if (lag == 3 && g - 3 >= 0) { + n_pop += lid_hmx_n_out(hctx, ith, nth, g - 3); + } + if (g >= 0 && g + 1 < n) { + n_pop += lid_hmx_in(hctx, th, ith, nth, g + 1, false); + } + if (lag == 2 && g - 2 >= 0) { + n_pop += lid_hmx_n_out(hctx, ith, nth, g - 2); + } + for (uint32_t i = 0; i < n_pop; ++i) { + dma_queue_pop(dma_q); + } + + if (hctx->nx_on && g >= 0) { + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_A_PREP, (uint16_t) g); + lid_hmx_q_chunk(hctx, th, ith, nth); + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_A_PREP, (uint16_t) g); + } + if (g + 2 < n) { + lid_hmx_push_in(hctx, th, ith, nth, g + 2); + } + if (g + 1 < n) { + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_W_PREP, (uint16_t) (g + 1)); + lid_hmx_prep_k(hctx, ith, nth, g + 1); + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_W_PREP, (uint16_t) (g + 1)); + } + if (g - 1 >= 0) { + lid_hmx_extract(hctx, th, ith, nth, g - 1); + } + if (g == n) { + uint32_t n_tail = lid_hmx_n_out(hctx, ith, nth, n - 1); + if (lag == 3 && n >= 2) { + n_tail += lid_hmx_n_out(hctx, ith, nth, n - 2); + } + for (uint32_t i = 0; i < n_tail; ++i) { + dma_queue_pop(dma_q); + } + } +} + +static void lid_hmx_stage_thread(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { + lid_hmx_stage((const struct htp_lid_hmx_context *) data, th, ith, nth); +} + +// Push the Q rows of channel tile c of set S, one DMA per token. Returns the number of DMA descriptors. +static uint32_t lid_hmx_push_q(const struct htp_lid_hmx_context * hctx, dma_queue * dma_q, uint8_t * qs, + const struct lid_hmx_set * S, uint32_t c) { + const struct htp_tensor * q = hctx->octx->src[0]; + const uint32_t nh = q->ne[1]; + const uint32_t ch = c * 32; + const uint32_t nr = MIN(32, S->nt * nh - ch); + + // channel ch + r is (token t, head h); a DMA ends at the token end, so the next one starts at h 0 + uint32_t t = fastdiv(ch, &hctx->kp->div_n_head); + uint32_t h = ch - t * nh; + uint32_t n = 0; + for (uint32_t r = 0; r < nr; ++t, h = 0) { + const uint32_t cnt = MIN(nh - h, nr - r); + dma_queue_push(dma_q, dma_make_data(qs + r * hctx->L.size_qs_row, + q->data + S->is * q->nb[3] + (S->t0 + t) * q->nb[2] + h * q->nb[1]), + hctx->L.size_qs_row, q->nb[1], q->ne[0] * sizeof(float), cnt); + r += cnt; + n++; + } + return n; +} + +// Scales (|w| per channel, f16 in the low half of each word) and G tile (sign(w) at row c, column token(c) % 32) +// of one channel tile. w points at the tile's 32 channels, the first one is (token t, head h). +static inline void lid_hmx_build_sc_g(uint8_t * restrict sc, __fp16 * restrict gt, const float * restrict w, + uint32_t t, uint32_t h, uint32_t nr, uint32_t nh) { + const HVX_Vector zero = Q6_V_vzero(); + const HVX_Vector wv = Q6_V_vmux_QVV(Q6_Q_vsetq2_R(nr * sizeof(float)), *(const HVX_Vector *) w, zero); + + HVX_Vector * scv = (HVX_Vector *) sc; + scv[0] = hvx_vec_f32_to_f16_shuff(Q6_V_vand_VV(wv, Q6_V_vsplat_R(0x7FFFFFFF)), zero); + scv[1] = zero; + + // sign(w) as f16 bits in the low half of each word, then packed: row r in halfword r + const HVX_VectorPred nz = Q6_Q_vcmp_gt_VuwVuw(Q6_V_vand_VV(wv, Q6_V_vsplat_R(0x7FFFFFFF)), zero); + const HVX_Vector sgn = Q6_V_vor_VV(Q6_V_vsplat_R(0x3C00), Q6_Vw_vasr_VwR(Q6_V_vand_VV(wv, Q6_V_vsplat_R(0x80000000)), 16)); + const HVX_Vector sv = Q6_Vh_vpacke_VwVw(zero, Q6_V_vmux_QVV(nz, sgn, zero)); + + // token column of each row; -1 for missing rows (only rows 0..31 are used) + int16_t tcol[64] __attribute__((aligned(128))) = { 0 }; + for (uint32_t r = 0; r < 32; ++r) { + tcol[r] = r < nr ? (int16_t) (t % 32) : -1; + if (++h == nh) { + h = 0; + t++; + } + } + const HVX_Vector tv = *(const HVX_Vector *) tcol; + + // halfword 2n and 2n + 1 of a G vector are column n + static const int16_t col[64] __attribute__((aligned(128))) = { + 0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8, 8, 9, 9, 10, 10, + 11, 11, 12, 12, 13, 13, 14, 14, 15, 15, 16, 16, 17, 17, 18, 18, 19, 19, 20, 20, 21, 21, + 22, 22, 23, 23, 24, 24, 25, 25, 26, 26, 27, 27, 28, 28, 29, 29, 30, 30, 31, 31, + }; + const HVX_Vector cv = *(const HVX_Vector *) col; + + // vector p holds rows 2p, 2p + 1 + HVX_Vector * gv = (HVX_Vector *) gt; + for (uint32_t p = 0; p < 16; ++p) { + const HVX_Vector tp = hvx_vec_repl4(Q6_V_vror_VR(tv, p * 4)); + const HVX_Vector sp = hvx_vec_repl4(Q6_V_vror_VR(sv, p * 4)); + gv[p] = Q6_V_vmux_QVV(Q6_Q_vcmp_eq_VhVh(cv, tp), sp, zero); + } +} + +// Q buffer slot of set +static inline uint32_t lid_hmx_slot(const struct htp_lid_hmx_context * hctx, uint32_t qbuf, uint32_t set) { + return qbuf * hctx->n_qsets + set; +} + +// Scales and G of channel tile c of a Q slot (needs the slot's w) +static __attribute__((noinline)) void lid_hmx_q_scg(const struct htp_lid_hmx_context * hctx, uint32_t slot, const struct lid_hmx_set * S, uint32_t c) { + const struct htp_lid_hmx_vtcm_layout * L = &hctx->L; + const uint32_t nh = hctx->octx->src[0]->ne[1]; + const uint32_t ch = c * 32; + const uint32_t t = fastdiv(ch, &hctx->kp->div_n_head); + + lid_hmx_build_sc_g(hctx->sc + (slot * L->n_ch + c) * 256, hctx->gw + (slot * L->n_ch + c) * HMX_FP16_TILE_N_ELMS, + hctx->w + slot * (L->size_w / sizeof(float)) + ch, t, ch - t * nh, MIN(32, S->nt * nh - ch), nh); +} + +// Staged f32 Q rows of channel tile c to f16, then to the weight tiles of a Q slot +static __attribute__((noinline)) void lid_hmx_q_tiles(const struct htp_lid_hmx_context * hctx, uint32_t slot, const struct lid_hmx_set * S, uint32_t c, + const uint8_t * qs, uint8_t * qh) { + const struct htp_lid_hmx_vtcm_layout * L = &hctx->L; + const uint32_t D = hctx->octx->src[0]->ne[0]; + const uint32_t nh = hctx->octx->src[0]->ne[1]; + const uint32_t nr = MIN(32, S->nt * nh - c * 32); + + for (uint32_t r = 0; r < nr; ++r) { + hvx_copy_f16_f32_aa(qh + r * L->size_qh_row, qs + r * L->size_qs_row, D); + } + + // missing channels must be zero (finite): zero the tiles, then scatter only the valid rows + __fp16 * qw = hctx->qw + (slot * L->n_ch + c) * L->n_dot * HMX_FP16_TILE_N_ELMS; + if (nr < 32) { + for (uint32_t i = 0; i < L->n_dot * 16; ++i) { + ((HVX_Vector *) qw)[i] = Q6_V_vzero(); + } + } + hmx_interleave_rows_to_tiles(qw, (const __fp16 *) qh, nr, D, L->size_qh_row / sizeof(__fp16), 0, hex_round_up(nr, 2)); + hvx_gather_sync(qw); +} + +static inline uint32_t lid_hmx_find_valid_gc(const struct htp_lid_hmx_context * hctx, uint32_t gc, uint32_t gc1) { + const struct fastdiv_values * div_n_ch = &hctx->kp->div_n_ch; + const uint32_t n_ch = hctx->L.n_ch; + while (gc < gc1) { + const uint32_t set = fastdiv(gc, div_n_ch); + const uint32_t c = gc - set * n_ch; + if (c < hctx->nx_sets[set].n_ch) { + return gc; + } + gc++; + } + return gc1; +} + +static inline uint32_t lid_hmx_q_chunk_push(const struct htp_lid_hmx_context * hctx, dma_queue * dma_q, + uint8_t * qs, uint32_t qb, uint32_t gc) { + const struct htp_ops_context * octx = hctx->octx; + const struct htp_tensor * w = octx->src[2]; + const struct htp_lid_hmx_vtcm_layout * L = &hctx->L; + const uint32_t nh = octx->src[0]->ne[1]; + + const uint32_t set = fastdiv(gc, &hctx->kp->div_n_ch); + const uint32_t c = gc - set * L->n_ch; + const struct lid_hmx_set * S = &hctx->nx_sets[set]; + + const uint32_t slot = lid_hmx_slot(hctx, qb, set); + const uint32_t ch = c * 32; + const uint32_t ta = fastdiv(ch, &hctx->kp->div_n_head); + const uint32_t tb = fastdiv(MIN(ch + 32, S->nt * nh) - 1, &hctx->kp->div_n_head) + 1; + + float * wd = hctx->w + slot * (L->size_w / sizeof(float)); + dma_queue_push(dma_q, dma_make_data(wd + ta * nh, w->data + S->is * w->nb[3] + (S->t0 + ta) * w->nb[1]), + nh * sizeof(float), w->nb[1], nh * sizeof(float), tb - ta); + return 1 + lid_hmx_push_q(hctx, dma_q, qs, S, c); +} + +// Stage g share of the next run's Q prep (the DMA queue of the thread is empty here) +static __attribute__((noinline)) void lid_hmx_q_chunk(const struct htp_lid_hmx_context * hctx, const htp_thread_t * th, uint32_t ith, uint32_t nth) { + const struct htp_ops_context * octx = hctx->octx; + const struct htp_lid_hmx_vtcm_layout * L = &hctx->L; + + const uint32_t n_tot = hctx->nx_n_run * L->n_ch; + const uint32_t a = fastdiv(hctx->g * n_tot, &hctx->nx_stg_div); + const uint32_t b = fastdiv((hctx->g + 1) * n_tot, &hctx->nx_stg_div); + const uint32_t per = fastdiv(b - a + octx->n_hvx - 1, &octx->n_hvx_div); + const uint32_t gc0 = MIN(a + ith * per, b); + const uint32_t gc1 = MIN(gc0 + per, b); + const uint32_t qb = hctx->qbuf ^ 1; + + dma_queue * dma_q = th->dma_q; + uint8_t * qs[2]; + qs[0] = hctx->qs + ith * L->size_qs; + qs[1] = qs[0] + 32 * L->size_qs_row; + uint8_t * qh = qs[0] + 2 * 32 * L->size_qs_row; + + uint32_t gc_cur = lid_hmx_find_valid_gc(hctx, gc0, gc1); + if (gc_cur >= gc1) { + return; + } + + const uint32_t gc_next = lid_hmx_find_valid_gc(hctx, gc_cur + 1, gc1); + uint32_t gc_prefetch = gc1; + + uint32_t n_dma[2] = {0, 0}; + n_dma[0] = lid_hmx_q_chunk_push(hctx, dma_q, qs[0], qb, gc_cur); + if (gc_next < gc1) { + n_dma[1] = lid_hmx_q_chunk_push(hctx, dma_q, qs[1], qb, gc_next); + gc_prefetch = lid_hmx_find_valid_gc(hctx, gc_next + 1, gc1); + } + + uint32_t slot = 0; + while (gc_cur < gc1) { + for (uint32_t i = 0; i < n_dma[slot]; ++i) { + dma_queue_pop(dma_q); + } + + const uint32_t set = fastdiv(gc_cur, &hctx->kp->div_n_ch); + const uint32_t c = gc_cur - set * L->n_ch; + const struct lid_hmx_set * S = &hctx->nx_sets[set]; + const uint32_t qslot = lid_hmx_slot(hctx, qb, set); + + lid_hmx_q_scg(hctx, qslot, S, c); + lid_hmx_q_tiles(hctx, qslot, S, c, qs[slot], qh); + + if (gc_prefetch < gc1) { + n_dma[slot] = lid_hmx_q_chunk_push(hctx, dma_q, qs[slot], qb, gc_prefetch); + gc_prefetch = lid_hmx_find_valid_gc(hctx, gc_prefetch + 1, gc1); + } else { + n_dma[slot] = 0; + } + + gc_cur = lid_hmx_find_valid_gc(hctx, gc_cur + 1, gc1); + slot ^= 1; + } +} + +// Q rows to weight tiles, per channel scales |w| and G tiles for all sets of the run, then stage -1 (prep block 0). +// Q work goes to the first threads (thread 0 starts first), the K rows of block 0 to the others. +static void lid_hmx_qprep_thread(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { + struct htp_lid_hmx_context * hctx = (struct htp_lid_hmx_context *) data; + const struct htp_ops_context * octx = hctx->octx; + const struct htp_tensor * q = octx->src[0]; + const struct htp_tensor * w = octx->src[2]; + const struct htp_lid_hmx_vtcm_layout * L = &hctx->L; + + const uint32_t nh = q->ne[1]; + + struct htp_thread_trace * tr = th->trace; + dma_queue * dma_q = th->dma_q; + + const uint32_t n_tot = hctx->n_run * L->n_ch; + const uint32_t per = fastdiv(n_tot + octx->n_hvx - 1, &octx->n_hvx_div); + const uint32_t gc0 = MIN(ith * per, n_tot); + const uint32_t gc1 = MIN(gc0 + per, n_tot); + + uint8_t * qs = hctx->qs + ith * L->size_qs; + uint8_t * qh = qs + 2 * 32 * L->size_qs_row; + + // all input DMAs in flight at once: w of the sets, Q of the first channel tile, then K and mask of block 0 + uint32_t n_dma = 0; + uint32_t first = gc1; + const struct fastdiv_values * div_n_ch = &hctx->kp->div_n_ch; + for (uint32_t set = fastdiv(gc0, div_n_ch); gc0 < gc1 && set <= fastdiv(gc1 - 1, div_n_ch); ++set) { + const struct lid_hmx_set S = hctx->sets[set]; + + const uint32_t c0 = MAX(gc0, set * L->n_ch) - set * L->n_ch; + const uint32_t c1 = MIN(MIN(gc1, (set + 1) * L->n_ch) - set * L->n_ch, S.n_ch); + if (c0 >= c1) { + continue; + } + first = MIN(first, set * L->n_ch + c0); + + const uint32_t ta = fastdiv(c0 * 32, &hctx->kp->div_n_head); + const uint32_t tb = fastdiv(MIN(c1 * 32, S.nt * nh) - 1, &hctx->kp->div_n_head) + 1; + float * wd = hctx->w + lid_hmx_slot(hctx, hctx->qbuf, set) * (L->size_w / sizeof(float)); + dma_queue_push(dma_q, dma_make_data(wd + ta * nh, w->data + S.is * w->nb[3] + (S.t0 + ta) * w->nb[1]), + nh * sizeof(float), w->nb[1], nh * sizeof(float), tb - ta); + n_dma++; + } + + if (first < gc1) { + const uint32_t set = fastdiv(first, div_n_ch); + const struct lid_hmx_set S = hctx->sets[set]; + n_dma += lid_hmx_push_q(hctx, dma_q, qs, &S, first - set * L->n_ch); + } + + uint32_t n_in = lid_hmx_push_in(hctx, th, ith, nth, 0); + + for (uint32_t i = 0; i < n_dma; ++i) { + dma_queue_pop(dma_q); + } + + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_A_PREP, (uint16_t) gc0); + + for (uint32_t gc = first; gc < gc1; ++gc) { + const uint32_t set = fastdiv(gc, div_n_ch); + const uint32_t c = gc - set * L->n_ch; + const struct lid_hmx_set S = hctx->sets[set]; + if (c >= S.n_ch) { + continue; + } + + uint32_t n_q = 0; + if (gc != first) { + n_q = lid_hmx_push_q(hctx, dma_q, qs, &S, c) + n_in; + n_in = 0; + } + + // scales and G while the Q rows land + const uint32_t slot = lid_hmx_slot(hctx, hctx->qbuf, set); + lid_hmx_q_scg(hctx, slot, &S, c); + + for (uint32_t i = 0; i < n_q; ++i) { + dma_queue_pop(dma_q); + } + + lid_hmx_q_tiles(hctx, slot, &S, c, qs, qh); + } + + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_A_PREP, (uint16_t) gc0); + + for (uint32_t i = 0; i < n_in; ++i) { + dma_queue_pop(dma_q); + } + + lid_hmx_stage(hctx, th, ith, nth); +} + +// Run start when its Q was prepped in the previous run: block 0 K and mask, then stage -1 +static void lid_hmx_first_thread(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { + const struct htp_lid_hmx_context * hctx = (const struct htp_lid_hmx_context *) data; + + const uint32_t n_in = lid_hmx_push_in(hctx, th, ith, nth, 0); + for (uint32_t i = 0; i < n_in; ++i) { + dma_queue_pop(th->dma_q); + } + lid_hmx_stage(hctx, th, ith, nth); +} + +static int lid_hmx_run(struct htp_ops_context * octx, const struct htp_lid_kernel_params * kparams) { + const struct htp_tensor * q = octx->src[0]; + + const uint32_t D = q->ne[0]; + const uint32_t n_head = q->ne[1]; + const uint32_t n_stream = q->ne[3]; + const uint32_t n_hvx = kparams->n_hvx; + const uint32_t n_qsets = kparams->n_qsets; + + struct htp_lid_hmx_context hctx; + memset(&hctx, 0, sizeof(hctx)); + hctx.octx = octx; + hctx.kp = kparams; + hctx.Bc = kparams->kv_block; + hctx.kv_seg = kparams->kv_seg; + hctx.n_tile = kparams->n_tokens_tile; + hctx.n_tt = kparams->n_tt; + hctx.n_qsets = n_qsets; + htp_lid_hmx_vtcm_layout_build(&hctx.L, D, n_head, hctx.n_tile, hctx.Bc, n_qsets * kparams->n_qbuf, n_hvx, + kparams->k_row_size); + + uint8_t * const base = octx->ctx->vtcm_base; + const struct htp_lid_hmx_vtcm_layout * L = &hctx.L; + + hctx.qw = VTCM_LAYOUT_PTR(__fp16, base, L->off_qw); + hctx.gw = VTCM_LAYOUT_PTR(__fp16, base, L->off_g); + hctx.sc = VTCM_LAYOUT_PTR(uint8_t, base, L->off_sc); + hctx.sc_id = VTCM_LAYOUT_PTR(uint8_t, base, L->off_sc_id); + hctx.s = VTCM_LAYOUT_PTR(__fp16, base, L->off_s); + hctx.w = VTCM_LAYOUT_PTR(float, base, L->off_w); + hctx.qs = VTCM_LAYOUT_PTR(uint8_t, base, L->off_qs); + for (int i = 0; i < 2; i++) { + hctx.k_raw[i] = VTCM_LAYOUT_PTR(uint8_t, base, L->off_k_raw[i]); + hctx.k_t[i] = VTCM_LAYOUT_PTR(__fp16, base, L->off_k_t[i]); + hctx.o[i] = VTCM_LAYOUT_PTR(__fp16, base, L->off_o[i]); + hctx.out[i] = VTCM_LAYOUT_PTR(uint8_t, base, L->off_out[i]); + } + for (int i = 0; i < 4; i++) { + hctx.m[i] = VTCM_LAYOUT_PTR(uint8_t, base, L->off_m[i]); + } + + hmx_init_column_scales(hctx.sc_id, Q6_V_vsplat_R(0x3c00)); // identity 1.0 f16 + + hmx_queue_t hmx_q = octx->ctx->hmx_queue; + lid_hmx_job_t job[2]; + + const uint32_t n_tiles = n_stream * hctx.n_tt; + + for (uint32_t tile0 = 0, r = 0; tile0 < n_tiles; tile0 += n_qsets, ++r) { + hctx.tile0 = tile0; + hctx.n_run = MIN(n_qsets, n_tiles - tile0); + hctx.qbuf = r & (kparams->n_qbuf - 1); // n_qbuf is 1 or 2 + + for (uint32_t i = 0; i < hctx.n_run; ++i) { + hctx.sets[i] = lid_hmx_set_init(&hctx, i); + } + + // all sets of a run have the same token count: one set per run, or all tokens in one tile + const struct lid_hmx_set S0 = hctx.sets[0]; + hctx.nt_run = S0.nt; + hctx.n_u_run = S0.n_u; + hctx.n_blocks = hctx.n_run == n_qsets ? kparams->n_kv_blocks : (hctx.n_run * hctx.kv_seg + hctx.Bc - 1) / hctx.Bc; + + const int32_t n = (int32_t) hctx.n_blocks; + const uint32_t n_tot = hctx.n_run * L->n_ch; + const uint32_t per = fastdiv(n_tot + n_hvx - 1, &octx->n_hvx_div); + + hctx.n_qthr = (n_tot + per - 1) / per; + hctx.g = -1; + + const bool q_ready = hctx.nx_on; + if (q_ready) { + hctx.n_qthr = 0; + hctx.nx_on = false; + work_queue_run(octx->ctx->hvx_queue, lid_hmx_first_thread, &hctx, n_hvx); + } else { + work_queue_run(octx->ctx->hvx_queue, lid_hmx_qprep_thread, &hctx, n_hvx); + } + + // with two Q buffers the stages of this run prep the Q of the next one + if (kparams->n_qbuf == 2 && tile0 + n_qsets < n_tiles) { + struct htp_lid_hmx_context nx = hctx; + nx.tile0 = tile0 + n_qsets; + hctx.nx_n_run = MIN(n_qsets, n_tiles - nx.tile0); + for (uint32_t i = 0; i < hctx.nx_n_run; ++i) { + hctx.nx_sets[i] = lid_hmx_set_init(&nx, i); + } + hctx.nx_stg_div = init_fastdiv_values(hctx.n_blocks + 1); + hctx.nx_on = true; + } + + for (int32_t g = 0; g < n; ++g) { + struct lid_hmx_seg seg[LID_HMX_MAX_SEGS]; + uint32_t n_rows; + const uint32_t n_seg = lid_hmx_segs(&hctx, g, seg, &n_rows); + + lid_hmx_job_t * j = &job[g % 2]; + #pragma clang loop vectorize(disable) + for (uint32_t i = 0; i < n_seg; ++i) { + const uint32_t slot = lid_hmx_slot(&hctx, hctx.qbuf, seg[i].set); + j->seg[i].qw = hctx.qw + slot * L->n_ch * L->n_dot * HMX_FP16_TILE_N_ELMS; + j->seg[i].gw = hctx.gw + slot * L->n_ch * HMX_FP16_TILE_N_ELMS; + j->seg[i].sc = hctx.sc + slot * L->n_ch * 256; + j->seg[i].rt0 = seg[i].x0 / 32; + j->seg[i].n_rt = (seg[i].nv + 31) / 32; + } + j->n_seg = n_seg; + j->k_t = hctx.k_t[g % 2]; + j->sc_id = hctx.sc_id; + j->s = hctx.s; + j->o = hctx.o[g % 2]; + j->n_ch = S0.n_ch; + j->n_u = S0.n_u; + j->n_dot = L->n_dot; + j->n_head = n_head; + j->n_sg = L->n_sg; + hmx_queue_push(hmx_q, hmx_queue_make_desc(lid_hmx_worker, j)); + + // stage g has work only if there is a next block, a previous one or next run Q work + hctx.g = g; + if (g + 1 < n || g >= 1 || hctx.nx_on) { + work_queue_run(octx->ctx->hvx_queue, lid_hmx_stage_thread, &hctx, n_hvx); + } + + hmx_queue_pop(hmx_q); + } + + // last stage only extracts; with one token only thread 0 has work (same token split for nth 1) + hctx.g = n; + const uint32_t n_last = hctx.nt_run == 1 && !hctx.nx_on ? 1 : n_hvx; + work_queue_run(octx->ctx->hvx_queue, lid_hmx_stage_thread, &hctx, n_last); + } + + return HTP_STATUS_OK; +} + +int op_lightning_indexer(struct htp_ops_context * octx) { + const struct htp_tensor * q = octx->src[0]; + const struct htp_tensor * m = octx->src[3]; + + // no multi-device split yet, device 0 does all the work + if (octx->ctx->mdev.count > 1 && octx->ctx->mdev.idx != 0) { + return HTP_STATUS_OK; + } + + const struct htp_lid_kernel_params * kparams = (const struct htp_lid_kernel_params *) octx->kernel_params; + + if (kparams->vtcm_size > octx->ctx->vtcm_size) { + return HTP_STATUS_VTCM_TOO_SMALL; + } + + if (!htp_ops_context_set_n_hvx(octx, kparams->n_hvx)) { + return HTP_STATUS_INVAL_PARAMS; + } + + if (kparams->kernel_type == HTP_LID_KERNEL_HMX) { + return lid_hmx_run(octx, kparams); + } + + struct htp_lid_context lctx; + lctx.octx = octx; + lctx.kp = kparams; + lctx.n_blocks = kparams->n_kv_blocks; + lctx.blocks_per_hvx = kparams->blocks_per_hvx; + + htp_lid_vtcm_layout_build(&lctx.L, q->ne[0], q->ne[1], kparams->n_tokens_tile, kparams->n_hvx, kparams->k_row_size); + + uint8_t * const base = octx->ctx->vtcm_base; + + lctx.spad_q = VTCM_LAYOUT_PTR(uint8_t, base, lctx.L.off_q); + lctx.spad_w = VTCM_LAYOUT_PTR(uint8_t, base, lctx.L.off_w); + lctx.spad_qs = VTCM_LAYOUT_PTR(uint8_t, base, lctx.L.off_qs); + lctx.spad_k = VTCM_LAYOUT_PTR(uint8_t, base, lctx.L.off_k); + lctx.spad_kt = VTCM_LAYOUT_PTR(uint8_t, base, lctx.L.off_kt); + lctx.spad_m = VTCM_LAYOUT_PTR(uint8_t, base, lctx.L.off_m); + lctx.spad_o = VTCM_LAYOUT_PTR(uint8_t, base, lctx.L.off_o); + + const uint32_t n_tokens = q->ne[2]; + const uint32_t n_stream = q->ne[3]; + const uint32_t n_tile = kparams->n_tokens_tile; + const uint32_t n_hvx = kparams->n_hvx; + + for (uint32_t is = 0; is < n_stream; ++is) { + lctx.is = is; + lctx.im3 = fastmodulo(is, m->ne[3], &kparams->div_m_ne3); + + for (uint32_t t0 = 0; t0 < n_tokens; t0 += n_tile) { + lctx.t0 = t0; + lctx.nt = MIN(n_tile, n_tokens - t0); + + work_queue_run(octx->ctx->hvx_queue, lid_prep_thread, &lctx, MIN(n_hvx, lctx.nt * q->ne[1])); + work_queue_run(octx->ctx->hvx_queue, lid_compute_thread, &lctx, n_hvx); + } + } + + return HTP_STATUS_OK; +} diff --git a/ggml/src/ggml-hexagon/htp/lightning-indexer-ops.h b/ggml/src/ggml-hexagon/htp/lightning-indexer-ops.h new file mode 100644 index 000000000000..14b3503f6e6d --- /dev/null +++ b/ggml/src/ggml-hexagon/htp/lightning-indexer-ops.h @@ -0,0 +1,213 @@ +#ifndef HTP_LIGHTNING_INDEXER_OPS_H +#define HTP_LIGHTNING_INDEXER_OPS_H + +#include +#include +#include + +#include "hex-common.h" +#include "hex-fastdiv.h" +#include "htp-vtcm.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#define HTP_LID_KV_BLOCK 64 // kv rows per block (HVX and HMX Bc) +#define HTP_LID_TOKENS_MAX 32 // max tokens per Q tile (HVX splat path) +#define HTP_LID_HMX_TOKENS_MAX 512 // HMX can keep more tokens; VTCM decides +#define HTP_LID_HMX_KV_BLOCK_MAX 2048 +#define HTP_LID_HMX_MAX_QSETS 8 // streams in one HMX pipeline run +#define HTP_LID_N_BUF 4 // K / mask / output buffers per thread (two blocks computed, two in flight) + +#ifndef HMX_FP16_TILE_N_COLS +#define HMX_FP16_TILE_N_COLS 32 +#endif +#ifndef HMX_FP16_TILE_SIZE +#define HMX_FP16_TILE_SIZE 2048 +#endif + +enum htp_lid_kernel_type { + HTP_LID_KERNEL_HVX = 0, + HTP_LID_KERNEL_HMX = 1, +}; + +struct htp_lid_kernel_params { + uint8_t kernel_type; // enum htp_lid_kernel_type + uint32_t n_hvx; + uint32_t n_tokens_tile; // tokens per Q tile + uint32_t n_kv_blocks; // HVX: per stream, HMX: per run of n_qsets sets + uint32_t blocks_per_hvx; + uint32_t vtcm_size; + uint32_t k_row_size; // K row bytes + uint32_t kv_block; // HMX: kv rows per block (multiple of 64) + uint32_t n_qsets; // HMX: Q sets kept in VTCM (streams run in one pipeline) + uint32_t n_qbuf; // HMX: Q buffers, 2: the next run's Q is prepped during the current run + uint32_t kv_seg; // HMX: rows per set, n_kv padded to 64 + uint32_t n_tt; // HMX: token tiles per stream + + struct fastdiv_values div_n_head; + struct fastdiv_values div_m_ne3; // mask streams (broadcast) + struct fastdiv_values div_kv_seg; // HMX + struct fastdiv_values div_n_tt; // HMX + struct fastdiv_values div_n_ch; // HMX: channel tiles per Q set +}; + +#if defined(__cplusplus) +static_assert(sizeof(struct htp_lid_kernel_params) <= 128, "htp_lid_kernel_params is too large"); +#else +_Static_assert(sizeof(struct htp_lid_kernel_params) <= 128, "htp_lid_kernel_params is too large"); +#endif + +// VTCM layout shared by host (fit check) and device (buffer offsets). +// Shared regions: Q splats (one f16 vector per q element) and head weight splats for the current token tile. +// Per-HVX regions: Q prep staging, K / mask / output buffers, two transposed K blocks and transpose scratch. +struct htp_lid_vtcm_layout { + size_t off_q; + size_t off_w; + size_t off_qs; + size_t off_k; + size_t off_kt; + size_t off_m; + size_t off_o; + + size_t size_q_row; // f16 Q row, padded + size_t size_qs_row; // f32 Q staging row, padded + size_t size_k_row; // f16 K row, padded + size_t size_k_raw_row; + size_t size_m_row; // f16 mask row of one kv block, padded + size_t size_o_row; // f32 output row of one kv block, padded + + size_t size_qs; // per thread + size_t size_k_buf; // one K buffer + size_t size_kt; // per thread: two transposed K blocks, two 64x64 scratch tiles, Q8_0 block scales + size_t size_m_buf; // one mask buffer + size_t size_o_buf; // one output buffer + + size_t total_bytes; +}; + +static inline void htp_lid_vtcm_layout_build(struct htp_lid_vtcm_layout * L, size_t D, size_t n_head, size_t n_tok, size_t n_hvx, size_t k_row_size) { + L->size_q_row = hex_round_up(D * sizeof(uint16_t), 128); + L->size_qs_row = hex_round_up(D * sizeof(float), 128); + L->size_k_row = hex_round_up(D * sizeof(uint16_t), 128); + L->size_k_raw_row = L->size_k_row > hex_round_up(k_row_size, 128) ? L->size_k_row : hex_round_up(k_row_size, 128); + L->size_m_row = hex_round_up(HTP_LID_KV_BLOCK * sizeof(uint16_t), 128); + L->size_o_row = hex_round_up(HTP_LID_KV_BLOCK * sizeof(float), 128); + + L->size_qs = 2 * (n_head * L->size_qs_row + hex_round_up(n_head * sizeof(float), 128)) + L->size_q_row; + L->size_k_buf = HTP_LID_KV_BLOCK * L->size_k_raw_row; + L->size_kt = (2 * hex_round_up(D, HTP_LID_KV_BLOCK) + 2 * HTP_LID_KV_BLOCK + D / 32) * 128; + L->size_m_buf = n_tok * L->size_m_row; + L->size_o_buf = n_tok * L->size_o_row; + + size_t off = 0; + VTCM_LAYOUT_ALLOC(off, off_q, n_tok * n_head * D * 128); + VTCM_LAYOUT_ALLOC(off, off_w, n_tok * n_head * 128); + VTCM_LAYOUT_ALLOC(off, off_qs, n_hvx * L->size_qs); + VTCM_LAYOUT_ALLOC(off, off_k, n_hvx * HTP_LID_N_BUF * L->size_k_buf); + VTCM_LAYOUT_ALLOC(off, off_kt, n_hvx * L->size_kt); + VTCM_LAYOUT_ALLOC(off, off_m, n_hvx * HTP_LID_N_BUF * L->size_m_buf); + VTCM_LAYOUT_ALLOC(off, off_o, n_hvx * HTP_LID_N_BUF * L->size_o_buf); + + L->total_bytes = off; +} + +static inline size_t htp_lid_compute_vtcm_usage(size_t D, size_t n_head, size_t n_tok, size_t n_hvx, size_t k_row_size) { + struct htp_lid_vtcm_layout L; + htp_lid_vtcm_layout_build(&L, D, n_head, n_tok, n_hvx, k_row_size); + return L.total_bytes; +} + +// HMX path. K is the activation (kv rows), Q the weights (one output channel per (token, head)). +// mm1: S = relu(K Q^T) * |w| per channel (HMX convert scale + pos clamp) +// mm2: O = S G, G[(t,h), t] = sign(w[t,h]). O is [kv, token], f16. +// Q weights, G, scales and w are per Q set (one per stream when all tokens fit one tile). +struct htp_lid_hmx_vtcm_layout { + size_t off_qw; // Q weight tiles [n_qsets][n_ch][n_dot] + size_t off_g; // G weight tiles [n_qsets][n_ch] + size_t off_sc; // per channel scales, 256 bytes per channel tile, [n_qsets][n_ch] + size_t off_sc_id; // identity scales + size_t off_s; // S tiles of one block and token tile, [n_rt][n_sg] + size_t off_w; // w of the token tile, [n_qsets][n_tok][n_head] f32 + size_t off_qs; // per thread: f32 and f16 staging for 32 Q rows, gather scratch + size_t off_k_raw[2]; // K rows, [Bc][size_k_row] + size_t off_k_t[2]; // K tiles [n_rt][n_dot] + size_t off_o[2]; // O tiles [n_rt][n_u] + size_t off_m[4]; // mask rows [n_tok][size_m_row] + size_t off_out[2]; // f32 output rows [n_tok][size_o_row] + + size_t size_qs_row; // f32 Q row, padded + size_t size_qh_row; // f16 Q row, padded + size_t size_k_row; // f16 K row, padded + size_t size_k_raw_row; + size_t size_m_row; + size_t size_o_row; + size_t size_qs; // per thread + size_t size_w; // per Q set + + uint32_t n_ch; // channel tiles: ceil(n_tok * n_head / 32) + uint32_t n_u; // token tiles: ceil(n_tok / 32) + uint32_t n_dot; // D / 32 + uint32_t n_rt; // kv row tiles per block: Bc / 32 + uint32_t n_sg; // S tiles per (row tile, token tile): min(n_head, n_ch) + + size_t total_bytes; +}; + +static inline void htp_lid_hmx_vtcm_layout_build(struct htp_lid_hmx_vtcm_layout * L, + size_t D, size_t n_head, size_t n_tok, size_t Bc, size_t n_qsets, size_t n_hvx, size_t k_row_size) { + L->n_ch = (uint32_t) ((n_tok * n_head + 31) / 32); + L->n_u = (uint32_t) ((n_tok + 31) / 32); + L->n_dot = (uint32_t) (D / 32); + L->n_rt = (uint32_t) (Bc / 32); + L->n_sg = (uint32_t) (n_head < L->n_ch ? n_head : L->n_ch); + + L->size_qs_row = hex_round_up(D * sizeof(float), 128); + L->size_qh_row = hex_round_up(D * sizeof(uint16_t), 128); + L->size_k_row = hex_round_up(D * sizeof(uint16_t), 128); + L->size_k_raw_row = L->size_k_row > hex_round_up(k_row_size, 128) ? L->size_k_row : hex_round_up(k_row_size, 128); + L->size_m_row = hex_round_up(Bc * sizeof(uint16_t), 128); + L->size_o_row = hex_round_up(Bc * sizeof(float), 128); + L->size_qs = 2 * 32 * L->size_qs_row + 32 * L->size_qh_row + hex_round_up(Bc / 64 * 128, 128); + L->size_w = hex_round_up(n_tok * n_head * sizeof(float), 128); + + const size_t tile = HMX_FP16_TILE_SIZE; + + // tile regions first: they need 2 KB alignment + size_t off = 0; + VTCM_LAYOUT_ALLOC(off, off_qw, n_qsets * L->n_ch * L->n_dot * tile); + VTCM_LAYOUT_ALLOC(off, off_g, n_qsets * L->n_ch * tile); + VTCM_LAYOUT_ALLOC(off, off_s, (size_t) L->n_rt * L->n_sg * tile); + VTCM_LAYOUT_ALLOC(off, off_k_t[0], (size_t) L->n_rt * L->n_dot * tile); + VTCM_LAYOUT_ALLOC(off, off_k_t[1], (size_t) L->n_rt * L->n_dot * tile); + VTCM_LAYOUT_ALLOC(off, off_o[0], (size_t) L->n_rt * L->n_u * tile); + VTCM_LAYOUT_ALLOC(off, off_o[1], (size_t) L->n_rt * L->n_u * tile); + VTCM_LAYOUT_ALLOC(off, off_sc, n_qsets * L->n_ch * 256); + VTCM_LAYOUT_ALLOC(off, off_sc_id, 256); + VTCM_LAYOUT_ALLOC(off, off_w, n_qsets * L->size_w); + VTCM_LAYOUT_ALLOC(off, off_qs, n_hvx * L->size_qs); + for (int i = 0; i < 2; i++) { + VTCM_LAYOUT_ALLOC(off, off_k_raw[i], Bc * L->size_k_raw_row); + } + for (int i = 0; i < 4; i++) { + VTCM_LAYOUT_ALLOC(off, off_m[i], n_tok * L->size_m_row); + } + for (int i = 0; i < 2; i++) { + VTCM_LAYOUT_ALLOC(off, off_out[i], n_tok * L->size_o_row); + } + + L->total_bytes = off; +} + +static inline size_t htp_lid_hmx_compute_vtcm_usage(size_t D, size_t n_head, size_t n_tok, size_t Bc, size_t n_qsets, size_t n_hvx, size_t k_row_size) { + struct htp_lid_hmx_vtcm_layout L; + htp_lid_hmx_vtcm_layout_build(&L, D, n_head, n_tok, Bc, n_qsets, n_hvx, k_row_size); + return L.total_bytes; +} + +#ifdef __cplusplus +} +#endif + +#endif /* HTP_LIGHTNING_INDEXER_OPS_H */ diff --git a/ggml/src/ggml-hexagon/htp/main.c b/ggml/src/ggml-hexagon/htp/main.c index 78adc9a495d5..ad457d1a6b3e 100644 --- a/ggml/src/ggml-hexagon/htp/main.c +++ b/ggml/src/ggml-hexagon/htp/main.c @@ -20,6 +20,7 @@ #include #include +#include "htp-thread.h" #include "hex-utils.h" #include "dma-queue.h" #include "hmx-queue.h" @@ -32,8 +33,9 @@ #include "htp-ops.h" #include "htp_iface.h" #include "work-queue.h" -#include "hex-profile.h" +#include "htp-trace.h" #include "allreduce-ops.h" +#include "dsv4-hc-ops.h" #include "htp-fence.h" #define HMX_QUEUE_CAPACITY 128 @@ -42,8 +44,7 @@ #define WORK_QUEUE_STACK_SIZE 16384 #define MAIN_THREAD_STACK_SIZE 32768 -_Static_assert(WORK_QUEUE_MAX_N_THREADS >= HTP_MAX_NTHREADS, - "work-queue thread cap must be >= HTP_MAX_NTHREADS"); +atomic_uint htp_thread_count = 0; struct htp_handle { struct htp_context * ctx; @@ -490,17 +491,23 @@ AEEResult htp_iface_start(remote_handle64 handle, uint32_t sess_id, uint64_t dsp qurt_sysenv_max_hthreads_t hw_threads; qurt_sysenv_get_max_hw_threads(&hw_threads); uint32_t hw_nhvx = (qurt_hvx_get_units() >> 8) & 0xFF; + uint32_t hw_nth = hw_threads.max_hthreads; + if (hw_nth > HTP_MAX_NTHREADS) { + hw_nth = HTP_MAX_NTHREADS; + } if (n_hvx == 0) { n_hvx = hw_nhvx; } - if (n_hvx > hw_threads.max_hthreads) { - n_hvx = hw_threads.max_hthreads; + if (n_hvx > HTP_MAX_NHVX) { + n_hvx = HTP_MAX_NHVX; } - if (n_hvx > HTP_MAX_NTHREADS) { - n_hvx = HTP_MAX_NTHREADS; + if (n_hvx > hw_nth - 1) { + n_hvx = hw_nth - 1; } + uint32_t n_threads = hw_nth; + // layout segments of our contiguous block // 1. htp_context : sits at the base (block is 4K-aligned via memalign below) @@ -519,13 +526,13 @@ AEEResult htp_iface_start(remote_handle64 handle, uint32_t sess_id, uint64_t dsp size_t wq_align = work_queue_alignof(); size_t offset_wq = hex_align_up(offset, wq_align); size_t size_wq = work_queue_sizeof(n_hvx, WORK_QUEUE_CAPACITY, WORK_QUEUE_STACK_SIZE); - offset = offset_wq + size_wq; + offset = offset_wq + size_wq; // 4. dma_queue size_t dma_align = dma_queue_alignof(); size_t offset_dma = hex_align_up(offset, dma_align); size_t size_dma = 0; - for (uint32_t i = 0; i < n_hvx; i++) { + for (uint32_t i = 0; i < n_threads; i++) { size_dma = hex_align_up(size_dma, dma_queue_alignof()); size_dma += dma_queue_sizeof(256); } @@ -656,16 +663,9 @@ AEEResult htp_iface_start(remote_handle64 handle, uint32_t sess_id, uint64_t dsp } #endif - ctx->hmx_enabled = n_hmx; - ctx->hmx_queue = NULL; - if (n_hmx) { - void * hmx_ptr = (void *) ((uintptr_t) block + offset_hmx); - ctx->hmx_queue = hmx_queue_init(hmx_ptr, HMX_QUEUE_CAPACITY, HMX_QUEUE_STACK_SIZE, ctx->vtcm_rctx, &ctx->trace[HTP_MAX_NTHREADS]); - } - FARF(HIGH, "HMX %s (n_hmx=%d)", ctx->hmx_enabled ? "enabled" : "disabled", n_hmx); - - ctx->n_threads = n_hvx; - ctx->n_threads_div = init_fastdiv_values(ctx->n_threads); + ctx->n_threads = n_threads; + ctx->n_hvx = n_hvx; + ctx->n_hvx_div = init_fastdiv_values(ctx->n_hvx); // Initialize DMA queues uint8_t * dma_ptr_curr = (uint8_t *) ((uintptr_t) block + offset_dma); @@ -680,8 +680,18 @@ AEEResult htp_iface_start(remote_handle64 handle, uint32_t sess_id, uint64_t dsp ctx->ddr_spad_size = HTP_DDR_SPAD_SIZE; ctx->ddr_spad_base = memalign(128, ctx->ddr_spad_size); + htp_thread_reset(); + void * wq_ptr = (void *) ((uintptr_t) block + offset_wq); - ctx->work_queue = work_queue_init(wq_ptr, n_hvx, WORK_QUEUE_CAPACITY, WORK_QUEUE_STACK_SIZE); + ctx->hvx_queue = work_queue_init(wq_ptr, n_hvx, WORK_QUEUE_CAPACITY, WORK_QUEUE_STACK_SIZE, ctx->dma, ctx->trace); + + ctx->hmx_enabled = n_hmx; + ctx->hmx_queue = NULL; + if (n_hmx) { + void * hmx_ptr = (void *) ((uintptr_t) block + offset_hmx); + ctx->hmx_queue = hmx_queue_init(hmx_ptr, HMX_QUEUE_CAPACITY, HMX_QUEUE_STACK_SIZE, ctx->vtcm_rctx, ctx->dma, ctx->trace); + } + FARF(HIGH, "HMX %s (n_hmx=%d)", ctx->hmx_enabled ? "enabled" : "disabled", n_hmx); ctx->main_stack = NULL; ctx->main_thread = 0; @@ -707,7 +717,7 @@ AEEResult htp_iface_start(remote_handle64 handle, uint32_t sess_id, uint64_t dsp } FARF(HIGH, "session %u started: n-hvx %u vtcm-size %zu vtcm-rctx %u n-threads %u thread-id %d thread-prio %d \n", - sess_id, hw_nhvx, ctx->vtcm_size, ctx->vtcm_rctx, ctx->n_threads, ctx->thread_id, ctx->thread_prio); + sess_id, ctx->n_hvx, ctx->vtcm_size, ctx->vtcm_rctx, ctx->n_threads, ctx->thread_id, ctx->thread_prio); return AEE_SUCCESS; } @@ -732,7 +742,7 @@ AEEResult htp_iface_stop(remote_handle64 handle) { return err; } - work_queue_free(ctx->work_queue); + work_queue_free(ctx->hvx_queue); for (int i = 0; i < ctx->n_threads; i++) { dma_queue_free(ctx->dma[i]); @@ -768,16 +778,20 @@ AEEResult htp_iface_hwinfo(remote_handle64 handle, uint32_t * n_threads, uint32_ qurt_sysenv_get_max_hw_threads(&hw_threads); uint32_t hw_nhvx = (qurt_hvx_get_units() >> 8) & 0xFF; + uint32_t n_threads_val = hw_threads.max_hthreads; + if (n_threads_val > HTP_MAX_NTHREADS) { + n_threads_val = HTP_MAX_NTHREADS; + } + uint32_t n_hvx_val = hw_nhvx; - if (n_hvx_val > hw_threads.max_hthreads) { - n_hvx_val = hw_threads.max_hthreads; + if (n_hvx_val > HTP_MAX_NHVX) { + n_hvx_val = HTP_MAX_NHVX; } - if (n_hvx_val > HTP_MAX_NTHREADS) { - n_hvx_val = HTP_MAX_NTHREADS; + if (n_hvx_val > n_threads_val - 1) { + n_hvx_val = n_threads_val - 1; } - // for now we force n_threads == n_hvx - *n_threads = n_hvx_val; + *n_threads = n_threads_val; *n_hvx = n_hvx_val; *n_hmx = 1; @@ -930,6 +944,7 @@ static int execute_op(struct htp_ops_context * octx) { case HTP_OP_NORM: case HTP_OP_RMS_NORM: case HTP_OP_RMS_NORM_MUL: + case HTP_OP_RMS_NORM_MUL_ADD: case HTP_OP_SCALE: case HTP_OP_CLAMP: case HTP_OP_LEAKY_RELU: @@ -1007,6 +1022,9 @@ static int execute_op(struct htp_ops_context * octx) { case HTP_OP_CUMSUM: return op_cumsum(octx); + case HTP_OP_TOP_K_MASK: + return op_top_k_mask(octx); + case HTP_OP_FILL: return op_fill(octx); @@ -1026,6 +1044,7 @@ static int execute_op(struct htp_ops_context * octx) { return op_im2col_3d(octx); case HTP_OP_CONV_2D: + case HTP_OP_CONV_3D: return op_conv_2d(octx); case HTP_OP_CONV_2D_DW: @@ -1040,12 +1059,21 @@ static int execute_op(struct htp_ops_context * octx) { case HTP_OP_POOL_1D: return op_pool_1d(octx); + case HTP_OP_DSV4_HC_PRE: + return op_dsv4_hc_pre(octx); + + case HTP_OP_DSV4_HC_POST: + return op_dsv4_hc_post(octx); + case HTP_OP_CONCAT: return op_concat(octx); case HTP_OP_GATED_DELTA_NET: return op_gated_delta_net(octx); + case HTP_OP_LIGHTNING_INDEXER: + return op_lightning_indexer(octx); + case HTP_OP_TRI: return op_unary(octx); @@ -1212,10 +1240,10 @@ static int proc_op_req(struct htp_ops_context * octx, struct htp_buf_desc * bufs struct htp_tensor * tens, uint32_t idx, struct htp_op_desc * op) { memcpy(octx->op_params, op->params, sizeof(octx->op_params)); memcpy(octx->kernel_params, op->kernel_params, sizeof(octx->kernel_params)); - octx->flags = op->flags; - octx->op = op->opcode; - octx->n_threads = octx->ctx->n_threads; - octx->n_threads_div = octx->ctx->n_threads_div; + octx->flags = op->flags; + octx->op = op->opcode; + octx->n_hvx = octx->ctx->n_hvx; + octx->n_hvx_div = octx->ctx->n_hvx_div; FARF(HIGH, "proc-op #%u: opcode %u flags 0x%x", idx, octx->op, octx->flags); @@ -1277,7 +1305,7 @@ static void process_opbatch(struct htp_context * ctx, const struct htp_opbatch_r const uint32_t t_size = sizeof(struct htp_tensor) * n_tens; const uint32_t o_size = sizeof(struct htp_op_desc) * n_ops; const uint32_t p_size = sizeof(struct htp_prof_desc) * n_ops; - const uint32_t tr_size = (HTP_MAX_NTHREADS + 1) * req->n_traces * sizeof(struct htp_trace_desc); + const uint32_t tr_size = ctx->n_threads * req->n_traces * sizeof(struct htp_trace_desc); if (dbuf->size < b_size + t_size + o_size + p_size + tr_size) { FARF(ERROR, "invalid opbatch memory block size %u (req %u)", dbuf->size, b_size + t_size + o_size + p_size + tr_size); @@ -1300,7 +1328,7 @@ static void process_opbatch(struct htp_context * ctx, const struct htp_opbatch_r memset(ctx->trace, 0, sizeof(ctx->trace)); if (ctx->profiler == HTP_PROF_TRACE) { struct htp_trace_desc * trace_events = (struct htp_trace_desc *) (m_ptr + p_size); - for (int t = 0; t <= HTP_MAX_NTHREADS; t++) { + for (uint32_t t = 0; t < ctx->n_threads; t++) { ctx->trace[t].events = &trace_events[t * req->n_traces]; ctx->trace[t].max_events = req->n_traces; } @@ -1324,7 +1352,7 @@ static void process_opbatch(struct htp_context * ctx, const struct htp_opbatch_r mdev_group_init(ctx, req); - work_queue_wakeup(ctx->work_queue); + work_queue_wakeup(ctx->hvx_queue); if (ctx->hmx_queue) { hmx_queue_wakeup(ctx->hmx_queue); } @@ -1357,7 +1385,7 @@ static void process_opbatch(struct htp_context * ctx, const struct htp_opbatch_r hmx_queue_suspend(ctx->hmx_queue); hmx_queue_flush(ctx->hmx_queue); } - work_queue_suspend(ctx->work_queue); + work_queue_suspend(ctx->hvx_queue); // Flush remaining dirty tensors at the end of the batch htp_trace_event_start(&ctx->trace[0], HTP_TRACE_EVT_L2FLUSH, 0); @@ -1380,7 +1408,7 @@ static void process_opbatch(struct htp_context * ctx, const struct htp_opbatch_r rsp.cycles_stop = batch_prof.cycles_stop; if (ctx->profiler == HTP_PROF_TRACE) { - for (int t = 0; t <= HTP_MAX_NTHREADS; t++) { + for (uint32_t t = 0; t < ctx->n_threads; t++) { rsp.n_traces[t] = ctx->trace[t].count; } } diff --git a/ggml/src/ggml-hexagon/htp/matmul-ops.c b/ggml/src/ggml-hexagon/htp/matmul-ops.c index b4b943058f18..2d0750954c1b 100644 --- a/ggml/src/ggml-hexagon/htp/matmul-ops.c +++ b/ggml/src/ggml-hexagon/htp/matmul-ops.c @@ -100,7 +100,7 @@ struct htp_mm_context { const float * restrict sz0, const float * restrict sz1); // Precomputed values - uint32_t src0_nrows_per_thread; + uint32_t src0_nrows_per_hvx; uint32_t src0_row_start; uint32_t src0_row_end; uint32_t src0_row_size_padded; @@ -115,12 +115,12 @@ struct htp_mm_context { struct fastdiv_values mm_div_ne11; // Precomputed block-parallel quantization values - uint32_t quant_ib_first[WORK_QUEUE_MAX_N_THREADS]; - uint32_t quant_ib_last[WORK_QUEUE_MAX_N_THREADS]; - uint32_t quant_r[WORK_QUEUE_MAX_N_THREADS]; - uint32_t quant_c[WORK_QUEUE_MAX_N_THREADS]; + uint32_t quant_ib_first[HTP_MAX_NHVX]; + uint32_t quant_ib_last[HTP_MAX_NHVX]; + uint32_t quant_r[HTP_MAX_NHVX]; + uint32_t quant_c[HTP_MAX_NHVX]; uint32_t n_quant_tasks; - uint32_t n_quant_rows_per_thread; + uint32_t n_quant_rows_per_hvx; // Fields for scattered mapping & HMX support in MUL_MAT_ID const uint32_t * matrix_row_counts; @@ -140,21 +140,21 @@ struct htp_mm_context { uint32_t vtcm_act_raw_stride; // Cached thread offsets/sizes - uint32_t vtcm_src0_size_per_thread; - uint32_t vtcm_act_size_per_thread; - uint32_t vtcm_dst_size_per_thread; + uint32_t vtcm_src0_size_per_hvx; + uint32_t vtcm_act_size_per_hvx; + uint32_t vtcm_dst_size_per_hvx; }; static int htp_mm_init_context( struct htp_ops_context * octx, const struct htp_mm_kernel_params * kparams ) { - if (!htp_ops_context_set_n_threads(octx, (uint32_t) kparams->n_threads)) { + if (!htp_ops_context_set_n_hvx(octx, (uint32_t) kparams->n_hvx)) { return HTP_STATUS_INVAL_PARAMS; } if (kparams->n_hmx) { - if (kparams->n_act_threads <= 0 || kparams->n_act_threads > (int32_t) octx->n_threads) { + if (kparams->n_act_hvx <= 0 || kparams->n_act_hvx > (int32_t) octx->n_hvx) { return HTP_STATUS_INVAL_PARAMS; } } @@ -236,8 +236,8 @@ static const uint8_t __attribute__((aligned(VLEN))) kvalues_mxfp4_lut[] = { #define htp_matmul_preamble \ struct htp_mm_context * mmctx = data; \ struct htp_ops_context * octx = mmctx->octx; \ - dma_queue * dma_q = octx->ctx->dma[ith]; \ - uint32_t src0_nrows_per_thread = mmctx->src0_nrows_per_thread; \ + dma_queue * dma_q = th->dma_q; \ + uint32_t src0_nrows_per_hvx = mmctx->src0_nrows_per_hvx; \ htp_matmul_tensors_preamble; @@ -248,17 +248,17 @@ static const uint8_t __attribute__((aligned(VLEN))) kvalues_mxfp4_lut[] = { #include "hvx-mm-kernels-float.h" #include "hmx-mm-kernels-tiled.h" -static void hvx_mm_2d_tiled(unsigned int nth, unsigned int ith, void * data) { +static void hvx_mm_2d_tiled(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { htp_matmul_preamble; const uint32_t src0_nrows = mmctx->src0_row_end - mmctx->src0_row_start; const uint32_t act_nrows = mmctx->cur_m_rows ? mmctx->cur_m_rows : (ne11 * ne12 * ne13); const uint32_t cur_m_start = mmctx->cur_m_start; - const uint32_t src0_start_row = mmctx->src0_row_start + src0_nrows_per_thread * ith; - const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, mmctx->src0_row_end); + const uint32_t src0_start_row = mmctx->src0_row_start + src0_nrows_per_hvx * ith; + const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_hvx, mmctx->src0_row_end); - struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + struct htp_thread_trace * tr = th->trace; const struct htp_mm_kernel_params * kparams = (const struct htp_mm_kernel_params *) octx->kernel_params; const uint32_t n_prefetch = kparams->n_prefetch; @@ -269,8 +269,8 @@ static void hvx_mm_2d_tiled(unsigned int nth, unsigned int ith, void * data) { const size_t act_stride = mmctx->vtcm_act_stride; const size_t src2_stride = src2 ? ((src2->ne[1] == 1) ? 0 : src2->nb[1]) : 0; - uint8_t * restrict vtcm_dst_ptr = mmctx->vtcm_dst + mmctx->vtcm_dst_size_per_thread * ith; - uint8_t * restrict vtcm_src0_ptr = mmctx->vtcm_src0 + mmctx->vtcm_src0_size_per_thread * ith; + uint8_t * restrict vtcm_dst_ptr = mmctx->vtcm_dst + mmctx->vtcm_dst_size_per_hvx * ith; + uint8_t * restrict vtcm_src0_ptr = mmctx->vtcm_src0 + mmctx->vtcm_src0_size_per_hvx * ith; uint8_t * restrict act_data = mmctx->vtcm_act; const dma_addr_t src0_row = src0->data; @@ -347,15 +347,15 @@ static void hvx_mm_2d_tiled(unsigned int nth, unsigned int ith, void * data) { } } -static void hvx_mv_2d_tiled(unsigned int nth, unsigned int ith, void * data) { +static void hvx_mv_2d_tiled(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { htp_matmul_preamble; const uint32_t src0_nrows = mmctx->src0_row_end - mmctx->src0_row_start; - const uint32_t src0_start_row = mmctx->src0_row_start + src0_nrows_per_thread * ith; - const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, mmctx->src0_row_end); + const uint32_t src0_start_row = mmctx->src0_row_start + src0_nrows_per_hvx * ith; + const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_hvx, mmctx->src0_row_end); - struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + struct htp_thread_trace * tr = th->trace; const struct htp_mm_kernel_params * kparams = (const struct htp_mm_kernel_params *) octx->kernel_params; const uint32_t n_prefetch = kparams->n_prefetch; @@ -365,8 +365,8 @@ static void hvx_mv_2d_tiled(unsigned int nth, unsigned int ith, void * data) { const size_t act_row_size = nb11; const size_t act_stride = mmctx->vtcm_act_stride; - uint8_t * vtcm_dst_ptr = mmctx->vtcm_dst + mmctx->vtcm_dst_size_per_thread * ith; - uint8_t * vtcm_src0_ptr = mmctx->vtcm_src0 + mmctx->vtcm_src0_size_per_thread * ith; + uint8_t * vtcm_dst_ptr = mmctx->vtcm_dst + mmctx->vtcm_dst_size_per_hvx * ith; + uint8_t * vtcm_src0_ptr = mmctx->vtcm_src0 + mmctx->vtcm_src0_size_per_hvx * ith; uint8_t * act_data = mmctx->vtcm_act; float * tmp = (float *) vtcm_dst_ptr; @@ -441,7 +441,7 @@ static void hvx_mv_2d_tiled(unsigned int nth, unsigned int ith, void * data) { } } -static void hvx_mm_nx_2d_tiled(unsigned int nth, unsigned int ith, void * data) { +static void hvx_mm_nx_2d_tiled(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { struct htp_mm_context * mmctx = data; struct htp_ops_context * octx = mmctx->octx; const struct htp_mm_kernel_params * kparams = (const struct htp_mm_kernel_params *) octx->kernel_params; @@ -452,10 +452,10 @@ static void hvx_mm_nx_2d_tiled(unsigned int nth, unsigned int ith, void * data) const uint32_t act_nrows = act->ne[1] * act->ne[2] * act->ne[3]; const size_t act_stride = mmctx->vtcm_act_stride; - uint8_t * restrict vtcm_weight_ptr = mmctx->vtcm_src0 + mmctx->vtcm_src0_size_per_thread * ith; + uint8_t * restrict vtcm_weight_ptr = mmctx->vtcm_src0 + mmctx->vtcm_src0_size_per_hvx * ith; uint8_t * restrict act_data = mmctx->vtcm_act; - struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + struct htp_thread_trace * tr = th->trace; const uint32_t n_prefetch = kparams->n_prefetch; assert(n_prefetch >= 2 && n_prefetch <= HTP_MM_MAX_PREFETCH && (n_prefetch & (n_prefetch - 1)) == 0); @@ -463,7 +463,7 @@ static void hvx_mm_nx_2d_tiled(unsigned int nth, unsigned int ith, void * data) const struct htp_tensor * restrict src_w = octx->src[widx]; const struct htp_tensor * restrict dst = octx->dsts[widx]; if (!src_w || !dst) continue; - dma_queue * dma_q = octx->ctx->dma[ith]; + dma_queue * dma_q = th->dma_q; const uint32_t tile_size = htp_tensor_get_tile_size(src_w->type); const uint32_t aligned_tile_size = htp_tensor_get_tile_padded_size(src_w->type); @@ -487,11 +487,11 @@ static void hvx_mm_nx_2d_tiled(unsigned int nth, unsigned int ith, void * data) } const uint32_t nrows = src0_end_row - src0_start_row; - uint32_t src0_nrows_per_thread = fastdiv(nrows + nth - 1, &octx->n_threads_div); - src0_nrows_per_thread = hex_round_up(src0_nrows_per_thread, 32); + uint32_t src0_nrows_per_hvx = fastdiv(nrows + nth - 1, &octx->n_hvx_div); + src0_nrows_per_hvx = hex_round_up(src0_nrows_per_hvx, 32); - const uint32_t start_row = src0_start_row + src0_nrows_per_thread * ith; - const uint32_t end_row = MIN(start_row + src0_nrows_per_thread, src0_end_row); + const uint32_t start_row = src0_start_row + src0_nrows_per_hvx * ith; + const uint32_t end_row = MIN(start_row + src0_nrows_per_hvx, src0_end_row); if (start_row >= end_row) continue; uint32_t ct_start = start_row / 32; @@ -612,25 +612,25 @@ static void hvx_mm_transfer_act_dma( } #define QUANTIZE_IMPL(name, log_name, kernel_fn, dst_row_size_expr) \ -static void name(unsigned int nth, unsigned int ith, void * data) { \ +static void name(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { \ (void) nth; \ struct htp_mm_context * mmctx = data; \ struct htp_ops_context * octx = mmctx->octx; \ const struct htp_tensor * src = mmctx->act; \ const uint32_t ne0 = src->ne[0]; \ const uint32_t nrows = mmctx->cur_m_rows ? mmctx->cur_m_rows : mmctx->act_nrows; \ - const uint32_t nrows_per_thread = mmctx->n_quant_rows_per_thread; \ + const uint32_t nrows_per_hvx = mmctx->n_quant_rows_per_hvx; \ \ - const uint32_t ir_first = nrows_per_thread * ith; \ + const uint32_t ir_first = nrows_per_hvx * ith; \ if (ir_first >= nrows) { \ return; \ } \ \ - struct htp_thread_trace * tr = &octx->ctx->trace[ith]; \ + struct htp_thread_trace * tr = th->trace; \ htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_A_QUANT, ir_first); \ \ uint8_t * restrict dst = mmctx->vtcm_act; \ - const uint32_t ir_last = MIN(ir_first + nrows_per_thread, nrows); \ + const uint32_t ir_last = MIN(ir_first + nrows_per_hvx, nrows); \ const size_t raw_row_size = mmctx->vtcm_act_raw_stride; \ const size_t dst_row_size = (dst_row_size_expr); \ \ @@ -650,14 +650,14 @@ QUANTIZE_IMPL(quantize_f32_f32, "quantize-f32-f32", quanti QUANTIZE_IMPL(quantize_f32_f16, "quantize-f32-f16", quantize_f32_f16_kernel, mmctx->vtcm_act_stride) QUANTIZE_IMPL(quantize_f16_f16, "quantize-f16-f16", quantize_f16_f16_kernel, mmctx->vtcm_act_stride) -static void quantize_f32_q8_0_tiled_block(unsigned int nth, unsigned int ith, void * data) { +static void quantize_f32_q8_0_tiled_block(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { (void) nth; struct htp_mm_context * mmctx = data; if (mmctx->quant_ib_first[ith] >= mmctx->quant_ib_last[ith]) { return; } struct htp_ops_context * octx = mmctx->octx; - struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + struct htp_thread_trace * tr = th->trace; htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_A_QUANT, mmctx->quant_ib_first[ith]); const struct htp_tensor * src = mmctx->act; @@ -678,14 +678,14 @@ static void quantize_f32_q8_0_tiled_block(unsigned int nth, unsigned int ith, vo htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_A_QUANT, mmctx->quant_ib_first[ith]); } -static void quantize_f32_q8_1_tiled_block(unsigned int nth, unsigned int ith, void * data) { +static void quantize_f32_q8_1_tiled_block(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { (void) nth; struct htp_mm_context * mmctx = data; if (mmctx->quant_ib_first[ith] >= mmctx->quant_ib_last[ith]) { return; } struct htp_ops_context * octx = mmctx->octx; - struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + struct htp_thread_trace * tr = th->trace; htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_A_QUANT, mmctx->quant_ib_first[ith]); const struct htp_tensor * src = mmctx->act; @@ -706,14 +706,14 @@ static void quantize_f32_q8_1_tiled_block(unsigned int nth, unsigned int ith, vo htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_A_QUANT, mmctx->quant_ib_first[ith]); } -static void quantize_f32_q8_1_s16_tiled_block(unsigned int nth, unsigned int ith, void * data) { +static void quantize_f32_q8_1_s16_tiled_block(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { (void) nth; struct htp_mm_context * mmctx = data; if (mmctx->quant_ib_first[ith] >= mmctx->quant_ib_last[ith]) { return; } struct htp_ops_context * octx = mmctx->octx; - struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + struct htp_thread_trace * tr = th->trace; htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_A_QUANT, mmctx->quant_ib_first[ith]); const struct htp_tensor * src = mmctx->act; @@ -734,14 +734,14 @@ static void quantize_f32_q8_1_s16_tiled_block(unsigned int nth, unsigned int ith htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_A_QUANT, mmctx->quant_ib_first[ith]); } -static void quantize_f32_f16_tiled_block(unsigned int nth, unsigned int ith, void * data) { +static void quantize_f32_f16_tiled_block(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { (void) nth; struct htp_mm_context * mmctx = data; if (mmctx->quant_ib_first[ith] >= mmctx->quant_ib_last[ith]) { return; } struct htp_ops_context * octx = mmctx->octx; - struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + struct htp_thread_trace * tr = th->trace; htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_A_QUANT, mmctx->quant_ib_first[ith]); const struct htp_tensor * src = mmctx->act; @@ -762,14 +762,14 @@ static void quantize_f32_f16_tiled_block(unsigned int nth, unsigned int ith, voi htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_A_QUANT, mmctx->quant_ib_first[ith]); } -static void quantize_f16_f16_tiled_block(unsigned int nth, unsigned int ith, void * data) { +static void quantize_f16_f16_tiled_block(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { (void) nth; struct htp_mm_context * mmctx = data; if (mmctx->quant_ib_first[ith] >= mmctx->quant_ib_last[ith]) { return; } struct htp_ops_context * octx = mmctx->octx; - struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + struct htp_thread_trace * tr = th->trace; htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_A_QUANT, mmctx->quant_ib_first[ith]); const struct htp_tensor * src = mmctx->act; @@ -791,7 +791,7 @@ static void quantize_f16_f16_tiled_block(unsigned int nth, unsigned int ith, voi } // q8_1 for weight types with offsets (q8_1_s16 for Q2_K), f16_tiled for F16, otherwise q8_0 -static inline worker_callback_t htp_mm_act_quant_row_func(int weight_type, int act_type) { +static inline work_queue_func_t htp_mm_act_quant_row_func(int weight_type, int act_type) { if (weight_type == HTP_TYPE_F16) { return (act_type == HTP_TYPE_F16) ? quantize_f16_f16_tiled : quantize_f32_f16_tiled; } @@ -801,7 +801,7 @@ static inline worker_callback_t htp_mm_act_quant_row_func(int weight_type, int a return htp_mm_weight_has_offset(weight_type) ? quantize_f32_q8_1_tiled : quantize_f32_q8_0_tiled; } -static inline worker_callback_t htp_mm_act_quant_block_func(int weight_type, int act_type) { +static inline work_queue_func_t htp_mm_act_quant_block_func(int weight_type, int act_type) { if (weight_type == HTP_TYPE_F16) { return (act_type == HTP_TYPE_F16) ? quantize_f16_f16_tiled_block : quantize_f32_f16_tiled_block; } @@ -811,17 +811,17 @@ static inline worker_callback_t htp_mm_act_quant_block_func(int weight_type, int return htp_mm_weight_has_offset(weight_type) ? quantize_f32_q8_1_tiled_block : quantize_f32_q8_0_tiled_block; } -static void hvx_mm_4d_tiled(unsigned int nth, unsigned int ith, void * data) { +static void hvx_mm_4d_tiled(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { htp_matmul_preamble; const uint32_t src0_nrows = mmctx->src0_row_end - mmctx->src0_row_start; const uint32_t cur_m_rows = mmctx->cur_m_rows ? mmctx->cur_m_rows : (ne11 * ne12 * ne13); const uint32_t cur_m_start = mmctx->cur_m_start; - const uint32_t src0_start_row = mmctx->src0_row_start + src0_nrows_per_thread * ith; - const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, mmctx->src0_row_end); + const uint32_t src0_start_row = mmctx->src0_row_start + src0_nrows_per_hvx * ith; + const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_hvx, mmctx->src0_row_end); - struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + struct htp_thread_trace * tr = th->trace; const struct htp_mm_kernel_params * kparams = (const struct htp_mm_kernel_params *) octx->kernel_params; const uint32_t n_prefetch = kparams->n_prefetch; @@ -830,7 +830,7 @@ static void hvx_mm_4d_tiled(unsigned int nth, unsigned int ith, void * data) { const size_t dst_row_size = nb1; const size_t act_stride = mmctx->vtcm_act_stride; - uint8_t * restrict vtcm_src0_ptr = mmctx->vtcm_src0 + mmctx->vtcm_src0_size_per_thread * ith; + uint8_t * restrict vtcm_src0_ptr = mmctx->vtcm_src0 + mmctx->vtcm_src0_size_per_hvx * ith; uint8_t * restrict act_data = mmctx->vtcm_act; const uint32_t tile_size = htp_tensor_get_tile_size(src0->type); @@ -935,7 +935,7 @@ static void hvx_mm_4d_tiled(unsigned int nth, unsigned int ith, void * data) { } } -static void hvx_mm_2d_flat(unsigned int nth, unsigned int ith, void * data) { +static void hvx_mm_2d_flat(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { htp_matmul_preamble; const struct htp_mm_kernel_params * kparams = (const struct htp_mm_kernel_params *) octx->kernel_params; @@ -947,11 +947,11 @@ static void hvx_mm_2d_flat(unsigned int nth, unsigned int ith, void * data) { const uint32_t act_nrows = mmctx->cur_m_rows ? mmctx->cur_m_rows : mmctx->act_nrows; const uint32_t cur_m_start = mmctx->cur_m_start; - const uint32_t src0_start_row = mmctx->src0_row_start + src0_nrows_per_thread * ith; - const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, mmctx->src0_row_end); + const uint32_t src0_start_row = mmctx->src0_row_start + src0_nrows_per_hvx * ith; + const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_hvx, mmctx->src0_row_end); const uint32_t src0_end_row_x2 = src0_start_row + ((src0_end_row - src0_start_row) & ~1U); - struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + struct htp_thread_trace * tr = th->trace; const size_t dst_row_size = nb1; const size_t src0_row_size = nb01; @@ -961,8 +961,8 @@ static void hvx_mm_2d_flat(unsigned int nth, unsigned int ith, void * data) { const size_t act_stride = mmctx->vtcm_act_stride; // Per-thread VTCMs for all tensors - uint8_t * restrict vtcm_dst_ptr = mmctx->vtcm_dst + mmctx->vtcm_dst_size_per_thread * ith; - uint8_t * restrict vtcm_src0_ptr = mmctx->vtcm_src0 + mmctx->vtcm_src0_size_per_thread * ith; + uint8_t * restrict vtcm_dst_ptr = mmctx->vtcm_dst + mmctx->vtcm_dst_size_per_hvx * ith; + uint8_t * restrict vtcm_src0_ptr = mmctx->vtcm_src0 + mmctx->vtcm_src0_size_per_hvx * ith; uint8_t * restrict act_data = mmctx->vtcm_act; const dma_addr_t src0_row = src0->data; @@ -1038,15 +1038,15 @@ static void hvx_mm_2d_flat(unsigned int nth, unsigned int ith, void * data) { } } -static void hvx_mv_2d_flat(unsigned int nth, unsigned int ith, void * data) { +static void hvx_mv_2d_flat(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { htp_matmul_preamble; const uint32_t src0_nrows = mmctx->src0_row_end - mmctx->src0_row_start; - const uint32_t src0_start_row = mmctx->src0_row_start + src0_nrows_per_thread * ith; - const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, mmctx->src0_row_end); + const uint32_t src0_start_row = mmctx->src0_row_start + src0_nrows_per_hvx * ith; + const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_hvx, mmctx->src0_row_end); - struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + struct htp_thread_trace * tr = th->trace; const size_t dst_row_size = nb1; const size_t src0_row_size = nb01; @@ -1056,8 +1056,8 @@ static void hvx_mv_2d_flat(unsigned int nth, unsigned int ith, void * data) { const size_t act_stride = mmctx->vtcm_act_stride; // Per-thread VTCMs for all tensors - uint8_t * vtcm_dst_ptr = mmctx->vtcm_dst + mmctx->vtcm_dst_size_per_thread * ith; - uint8_t * vtcm_src0_ptr = mmctx->vtcm_src0 + mmctx->vtcm_src0_size_per_thread * ith; + uint8_t * vtcm_dst_ptr = mmctx->vtcm_dst + mmctx->vtcm_dst_size_per_hvx * ith; + uint8_t * vtcm_src0_ptr = mmctx->vtcm_src0 + mmctx->vtcm_src0_size_per_hvx * ith; uint8_t * act_data = mmctx->vtcm_act; float * tmp = (float *) vtcm_dst_ptr; @@ -1143,7 +1143,7 @@ static void hvx_mv_2d_flat(unsigned int nth, unsigned int ith, void * data) { } } -static void hvx_mm_4d_flat(unsigned int nth, unsigned int ith, void * data) { +static void hvx_mm_4d_flat(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { htp_matmul_preamble; const struct htp_mm_kernel_params * kparams = (const struct htp_mm_kernel_params *) octx->kernel_params; @@ -1155,18 +1155,18 @@ static void hvx_mm_4d_flat(unsigned int nth, unsigned int ith, void * data) { const uint32_t cur_m_rows = mmctx->cur_m_rows ? mmctx->cur_m_rows : (ne11 * ne12 * ne13); const uint32_t cur_m_start = mmctx->cur_m_start; - const uint32_t src0_start_row = mmctx->src0_row_start + src0_nrows_per_thread * ith; - const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, mmctx->src0_row_end); + const uint32_t src0_start_row = mmctx->src0_row_start + src0_nrows_per_hvx * ith; + const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_hvx, mmctx->src0_row_end); const uint32_t src0_end_row_x2 = src0_start_row + ((src0_end_row - src0_start_row) & ~1U); - struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + struct htp_thread_trace * tr = th->trace; const size_t dst_row_size = nb1; const size_t src0_row_size = nb01; const size_t src0_stride = mmctx->vtcm_src0_stride; const size_t act_stride = mmctx->vtcm_act_stride; - uint8_t * restrict vtcm_src0_ptr = mmctx->vtcm_src0 + mmctx->vtcm_src0_size_per_thread * ith; + uint8_t * restrict vtcm_src0_ptr = mmctx->vtcm_src0 + mmctx->vtcm_src0_size_per_hvx * ith; uint8_t * restrict act_data = mmctx->vtcm_act; @@ -1292,22 +1292,22 @@ static void hvx_mm_4d_flat(unsigned int nth, unsigned int ith, void * data) { #define MMID_MATRIX_ROW(row_id, i1) matrix_rows[(row_id) * mmctx->mapping_stride + (i1)] -static void hvx_mm_id_tiled(unsigned int nth, unsigned int ith, void * data) { +static void hvx_mm_id_tiled(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { htp_matmul_preamble; const struct htp_tensor * restrict ids = octx->src[2]; const uint32_t src0_nrows = mmctx->src0_row_end - mmctx->src0_row_start; // src0 rows per expert const uint32_t act_nrows = ne11; - const uint32_t src0_start_row = mmctx->src0_row_start + src0_nrows_per_thread * ith; - const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, mmctx->src0_row_end); + const uint32_t src0_start_row = mmctx->src0_row_start + src0_nrows_per_hvx * ith; + const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_hvx, mmctx->src0_row_end); if (src0_start_row >= src0_end_row) { return; } - struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + struct htp_thread_trace * tr = th->trace; const struct htp_mm_kernel_params * kparams = (const struct htp_mm_kernel_params *) octx->kernel_params; const uint32_t n_prefetch = kparams->n_prefetch; @@ -1325,7 +1325,7 @@ static void hvx_mm_id_tiled(unsigned int nth, unsigned int ith, void * data) { const size_t act_stride = mmctx->vtcm_act_stride; // Per-thread VTCMs for all tensors - uint8_t * restrict vtcm_src0_ptr = mmctx->vtcm_src0 + mmctx->vtcm_src0_size_per_thread * ith; + uint8_t * restrict vtcm_src0_ptr = mmctx->vtcm_src0 + mmctx->vtcm_src0_size_per_hvx * ith; uint8_t * restrict act_data = mmctx->vtcm_act; for (uint32_t cur_a = 0; cur_a < n_as; ++cur_a) { @@ -1380,21 +1380,21 @@ static void hvx_mm_id_tiled(unsigned int nth, unsigned int ith, void * data) { } } -static void hvx_mv_id_tiled(unsigned int nth, unsigned int ith, void * data) { +static void hvx_mv_id_tiled(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { htp_matmul_preamble; const struct htp_tensor * restrict ids = octx->src[2]; const uint32_t src0_nrows = mmctx->src0_row_end - mmctx->src0_row_start; // src0 rows per expert - const uint32_t src0_start_row = mmctx->src0_row_start + src0_nrows_per_thread * ith; - const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, mmctx->src0_row_end); + const uint32_t src0_start_row = mmctx->src0_row_start + src0_nrows_per_hvx * ith; + const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_hvx, mmctx->src0_row_end); if (src0_start_row >= src0_end_row) { return; } - struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + struct htp_thread_trace * tr = th->trace; const struct htp_mm_kernel_params * kparams = (const struct htp_mm_kernel_params *) octx->kernel_params; const uint32_t n_prefetch = kparams->n_prefetch; @@ -1409,7 +1409,7 @@ static void hvx_mv_id_tiled(unsigned int nth, unsigned int ith, void * data) { const uint32_t n_ids = ne02; // num experts // Per-thread VTCMs for all tensors - uint8_t * restrict vtcm_src0_ptr = mmctx->vtcm_src0 + mmctx->vtcm_src0_size_per_thread * ith; + uint8_t * restrict vtcm_src0_ptr = mmctx->vtcm_src0 + mmctx->vtcm_src0_size_per_hvx * ith; uint8_t * restrict act_data = mmctx->vtcm_act; for (uint32_t ie1 = 0; ie1 < n_aids; ++ie1) { // for each expert @@ -1457,7 +1457,7 @@ static void hvx_mv_id_tiled(unsigned int nth, unsigned int ith, void * data) { } } -static void hvx_mv_id_nx_tiled(unsigned int nth, unsigned int ith, void * data) { +static void hvx_mv_id_nx_tiled(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { struct htp_mm_context * mmctx = (struct htp_mm_context *) data; struct htp_ops_context * octx = mmctx->octx; const struct htp_mm_kernel_params * kparams = (const struct htp_mm_kernel_params *) octx->kernel_params; @@ -1467,7 +1467,7 @@ static void hvx_mv_id_nx_tiled(unsigned int nth, unsigned int ith, void * data) const struct htp_tensor * restrict ids = octx->src[n_weights + 1]; - struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + struct htp_thread_trace * tr = th->trace; const uint32_t n_prefetch = kparams->n_prefetch; assert(n_prefetch >= 2 && n_prefetch <= HTP_MM_MAX_PREFETCH && (n_prefetch & (n_prefetch - 1)) == 0); @@ -1475,7 +1475,7 @@ static void hvx_mv_id_nx_tiled(unsigned int nth, unsigned int ith, void * data) const uint32_t n_aids = ids->ne[0]; const uint32_t n_ids = src0->ne[2]; - uint8_t * restrict vtcm_src0_ptr = mmctx->vtcm_src0 + mmctx->vtcm_src0_size_per_thread * ith; + uint8_t * restrict vtcm_src0_ptr = mmctx->vtcm_src0 + mmctx->vtcm_src0_size_per_hvx * ith; uint8_t * restrict act_data = mmctx->vtcm_act; for (uint32_t ie1 = 0; ie1 < n_aids; ++ie1) { @@ -1487,7 +1487,7 @@ static void hvx_mv_id_nx_tiled(unsigned int nth, unsigned int ith, void * data) const struct htp_tensor * restrict src_w = octx->src[p]; const struct htp_tensor * restrict dst = octx->dsts[p]; if (!src_w || !dst) continue; - dma_queue * dma_q = octx->ctx->dma[ith]; + dma_queue * dma_q = th->dma_q; const uint32_t ne01 = src_w->ne[1]; uint32_t start_row = 0; @@ -1500,11 +1500,11 @@ static void hvx_mv_id_nx_tiled(unsigned int nth, unsigned int ith, void * data) } const uint32_t nrows = end_row - start_row; - uint32_t src0_nrows_per_thread = fastdiv(nrows + nth - 1, &octx->n_threads_div); - src0_nrows_per_thread = hex_round_up(src0_nrows_per_thread, 32); + uint32_t src0_nrows_per_hvx = fastdiv(nrows + nth - 1, &octx->n_hvx_div); + src0_nrows_per_hvx = hex_round_up(src0_nrows_per_hvx, 32); - const uint32_t src0_start_row = start_row + src0_nrows_per_thread * ith; - const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, end_row); + const uint32_t src0_start_row = start_row + src0_nrows_per_hvx * ith; + const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_hvx, end_row); if (src0_start_row >= src0_end_row) continue; const dma_addr_t src0_row = src_w->data + eid * src_w->nb[2]; @@ -1546,7 +1546,7 @@ static void hvx_mv_id_nx_tiled(unsigned int nth, unsigned int ith, void * data) } } -static void hvx_mm_id_nx_tiled(unsigned int nth, unsigned int ith, void * data) { +static void hvx_mm_id_nx_tiled(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { struct htp_mm_context * mmctx = (struct htp_mm_context *) data; struct htp_ops_context * octx = mmctx->octx; const struct htp_mm_kernel_params * kparams = (const struct htp_mm_kernel_params *) octx->kernel_params; @@ -1556,7 +1556,7 @@ static void hvx_mm_id_nx_tiled(unsigned int nth, unsigned int ith, void * data) const struct htp_tensor * restrict ids = octx->src[n_weights + 1]; - struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + struct htp_thread_trace * tr = th->trace; const uint32_t n_prefetch = kparams->n_prefetch; assert(n_prefetch >= 2 && n_prefetch <= HTP_MM_MAX_PREFETCH && (n_prefetch & (n_prefetch - 1)) == 0); @@ -1568,7 +1568,7 @@ static void hvx_mm_id_nx_tiled(unsigned int nth, unsigned int ith, void * data) const size_t act_stride = mmctx->vtcm_act_stride; - uint8_t * restrict vtcm_src0_ptr = mmctx->vtcm_src0 + mmctx->vtcm_src0_size_per_thread * ith; + uint8_t * restrict vtcm_src0_ptr = mmctx->vtcm_src0 + mmctx->vtcm_src0_size_per_hvx * ith; uint8_t * restrict act_data = mmctx->vtcm_act; for (uint32_t cur_a = 0; cur_a < n_as; ++cur_a) { @@ -1579,7 +1579,7 @@ static void hvx_mm_id_nx_tiled(unsigned int nth, unsigned int ith, void * data) const struct htp_tensor * restrict src_w = octx->src[p]; const struct htp_tensor * restrict dst = octx->dsts[p]; if (!src_w || !dst) continue; - dma_queue * dma_q = octx->ctx->dma[ith]; + dma_queue * dma_q = th->dma_q; const uint32_t ne01 = src_w->ne[1]; uint32_t start_row = 0; @@ -1592,11 +1592,11 @@ static void hvx_mm_id_nx_tiled(unsigned int nth, unsigned int ith, void * data) } const uint32_t nrows = end_row - start_row; - uint32_t src0_nrows_per_thread = fastdiv(nrows + nth - 1, &octx->n_threads_div); - src0_nrows_per_thread = hex_round_up(src0_nrows_per_thread, 32); + uint32_t src0_nrows_per_hvx = fastdiv(nrows + nth - 1, &octx->n_hvx_div); + src0_nrows_per_hvx = hex_round_up(src0_nrows_per_hvx, 32); - const uint32_t src0_start_row = start_row + src0_nrows_per_thread * ith; - const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, end_row); + const uint32_t src0_start_row = start_row + src0_nrows_per_hvx * ith; + const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_hvx, end_row); if (src0_start_row >= src0_end_row) continue; const dma_addr_t src0_row = src_w->data + cur_a * src_w->nb[2]; @@ -1664,6 +1664,12 @@ static int hvx_mm_init_tiled_vec_dot(struct htp_mm_context * mmctx, enum htp_dat mmctx->vec_dot_32x1 = tiled_vec_dot_q8_0_32x1; mmctx->vec_dot_32x2 = tiled_vec_dot_q8_0_32x2; return 0; + case HTP_TYPE_Q5_0: + mmctx->type = "q5_0_tiled-f32"; + mmctx->vec_dot_32x1 = tiled_vec_dot_q5_0_32x1; + mmctx->vec_dot_32x2 = tiled_vec_dot_q5_0_32x2; + return 0; + case HTP_TYPE_Q5_1: case HTP_TYPE_Q5_K: mmctx->type = "q5_k_tiled-f32"; mmctx->vec_dot_32x1 = tiled_vec_dot_q5_k_32x1; @@ -1751,12 +1757,12 @@ static int hvx_mm_matmul(struct htp_ops_context * octx) { const bool is_tiled = htp_tensor_is_tiled(src0); - // Compute src0_nrows_per_thread - mmctx->src0_nrows_per_thread = fastdiv(nrows + octx->n_threads - 1, &octx->n_threads_div); + // Compute src0_nrows_per_hvx + mmctx->src0_nrows_per_hvx = fastdiv(nrows + octx->n_hvx - 1, &octx->n_hvx_div); if (is_tiled) { - mmctx->src0_nrows_per_thread = hex_round_up(mmctx->src0_nrows_per_thread, 32); + mmctx->src0_nrows_per_hvx = hex_round_up(mmctx->src0_nrows_per_hvx, 32); } else { - mmctx->src0_nrows_per_thread += (mmctx->src0_nrows_per_thread & 1); // round up to even + mmctx->src0_nrows_per_hvx += (mmctx->src0_nrows_per_hvx & 1); // round up to even } const size_t src0_row_size = nb01; @@ -1765,8 +1771,8 @@ static int hvx_mm_matmul(struct htp_ops_context * octx) { const size_t src0_row_size_padded = hex_round_up(src0_row_size, 128); - worker_callback_t quant_task_func; - worker_callback_t matmul_job_func; + work_queue_func_t quant_task_func; + work_queue_func_t matmul_job_func; uint32_t n_quant_tasks = 1; const bool is_batched = (ne12 > 1 || ne13 > 1 || ne02 > 1 || ne03 > 1); if (is_batched) { @@ -1821,8 +1827,8 @@ static int hvx_mm_matmul(struct htp_ops_context * octx) { const uint32_t nb = (ne10 + qk - 1) / qk; const uint32_t total_nb = act_nrows * nb; - if (act_nrows < octx->n_threads && !is_batched) { - n_quant_tasks = MIN(total_nb, octx->n_threads); + if (act_nrows < octx->n_hvx && !is_batched) { + n_quant_tasks = MIN(total_nb, octx->n_hvx); quant_task_func = htp_mm_act_quant_block_func(src0->type, act->type); for (uint32_t ith = 0; ith < n_quant_tasks; ++ith) { uint32_t ib_first = (total_nb * ith) / n_quant_tasks; @@ -1833,7 +1839,7 @@ static int hvx_mm_matmul(struct htp_ops_context * octx) { mmctx->quant_c[ith] = ib_first % nb; } } else { - n_quant_tasks = MIN(act_nrows, octx->n_threads); + n_quant_tasks = MIN(act_nrows, octx->n_hvx); quant_task_func = htp_mm_act_quant_row_func(src0->type, act->type); } act_row_size = kparams->act_row_size; @@ -1846,7 +1852,7 @@ static int hvx_mm_matmul(struct htp_ops_context * octx) { const uint32_t m_layout_rows = m_chunk; struct htp_mm_hvx_vtcm_layout L; - htp_mm_hvx_vtcm_layout_build(&L, kparams->kernel_type, src0->type, ne10, m_layout_rows, octx->n_threads, + htp_mm_hvx_vtcm_layout_build(&L, kparams->kernel_type, src0->type, ne10, m_layout_rows, octx->n_hvx, dst_row_size, src0_row_size, act_row_size, src2 ? src2->nb[1] : 0, kparams->n_prefetch, false, false); if (kparams->kernel_type == HTP_MM_KERNEL_HVX_F16_F16_VTCM || @@ -1854,13 +1860,13 @@ static int hvx_mm_matmul(struct htp_ops_context * octx) { kparams->kernel_type == HTP_MM_KERNEL_HVX_BF16_F32_VTCM || kparams->kernel_type == HTP_MM_KERNEL_HVX_QUANT_ROW || kparams->kernel_type == HTP_MM_KERNEL_HVX_QUANT_BLOCK) { - mmctx->vtcm_act_size_per_thread = L.act_bytes; + mmctx->vtcm_act_size_per_hvx = L.act_bytes; } else { - mmctx->vtcm_act_size_per_thread = fastdiv(L.act_bytes, &octx->n_threads_div); + mmctx->vtcm_act_size_per_hvx = fastdiv(L.act_bytes, &octx->n_hvx_div); } - mmctx->vtcm_src0_size_per_thread = fastdiv(L.src0_bytes, &octx->n_threads_div); - mmctx->vtcm_dst_size_per_thread = fastdiv(L.dst_bytes, &octx->n_threads_div); + mmctx->vtcm_src0_size_per_hvx = fastdiv(L.src0_bytes, &octx->n_hvx_div); + mmctx->vtcm_dst_size_per_hvx = fastdiv(L.dst_bytes, &octx->n_hvx_div); const size_t vtcm_size = L.total_bytes; @@ -1910,8 +1916,8 @@ static int hvx_mm_matmul(struct htp_ops_context * octx) { const uint32_t total_nb = cur_m_rows * nb; uint32_t quant_tasks; work_queue_func_t q_func; - if (cur_m_rows < octx->n_threads && (kparams->kernel_type == HTP_MM_KERNEL_HVX_QUANT_BLOCK || kparams->kernel_type == HTP_MM_KERNEL_HVX_QUANT_ROW)) { - quant_tasks = MIN(total_nb, octx->n_threads); + if (cur_m_rows < octx->n_hvx && (kparams->kernel_type == HTP_MM_KERNEL_HVX_QUANT_BLOCK || kparams->kernel_type == HTP_MM_KERNEL_HVX_QUANT_ROW)) { + quant_tasks = MIN(total_nb, octx->n_hvx); q_func = htp_mm_act_quant_block_func(src0->type, act->type); for (uint32_t ith = 0; ith < quant_tasks; ++ith) { uint32_t ib_first = (total_nb * ith) / quant_tasks; @@ -1922,17 +1928,17 @@ static int hvx_mm_matmul(struct htp_ops_context * octx) { mmctx->quant_c[ith] = ib_first % nb; } } else { - quant_tasks = MIN(cur_m_rows, octx->n_threads); + quant_tasks = MIN(cur_m_rows, octx->n_hvx); q_func = quant_task_func; - mmctx->n_quant_rows_per_thread = (cur_m_rows + quant_tasks - 1) / quant_tasks; + mmctx->n_quant_rows_per_hvx = (cur_m_rows + quant_tasks - 1) / quant_tasks; } mmctx->n_quant_tasks = quant_tasks; - work_queue_run(octx->ctx->work_queue, q_func, mmctx, quant_tasks); + work_queue_run(octx->ctx->hvx_queue, q_func, mmctx, quant_tasks); } else { hvx_mm_transfer_act_dma(octx, kparams, act, mmctx->vtcm_act, mmctx->vtcm_act_stride, m_start, cur_m_rows); } - work_queue_run(octx->ctx->work_queue, matmul_job_func, mmctx, octx->n_threads); + work_queue_run(octx->ctx->hvx_queue, matmul_job_func, mmctx, octx->n_hvx); } } else { mmctx->cur_m_start = 0; @@ -1940,22 +1946,22 @@ static int hvx_mm_matmul(struct htp_ops_context * octx) { if (need_quant) { hvx_mm_transfer_act_dma(octx, kparams, act, mmctx->vtcm_act_raw, mmctx->vtcm_act_raw_stride, 0, act_nrows); - mmctx->n_quant_rows_per_thread = (act_nrows + n_quant_tasks - 1) / n_quant_tasks; + mmctx->n_quant_rows_per_hvx = (act_nrows + n_quant_tasks - 1) / n_quant_tasks; mmctx->n_quant_tasks = n_quant_tasks; - work_queue_run(octx->ctx->work_queue, quant_task_func, mmctx, n_quant_tasks); + work_queue_run(octx->ctx->hvx_queue, quant_task_func, mmctx, n_quant_tasks); } else { hvx_mm_transfer_act_dma(octx, kparams, act, mmctx->vtcm_act, mmctx->vtcm_act_stride, 0, act_nrows); } - work_queue_run(octx->ctx->work_queue, matmul_job_func, mmctx, octx->n_threads); + work_queue_run(octx->ctx->hvx_queue, matmul_job_func, mmctx, octx->n_hvx); } return HTP_STATUS_OK; } -static void dequantize_tiled_worker_loop(unsigned int n, unsigned int i, void * data) { +static void dequantize_tiled_worker_loop(const htp_thread_t * th, unsigned int n, unsigned int i, void * data) { tiled_dequantize_state_t * state = (tiled_dequantize_state_t *) data; - struct htp_thread_trace * tr = &state->traces[i]; + struct htp_thread_trace * tr = th->trace; htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_W_DEQUANT, i); for (unsigned int task_id = i; task_id < (unsigned int) state->n_tasks; task_id += n) { int start = task_id * state->n_tiles_per_task; @@ -1965,10 +1971,10 @@ static void dequantize_tiled_worker_loop(unsigned int n, unsigned int i, void * htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_W_DEQUANT, i); } -static void transfer_output_chunk_worker_fn(unsigned int n, unsigned int i, void *data) { +static void transfer_output_chunk_worker_fn(const htp_thread_t * th, unsigned int n, unsigned int i, void *data) { output_transfer_task_state_t *st = (output_transfer_task_state_t *) data; - struct htp_thread_trace * tr = &st->traces[i]; + struct htp_thread_trace * tr = th->trace; int start_chunk_idx = i * st->n_chunks_per_task; htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_O_PROC, start_chunk_idx); @@ -1986,8 +1992,6 @@ static void transfer_output_chunk_worker_fn(unsigned int n, unsigned int i, void } typedef struct { - struct htp_context * ctx; - struct htp_thread_trace * traces; __fp16 * dst; dma_addr_t act_dma_addr; float * vtcm_f32_act; @@ -1997,15 +2001,13 @@ typedef struct { uint32_t k_block; uint32_t k_stride; uint32_t k_valid; - size_t vtcm_f32_act_bytes_per_thread; + size_t vtcm_f32_act_bytes_per_hvx; uint32_t dma_step_rows; uint32_t dma_step_rows_shift; uint32_t act_elem_size; // 4=F32 src1, 2=F16 src1 } activation_transfer_task_state_t; typedef struct { - struct htp_context * ctx; - struct htp_thread_trace * traces; __fp16 * dst; dma_addr_t act_dma_addr; float * vtcm_f32_act; @@ -2014,7 +2016,7 @@ typedef struct { uint32_t k_stride; uint32_t k_valid; uint32_t n_col_chunks; - struct fastdiv_values n_threads_div; + struct fastdiv_values n_hvx_div; size_t vtcm_f32_act_bytes; uint32_t dma_step_rows; uint32_t dma_step_rows_shift; @@ -2088,16 +2090,16 @@ static void transfer_activation_chunk_fp32_to_fp16_dma_pipelined_col_chunk( } } -static void transfer_activation_chunk_col_chunk_worker_fn(unsigned int n, unsigned int i, void *data) { +static void transfer_activation_chunk_col_chunk_worker_fn(const htp_thread_t * th, unsigned int n, unsigned int i, void *data) { activation_transfer_col_chunk_state_t *st = (activation_transfer_col_chunk_state_t *) data; - struct htp_thread_trace * tr = &st->traces[i]; + struct htp_thread_trace * tr = th->trace; uint32_t n_blocks = st->k_block / 32; - uint32_t b_first = fastdiv(n_blocks * i, &st->n_threads_div); - uint32_t b_last = fastdiv(n_blocks * (i + 1), &st->n_threads_div); + uint32_t b_first = fastdiv(n_blocks * i, &st->n_hvx_div); + uint32_t b_last = fastdiv(n_blocks * (i + 1), &st->n_hvx_div); uint32_t c_first = b_first * 32; - uint32_t c_last = b_last * 32; - uint32_t c_len = c_last - c_first; + uint32_t c_last = b_last * 32; + uint32_t c_len = c_last - c_first; if (c_len == 0) { return; @@ -2110,11 +2112,11 @@ static void transfer_activation_chunk_col_chunk_worker_fn(unsigned int n, unsign __fp16 *dst = st->dst; - size_t thread_scratch_bytes = hex_align_down(fastdiv(st->vtcm_f32_act_bytes, &st->n_threads_div), 128); + size_t thread_scratch_bytes = hex_align_down(fastdiv(st->vtcm_f32_act_bytes, &st->n_hvx_div), 128); float *thread_f32_act = (float *)((char *)st->vtcm_f32_act + i * thread_scratch_bytes); transfer_activation_chunk_fp32_to_fp16_dma_pipelined_col_chunk( - st->ctx->dma[i], dst, st->act_dma_addr, st->n_rows, st->k_block, st->k_stride, k_chunk_valid, + th->dma_q, dst, st->act_dma_addr, st->n_rows, st->k_block, st->k_stride, k_chunk_valid, c_first, c_len, thread_f32_act, tr, st->dma_step_rows, st->dma_step_rows_shift ); } @@ -2202,10 +2204,10 @@ static void transfer_activation_chunk_to_fp16_dma_pipelined( } } -static void transfer_activation_chunk_worker_fn(unsigned int n, unsigned int i, void *data) { +static void transfer_activation_chunk_worker_fn(const htp_thread_t * th, unsigned int n, unsigned int i, void *data) { activation_transfer_task_state_t *st = (activation_transfer_task_state_t *) data; - struct htp_thread_trace * tr = &st->traces[i]; + struct htp_thread_trace * tr = th->trace; for (unsigned int task_id = i; task_id < (unsigned int)st->n_tasks; task_id += n) { int chunk_idx = task_id * st->n_chunks_per_task; @@ -2214,15 +2216,14 @@ static void transfer_activation_chunk_worker_fn(unsigned int n, unsigned int i, __fp16 *dst = st->dst + chunk_idx * st->k_block; const dma_addr_t act_dma_addr = st->act_dma_addr + (size_t) chunk_idx * st->k_stride * st->act_elem_size; - uint8_t *thread_act = (uint8_t *) st->vtcm_f32_act + i * st->vtcm_f32_act_bytes_per_thread; + uint8_t *thread_act = (uint8_t *) st->vtcm_f32_act + i * st->vtcm_f32_act_bytes_per_hvx; transfer_activation_chunk_to_fp16_dma_pipelined( - st->ctx->dma[i], dst, act_dma_addr, chunk_size, st->k_block, st->k_stride, st->k_valid, thread_act, st->act_elem_size, tr, st->dma_step_rows, st->dma_step_rows_shift + th->dma_q, dst, act_dma_addr, chunk_size, st->k_block, st->k_stride, st->k_valid, thread_act, st->act_elem_size, tr, st->dma_step_rows, st->dma_step_rows_shift ); } } typedef struct { - struct htp_thread_trace * traces; const struct mmid_row_mapping * matrix_rows; __fp16 * dst; const float * src; @@ -2242,7 +2243,6 @@ typedef struct { } activation_transfer_gathered_task_state_t; typedef struct { - struct htp_thread_trace * traces; const struct mmid_row_mapping * matrix_rows; const __fp16 * vtcm_src; float * dst; @@ -2258,9 +2258,9 @@ typedef struct { uint32_t cne1; } output_transfer_scattered_task_state_t; -static void transfer_activation_chunk_gathered_worker_fn(unsigned int n, unsigned int i, void *data) { +static void transfer_activation_chunk_gathered_worker_fn(const htp_thread_t * th, unsigned int n, unsigned int i, void *data) { activation_transfer_gathered_task_state_t *st = data; - struct htp_thread_trace * tr = &st->traces[i]; + struct htp_thread_trace * tr = th->trace; int chunk_idx = i; int chunk_size = st->n_chunks_per_task; int vtcm_start_row = chunk_idx * chunk_size; @@ -2276,9 +2276,9 @@ static void transfer_activation_chunk_gathered_worker_fn(unsigned int n, unsigne } } -static void transfer_activation_chunk_gathered_worker_flat_fn(unsigned int n, unsigned int i, void *data) { +static void transfer_activation_chunk_gathered_worker_flat_fn(const htp_thread_t * th, unsigned int n, unsigned int i, void *data) { activation_transfer_gathered_task_state_t *st = data; - struct htp_thread_trace * tr = &st->traces[i]; + struct htp_thread_trace * tr = th->trace; int chunk_idx = i; int chunk_size = st->n_chunks_per_task; int vtcm_start_row = chunk_idx * chunk_size; @@ -2294,9 +2294,9 @@ static void transfer_activation_chunk_gathered_worker_flat_fn(unsigned int n, un } } -static void transfer_output_chunk_scattered_worker_fn(unsigned int n, unsigned int i, void *data) { +static void transfer_output_chunk_scattered_worker_fn(const htp_thread_t * th, unsigned int n, unsigned int i, void *data) { output_transfer_scattered_task_state_t *st = data; - struct htp_thread_trace * tr = &st->traces[i]; + struct htp_thread_trace * tr = th->trace; int chunk_idx = i; int chunk_size = st->n_chunks_per_task; int vtcm_start_row = chunk_idx * chunk_size; @@ -2322,6 +2322,8 @@ static inline tiled_dequant_task_fn_t hmx_get_dequant_worker_fn(int weight_type, case HTP_TYPE_Q4_K: return dequantize_tiled_weight_to_fp16_task_q4_1; case HTP_TYPE_MXFP4: return dequantize_tiled_weight_to_fp16_task_mxfp4; case HTP_TYPE_Q8_0: return dequantize_tiled_weight_to_fp16_task_q8_0; + case HTP_TYPE_Q5_0: return dequantize_tiled_weight_to_fp16_task_q5_0; + case HTP_TYPE_Q5_1: case HTP_TYPE_Q5_K: return dequantize_tiled_weight_to_fp16_task_q5_k; case HTP_TYPE_Q6_K: return dequantize_tiled_weight_to_fp16_task_q6_k; case HTP_TYPE_Q3_K: return dequantize_tiled_weight_to_fp16_task_q3_k; @@ -2341,7 +2343,7 @@ static void dequantize_tiled_weight_chunk_to_fp16_tiles( int n_cols, int k_block, size_t row_stride, int weight_type, int n_k_tiles, struct fastdiv_values n_k_tiles_div, - tiled_dequant_task_fn_t dequant_worker_fn, int n_threads) { + tiled_dequant_task_fn_t dequant_worker_fn, int n_hvx) { if (!dequant_worker_fn) { return; @@ -2353,7 +2355,7 @@ static void dequantize_tiled_weight_chunk_to_fp16_tiles( const uint32_t n_weight_tiles = k_block / htp_tensor_get_tile_k(weight_type); const size_t n_tot_tiles = n_col_tiles * n_weight_tiles; - size_t n_tiles_per_task = (n_threads == 1) ? n_tot_tiles : hmx_ceil_div(n_tot_tiles, n_threads); + size_t n_tiles_per_task = (n_hvx == 1) ? n_tot_tiles : hmx_ceil_div(n_tot_tiles, n_hvx); tiled_dequantize_state_t state; state.n_tasks = (n_tot_tiles + n_tiles_per_task - 1) / n_tiles_per_task; @@ -2369,23 +2371,15 @@ static void dequantize_tiled_weight_chunk_to_fp16_tiles( state.n_k_tiles = n_k_tiles; state.n_weight_tiles = n_weight_tiles; state.n_k_tiles_div = n_k_tiles_div; - state.traces = ctx->trace; - state.ctx = ctx; state.tile_size = htp_tensor_get_tile_size(weight_type); state.aligned_tile_size = htp_tensor_get_tile_padded_size(weight_type); - if (state.n_tasks == 1 || n_threads == 1) { - dequantize_tiled_worker_loop(1, 0, &state); - } else { - int n_tasks = hex_smin((int) state.n_tasks, n_threads); - worker_pool_run_func(ctx->worker_pool, dequantize_tiled_worker_loop, &state, n_tasks); - } + int n_tasks = hex_smin((int) state.n_tasks, n_hvx); + work_queue_run(ctx->hvx_queue, dequantize_tiled_worker_loop, &state, n_tasks); } typedef struct { - struct htp_context * ctx; - struct htp_thread_trace * traces; float * dst; const __fp16 * vtcm_src; const float * src2; @@ -2394,19 +2388,19 @@ typedef struct { uint32_t dst_stride; uint32_t src2_stride; uint32_t dst_cols; - struct fastdiv_values n_threads_div; + struct fastdiv_values n_hvx_div; } output_transfer_col_chunk_state_t; -static void transfer_output_chunk_col_chunk_worker_fn(unsigned int n, unsigned int i, void *data) { +static void transfer_output_chunk_col_chunk_worker_fn(const htp_thread_t * th, unsigned int n, unsigned int i, void *data) { (void) n; output_transfer_col_chunk_state_t *st = (output_transfer_col_chunk_state_t *) data; - struct htp_thread_trace * tr = &st->traces[i]; + struct htp_thread_trace * tr = th->trace; // Round up: the last block is partial when N is not 32-aligned. Its pad // columns are dropped by the dst_cols clamp inside the store. uint32_t n_blocks = hmx_ceil_div(st->n_cols, 32); - uint32_t b_first = fastdiv(n_blocks * i, &st->n_threads_div); - uint32_t b_last = fastdiv(n_blocks * (i + 1), &st->n_threads_div); + uint32_t b_first = fastdiv(n_blocks * i, &st->n_hvx_div); + uint32_t b_last = fastdiv(n_blocks * (i + 1), &st->n_hvx_div); uint32_t c_first = b_first * 32; uint32_t c_last = b_last * 32; uint32_t c_len = c_last - c_first; @@ -2431,12 +2425,12 @@ static void transfer_output_chunk_col_chunk_worker_fn(unsigned int n, unsigned i } static void transfer_output_chunk_threaded(struct htp_context *ctx, float *dst, const float *src2, const __fp16 *vtcm_src, - int n_rows, int n_cols, int dst_stride, uint32_t src2_stride, int dst_cols, int n_threads) { + int n_rows, int n_cols, int dst_stride, uint32_t src2_stride, int dst_cols, int n_hvx) { if (n_rows <= 0) return; uint32_t n_blocks = hmx_ceil_div((uint32_t) n_cols, 32); - if (n_threads > 1 && n_blocks >= (uint32_t)n_threads) { - struct fastdiv_values n_threads_div = (n_threads == (int)ctx->n_threads) ? ctx->n_threads_div : init_fastdiv_values(n_threads); + if (n_hvx > 1 && n_blocks >= (uint32_t)n_hvx) { + struct fastdiv_values n_hvx_div = (n_hvx == (int)ctx->n_hvx) ? ctx->n_hvx_div : init_fastdiv_values(n_hvx); output_transfer_col_chunk_state_t col_state; col_state.dst = dst; col_state.src2 = src2; @@ -2446,22 +2440,20 @@ static void transfer_output_chunk_threaded(struct htp_context *ctx, float *dst, col_state.dst_stride = (uint32_t)dst_stride; col_state.src2_stride = src2_stride; col_state.dst_cols = (uint32_t)dst_cols; - col_state.n_threads_div = n_threads_div; - col_state.traces = ctx->trace; - col_state.ctx = ctx; + col_state.n_hvx_div = n_hvx_div; - worker_pool_run_func(ctx->worker_pool, transfer_output_chunk_col_chunk_worker_fn, &col_state, n_threads); + work_queue_run(ctx->hvx_queue, transfer_output_chunk_col_chunk_worker_fn, &col_state, n_hvx); return; } size_t n_tot_chunks = n_rows; - size_t n_chunks_per_task = (n_threads == 1) ? n_tot_chunks : hmx_ceil_div(n_rows, n_threads); + size_t n_chunks_per_task = (n_hvx == 1) ? n_tot_chunks : hmx_ceil_div(n_rows, n_hvx); n_chunks_per_task = hex_align_up(n_chunks_per_task, 2); - int actual_threads = hmx_ceil_div(n_rows, n_chunks_per_task); + int actual_hvx = hmx_ceil_div(n_rows, n_chunks_per_task); output_transfer_task_state_t state; - state.n_tasks = actual_threads; + state.n_tasks = actual_hvx; state.n_tot_chunks = n_tot_chunks; state.n_chunks_per_task = n_chunks_per_task; state.dst = dst; @@ -2471,13 +2463,8 @@ static void transfer_output_chunk_threaded(struct htp_context *ctx, float *dst, state.dst_stride = dst_stride; state.src2_stride = src2_stride; state.dst_cols = dst_cols; - state.traces = ctx->trace; - if (actual_threads <= 1) { - transfer_output_chunk_worker_fn(1, 0, &state); - } else { - worker_pool_run_func(ctx->worker_pool, transfer_output_chunk_worker_fn, &state, actual_threads); - } + work_queue_run(ctx->hvx_queue, transfer_output_chunk_worker_fn, &state, actual_hvx); } struct activation_transfer_params { @@ -2487,8 +2474,8 @@ struct activation_transfer_params { int n_rows; int k_block; int k_stride; - int n_threads; - const struct fastdiv_values * act_threads_div; + int n_hvx; + const struct fastdiv_values * act_hvx_div; const struct fastdiv_values * k_div; int k_valid; float * vtcm_f32_act; @@ -2503,8 +2490,8 @@ static void transfer_activation_chunk_threaded(const struct activation_transfer_ int n_rows = params->n_rows; int k_block = params->k_block; int k_stride = params->k_stride; - int n_threads = params->n_threads; - const struct fastdiv_values * act_threads_div = params->act_threads_div; + int n_hvx = params->n_hvx; + const struct fastdiv_values * act_hvx_div = params->act_hvx_div; const struct fastdiv_values * k_div = params->k_div; int k_valid = params->k_valid; float * vtcm_f32_act = params->vtcm_f32_act; @@ -2517,12 +2504,12 @@ static void transfer_activation_chunk_threaded(const struct activation_transfer_ } const size_t n_tasks = (n_rows + 31) >> 5; - if (act_elem_size == sizeof(float) && n_threads > 1 && k_block > 32 && n_tasks < (size_t) n_threads) { + if (act_elem_size == sizeof(float) && n_hvx > 1 && k_block > 32 && n_tasks < (size_t) n_hvx) { // Calculate step rows parameters for column-chunked dma pipelining uint32_t dma_step_rows = 2; uint32_t dma_step_rows_shift = 1; if (vtcm_f32_act_bytes > 0) { - size_t thread_scratch_bytes = hex_align_down(fastdiv(vtcm_f32_act_bytes, act_threads_div), 128); + size_t thread_scratch_bytes = hex_align_down(fastdiv(vtcm_f32_act_bytes, act_hvx_div), 128); size_t thread_scratch_elements = thread_scratch_bytes / sizeof(float); size_t dma_step_rows_max = fastdiv(thread_scratch_elements / 2, k_div); if (dma_step_rows_max >= 4) { @@ -2538,26 +2525,24 @@ static void transfer_activation_chunk_threaded(const struct activation_transfer_ col_state.k_block = k_block; col_state.k_stride = k_stride; col_state.k_valid = k_valid; - col_state.n_col_chunks = n_threads; - col_state.n_threads_div = *act_threads_div; + col_state.n_col_chunks = n_hvx; + col_state.n_hvx_div = *act_hvx_div; col_state.vtcm_f32_act = vtcm_f32_act; col_state.vtcm_f32_act_bytes = vtcm_f32_act_bytes; - col_state.traces = ctx->trace; - col_state.ctx = ctx; col_state.dma_step_rows = dma_step_rows; col_state.dma_step_rows_shift = dma_step_rows_shift; - worker_pool_run_func(ctx->worker_pool, transfer_activation_chunk_col_chunk_worker_fn, &col_state, n_threads); + work_queue_run(ctx->hvx_queue, transfer_activation_chunk_col_chunk_worker_fn, &col_state, n_hvx); return; } assert(k_block % HTP_MM_HMX_TILE_N_COLS == 0 && k_stride % HTP_MM_HMX_TILE_N_COLS == 0); size_t n_tot_chunks = n_rows; - size_t n_chunks_per_task = (n_threads == 1) ? n_tot_chunks : 32; // must be multiple of 32 to ensure correct destination address + size_t n_chunks_per_task = (n_hvx == 1) ? n_tot_chunks : 32; // must be multiple of 32 to ensure correct destination address activation_transfer_task_state_t state; - state.n_tasks = (n_threads == 1) ? 1 : hmx_ceil_div(n_tot_chunks, 32); + state.n_tasks = (n_hvx == 1) ? 1 : hmx_ceil_div(n_tot_chunks, 32); state.n_tot_chunks = n_tot_chunks; state.n_chunks_per_task = n_chunks_per_task; state.dst = dst; @@ -2565,17 +2550,15 @@ static void transfer_activation_chunk_threaded(const struct activation_transfer_ state.k_block = k_block; state.k_stride = k_stride; state.k_valid = k_valid; - state.traces = ctx->trace; - state.ctx = ctx; state.vtcm_f32_act = vtcm_f32_act; - state.act_elem_size = act_elem_size; + state.act_elem_size = act_elem_size; - state.vtcm_f32_act_bytes_per_thread = hex_align_down(fastdiv(vtcm_f32_act_bytes, act_threads_div), 128); + state.vtcm_f32_act_bytes_per_hvx = hex_align_down(fastdiv(vtcm_f32_act_bytes, act_hvx_div), 128); uint32_t dma_step_rows = 2; uint32_t dma_step_rows_shift = 1; - if (state.vtcm_f32_act_bytes_per_thread > 0) { - size_t thread_scratch_elements = state.vtcm_f32_act_bytes_per_thread / sizeof(float); + if (state.vtcm_f32_act_bytes_per_hvx > 0) { + size_t thread_scratch_elements = state.vtcm_f32_act_bytes_per_hvx / sizeof(float); size_t dma_step_rows_max = fastdiv(thread_scratch_elements / 2, k_div); if (dma_step_rows_max >= 4) { dma_step_rows = 4; @@ -2585,12 +2568,8 @@ static void transfer_activation_chunk_threaded(const struct activation_transfer_ state.dma_step_rows = dma_step_rows; state.dma_step_rows_shift = dma_step_rows_shift; - int active_threads = hex_smin(n_threads, (int)state.n_tasks); - if (state.n_tasks == 1 || n_threads == 1) { - transfer_activation_chunk_worker_fn(1, 0, &state); - } else { - worker_pool_run_func(ctx->worker_pool, transfer_activation_chunk_worker_fn, &state, active_threads); - } + int active_hvx = hex_smin(n_hvx, (int)state.n_tasks); + work_queue_run(ctx->hvx_queue, transfer_activation_chunk_worker_fn, &state, active_hvx); } // --- Async HMX matmul job (for pipeline overlap) --- @@ -2604,7 +2583,7 @@ typedef struct { uint32_t n_dot_tiles; } hmx_matmul_job_t; -static void hmx_matmul_worker_fn(void * data) { +static void hmx_matmul_worker_fn(const htp_thread_t * th, void * data) { hmx_matmul_job_t * job = (hmx_matmul_job_t *) data; FARF(HIGH, "hmx-mm-job: n_row_tiles %u n_col_tiles %u n_dot_tiles %u", job->n_row_tiles, job->n_col_tiles, job->n_dot_tiles); core_dot_chunk_fp16(job->output, job->activation, job->weight, job->scales, job->n_row_tiles, job->n_col_tiles, job->n_dot_tiles); @@ -2646,9 +2625,9 @@ static int hmx_mm_2d_f32(struct htp_context *ctx, int m_chunk, int n_chunk, int pipeline, - int n_threads, - int act_threads, - const struct fastdiv_values * act_threads_div, + int n_hvx, + int act_hvx, + const struct fastdiv_values * act_hvx_div, const struct fastdiv_values * k_div, int tile_size, int aligned_tile_size, @@ -2686,7 +2665,7 @@ static int hmx_mm_2d_f32(struct htp_context *ctx, size_t vtcm_used = vtcm_size; struct htp_mm_hmx_vtcm_layout L; - htp_mm_hmx_vtcm_layout_build(&L, HTP_MM_KERNEL_HMX_2D, weight_type, k, m_chunk_n_rows, n_chunk_n_cols, 1, pipeline, act_threads, aligned_tile_size, src2_bytes); + htp_mm_hmx_vtcm_layout_build(&L, HTP_MM_KERNEL_HMX_2D, weight_type, k, m_chunk_n_rows, n_chunk_n_cols, 1, pipeline, act_hvx, aligned_tile_size, src2_bytes); vtcm_used = L.total_bytes; if (vtcm_used > vtcm_budget) { @@ -2751,8 +2730,8 @@ static int hmx_mm_2d_f32(struct htp_context *ctx, .n_rows = (int) n_rows, .k_block = k, .k_stride = act_stride, - .n_threads = act_threads, - .act_threads_div = act_threads_div, + .n_hvx = act_hvx, + .act_hvx_div = act_hvx_div, .k_div = k_div, .k_valid = k_valid, .vtcm_f32_act = vtcm_f32_act, @@ -2789,7 +2768,7 @@ static int hmx_mm_2d_f32(struct htp_context *ctx, dequantize_tiled_weight_chunk_to_fp16_tiles( ctx, vtcm_weight_bufs[i % 4], curr_raw, n_cols, k, row_stride, weight_type, - n_k_tiles, n_k_tiles_div, dequant_worker_fn, n_threads); + n_k_tiles, n_k_tiles_div, dequant_worker_fn, n_hvx); // 3. push A_{i+2} (if i+2 < n_chunk_cnt) if (i + 2 < n_chunk_cnt) { @@ -2814,7 +2793,7 @@ static int hmx_mm_2d_f32(struct htp_context *ctx, const float *bias_chunk = has_bias ? (vtcm_bias + mr * src2_stride + nc_prev) : NULL; int chunk_dst_cols = dst_cols - (int)nc_prev; if (chunk_dst_cols > 0) { - transfer_output_chunk_threaded(ctx, output_chunk, bias_chunk, vtcm_output_bufs[(i - 1) % 2], n_rows, n_cols_prev, dst_stride, src2_stride, chunk_dst_cols, n_threads); + transfer_output_chunk_threaded(ctx, output_chunk, bias_chunk, vtcm_output_bufs[(i - 1) % 2], n_rows, n_cols_prev, dst_stride, src2_stride, chunk_dst_cols, n_hvx); } } } @@ -2827,7 +2806,7 @@ static int hmx_mm_2d_f32(struct htp_context *ctx, const float *bias_chunk = has_bias ? (vtcm_bias + mr * src2_stride + nc_last) : NULL; int chunk_dst_cols = dst_cols - (int)nc_last; if (chunk_dst_cols > 0) { - transfer_output_chunk_threaded(ctx, output_chunk, bias_chunk, vtcm_output_bufs[(n_chunk_cnt - 1) % 2], n_rows, n_cols_last, dst_stride, src2_stride, chunk_dst_cols, n_threads); + transfer_output_chunk_threaded(ctx, output_chunk, bias_chunk, vtcm_output_bufs[(n_chunk_cnt - 1) % 2], n_rows, n_cols_last, dst_stride, src2_stride, chunk_dst_cols, n_hvx); } } } else { @@ -2846,8 +2825,8 @@ static int hmx_mm_2d_f32(struct htp_context *ctx, .n_rows = (int) n_rows, .k_block = k, .k_stride = act_stride, - .n_threads = act_threads, - .act_threads_div = act_threads_div, + .n_hvx = act_hvx, + .act_hvx_div = act_hvx_div, .k_div = k_div, .k_valid = k_valid, .vtcm_f32_act = vtcm_f32_act, @@ -2876,7 +2855,7 @@ static int hmx_mm_2d_f32(struct htp_context *ctx, dequantize_tiled_weight_chunk_to_fp16_tiles( ctx, vtcm_weight_bufs[i % 2], curr_raw, n_cols, k, row_stride, weight_type, - n_k_tiles, n_k_tiles_div, dequant_worker_fn, n_threads); + n_k_tiles, n_k_tiles_div, dequant_worker_fn, n_hvx); // Start weight DMA for the next chunk early const size_t nc_next = nc + n_chunk_n_cols; @@ -2896,7 +2875,7 @@ static int hmx_mm_2d_f32(struct htp_context *ctx, const float *bias_chunk = has_bias ? (vtcm_bias + mr * src2_stride + nc) : NULL; int chunk_dst_cols = dst_cols - (int)nc; if (chunk_dst_cols > 0) { - transfer_output_chunk_threaded(ctx, output_chunk, bias_chunk, vtcm_output, n_rows, n_cols, dst_stride, src2_stride, chunk_dst_cols, n_threads); + transfer_output_chunk_threaded(ctx, output_chunk, bias_chunk, vtcm_output, n_rows, n_cols, dst_stride, src2_stride, chunk_dst_cols, n_hvx); } } } @@ -2947,16 +2926,16 @@ static int hmx_mm_nx_2d_f32(struct htp_ops_context * octx, const struct htp_mm_k const int m_chunk_n_rows = kparams->m_chunk; const int n_chunk_n_cols = kparams->n_chunk; const int pipeline = kparams->pipeline; - const int n_threads = octx->n_threads; - const int act_threads = kparams->n_act_threads; - const struct fastdiv_values * act_threads_div = &kparams->div_n_act_threads; + const int n_hvx = octx->n_hvx; + const int act_hvx = kparams->n_act_hvx; + const struct fastdiv_values * act_hvx_div = &kparams->div_n_act_hvx; const struct fastdiv_values * k_div = &kparams->div_ne00_padded; const uint32_t dma_dst_stride = is_tiled ? aligned_tile_size : row_stride; const uint32_t dma_width_bytes = is_tiled ? tile_size : row_stride; struct htp_mm_hmx_vtcm_layout L; - htp_mm_hmx_vtcm_layout_build(&L, HTP_MM_KERNEL_HMX_2D, weight_type, k, m_chunk_n_rows, n_chunk_n_cols, 1, pipeline, act_threads, aligned_tile_size, 0); + htp_mm_hmx_vtcm_layout_build(&L, HTP_MM_KERNEL_HMX_2D, weight_type, k, m_chunk_n_rows, n_chunk_n_cols, 1, pipeline, act_hvx, aligned_tile_size, 0); if (L.total_bytes > vtcm_budget) { FARF(ERROR, "hmx-mm-nx-2d: VTCM overflow: used %zu budget %zu, m %d k %d mc %d nc %d", @@ -3034,8 +3013,8 @@ static int hmx_mm_nx_2d_f32(struct htp_ops_context * octx, const struct htp_mm_k .n_rows = (int) n_rows, .k_block = k, .k_stride = act_stride, - .n_threads = act_threads, - .act_threads_div = act_threads_div, + .n_hvx = act_hvx, + .act_hvx_div = act_hvx_div, .k_div = k_div, .k_valid = k_valid, .vtcm_f32_act = vtcm_f32_act, @@ -3085,7 +3064,7 @@ static int hmx_mm_nx_2d_f32(struct htp_ops_context * octx, const struct htp_mm_k dequantize_tiled_weight_chunk_to_fp16_tiles( ctx, vtcm_weight_bufs[i % 4], curr_raw, n_cols, k, row_stride, weight_type, - n_k_tiles, n_k_tiles_div, dequant_worker_fn, n_threads); + n_k_tiles, n_k_tiles_div, dequant_worker_fn, n_hvx); if (i + 2 < n_chunk_cnt) { const uint32_t height_p2 = is_tiled ? hmx_ceil_div(n_cols_p2, 32) * n_weight_tiles : n_cols_p2; @@ -3106,7 +3085,7 @@ static int hmx_mm_nx_2d_f32(struct htp_ops_context * octx, const struct htp_mm_k float *output_chunk = dst_ptr + (mr * dst_stride + nc_prev); int chunk_dst_cols = dst_cols - (int)nc_prev; if (chunk_dst_cols > 0) { - transfer_output_chunk_threaded(ctx, output_chunk, NULL, vtcm_output_bufs[(i - 1) % 2], n_rows, n_cols_prev, dst_stride, 0, chunk_dst_cols, n_threads); + transfer_output_chunk_threaded(ctx, output_chunk, NULL, vtcm_output_bufs[(i - 1) % 2], n_rows, n_cols_prev, dst_stride, 0, chunk_dst_cols, n_hvx); } } } @@ -3117,7 +3096,7 @@ static int hmx_mm_nx_2d_f32(struct htp_ops_context * octx, const struct htp_mm_k float *output_chunk = dst_ptr + (mr * dst_stride + nc_last); int chunk_dst_cols = dst_cols - (int)nc_last; if (chunk_dst_cols > 0) { - transfer_output_chunk_threaded(ctx, output_chunk, NULL, vtcm_output_bufs[(n_chunk_cnt - 1) % 2], n_rows, n_cols_last, dst_stride, 0, chunk_dst_cols, n_threads); + transfer_output_chunk_threaded(ctx, output_chunk, NULL, vtcm_output_bufs[(n_chunk_cnt - 1) % 2], n_rows, n_cols_last, dst_stride, 0, chunk_dst_cols, n_hvx); } } } @@ -3136,8 +3115,8 @@ static int hmx_mm_nx_2d_f32(struct htp_ops_context * octx, const struct htp_mm_k .n_rows = (int) n_rows, .k_block = k, .k_stride = act_stride, - .n_threads = act_threads, - .act_threads_div = act_threads_div, + .n_hvx = act_hvx, + .act_hvx_div = act_hvx_div, .k_div = k_div, .k_valid = k_valid, .vtcm_f32_act = vtcm_f32_act, @@ -3180,7 +3159,7 @@ static int hmx_mm_nx_2d_f32(struct htp_ops_context * octx, const struct htp_mm_k dequantize_tiled_weight_chunk_to_fp16_tiles( ctx, vtcm_weight_bufs[i % 2], curr_raw, n_cols, k, row_stride, weight_type, - n_k_tiles, n_k_tiles_div, dequant_worker_fn, n_threads); + n_k_tiles, n_k_tiles_div, dequant_worker_fn, n_hvx); const size_t nc_next = nc + n_chunk_n_cols; if (nc_next < n) { @@ -3196,7 +3175,7 @@ static int hmx_mm_nx_2d_f32(struct htp_ops_context * octx, const struct htp_mm_k float *output_chunk = dst_ptr + (mr * dst_stride + nc); int chunk_dst_cols = dst_cols - (int)nc; if (chunk_dst_cols > 0) { - transfer_output_chunk_threaded(ctx, output_chunk, NULL, vtcm_output, n_rows, n_cols, dst_stride, 0, chunk_dst_cols, n_threads); + transfer_output_chunk_threaded(ctx, output_chunk, NULL, vtcm_output, n_rows, n_cols, dst_stride, 0, chunk_dst_cols, n_hvx); } } } @@ -3228,8 +3207,8 @@ static inline float *hmx_mm_dst_batch_ptr(const hmx_mm_batched_params_t *params, static int hmx_mm_batched_simple(struct htp_context *ctx, const hmx_mm_batched_params_t *params, - int m_chunk, int n_chunk, int pipeline, int n_threads, int act_threads, int vtcm_size, - const struct fastdiv_values * act_threads_div, const struct fastdiv_values * k_div) { + int m_chunk, int n_chunk, int pipeline, int n_hvx, int act_hvx, int vtcm_size, + const struct fastdiv_values * act_hvx_div, const struct fastdiv_values * k_div) { int ret = 0; for (int b3 = 0; b3 < params->ne13 && ret == 0; ++b3) { for (int b2 = 0; b2 < params->ne12 && ret == 0; ++b2) { @@ -3243,16 +3222,16 @@ static int hmx_mm_batched_simple(struct htp_context *ctx, params->m, params->k, params->n, params->act_stride, params->act_elem_size, params->weight_stride, params->weight_type, params->k, params->dst_stride, params->bias_stride, params->dst_cols, - m_chunk, n_chunk, pipeline, n_threads, act_threads, - act_threads_div, k_div, params->tile_size, params->aligned_tile_size, vtcm_size); + m_chunk, n_chunk, pipeline, n_hvx, act_hvx, + act_hvx_div, k_div, params->tile_size, params->aligned_tile_size, vtcm_size); } } return ret; } static int hmx_mm_batched(struct htp_context *ctx, const hmx_mm_batched_params_t *params, - int m_chunk, int n_chunk, int pipeline, int n_threads, int act_threads, - const struct fastdiv_values * act_threads_div, + int m_chunk, int n_chunk, int pipeline, int n_hvx, int act_hvx, + const struct fastdiv_values * act_hvx_div, const struct fastdiv_values * k_div, int vtcm_size) { const size_t row_stride = htp_tensor_get_row_size(params->weight_type, params->k); @@ -3272,7 +3251,7 @@ static int hmx_mm_batched(struct htp_context *ctx, const hmx_mm_batched_params_t // Grouped path is only valid for flat F16 if group_size > 1 and it fits within VTCM budget. bool run_grouped = (params->weight_type == HTP_TYPE_F16 && params->tile_size == 0 && group_size > 1 && (size_t)vtcm_size <= vtcm_budget); if (!run_grouped) { - return hmx_mm_batched_simple(ctx, params, m_chunk, n_chunk, pipeline, n_threads, act_threads, vtcm_size, act_threads_div, k_div); + return hmx_mm_batched_simple(ctx, params, m_chunk, n_chunk, pipeline, n_hvx, act_hvx, vtcm_size, act_hvx_div, k_div); } struct htp_thread_trace * tr = &ctx->trace[0]; @@ -3282,12 +3261,12 @@ static int hmx_mm_batched(struct htp_context *ctx, const hmx_mm_batched_params_t size_t n_chunk_n_cols = n_chunk; struct htp_mm_hmx_vtcm_layout L; - htp_mm_hmx_vtcm_layout_build(&L, HTP_MM_KERNEL_HMX_BATCHED, HTP_TYPE_F16, params->k, m_chunk_n_rows, n_chunk_n_cols, group_size, false, act_threads, 0, params->bias_bytes); + htp_mm_hmx_vtcm_layout_build(&L, HTP_MM_KERNEL_HMX_BATCHED, HTP_TYPE_F16, params->k, m_chunk_n_rows, n_chunk_n_cols, group_size, false, act_hvx, 0, params->bias_bytes); if (L.total_bytes > vtcm_budget) { FARF(HIGH, "%s: grouped layout overflowed VTCM, falling back to simple batched loop", __func__); htp_trace_event_stop(tr, HTP_TRACE_EVT_INIT, 0); - return hmx_mm_batched_simple(ctx, params, m_chunk, n_chunk, pipeline, n_threads, act_threads, vtcm_size, act_threads_div, k_div); + return hmx_mm_batched_simple(ctx, params, m_chunk, n_chunk, pipeline, n_hvx, act_hvx, vtcm_size, act_hvx_div, k_div); } uint8_t * const base = (uint8_t *) ctx->vtcm_base; @@ -3345,8 +3324,8 @@ static int hmx_mm_batched(struct htp_context *ctx, const hmx_mm_batched_params_t .n_rows = (int) n_rows, .k_block = params->k, .k_stride = params->act_stride, - .n_threads = act_threads, - .act_threads_div = act_threads_div, + .n_hvx = act_hvx, + .act_hvx_div = act_hvx_div, .k_div = k_div, .k_valid = params->k, .vtcm_f32_act = vtcm_f32_act, @@ -3405,7 +3384,7 @@ static int hmx_mm_batched(struct htp_context *ctx, const hmx_mm_batched_params_t int chunk_dst_cols = params->dst_cols - (int)nc; if (chunk_dst_cols > 0) { transfer_output_chunk_threaded(ctx, output, bias_chunk, vtcm_output, (int) n_rows, (int) n_cols, - params->dst_stride, params->bias_stride, chunk_dst_cols, n_threads); + params->dst_stride, params->bias_stride, chunk_dst_cols, n_hvx); } } } @@ -3431,20 +3410,20 @@ static void transfer_activation_chunk_gathered_threaded( size_t nb11, size_t nb12, int cne1, - int n_threads, + int n_hvx, int k_valid) { if (n_rows <= 0) return; - int chunks_per_thread = hmx_ceil_div(n_rows, n_threads); - chunks_per_thread = hex_align_up(chunks_per_thread, 2); + int chunks_per_hvx = hmx_ceil_div(n_rows, n_hvx); + chunks_per_hvx = hex_align_up(chunks_per_hvx, 2); - int actual_threads = hmx_ceil_div(n_rows, chunks_per_thread); + int actual_hvx = hmx_ceil_div(n_rows, chunks_per_hvx); activation_transfer_gathered_task_state_t state = { .dst = dst, .src = src, - .n_tasks = actual_threads, + .n_tasks = actual_hvx, .n_tot_chunks = n_rows, - .n_chunks_per_task = chunks_per_thread, + .n_chunks_per_task = chunks_per_hvx, .k_block = k_block, .matrix_rows = matrix_rows, .cur_a = cur_a, @@ -3456,17 +3435,12 @@ static void transfer_activation_chunk_gathered_threaded( .start_row = start_row, .cne1 = cne1, .k_valid = k_valid, - .traces = ctx->trace, }; - worker_callback_t worker_fn = ne11 == 1 ? transfer_activation_chunk_gathered_worker_flat_fn : + work_queue_func_t worker_fn = ne11 == 1 ? transfer_activation_chunk_gathered_worker_flat_fn : transfer_activation_chunk_gathered_worker_fn; - if (actual_threads <= 1) { - worker_fn(1, 0, &state); - } else { - worker_pool_run_func(ctx->worker_pool, worker_fn, &state, actual_threads); - } + work_queue_run(ctx->hvx_queue, worker_fn, &state, actual_hvx); } static void transfer_output_chunk_scattered_threaded( @@ -3482,19 +3456,19 @@ static void transfer_output_chunk_scattered_threaded( size_t dst_nb1, size_t dst_nb2, int cne1, - int n_threads) { + int n_hvx) { if (n_rows <= 0) return; - int chunks_per_thread = hmx_ceil_div(n_rows, n_threads); - chunks_per_thread = hex_align_up(chunks_per_thread, 2); + int chunks_per_hvx = hmx_ceil_div(n_rows, n_hvx); + chunks_per_hvx = hex_align_up(chunks_per_hvx, 2); - int actual_threads = hmx_ceil_div(n_rows, chunks_per_thread); + int actual_hvx = hmx_ceil_div(n_rows, chunks_per_hvx); output_transfer_scattered_task_state_t state = { .vtcm_src = vtcm_src, .dst = dst, - .n_tasks = actual_threads, + .n_tasks = actual_hvx, .n_tot_chunks = n_rows, - .n_chunks_per_task = chunks_per_thread, + .n_chunks_per_task = chunks_per_hvx, .n_cols = n_cols, .matrix_rows = matrix_rows, .cur_a = cur_a, @@ -3503,14 +3477,9 @@ static void transfer_output_chunk_scattered_threaded( .dst_nb2 = dst_nb2, .start_row = start_row, .cne1 = cne1, - .traces = ctx->trace, }; - if (actual_threads <= 1) { - transfer_output_chunk_scattered_worker_fn(1, 0, &state); - } else { - worker_pool_run_func(ctx->worker_pool, transfer_output_chunk_scattered_worker_fn, &state, actual_threads); - } + work_queue_run(ctx->hvx_queue, transfer_output_chunk_scattered_worker_fn, &state, actual_hvx); } static int hmx_mm_id_2d_f32(struct htp_context *ctx, @@ -3532,7 +3501,7 @@ static int hmx_mm_id_2d_f32(struct htp_context *ctx, int mapping_stride, int m_start, int m_end, - int n_threads) { + int n_hvx) { struct htp_thread_trace * tr = &ctx->trace[0]; htp_trace_event_start(tr, HTP_TRACE_EVT_INIT, 0); @@ -3614,7 +3583,7 @@ static int hmx_mm_id_2d_f32(struct htp_context *ctx, transfer_activation_chunk_gathered_threaded( ctx, vtcm_f16_act, activation, (int) mr, (int) n_rows, k, - matrix_rows, cur_a, mapping_stride, ne11, act_nb1, act_nb2, cne1, n_threads, k_valid); + matrix_rows, cur_a, mapping_stride, ne11, act_nb1, act_nb2, cne1, n_hvx, k_valid); // A0: Pre-fetch the first weight chunk (nc = 0) if (n > 0) { @@ -3636,7 +3605,7 @@ static int hmx_mm_id_2d_f32(struct htp_context *ctx, dequantize_tiled_weight_chunk_to_fp16_tiles( ctx, vtcm_weight_bufs[i % 2], curr_raw, n_cols, k, row_stride, weight_type, - n_k_tiles, n_k_tiles_div, dequant_worker_fn, n_threads + n_k_tiles, n_k_tiles_div, dequant_worker_fn, n_hvx ); // Start weight DMA for the next chunk early @@ -3656,7 +3625,7 @@ static int hmx_mm_id_2d_f32(struct htp_context *ctx, // D: Output Store transfer_output_chunk_scattered_threaded( ctx, dst + nc, vtcm_output, (int) mr, (int) n_rows, (int) n_cols, - matrix_rows, cur_a, mapping_stride, dst_nb1, dst_nb2, cne1, n_threads); + matrix_rows, cur_a, mapping_stride, dst_nb1, dst_nb2, cne1, n_hvx); } } @@ -3710,7 +3679,7 @@ static int hmx_mm_op_matmul(struct htp_ops_context * octx, const struct htp_mm_k const dma_addr_t act_addr = act->data + (size_t) m_start * act_stride * act_elem_size; int ret = -1; - const int n_threads = kparams->n_threads; + const int n_hvx = kparams->n_hvx; if (kparams->kernel_type == HTP_MM_KERNEL_HMX_BATCHED) { hmx_mm_batched_params_t batch_params = { .dst = dst_ptr, @@ -3750,9 +3719,9 @@ static int hmx_mm_op_matmul(struct htp_ops_context * octx, const struct htp_mm_k }; ret = hmx_mm_batched(octx->ctx, &batch_params, kparams->m_chunk, kparams->n_chunk, - kparams->pipeline, n_threads, - kparams->n_act_threads, - &kparams->div_n_act_threads, + kparams->pipeline, n_hvx, + kparams->n_act_hvx, + &kparams->div_n_act_hvx, &kparams->div_ne00_padded, kparams->vtcm_size); } else { @@ -3761,9 +3730,9 @@ static int hmx_mm_op_matmul(struct htp_ops_context * octx, const struct htp_mm_k act_addr, src0->data, m_rows, k, n, act_stride, act_elem_size, (int) src0->nb[1], (int) src0->type, (int) act->ne[0], dst_stride, src2_stride, (int)dst->ne[0], - kparams->m_chunk, kparams->n_chunk, kparams->pipeline, n_threads, - kparams->n_act_threads, - &kparams->div_n_act_threads, + kparams->m_chunk, kparams->n_chunk, kparams->pipeline, n_hvx, + kparams->n_act_hvx, + &kparams->div_n_act_hvx, &kparams->div_ne00_padded, kparams->tile_size, kparams->aligned_tile_size, kparams->vtcm_size ); @@ -3849,7 +3818,7 @@ static int hmx_mm_op_matmul_id( (int) src0->nb[1], (int) src0->type, kparams->tile_size, kparams->aligned_tile_size, matrix_rows, cur_a, mmctx->mapping_stride, - m_start, m_end, (int) octx->n_threads); + m_start, m_end, (int) octx->n_hvx); if (ret != 0) { FARF(ERROR, "HMX matmul failed for expert %u, error %d\n", cur_a, ret); return HTP_STATUS_NO_SUPPORT; @@ -3881,8 +3850,8 @@ static int hvx_mm_matmul_id( work_queue_func_t quant_task_func; uint32_t n_quant_tasks = 1; - if (act_nrows < octx->n_threads) { - n_quant_tasks = MIN(total_nb, octx->n_threads); + if (act_nrows < octx->n_hvx) { + n_quant_tasks = MIN(total_nb, octx->n_hvx); quant_task_func = htp_mm_act_quant_block_func(src0->type, act->type); for (uint32_t ith = 0; ith < n_quant_tasks; ++ith) { uint32_t ib_first = (total_nb * ith) / n_quant_tasks; @@ -3893,13 +3862,13 @@ static int hvx_mm_matmul_id( mmctx->quant_c[ith] = ib_first % nb; } } else { - n_quant_tasks = MIN(act_nrows, octx->n_threads); + n_quant_tasks = MIN(act_nrows, octx->n_hvx); quant_task_func = htp_mm_act_quant_row_func(src0->type, act->type); } size_t act_row_size = htp_mm_tiled_act_row_size(src0->type, ne10); struct htp_mm_hvx_vtcm_layout L; - htp_mm_hvx_vtcm_layout_build(&L, kparams->kernel_type, src0->type, ne10, act_nrows, octx->n_threads, + htp_mm_hvx_vtcm_layout_build(&L, kparams->kernel_type, src0->type, ne10, act_nrows, octx->n_hvx, 0, src0_row_size, act_row_size, 0, kparams->n_prefetch, true, false); const size_t vtcm_size = L.total_bytes; @@ -3929,9 +3898,9 @@ static int hvx_mm_matmul_id( mmctx->vtcm_act_stride = act_row_size; mmctx->vtcm_act_raw_stride = htp_mm_act_raw_row_size(ne10); - mmctx->vtcm_src0_size_per_thread = fastdiv(L.src0_bytes, &octx->n_threads_div); - mmctx->vtcm_act_size_per_thread = L.act_bytes; - mmctx->vtcm_dst_size_per_thread = fastdiv(L.dst_bytes, &octx->n_threads_div); + mmctx->vtcm_src0_size_per_hvx = fastdiv(L.src0_bytes, &octx->n_hvx_div); + mmctx->vtcm_act_size_per_hvx = L.act_bytes; + mmctx->vtcm_dst_size_per_hvx = fastdiv(L.dst_bytes, &octx->n_hvx_div); mmctx->cur_m_start = 0; mmctx->cur_m_rows = act_nrows; @@ -3940,11 +3909,11 @@ static int hvx_mm_matmul_id( hvx_mm_transfer_act_dma(octx, kparams, act, mmctx->vtcm_act_raw, mmctx->vtcm_act_raw_stride, 0, act_nrows); - mmctx->n_quant_rows_per_thread = (act_nrows + n_quant_tasks - 1) / n_quant_tasks; + mmctx->n_quant_rows_per_hvx = (act_nrows + n_quant_tasks - 1) / n_quant_tasks; mmctx->n_quant_tasks = n_quant_tasks; - work_queue_run(octx->ctx->work_queue, quant_task_func, mmctx, n_quant_tasks); + work_queue_run(octx->ctx->hvx_queue, quant_task_func, mmctx, n_quant_tasks); - work_queue_run(octx->ctx->work_queue, hvx_mmid_task_func, mmctx, octx->n_threads); + work_queue_run(octx->ctx->hvx_queue, hvx_mmid_task_func, mmctx, octx->n_hvx); return HTP_STATUS_OK; } @@ -4013,7 +3982,7 @@ static int hmx_mm_op_matmul_id_nx( (int) src_w->nb[1], (int) src_w->type, kparams->tile_size, kparams->aligned_tile_size, matrix_rows, cur_a, mmctx->mapping_stride, - m_start, m_end, (int) octx->n_threads); + m_start, m_end, (int) octx->n_hvx); if (ret != 0) { FARF(ERROR, "HMX matmul ID NX failed for expert %u weight %u, error %d\n", cur_a, p, ret); return HTP_STATUS_NO_SUPPORT; @@ -4048,8 +4017,8 @@ static int hvx_mm_matmul_id_nx( work_queue_func_t quant_task_func; uint32_t n_quant_tasks = 1; - if (act_nrows < octx->n_threads) { - n_quant_tasks = MIN(total_nb, octx->n_threads); + if (act_nrows < octx->n_hvx) { + n_quant_tasks = MIN(total_nb, octx->n_hvx); quant_task_func = htp_mm_act_quant_block_func(src0->type, act->type); for (uint32_t ith = 0; ith < n_quant_tasks; ++ith) { uint32_t ib_first = (total_nb * ith) / n_quant_tasks; @@ -4060,13 +4029,13 @@ static int hvx_mm_matmul_id_nx( mmctx->quant_c[ith] = ib_first % nb; } } else { - n_quant_tasks = MIN(act_nrows, octx->n_threads); + n_quant_tasks = MIN(act_nrows, octx->n_hvx); quant_task_func = htp_mm_act_quant_row_func(src0->type, act->type); } size_t act_row_size = htp_mm_tiled_act_row_size(src0->type, act->ne[0]); struct htp_mm_hvx_vtcm_layout L; - htp_mm_hvx_vtcm_layout_build(&L, kparams->kernel_type, src0->type, act->ne[0], act_nrows, octx->n_threads, + htp_mm_hvx_vtcm_layout_build(&L, kparams->kernel_type, src0->type, act->ne[0], act_nrows, octx->n_hvx, 0, src0_row_size, act_row_size, 0, kparams->n_prefetch, true, false); const size_t vtcm_size = L.total_bytes; @@ -4087,27 +4056,27 @@ static int hvx_mm_matmul_id_nx( mmctx->vtcm_act_stride = act_row_size; mmctx->vtcm_act_raw_stride = htp_mm_act_raw_row_size(act->ne[0]); - mmctx->vtcm_src0_size_per_thread = fastdiv(L.src0_bytes, &octx->n_threads_div); - mmctx->vtcm_act_size_per_thread = L.act_bytes; - mmctx->vtcm_dst_size_per_thread = fastdiv(L.dst_bytes, &octx->n_threads_div); + mmctx->vtcm_src0_size_per_hvx = fastdiv(L.src0_bytes, &octx->n_hvx_div); + mmctx->vtcm_act_size_per_hvx = L.act_bytes; + mmctx->vtcm_dst_size_per_hvx = fastdiv(L.dst_bytes, &octx->n_hvx_div); mmctx->cur_m_start = 0; mmctx->cur_m_rows = act_nrows; - FARF(HIGH, "matmul-id-nx: src0 %d:%d:%d type %s nrows %u, act %d:%d:%d nrows %u, vtcm %zu/%zu, threads %d\n", + FARF(HIGH, "matmul-id-nx: src0 %d:%d:%d type %s nrows %u, act %d:%d:%d nrows %u, vtcm %zu/%zu, nhvx %d\n", src0->ne[0], src0->ne[1], src0->ne[2], mmctx->type, src0->ne[1], act->ne[0], act->ne[1], act->ne[2], act_nrows, - L.total_bytes, octx->ctx->vtcm_size, octx->n_threads); + L.total_bytes, octx->ctx->vtcm_size, octx->n_hvx); htp_trace_event_stop(tr, HTP_TRACE_EVT_INIT, 0); hvx_mm_transfer_act_dma(octx, kparams, act, mmctx->vtcm_act_raw, mmctx->vtcm_act_raw_stride, 0, act_nrows); - mmctx->n_quant_rows_per_thread = (act_nrows + n_quant_tasks - 1) / n_quant_tasks; + mmctx->n_quant_rows_per_hvx = (act_nrows + n_quant_tasks - 1) / n_quant_tasks; mmctx->n_quant_tasks = n_quant_tasks; - work_queue_run(octx->ctx->work_queue, quant_task_func, mmctx, n_quant_tasks); + work_queue_run(octx->ctx->hvx_queue, quant_task_func, mmctx, n_quant_tasks); - work_queue_run(octx->ctx->work_queue, hvx_mmid_task_func, mmctx, octx->n_threads); + work_queue_run(octx->ctx->hvx_queue, hvx_mmid_task_func, mmctx, octx->n_hvx); return HTP_STATUS_OK; } @@ -4386,8 +4355,8 @@ int op_matmul_id(struct htp_ops_context * octx) { mmctx->src0_row_start = src0_row_start; mmctx->src0_row_end = src0_row_end; - mmctx->src0_nrows_per_thread = fastdiv(nrows + octx->n_threads - 1, &octx->n_threads_div); - mmctx->src0_nrows_per_thread = hex_round_up(mmctx->src0_nrows_per_thread, 32); + mmctx->src0_nrows_per_hvx = fastdiv(nrows + octx->n_hvx - 1, &octx->n_hvx_div); + mmctx->src0_nrows_per_hvx = hex_round_up(mmctx->src0_nrows_per_hvx, 32); if (hvx_mm_init_tiled_vec_dot(mmctx, src0->type) == 0) { s = hvx_mm_matmul_id(octx, mmctx, act_nrows > 1 ? hvx_mm_id_tiled : hvx_mv_id_tiled); @@ -4530,10 +4499,10 @@ static int op_matmul_nx_impl(struct htp_ops_context * octx) { const uint32_t nb = (act->ne[0] + qk - 1) / qk; const uint32_t total_nb = act_nrows * nb; - worker_callback_t quant_task_func; + work_queue_func_t quant_task_func; uint32_t n_quant_tasks = 1; - if (act_nrows < octx->n_threads) { - n_quant_tasks = MIN(total_nb, octx->n_threads); + if (act_nrows < octx->n_hvx) { + n_quant_tasks = MIN(total_nb, octx->n_hvx); quant_task_func = htp_mm_act_quant_block_func(src0->type, act->type); for (uint32_t ith = 0; ith < n_quant_tasks; ++ith) { uint32_t ib_first = (total_nb * ith) / n_quant_tasks; @@ -4544,14 +4513,14 @@ static int op_matmul_nx_impl(struct htp_ops_context * octx) { mmctx->quant_c[ith] = ib_first % nb; } } else { - n_quant_tasks = MIN(act_nrows, octx->n_threads); + n_quant_tasks = MIN(act_nrows, octx->n_hvx); quant_task_func = htp_mm_act_quant_row_func(src0->type, act->type); } const size_t act_row_size = htp_mm_tiled_act_row_size(src0->type, act->ne[0]); struct htp_mm_hvx_vtcm_layout L; - htp_mm_hvx_vtcm_layout_build(&L, kparams->kernel_type, src0->type, act->ne[0], act_nrows, octx->n_threads, + htp_mm_hvx_vtcm_layout_build(&L, kparams->kernel_type, src0->type, act->ne[0], act_nrows, octx->n_hvx, 0, src0_row_size, act_row_size, 0, kparams->n_prefetch, false, true); const size_t vtcm_size = L.total_bytes; @@ -4572,19 +4541,19 @@ static int op_matmul_nx_impl(struct htp_ops_context * octx) { mmctx->vtcm_act_stride = act_row_size; mmctx->vtcm_act_raw_stride = htp_mm_act_raw_row_size(act->ne[0]); - mmctx->vtcm_src0_size_per_thread = fastdiv(L.src0_bytes, &octx->n_threads_div); - mmctx->vtcm_act_size_per_thread = L.act_bytes; - mmctx->vtcm_dst_size_per_thread = fastdiv(L.dst_bytes, &octx->n_threads_div); + mmctx->vtcm_src0_size_per_hvx = fastdiv(L.src0_bytes, &octx->n_hvx_div); + mmctx->vtcm_act_size_per_hvx = L.act_bytes; + mmctx->vtcm_dst_size_per_hvx = fastdiv(L.dst_bytes, &octx->n_hvx_div); htp_trace_event_stop(tr, HTP_TRACE_EVT_INIT, 0); hvx_mm_transfer_act_dma(octx, kparams, act, mmctx->vtcm_act_raw, mmctx->vtcm_act_raw_stride, 0, act_nrows); - mmctx->n_quant_rows_per_thread = (act_nrows + n_quant_tasks - 1) / n_quant_tasks; + mmctx->n_quant_rows_per_hvx = (act_nrows + n_quant_tasks - 1) / n_quant_tasks; mmctx->n_quant_tasks = n_quant_tasks; - work_queue_run(octx->ctx->work_queue, quant_task_func, mmctx, n_quant_tasks); + work_queue_run(octx->ctx->hvx_queue, quant_task_func, mmctx, n_quant_tasks); - work_queue_run(octx->ctx->work_queue, hvx_mm_nx_2d_tiled, mmctx, octx->n_threads); + work_queue_run(octx->ctx->hvx_queue, hvx_mm_nx_2d_tiled, mmctx, octx->n_hvx); return HTP_STATUS_OK; } diff --git a/ggml/src/ggml-hexagon/htp/matmul-ops.h b/ggml/src/ggml-hexagon/htp/matmul-ops.h index 4971ed958b75..7a85624cf9b6 100644 --- a/ggml/src/ggml-hexagon/htp/matmul-ops.h +++ b/ggml/src/ggml-hexagon/htp/matmul-ops.h @@ -64,8 +64,8 @@ struct htp_mm_kernel_params { uint8_t unused; // reserved uint8_t n_hmx; // 1 = use HMX, 0 = use HVX - uint8_t n_threads; // Number of threads to spawn - uint8_t n_act_threads; // Number of threads for activation preparation + uint8_t n_hvx; // Number of threads to spawn + uint8_t n_act_hvx; // Number of threads for activation preparation uint8_t n_prefetch; // Prefetch lookahead buffers/rows in VTCM uint8_t n_weights; // Number of weights for fused NX @@ -83,7 +83,7 @@ struct htp_mm_kernel_params { struct fastdiv_values div_r2; struct fastdiv_values div_r3; struct fastdiv_values div_ne12; - struct fastdiv_values div_n_act_threads; + struct fastdiv_values div_n_act_hvx; struct fastdiv_values div_ne00_padded; }; @@ -185,7 +185,7 @@ static inline int htp_mm_hmx_compute_chunks(size_t vtcm_total, // (Q2_K: per-16 k sums, q8_1_s16) static inline bool htp_mm_weight_has_offset(int weight_type) { return weight_type == HTP_TYPE_Q4_1 || weight_type == HTP_TYPE_Q4_K || weight_type == HTP_TYPE_Q5_K || - weight_type == HTP_TYPE_Q2_K || weight_type == HTP_TYPE_Q2_0; + weight_type == HTP_TYPE_Q2_K || weight_type == HTP_TYPE_Q2_0 || weight_type == HTP_TYPE_Q5_1; } // --- Activation/Row Size Helpers --- @@ -310,7 +310,7 @@ static inline void htp_mm_hmx_vtcm_layout_build( size_t nc, uint32_t group_size, bool pipeline, - uint32_t act_threads, + uint32_t act_hvx, uint32_t aligned_tile_size, size_t bias_size ) { @@ -323,7 +323,7 @@ static inline void htp_mm_hmx_vtcm_layout_build( const size_t activation_area_size = hex_align_up(group_size * act_head_stride * sizeof(uint16_t), HTP_MM_HMX_TILE_SIZE); const size_t output_area_size = hex_align_up(group_size * mc * nc * sizeof(uint16_t), HTP_MM_HMX_TILE_SIZE); const size_t scratch_area_size = hex_align_up(nc * vec_dot_size, HTP_MM_HMX_TILE_SIZE); - const size_t min_f32_size = hex_align_up(act_threads * HTP_MM_DMA_ACT_MULTIPLIER * k * sizeof(float), 128); + const size_t min_f32_size = hex_align_up(act_hvx * HTP_MM_DMA_ACT_MULTIPLIER * k * sizeof(float), 128); // Group A: Permanent activation tiles and scales size_t off_group_a = 0; @@ -343,7 +343,7 @@ static inline void htp_mm_hmx_vtcm_layout_build( const size_t group_b_size = off_group_b - off_group_a; // Group C: Activation prep temporary buffer (overlaps Group B, starting at off_group_a) - const size_t max_f32_size = act_threads * 64 * k * sizeof(float); + const size_t max_f32_size = act_hvx * 64 * k * sizeof(float); const size_t act_f32_size = hex_align_up(hex_smin(max_f32_size, hex_smax(min_f32_size, group_b_size)), 128); size_t off_group_c = off_group_a; VTCM_LAYOUT_ALLOC(off_group_c, off_act_f32, act_f32_size); @@ -367,7 +367,7 @@ static inline void htp_mm_hmx_vtcm_layout_build( const size_t vec_dot_size = k * sizeof(uint16_t); const uint32_t n_weight_tiles = k / htp_tensor_get_tile_k(wtype); - const size_t min_f32_size = hex_align_up(act_threads * HTP_MM_DMA_ACT_MULTIPLIER * k * sizeof(float), 128); + const size_t min_f32_size = hex_align_up(act_hvx * HTP_MM_DMA_ACT_MULTIPLIER * k * sizeof(float), 128); const size_t weight_area_size = is_quant ? hex_align_up((nc / 32) * n_weight_tiles * aligned_tile_size, HTP_MM_HMX_TILE_SIZE) : hex_align_up(nc * row_stride, HTP_MM_HMX_TILE_SIZE); @@ -395,7 +395,7 @@ static inline void htp_mm_hmx_vtcm_layout_build( const size_t group_b_size = off_group_b - off_group_a; // Group C: Activation prep temporary buffer (overlaps Group B, starting at off_group_a) - const size_t max_f32_size = act_threads * 64 * k * sizeof(float); + const size_t max_f32_size = act_hvx * 64 * k * sizeof(float); const size_t act_f32_size = hex_align_up(hex_smin(max_f32_size, hex_smax(min_f32_size, group_b_size)), 128); size_t off_group_c = off_group_a; VTCM_LAYOUT_ALLOC(off_group_c, off_act_f32, act_f32_size); @@ -423,7 +423,7 @@ static inline void htp_mm_hvx_vtcm_layout_build( int wtype, uint32_t ne10, // k uint32_t act_nrows, // m_total - uint32_t n_threads, + uint32_t n_hvx, size_t dst_row_size, size_t src0_row_size, size_t act_row_size, @@ -443,13 +443,13 @@ static inline void htp_mm_hvx_vtcm_layout_build( uint32_t aligned_tile_size = htp_tensor_get_tile_padded_size(wtype); uint32_t n_k_tiles = (ne10 + htp_tensor_get_tile_k(wtype) - 1) / htp_tensor_get_tile_k(wtype); uint32_t tile_row_size = n_k_tiles * aligned_tile_size; - size_t weight_sz_per_thread = hex_round_up(n_prefetch * tile_row_size, 128); + size_t weight_sz_per_hvx = hex_round_up(n_prefetch * tile_row_size, 128); size_t tiled_act_row_size = htp_mm_tiled_act_row_size(wtype, ne10); size_t q_act_sz = hex_round_up(tiled_act_row_size * act_nrows, 128); size_t raw_row_size = htp_mm_act_raw_row_size(ne10); - src0_sz = weight_sz_per_thread * n_threads; // shared single-weight prefetch buffer + src0_sz = weight_sz_per_hvx * n_hvx; // shared single-weight prefetch buffer act_sz = q_act_sz; // quantized activation buffer bias_sz = 0; dst_sz = 0; @@ -459,13 +459,13 @@ static inline void htp_mm_hvx_vtcm_layout_build( const uint32_t aligned_tile_size = htp_tensor_get_tile_padded_size(wtype); const uint32_t n_k_tiles = (ne10 + htp_tensor_get_tile_k(wtype) - 1) / htp_tensor_get_tile_k(wtype); const uint32_t tile_row_size = n_k_tiles * aligned_tile_size; - size_t src0_sz_per_thread = htp_mm_round_up(n_prefetch * tile_row_size, 256); + size_t src0_sz_per_hvx = htp_mm_round_up(n_prefetch * tile_row_size, 256); act_sz = htp_mm_round_up(act_row_size_tiled * act_nrows, 256); size_t raw_row_size = htp_mm_act_raw_row_size(ne10); - src0_sz = src0_sz_per_thread * n_threads; + src0_sz = src0_sz_per_hvx * n_hvx; dst_sz = 0; bias_sz = 0; act_raw_sz = hex_round_up(raw_row_size * act_nrows, 128); @@ -477,8 +477,8 @@ static inline void htp_mm_hvx_vtcm_layout_build( case HTP_MM_KERNEL_HVX_F16_F16_VTCM: { size_t f16_act_row_size = htp_mm_round_up(ne10 * 2, 128); act_sz = htp_mm_round_up(f16_act_row_size * act_nrows, 256); - src0_sz = htp_mm_round_up(n_prefetch * src0_row_size_padded, 256) * n_threads; - dst_sz = dst_nrows > 0 ? htp_mm_round_up(dst_row_size, 128) * n_threads : 0; + src0_sz = htp_mm_round_up(n_prefetch * src0_row_size_padded, 256) * n_hvx; + dst_sz = dst_nrows > 0 ? htp_mm_round_up(dst_row_size, 128) * n_hvx : 0; act_raw_sz = hex_round_up(hex_round_up(ne10 * sizeof(float), 128) * act_nrows, 128); break; } @@ -486,8 +486,8 @@ static inline void htp_mm_hvx_vtcm_layout_build( case HTP_MM_KERNEL_HVX_BF16_F32_VTCM: { size_t f32_act_row_size = htp_mm_round_up(ne10 * 4, 128); act_sz = htp_mm_round_up(f32_act_row_size * act_nrows, 256); - src0_sz = htp_mm_round_up(n_prefetch * src0_row_size_padded, 256) * n_threads; - dst_sz = dst_nrows > 0 ? htp_mm_round_up(dst_row_size, 128) * n_threads : 0; + src0_sz = htp_mm_round_up(n_prefetch * src0_row_size_padded, 256) * n_hvx; + dst_sz = dst_nrows > 0 ? htp_mm_round_up(dst_row_size, 128) * n_hvx : 0; act_raw_sz = 0; break; } @@ -499,11 +499,11 @@ static inline void htp_mm_hvx_vtcm_layout_build( uint32_t tile_row_size = n_weight_tiles * aligned_tile_size; size_t tiled_vtcm_size = htp_mm_round_up(n_prefetch * tile_row_size, 256); - src0_sz = tiled_vtcm_size * n_threads; + src0_sz = tiled_vtcm_size * n_hvx; act_sz = htp_mm_round_up(q_act_row_size * act_nrows, 256); - size_t dst_slice_per_thread = (dst_nrows > 0 && act_nrows == 1) ? htp_mm_round_up((dst_row_size + n_threads - 1) / n_threads, 128) : 0; - dst_sz = dst_slice_per_thread * n_threads; + size_t dst_slice_per_hvx = (dst_nrows > 0 && act_nrows == 1) ? htp_mm_round_up((dst_row_size + n_hvx - 1) / n_hvx, 128) : 0; + dst_sz = dst_slice_per_hvx * n_hvx; size_t raw_row_size = htp_mm_act_raw_row_size(ne10); act_raw_sz = hex_round_up(raw_row_size * act_nrows, 128); break; @@ -542,7 +542,7 @@ static inline bool htp_mm_hvx_solve_vtcm_params( int wtype, uint32_t ne10, uint32_t act_nrows, - uint32_t n_threads, + uint32_t n_hvx, size_t dst_row_size, size_t src0_row_size, size_t act_row_size, @@ -554,7 +554,7 @@ static inline bool htp_mm_hvx_solve_vtcm_params( ) { struct htp_mm_hvx_vtcm_layout L; htp_mm_hvx_vtcm_layout_build( - &L, kernel_type, wtype, ne10, act_nrows, n_threads, + &L, kernel_type, wtype, ne10, act_nrows, n_hvx, dst_row_size, src0_row_size, act_row_size, bias_row_size, n_prefetch, false, false ); @@ -599,14 +599,14 @@ static inline bool htp_mm_hvx_solve_vtcm_params( } htp_mm_hvx_vtcm_layout_build( - &L, kernel_type, wtype, ne10, m_chunk, n_threads, + &L, kernel_type, wtype, ne10, m_chunk, n_hvx, dst_row_size, src0_row_size, act_row_size, bias_row_size, n_prefetch, false, false ); while (m_chunk > 2 && L.total_bytes > vtcm_budget) { m_chunk -= 2; htp_mm_hvx_vtcm_layout_build( - &L, kernel_type, wtype, ne10, m_chunk, n_threads, + &L, kernel_type, wtype, ne10, m_chunk, n_hvx, dst_row_size, src0_row_size, act_row_size, bias_row_size, n_prefetch, false, false ); } @@ -621,18 +621,18 @@ static inline bool htp_mm_hvx_solve_vtcm_params( } static inline size_t htp_mm_hmx_get_2d_vtcm_size( - int wtype, uint32_t k, size_t mc, size_t nc, bool pipeline, uint32_t act_threads, uint32_t aligned_tile_size, size_t bias_size + int wtype, uint32_t k, size_t mc, size_t nc, bool pipeline, uint32_t act_hvx, uint32_t aligned_tile_size, size_t bias_size ) { struct htp_mm_hmx_vtcm_layout L; - htp_mm_hmx_vtcm_layout_build(&L, HTP_MM_KERNEL_HMX_2D, wtype, k, mc, nc, 1, pipeline, act_threads, aligned_tile_size, bias_size); + htp_mm_hmx_vtcm_layout_build(&L, HTP_MM_KERNEL_HMX_2D, wtype, k, mc, nc, 1, pipeline, act_hvx, aligned_tile_size, bias_size); return L.total_bytes; } static inline size_t htp_mm_hmx_get_batched_vtcm_size( - int wtype, uint32_t k, size_t mc, size_t nc, uint32_t group_size, bool pipeline, uint32_t act_threads, size_t bias_size) { + int wtype, uint32_t k, size_t mc, size_t nc, uint32_t group_size, bool pipeline, uint32_t act_hvx, size_t bias_size) { (void)pipeline; struct htp_mm_hmx_vtcm_layout L; - htp_mm_hmx_vtcm_layout_build(&L, HTP_MM_KERNEL_HMX_BATCHED, wtype, k, mc, nc, group_size, false, act_threads, 0, bias_size); + htp_mm_hmx_vtcm_layout_build(&L, HTP_MM_KERNEL_HMX_BATCHED, wtype, k, mc, nc, group_size, false, act_hvx, 0, bias_size); return L.total_bytes; } @@ -642,23 +642,23 @@ static inline bool htp_mm_hmx_solve_batched_params( uint32_t ne01_padded, uint32_t ne11, uint32_t group_size, - int n_threads, + int n_hvx, bool pipeline, size_t bias_size, size_t vtcm_budget, size_t * m_chunk_out, size_t * n_chunk_out, - int * act_threads_out, + int * act_hvx_out, size_t * vtcm_size_out ) { size_t best_mblocks = SIZE_MAX; - int best_act_threads = 0; + int best_act_hvx = 0; size_t best_m_chunk = 0; size_t best_n_chunk = 0; size_t best_vtcm_size = 0; - int act_threads = n_threads; - while (act_threads >= 1) { + int act_hvx = n_hvx; + while (act_hvx >= 1) { size_t group_overhead = htp_mm_hmx_get_batched_overhead() + (bias_size > 0 ? hex_align_up(bias_size, HTP_MM_HMX_TILE_SIZE) : 0); size_t group_size_per_n, group_size_per_m, group_size_per_mn; htp_mm_hmx_get_batched_chunk_costs(k, group_size, &group_size_per_n, &group_size_per_m, &group_size_per_mn); @@ -670,30 +670,30 @@ static inline bool htp_mm_hmx_solve_batched_params( if (htp_mm_hmx_compute_chunks(vtcm_budget, group_overhead, group_size_per_n, group_size_per_m, group_size_per_mn, hex_align_up(ne11, 32), ne01_padded, (size_t) ne01_padded * HTP_MM_HMX_COST_W_DEQUANT, (size_t) ne11 * HTP_MM_HMX_COST_A_CONVERT, &m_chunk_candidate, &n_chunk_candidate, &vtcm_size_candidate) == 0) { - size_t exact_size = htp_mm_hmx_get_batched_vtcm_size(wtype, k, m_chunk_candidate, n_chunk_candidate, group_size, pipeline, act_threads, bias_size); + size_t exact_size = htp_mm_hmx_get_batched_vtcm_size(wtype, k, m_chunk_candidate, n_chunk_candidate, group_size, pipeline, act_hvx, bias_size); if (exact_size <= vtcm_budget) { size_t mblocks = ((size_t) ne11 + m_chunk_candidate - 1) / m_chunk_candidate; - if (mblocks < best_mblocks || (mblocks == best_mblocks && act_threads > best_act_threads)) { + if (mblocks < best_mblocks || (mblocks == best_mblocks && act_hvx > best_act_hvx)) { best_mblocks = mblocks; - best_act_threads = act_threads; + best_act_hvx = act_hvx; best_m_chunk = m_chunk_candidate; best_n_chunk = n_chunk_candidate; best_vtcm_size = exact_size; } } } - if (act_threads == 1) { - act_threads = 0; + if (act_hvx == 1) { + act_hvx = 0; } else { - act_threads /= 2; + act_hvx /= 2; } } - if (best_act_threads > 0) { + if (best_act_hvx > 0) { *m_chunk_out = best_m_chunk; *n_chunk_out = best_n_chunk; *vtcm_size_out = best_vtcm_size; - *act_threads_out = best_act_threads; + *act_hvx_out = best_act_hvx; return true; } return false; @@ -706,7 +706,7 @@ static inline bool htp_mm_hmx_solve_2d_params( uint32_t ne01_padded, uint32_t ne11_padded, uint32_t m_for_cost, - int n_threads, + int n_hvx, bool pipeline, bool is_matmul_id, uint32_t aligned_tile_size, @@ -714,19 +714,19 @@ static inline bool htp_mm_hmx_solve_2d_params( size_t vtcm_budget, size_t * m_chunk_out, size_t * n_chunk_out, - int * act_threads_out, + int * act_hvx_out, size_t * vtcm_size_out ) { size_t best_mblocks = SIZE_MAX; - int best_act_threads = 0; + int best_act_hvx = 0; size_t best_m_chunk = 0; size_t best_n_chunk = 0; size_t best_vtcm_size = 0; const int m_for_chunks = is_matmul_id ? hex_align_up(m_id_rows, 32) : ne11_padded; - int act_threads = n_threads; - while (act_threads >= 1) { + int act_hvx = n_hvx; + while (act_hvx >= 1) { size_t simple_2d_overhead = htp_mm_hmx_get_2d_overhead(pipeline, is_matmul_id) + (bias_size > 0 ? hex_align_up(bias_size, HTP_MM_HMX_TILE_SIZE) : 0); size_t simple_2d_size_per_n, simple_2d_size_per_m, simple_2d_size_per_mn; htp_mm_hmx_get_2d_chunk_costs(wtype, k, pipeline, aligned_tile_size, &simple_2d_size_per_n, &simple_2d_size_per_m, &simple_2d_size_per_mn); @@ -738,30 +738,30 @@ static inline bool htp_mm_hmx_solve_2d_params( if (htp_mm_hmx_compute_chunks(vtcm_budget, simple_2d_overhead, simple_2d_size_per_n, simple_2d_size_per_m, simple_2d_size_per_mn, m_for_chunks, ne01_padded, (size_t) ne01_padded * HTP_MM_HMX_COST_W_DEQUANT, (size_t) m_for_cost * HTP_MM_HMX_COST_A_CONVERT, &m_chunk_candidate, &n_chunk_candidate, &vtcm_size_candidate) == 0) { - size_t exact_size = htp_mm_hmx_get_2d_vtcm_size(wtype, k, m_chunk_candidate, n_chunk_candidate, pipeline, is_matmul_id ? 0 : act_threads, aligned_tile_size, bias_size); + size_t exact_size = htp_mm_hmx_get_2d_vtcm_size(wtype, k, m_chunk_candidate, n_chunk_candidate, pipeline, is_matmul_id ? 0 : act_hvx, aligned_tile_size, bias_size); if (exact_size <= vtcm_budget) { size_t mblocks = ((size_t) m_for_cost + m_chunk_candidate - 1) / m_chunk_candidate; - if (mblocks < best_mblocks || (mblocks == best_mblocks && act_threads > best_act_threads)) { + if (mblocks < best_mblocks || (mblocks == best_mblocks && act_hvx > best_act_hvx)) { best_mblocks = mblocks; - best_act_threads = act_threads; + best_act_hvx = act_hvx; best_m_chunk = m_chunk_candidate; best_n_chunk = n_chunk_candidate; best_vtcm_size = exact_size; } } } - if (act_threads == 1) { - act_threads = 0; + if (act_hvx == 1) { + act_hvx = 0; } else { - act_threads /= 2; + act_hvx /= 2; } } - if (best_act_threads > 0) { + if (best_act_hvx > 0) { *m_chunk_out = best_m_chunk; *n_chunk_out = best_n_chunk; *vtcm_size_out = best_vtcm_size; - *act_threads_out = best_act_threads; + *act_hvx_out = best_act_hvx; return true; } return false; diff --git a/ggml/src/ggml-hexagon/htp/pad-ops.c b/ggml/src/ggml-hexagon/htp/pad-ops.c index cda6df318c21..b77e95a902bb 100644 --- a/ggml/src/ggml-hexagon/htp/pad-ops.c +++ b/ggml/src/ggml-hexagon/htp/pad-ops.c @@ -13,7 +13,7 @@ #define GGML_COMMON_DECL_C #include "ggml-common.h" #include "hex-common.h" -#include "hex-profile.h" +#include "htp-trace.h" #include "htp-ctx.h" #include "htp-ops.h" #include "htp-tensor.h" @@ -100,7 +100,7 @@ struct htp_pad_context { int32_t lp2, rp2; int32_t lp3, rp3; - uint32_t nrows_per_thread; + uint32_t nrows_per_hvx; uint32_t total_dst_rows; uint32_t row_start; @@ -136,8 +136,8 @@ struct htp_pad_context { \ const size_t type_size = pctx->type_size; \ \ - const uint32_t row_start = pctx->row_start + pctx->nrows_per_thread * ith; \ - const uint32_t row_end = MIN(row_start + pctx->nrows_per_thread, pctx->row_start + pctx->total_dst_rows); + const uint32_t row_start = pctx->row_start + pctx->nrows_per_hvx * ith; \ + const uint32_t row_end = MIN(row_start + pctx->nrows_per_hvx, pctx->row_start + pctx->total_dst_rows); #define htp_pad_dma_preamble \ @@ -146,21 +146,21 @@ struct htp_pad_context { const size_t dst_row_size = pctx->dst_row_size; \ const size_t dst_row_size_aligned = pctx->dst_row_size_aligned; \ \ - uint8_t * src_spad_base = octx->src0_spad.data + ith * octx->src0_spad.size_per_thread; \ - uint8_t * dst_spad_base = octx->dst_spad.data + ith * octx->dst_spad.size_per_thread; \ + uint8_t * src_spad_base = octx->src0_spad.data + ith * octx->src0_spad.size_per_hvx; \ + uint8_t * dst_spad_base = octx->dst_spad.data + ith * octx->dst_spad.size_per_hvx; \ \ - dma_queue * dma_q = octx->ctx->dma[ith]; + dma_queue * dma_q = th->dma_q; // --------------------------------------------------------------------------- // HVX vectorized PAD kernel // --------------------------------------------------------------------------- -static void pad_job_per_thread_hvx(unsigned int nth, unsigned int ith, void * data) { +static void pad_job_per_hvx_hvx(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { const struct htp_pad_context * pctx = (const struct htp_pad_context *) data; struct htp_ops_context * octx = pctx->octx; htp_pad_preamble; - struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + struct htp_thread_trace * tr = th->trace; htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, row_start); for (uint32_t dst_row = row_start; dst_row < row_end; dst_row++) { @@ -213,7 +213,7 @@ static void pad_job_per_thread_hvx(unsigned int nth, unsigned int ith, void * da // HVX + DMA PAD kernel - aligned, double-buffered // --------------------------------------------------------------------------- -static void pad_job_per_thread_hvx_dma(unsigned int nth, unsigned int ith, void * data) { +static void pad_job_per_hvx_hvx_dma(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { const struct htp_pad_context * pctx = (const struct htp_pad_context *) data; struct htp_ops_context * octx = pctx->octx; htp_pad_preamble; @@ -252,7 +252,7 @@ static void pad_job_per_thread_hvx_dma(unsigned int nth, unsigned int ith, void // Main loop: pop completed DMAs, compute in VTCM with aligned HVX ops, // push dst DMA and prefetch src for the next+1 row. // ----------------------------------------------------------------------- - struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + struct htp_thread_trace * tr = th->trace; for (uint32_t ir = row_start; ir < row_end; ir++) { uint8_t * dst_spad_cur = (uint8_t *) dma_queue_pop(dma_q).src; @@ -318,12 +318,12 @@ static void pad_job_per_thread_hvx_dma(unsigned int nth, unsigned int ith, void // HVX circular PAD kernel // --------------------------------------------------------------------------- -static void pad_job_per_thread_hvx_circular(unsigned int nth, unsigned int ith, void * data) { +static void pad_job_per_hvx_hvx_circular(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { const struct htp_pad_context * pctx = (const struct htp_pad_context *) data; struct htp_ops_context * octx = pctx->octx; htp_pad_preamble; - struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + struct htp_thread_trace * tr = th->trace; htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, row_start); for (uint32_t dst_row = row_start; dst_row < row_end; dst_row++) { @@ -387,7 +387,7 @@ static void pad_job_per_thread_hvx_circular(unsigned int nth, unsigned int ith, // HVX + DMA circular PAD kernel - aligned, double-buffered // --------------------------------------------------------------------------- -static void pad_job_per_thread_hvx_circular_dma(unsigned int nth, unsigned int ith, void * data) { +static void pad_job_per_hvx_hvx_circular_dma(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { const struct htp_pad_context * pctx = (const struct htp_pad_context *) data; struct htp_ops_context * octx = pctx->octx; htp_pad_preamble; @@ -417,7 +417,7 @@ static void pad_job_per_thread_hvx_circular_dma(unsigned int nth, unsigned int i // Main loop: pop completed DMAs, assemble circular row in VTCM with // aligned HVX ops, push dst DMA and prefetch src for the next+1 row. // ----------------------------------------------------------------------- - struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + struct htp_thread_trace * tr = th->trace; for (uint32_t ir = row_start; ir < row_end; ir++) { uint8_t * dst_spad_cur = (uint8_t *) dma_queue_pop(dma_q).src; @@ -521,11 +521,11 @@ int op_pad(struct htp_ops_context * octx) { return HTP_STATUS_OK; } - if (!htp_ops_context_set_n_threads(octx, MIN(kparams->n_threads, nrows))) { + if (!htp_ops_context_set_n_hvx(octx, MIN(kparams->n_hvx, nrows))) { return HTP_STATUS_INVAL_PARAMS; } - const uint32_t n_threads = octx->n_threads; + const uint32_t n_hvx = octx->n_hvx; if (!kparams->use_dma && (htp_tensor_is_extended(src0) || htp_tensor_is_extended(dst))) { return HTP_STATUS_NO_SUPPORT; @@ -533,12 +533,12 @@ int op_pad(struct htp_ops_context * octx) { struct htp_pad_vtcm_layout layout = { 0 }; if (kparams->use_dma) { - htp_pad_vtcm_layout_build(&layout, src_row_size, dst_row_size, n_threads); + htp_pad_vtcm_layout_build(&layout, src_row_size, dst_row_size, n_hvx); - octx->src0_spad.size_per_thread = layout.src0_bytes_per_thread; - octx->dst_spad.size_per_thread = layout.dst_bytes_per_thread; - octx->src0_spad.size = n_threads * layout.src0_bytes_per_thread; - octx->dst_spad.size = n_threads * layout.dst_bytes_per_thread; + octx->src0_spad.size_per_hvx = layout.src0_bytes_per_hvx; + octx->dst_spad.size_per_hvx = layout.dst_bytes_per_hvx; + octx->src0_spad.size = n_hvx * layout.src0_bytes_per_hvx; + octx->dst_spad.size = n_hvx * layout.dst_bytes_per_hvx; octx->src0_spad.data = VTCM_LAYOUT_PTR(uint8_t, octx->ctx->vtcm_base, layout.off_src0); octx->dst_spad.data = VTCM_LAYOUT_PTR(uint8_t, octx->ctx->vtcm_base, layout.off_dst); octx->src0_spad.src = NULL; @@ -552,7 +552,7 @@ int op_pad(struct htp_ops_context * octx) { .lp1 = lp1, .rp1 = rp1, .lp2 = lp2, .rp2 = rp2, .lp3 = lp3, .rp3 = rp3, - .nrows_per_thread = fastdiv(nrows + n_threads - 1, &octx->n_threads_div), + .nrows_per_hvx = fastdiv(nrows + n_hvx - 1, &octx->n_hvx_div), .total_dst_rows = nrows, .row_start = row_start, .type_size = type_size, @@ -571,16 +571,16 @@ int op_pad(struct htp_ops_context * octx) { switch (kparams->kernel_type) { case HTP_PAD_KERNEL_HVX_CIRCULAR_DMA: - work_queue_run(octx->ctx->work_queue, pad_job_per_thread_hvx_circular_dma, &pctx, n_threads); + work_queue_run(octx->ctx->hvx_queue, pad_job_per_hvx_hvx_circular_dma, &pctx, n_hvx); break; case HTP_PAD_KERNEL_HVX_CIRCULAR: - work_queue_run(octx->ctx->work_queue, pad_job_per_thread_hvx_circular, &pctx, n_threads); + work_queue_run(octx->ctx->hvx_queue, pad_job_per_hvx_hvx_circular, &pctx, n_hvx); break; case HTP_PAD_KERNEL_HVX_DMA: - work_queue_run(octx->ctx->work_queue, pad_job_per_thread_hvx_dma, &pctx, n_threads); + work_queue_run(octx->ctx->hvx_queue, pad_job_per_hvx_hvx_dma, &pctx, n_hvx); break; default: - work_queue_run(octx->ctx->work_queue, pad_job_per_thread_hvx, &pctx, n_threads); + work_queue_run(octx->ctx->hvx_queue, pad_job_per_hvx_hvx, &pctx, n_hvx); break; } diff --git a/ggml/src/ggml-hexagon/htp/pad-ops.h b/ggml/src/ggml-hexagon/htp/pad-ops.h index 5b8557538c60..15357e5f67de 100644 --- a/ggml/src/ggml-hexagon/htp/pad-ops.h +++ b/ggml/src/ggml-hexagon/htp/pad-ops.h @@ -19,31 +19,31 @@ struct htp_pad_vtcm_layout { size_t off_dst; size_t src_row_size_aligned; size_t dst_row_size_aligned; - size_t src0_bytes_per_thread; - size_t dst_bytes_per_thread; + size_t src0_bytes_per_hvx; + size_t dst_bytes_per_hvx; }; static inline void htp_pad_vtcm_layout_build( struct htp_pad_vtcm_layout * L, size_t src_row_size, size_t dst_row_size, - uint32_t n_threads + uint32_t n_hvx ) { L->src_row_size_aligned = hex_round_up(src_row_size, 128); L->dst_row_size_aligned = hex_round_up(dst_row_size, 128); - L->src0_bytes_per_thread = 2 * L->src_row_size_aligned; - L->dst_bytes_per_thread = 2 * L->dst_row_size_aligned; + L->src0_bytes_per_hvx = 2 * L->src_row_size_aligned; + L->dst_bytes_per_hvx = 2 * L->dst_row_size_aligned; size_t off = 0; - VTCM_LAYOUT_ALLOC(off, off_src0, L->src0_bytes_per_thread * n_threads); - VTCM_LAYOUT_ALLOC(off, off_dst, L->dst_bytes_per_thread * n_threads); + VTCM_LAYOUT_ALLOC(off, off_src0, L->src0_bytes_per_hvx * n_hvx); + VTCM_LAYOUT_ALLOC(off, off_dst, L->dst_bytes_per_hvx * n_hvx); L->total_bytes = off; } struct htp_pad_kernel_params { uint8_t kernel_type; - uint32_t n_threads; + uint32_t n_hvx; uint32_t total_rows; uint32_t use_dma; uint32_t elem_size; diff --git a/ggml/src/ggml-hexagon/htp/pool-ops.c b/ggml/src/ggml-hexagon/htp/pool-ops.c index b41c7bed23c7..12cd8ea90bf8 100644 --- a/ggml/src/ggml-hexagon/htp/pool-ops.c +++ b/ggml/src/ggml-hexagon/htp/pool-ops.c @@ -5,7 +5,7 @@ #include "hex-common.h" #include "dma-queue.h" -#include "hex-profile.h" +#include "htp-trace.h" #include "htp-ctx.h" #include "htp-ops.h" #include "htp-tensor.h" @@ -249,29 +249,29 @@ struct pool_2d_context { const struct htp_pool_2d_kernel_params * kparams; struct htp_pool_vtcm_layout layout; pool_plane_fn_t pool_plane; - uint32_t n_threads; + uint32_t n_hvx; uint32_t plane_start; uint32_t plane_count; - uint32_t planes_per_thread; + uint32_t planes_per_hvx; }; -static void pool_2d_thread(unsigned int nth, unsigned int ith, void * data) { +static void pool_2d_thread(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { struct pool_2d_context * ctx = (struct pool_2d_context *) data; const struct htp_pool_2d_kernel_params * p = ctx->kparams; const struct htp_pool_vtcm_layout * L = &ctx->layout; const struct htp_tensor * src0 = ctx->octx->src[0]; const struct htp_tensor * dst = ctx->octx->dst; pool_plane_fn_t pool_plane = ctx->pool_plane; - const uint32_t planes_per_thread = ctx->planes_per_thread; - const uint32_t first = ctx->plane_start + ith * planes_per_thread; - const uint32_t last = MIN(first + planes_per_thread, ctx->plane_start + ctx->plane_count); + const uint32_t planes_per_hvx = ctx->planes_per_hvx; + const uint32_t first = ctx->plane_start + ith * planes_per_hvx; + const uint32_t last = MIN(first + planes_per_hvx, ctx->plane_start + ctx->plane_count); if (first >= last) { return; } - struct htp_thread_trace * tr = &ctx->octx->ctx->trace[ith]; - dma_queue * dma_queue = ctx->octx->ctx->dma[ith]; + struct htp_thread_trace * tr = th->trace; + dma_queue * dma_queue = th->dma_q; const uint32_t src_plane_bytes = L->src_plane_bytes; const uint32_t dst_plane_bytes = L->dst_plane_bytes; @@ -279,8 +279,8 @@ static void pool_2d_thread(unsigned int nth, unsigned int ith, void * data) { const uint32_t dst_spad_half = (uint32_t) L->dst_spad_half_size; uint8_t * vtcm_base = (uint8_t *) ctx->octx->ctx->vtcm_base; - uint8_t * src_spad = VTCM_LAYOUT_PTR(uint8_t, vtcm_base, L->off_src) + ith * L->src_bytes_per_thread; - uint8_t * dst_spad = VTCM_LAYOUT_PTR(uint8_t, vtcm_base, L->off_dst) + ith * L->dst_bytes_per_thread; + uint8_t * src_spad = VTCM_LAYOUT_PTR(uint8_t, vtcm_base, L->off_src) + ith * L->src_bytes_per_hvx; + uint8_t * dst_spad = VTCM_LAYOUT_PTR(uint8_t, vtcm_base, L->off_dst) + ith * L->dst_bytes_per_hvx; float * srcb2[2] = { (float *) src_spad, (float *) (src_spad + src_spad_half) }; float * dstb2[2] = { (float *) dst_spad, (float *) (dst_spad + dst_spad_half) }; @@ -355,12 +355,12 @@ int op_pool_2d(struct htp_ops_context * octx) { return HTP_STATUS_OK; } - const uint32_t n_threads = MIN(p->n_threads, plane_count); - if (!htp_ops_context_set_n_threads(octx, n_threads)) { + const uint32_t n_hvx = MIN(p->n_hvx, plane_count); + if (!htp_ops_context_set_n_hvx(octx, n_hvx)) { return HTP_STATUS_INVAL_PARAMS; } - const uint32_t planes_per_thread = fastdiv(plane_count + n_threads - 1, &octx->n_threads_div); + const uint32_t planes_per_hvx = fastdiv(plane_count + n_hvx - 1, &octx->n_hvx_div); pool_plane_fn_t pool_plane; switch (p->kernel_type) { @@ -375,13 +375,13 @@ int op_pool_2d(struct htp_ops_context * octx) { .octx = octx, .kparams = p, .pool_plane = pool_plane, - .n_threads = n_threads, + .n_hvx = n_hvx, .plane_start = plane_start, .plane_count = plane_count, - .planes_per_thread = planes_per_thread, + .planes_per_hvx = planes_per_hvx, }; - htp_pool_vtcm_layout_build(&ctx.layout, p->src_x, p->src_y, p->dst_x, p->dst_y, n_threads); - work_queue_run(octx->ctx->work_queue, pool_2d_thread, &ctx, n_threads); + htp_pool_vtcm_layout_build(&ctx.layout, p->src_x, p->src_y, p->dst_x, p->dst_y, n_hvx); + work_queue_run(octx->ctx->hvx_queue, pool_2d_thread, &ctx, n_hvx); return HTP_STATUS_OK; } diff --git a/ggml/src/ggml-hexagon/htp/pool-ops.h b/ggml/src/ggml-hexagon/htp/pool-ops.h index 178e5d00fc41..e113bb9dd30d 100644 --- a/ggml/src/ggml-hexagon/htp/pool-ops.h +++ b/ggml/src/ggml-hexagon/htp/pool-ops.h @@ -29,7 +29,7 @@ struct htp_pool_2d_kernel_params { uint32_t stride_y; int32_t pad_x; int32_t pad_y; - uint32_t n_threads; + uint32_t n_hvx; uint32_t planes; uint32_t planes_per_chunk; uint32_t pool_op; @@ -51,8 +51,8 @@ struct htp_pool_vtcm_layout { size_t off_dst; uint32_t src_plane_bytes; uint32_t dst_plane_bytes; - size_t src_bytes_per_thread; - size_t dst_bytes_per_thread; + size_t src_bytes_per_hvx; + size_t dst_bytes_per_hvx; size_t src_spad_half_size; size_t dst_spad_half_size; }; @@ -63,19 +63,19 @@ static inline void htp_pool_vtcm_layout_build( uint32_t src_y, uint32_t dst_x, uint32_t dst_y, - uint32_t n_threads + uint32_t n_hvx ) { // Full-plane double buffering (256 bytes guard space for vector loads) L->src_plane_bytes = src_x * src_y * sizeof(float); L->dst_plane_bytes = dst_x * dst_y * sizeof(float); L->src_spad_half_size = hex_round_up(L->src_plane_bytes + 256, 128); L->dst_spad_half_size = hex_round_up(L->dst_plane_bytes, 128); - L->src_bytes_per_thread = 2 * L->src_spad_half_size; - L->dst_bytes_per_thread = 2 * L->dst_spad_half_size; + L->src_bytes_per_hvx = 2 * L->src_spad_half_size; + L->dst_bytes_per_hvx = 2 * L->dst_spad_half_size; size_t off = 0; - VTCM_LAYOUT_ALLOC(off, off_src, L->src_bytes_per_thread * n_threads); - VTCM_LAYOUT_ALLOC(off, off_dst, L->dst_bytes_per_thread * n_threads); + VTCM_LAYOUT_ALLOC(off, off_src, L->src_bytes_per_hvx * n_hvx); + VTCM_LAYOUT_ALLOC(off, off_dst, L->dst_bytes_per_hvx * n_hvx); L->total_bytes = off; } @@ -85,10 +85,10 @@ static inline bool htp_pool_solve_layout( uint32_t src_y, uint32_t dst_x, uint32_t dst_y, - uint32_t n_threads, + uint32_t n_hvx, size_t vtcm_budget ) { - htp_pool_vtcm_layout_build(layout, src_x, src_y, dst_x, dst_y, n_threads); + htp_pool_vtcm_layout_build(layout, src_x, src_y, dst_x, dst_y, n_hvx); return layout->total_bytes <= vtcm_budget; } diff --git a/ggml/src/ggml-hexagon/htp/repeat-ops.c b/ggml/src/ggml-hexagon/htp/repeat-ops.c index 827ef24c575b..a9e4d5c8636f 100644 --- a/ggml/src/ggml-hexagon/htp/repeat-ops.c +++ b/ggml/src/ggml-hexagon/htp/repeat-ops.c @@ -13,7 +13,7 @@ #include "ggml-common.h" #include "htp-ctx.h" #include "hex-common.h" -#include "hex-profile.h" +#include "htp-trace.h" #include "htp-ops.h" #include "htp-tensor.h" #include "repeat-ops.h" @@ -22,14 +22,14 @@ struct htp_repeat_context { struct htp_ops_context * octx; const struct htp_repeat_kernel_params * kparams; - uint32_t nrows_per_thread; + uint32_t nrows_per_hvx; uint32_t total_dst_rows; // ne1 * ne2 * ne3 uint32_t row_start; size_t type_size; }; -static void repeat_job_per_thread(unsigned int nth, unsigned int ith, void * data) { +static void repeat_job_per_hvx(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { const struct htp_repeat_context * rctx = (const struct htp_repeat_context *) data; struct htp_ops_context * octx = rctx->octx; const struct htp_repeat_kernel_params * kparams = rctx->kparams; @@ -60,13 +60,13 @@ static void repeat_job_per_thread(unsigned int nth, unsigned int ith, void * dat const size_t row_bytes = ne00 * rctx->type_size; - const uint32_t row_start = rctx->row_start + rctx->nrows_per_thread * ith; - const uint32_t row_end = MIN(row_start + rctx->nrows_per_thread, rctx->row_start + rctx->total_dst_rows); + const uint32_t row_start = rctx->row_start + rctx->nrows_per_hvx * ith; + const uint32_t row_end = MIN(row_start + rctx->nrows_per_hvx, rctx->row_start + rctx->total_dst_rows); if (row_start >= row_end) { return; } - struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + struct htp_thread_trace * tr = th->trace; htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, row_start); for (uint32_t dst_row = row_start; dst_row < row_end; dst_row++) { @@ -119,7 +119,7 @@ int op_repeat(struct htp_ops_context * octx) { const struct htp_repeat_kernel_params * kparams = (const struct htp_repeat_kernel_params *) octx->kernel_params; - if (!htp_ops_context_set_n_threads(octx, kparams->n_threads)) { + if (!htp_ops_context_set_n_hvx(octx, kparams->n_hvx)) { return HTP_STATUS_INVAL_PARAMS; } @@ -141,12 +141,12 @@ int op_repeat(struct htp_ops_context * octx) { return HTP_STATUS_OK; } - const uint32_t n_threads = octx->n_threads; + const uint32_t n_hvx = octx->n_hvx; struct htp_repeat_context rctx = { .octx = octx, .kparams = kparams, - .nrows_per_thread = fastdiv(nrows + n_threads - 1, &octx->n_threads_div), + .nrows_per_hvx = fastdiv(nrows + n_hvx - 1, &octx->n_hvx_div), .total_dst_rows = nrows, .row_start = row_start, .type_size = type_size, @@ -157,7 +157,7 @@ int op_repeat(struct htp_ops_context * octx) { dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3], kparams->nr0, kparams->nr1, kparams->nr2, kparams->nr3); - work_queue_run(octx->ctx->work_queue, repeat_job_per_thread, &rctx, n_threads); + work_queue_run(octx->ctx->hvx_queue, repeat_job_per_hvx, &rctx, n_hvx); return HTP_STATUS_OK; } diff --git a/ggml/src/ggml-hexagon/htp/repeat-ops.h b/ggml/src/ggml-hexagon/htp/repeat-ops.h index 47759e037d08..1561994d49db 100644 --- a/ggml/src/ggml-hexagon/htp/repeat-ops.h +++ b/ggml/src/ggml-hexagon/htp/repeat-ops.h @@ -5,7 +5,7 @@ #include struct htp_repeat_kernel_params { - uint32_t n_threads; + uint32_t n_hvx; uint32_t nr0; uint32_t nr1; uint32_t nr2; diff --git a/ggml/src/ggml-hexagon/htp/roll-ops.c b/ggml/src/ggml-hexagon/htp/roll-ops.c index 77ded0fe9022..aa7a0ad4b754 100644 --- a/ggml/src/ggml-hexagon/htp/roll-ops.c +++ b/ggml/src/ggml-hexagon/htp/roll-ops.c @@ -14,7 +14,7 @@ #include "ggml-common.h" #include "htp-ctx.h" #include "hex-common.h" -#include "hex-profile.h" +#include "htp-trace.h" #include "htp-ops.h" #include "htp-tensor.h" #include "roll-ops.h" @@ -25,7 +25,7 @@ struct htp_roll_context { uint32_t row_start; uint32_t nrows; - uint32_t nrows_per_thread; + uint32_t nrows_per_hvx; }; static inline uint32_t htp_roll_wrap(int32_t i, uint32_t ne) { @@ -199,19 +199,19 @@ static int roll_dma_f32_strided(struct htp_ops_context * octx) { return HTP_STATUS_OK; } -static void roll_thread_f32(unsigned int nth, unsigned int ith, void * data) { +static void roll_thread_f32(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { struct htp_roll_context * rctx = (struct htp_roll_context *) data; struct htp_ops_context * octx = rctx->octx; htp_roll_preamble; - const uint32_t row_start = rctx->row_start + rctx->nrows_per_thread * ith; - const uint32_t row_end = MIN(row_start + rctx->nrows_per_thread, rctx->row_start + rctx->nrows); + const uint32_t row_start = rctx->row_start + rctx->nrows_per_hvx * ith; + const uint32_t row_end = MIN(row_start + rctx->nrows_per_hvx, rctx->row_start + rctx->nrows); if (row_start >= row_end) { return; } - struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + struct htp_thread_trace * tr = th->trace; htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, row_start); for (uint32_t row = row_start; row < row_end; row++) { @@ -275,20 +275,20 @@ int execute_op_roll_f32(struct htp_ops_context * octx) { return HTP_STATUS_NO_SUPPORT; } - if (!htp_ops_context_set_n_threads(octx, MIN(kparams->n_threads, nrows))) { + if (!htp_ops_context_set_n_hvx(octx, MIN(kparams->n_hvx, nrows))) { return HTP_STATUS_INVAL_PARAMS; } - const uint32_t n_threads = octx->n_threads; + const uint32_t n_hvx = octx->n_hvx; struct htp_roll_context rctx = { .octx = octx, .kparams = kparams, .row_start = row_start, .nrows = nrows, - .nrows_per_thread = fastdiv(nrows + n_threads - 1, &octx->n_threads_div), + .nrows_per_hvx = fastdiv(nrows + n_hvx - 1, &octx->n_hvx_div), }; - work_queue_run(octx->ctx->work_queue, roll_thread_f32, &rctx, n_threads); + work_queue_run(octx->ctx->hvx_queue, roll_thread_f32, &rctx, n_hvx); return HTP_STATUS_OK; } diff --git a/ggml/src/ggml-hexagon/htp/roll-ops.h b/ggml/src/ggml-hexagon/htp/roll-ops.h index 2646aaa4c99c..f98fb8ea9074 100644 --- a/ggml/src/ggml-hexagon/htp/roll-ops.h +++ b/ggml/src/ggml-hexagon/htp/roll-ops.h @@ -12,7 +12,7 @@ enum htp_roll_kernel_type { struct htp_roll_kernel_params { uint8_t kernel_type; - uint32_t n_threads; + uint32_t n_hvx; uint32_t total_rows; uint32_t i0_src0; uint32_t n0; diff --git a/ggml/src/ggml-hexagon/htp/rope-ops.c b/ggml/src/ggml-hexagon/htp/rope-ops.c index 8c55ac3af49e..1c73de10b50f 100644 --- a/ggml/src/ggml-hexagon/htp/rope-ops.c +++ b/ggml/src/ggml-hexagon/htp/rope-ops.c @@ -66,12 +66,12 @@ struct htp_rope_context { float theta_powers[32]; float corr_dims[2]; - uint32_t src0_nrows_per_thread; + uint32_t src0_nrows_per_hvx; struct htp_ops_context * octx; uint8_t * vtcm_base; - size_t spad_per_thread; + size_t spad_per_hvx; size_t theta_cache_offset; size_t src0_row_size; @@ -532,7 +532,7 @@ static void inline rope_neox_f32_inplace(struct htp_rope_context * rctx, uint8_t } } -static void rope_job_f32(unsigned int nth, unsigned int ith, void * data) { +static void rope_job_f32(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { struct htp_rope_context * rctx = (struct htp_rope_context *) data; struct htp_ops_context * octx = rctx->octx; @@ -544,10 +544,10 @@ static void rope_job_f32(unsigned int nth, unsigned int ith, void * data) { htp_rope_preamble; const uint32_t src0_nrows = rctx->nrows; - const uint32_t src0_nrows_per_thread = rctx->src0_nrows_per_thread; + const uint32_t src0_nrows_per_hvx = rctx->src0_nrows_per_hvx; - const uint32_t src0_start_row = rctx->row_start + src0_nrows_per_thread * ith; - const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, rctx->row_start + src0_nrows); + const uint32_t src0_start_row = rctx->row_start + src0_nrows_per_hvx * ith; + const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_hvx, rctx->row_start + src0_nrows); // no work for this thread if (src0_start_row >= src0_end_row) { @@ -560,12 +560,12 @@ static void rope_job_f32(unsigned int nth, unsigned int ith, void * data) { const bool is_vision = (mode == HTP_ROPE_TYPE_VISION); // VTCM setup - uint8_t * src0_spad_base = rctx->vtcm_base + (ith * rctx->spad_per_thread); + uint8_t * src0_spad_base = rctx->vtcm_base + (ith * rctx->spad_per_hvx); float * theta_cache = (float *) (src0_spad_base); src0_spad_base = src0_spad_base + rctx->theta_cache_offset; - dma_queue * dma_q = octx->ctx->dma[ith]; - struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + dma_queue * dma_q = th->dma_q; + struct htp_thread_trace * tr = th->trace; const int32_t * pos = (const int32_t *) (uintptr_t) src1->data; const float * freq_factors = rctx->freq_factors; @@ -701,7 +701,7 @@ static int execute_op_rope_f32(struct htp_ops_context * octx) { const struct htp_tensor * dst = octx->dst; const struct htp_rope_kernel_params * kparams = (const struct htp_rope_kernel_params *) octx->kernel_params; - if (!htp_ops_context_set_n_threads(octx, kparams->n_threads)) { + if (!htp_ops_context_set_n_hvx(octx, kparams->n_hvx)) { return HTP_STATUS_INVAL_PARAMS; } assert(octx->ctx->vtcm_size >= kparams->vtcm_size); @@ -729,7 +729,7 @@ static int execute_op_rope_f32(struct htp_ops_context * octx) { return HTP_STATUS_OK; } - const uint32_t n_threads = octx->n_threads; + const uint32_t n_hvx = octx->n_hvx; const uint32_t ne0 = dst->ne[0]; const size_t src0_row_size = src0->ne[0] * sizeof(float); @@ -738,14 +738,14 @@ static int execute_op_rope_f32(struct htp_ops_context * octx) { const size_t dst_row_stride = dst->nb[1]; struct htp_rope_vtcm_layout layout; - htp_rope_vtcm_layout_build(&layout, src0->ne[0], kparams->n_threads, src2 ? src2->ne[0] : 0); + htp_rope_vtcm_layout_build(&layout, src0->ne[0], kparams->n_hvx, src2 ? src2->ne[0] : 0); struct htp_rope_context rctx; memset(&rctx, 0, sizeof(struct htp_rope_context)); rctx.octx = octx; rctx.vtcm_base = VTCM_LAYOUT_PTR(uint8_t, octx->ctx->vtcm_base, layout.off_spad); - rctx.spad_per_thread = layout.bytes_per_thread; + rctx.spad_per_hvx = layout.bytes_per_hvx; rctx.theta_cache_offset = layout.theta_cache_size_aligned; if (src2) { @@ -790,14 +790,14 @@ static int execute_op_rope_f32(struct htp_ops_context * octx) { rctx.src0_nrows = nrows; rctx.nrows = nrows; rctx.row_start = row_start; - rctx.src0_nrows_per_thread = fastdiv(nrows + n_threads - 1, &octx->n_threads_div); + rctx.src0_nrows_per_hvx = fastdiv(nrows + n_hvx - 1, &octx->n_hvx_div); rctx.div_ne2_ne1 = kparams->div_ne2_ne1; rctx.div_ne1 = kparams->div_ne1; FARF(HIGH, "rope-f32 n-rows %u n-dims %d ne0 %u ext-factor %.6f theta-scale %.6f attn-factor %.6f\n", rctx.src0_nrows, rctx.n_dims, ne0, rctx.ext_factor, rctx.theta_scale, rctx.attn_factor); - work_queue_run(octx->ctx->work_queue, rope_job_f32, &rctx, n_threads); + work_queue_run(octx->ctx->hvx_queue, rope_job_f32, &rctx, n_hvx); return err; } diff --git a/ggml/src/ggml-hexagon/htp/rope-ops.h b/ggml/src/ggml-hexagon/htp/rope-ops.h index 0c28ef826d5f..a21b85c0600c 100644 --- a/ggml/src/ggml-hexagon/htp/rope-ops.h +++ b/ggml/src/ggml-hexagon/htp/rope-ops.h @@ -10,7 +10,7 @@ #define HTP_ROPE_SPAD_NROWS (HTP_ROPE_SPAD_BLOCK * HTP_ROPE_SPAD_NSLOTS) struct htp_rope_kernel_params { - uint32_t n_threads; + uint32_t n_hvx; uint32_t vtcm_size; struct fastdiv_values div_ne2_ne1; @@ -27,7 +27,7 @@ struct htp_rope_vtcm_layout { size_t total_bytes; size_t off_spad; size_t off_freq_factors; - size_t bytes_per_thread; + size_t bytes_per_hvx; size_t theta_cache_size_aligned; size_t src0_row_size_aligned; size_t freq_factors_size_aligned; @@ -36,7 +36,7 @@ struct htp_rope_vtcm_layout { static inline void htp_rope_vtcm_layout_build( struct htp_rope_vtcm_layout * L, uint32_t ne00, - uint32_t n_threads, + uint32_t n_hvx, uint32_t n_freq_factors ) { const size_t src0_row_size = ne00 * sizeof(float); @@ -47,10 +47,10 @@ static inline void htp_rope_vtcm_layout_build( L->src0_row_size_aligned = src0_row_size_aligned; L->theta_cache_size_aligned = theta_cache_size_aligned; L->freq_factors_size_aligned = freq_factors_size_aligned; - L->bytes_per_thread = theta_cache_size_aligned + HTP_ROPE_SPAD_NROWS * src0_row_size_aligned; + L->bytes_per_hvx = theta_cache_size_aligned + HTP_ROPE_SPAD_NROWS * src0_row_size_aligned; size_t off = 0; - VTCM_LAYOUT_ALLOC(off, off_spad, L->bytes_per_thread * n_threads); + VTCM_LAYOUT_ALLOC(off, off_spad, L->bytes_per_hvx * n_hvx); VTCM_LAYOUT_ALLOC_OPTIONAL(off, off_freq_factors, freq_factors_size_aligned, n_freq_factors > 0); L->total_bytes = off; } diff --git a/ggml/src/ggml-hexagon/htp/set-rows-ops.c b/ggml/src/ggml-hexagon/htp/set-rows-ops.c index 191cf0c6afba..add91408fe5e 100644 --- a/ggml/src/ggml-hexagon/htp/set-rows-ops.c +++ b/ggml/src/ggml-hexagon/htp/set-rows-ops.c @@ -61,32 +61,33 @@ struct set_rows_context { uint8_t * vtcm_base; uint32_t task_start; uint32_t tasks; - uint32_t tasks_per_thread; + uint32_t tasks_per_hvx; }; #define SET_ROWS_THREAD_DMA_FN(TYPE_NAME, IDX_TYPE, COMPUTE_EXPR) \ -static void set_rows_thread_dma_##TYPE_NAME##_##IDX_TYPE(unsigned int nth, unsigned int ith, void *data) { \ +static void set_rows_thread_dma_##TYPE_NAME##_##IDX_TYPE(const htp_thread_t * th, unsigned int nth, \ + unsigned int ith, void *data) { \ struct set_rows_context * srctx = (struct set_rows_context *)data; \ struct htp_ops_context * octx = srctx->octx; \ const struct htp_set_rows_kernel_params * kparams = srctx->kparams; \ set_rows_preamble; \ - struct htp_thread_trace * tr = &octx->ctx->trace[ith]; \ - const uint32_t dr = srctx->tasks_per_thread; \ + struct htp_thread_trace * tr = th->trace; \ + const uint32_t dr = srctx->tasks_per_hvx; \ const uint32_t ir0 = srctx->task_start + dr * ith; \ if (ir0 >= srctx->task_start + srctx->tasks) { \ return; \ } \ const uint32_t ir1 = MIN(ir0 + dr, srctx->task_start + srctx->tasks); \ - dma_queue * dma_q = octx->ctx->dma[ith]; \ + dma_queue * dma_q = th->dma_q; \ const struct htp_set_rows_vtcm_layout * vtcm_layout = &srctx->vtcm_layout; \ uint8_t * vtcm_src0 = VTCM_LAYOUT_PTR(uint8_t, srctx->vtcm_base, vtcm_layout->off_src0) + \ - ith * vtcm_layout->src0_bytes_per_thread; \ + ith * vtcm_layout->src0_bytes_per_hvx; \ uint8_t * vtcm_dst = VTCM_LAYOUT_PTR(uint8_t, srctx->vtcm_base, vtcm_layout->off_dst) + \ - ith * vtcm_layout->dst_bytes_per_thread; \ + ith * vtcm_layout->dst_bytes_per_hvx; \ const uint32_t src0_row_size = ne00 * sizeof(float); \ const uint32_t dst_row_size = htp_tensor_get_row_size(octx->dst->type, ne00); \ - const uint32_t nrows_per_thread = ir1 - ir0; \ - const uint32_t total_steps = ne03 * ne02 * nrows_per_thread; \ + const uint32_t nrows_per_hvx = ir1 - ir0; \ + const uint32_t total_steps = ne03 * ne02 * nrows_per_hvx; \ uint32_t pi_step = 0; \ uint32_t pi02 = 0; \ uint32_t pi03 = 0; \ @@ -102,7 +103,7 @@ static void set_rows_thread_dma_##TYPE_NAME##_##IDX_TYPE(unsigned int nth, unsig src0_data), \ vtcm_layout->src0_spad_half_size, src0_row_size, src0_row_size, 1); \ pi_step++; \ - if (pi_step == nrows_per_thread) { \ + if (pi_step == nrows_per_hvx) { \ pi_step = 0; \ pi02++; \ if (pi02 == ne02) { \ @@ -147,7 +148,7 @@ static void set_rows_thread_dma_##TYPE_NAME##_##IDX_TYPE(unsigned int nth, unsig dma_make_data(src_spad, psrc0_data), \ vtcm_layout->src0_spad_half_size, src0_row_size, src0_row_size, 1); \ pi_step++; \ - if (pi_step == nrows_per_thread) { \ + if (pi_step == nrows_per_hvx) { \ pi_step = 0; \ pi02++; \ if (pi02 == ne02) { \ @@ -157,7 +158,7 @@ static void set_rows_thread_dma_##TYPE_NAME##_##IDX_TYPE(unsigned int nth, unsig } \ } \ ci_step++; \ - if (ci_step == nrows_per_thread) { \ + if (ci_step == nrows_per_hvx) { \ ci_step = 0; \ ci02++; \ ci11_base++; \ @@ -211,11 +212,11 @@ int op_set_rows(struct htp_ops_context * octx) { return HTP_STATUS_OK; } - if (!htp_ops_context_set_n_threads(octx, (uint32_t) kparams->n_threads)) { + if (!htp_ops_context_set_n_hvx(octx, (uint32_t) kparams->n_hvx)) { return HTP_STATUS_INVAL_PARAMS; } - const uint32_t n_threads = octx->n_threads; + const uint32_t n_hvx = octx->n_hvx; // l2fetch the src1 (indices) tensor in the main thread hex_l2fetch_block((const void *)octx->src[1]->data, octx->src[1]->ne[3] * octx->src[1]->nb[3]); @@ -225,9 +226,9 @@ int op_set_rows(struct htp_ops_context * octx) { srctx.kparams = kparams; srctx.task_start = task_start; srctx.tasks = tasks; - srctx.tasks_per_thread = fastdiv(tasks + n_threads - 1, &octx->n_threads_div); + srctx.tasks_per_hvx = fastdiv(tasks + n_hvx - 1, &octx->n_hvx_div); - htp_set_rows_vtcm_layout_build(&srctx.vtcm_layout, octx->dst->type, ne00, n_threads); + htp_set_rows_vtcm_layout_build(&srctx.vtcm_layout, octx->dst->type, ne00, n_hvx); srctx.vtcm_base = (uint8_t *)octx->ctx->vtcm_base; work_queue_func_t q_func = NULL; @@ -240,15 +241,15 @@ int op_set_rows(struct htp_ops_context * octx) { default: return HTP_STATUS_NO_SUPPORT; } - FARF(HIGH, "set-rows: (%ux%ux%ux%u) x (%ux%ux%ux%u) -> (%ux%ux%ux%u) : src0-vtcm-size %zu dst-vtcm-size %zu n-threads %d\n", + FARF(HIGH, "set-rows: (%ux%ux%ux%u) x (%ux%ux%ux%u) -> (%ux%ux%ux%u) : src0-vtcm-size %zu dst-vtcm-size %zu n-hvx %d\n", octx->src[0]->ne[0], octx->src[0]->ne[1], octx->src[0]->ne[2], octx->src[0]->ne[3], octx->src[1]->ne[0], octx->src[1]->ne[1], octx->src[1]->ne[2], octx->src[1]->ne[3], octx->dst->ne[0], octx->dst->ne[1], octx->dst->ne[2], octx->dst->ne[3], - srctx.vtcm_layout.src0_bytes_per_thread * n_threads, - srctx.vtcm_layout.dst_bytes_per_thread * n_threads, - n_threads); + srctx.vtcm_layout.src0_bytes_per_hvx * n_hvx, + srctx.vtcm_layout.dst_bytes_per_hvx * n_hvx, + n_hvx); - work_queue_run(octx->ctx->work_queue, q_func, &srctx, n_threads); + work_queue_run(octx->ctx->hvx_queue, q_func, &srctx, n_hvx); return HTP_STATUS_OK; } diff --git a/ggml/src/ggml-hexagon/htp/set-rows-ops.h b/ggml/src/ggml-hexagon/htp/set-rows-ops.h index b9cb6cc90a56..85086b08ef6a 100644 --- a/ggml/src/ggml-hexagon/htp/set-rows-ops.h +++ b/ggml/src/ggml-hexagon/htp/set-rows-ops.h @@ -6,7 +6,7 @@ #include "htp-vtcm.h" struct htp_set_rows_kernel_params { - uint32_t n_threads; + uint32_t n_hvx; uint32_t total_tasks; uint32_t vtcm_size; }; @@ -16,8 +16,8 @@ struct htp_set_rows_vtcm_layout { size_t off_src0; size_t off_dst; - size_t src0_bytes_per_thread; - size_t dst_bytes_per_thread; + size_t src0_bytes_per_hvx; + size_t dst_bytes_per_hvx; size_t src0_spad_half_size; size_t dst_spad_half_size; @@ -27,7 +27,7 @@ static inline void htp_set_rows_vtcm_layout_build( struct htp_set_rows_vtcm_layout * L, int dst_type, uint32_t ne00, - uint32_t n_threads) { + uint32_t n_hvx) { size_t src0_row_size = ne00 * sizeof(float); size_t dst_row_size = htp_tensor_get_row_size(dst_type, ne00); @@ -38,12 +38,12 @@ static inline void htp_set_rows_vtcm_layout_build( L->src0_spad_half_size = src0_row_size_aligned; L->dst_spad_half_size = dst_row_size_aligned; - L->src0_bytes_per_thread = src0_row_size_aligned * 2; - L->dst_bytes_per_thread = dst_row_size_aligned * 2; + L->src0_bytes_per_hvx = src0_row_size_aligned * 2; + L->dst_bytes_per_hvx = dst_row_size_aligned * 2; size_t off = 0; - VTCM_LAYOUT_ALLOC(off, off_src0, L->src0_bytes_per_thread * n_threads); - VTCM_LAYOUT_ALLOC(off, off_dst, L->dst_bytes_per_thread * n_threads); + VTCM_LAYOUT_ALLOC(off, off_src0, L->src0_bytes_per_hvx * n_hvx); + VTCM_LAYOUT_ALLOC(off, off_dst, L->dst_bytes_per_hvx * n_hvx); L->total_bytes = off; } diff --git a/ggml/src/ggml-hexagon/htp/softmax-ops.c b/ggml/src/ggml-hexagon/htp/softmax-ops.c index d65c3fae33a6..22aebcd5e650 100644 --- a/ggml/src/ggml-hexagon/htp/softmax-ops.c +++ b/ggml/src/ggml-hexagon/htp/softmax-ops.c @@ -13,7 +13,7 @@ #include "hvx-utils.h" #include "hex-fastdiv.h" #include "hex-common.h" -#include "hex-profile.h" +#include "htp-trace.h" #define GGML_COMMON_DECL_C #include "ggml-common.h" @@ -38,9 +38,9 @@ struct htp_softmax_context { uint8_t * vtcm_src1; uint8_t * vtcm_dst; - uint32_t vtcm_src0_size_per_thread; - uint32_t vtcm_src1_size_per_thread; - uint32_t vtcm_dst_size_per_thread; + uint32_t vtcm_src0_size_per_hvx; + uint32_t vtcm_src1_size_per_hvx; + uint32_t vtcm_dst_size_per_hvx; uint32_t src0_spad_half_size; uint32_t src1_spad_half_size; @@ -65,7 +65,7 @@ struct htp_softmax_context { struct fastdiv_values div_ne12; struct fastdiv_values div_ne13; - uint32_t src0_nrows_per_thread; + uint32_t src0_nrows_per_hvx; uint32_t row_start; uint32_t nrows; @@ -287,19 +287,19 @@ static const softmax_compute_fn_t softmax_kernels[HTP_SOFTMAX_KERNEL_COUNT] = { [HTP_SOFTMAX_KERNEL_MASK_F16] = compute_fast_softmax_f32_mask_f16, }; -static void softmax_thread_dma(unsigned int nth, unsigned int ith, void * data) { +static void softmax_thread_dma(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { (void) nth; const struct htp_softmax_context * smctx = (const struct htp_softmax_context *) data; struct htp_ops_context * octx = smctx->octx; const struct htp_tensor * src0 = octx->src[0]; const struct htp_tensor * dst = octx->dst; - struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + struct htp_thread_trace * tr = th->trace; const uint32_t src0_nrows = smctx->nrows; - const uint32_t src0_nrows_per_thread = smctx->src0_nrows_per_thread; + const uint32_t src0_nrows_per_hvx = smctx->src0_nrows_per_hvx; - const uint32_t src0_start_row = smctx->row_start + src0_nrows_per_thread * ith; - const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, smctx->row_start + src0_nrows); + const uint32_t src0_start_row = smctx->row_start + src0_nrows_per_hvx * ith; + const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_hvx, smctx->row_start + src0_nrows); if (src0_start_row >= src0_end_row) { return; @@ -311,13 +311,13 @@ static void softmax_thread_dma(unsigned int nth, unsigned int ith, void * data) const size_t src0_row_size = src0->ne[0] * sizeof(float); const size_t dst_row_size = src0->ne[0] * sizeof(float); - uint8_t * src0_vtcm_base = smctx->vtcm_src0 + (ith * smctx->vtcm_src0_size_per_thread); - uint8_t * dst_vtcm_base = smctx->vtcm_dst + (ith * smctx->vtcm_dst_size_per_thread); + uint8_t * src0_vtcm_base = smctx->vtcm_src0 + (ith * smctx->vtcm_src0_size_per_hvx); + uint8_t * dst_vtcm_base = smctx->vtcm_dst + (ith * smctx->vtcm_dst_size_per_hvx); const size_t src0_vtcm_half = smctx->src0_spad_half_size; const size_t dst_vtcm_half = smctx->dst_spad_half_size; - dma_queue * dma_q = octx->ctx->dma[ith]; + dma_queue * dma_q = th->dma_q; for (uint32_t r = src0_start_row, idx = 0; r < src0_end_row && idx < 2; r++, idx++) { dma_addr_t cur_dst = data_dst + r * dst_row_size; @@ -357,20 +357,20 @@ static void softmax_thread_dma(unsigned int nth, unsigned int ith, void * data) dma_queue_flush(dma_q); } -static void softmax_thread_mask_dma(unsigned int nth, unsigned int ith, void * data) { +static void softmax_thread_mask_dma(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { (void) nth; const struct htp_softmax_context * smctx = (const struct htp_softmax_context *) data; struct htp_ops_context * octx = smctx->octx; const struct htp_tensor * src0 = octx->src[0]; const struct htp_tensor * src1 = octx->src[1]; const struct htp_tensor * dst = octx->dst; - struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + struct htp_thread_trace * tr = th->trace; const uint32_t src0_nrows = smctx->nrows; - const uint32_t src0_nrows_per_thread = smctx->src0_nrows_per_thread; + const uint32_t src0_nrows_per_hvx = smctx->src0_nrows_per_hvx; - const uint32_t src0_start_row = smctx->row_start + src0_nrows_per_thread * ith; - const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, smctx->row_start + src0_nrows); + const uint32_t src0_start_row = smctx->row_start + src0_nrows_per_hvx * ith; + const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_hvx, smctx->row_start + src0_nrows); if (src0_start_row >= src0_end_row) { return; @@ -384,9 +384,9 @@ static void softmax_thread_mask_dma(unsigned int nth, unsigned int ith, void * d const size_t dst_row_size = src0->ne[0] * sizeof(float); const size_t mask_row_size = smctx->use_f16 ? (src1->ne[0] * sizeof(__fp16)) : (src1->ne[0] * sizeof(float)); - uint8_t * src0_vtcm_base = smctx->vtcm_src0 + (ith * smctx->vtcm_src0_size_per_thread); - uint8_t * src1_vtcm_base = smctx->vtcm_src1 + (ith * smctx->vtcm_src1_size_per_thread); - uint8_t * dst_vtcm_base = smctx->vtcm_dst + (ith * smctx->vtcm_dst_size_per_thread); + uint8_t * src0_vtcm_base = smctx->vtcm_src0 + (ith * smctx->vtcm_src0_size_per_hvx); + uint8_t * src1_vtcm_base = smctx->vtcm_src1 + (ith * smctx->vtcm_src1_size_per_hvx); + uint8_t * dst_vtcm_base = smctx->vtcm_dst + (ith * smctx->vtcm_dst_size_per_hvx); const size_t src0_vtcm_half = smctx->src0_spad_half_size; const size_t src1_vtcm_half = smctx->src1_spad_half_size; @@ -408,7 +408,7 @@ static void softmax_thread_mask_dma(unsigned int nth, unsigned int ith, void * d const struct fastdiv_values * div_ne12 = &smctx->div_ne12; const struct fastdiv_values * div_ne13 = &smctx->div_ne13; - dma_queue * dma_q = octx->ctx->dma[ith]; + dma_queue * dma_q = th->dma_q; for (uint32_t r = src0_start_row, idx = 0; r < src0_end_row && idx < 2; r++, idx++) { dma_addr_t cur_dst = data_dst + r * dst_row_size; @@ -492,7 +492,7 @@ static int execute_op_softmax_f32(struct htp_ops_context * octx) { const struct htp_softmax_kernel_params * kparams = (const struct htp_softmax_kernel_params *) octx->kernel_params; - if (!htp_ops_context_set_n_threads(octx, kparams->n_threads)) { + if (!htp_ops_context_set_n_hvx(octx, kparams->n_hvx)) { return HTP_STATUS_INVAL_PARAMS; } @@ -520,8 +520,8 @@ static int execute_op_softmax_f32(struct htp_ops_context * octx) { src0_nrows, rows_per_chunk, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div); row_start = range.start; nrows = range.count; - if (nrows < octx->n_threads) { - htp_ops_context_set_n_threads(octx, nrows ? nrows : 1); + if (nrows < octx->n_hvx) { + htp_ops_context_set_n_hvx(octx, nrows ? nrows : 1); } } @@ -529,7 +529,7 @@ static int execute_op_softmax_f32(struct htp_ops_context * octx) { return HTP_STATUS_OK; } - const uint32_t n_threads = octx->n_threads; + const uint32_t n_hvx = octx->n_hvx; uint8_t * const vtcm_base = (uint8_t *) octx->ctx->vtcm_base; struct htp_softmax_vtcm_layout layout; @@ -538,7 +538,7 @@ static int execute_op_softmax_f32(struct htp_ops_context * octx) { kparams->use_src1 ? octx->src[1]->ne[0] : 0, kparams->use_src1 != 0, kparams->use_f16 != 0, - kparams->n_threads); + kparams->n_hvx); struct htp_softmax_context smctx = { .octx = octx, @@ -553,9 +553,9 @@ static int execute_op_softmax_f32(struct htp_ops_context * octx) { .vtcm_dst = VTCM_LAYOUT_PTR(uint8_t, vtcm_base, layout.off_dst), .vtcm_src1 = VTCM_LAYOUT_PTR_OPTIONAL(uint8_t, vtcm_base, layout.off_src1, kparams->use_src1), - .vtcm_src0_size_per_thread = (uint32_t) layout.src0_bytes_per_thread, - .vtcm_src1_size_per_thread = (uint32_t) layout.src1_bytes_per_thread, - .vtcm_dst_size_per_thread = (uint32_t) layout.dst_bytes_per_thread, + .vtcm_src0_size_per_hvx = (uint32_t) layout.src0_bytes_per_hvx, + .vtcm_src1_size_per_hvx = (uint32_t) layout.src1_bytes_per_hvx, + .vtcm_dst_size_per_hvx = (uint32_t) layout.dst_bytes_per_hvx, .src0_spad_half_size = (uint32_t) layout.src0_spad_half_size, .src1_spad_half_size = (uint32_t) layout.src1_spad_half_size, @@ -580,7 +580,7 @@ static int execute_op_softmax_f32(struct htp_ops_context * octx) { .div_ne12 = kparams->div_ne12, .div_ne13 = kparams->div_ne13, - .src0_nrows_per_thread = fastdiv(nrows + n_threads - 1, &octx->n_threads_div), + .src0_nrows_per_hvx = fastdiv(nrows + n_hvx - 1, &octx->n_hvx_div), .row_start = row_start, .nrows = nrows, }; @@ -596,7 +596,7 @@ static int execute_op_softmax_f32(struct htp_ops_context * octx) { } work_queue_func_t task_func = kparams->use_src1 ? softmax_thread_mask_dma : softmax_thread_dma; - work_queue_run(octx->ctx->work_queue, task_func, &smctx, n_threads); + work_queue_run(octx->ctx->hvx_queue, task_func, &smctx, n_hvx); return HTP_STATUS_OK; } diff --git a/ggml/src/ggml-hexagon/htp/softmax-ops.h b/ggml/src/ggml-hexagon/htp/softmax-ops.h index ef6567306d42..9e5f9c759f51 100644 --- a/ggml/src/ggml-hexagon/htp/softmax-ops.h +++ b/ggml/src/ggml-hexagon/htp/softmax-ops.h @@ -18,7 +18,7 @@ enum htp_softmax_kernel_type { struct htp_softmax_kernel_params { uint8_t kernel_type; - uint32_t n_threads; + uint32_t n_hvx; uint32_t vtcm_size; uint32_t n_head; @@ -49,9 +49,9 @@ struct htp_softmax_vtcm_layout { size_t off_dst; size_t off_src1; - size_t src0_bytes_per_thread; - size_t dst_bytes_per_thread; - size_t src1_bytes_per_thread; + size_t src0_bytes_per_hvx; + size_t dst_bytes_per_hvx; + size_t src1_bytes_per_hvx; size_t src0_spad_half_size; size_t dst_spad_half_size; @@ -64,7 +64,7 @@ static inline void htp_softmax_vtcm_layout_build( uint32_t ne10, bool use_src1, bool use_f16, - uint32_t n_threads + uint32_t n_hvx ) { size_t src0_row_size = ne00 * sizeof(float); size_t dst_row_size = ne00 * sizeof(float); @@ -79,14 +79,14 @@ static inline void htp_softmax_vtcm_layout_build( L->src1_spad_half_size = src1_row_size_aligned; // Double buffering: 2 half-buffers per thread - L->src0_bytes_per_thread = src0_row_size_aligned * 2; - L->dst_bytes_per_thread = dst_row_size_aligned * 2; - L->src1_bytes_per_thread = src1_row_size_aligned * 2; + L->src0_bytes_per_hvx = src0_row_size_aligned * 2; + L->dst_bytes_per_hvx = dst_row_size_aligned * 2; + L->src1_bytes_per_hvx = src1_row_size_aligned * 2; size_t off = 0; - VTCM_LAYOUT_ALLOC(off, off_src0, L->src0_bytes_per_thread * n_threads); - VTCM_LAYOUT_ALLOC(off, off_dst, L->dst_bytes_per_thread * n_threads); - VTCM_LAYOUT_ALLOC_OPTIONAL(off, off_src1, L->src1_bytes_per_thread * n_threads, use_src1); + VTCM_LAYOUT_ALLOC(off, off_src0, L->src0_bytes_per_hvx * n_hvx); + VTCM_LAYOUT_ALLOC(off, off_dst, L->dst_bytes_per_hvx * n_hvx); + VTCM_LAYOUT_ALLOC_OPTIONAL(off, off_src1, L->src1_bytes_per_hvx * n_hvx, use_src1); L->total_bytes = off; } diff --git a/ggml/src/ggml-hexagon/htp/solve-tri-ops.c b/ggml/src/ggml-hexagon/htp/solve-tri-ops.c index 628ca65e8451..73eff927edff 100644 --- a/ggml/src/ggml-hexagon/htp/solve-tri-ops.c +++ b/ggml/src/ggml-hexagon/htp/solve-tri-ops.c @@ -4,7 +4,7 @@ #include #include "hex-common.h" -#include "hex-profile.h" +#include "htp-trace.h" #define GGML_COMMON_DECL_C #include "ggml-common.h" @@ -18,7 +18,7 @@ struct htp_solve_tri_context { struct htp_ops_context * octx; const struct htp_solve_tri_kernel_params * kparams; - uint32_t jobs_per_thread; + uint32_t jobs_per_hvx; uint32_t total_jobs; uint32_t job_start; uint32_t k_chunks; @@ -79,7 +79,7 @@ static inline void solve_tri_row_hvx(const float * A_row, } // Batch-level thread: each job is one full batch. -static void solve_tri_batch_thread_f32(unsigned int nth, unsigned int ith, void * data) { +static void solve_tri_batch_thread_f32(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { struct htp_solve_tri_context * sctx = (struct htp_solve_tri_context *) data; struct htp_ops_context * octx = sctx->octx; const struct htp_solve_tri_kernel_params * kparams = sctx->kparams; @@ -96,10 +96,10 @@ static void solve_tri_batch_thread_f32(unsigned int nth, unsigned int ith, void const uint32_t col_block = VLEN_FP32; const uint32_t k_full = k & ~(col_block - 1); - const uint32_t start_batch = sctx->job_start + sctx->jobs_per_thread * ith; - const uint32_t end_batch = MIN(start_batch + sctx->jobs_per_thread, sctx->job_start + sctx->total_jobs); + const uint32_t start_batch = sctx->job_start + sctx->jobs_per_hvx * ith; + const uint32_t end_batch = MIN(start_batch + sctx->jobs_per_hvx, sctx->job_start + sctx->total_jobs); - struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + struct htp_thread_trace * tr = th->trace; htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) start_batch); for (uint32_t batch = start_batch; batch < end_batch; ++batch) { @@ -141,7 +141,7 @@ static void solve_tri_batch_thread_f32(unsigned int nth, unsigned int ith, void } // Chunk-level thread: each job is one (batch, col_chunk) pair. -static void solve_tri_chunk_thread_f32(unsigned int nth, unsigned int ith, void * data) { +static void solve_tri_chunk_thread_f32(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { struct htp_solve_tri_context * sctx = (struct htp_solve_tri_context *) data; struct htp_ops_context * octx = sctx->octx; const struct htp_solve_tri_kernel_params * kparams = sctx->kparams; @@ -155,10 +155,10 @@ static void solve_tri_chunk_thread_f32(unsigned int nth, unsigned int ith, void const uint32_t ne02 = src0->ne[2]; - const uint32_t start_job = sctx->job_start + sctx->jobs_per_thread * ith; - const uint32_t end_job = MIN(start_job + sctx->jobs_per_thread, sctx->job_start + sctx->total_jobs); + const uint32_t start_job = sctx->job_start + sctx->jobs_per_hvx * ith; + const uint32_t end_job = MIN(start_job + sctx->jobs_per_hvx, sctx->job_start + sctx->total_jobs); - struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + struct htp_thread_trace * tr = th->trace; htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) start_job); for (uint32_t job = start_job; job < end_job; ++job) { @@ -233,23 +233,23 @@ int op_solve_tri(struct htp_ops_context * octx) { return HTP_STATUS_OK; } - if (!htp_ops_context_set_n_threads(octx, MIN(kparams->n_threads, njobs))) { + if (!htp_ops_context_set_n_hvx(octx, MIN(kparams->n_hvx, njobs))) { return HTP_STATUS_INVAL_PARAMS; } - const uint32_t n_threads = octx->n_threads; + const uint32_t n_hvx = octx->n_hvx; struct htp_solve_tri_context sctx = { .octx = octx, .kparams = kparams, - .jobs_per_thread = fastdiv(njobs + n_threads - 1, &octx->n_threads_div), + .jobs_per_hvx = fastdiv(njobs + n_hvx - 1, &octx->n_hvx_div), .total_jobs = njobs, .job_start = job_start, .k_chunks = k_chunks, .col_block = col_block, }; - work_queue_run(octx->ctx->work_queue, solve_tri_batch_thread_f32, &sctx, n_threads); + work_queue_run(octx->ctx->hvx_queue, solve_tri_batch_thread_f32, &sctx, n_hvx); } else { const uint32_t total_jobs = kparams->total_jobs; @@ -267,23 +267,23 @@ int op_solve_tri(struct htp_ops_context * octx) { return HTP_STATUS_OK; } - if (!htp_ops_context_set_n_threads(octx, MIN(kparams->n_threads, njobs))) { + if (!htp_ops_context_set_n_hvx(octx, MIN(kparams->n_hvx, njobs))) { return HTP_STATUS_INVAL_PARAMS; } - const uint32_t n_threads = octx->n_threads; + const uint32_t n_hvx = octx->n_hvx; struct htp_solve_tri_context sctx = { .octx = octx, .kparams = kparams, - .jobs_per_thread = fastdiv(njobs + n_threads - 1, &octx->n_threads_div), + .jobs_per_hvx = fastdiv(njobs + n_hvx - 1, &octx->n_hvx_div), .total_jobs = njobs, .job_start = job_start, .k_chunks = k_chunks, .col_block = col_block, }; - work_queue_run(octx->ctx->work_queue, solve_tri_chunk_thread_f32, &sctx, n_threads); + work_queue_run(octx->ctx->hvx_queue, solve_tri_chunk_thread_f32, &sctx, n_hvx); } return HTP_STATUS_OK; diff --git a/ggml/src/ggml-hexagon/htp/solve-tri-ops.h b/ggml/src/ggml-hexagon/htp/solve-tri-ops.h index 327415814bf4..6a46cfdc414d 100644 --- a/ggml/src/ggml-hexagon/htp/solve-tri-ops.h +++ b/ggml/src/ggml-hexagon/htp/solve-tri-ops.h @@ -5,7 +5,7 @@ #include struct htp_solve_tri_kernel_params { - uint32_t n_threads; + uint32_t n_hvx; uint32_t total_batches; uint32_t total_jobs; uint32_t k_chunks; diff --git a/ggml/src/ggml-hexagon/htp/ssm-conv.c b/ggml/src/ggml-hexagon/htp/ssm-conv.c index 4b06b18f2cd2..d75304fef11d 100644 --- a/ggml/src/ggml-hexagon/htp/ssm-conv.c +++ b/ggml/src/ggml-hexagon/htp/ssm-conv.c @@ -15,7 +15,7 @@ #include "ggml-common.h" #include "htp-ctx.h" #include "dma-queue.h" -#include "hex-profile.h" +#include "htp-trace.h" #include "htp-ops.h" #include "htp-tensor.h" #include "hvx-utils.h" @@ -24,7 +24,7 @@ struct htp_ssm_conv_context { struct htp_ops_context * octx; const struct htp_ssm_conv_kernel_params * kparams; - uint32_t nrows_per_thread; + uint32_t nrows_per_hvx; uint32_t d_inner_tile; uint32_t row_start; uint32_t nrows; @@ -80,8 +80,8 @@ static inline void hvx_transpose_32x32_f32(HVX_Vector m[32]) { } // HVX deinterleave for d_conv == 4: channel-major raw VTCM -> tap-major T VTCM -static inline void hvx_ssm_conv_unpack_to_T_4(const float * raw, float * T, uint32_t d_inner_per_thread, uint32_t d_inner_stride) { - for (uint32_t cb = 0; cb < d_inner_per_thread; cb += VLEN_FP32) { +static inline void hvx_ssm_conv_unpack_to_T_4(const float * raw, float * T, uint32_t d_inner_per_hvx, uint32_t d_inner_stride) { + for (uint32_t cb = 0; cb < d_inner_per_hvx; cb += VLEN_FP32) { HVX_Vector v0 = *(const HVX_Vector *)(raw + (cb + 0) * 4); HVX_Vector v1 = *(const HVX_Vector *)(raw + (cb + 8) * 4); HVX_Vector v2 = *(const HVX_Vector *)(raw + (cb + 16) * 4); @@ -101,15 +101,15 @@ static inline void hvx_ssm_conv_unpack_to_T_4(const float * raw, float * T, uint } // HVX transpose for general d_conv <= 32: channel-major raw VTCM -> tap-major T VTCM -static inline void hvx_ssm_conv_unpack_to_T_gen(const float * raw, float * T, uint32_t d_inner_per_thread, uint32_t d_inner_stride, uint32_t d_conv) { +static inline void hvx_ssm_conv_unpack_to_T_gen(const float * raw, float * T, uint32_t d_inner_per_hvx, uint32_t d_inner_stride, uint32_t d_conv) { uint32_t __attribute__((aligned(VLEN))) mask_buf[VLEN_FP32] = { 0 }; for (uint32_t j = 0; j < d_conv; ++j) { mask_buf[j] = 0xFFFFFFFF; } const HVX_Vector mask = *(const HVX_Vector *) mask_buf; - for (uint32_t cb = 0; cb < d_inner_per_thread; cb += VLEN_FP32) { - const uint32_t cb_n = MIN(VLEN_FP32, d_inner_per_thread - cb); + for (uint32_t cb = 0; cb < d_inner_per_hvx; cb += VLEN_FP32) { + const uint32_t cb_n = MIN(VLEN_FP32, d_inner_per_hvx - cb); HVX_Vector sub[32]; for (uint32_t r = 0; r < cb_n; ++r) { const float * ch_ptr = raw + (cb + r) * d_conv; @@ -127,11 +127,11 @@ static inline void hvx_ssm_conv_unpack_to_T_gen(const float * raw, float * T, ui } } -static inline void hvx_ssm_conv_unpack_to_T(const float * raw, float * T, uint32_t d_inner_per_thread, uint32_t d_inner_stride, uint32_t d_conv) { - if (d_conv == 4 && (d_inner_per_thread % VLEN_FP32 == 0)) { - hvx_ssm_conv_unpack_to_T_4(raw, T, d_inner_per_thread, d_inner_stride); +static inline void hvx_ssm_conv_unpack_to_T(const float * raw, float * T, uint32_t d_inner_per_hvx, uint32_t d_inner_stride, uint32_t d_conv) { + if (d_conv == 4 && (d_inner_per_hvx % VLEN_FP32 == 0)) { + hvx_ssm_conv_unpack_to_T_4(raw, T, d_inner_per_hvx, d_inner_stride); } else { - hvx_ssm_conv_unpack_to_T_gen(raw, T, d_inner_per_thread, d_inner_stride, d_conv); + hvx_ssm_conv_unpack_to_T_gen(raw, T, d_inner_per_hvx, d_inner_stride, d_conv); } } @@ -186,7 +186,7 @@ static inline void hvx_ssm_conv_transpose_block(const float * raw_block, } // Single-token decode worker (n_t == 1) -static void ssm_conv_thread_f32_decode(unsigned int nth, unsigned int ith, void * data) { +static void ssm_conv_thread_f32_decode(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { struct htp_ssm_conv_context * scctx = (struct htp_ssm_conv_context *) data; struct htp_ops_context * octx = scctx->octx; const struct htp_ssm_conv_kernel_params * kparams = scctx->kparams; @@ -195,14 +195,14 @@ static void ssm_conv_thread_f32_decode(unsigned int nth, unsigned int ith, void const struct htp_tensor * restrict src1 = octx->src[1]; const struct htp_tensor * restrict dst = octx->dst; - dma_queue * dma_q = octx->ctx->dma[ith]; + dma_queue * dma_q = th->dma_q; const uint32_t d_conv = kparams->d_conv; const uint32_t d_inner = kparams->d_inner; const uint32_t n_s = kparams->n_s; const bool apply_silu = kparams->silu != 0; - const uint32_t dr = scctx->nrows_per_thread; + const uint32_t dr = scctx->nrows_per_hvx; const uint32_t ir0 = scctx->row_start + dr * ith; const uint32_t ir1 = MIN(ir0 + dr, scctx->row_start + scctx->nrows); @@ -210,19 +210,19 @@ static void ssm_conv_thread_f32_decode(unsigned int nth, unsigned int ith, void return; } - const uint32_t d_inner_per_thread = ir1 - ir0; - const uint32_t d_inner_stride = hex_round_up(d_inner_per_thread, VLEN_FP32); + const uint32_t d_inner_per_hvx = ir1 - ir0; + const uint32_t d_inner_stride = hex_round_up(d_inner_per_hvx, VLEN_FP32); const uint32_t d_inner_tile = scctx->d_inner_tile; const size_t src0_stride_inner_bytes = src0->nb[1]; const size_t src0_stride_seq_bytes = src0->nb[2]; const size_t dst_stride_seq_bytes = dst->nb[2]; - uint8_t * src1_spad_base = octx->src1_spad.data + ith * octx->src1_spad.size_per_thread; - uint8_t * src0_spad_base = octx->src0_spad.data + ith * octx->src0_spad.size_per_thread; - uint8_t * dst_spad_base = octx->dst_spad.data + ith * octx->dst_spad.size_per_thread; + uint8_t * src1_spad_base = octx->src1_spad.data + ith * octx->src1_spad.size_per_hvx; + uint8_t * src0_spad_base = octx->src0_spad.data + ith * octx->src0_spad.size_per_hvx; + uint8_t * dst_spad_base = octx->dst_spad.data + ith * octx->dst_spad.size_per_hvx; - const size_t weight_bytes = (size_t) d_inner_per_thread * d_conv * sizeof(float); + const size_t weight_bytes = (size_t) d_inner_per_hvx * d_conv * sizeof(float); const size_t weight_raw_size = hex_round_up(weight_bytes, 128); float * src1_raw = (float *) src1_spad_base; @@ -236,12 +236,12 @@ static void ssm_conv_thread_f32_decode(unsigned int nth, unsigned int ith, void float * dst_tile[2] = { (float *) dst_spad_base, (float *) (dst_spad_base + dst_tile_bytes) }; - struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + struct htp_thread_trace * tr = th->trace; // raw_dot keeps both operands in the DMA layout, so src1 needs no prep pass - const bool raw_dot = (d_conv == 4) && (d_inner_per_thread % VLEN_FP32 == 0); + const bool raw_dot = (d_conv == 4) && (d_inner_per_hvx % VLEN_FP32 == 0); - const uint32_t n_tiles = fastdiv(d_inner_per_thread + d_inner_tile - 1, &kparams->div_d_inner_tile); + const uint32_t n_tiles = fastdiv(d_inner_per_hvx + d_inner_tile - 1, &kparams->div_d_inner_tile); const uint32_t n_chunks = n_s * n_tiles; const size_t row_bytes = d_conv * sizeof(float); @@ -250,14 +250,14 @@ static void ssm_conv_thread_f32_decode(unsigned int nth, unsigned int ith, void #define SSM_CONV_DECODE_PUSH_FETCH(c) \ do { \ - const uint32_t cur_tile_n = MIN(d_inner_tile, d_inner_per_thread - tile_off_fetch); \ + const uint32_t cur_tile_n = MIN(d_inner_tile, d_inner_per_hvx - tile_off_fetch); \ const dma_addr_t fetch_ddr = src0->data + s_fetch * src0_stride_seq_bytes + \ (ir0 + tile_off_fetch) * src0_stride_inner_bytes; \ dma_queue_push(dma_q, \ dma_make_data((uint8_t *) src0_tile_raw[(c) & 1], fetch_ddr), \ row_bytes, src0_stride_inner_bytes, row_bytes, cur_tile_n); \ tile_off_fetch += d_inner_tile; \ - if (tile_off_fetch >= d_inner_per_thread) { \ + if (tile_off_fetch >= d_inner_per_hvx) { \ tile_off_fetch = 0; \ s_fetch++; \ } \ @@ -277,7 +277,7 @@ static void ssm_conv_thread_f32_decode(unsigned int nth, unsigned int ith, void if (!raw_dot) { // Unpack/transpose src1_raw into src1_T {d_conv, d_inner_stride} htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_W_PREP, (uint16_t) ir0); - hvx_ssm_conv_unpack_to_T(src1_raw, src1_T, d_inner_per_thread, d_inner_stride, d_conv); + hvx_ssm_conv_unpack_to_T(src1_raw, src1_T, d_inner_per_hvx, d_inner_stride, d_conv); htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_W_PREP, (uint16_t) ir0); } @@ -285,7 +285,7 @@ static void ssm_conv_thread_f32_decode(unsigned int nth, unsigned int ith, void uint32_t tile_off = 0; for (uint32_t c = 0; c < n_chunks; ++c) { - const uint32_t tile_n = MIN(d_inner_tile, d_inner_per_thread - tile_off); + const uint32_t tile_n = MIN(d_inner_tile, d_inner_per_hvx - tile_off); if (c >= 2) { dma_queue_pop(dma_q); // writeback of chunk c-2, frees dst_tile[c & 1] @@ -328,7 +328,7 @@ static void ssm_conv_thread_f32_decode(unsigned int nth, unsigned int ith, void } tile_off += d_inner_tile; - if (tile_off >= d_inner_per_thread) { + if (tile_off >= d_inner_per_hvx) { tile_off = 0; i3++; } @@ -347,7 +347,7 @@ static void ssm_conv_thread_f32_decode(unsigned int nth, unsigned int ith, void } // Multi-token prefill worker (n_t > 1) -static void ssm_conv_thread_f32_prefill(unsigned int nth, unsigned int ith, void * data) { +static void ssm_conv_thread_f32_prefill(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { struct htp_ssm_conv_context * scctx = (struct htp_ssm_conv_context *) data; struct htp_ops_context * octx = scctx->octx; const struct htp_ssm_conv_kernel_params * kparams = scctx->kparams; @@ -356,7 +356,7 @@ static void ssm_conv_thread_f32_prefill(unsigned int nth, unsigned int ith, void const struct htp_tensor * restrict src1 = octx->src[1]; const struct htp_tensor * restrict dst = octx->dst; - dma_queue * dma_q = octx->ctx->dma[ith]; + dma_queue * dma_q = th->dma_q; const uint32_t d_conv = kparams->d_conv; const uint32_t d_inner = kparams->d_inner; @@ -365,7 +365,7 @@ static void ssm_conv_thread_f32_prefill(unsigned int nth, unsigned int ith, void const uint32_t ncs = src0->ne[0]; const bool apply_silu = kparams->silu != 0; - const uint32_t dr = scctx->nrows_per_thread; + const uint32_t dr = scctx->nrows_per_hvx; const uint32_t ir0 = scctx->row_start + dr * ith; const uint32_t ir1 = MIN(ir0 + dr, scctx->row_start + scctx->nrows); @@ -373,8 +373,8 @@ static void ssm_conv_thread_f32_prefill(unsigned int nth, unsigned int ith, void return; } - const uint32_t d_inner_per_thread = ir1 - ir0; - const uint32_t d_inner_stride = hex_round_up(d_inner_per_thread, VLEN_FP32); + const uint32_t d_inner_per_hvx = ir1 - ir0; + const uint32_t d_inner_stride = hex_round_up(d_inner_per_hvx, VLEN_FP32); const uint32_t d_inner_tile = scctx->d_inner_tile; const size_t src0_stride_inner_bytes = src0->nb[1]; @@ -382,11 +382,11 @@ static void ssm_conv_thread_f32_prefill(unsigned int nth, unsigned int ith, void const size_t dst_stride_token_bytes = dst->nb[1]; const size_t dst_stride_seq_bytes = dst->nb[2]; - uint8_t * src1_spad_base = octx->src1_spad.data + ith * octx->src1_spad.size_per_thread; - uint8_t * src0_spad_base = octx->src0_spad.data + ith * octx->src0_spad.size_per_thread; - uint8_t * dst_spad_base = octx->dst_spad.data + ith * octx->dst_spad.size_per_thread; + uint8_t * src1_spad_base = octx->src1_spad.data + ith * octx->src1_spad.size_per_hvx; + uint8_t * src0_spad_base = octx->src0_spad.data + ith * octx->src0_spad.size_per_hvx; + uint8_t * dst_spad_base = octx->dst_spad.data + ith * octx->dst_spad.size_per_hvx; - const size_t weight_bytes = (size_t) d_inner_per_thread * d_conv * sizeof(float); + const size_t weight_bytes = (size_t) d_inner_per_hvx * d_conv * sizeof(float); const size_t weight_raw_size = hex_round_up(weight_bytes, 128); float * src1_raw = (float *) src1_spad_base; @@ -403,9 +403,9 @@ static void ssm_conv_thread_f32_prefill(unsigned int nth, unsigned int ith, void float * dst_tile[2] = { (float *) dst_spad_base, (float *) (dst_spad_base + dst_tile_bytes) }; - struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + struct htp_thread_trace * tr = th->trace; - const uint32_t n_tiles = fastdiv(d_inner_per_thread + d_inner_tile - 1, &kparams->div_d_inner_tile); + const uint32_t n_tiles = fastdiv(d_inner_per_hvx + d_inner_tile - 1, &kparams->div_d_inner_tile); const uint32_t n_chunks = n_s * n_tiles; const size_t row_bytes = ncs * sizeof(float); @@ -416,14 +416,14 @@ static void ssm_conv_thread_f32_prefill(unsigned int nth, unsigned int ith, void // Two fetches run ahead, so the queue order is F0 F1 W0 F2 W1 ... and pops follow it. #define SSM_CONV_PUSH_FETCH(c) \ do { \ - const uint32_t cur_tile_n = MIN(d_inner_tile, d_inner_per_thread - tile_off_fetch); \ + const uint32_t cur_tile_n = MIN(d_inner_tile, d_inner_per_hvx - tile_off_fetch); \ const dma_addr_t fetch_ddr = src0->data + s_fetch * src0_stride_seq_bytes + \ (ir0 + tile_off_fetch) * src0_stride_inner_bytes; \ dma_queue_push(dma_q, \ dma_make_data((uint8_t *) src0_tile_raw[(c) & 1], fetch_ddr), \ raw_stride_bytes, src0_stride_inner_bytes, row_bytes, cur_tile_n); \ tile_off_fetch += d_inner_tile; \ - if (tile_off_fetch >= d_inner_per_thread) { \ + if (tile_off_fetch >= d_inner_per_hvx) { \ tile_off_fetch = 0; \ s_fetch++; \ } \ @@ -442,7 +442,7 @@ static void ssm_conv_thread_f32_prefill(unsigned int nth, unsigned int ith, void // Unpack/transpose src1_raw into src1_T {d_conv, d_inner_stride} htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_W_PREP, (uint16_t) ir0); - hvx_ssm_conv_unpack_to_T(src1_raw, src1_T, d_inner_per_thread, d_inner_stride, d_conv); + hvx_ssm_conv_unpack_to_T(src1_raw, src1_T, d_inner_per_hvx, d_inner_stride, d_conv); htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_W_PREP, (uint16_t) ir0); const uint32_t C_TILE = VLEN_FP32; @@ -454,7 +454,7 @@ static void ssm_conv_thread_f32_prefill(unsigned int nth, unsigned int ith, void uint32_t tile_off = 0; for (uint32_t c = 0; c < n_chunks; ++c) { - const uint32_t tile_n = MIN(d_inner_tile, d_inner_per_thread - tile_off); + const uint32_t tile_n = MIN(d_inner_tile, d_inner_per_hvx - tile_off); const float * restrict raw = src0_tile_raw[c & 1]; float * restrict out = dst_tile[c & 1]; @@ -533,7 +533,7 @@ static void ssm_conv_thread_f32_prefill(unsigned int nth, unsigned int ith, void } tile_off += d_inner_tile; - if (tile_off >= d_inner_per_thread) { + if (tile_off >= d_inner_per_hvx) { tile_off = 0; i3++; } @@ -558,7 +558,7 @@ int op_ssm_conv_f32(struct htp_ops_context * octx) { const struct htp_ssm_conv_kernel_params * kparams = (const struct htp_ssm_conv_kernel_params *) octx->kernel_params; - if (!htp_ops_context_set_n_threads(octx, kparams->n_threads)) { + if (!htp_ops_context_set_n_hvx(octx, kparams->n_hvx)) { return HTP_STATUS_INVAL_PARAMS; } @@ -586,18 +586,18 @@ int op_ssm_conv_f32(struct htp_ops_context * octx) { return HTP_STATUS_VTCM_TOO_SMALL; } - const uint32_t n_threads = octx->n_threads; - const uint32_t raw_rpt = fastdiv(nrows + n_threads - 1, &octx->n_threads_div); - const uint32_t d_inner_per_thread = hex_round_up(raw_rpt, VLEN_FP32); + const uint32_t n_hvx = octx->n_hvx; + const uint32_t raw_rpt = fastdiv(nrows + n_hvx - 1, &octx->n_hvx_div); + const uint32_t d_inner_per_hvx = hex_round_up(raw_rpt, VLEN_FP32); struct htp_ssm_conv_vtcm_layout L; - htp_ssm_conv_vtcm_layout_build(&L, d_inner_per_thread, kparams->d_conv, kparams->n_t, kparams->d_inner_tile, n_threads); + htp_ssm_conv_vtcm_layout_build(&L, d_inner_per_hvx, kparams->d_conv, kparams->n_t, kparams->d_inner_tile, n_hvx); uint8_t * const base = (uint8_t *) octx->ctx->vtcm_base; - octx->src0_spad.size_per_thread = L.src0_bytes_per_thread; - octx->src1_spad.size_per_thread = L.src1_bytes_per_thread; - octx->dst_spad.size_per_thread = L.dst_bytes_per_thread; + octx->src0_spad.size_per_hvx = L.src0_bytes_per_hvx; + octx->src1_spad.size_per_hvx = L.src1_bytes_per_hvx; + octx->dst_spad.size_per_hvx = L.dst_bytes_per_hvx; octx->src0_spad.size = L.src0_bytes; octx->src1_spad.size = L.src1_bytes; @@ -613,7 +613,7 @@ int op_ssm_conv_f32(struct htp_ops_context * octx) { struct htp_ssm_conv_context scctx = { .octx = octx, .kparams = kparams, - .nrows_per_thread = d_inner_per_thread, + .nrows_per_hvx = d_inner_per_hvx, .d_inner_tile = kparams->d_inner_tile, .row_start = row_start, .nrows = nrows, @@ -626,9 +626,9 @@ int op_ssm_conv_f32(struct htp_ops_context * octx) { kparams->n_t == 1 ? "decode" : "prefill"); if (kparams->n_t == 1) { - work_queue_run(octx->ctx->work_queue, ssm_conv_thread_f32_decode, &scctx, n_threads); + work_queue_run(octx->ctx->hvx_queue, ssm_conv_thread_f32_decode, &scctx, n_hvx); } else { - work_queue_run(octx->ctx->work_queue, ssm_conv_thread_f32_prefill, &scctx, n_threads); + work_queue_run(octx->ctx->hvx_queue, ssm_conv_thread_f32_prefill, &scctx, n_hvx); } return HTP_STATUS_OK; diff --git a/ggml/src/ggml-hexagon/htp/ssm-conv.h b/ggml/src/ggml-hexagon/htp/ssm-conv.h index cd4180857294..7c3f3041ffdf 100644 --- a/ggml/src/ggml-hexagon/htp/ssm-conv.h +++ b/ggml/src/ggml-hexagon/htp/ssm-conv.h @@ -9,7 +9,7 @@ #include "htp-vtcm.h" struct htp_ssm_conv_kernel_params { - uint32_t n_threads; + uint32_t n_hvx; uint32_t d_conv; uint32_t d_inner; uint32_t n_t; @@ -27,9 +27,9 @@ struct htp_ssm_conv_vtcm_layout { size_t off_src0; size_t off_src1; size_t off_dst; - size_t src0_bytes_per_thread; - size_t src1_bytes_per_thread; - size_t dst_bytes_per_thread; + size_t src0_bytes_per_hvx; + size_t src1_bytes_per_hvx; + size_t dst_bytes_per_hvx; size_t src0_bytes; size_t src1_bytes; size_t dst_bytes; @@ -37,34 +37,34 @@ struct htp_ssm_conv_vtcm_layout { static inline void htp_ssm_conv_vtcm_layout_build( struct htp_ssm_conv_vtcm_layout * L, - uint32_t d_inner_per_thread, + uint32_t d_inner_per_hvx, uint32_t d_conv, uint32_t n_t, uint32_t d_inner_tile, - uint32_t n_threads + uint32_t n_hvx ) { - const size_t src1_raw_bytes = hex_round_up((size_t) d_inner_per_thread * d_conv * sizeof(float), 128) + 128; - const size_t src1_T_bytes = hex_round_up((size_t) d_conv * d_inner_per_thread * sizeof(float), 128); + const size_t src1_raw_bytes = hex_round_up((size_t) d_inner_per_hvx * d_conv * sizeof(float), 128) + 128; + const size_t src1_T_bytes = hex_round_up((size_t) d_conv * d_inner_per_hvx * sizeof(float), 128); - L->src1_bytes_per_thread = src1_raw_bytes + src1_T_bytes; + L->src1_bytes_per_hvx = src1_raw_bytes + src1_T_bytes; if (n_t == 1) { const size_t src0_tile_raw_bytes = hex_round_up((size_t) d_inner_tile * d_conv * sizeof(float), 128); const size_t src0_T_bytes = hex_round_up((size_t) d_conv * d_inner_tile * sizeof(float), 128); - L->src0_bytes_per_thread = 2 * src0_tile_raw_bytes + src0_T_bytes; - L->dst_bytes_per_thread = 2 * hex_round_up((size_t) d_inner_tile * sizeof(float), 128); + L->src0_bytes_per_hvx = 2 * src0_tile_raw_bytes + src0_T_bytes; + L->dst_bytes_per_hvx = 2 * hex_round_up((size_t) d_inner_tile * sizeof(float), 128); } else { const uint32_t ncs = d_conv - 1 + n_t; const size_t raw_stride_bytes = ncs * sizeof(float) + sizeof(float); const size_t src0_block_T = hex_round_up((size_t) ncs * 32 * sizeof(float), 128); const size_t src0_tile_raw = hex_round_up((size_t) d_inner_tile * raw_stride_bytes, 128); - L->src0_bytes_per_thread = 2 * src0_tile_raw + src0_block_T; - L->dst_bytes_per_thread = 2 * hex_round_up((size_t) d_inner_tile * n_t * sizeof(float), 128); + L->src0_bytes_per_hvx = 2 * src0_tile_raw + src0_block_T; + L->dst_bytes_per_hvx = 2 * hex_round_up((size_t) d_inner_tile * n_t * sizeof(float), 128); } - L->src0_bytes = L->src0_bytes_per_thread * n_threads; - L->src1_bytes = L->src1_bytes_per_thread * n_threads; - L->dst_bytes = L->dst_bytes_per_thread * n_threads; + L->src0_bytes = L->src0_bytes_per_hvx * n_hvx; + L->src1_bytes = L->src1_bytes_per_hvx * n_hvx; + L->dst_bytes = L->dst_bytes_per_hvx * n_hvx; size_t off = 0; VTCM_LAYOUT_ALLOC(off, off_src0, L->src0_bytes); diff --git a/ggml/src/ggml-hexagon/htp/sum-rows-ops.c b/ggml/src/ggml-hexagon/htp/sum-rows-ops.c index 3d931ed19305..b3f8a7090902 100644 --- a/ggml/src/ggml-hexagon/htp/sum-rows-ops.c +++ b/ggml/src/ggml-hexagon/htp/sum-rows-ops.c @@ -14,7 +14,7 @@ #define GGML_COMMON_DECL_C #include "ggml-common.h" #include "hex-common.h" -#include "hex-profile.h" +#include "htp-trace.h" #include "htp-ctx.h" #include "htp-ops.h" #include "htp-tensor.h" @@ -51,18 +51,18 @@ struct sum_rows_context { uint32_t ne00; size_t src_stride; size_t dst_stride; - uint32_t rows_per_thread; + uint32_t rows_per_hvx; uint32_t total_rows; }; -static void sum_rows_thread_f32(unsigned int nth, unsigned int ith, void *data) { +static void sum_rows_thread_f32(const htp_thread_t * th, unsigned int nth, unsigned int ith, void *data) { const struct sum_rows_context * smctx = (const struct sum_rows_context *) data; - const uint32_t rows_per_thread = smctx->rows_per_thread; + const uint32_t rows_per_hvx = smctx->rows_per_hvx; const uint32_t total_rows = smctx->total_rows; - const uint32_t start_row = rows_per_thread * ith; - const uint32_t end_row = MIN(start_row + rows_per_thread, total_rows); + const uint32_t start_row = rows_per_hvx * ith; + const uint32_t end_row = MIN(start_row + rows_per_hvx, total_rows); if (start_row >= end_row) { return; @@ -78,7 +78,7 @@ static void sum_rows_thread_f32(unsigned int nth, unsigned int ith, void *data) // Calculate actual number of rows for this thread const uint32_t n_rows = end_row - start_row; - struct htp_thread_trace * tr = &smctx->octx->ctx->trace[ith]; + struct htp_thread_trace * tr = th->trace; htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) start_row); for (uint32_t ir = 0; ir < n_rows; ir++) { @@ -109,7 +109,7 @@ int op_sum_rows(struct htp_ops_context * octx) { const struct htp_sum_rows_kernel_params * kparams = (const struct htp_sum_rows_kernel_params *) octx->kernel_params; - if (!htp_ops_context_set_n_threads(octx, kparams->n_threads)) { + if (!htp_ops_context_set_n_hvx(octx, kparams->n_hvx)) { return HTP_STATUS_INVAL_PARAMS; } @@ -131,8 +131,8 @@ int op_sum_rows(struct htp_ops_context * octx) { return HTP_STATUS_OK; } - const uint32_t n_threads = octx->n_threads; - const uint32_t rows_per_thread = fastdiv(nrows + n_threads - 1, &octx->n_threads_div); + const uint32_t n_hvx = octx->n_hvx; + const uint32_t rows_per_hvx = fastdiv(nrows + n_hvx - 1, &octx->n_hvx_div); struct sum_rows_context smctx = { .octx = octx, @@ -141,11 +141,11 @@ int op_sum_rows(struct htp_ops_context * octx) { .ne00 = ne00, .src_stride = nb01, .dst_stride = nb1, - .rows_per_thread = rows_per_thread, + .rows_per_hvx = rows_per_hvx, .total_rows = nrows, }; - work_queue_run(octx->ctx->work_queue, sum_rows_thread_f32, &smctx, n_threads); + work_queue_run(octx->ctx->hvx_queue, sum_rows_thread_f32, &smctx, n_hvx); return HTP_STATUS_OK; } @@ -153,15 +153,15 @@ int op_sum_rows(struct htp_ops_context * octx) { struct sum_context { struct htp_ops_context * octx; const float * src_data; - float partial_sums[HTP_MAX_NTHREADS]; + float partial_sums[HTP_MAX_NHVX]; uint32_t total_elems; - uint32_t elems_per_thread; + uint32_t elems_per_hvx; }; -static void sum_thread_f32(unsigned int nth, unsigned int ith, void * data) { +static void sum_thread_f32(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { struct sum_context * sctx = (struct sum_context *) data; - const uint32_t start = sctx->elems_per_thread * ith; - const uint32_t end = MIN(start + sctx->elems_per_thread, sctx->total_elems); + const uint32_t start = sctx->elems_per_hvx * ith; + const uint32_t end = MIN(start + sctx->elems_per_hvx, sctx->total_elems); if (start >= end) { sctx->partial_sums[ith] = 0.0f; @@ -171,7 +171,7 @@ static void sum_thread_f32(unsigned int nth, unsigned int ith, void * data) { const uint32_t n = end - start; const float * src = sctx->src_data + start; - struct htp_thread_trace * tr = &sctx->octx->ctx->trace[ith]; + struct htp_thread_trace * tr = th->trace; htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) start); hex_l2fetch_block((const void *) src, n * sizeof(float)); @@ -196,7 +196,7 @@ int op_sum(struct htp_ops_context * octx) { const struct htp_sum_rows_kernel_params * kparams = (const struct htp_sum_rows_kernel_params *) octx->kernel_params; - if (!htp_ops_context_set_n_threads(octx, kparams->n_threads)) { + if (!htp_ops_context_set_n_hvx(octx, kparams->n_hvx)) { return HTP_STATUS_INVAL_PARAMS; } @@ -210,20 +210,20 @@ int op_sum(struct htp_ops_context * octx) { return HTP_STATUS_OK; } - const uint32_t n_threads = octx->n_threads; - const uint32_t elems_per_thread = kparams->elems_per_thread; + const uint32_t n_hvx = octx->n_hvx; + const uint32_t elems_per_hvx = kparams->elems_per_hvx; struct sum_context sctx = { .octx = octx, .src_data = (const float *) src0->data, .total_elems = total_elems, - .elems_per_thread = elems_per_thread, + .elems_per_hvx = elems_per_hvx, }; - work_queue_run(octx->ctx->work_queue, sum_thread_f32, &sctx, n_threads); + work_queue_run(octx->ctx->hvx_queue, sum_thread_f32, &sctx, n_hvx); float sum = 0.0f; - for (uint32_t i = 0; i < n_threads; i++) { + for (uint32_t i = 0; i < n_hvx; i++) { sum += sctx.partial_sums[i]; } ((float *) dst->data)[0] = sum; @@ -250,17 +250,17 @@ struct argmax_context { uint32_t dst_stride; uint32_t row_start; uint32_t nrows; - uint32_t rows_per_thread; - uint32_t elems_per_thread; + uint32_t rows_per_hvx; + uint32_t elems_per_hvx; - float partial_max[HTP_MAX_NTHREADS]; - int32_t partial_idx[HTP_MAX_NTHREADS]; + float partial_max[HTP_MAX_NHVX]; + int32_t partial_idx[HTP_MAX_NHVX]; }; -static void argmax_thread_single_row(unsigned int nth, unsigned int ith, void * data) { +static void argmax_thread_single_row(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { struct argmax_context * actx = (struct argmax_context *) data; - const uint32_t start = actx->elems_per_thread * ith; - const uint32_t end = MIN(start + actx->elems_per_thread, actx->ne00); + const uint32_t start = actx->elems_per_hvx * ith; + const uint32_t end = MIN(start + actx->elems_per_hvx, actx->ne00); if (start >= end) { actx->partial_max[ith] = -INFINITY; @@ -271,7 +271,7 @@ static void argmax_thread_single_row(unsigned int nth, unsigned int ith, void * const uint32_t n = end - start; const float * src = actx->src_data + start; - struct htp_thread_trace * tr = &actx->octx->ctx->trace[ith]; + struct htp_thread_trace * tr = th->trace; htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) start); hex_l2fetch_block((const void *) src, n * sizeof(float)); @@ -286,16 +286,16 @@ static void argmax_thread_single_row(unsigned int nth, unsigned int ith, void * htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) start); } -static void argmax_thread_multi_row(unsigned int nth, unsigned int ith, void * data) { +static void argmax_thread_multi_row(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { struct argmax_context * actx = (struct argmax_context *) data; - const uint32_t r0 = actx->row_start + actx->rows_per_thread * ith; - const uint32_t r1 = MIN(r0 + actx->rows_per_thread, actx->row_start + actx->nrows); + const uint32_t r0 = actx->row_start + actx->rows_per_hvx * ith; + const uint32_t r1 = MIN(r0 + actx->rows_per_hvx, actx->row_start + actx->nrows); if (r0 >= r1) { return; } - struct htp_thread_trace * tr = &actx->octx->ctx->trace[ith]; + struct htp_thread_trace * tr = th->trace; htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) r0); for (uint32_t r = r0; r < r1; r++) { @@ -329,7 +329,7 @@ int op_argmax(struct htp_ops_context * octx) { const struct htp_sum_rows_kernel_params * kparams = (const struct htp_sum_rows_kernel_params *) octx->kernel_params; - if (!htp_ops_context_set_n_threads(octx, kparams->n_threads)) { + if (!htp_ops_context_set_n_hvx(octx, kparams->n_hvx)) { return HTP_STATUS_INVAL_PARAMS; } @@ -350,8 +350,8 @@ int op_argmax(struct htp_ops_context * octx) { return HTP_STATUS_OK; } - const uint32_t n_threads = octx->n_threads; - const uint32_t elems_per_thread = kparams->elems_per_thread; + const uint32_t n_hvx = octx->n_hvx; + const uint32_t elems_per_hvx = kparams->elems_per_hvx; struct argmax_context actx = { .octx = octx, @@ -362,15 +362,15 @@ int op_argmax(struct htp_ops_context * octx) { .dst_stride = dst->nb[0] > 0 ? (uint32_t) dst->nb[0] : (uint32_t) sizeof(int32_t), .row_start = 0, .nrows = 1, - .rows_per_thread = 1, - .elems_per_thread = elems_per_thread, + .rows_per_hvx = 1, + .elems_per_hvx = elems_per_hvx, }; - work_queue_run(octx->ctx->work_queue, argmax_thread_single_row, &actx, n_threads); + work_queue_run(octx->ctx->hvx_queue, argmax_thread_single_row, &actx, n_hvx); float best_val = actx.partial_max[0]; int32_t best_idx = actx.partial_idx[0]; - for (uint32_t i = 1; i < n_threads; i++) { + for (uint32_t i = 1; i < n_hvx; i++) { if (actx.partial_max[i] > best_val) { best_val = actx.partial_max[i]; best_idx = actx.partial_idx[i]; @@ -397,8 +397,8 @@ int op_argmax(struct htp_ops_context * octx) { return HTP_STATUS_OK; } - const uint32_t n_threads = octx->n_threads; - const uint32_t rows_per_thread = fastdiv(nrows + n_threads - 1, &octx->n_threads_div); + const uint32_t n_hvx = octx->n_hvx; + const uint32_t rows_per_hvx = fastdiv(nrows + n_hvx - 1, &octx->n_hvx_div); struct argmax_context actx = { .octx = octx, @@ -409,10 +409,10 @@ int op_argmax(struct htp_ops_context * octx) { .dst_stride = dst->nb[0] > 0 ? (uint32_t) dst->nb[0] : (uint32_t) sizeof(int32_t), .row_start = row_start, .nrows = nrows, - .rows_per_thread = rows_per_thread, - .elems_per_thread = 0, + .rows_per_hvx = rows_per_hvx, + .elems_per_hvx = 0, }; - work_queue_run(octx->ctx->work_queue, argmax_thread_multi_row, &actx, n_threads); + work_queue_run(octx->ctx->hvx_queue, argmax_thread_multi_row, &actx, n_hvx); return HTP_STATUS_OK; } diff --git a/ggml/src/ggml-hexagon/htp/sum-rows-ops.h b/ggml/src/ggml-hexagon/htp/sum-rows-ops.h index cc9ddd483913..0b8acf030b0a 100644 --- a/ggml/src/ggml-hexagon/htp/sum-rows-ops.h +++ b/ggml/src/ggml-hexagon/htp/sum-rows-ops.h @@ -4,8 +4,8 @@ #include struct htp_sum_rows_kernel_params { - uint32_t n_threads; - uint32_t elems_per_thread; + uint32_t n_hvx; + uint32_t elems_per_hvx; }; #if defined(__cplusplus) diff --git a/ggml/src/ggml-hexagon/htp/top-k-mask-ops.c b/ggml/src/ggml-hexagon/htp/top-k-mask-ops.c new file mode 100644 index 000000000000..9c4dc583c495 --- /dev/null +++ b/ggml/src/ggml-hexagon/htp/top-k-mask-ops.c @@ -0,0 +1,665 @@ +#pragma clang diagnostic ignored "-Wunused-variable" +#pragma clang diagnostic ignored "-Wunused-function" +#pragma clang diagnostic ignored "-Wunused-but-set-variable" + +#include +#include +#include + +#define GGML_COMMON_DECL_C +#include "ggml-common.h" +#include "hex-common.h" +#include "htp-trace.h" +#include "htp-ctx.h" +#include "htp-ops.h" +#include "htp-tensor.h" +#include "htp-vtcm.h" +#include "hvx-types.h" +#include "hvx-utils.h" +#include "dma-queue.h" +#include "top-k-mask-ops.h" + +#define TOP_K_MASK_NINF_F32 0xff800000u // f32 score compare + +struct htp_top_k_mask_context { + struct htp_ops_context * octx; + const struct htp_top_k_mask_kernel_params * kparams; + + uint32_t rows_per_hvx; + uint32_t row0, row1; // rows this device handles + uint32_t off_extra, off_score; // parts inside one row input slot + uint32_t col0, ncols_out; // PICKS: columns this device writes back + atomic_uint barrier; // PICKS: arrivals, barrier k waits for k * n_hvx + + uint8_t * blocks; + uint8_t * mask; + uint8_t * rowin; + uint8_t * tiles; + uint8_t * mtiles; + uint8_t * scratch; +}; + +// DMA queue pop with a trace event around the wait +static inline void top_k_mask_pop(dma_queue * dma_q, struct htp_thread_trace * tr, uint16_t info) { + htp_trace_event_start(tr, HTP_TRACE_EVT_DMA, info); + dma_queue_pop(dma_q); + htp_trace_event_stop(tr, HTP_TRACE_EVT_DMA, info); +} + +// Lanes 0..31 as words, in registers: inclusive scan of ones, minus one +static inline HVX_Vector top_k_mask_lane_iota(void) { + const HVX_Vector zero = Q6_V_vzero(); + const HVX_Vector one = Q6_V_vsplat_R(1); + HVX_Vector v = one; + v = Q6_Vw_vadd_VwVw(v, Q6_V_vlalign_VVR(v, zero, 4)); + v = Q6_Vw_vadd_VwVw(v, Q6_V_vlalign_VVR(v, zero, 8)); + v = Q6_Vw_vadd_VwVw(v, Q6_V_vlalign_VVR(v, zero, 16)); + v = Q6_Vw_vadd_VwVw(v, Q6_V_vlalign_VVR(v, zero, 32)); + v = Q6_Vw_vadd_VwVw(v, Q6_V_vlalign_VVR(v, zero, 64)); + return Q6_Vw_vsub_VwVw(v, one); +} + +static __attribute__((noinline)) void top_k_mask_fill(uint8_t * restrict dst, uint32_t nbytes, uint32_t value) { + const HVX_Vector v = Q6_V_vsplat_R(value); + HVX_Vector * vdst = (HVX_Vector *) dst; + const uint32_t nvec = nbytes / VLEN; + + #pragma unroll(4) + for (uint32_t i = 0; i < nvec; i++) { + vdst[i] = v; + } +} + +#define TOP_K_MASK_DEAD 0x7fffffffu // cell value of a dead lane: c - c0 >=u ncols for every tile, signed max + +// Byte offsets of the cells in [c0, c0 + ncols) relative to c0, 0 for the others +static inline HVX_Vector top_k_mask_keep_off(HVX_Vector c, HVX_Vector v_c0, HVX_Vector v_ncols, const bool is_f16) { + const HVX_Vector cr = Q6_Vw_vsub_VwVw(c, v_c0); + return Q6_Vw_vasl_VwR(Q6_V_vmux_QVV(Q6_Q_vcmp_gt_VuwVuw(v_ncols, cr), cr, Q6_V_vzero()), is_f16 ? 1 : 2); +} + +// Mask value of each cell lane in [c0, c0 + ncols) into tmp (predicated gather), returns the scatter offsets +static inline HVX_Vector top_k_mask_keep_gather(HVX_Vector c, const uint8_t * restrict mrow, HVX_Vector v_c0, HVX_Vector v_ncols, + uint32_t mu, HVX_Vector * restrict tmp, const bool is_f16) { + const HVX_VectorPred q = Q6_Q_vcmp_gt_VuwVuw(v_ncols, Q6_Vw_vsub_VwVw(c, v_c0)); + const HVX_Vector off = top_k_mask_keep_off(c, v_c0, v_ncols, is_f16); + + // Halfword data with word offsets: both halves of the pair hold the same offsets, the word predicate covers both halfwords of a lane + if (is_f16) { + Q6_vgather_AQRMWw(tmp, q, (size_t) mrow, mu, Q6_W_vcombine_VV(off, off)); + } else { + Q6_vgather_AQRMVw(tmp, q, (size_t) mrow, mu, off); + } + return off; +} + +static inline void top_k_mask_keep_scatter(HVX_Vector c, HVX_Vector off, uint8_t * restrict tile, HVX_Vector v_c0, HVX_Vector v_ncols, + uint32_t mu, HVX_Vector val, const bool is_f16) { + const HVX_VectorPred q = Q6_Q_vcmp_gt_VuwVuw(v_ncols, Q6_Vw_vsub_VwVw(c, v_c0)); + if (is_f16) { + Q6_vscatter_QRMWwV(q, (size_t) tile, mu, Q6_W_vcombine_VV(off, off), val); + } else { + Q6_vscatter_QRMVwV(q, (size_t) tile, mu, off, val); + } +} + +// Keep 4 cell vectors (dead lanes = TOP_K_MASK_DEAD): tile[c - c0] = mrow[c - c0] for c in [c0, c0 + ncols). One gather sync. +// flag: no mask, tile[c - c0] = all ones (keep flag) +static inline void top_k_mask_keep4(HVX_Vector c0v, HVX_Vector c1v, HVX_Vector c2v, HVX_Vector c3v, uint8_t * restrict tile, + const uint8_t * restrict mrow, HVX_Vector v_c0, HVX_Vector v_ncols, uint32_t mu, + HVX_Vector * restrict tmp, const bool is_f16, const bool flag) { + if (flag) { + const HVX_Vector ones = Q6_V_vsplat_R(0xffffffffu); + top_k_mask_keep_scatter(c0v, top_k_mask_keep_off(c0v, v_c0, v_ncols, is_f16), tile, v_c0, v_ncols, mu, ones, is_f16); + top_k_mask_keep_scatter(c1v, top_k_mask_keep_off(c1v, v_c0, v_ncols, is_f16), tile, v_c0, v_ncols, mu, ones, is_f16); + top_k_mask_keep_scatter(c2v, top_k_mask_keep_off(c2v, v_c0, v_ncols, is_f16), tile, v_c0, v_ncols, mu, ones, is_f16); + top_k_mask_keep_scatter(c3v, top_k_mask_keep_off(c3v, v_c0, v_ncols, is_f16), tile, v_c0, v_ncols, mu, ones, is_f16); + return; + } + const HVX_Vector o0 = top_k_mask_keep_gather(c0v, mrow, v_c0, v_ncols, mu, tmp + 0, is_f16); + const HVX_Vector o1 = top_k_mask_keep_gather(c1v, mrow, v_c0, v_ncols, mu, tmp + 1, is_f16); + const HVX_Vector o2 = top_k_mask_keep_gather(c2v, mrow, v_c0, v_ncols, mu, tmp + 2, is_f16); + const HVX_Vector o3 = top_k_mask_keep_gather(c3v, mrow, v_c0, v_ncols, mu, tmp + 3, is_f16); + hvx_gather_sync(tmp + 3); + top_k_mask_keep_scatter(c0v, o0, tile, v_c0, v_ncols, mu, tmp[0], is_f16); + top_k_mask_keep_scatter(c1v, o1, tile, v_c0, v_ncols, mu, tmp[1], is_f16); + top_k_mask_keep_scatter(c2v, o2, tile, v_c0, v_ncols, mu, tmp[2], is_f16); + top_k_mask_keep_scatter(c3v, o3, tile, v_c0, v_ncols, mu, tmp[3], is_f16); +} + +// Pick vector j: picks below j1 and n_pick are valid, returns them with dead lanes marked. v_rem = j1 - lane index. +static inline HVX_Vector top_k_mask_load_picks(const int32_t * restrict tk, uint32_t j, HVX_Vector v_rem, HVX_Vector v_npick) { + const HVX_Vector p_raw = hvx_vmem(tk + j); + // lane + j = n_pick + const HVX_VectorPred valid = Q6_Q_vcmp_gtand_QVuwVuw(Q6_Q_vcmp_gt_VwVw(v_rem, Q6_V_vsplat_R(j)), v_npick, p_raw); + return Q6_V_vmux_QVV(valid, p_raw, Q6_V_vsplat_R(TOP_K_MASK_DEAD)); +} + +// Scatter the kept cells of picks [j0, j1) (and the extras) into the tile of columns [c0, c0 + ncols). slot holds the row inputs, mrow the mask columns. +// Gathers are issued in batches with one sync per batch: tmp[0..3] scores/blocks, tmp[4] score with blocks, tmp[5..8] mask values. +static inline __attribute__((always_inline)) void top_k_mask_scatter_tile( + const struct htp_top_k_mask_context * tctx, const uint8_t * restrict slot, const uint8_t * restrict mrow, uint8_t * restrict tile, + HVX_Vector * restrict tmp, uint32_t c0, uint32_t ncols, uint32_t j0, uint32_t j1, bool do_extra, + const bool is_f16, const bool flag) { + const struct htp_top_k_mask_kernel_params * kp = tctx->kparams; + + const int32_t * restrict tk = (const int32_t *) slot; + const int32_t * restrict ex = (const int32_t *) (slot + tctx->off_extra); + const uint8_t * restrict sc = slot + tctx->off_score; + const uint8_t * restrict blk = tctx->blocks; + + const bool has_score = kp->flags & HTP_TOP_K_MASK_HAS_SCORE; + const bool has_blocks = kp->flags & HTP_TOP_K_MASK_HAS_BLOCKS; + const uint32_t n_blk = kp->n_blk; + const uint32_t n_extra = do_extra ? kp->n_extra : 0; + const uint32_t mu = ncols * kp->elem_size - 1; + const uint32_t sc_mu = kp->score_bytes - 1; + const uint32_t blk_mu = kp->blocks_bytes - 1; + const uint32_t nb1_h = ((uint32_t) kp->blocks_nb1 << 16) | kp->blocks_nb1; + + const HVX_Vector v_lane = top_k_mask_lane_iota(); // in the HVX_COMP scope of the caller + const HVX_Vector v_c0 = Q6_V_vsplat_R(c0); + const HVX_Vector v_ncols = Q6_V_vsplat_R(ncols); + const HVX_Vector v_npick = Q6_V_vsplat_R(kp->n_pick); + const HVX_Vector v_pmax = Q6_V_vsplat_R(kp->n_pick - 1); // dead picks gather the last entry, every gather stays in its region + const HVX_Vector v_rem = Q6_Vw_vsub_VwVw(Q6_V_vsplat_R(j1), v_lane); + const HVX_Vector v_ninf = Q6_V_vsplat_R(TOP_K_MASK_NINF_F32); + const HVX_Vector v_dead = Q6_V_vsplat_R(TOP_K_MASK_DEAD); + + HVX_Vector * restrict mtmp = tmp + 5; + + if (has_blocks) { + for (uint32_t j = j0; j < j1; j += VLEN_FP32) { + HVX_Vector cell = top_k_mask_load_picks(tk, j, v_rem, v_npick); + const HVX_Vector p = Q6_Vw_vmin_VwVw(cell, v_pmax); + if (has_score) { + Q6_vgather_ARMVw(tmp + 4, (size_t) sc, sc_mu, Q6_Vw_vasl_VwR(p, 2)); + } + const HVX_Vector boff = Q6_Vw_vmpyi_VwRh(p, nb1_h); + for (uint32_t k = 0; k < n_blk; k += 4) { + // blocks k..k+3 of every pick; the missing ones (k + i >= n_blk) are dead + const uint32_t nk = MIN(n_blk - k, 4); + const HVX_Vector bo = Q6_Vw_vadd_VwVw(boff, Q6_V_vsplat_R(k * 4)); + for (uint32_t i = 0; i < nk; i++) { + Q6_vgather_ARMVw(tmp + i, (size_t) blk, blk_mu, Q6_Vw_vadd_VwVw(bo, Q6_V_vsplat_R(i * 4))); + } + hvx_gather_sync(tmp + nk - 1); + if (has_score && k == 0) { + // IEEE compare: NaN and -inf scores are dead + cell = Q6_V_vmux_QVV(Q6_Q_vcmp_gt_VsfVsf(tmp[4], v_ninf), cell, v_dead); + } + const HVX_VectorPred dead = Q6_Q_vcmp_eq_VwVw(cell, v_dead); + top_k_mask_keep4(Q6_V_vmux_QVV(dead, v_dead, tmp[0]), + nk > 1 ? Q6_V_vmux_QVV(dead, v_dead, tmp[1]) : v_dead, + nk > 2 ? Q6_V_vmux_QVV(dead, v_dead, tmp[2]) : v_dead, + nk > 3 ? Q6_V_vmux_QVV(dead, v_dead, tmp[3]) : v_dead, + tile, mrow, v_c0, v_ncols, mu, mtmp, is_f16, flag); + } + } + } else { + // 4 pick vectors per batch, the vectors at or past j1 are dead and not loaded + for (uint32_t j = j0; j < j1; j += 4 * VLEN_FP32) { + HVX_Vector c0v = top_k_mask_load_picks(tk, j, v_rem, v_npick); + HVX_Vector c1v = j + 1 * VLEN_FP32 < j1 ? top_k_mask_load_picks(tk, j + 1 * VLEN_FP32, v_rem, v_npick) : v_dead; + HVX_Vector c2v = j + 2 * VLEN_FP32 < j1 ? top_k_mask_load_picks(tk, j + 2 * VLEN_FP32, v_rem, v_npick) : v_dead; + HVX_Vector c3v = j + 3 * VLEN_FP32 < j1 ? top_k_mask_load_picks(tk, j + 3 * VLEN_FP32, v_rem, v_npick) : v_dead; + if (has_score) { + Q6_vgather_ARMVw(tmp + 0, (size_t) sc, sc_mu, Q6_Vw_vasl_VwR(Q6_Vw_vmin_VwVw(c0v, v_pmax), 2)); + Q6_vgather_ARMVw(tmp + 1, (size_t) sc, sc_mu, Q6_Vw_vasl_VwR(Q6_Vw_vmin_VwVw(c1v, v_pmax), 2)); + Q6_vgather_ARMVw(tmp + 2, (size_t) sc, sc_mu, Q6_Vw_vasl_VwR(Q6_Vw_vmin_VwVw(c2v, v_pmax), 2)); + Q6_vgather_ARMVw(tmp + 3, (size_t) sc, sc_mu, Q6_Vw_vasl_VwR(Q6_Vw_vmin_VwVw(c3v, v_pmax), 2)); + hvx_gather_sync(tmp + 3); + c0v = Q6_V_vmux_QVV(Q6_Q_vcmp_gt_VsfVsf(tmp[0], v_ninf), c0v, v_dead); + c1v = Q6_V_vmux_QVV(Q6_Q_vcmp_gt_VsfVsf(tmp[1], v_ninf), c1v, v_dead); + c2v = Q6_V_vmux_QVV(Q6_Q_vcmp_gt_VsfVsf(tmp[2], v_ninf), c2v, v_dead); + c3v = Q6_V_vmux_QVV(Q6_Q_vcmp_gt_VsfVsf(tmp[3], v_ninf), c3v, v_dead); + } + top_k_mask_keep4(c0v, c1v, c2v, c3v, tile, mrow, v_c0, v_ncols, mu, mtmp, is_f16, flag); + } + } + + const HVX_Vector v_rem_ex = Q6_Vw_vsub_VwVw(Q6_V_vsplat_R(n_extra), v_lane); + for (uint32_t j = 0; j < n_extra; j += VLEN_FP32) { + const HVX_VectorPred tail = Q6_Q_vcmp_gt_VwVw(v_rem_ex, Q6_V_vsplat_R(j)); + top_k_mask_keep4(Q6_V_vmux_QVV(tail, hvx_vmem(ex + j), v_dead), v_dead, v_dead, v_dead, tile, mrow, v_c0, v_ncols, mu, mtmp, is_f16, flag); + } + + // release the scatters before DMA reads the tile + hvx_gather_sync(tile); +} + +static __attribute__((noinline)) void top_k_mask_scatter_tile_f32(const struct htp_top_k_mask_context * tctx, const uint8_t * restrict slot, + const uint8_t * restrict mrow, uint8_t * restrict tile, HVX_Vector * restrict tmp, + uint32_t c0, uint32_t ncols, uint32_t j0, uint32_t j1, bool do_extra) { + top_k_mask_scatter_tile(tctx, slot, mrow, tile, tmp, c0, ncols, j0, j1, do_extra, false, false); +} + +static __attribute__((noinline)) void top_k_mask_scatter_tile_f16(const struct htp_top_k_mask_context * tctx, const uint8_t * restrict slot, + const uint8_t * restrict mrow, uint8_t * restrict tile, HVX_Vector * restrict tmp, + uint32_t c0, uint32_t ncols, uint32_t j0, uint32_t j1, bool do_extra) { + top_k_mask_scatter_tile(tctx, slot, mrow, tile, tmp, c0, ncols, j0, j1, do_extra, true, false); +} + +// PICKS: keep flags (all ones) of the picks [j0, j1) into the flag tile of the whole row, the element size picks the flag width +static __attribute__((noinline)) void top_k_mask_flag_tile_f32(const struct htp_top_k_mask_context * tctx, const uint8_t * restrict slot, + uint8_t * restrict flags, HVX_Vector * restrict tmp, uint32_t j0, uint32_t j1, bool do_extra) { + top_k_mask_scatter_tile(tctx, slot, NULL, flags, tmp, 0, tctx->kparams->n_kv, j0, j1, do_extra, false, true); +} + +static __attribute__((noinline)) void top_k_mask_flag_tile_f16(const struct htp_top_k_mask_context * tctx, const uint8_t * restrict slot, + uint8_t * restrict flags, HVX_Vector * restrict tmp, uint32_t j0, uint32_t j1, bool do_extra) { + top_k_mask_scatter_tile(tctx, slot, NULL, flags, tmp, 0, tctx->kparams->n_kv, j0, j1, do_extra, true, true); +} + +// out = flag ? out : -inf over nbytes (a multiple of 128) of the row tile +static __attribute__((noinline)) void top_k_mask_select(uint8_t * restrict out, const uint8_t * restrict flags, uint32_t nbytes, uint32_t ninf) { + const HVX_Vector v_ninf = Q6_V_vsplat_R(ninf); + HVX_Vector * vout = (HVX_Vector *) out; + const HVX_Vector * vflag = (const HVX_Vector *) flags; + const uint32_t nvec = nbytes / VLEN; + + #pragma unroll(4) + for (uint32_t i = 0; i < nvec; i++) { + vout[i] = Q6_V_vxor_VV(v_ninf, Q6_V_vand_VV(Q6_V_vxor_VV(vout[i], v_ninf), vflag[i])); + } +} + +// Push the row inputs of flat row r into slot: top_k, extra, score. Returns the push count. +static inline uint32_t top_k_mask_push_rowin(const struct htp_top_k_mask_context * tctx, dma_queue * dma_q, uint32_t r, uint8_t * slot) { + const struct htp_ops_context * octx = tctx->octx; + const struct htp_top_k_mask_kernel_params * kp = tctx->kparams; + const struct htp_tensor * top_k = octx->src[1]; + const struct htp_tensor * score = octx->src[2]; + const struct htp_tensor * extra = octx->src[4]; + + const uint32_t tk_bytes = kp->n_top * sizeof(int32_t); + dma_queue_push(dma_q, dma_make_data(slot, top_k->data + r * kp->u.rows.rs_top), tk_bytes, tk_bytes, tk_bytes, 1); + uint32_t n = 1; + + if (kp->flags & HTP_TOP_K_MASK_HAS_EXTRA) { + const uint32_t ex_bytes = kp->n_extra * sizeof(int32_t); + dma_queue_push(dma_q, dma_make_data(slot + tctx->off_extra, extra->data + r * kp->u.rows.rs_extra), ex_bytes, ex_bytes, ex_bytes, 1); + n++; + } + + if (kp->flags & HTP_TOP_K_MASK_HAS_SCORE) { + dma_queue_push(dma_q, dma_make_data(slot + tctx->off_score, score->data + r * kp->u.rows.rs_score), kp->score_bytes, kp->score_bytes, kp->score_bytes, 1); + n++; + } + + return n; +} + +static inline struct fastdiv_values top_k_mask_fd(const struct htp_top_k_mask_kernel_params * kp, uint32_t k) { + const struct fastdiv_values fd = { kp->u.rows.m.dec.mp[k], kp->u.rows.m.dec.l[k] }; + return fd; +} + +// Mask row of flat row r: (i1, i2, i3) of dst, each taken modulo the mask dims, rows dense above the mask ne1 +static inline uint32_t top_k_mask_mask_row(const struct htp_top_k_mask_kernel_params * kp, uint32_t r) { + const uint16_t * ne = kp->u.rows.m.dec.ne; + const struct fastdiv_values fd1 = top_k_mask_fd(kp, 0); + const struct fastdiv_values fd2 = top_k_mask_fd(kp, 1); + const struct fastdiv_values fm1 = top_k_mask_fd(kp, 2); + const struct fastdiv_values fm2 = top_k_mask_fd(kp, 3); + const struct fastdiv_values fm3 = top_k_mask_fd(kp, 4); + + const uint32_t q1 = fastdiv(r, &fd1); + const uint32_t i1 = r - q1 * ne[0]; + const uint32_t i3 = fastdiv(q1, &fd2); + const uint32_t i2 = q1 - i3 * ne[1]; + + const uint32_t m1 = fastmodulo(i1, ne[2], &fm1); + const uint32_t m2 = fastmodulo(i2, ne[3], &fm2); + const uint32_t m3 = fastmodulo(i3, ne[4], &fm3); + return (m3 * ne[3] + m2) * ne[2] + m1; +} + +// Push the inputs of item (row r, col tile ct): row inputs on the first tile of a row, then the mask tile. Returns the push count. +static inline uint32_t top_k_mask_push_item(const struct htp_top_k_mask_context * tctx, dma_queue * dma_q, uint32_t r, uint32_t ct, + uint8_t * slot, uint8_t * mslot) { + const struct htp_ops_context * octx = tctx->octx; + const struct htp_top_k_mask_kernel_params * kp = tctx->kparams; + const struct htp_tensor * mask = octx->src[0]; + + const uint32_t c0 = ct * kp->tile_cols; + + uint32_t n = 0; + if (ct == 0) { + n += top_k_mask_push_rowin(tctx, dma_q, r, slot); + } + + if (mslot) { + const uint32_t m_off = top_k_mask_mask_row(kp, r) * kp->u.rows.mask_nb1 + c0 * kp->elem_size; + const uint32_t nb = MIN(kp->tile_cols, kp->n_kv - c0) * kp->elem_size; + dma_queue_push(dma_q, dma_make_data(mslot, mask->data + m_off), nb, nb, nb, 1); + n++; + } + + return n; +} + +static inline void top_k_mask_next(uint32_t * r, uint32_t * ct, uint32_t n_col_tiles) { + if (++(*ct) == n_col_tiles) { + *ct = 0; + ++(*r); + } +} + +// ROWS: the thread count follows the device rows (mdev), so the slice of an n byte copy is sized here +static inline uint32_t top_k_mask_slice(const struct htp_ops_context * octx, unsigned int nth, uint32_t n) { + return hex_round_up(fastdiv(n + nth - 1, &octx->n_hvx_div), HTP_TOP_K_MASK_UNIT); +} + +// Push this thread's slice (128B aligned) of a contiguous n byte copy. Returns the push count. +static inline uint32_t top_k_mask_push_slice(dma_queue * dma_q, unsigned int ith, dma_addr_t dst, dma_addr_t src, uint32_t n, uint32_t slice) { + const uint32_t b0 = ith * slice; + if (b0 >= n) { + return 0; + } + const uint32_t nb = MIN(slice, n - b0); + dma_queue_push(dma_q, dma_make_data(dst + b0, src + b0), nb, nb, nb, 1); + return 1; +} + +// Pass 1 (ROWS): load the blocks table and the resident mask row, split across threads. Fill the tiles of this thread with -inf meanwhile. +static void top_k_mask_preload(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { + struct htp_top_k_mask_context * tctx = (struct htp_top_k_mask_context *) data; + const struct htp_top_k_mask_kernel_params * kp = tctx->kparams; + struct htp_ops_context * octx = tctx->octx; + dma_queue * dma_q = th->dma_q; + struct htp_thread_trace * tr = th->trace; + + uint32_t n_pushed = 0; + + if (kp->blocks_bytes) { + n_pushed += top_k_mask_push_slice(dma_q, ith, (dma_addr_t) tctx->blocks, octx->src[3]->data, kp->blocks_bytes, top_k_mask_slice(octx, nth, kp->blocks_bytes)); + } + if (kp->flags & HTP_TOP_K_MASK_MASK_RESIDENT) { + const uint32_t mask_bytes = kp->u.rows.m.mask_bytes; + n_pushed += top_k_mask_push_slice(dma_q, ith, (dma_addr_t) tctx->mask, octx->src[0]->data, mask_bytes, top_k_mask_slice(octx, nth, mask_bytes)); + } + + const uint32_t tile_bytes = kp->tile_units * HTP_TOP_K_MASK_UNIT; + + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ith); + top_k_mask_fill(tctx->tiles + ith * 2 * tile_bytes, 2 * tile_bytes, kp->ninf); + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ith); + + for (uint32_t i = 0; i < n_pushed; i++) { + top_k_mask_pop(dma_q, tr, (uint16_t) i); + } +} + +// Pass 2 (ROWS): depth-2 DMA pipeline over the items of rows [row_first, row_last), item = (row, col tile). +// The row inputs alternate between 2 slots by row. +static void top_k_mask_run_items(const struct htp_top_k_mask_context * tctx, const htp_thread_t * th, unsigned int ith, uint32_t row_first, uint32_t row_last) { + struct htp_ops_context * octx = tctx->octx; + const struct htp_top_k_mask_kernel_params * kp = tctx->kparams; + const struct htp_tensor * dst = octx->dst; + dma_queue * dma_q = th->dma_q; + struct htp_thread_trace * tr = th->trace; + + const bool is_f16 = kp->flags & HTP_TOP_K_MASK_F16; + const uint32_t elem_size = kp->elem_size; + + const uint32_t rowin_bytes = kp->rowin_units * HTP_TOP_K_MASK_UNIT; + const uint32_t tile_bytes = kp->tile_units * HTP_TOP_K_MASK_UNIT; + const uint32_t tile_cols = kp->tile_cols; + const uint32_t n_col_tiles = kp->n_col_tiles; + const uint32_t n_items = (row_last - row_first) * n_col_tiles; + + // 2 row input slots per thread, alternating by row + uint8_t * rowin = tctx->rowin + ith * 2 * rowin_bytes; + uint8_t * tiles = tctx->tiles + ith * 2 * tile_bytes; + uint8_t * mtiles = (kp->flags & HTP_TOP_K_MASK_MASK_RESIDENT) ? NULL : tctx->mtiles + ith * 2 * tile_bytes; + + HVX_Vector * tmp = (HVX_Vector *) (tctx->scratch + ith * HTP_TOP_K_MASK_SCRATCH_BYTES); + + const dma_addr_t dst_data = dst->data; + + uint32_t n_in[2] = { 0, 0 }; + + // prefetch cursor + uint32_t pr = row_first, pct = 0; + + for (uint32_t it = 0; it < MIN(n_items, 2); it++) { + // Dummy dst writeback to establish queue ordering + dma_queue_push(dma_q, dma_make_data(dst_data, tiles + it * tile_bytes), 0, 0, 0, 0); + n_in[it] = top_k_mask_push_item(tctx, dma_q, pr, pct, rowin + ((pr - row_first) & 1) * rowin_bytes, mtiles ? mtiles + it * tile_bytes : NULL); + top_k_mask_next(&pr, &pct, n_col_tiles); + } + + uint32_t r = row_first, ct = 0; + + for (uint32_t it = 0; it < n_items; it++) { + const uint32_t s = it & 1; + uint8_t * tile = tiles + s * tile_bytes; + + // writeback of item it - 2 (or the dummy), then the tile is free again + top_k_mask_pop(dma_q, tr, (uint16_t) it); + if (it >= 2) { + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) it); + top_k_mask_fill(tile, tile_bytes, kp->ninf); + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) it); + } + + for (uint32_t i = 0; i < n_in[s]; i++) { + top_k_mask_pop(dma_q, tr, (uint16_t) it); + } + + const uint32_t c0 = ct * tile_cols; + const uint32_t ncols = MIN(tile_cols, kp->n_kv - c0); + + const uint8_t * slot = rowin + ((r - row_first) & 1) * rowin_bytes; + const uint8_t * mrow = mtiles ? mtiles + s * tile_bytes : tctx->mask + c0 * elem_size; + + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) it); + if (is_f16) { + top_k_mask_scatter_tile_f16(tctx, slot, mrow, tile, tmp, c0, ncols, 0, kp->n_top, true); + } else { + top_k_mask_scatter_tile_f32(tctx, slot, mrow, tile, tmp, c0, ncols, 0, kp->n_top, true); + } + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) it); + + const uint32_t d_off = r * kp->u.rows.rs_dst + c0 * elem_size; + const uint32_t nb = ncols * elem_size; + dma_queue_push(dma_q, dma_make_data(dst_data + d_off, tile), nb, nb, nb, 1); + + if (it + 2 < n_items) { + n_in[s] = top_k_mask_push_item(tctx, dma_q, pr, pct, rowin + ((pr - row_first) & 1) * rowin_bytes, mtiles ? mtiles + s * tile_bytes : NULL); + top_k_mask_next(&pr, &pct, n_col_tiles); + } + top_k_mask_next(&r, &ct, n_col_tiles); + } + + const uint32_t n_wb = MIN(n_items, 2); + for (uint32_t i = 0; i < n_wb; i++) { + top_k_mask_pop(dma_q, tr, (uint16_t) i); + } +} + +// Pass 2 (ROWS): whole rows split across threads, the col tiles of a row in order +static void top_k_mask_thread_rows(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { + const struct htp_top_k_mask_context * tctx = (const struct htp_top_k_mask_context *) data; + + const uint32_t ir0 = tctx->row0 + ith * tctx->rows_per_hvx; + const uint32_t ir1 = MIN(ir0 + tctx->rows_per_hvx, tctx->row1); + if (ir0 >= ir1) { + return; + } + + top_k_mask_run_items(tctx, th, ith, ir0, ir1); +} + +static inline void top_k_mask_barrier(atomic_uint * b, uint32_t target, struct htp_thread_trace * tr, uint16_t info) { + htp_trace_event_start(tr, HTP_TRACE_EVT_FENCE, info); + atomic_fetch_add_explicit(b, 1, memory_order_release); + while (atomic_load_explicit(b, memory_order_acquire) < target) { + hex_pause(); + } + htp_trace_event_stop(tr, HTP_TRACE_EVT_FENCE, info); +} + +// PICKS, one pass over n_virt = kp->n_hvx virtual workers (host slices); real worker ith runs the virtual workers ith, ith + nth, ... +// A: shared preload and a slice of the flag clear per virtual worker, the mask slice of this worker into the row tile; +// barrier; B: scatter keep flags of the picks of each virtual worker; barrier; C: select mask or -inf in this worker's slice and write it back +static void top_k_mask_thread_picks(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { + struct htp_top_k_mask_context * tctx = (struct htp_top_k_mask_context *) data; + const struct htp_top_k_mask_kernel_params * kp = tctx->kparams; + struct htp_ops_context * octx = tctx->octx; + dma_queue * dma_q = th->dma_q; + struct htp_thread_trace * tr = th->trace; + + const struct htp_tensor * top_k = octx->src[1]; + const struct htp_tensor * score = octx->src[2]; + const struct htp_tensor * extra = octx->src[4]; + + uint8_t * slot = tctx->rowin; + const uint32_t tile_bytes = kp->tile_units * HTP_TOP_K_MASK_UNIT; + uint8_t * out = tctx->tiles; + uint8_t * flags = tctx->tiles + tile_bytes; + + const uint32_t n_virt = kp->n_hvx; + + // this worker's slice of the device columns: the mask goes straight into the row tile, it is popped after the scan. + // The device share of the columns depends on mdev, so this slice is sized here. + const uint32_t out_b0 = tctx->col0 * kp->elem_size; + const uint32_t out_nb = tctx->ncols_out * kp->elem_size; + const uint32_t wslice = hex_round_up(fastdiv(out_nb + nth - 1, &octx->n_hvx_div), 128); + const uint32_t wb0 = out_b0 + ith * wslice; + const uint32_t wnb = ith * wslice < out_nb ? MIN(wslice, out_nb - ith * wslice) : 0; + + const uint32_t fslice = kp->u.picks.flags_slice_u * HTP_TOP_K_MASK_UNIT; + + for (uint32_t v = ith; v < n_virt; v += nth) { + uint32_t n_pushed = 0; + if (kp->blocks_bytes) { + n_pushed += top_k_mask_push_slice(dma_q, v, (dma_addr_t) tctx->blocks, octx->src[3]->data, kp->blocks_bytes, kp->u.picks.blocks_slice_u * HTP_TOP_K_MASK_UNIT); + } + n_pushed += top_k_mask_push_slice(dma_q, v, (dma_addr_t) slot, top_k->data, kp->n_top * sizeof(int32_t), kp->u.picks.top_slice_u * HTP_TOP_K_MASK_UNIT); + if (kp->flags & HTP_TOP_K_MASK_HAS_EXTRA) { + n_pushed += top_k_mask_push_slice(dma_q, v, (dma_addr_t) (slot + tctx->off_extra), extra->data, kp->n_extra * sizeof(int32_t), kp->u.picks.extra_slice_u * HTP_TOP_K_MASK_UNIT); + } + if (kp->flags & HTP_TOP_K_MASK_HAS_SCORE) { + n_pushed += top_k_mask_push_slice(dma_q, v, (dma_addr_t) (slot + tctx->off_score), score->data, kp->score_bytes, kp->u.picks.score_slice_u * HTP_TOP_K_MASK_UNIT); + } + + // the mask slice follows the last preload in the queue, so the pops below leave it in flight + if (wnb && v + nth >= n_virt) { + dma_queue_push(dma_q, dma_make_data(out + wb0, octx->src[0]->data + wb0), wnb, wnb, wnb, 1); + } + + const uint32_t fb0 = v * fslice; + if (fb0 < tile_bytes) { + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) v); + top_k_mask_fill(flags + fb0, MIN(fslice, tile_bytes - fb0), 0); // clear the flags + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) v); + } + + for (uint32_t i = 0; i < n_pushed; i++) { + top_k_mask_pop(dma_q, tr, (uint16_t) i); + } + } + + top_k_mask_barrier(&tctx->barrier, nth, tr, 0); + + HVX_Vector * tmp = (HVX_Vector *) (tctx->scratch + ith * HTP_TOP_K_MASK_SCRATCH_BYTES); + + for (uint32_t v = ith; v < n_virt; v += nth) { + const uint32_t j0 = MIN(v * kp->u.picks.picks_per_hvx, kp->n_top); + const uint32_t j1 = MIN(j0 + kp->u.picks.picks_per_hvx, kp->n_top); + const bool do_extra = v == n_virt - 1; + + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) v); + if (kp->flags & HTP_TOP_K_MASK_F16) { + top_k_mask_flag_tile_f16(tctx, slot, flags, tmp, j0, j1, do_extra); + } else { + top_k_mask_flag_tile_f32(tctx, slot, flags, tmp, j0, j1, do_extra); + } + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) v); + } + + top_k_mask_barrier(&tctx->barrier, 2 * nth, tr, 1); + + if (wnb) { + top_k_mask_pop(dma_q, tr, (uint16_t) ith); + // wb0 is 128B aligned (col0 is a multiple of 64 columns), the select may run past the row inside the tile + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ith); + top_k_mask_select(out + wb0, flags + wb0, hex_round_up(wnb, 128), kp->ninf); + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ith); + dma_queue_push(dma_q, dma_make_data(octx->dst->data + wb0, out + wb0), wnb, wnb, wnb, 1); + top_k_mask_pop(dma_q, tr, (uint16_t) ith); + } +} + +int op_top_k_mask(struct htp_ops_context * octx) { + const struct htp_top_k_mask_kernel_params * kp = (const struct htp_top_k_mask_kernel_params *) octx->kernel_params; + + if (kp->vtcm_size > octx->ctx->vtcm_size) { + return HTP_STATUS_VTCM_TOO_SMALL; + } + + const bool picks = kp->kernel_type == HTP_TOP_K_MASK_KERNEL_PICKS; + + // mdev: ROWS gives this device a contiguous share of the rows, DMA writes are exempt from the 128B rule. + // PICKS: every device scans the row, the write back is split by 64 column chunks. + uint32_t dev_row0 = 0; + uint32_t dev_rows = picks ? 1 : kp->u.rows.total_rows; + uint32_t dev_c0 = 0; + uint32_t dev_cols = kp->n_kv; + if (octx->ctx->mdev.count > 1) { + const struct htp_tensor_mdev_range range = picks ? + htp_tensor_mdev_partition(kp->n_kv, 64, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div) : + htp_tensor_mdev_partition(kp->u.rows.total_rows, 1, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div); + if (range.count == 0) { + return HTP_STATUS_OK; + } + if (picks) { + dev_c0 = range.start; + dev_cols = range.count; + } else { + dev_row0 = range.start; + dev_rows = range.count; + } + } + + // fewer workers than kparams->n_hvx always fit the VTCM layout: ROWS runs one row range per worker, PICKS loops over the host slices + const uint32_t max_hvx = MIN((uint32_t) kp->n_hvx, octx->ctx->n_hvx); + if (!htp_ops_context_set_n_hvx(octx, picks ? max_hvx : MIN(max_hvx, dev_rows))) { + return HTP_STATUS_INVAL_PARAMS; + } + const uint32_t n_hvx = octx->n_hvx; + + uint8_t * vtcm = octx->ctx->vtcm_base; + + struct htp_top_k_mask_context tctx = { + .octx = octx, + .kparams = kp, + .off_extra = kp->off_extra_units * HTP_TOP_K_MASK_UNIT, + .off_score = kp->off_score_units * HTP_TOP_K_MASK_UNIT, + .row0 = dev_row0, + .row1 = dev_row0 + dev_rows, + .col0 = dev_c0, + .ncols_out = dev_cols, + .blocks = VTCM_LAYOUT_PTR(uint8_t, vtcm, kp->off_blocks_u * HTP_TOP_K_MASK_UNIT), + .mask = VTCM_LAYOUT_PTR(uint8_t, vtcm, kp->off_mask_u * HTP_TOP_K_MASK_UNIT), + .rowin = VTCM_LAYOUT_PTR(uint8_t, vtcm, kp->off_rowin_u * HTP_TOP_K_MASK_UNIT), + .tiles = VTCM_LAYOUT_PTR(uint8_t, vtcm, kp->off_tiles_u * HTP_TOP_K_MASK_UNIT), + .mtiles = VTCM_LAYOUT_PTR(uint8_t, vtcm, kp->off_mtiles_u * HTP_TOP_K_MASK_UNIT), + .scratch = VTCM_LAYOUT_PTR(uint8_t, vtcm, kp->off_scratch_u * HTP_TOP_K_MASK_UNIT), + }; + + if (picks) { + atomic_init(&tctx.barrier, 0); + work_queue_run(octx->ctx->hvx_queue, top_k_mask_thread_picks, &tctx, n_hvx); + return HTP_STATUS_OK; + } + + tctx.rows_per_hvx = fastdiv(dev_rows + n_hvx - 1, &octx->n_hvx_div); + + work_queue_run(octx->ctx->hvx_queue, top_k_mask_preload, &tctx, n_hvx); + work_queue_run(octx->ctx->hvx_queue, top_k_mask_thread_rows, &tctx, n_hvx); + + return HTP_STATUS_OK; +} diff --git a/ggml/src/ggml-hexagon/htp/top-k-mask-ops.h b/ggml/src/ggml-hexagon/htp/top-k-mask-ops.h new file mode 100644 index 000000000000..9dcdfaa4808d --- /dev/null +++ b/ggml/src/ggml-hexagon/htp/top-k-mask-ops.h @@ -0,0 +1,131 @@ +#ifndef HTP_TOP_K_MASK_OPS_H +#define HTP_TOP_K_MASK_OPS_H + +#include "hex-common.h" +#include "hex-fastdiv.h" +#include "htp-vtcm.h" +#include +#include +#include + +// HVX gather results land here (gathers write to VTCM only): 10 vectors per thread +#define HTP_TOP_K_MASK_SCRATCH_BYTES 1280 + +enum htp_top_k_mask_kernel { + HTP_TOP_K_MASK_KERNEL_ROWS = 0, // threads split rows, row inputs streamed per row + HTP_TOP_K_MASK_KERNEL_PICKS = 1, // one row with one resident mask row: threads split the picks, keep flags select mask or -inf +}; + +#define HTP_TOP_K_MASK_F16 (1u << 0) +#define HTP_TOP_K_MASK_HAS_SCORE (1u << 1) +#define HTP_TOP_K_MASK_HAS_BLOCKS (1u << 2) +#define HTP_TOP_K_MASK_HAS_EXTRA (1u << 3) +#define HTP_TOP_K_MASK_MASK_RESIDENT (1u << 4) // mask has exactly 1 row + +#define HTP_TOP_K_MASK_UNIT 128 // unit of the compact u16 sizes in kernel_params + +struct htp_top_k_mask_shape { + uint32_t n_kv, elem_size, n_top, n_pick, n_blk, n_extra; + uint32_t has_score, has_blocks, mask_resident; +}; + +struct htp_top_k_mask_vtcm_layout { + size_t total_bytes; + size_t off_blocks, off_mask, off_rowin, off_tiles, off_mtiles; // 0 when unused + size_t off_scratch; // HTP_TOP_K_MASK_SCRATCH_BYTES per thread + size_t rowin_bytes; // one slot of top_k + extra + score row, parts at htp_top_k_mask_rowin_off_* + size_t tile_bytes; // one output tile slot, 128B aligned + size_t blocks_bytes, mask_bytes; // exact DMA sizes, 0 when unused +}; + +static inline uint64_t htp_top_k_mask_align128(uint64_t n) { + return (n + 127) & ~(uint64_t) 127; +} + +// Parts of one row input slot: top_k at 0, then extra, then score; used by the host layout +static inline uint64_t htp_top_k_mask_rowin_off_extra(uint32_t n_top) { + return htp_top_k_mask_align128((uint64_t) n_top * 4); +} + +static inline uint64_t htp_top_k_mask_rowin_off_score(uint32_t n_top, uint32_t n_extra) { + return htp_top_k_mask_rowin_off_extra(n_top) + htp_top_k_mask_align128((uint64_t) n_extra * 4); +} + +// the caller compares total_bytes against the VTCM size +static inline void htp_top_k_mask_vtcm_layout_build(struct htp_top_k_mask_vtcm_layout * L, + const struct htp_top_k_mask_shape * S, uint32_t kernel_type, uint32_t n_hvx, uint32_t tile_cols) { + memset(L, 0, sizeof(*L)); + + // PICKS streams the mask straight into its row tile + const bool picks = kernel_type == HTP_TOP_K_MASK_KERNEL_PICKS; + const uint64_t blocks_exact = S->has_blocks ? (uint64_t) S->n_blk * S->n_pick * 4 : 0; + const uint64_t blocks_bytes = htp_top_k_mask_align128(blocks_exact); + const uint64_t mask_exact = S->mask_resident && !picks ? (uint64_t) S->n_kv * S->elem_size : 0; + const uint64_t mask_bytes = htp_top_k_mask_align128(mask_exact); + const uint64_t rowin_bytes = htp_top_k_mask_rowin_off_score(S->n_top, S->n_extra) + + (S->has_score ? htp_top_k_mask_align128((uint64_t) S->n_pick * 4) : 0); + const uint64_t tile_bytes = htp_top_k_mask_align128((uint64_t) tile_cols * S->elem_size); + + // ROWS streams row inputs into 2 slots per thread. PICKS: one row, a row tile and a flag tile. + const uint64_t rowin_total = (picks ? 1 : 2 * (uint64_t) n_hvx) * rowin_bytes; + const uint64_t tiles_total = picks ? 2 * tile_bytes : 2 * (uint64_t) n_hvx * tile_bytes; + const uint64_t mtiles_total = S->mask_resident ? 0 : tiles_total; + const uint64_t scratch = (uint64_t) n_hvx * HTP_TOP_K_MASK_SCRATCH_BYTES; + + size_t off = 0; + VTCM_LAYOUT_ALLOC_OPTIONAL(off, off_blocks, (size_t) blocks_bytes, blocks_bytes > 0); + VTCM_LAYOUT_ALLOC_OPTIONAL(off, off_mask, (size_t) mask_bytes, mask_bytes > 0); + VTCM_LAYOUT_ALLOC(off, off_rowin, (size_t) rowin_total); + VTCM_LAYOUT_ALLOC(off, off_tiles, (size_t) tiles_total); + VTCM_LAYOUT_ALLOC_OPTIONAL(off, off_mtiles, (size_t) mtiles_total, !S->mask_resident); + VTCM_LAYOUT_ALLOC(off, off_scratch, (size_t) scratch); + + L->rowin_bytes = (size_t) rowin_bytes; + L->tile_bytes = (size_t) tile_bytes; + L->blocks_bytes = (size_t) blocks_exact; + L->mask_bytes = (size_t) mask_exact; + L->total_bytes = off; +} + +// Set by the host, used as is by the NPU: no tensor shape, stride or layout is derived on the NPU. +// Sizes of one slot or tile, the slot parts and the VTCM offsets are in HTP_TOP_K_MASK_UNIT bytes as u16. +// Row strides: byte offset of flat row r of top_k, extra, score and dst is r * stride (the host requires row flattenable inputs). +// PICKS splits the work in n_hvx virtual workers; fewer real workers loop over them. +struct htp_top_k_mask_kernel_params { + uint8_t n_hvx, kernel_type, flags, elem_size; + uint32_t vtcm_size; + uint32_t n_kv, n_top, n_pick, n_extra; + uint32_t ninf; // -inf cell pattern of elem_size + uint32_t blocks_bytes, score_bytes; // exact DMA sizes, gather regions are [0, bytes) + uint32_t tile_cols, n_col_tiles; + uint16_t n_blk, blocks_nb1; // blocks_nb1 = n_blk * 4, fits a signed 16-bit multiply + uint16_t off_blocks_u, off_mask_u, off_rowin_u, off_tiles_u, off_mtiles_u, off_scratch_u; + uint16_t rowin_units, tile_units, off_extra_units, off_score_units; + union { + struct { + uint32_t total_rows; + uint32_t rs_top, rs_extra, rs_score, rs_dst; // flat row strides (bytes of row 1) + uint32_t mask_nb1; // mask row stride, rows of the mask are dense above it + union { + uint32_t mask_bytes; // resident mask (1 row) + struct { // broadcast mask: row r -> (i1, i2, i3) -> mask row (i mod mne) + uint32_t mp[5]; // fastdiv multipliers of dst ne1, ne2 and mask ne1, ne2, ne3 + uint16_t ne[5]; // the divisors + uint8_t l[5]; // fastdiv shifts + } dec; + } m; + } rows; + struct { + uint32_t picks_per_hvx; // per virtual worker, multiple of the lanes of one vector + uint16_t blocks_slice_u, top_slice_u, extra_slice_u, score_slice_u, flags_slice_u; // per virtual worker slices + } picks; + } u; +}; + +#if defined(__cplusplus) +static_assert(sizeof(struct htp_top_k_mask_kernel_params) <= 128, "htp_top_k_mask_kernel_params is too large for kernel_params blob"); +#else +_Static_assert(sizeof(struct htp_top_k_mask_kernel_params) <= 128, "htp_top_k_mask_kernel_params is too large for kernel_params blob"); +#endif + +#endif // HTP_TOP_K_MASK_OPS_H diff --git a/ggml/src/ggml-hexagon/htp/unary-ops.c b/ggml/src/ggml-hexagon/htp/unary-ops.c index 3b63ac674d73..19ade22f376b 100644 --- a/ggml/src/ggml-hexagon/htp/unary-ops.c +++ b/ggml/src/ggml-hexagon/htp/unary-ops.c @@ -21,7 +21,7 @@ #include "htp-ops.h" #include "htp-tensor.h" #include "htp-vtcm.h" -#include "hex-profile.h" +#include "htp-trace.h" struct htp_unary_context; @@ -36,6 +36,13 @@ typedef void (*unary_rms_norm_mul_compute_fn_t)(const void * restrict src, uint32_t num_rows, const struct htp_unary_context * uctx); +typedef void (*unary_rms_norm_mul_add_compute_fn_t)(const void * restrict src, + const void * restrict weight, + const void * restrict add, + void * restrict dst, + uint32_t num_rows, + const struct htp_unary_context * uctx); + typedef void (*unary_tri_compute_fn_t)(const void * restrict src, void * restrict dst, uint32_t num_rows, @@ -63,6 +70,7 @@ struct htp_unary_context { dma_addr_t data_src0; dma_addr_t data_src1; // weight/scale tensor for RMS_NORM_MUL + dma_addr_t data_src2; // residual tensor for RMS_NORM_MUL_ADD dma_addr_t data_dst; size_t src0_data_row_size; // actual data bytes per row @@ -79,7 +87,7 @@ struct htp_unary_context { uint32_t block; uint32_t src0_nrows; - uint32_t src0_nrows_per_thread; + uint32_t src0_nrows_per_hvx; uint32_t row_start; uint32_t nc; uint32_t col_tile; // tiled mode @@ -87,11 +95,13 @@ struct htp_unary_context { uint8_t * vtcm_src0; uint8_t * vtcm_src1; + uint8_t * vtcm_src2; uint8_t * vtcm_dst; - size_t vtcm_src0_size_per_thread; - size_t vtcm_src1_size_per_thread; - size_t vtcm_dst_size_per_thread; + size_t vtcm_src0_size_per_hvx; + size_t vtcm_src1_size_per_hvx; + size_t vtcm_src2_size_per_hvx; + size_t vtcm_dst_size_per_hvx; }; // Convert flat row index to DDR byte offset using the tensor's actual strides. @@ -244,6 +254,29 @@ static void rms_norm_mul_f32(const void * restrict src, } } +static void rms_norm_mul_add_f32(const void * restrict src, + const void * restrict weight, + const void * restrict add, + void * restrict dst, + const uint32_t num_rows, + const struct htp_unary_context * uctx) { + htp_unary_op_preamble; + float epsilon = 0.f; + float scale = 0.f; + memcpy(&epsilon, &op_params[0], sizeof(float)); + memcpy(&scale, &op_params[1], sizeof(float)); + + const uint8_t * restrict w_local = (const uint8_t *) weight; + + for (uint32_t ir = 0; ir < num_rows; ir++) { + const uint8_t * restrict src_local = (const uint8_t *) src + (ir * src0_row_size_aligned); + const uint8_t * restrict add_local = (const uint8_t *) add + (ir * src0_row_size_aligned); + uint8_t * restrict dst_local = (uint8_t *) dst + (ir * dst_row_size_aligned); + + hvx_fast_rms_norm_mul_add_f32(src_local, w_local, add_local, dst_local, ne0, epsilon, scale); + } +} + static void norm_f32(const void * restrict src, void * restrict dst, const uint32_t num_rows, @@ -1200,25 +1233,25 @@ static void tri_apply_tile_f32(const void * restrict src, void * restrict dst, } // 1. Standard row-block unary task (F32 and F16). -static void unary_thread_row_block(unsigned int nth, unsigned int ith, void * data) { +static void unary_thread_row_block(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { (void) nth; const struct htp_unary_context * uctx = (const struct htp_unary_context *) data; struct htp_ops_context * octx = uctx->octx; const struct htp_tensor * src = octx->src[0]; const struct htp_tensor * dst = octx->dst; - struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + struct htp_thread_trace * tr = th->trace; htp_unary_preamble; - const uint32_t src0_nrows_per_thread = uctx->src0_nrows_per_thread; + const uint32_t src0_nrows_per_hvx = uctx->src0_nrows_per_hvx; const size_t src0_data_row_size = uctx->src0_data_row_size; const size_t dst_data_row_size = uctx->dst_data_row_size; const size_t src0_row_size_aligned = uctx->src0_row_size_aligned; const size_t dst_row_size_aligned = uctx->dst_row_size_aligned; const uint32_t src0_nrows = uctx->src0_nrows; - const uint32_t src0_start_row = uctx->row_start + src0_nrows_per_thread * ith; - const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, uctx->row_start + src0_nrows); + const uint32_t src0_start_row = uctx->row_start + src0_nrows_per_hvx * ith; + const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_hvx, uctx->row_start + src0_nrows); if (src0_start_row >= src0_end_row) { return; @@ -1227,8 +1260,8 @@ static void unary_thread_row_block(unsigned int nth, unsigned int ith, void * da const dma_addr_t data_src = uctx->data_src0; const dma_addr_t data_dst = uctx->data_dst; - uint8_t * src0_vtcm_data = uctx->vtcm_src0 + (ith * uctx->vtcm_src0_size_per_thread); - uint8_t * dst_vtcm_data = uctx->vtcm_dst + (ith * uctx->vtcm_dst_size_per_thread); + uint8_t * src0_vtcm_data = uctx->vtcm_src0 + (ith * uctx->vtcm_src0_size_per_hvx); + uint8_t * dst_vtcm_data = uctx->vtcm_dst + (ith * uctx->vtcm_dst_size_per_hvx); const size_t src0_vtcm_half_size = uctx->src0_vtcm_half_size; const size_t dst_vtcm_half_size = uctx->dst_vtcm_half_size; @@ -1240,7 +1273,7 @@ static void unary_thread_row_block(unsigned int nth, unsigned int ith, void * da const struct fastdiv_values * div_ne02 = &uctx->kparams->div_ne02; const struct fastdiv_values * div_ne012 = &uctx->kparams->div_ne012; - dma_queue * dma_q = octx->ctx->dma[ith]; + dma_queue * dma_q = th->dma_q; for (uint32_t ir = src0_start_row, vtcm_idx = 0; ir < src0_end_row && vtcm_idx < 2; vtcm_idx++) { const uint32_t block_size = unary_block_size(ir, src0_end_row, BLOCK, src0_contig, ne01, div_ne01); @@ -1297,25 +1330,25 @@ static void unary_thread_row_block(unsigned int nth, unsigned int ith, void * da } // 2. RMS_NORM_MUL row-block task with weight buffer. -static void unary_thread_rms_norm_mul_f32(unsigned int nth, unsigned int ith, void * data) { +static void unary_thread_rms_norm_mul_f32(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { (void) nth; const struct htp_unary_context * uctx = (const struct htp_unary_context *) data; struct htp_ops_context * octx = uctx->octx; const struct htp_tensor * src = octx->src[0]; const struct htp_tensor * dst = octx->dst; - struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + struct htp_thread_trace * tr = th->trace; htp_unary_preamble; - const uint32_t src0_nrows_per_thread = uctx->src0_nrows_per_thread; + const uint32_t src0_nrows_per_hvx = uctx->src0_nrows_per_hvx; const size_t src0_data_row_size = uctx->src0_data_row_size; const size_t dst_data_row_size = uctx->dst_data_row_size; const size_t src0_row_size_aligned = uctx->src0_row_size_aligned; const size_t dst_row_size_aligned = uctx->dst_row_size_aligned; const uint32_t src0_nrows = uctx->src0_nrows; - const uint32_t src0_start_row = uctx->row_start + src0_nrows_per_thread * ith; - const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, uctx->row_start + src0_nrows); + const uint32_t src0_start_row = uctx->row_start + src0_nrows_per_hvx * ith; + const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_hvx, uctx->row_start + src0_nrows); if (src0_start_row >= src0_end_row) { return; @@ -1330,9 +1363,9 @@ static void unary_thread_rms_norm_mul_f32(unsigned int nth, unsigned int ith, vo const uint32_t nb12 = src1->nb[2]; const uint32_t nb13 = src1->nb[3]; - uint8_t * src0_vtcm_data = uctx->vtcm_src0 + (ith * uctx->vtcm_src0_size_per_thread); - uint8_t * src1_vtcm_data = uctx->vtcm_src1 ? (uctx->vtcm_src1 + (ith * uctx->vtcm_src1_size_per_thread)) : NULL; - uint8_t * dst_vtcm_data = uctx->vtcm_dst + (ith * uctx->vtcm_dst_size_per_thread); + uint8_t * src0_vtcm_data = uctx->vtcm_src0 + (ith * uctx->vtcm_src0_size_per_hvx); + uint8_t * src1_vtcm_data = uctx->vtcm_src1 ? (uctx->vtcm_src1 + (ith * uctx->vtcm_src1_size_per_hvx)) : NULL; + uint8_t * dst_vtcm_data = uctx->vtcm_dst + (ith * uctx->vtcm_dst_size_per_hvx); const size_t src0_vtcm_half_size = uctx->src0_vtcm_half_size; const size_t src1_vtcm_half_size = uctx->src1_vtcm_half_size; @@ -1347,7 +1380,7 @@ static void unary_thread_rms_norm_mul_f32(unsigned int nth, unsigned int ith, vo const struct fastdiv_values * div_ne02 = &uctx->kparams->div_ne02; const struct fastdiv_values * div_ne012 = &uctx->kparams->div_ne012; - dma_queue * dma_q = octx->ctx->dma[ith]; + dma_queue * dma_q = th->dma_q; if (uctx->broadcast_weight) { dma_queue_push(dma_q, dma_make_data(src1_vtcm_data, data_src1), @@ -1430,26 +1463,138 @@ static void unary_thread_rms_norm_mul_f32(unsigned int nth, unsigned int ith, vo dma_queue_flush(dma_q); } +// 2b. RMS_NORM_MUL_ADD row-block task with broadcast weight and residual add. +static void unary_thread_rms_norm_mul_add_f32(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { + (void) nth; + const struct htp_unary_context * uctx = (const struct htp_unary_context *) data; + struct htp_ops_context * octx = uctx->octx; + const struct htp_tensor * src = octx->src[0]; + const struct htp_tensor * dst = octx->dst; + struct htp_thread_trace * tr = th->trace; + + htp_unary_preamble; + + const uint32_t src0_nrows_per_hvx = uctx->src0_nrows_per_hvx; + const size_t src0_data_row_size = uctx->src0_data_row_size; + const size_t dst_data_row_size = uctx->dst_data_row_size; + const size_t src0_row_size_aligned = uctx->src0_row_size_aligned; + const size_t dst_row_size_aligned = uctx->dst_row_size_aligned; + + const uint32_t src0_nrows = uctx->src0_nrows; + const uint32_t src0_start_row = uctx->row_start + src0_nrows_per_hvx * ith; + const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_hvx, uctx->row_start + src0_nrows); + + if (src0_start_row >= src0_end_row) { + return; + } + + const dma_addr_t data_src = uctx->data_src0; + const dma_addr_t data_src1 = uctx->data_src1; + const dma_addr_t data_src2 = uctx->data_src2; + const dma_addr_t data_dst = uctx->data_dst; + + uint8_t * src0_vtcm_data = uctx->vtcm_src0 + (ith * uctx->vtcm_src0_size_per_hvx); + uint8_t * src1_vtcm_data = uctx->vtcm_src1 + (ith * uctx->vtcm_src1_size_per_hvx); + uint8_t * src2_vtcm_data = uctx->vtcm_src2 + (ith * uctx->vtcm_src2_size_per_hvx); + uint8_t * dst_vtcm_data = uctx->vtcm_dst + (ith * uctx->vtcm_dst_size_per_hvx); + + const size_t src0_vtcm_half_size = uctx->src0_vtcm_half_size; + const size_t dst_vtcm_half_size = uctx->dst_vtcm_half_size; + + const bool src0_contig = uctx->kparams->src0_contig; + const uint32_t BLOCK = uctx->block; + + const struct fastdiv_values * div_ne01 = &uctx->kparams->div_ne01; + const struct fastdiv_values * div_ne02 = &uctx->kparams->div_ne02; + const struct fastdiv_values * div_ne012 = &uctx->kparams->div_ne012; + + dma_queue * dma_q = th->dma_q; + + dma_queue_push(dma_q, dma_make_data(src1_vtcm_data, data_src1), + uctx->src1_row_size_aligned, 0, uctx->src1_data_row_size, 1); + dma_queue_flush(dma_q); + + for (uint32_t ir = src0_start_row, vtcm_idx = 0; ir < src0_end_row && vtcm_idx < 2; vtcm_idx++) { + const uint32_t block_size = unary_block_size(ir, src0_end_row, BLOCK, src0_contig, ne01, div_ne01); + + dma_queue_push(dma_q, + dma_make_data(data_dst, dst_vtcm_data + (vtcm_idx * dst_vtcm_half_size)), + nb1, dst_row_size_aligned, dst_data_row_size, 0); + + const size_t src0_off = src0_contig ? ((size_t) ir * nb01) : + unary_row_offset(ir, ne01, ne02, div_ne01, div_ne02, div_ne012, nb01, nb02, nb03); + dma_queue_push(dma_q, + dma_make_data(src0_vtcm_data + (vtcm_idx * src0_vtcm_half_size), data_src + src0_off), + src0_row_size_aligned, nb01, src0_data_row_size, block_size); + dma_queue_push(dma_q, + dma_make_data(src2_vtcm_data + (vtcm_idx * src0_vtcm_half_size), data_src2 + src0_off), + src0_row_size_aligned, nb01, src0_data_row_size, block_size); + + ir += block_size; + } + + unary_rms_norm_mul_add_compute_fn_t compute = (unary_rms_norm_mul_add_compute_fn_t) uctx->compute; + + for (uint32_t ir = src0_start_row; ir < src0_end_row; ) { + const uint32_t block_size = unary_block_size(ir, src0_end_row, BLOCK, src0_contig, ne01, div_ne01); + + void * dst_vtcm = (void *) (uintptr_t) dma_queue_pop(dma_q).src; + void * src0_vtcm = (void *) (uintptr_t) dma_queue_pop(dma_q).dst; + void * src2_vtcm = (void *) (uintptr_t) dma_queue_pop(dma_q).dst; + + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, ir); + compute(src0_vtcm, src1_vtcm_data, src2_vtcm, dst_vtcm, block_size, uctx); + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, ir); + + const size_t dst_off = (size_t) ir * nb1; + dma_queue_push(dma_q, + dma_make_data(data_dst + dst_off, dst_vtcm), + nb1, dst_row_size_aligned, dst_data_row_size, block_size); + + const uint32_t next_ir = ir + block_size; + if (next_ir < src0_end_row) { + const uint32_t next_block_size = unary_block_size(next_ir, src0_end_row, BLOCK, src0_contig, + ne01, div_ne01); + const uint32_t pref_ir = next_ir + next_block_size; + if (pref_ir < src0_end_row) { + const uint32_t pref_block_size = unary_block_size(pref_ir, src0_end_row, BLOCK, src0_contig, + ne01, div_ne01); + const size_t src0_pref_off = src0_contig ? ((size_t) pref_ir * nb01) : + unary_row_offset(pref_ir, ne01, ne02, div_ne01, div_ne02, div_ne012, nb01, nb02, nb03); + dma_queue_push(dma_q, + dma_make_data(src0_vtcm, data_src + src0_pref_off), + src0_row_size_aligned, nb01, src0_data_row_size, pref_block_size); + dma_queue_push(dma_q, + dma_make_data(src2_vtcm, data_src2 + src0_pref_off), + src0_row_size_aligned, nb01, src0_data_row_size, pref_block_size); + } + } + ir += block_size; + } + + dma_queue_flush(dma_q); +} + // 3. TRI row-block task with row index ir. -static void unary_thread_tri_f32(unsigned int nth, unsigned int ith, void * data) { +static void unary_thread_tri_f32(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { (void) nth; const struct htp_unary_context * uctx = (const struct htp_unary_context *) data; struct htp_ops_context * octx = uctx->octx; const struct htp_tensor * src = octx->src[0]; const struct htp_tensor * dst = octx->dst; - struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + struct htp_thread_trace * tr = th->trace; htp_unary_preamble; - const uint32_t src0_nrows_per_thread = uctx->src0_nrows_per_thread; + const uint32_t src0_nrows_per_hvx = uctx->src0_nrows_per_hvx; const size_t src0_data_row_size = uctx->src0_data_row_size; const size_t dst_data_row_size = uctx->dst_data_row_size; const size_t src0_row_size_aligned = uctx->src0_row_size_aligned; const size_t dst_row_size_aligned = uctx->dst_row_size_aligned; const uint32_t src0_nrows = uctx->src0_nrows; - const uint32_t src0_start_row = uctx->row_start + src0_nrows_per_thread * ith; - const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, uctx->row_start + src0_nrows); + const uint32_t src0_start_row = uctx->row_start + src0_nrows_per_hvx * ith; + const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_hvx, uctx->row_start + src0_nrows); if (src0_start_row >= src0_end_row) { return; @@ -1458,14 +1603,14 @@ static void unary_thread_tri_f32(unsigned int nth, unsigned int ith, void * data const dma_addr_t data_src = uctx->data_src0; const dma_addr_t data_dst = uctx->data_dst; - uint8_t * src0_vtcm_data = uctx->vtcm_src0 + (ith * uctx->vtcm_src0_size_per_thread); - uint8_t * dst_vtcm_data = uctx->vtcm_dst + (ith * uctx->vtcm_dst_size_per_thread); + uint8_t * src0_vtcm_data = uctx->vtcm_src0 + (ith * uctx->vtcm_src0_size_per_hvx); + uint8_t * dst_vtcm_data = uctx->vtcm_dst + (ith * uctx->vtcm_dst_size_per_hvx); const size_t src0_vtcm_half_size = uctx->src0_vtcm_half_size; const size_t dst_vtcm_half_size = uctx->dst_vtcm_half_size; const uint32_t BLOCK = uctx->block; - dma_queue * dma_q = octx->ctx->dma[ith]; + dma_queue * dma_q = th->dma_q; for (uint32_t ir = src0_start_row, vtcm_idx = 0; ir < src0_end_row && vtcm_idx < 2; vtcm_idx++) { const uint32_t block_size = MIN(BLOCK, src0_end_row - ir); @@ -1518,22 +1663,22 @@ static void unary_thread_tri_f32(unsigned int nth, unsigned int ith, void * data } // 4. Pointwise tiled unary task. -static void unary_thread_tiled(unsigned int nth, unsigned int ith, void * data) { +static void unary_thread_tiled(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { (void) nth; const struct htp_unary_context * uctx = (const struct htp_unary_context *) data; struct htp_ops_context * octx = uctx->octx; const struct htp_tensor * src = octx->src[0]; const struct htp_tensor * dst = octx->dst; - struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + struct htp_thread_trace * tr = th->trace; htp_unary_preamble; - const uint32_t src0_nrows_per_thread = uctx->src0_nrows_per_thread; + const uint32_t src0_nrows_per_hvx = uctx->src0_nrows_per_hvx; const uint32_t col_tile = uctx->col_tile; const uint32_t src0_nrows = uctx->src0_nrows; - const uint32_t src0_start_row = uctx->row_start + src0_nrows_per_thread * ith; - const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, uctx->row_start + src0_nrows); + const uint32_t src0_start_row = uctx->row_start + src0_nrows_per_hvx * ith; + const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_hvx, uctx->row_start + src0_nrows); if (src0_start_row >= src0_end_row) { return; @@ -1542,13 +1687,13 @@ static void unary_thread_tiled(unsigned int nth, unsigned int ith, void * data) const dma_addr_t data_src = uctx->data_src0; const dma_addr_t data_dst = uctx->data_dst; - uint8_t * src0_vtcm_data = uctx->vtcm_src0 + (ith * uctx->vtcm_src0_size_per_thread); - uint8_t * dst_vtcm_data = uctx->vtcm_dst + (ith * uctx->vtcm_dst_size_per_thread); + uint8_t * src0_vtcm_data = uctx->vtcm_src0 + (ith * uctx->vtcm_src0_size_per_hvx); + uint8_t * dst_vtcm_data = uctx->vtcm_dst + (ith * uctx->vtcm_dst_size_per_hvx); const size_t src0_half = uctx->src0_vtcm_half_size; const size_t dst_half = uctx->dst_vtcm_half_size; - dma_queue * dma_q = octx->ctx->dma[ith]; + dma_queue * dma_q = th->dma_q; const struct fastdiv_values * div_ne01 = &uctx->kparams->div_ne01; const struct fastdiv_values * div_ne02 = &uctx->kparams->div_ne02; @@ -1629,23 +1774,23 @@ static void unary_thread_tiled(unsigned int nth, unsigned int ith, void * data) } // 5. TRI tiled task. -static void unary_thread_tiled_tri_f32(unsigned int nth, unsigned int ith, void * data) { +static void unary_thread_tiled_tri_f32(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { (void) nth; const struct htp_unary_context * uctx = (const struct htp_unary_context *) data; struct htp_ops_context * octx = uctx->octx; const struct htp_tensor * src = octx->src[0]; const struct htp_tensor * dst = octx->dst; - struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + struct htp_thread_trace * tr = th->trace; htp_unary_preamble; - const uint32_t src0_nrows_per_thread = uctx->src0_nrows_per_thread; + const uint32_t src0_nrows_per_hvx = uctx->src0_nrows_per_hvx; const int32_t * op_params = octx->op_params; const uint32_t col_tile = uctx->col_tile; const uint32_t src0_nrows = uctx->src0_nrows; - const uint32_t src0_start_row = uctx->row_start + src0_nrows_per_thread * ith; - const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, uctx->row_start + src0_nrows); + const uint32_t src0_start_row = uctx->row_start + src0_nrows_per_hvx * ith; + const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_hvx, uctx->row_start + src0_nrows); if (src0_start_row >= src0_end_row) { return; @@ -1654,13 +1799,13 @@ static void unary_thread_tiled_tri_f32(unsigned int nth, unsigned int ith, void const dma_addr_t data_src = uctx->data_src0; const dma_addr_t data_dst = uctx->data_dst; - uint8_t * src0_vtcm_data = uctx->vtcm_src0 + (ith * uctx->vtcm_src0_size_per_thread); - uint8_t * dst_vtcm_data = uctx->vtcm_dst + (ith * uctx->vtcm_dst_size_per_thread); + uint8_t * src0_vtcm_data = uctx->vtcm_src0 + (ith * uctx->vtcm_src0_size_per_hvx); + uint8_t * dst_vtcm_data = uctx->vtcm_dst + (ith * uctx->vtcm_dst_size_per_hvx); const size_t src0_half = uctx->src0_vtcm_half_size; const size_t dst_half = uctx->dst_vtcm_half_size; - dma_queue * dma_q = octx->ctx->dma[ith]; + dma_queue * dma_q = th->dma_q; const struct fastdiv_values * div_ne01 = &uctx->kparams->div_ne01; const struct fastdiv_values * div_tpr = &uctx->kparams->div_tpr; @@ -1749,9 +1894,10 @@ static int execute_op_unary(struct htp_ops_context * octx) { const char * op_type = NULL; switch (octx->op) { - case HTP_OP_NORM: op_type = is_f16 ? "norm-f16" : "norm-f32"; break; - case HTP_OP_RMS_NORM: op_type = is_f16 ? "rmsnorm-f16" : "rmsnorm-f32"; break; - case HTP_OP_RMS_NORM_MUL: op_type = "rmsnorm-mul-f32"; break; + case HTP_OP_NORM: op_type = is_f16 ? "norm-f16" : "norm-f32"; break; + case HTP_OP_RMS_NORM: op_type = is_f16 ? "rmsnorm-f16" : "rmsnorm-f32"; break; + case HTP_OP_RMS_NORM_MUL: op_type = "rmsnorm-mul-f32"; break; + case HTP_OP_RMS_NORM_MUL_ADD: op_type = "rmsnorm-mul-add-f32"; break; case HTP_OP_SCALE: op_type = is_f16 ? "scale-f16" : "scale-f32"; break; case HTP_OP_CLAMP: op_type = is_f16 ? "clamp-f16" : "clamp-f32"; break; case HTP_OP_LEAKY_RELU: op_type = "leaky-relu-f32"; break; @@ -1782,7 +1928,7 @@ static int execute_op_unary(struct htp_ops_context * octx) { const struct htp_unary_kernel_params * kparams = (const struct htp_unary_kernel_params *) octx->kernel_params; - if (!htp_ops_context_set_n_threads(octx, kparams->n_threads)) { + if (!htp_ops_context_set_n_hvx(octx, kparams->n_hvx)) { return HTP_STATUS_INVAL_PARAMS; } @@ -1807,7 +1953,7 @@ static int execute_op_unary(struct htp_ops_context * octx) { return HTP_STATUS_OK; } - const uint32_t n_threads = octx->n_threads; + const uint32_t n_hvx = octx->n_hvx; // Always 0 for F16 - htp_unary_solve_layout() keeps F16 on the row-block path, // since only F32 has unary_task_f32_tiled_* kernels. @@ -1817,9 +1963,12 @@ static int execute_op_unary(struct htp_ops_context * octx) { bool broadcast_weight = kparams->broadcast_weight; const struct htp_tensor * src1 = NULL; - // RMS_NORM_MUL fusion is F32-only (its weight tensor is always F32; see + const bool has_weight = (octx->op == HTP_OP_RMS_NORM_MUL || octx->op == HTP_OP_RMS_NORM_MUL_ADD); + const bool has_src2 = (octx->op == HTP_OP_RMS_NORM_MUL_ADD); + + // RMS_NORM_MUL* fusion is F32-only (its weight tensor is always F32; see // try_fuse_node()'s type guard), so this never triggers when is_f16 is true. - if (octx->op == HTP_OP_RMS_NORM_MUL) { + if (has_weight) { src1 = octx->src[1]; src1_data_row_size = src1->ne[0] * sizeof(float); } @@ -1835,22 +1984,23 @@ static int execute_op_unary(struct htp_ops_context * octx) { struct htp_unary_vtcm_layout L; htp_unary_vtcm_layout_build(&L, octx->op, src0_data_row_size, dst_data_row_size, src1_data_row_size, - src0_elem_size, col_tile, kparams->block, broadcast_weight, kparams->n_threads); + src0_elem_size, col_tile, kparams->block, broadcast_weight, kparams->n_hvx); FARF(HIGH, "%s: (%ux%ux%ux%u) -> (%ux%ux%ux%u) : src0-vtcm-size %zu src1-vtcm-size %zu dst-vtcm-size %zu\n", op_type, src0->ne[0], src0->ne[1], src0->ne[2], src0->ne[3], dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3], - L.src0_bytes * n_threads, L.src1_bytes * n_threads, L.dst_bytes * n_threads); + L.src0_bytes * n_hvx, L.src1_bytes * n_hvx, L.dst_bytes * n_hvx); uint8_t * const base = (uint8_t *) octx->ctx->vtcm_base; struct htp_unary_context uctx = { .octx = octx, .kparams = kparams, - .src0_nrows_per_thread = fastdiv(nrows + n_threads - 1, &octx->n_threads_div), + .src0_nrows_per_hvx = fastdiv(nrows + n_hvx - 1, &octx->n_hvx_div), .src0_nrows = nrows, .row_start = row_start, .data_src0 = src0->data, - .data_src1 = (octx->op == HTP_OP_RMS_NORM_MUL) ? src1->data : 0, + .data_src1 = has_weight ? src1->data : 0, + .data_src2 = has_src2 ? octx->src[2]->data : 0, .data_dst = dst->data, .src0_data_row_size = src0_data_row_size, @@ -1862,7 +2012,7 @@ static int execute_op_unary(struct htp_ops_context * octx) { .dst_row_size_aligned = L.dst_row_size_aligned, .src0_vtcm_half_size = L.src0_bytes >> 1, - .src1_vtcm_half_size = (octx->op == HTP_OP_RMS_NORM_MUL) ? (L.src1_bytes >> (broadcast_weight ? 0 : 1)) : 0, + .src1_vtcm_half_size = has_weight ? (L.src1_bytes >> (broadcast_weight ? 0 : 1)) : 0, .dst_vtcm_half_size = L.dst_bytes >> 1, .block = kparams->block, @@ -1872,16 +2022,18 @@ static int execute_op_unary(struct htp_ops_context * octx) { .vtcm_src0 = VTCM_LAYOUT_PTR(uint8_t, base, L.off_src0), .vtcm_src1 = VTCM_LAYOUT_PTR_OPTIONAL(uint8_t, base, L.off_src1, L.src1_bytes > 0), + .vtcm_src2 = VTCM_LAYOUT_PTR_OPTIONAL(uint8_t, base, L.off_src2, L.src2_bytes > 0), .vtcm_dst = VTCM_LAYOUT_PTR(uint8_t, base, L.off_dst), - .vtcm_src0_size_per_thread = L.src0_bytes, - .vtcm_src1_size_per_thread = L.src1_bytes, - .vtcm_dst_size_per_thread = L.dst_bytes, + .vtcm_src0_size_per_hvx = L.src0_bytes, + .vtcm_src1_size_per_hvx = L.src1_bytes, + .vtcm_src2_size_per_hvx = L.src2_bytes, + .vtcm_dst_size_per_hvx = L.dst_bytes, }; FARF(HIGH, "%s: %s mode (col_tile %u)\n", op_type, col_tile ? "tiled" : "row-block", col_tile); - worker_callback_t task_func = NULL; + work_queue_func_t task_func = NULL; void * compute_func = NULL; if (col_tile) { @@ -1940,6 +2092,10 @@ static int execute_op_unary(struct htp_ops_context * octx) { task_func = unary_thread_rms_norm_mul_f32; compute_func = (void *) rms_norm_mul_f32; break; + case HTP_OP_RMS_NORM_MUL_ADD: + task_func = unary_thread_rms_norm_mul_add_f32; + compute_func = (void *) rms_norm_mul_add_f32; + break; case HTP_OP_SCALE: compute_func = (void *) scale_f32; break; case HTP_OP_CLAMP: compute_func = (void *) clamp_f32; break; case HTP_OP_LEAKY_RELU: compute_func = (void *) leaky_relu_f32; break; @@ -1976,7 +2132,7 @@ static int execute_op_unary(struct htp_ops_context * octx) { } uctx.compute = compute_func; - work_queue_run(octx->ctx->work_queue, task_func, &uctx, n_threads); + work_queue_run(octx->ctx->hvx_queue, task_func, &uctx, n_hvx); return err; } diff --git a/ggml/src/ggml-hexagon/htp/unary-ops.h b/ggml/src/ggml-hexagon/htp/unary-ops.h index 713e40d50b75..e03b2ac73fac 100644 --- a/ggml/src/ggml-hexagon/htp/unary-ops.h +++ b/ggml/src/ggml-hexagon/htp/unary-ops.h @@ -8,7 +8,7 @@ // Op-specific struct for precomputed unary params struct htp_unary_kernel_params { - uint32_t n_threads; + uint32_t n_hvx; uint32_t col_tile; uint32_t block; uint32_t broadcast_weight; @@ -39,6 +39,7 @@ static inline bool htp_op_is_unary(uint32_t opcode) { case HTP_OP_NORM: case HTP_OP_RMS_NORM: case HTP_OP_RMS_NORM_MUL: + case HTP_OP_RMS_NORM_MUL_ADD: case HTP_OP_SCALE: case HTP_OP_SQR: case HTP_OP_SQRT: @@ -74,6 +75,7 @@ static inline bool htp_unary_is_elementwise(uint32_t opcode) { case HTP_OP_NORM: case HTP_OP_RMS_NORM: case HTP_OP_RMS_NORM_MUL: + case HTP_OP_RMS_NORM_MUL_ADD: case HTP_OP_L2_NORM: case HTP_OP_HADAMARD: case HTP_OP_TRI: @@ -87,6 +89,7 @@ struct htp_unary_vtcm_layout { size_t total_bytes; size_t off_src0; size_t off_src1; + size_t off_src2; size_t off_dst; size_t src0_row_size_aligned; @@ -95,6 +98,7 @@ struct htp_unary_vtcm_layout { size_t src0_bytes; size_t src1_bytes; + size_t src2_bytes; size_t dst_bytes; }; @@ -108,21 +112,25 @@ static inline void htp_unary_vtcm_layout_build( uint32_t col_tile, uint32_t block, bool broadcast_weight, - uint32_t n_threads + uint32_t n_hvx ) { + const bool has_weight = (op == HTP_OP_RMS_NORM_MUL || op == HTP_OP_RMS_NORM_MUL_ADD); + L->src0_row_size_aligned = hex_round_up(src0_data_row_size, 128); L->dst_row_size_aligned = hex_round_up(dst_data_row_size, 128); - L->src1_row_size_aligned = (op == HTP_OP_RMS_NORM_MUL) ? hex_round_up(src1_data_row_size, 128) : 0; + L->src1_row_size_aligned = has_weight ? hex_round_up(src1_data_row_size, 128) : 0; if (col_tile > 0) { const size_t col_tile_bytes = col_tile * elem_size; L->src0_bytes = col_tile_bytes * 2; L->dst_bytes = col_tile_bytes * 2; L->src1_bytes = 0; + L->src2_bytes = 0; } else { L->src0_bytes = L->src0_row_size_aligned * block * 2; + L->src2_bytes = (op == HTP_OP_RMS_NORM_MUL_ADD) ? L->src0_bytes : 0; L->dst_bytes = L->dst_row_size_aligned * block * 2; - if (op == HTP_OP_RMS_NORM_MUL) { + if (has_weight) { L->src1_bytes = broadcast_weight ? L->src1_row_size_aligned : (L->src1_row_size_aligned * block * 2); } else { L->src1_bytes = 0; @@ -130,9 +138,10 @@ static inline void htp_unary_vtcm_layout_build( } size_t off = 0; - VTCM_LAYOUT_ALLOC(off, off_src0, L->src0_bytes * n_threads); - VTCM_LAYOUT_ALLOC_OPTIONAL(off, off_src1, L->src1_bytes * n_threads, L->src1_bytes > 0); - VTCM_LAYOUT_ALLOC(off, off_dst, L->dst_bytes * n_threads); + VTCM_LAYOUT_ALLOC(off, off_src0, L->src0_bytes * n_hvx); + VTCM_LAYOUT_ALLOC_OPTIONAL(off, off_src1, L->src1_bytes * n_hvx, L->src1_bytes > 0); + VTCM_LAYOUT_ALLOC_OPTIONAL(off, off_src2, L->src2_bytes * n_hvx, L->src2_bytes > 0); + VTCM_LAYOUT_ALLOC(off, off_dst, L->dst_bytes * n_hvx); L->total_bytes = off; } @@ -144,72 +153,79 @@ static inline void htp_unary_solve_layout( uint32_t ne11, uint32_t max_block, bool broadcast_weight, - uint32_t n_threads, + uint32_t n_hvx, size_t vtcm_size, size_t elem_size, uint32_t * out_col_tile, - uint32_t * out_vtcm_row_per_thread + uint32_t * out_vtcm_row_per_hvx ) { + const bool has_weight = (op == HTP_OP_RMS_NORM_MUL || op == HTP_OP_RMS_NORM_MUL_ADD); + const size_t dst_elem_size = (op == HTP_OP_HADAMARD) ? sizeof(float) : elem_size; const size_t src0_data_row_size = ne00 * elem_size; const size_t dst_data_row_size = ne10 * dst_elem_size; - const size_t src1_data_row_size = (op == HTP_OP_RMS_NORM_MUL) ? (ne11 * sizeof(float)) : 0; + const size_t src1_data_row_size = has_weight ? (ne11 * sizeof(float)) : 0; const size_t src0_row_size_aligned = hex_round_up(src0_data_row_size, 128); const size_t dst_row_size_aligned = hex_round_up(dst_data_row_size, 128); - const size_t src1_row_size_aligned = (op == HTP_OP_RMS_NORM_MUL) ? hex_round_up(src1_data_row_size, 128) : 0; + const size_t src1_row_size_aligned = has_weight ? hex_round_up(src1_data_row_size, 128) : 0; size_t vtcm_size_per_row = 0; - size_t vtcm_row_per_thread = 0; - - if (op == HTP_OP_RMS_NORM_MUL) { + size_t vtcm_row_per_hvx = 0; + + if (op == HTP_OP_RMS_NORM_MUL_ADD) { + size_t src1_vtcm_total = n_hvx * src1_row_size_aligned; + size_t available_vtcm = (vtcm_size > src1_vtcm_total) ? (vtcm_size - src1_vtcm_total) : 0; + vtcm_size_per_row = 2 * (2 * src0_row_size_aligned + dst_row_size_aligned); + vtcm_row_per_hvx = available_vtcm / (n_hvx * vtcm_size_per_row); + } else if (op == HTP_OP_RMS_NORM_MUL) { if (broadcast_weight) { size_t available_vtcm = vtcm_size; - size_t src1_vtcm_total = n_threads * src1_row_size_aligned; + size_t src1_vtcm_total = n_hvx * src1_row_size_aligned; if (available_vtcm > src1_vtcm_total) { available_vtcm -= src1_vtcm_total; } else { available_vtcm = 0; } vtcm_size_per_row = 2 * (src0_row_size_aligned + dst_row_size_aligned); - vtcm_row_per_thread = available_vtcm / (n_threads * vtcm_size_per_row); + vtcm_row_per_hvx = available_vtcm / (n_hvx * vtcm_size_per_row); } else { vtcm_size_per_row = 2 * (src0_row_size_aligned + dst_row_size_aligned + src1_row_size_aligned); - vtcm_row_per_thread = vtcm_size / (n_threads * vtcm_size_per_row); + vtcm_row_per_hvx = vtcm_size / (n_hvx * vtcm_size_per_row); } } else { vtcm_size_per_row = 2 * (src0_row_size_aligned + dst_row_size_aligned); - vtcm_row_per_thread = vtcm_size / (n_threads * vtcm_size_per_row); + vtcm_row_per_hvx = vtcm_size / (n_hvx * vtcm_size_per_row); } - if (max_block > 0 && vtcm_row_per_thread > max_block) { - vtcm_row_per_thread = max_block; + if (max_block > 0 && vtcm_row_per_hvx > max_block) { + vtcm_row_per_hvx = max_block; } const bool is_reduction = (op == HTP_OP_NORM || op == HTP_OP_RMS_NORM || - op == HTP_OP_RMS_NORM_MUL || op == HTP_OP_L2_NORM || - op == HTP_OP_HADAMARD); + op == HTP_OP_RMS_NORM_MUL || op == HTP_OP_RMS_NORM_MUL_ADD || + op == HTP_OP_L2_NORM || op == HTP_OP_HADAMARD); // The tiled fallback path below only has F32 task functions (unary_task_f32_tiled_*); // F16 has no tiled kernels, so it must stay on the row-block path like reduction ops. - // NOTE: if F16 ends up with vtcm_row_per_thread == 0 here (row too large for the VTCM + // NOTE: if F16 ends up with vtcm_row_per_hvx == 0 here (row too large for the VTCM // budget), execute_op_unary() will see BLOCK == 0 and skip computation for that op // (logged via FARF(ERROR, ...)) since there is no F16 tiled fallback. This is a known // limitation; supporting it would require adding F16 tiled kernels. const bool is_f16 = (elem_size == sizeof(_Float16)); uint32_t col_tile = 0; - if (vtcm_row_per_thread == 0 && !is_reduction && !is_f16) { - const size_t per_thread_budget = vtcm_size / n_threads; - const size_t col_tile_bytes = hex_align_down(per_thread_budget / 4, 128); + if (vtcm_row_per_hvx == 0 && !is_reduction && !is_f16) { + const size_t per_hvx_budget = vtcm_size / n_hvx; + const size_t col_tile_bytes = hex_align_down(per_hvx_budget / 4, 128); col_tile = (uint32_t) (col_tile_bytes / elem_size); } - const uint32_t block = col_tile ? 0 : (uint32_t) vtcm_row_per_thread; + const uint32_t block = col_tile ? 0 : (uint32_t) vtcm_row_per_hvx; htp_unary_vtcm_layout_build(L, op, src0_data_row_size, dst_data_row_size, src1_data_row_size, - elem_size, col_tile, block, broadcast_weight, n_threads); + elem_size, col_tile, block, broadcast_weight, n_hvx); *out_col_tile = col_tile; - *out_vtcm_row_per_thread = (uint32_t) vtcm_row_per_thread; + *out_vtcm_row_per_hvx = (uint32_t) vtcm_row_per_hvx; } #endif /* HTP_UNARY_OPS_H */ diff --git a/ggml/src/ggml-hexagon/htp/upscale-ops.c b/ggml/src/ggml-hexagon/htp/upscale-ops.c index 62070cf5d7cd..59ab7914ad62 100644 --- a/ggml/src/ggml-hexagon/htp/upscale-ops.c +++ b/ggml/src/ggml-hexagon/htp/upscale-ops.c @@ -6,7 +6,7 @@ #include "hex-common.h" #include "dma-queue.h" -#include "hex-profile.h" +#include "htp-trace.h" #include "htp-ctx.h" #include "htp-ops.h" #include "htp-tensor.h" @@ -306,7 +306,7 @@ struct upscale_context { const void * lut_y; uint32_t chunk_start; uint32_t chunk_count; - uint32_t chunks_per_thread; + uint32_t chunks_per_hvx; }; static void compute_nearest_chunk( @@ -756,31 +756,31 @@ static inline void dma_push_src_chunk( } } -static void upscale_worker(unsigned int nth, unsigned int ith, void * data) { +static void upscale_worker(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { struct upscale_context * ctx = (struct upscale_context *) data; const struct htp_upscale_kernel_params * p = ctx->kparams; const struct htp_upscale_vtcm_layout * L = &ctx->layout; const struct htp_tensor * src0 = ctx->octx->src[0]; const struct htp_tensor * dst = ctx->octx->dst; - const uint32_t first = ctx->chunk_start + ith * ctx->chunks_per_thread; - const uint32_t last = MIN(first + ctx->chunks_per_thread, ctx->chunk_start + ctx->chunk_count); + const uint32_t first = ctx->chunk_start + ith * ctx->chunks_per_hvx; + const uint32_t last = MIN(first + ctx->chunks_per_hvx, ctx->chunk_start + ctx->chunk_count); if (first >= last) { return; } - struct htp_thread_trace * tr = &ctx->octx->ctx->trace[ith]; - dma_queue * dma_q = ctx->octx->ctx->dma[ith]; + struct htp_thread_trace * tr = th->trace; + dma_queue * dma_q = th->dma_q; const uint32_t src_spad_half = (uint32_t) L->src_spad_half_size; const uint32_t dst_spad_half = (uint32_t) L->dst_spad_half_size; uint8_t * vtcm_base = (uint8_t *) ctx->octx->ctx->vtcm_base; - uint8_t * src_spad = VTCM_LAYOUT_PTR(uint8_t, vtcm_base, L->off_src) + ith * L->src_bytes_per_thread; - uint8_t * dst_spad = VTCM_LAYOUT_PTR(uint8_t, vtcm_base, L->off_dst) + ith * L->dst_bytes_per_thread; - float * R_buf = VTCM_LAYOUT_PTR(float, vtcm_base, L->off_r) + ith * (L->r_bytes_per_thread / sizeof(float)); - float * gather_tmp = VTCM_LAYOUT_PTR(float, vtcm_base, L->off_gather) + ith * (L->gather_bytes_per_thread / sizeof(float)); + uint8_t * src_spad = VTCM_LAYOUT_PTR(uint8_t, vtcm_base, L->off_src) + ith * L->src_bytes_per_hvx; + uint8_t * dst_spad = VTCM_LAYOUT_PTR(uint8_t, vtcm_base, L->off_dst) + ith * L->dst_bytes_per_hvx; + float * R_buf = VTCM_LAYOUT_PTR(float, vtcm_base, L->off_r) + ith * (L->r_bytes_per_hvx / sizeof(float)); + float * gather_tmp = VTCM_LAYOUT_PTR(float, vtcm_base, L->off_gather) + ith * (L->gather_bytes_per_hvx / sizeof(float)); const void * lut_y = ctx->lut_y; float * srcb2[2] = { (float *) src_spad, (float *) (src_spad + src_spad_half) }; @@ -866,12 +866,12 @@ int op_upscale(struct htp_ops_context * octx) { return HTP_STATUS_OK; } - const uint32_t n_threads = MIN(p->n_threads, chunk_count); - if (!htp_ops_context_set_n_threads(octx, n_threads)) { + const uint32_t n_hvx = MIN(p->n_hvx, chunk_count); + if (!htp_ops_context_set_n_hvx(octx, n_hvx)) { return HTP_STATUS_INVAL_PARAMS; } - const uint32_t chunks_per_thread = fastdiv(chunk_count + n_threads - 1, &octx->n_threads_div); + const uint32_t chunks_per_hvx = fastdiv(chunk_count + n_hvx - 1, &octx->n_hvx_div); size_t lut_x_size = 0; size_t elem_y_sz = sizeof(uint32_t); @@ -946,13 +946,13 @@ int op_upscale(struct htp_ops_context * octx) { .lut_y = lut_y, .chunk_start = chunk_start, .chunk_count = chunk_count, - .chunks_per_thread = chunks_per_thread, + .chunks_per_hvx = chunks_per_hvx, }; htp_upscale_vtcm_layout_build(&ctx.layout, src_x, src_y, dst_x, dst_y, - p->dst_nrows_per_chunk, p->inv_sf1, p->filter_radius1, n_threads); + p->dst_nrows_per_chunk, p->inv_sf1, p->filter_radius1, n_hvx); - work_queue_run(octx->ctx->work_queue, upscale_worker, &ctx, n_threads); + work_queue_run(octx->ctx->hvx_queue, upscale_worker, &ctx, n_hvx); return HTP_STATUS_OK; } diff --git a/ggml/src/ggml-hexagon/htp/upscale-ops.h b/ggml/src/ggml-hexagon/htp/upscale-ops.h index aff7854b50f3..592bf7f000a8 100644 --- a/ggml/src/ggml-hexagon/htp/upscale-ops.h +++ b/ggml/src/ggml-hexagon/htp/upscale-ops.h @@ -43,7 +43,7 @@ struct htp_upscale_kernel_params { float invscale0; float invscale1; - uint32_t n_threads; + uint32_t n_hvx; uint32_t chunks_per_group; uint32_t dst_nrows_per_chunk; @@ -103,10 +103,10 @@ struct htp_upscale_vtcm_layout { size_t off_dst; size_t off_r; size_t off_gather; - size_t src_bytes_per_thread; - size_t dst_bytes_per_thread; - size_t r_bytes_per_thread; - size_t gather_bytes_per_thread; + size_t src_bytes_per_hvx; + size_t dst_bytes_per_hvx; + size_t r_bytes_per_hvx; + size_t gather_bytes_per_hvx; size_t src_spad_half_size; size_t dst_spad_half_size; uint32_t max_src_chunk_rows; @@ -122,7 +122,7 @@ static inline void htp_upscale_vtcm_layout_build( uint32_t dst_chunk_rows, float inv_sf1, float filter_radius1, - uint32_t n_threads + uint32_t n_hvx ) { uint32_t max_src_rows; if (dst_chunk_rows >= dst_y) { @@ -142,16 +142,16 @@ static inline void htp_upscale_vtcm_layout_build( L->dst_chunk_rows = dst_chunk_rows; L->src_spad_half_size = hex_round_up(src_chunk_bytes + 256, 128); L->dst_spad_half_size = hex_round_up(dst_chunk_bytes, 128); - L->src_bytes_per_thread = 2 * L->src_spad_half_size; - L->dst_bytes_per_thread = 2 * L->dst_spad_half_size; - L->r_bytes_per_thread = hex_round_up(src_x * sizeof(float), 128); - L->gather_bytes_per_thread = 256; + L->src_bytes_per_hvx = 2 * L->src_spad_half_size; + L->dst_bytes_per_hvx = 2 * L->dst_spad_half_size; + L->r_bytes_per_hvx = hex_round_up(src_x * sizeof(float), 128); + L->gather_bytes_per_hvx = 256; size_t off = 0; - VTCM_LAYOUT_ALLOC(off, off_src, L->src_bytes_per_thread * n_threads); - VTCM_LAYOUT_ALLOC(off, off_dst, L->dst_bytes_per_thread * n_threads); - VTCM_LAYOUT_ALLOC(off, off_r, L->r_bytes_per_thread * n_threads); - VTCM_LAYOUT_ALLOC(off, off_gather, L->gather_bytes_per_thread * n_threads); + VTCM_LAYOUT_ALLOC(off, off_src, L->src_bytes_per_hvx * n_hvx); + VTCM_LAYOUT_ALLOC(off, off_dst, L->dst_bytes_per_hvx * n_hvx); + VTCM_LAYOUT_ALLOC(off, off_r, L->r_bytes_per_hvx * n_hvx); + VTCM_LAYOUT_ALLOC(off, off_gather, L->gather_bytes_per_hvx * n_hvx); L->total_bytes = off; } @@ -163,13 +163,13 @@ static inline bool htp_upscale_solve_layout( uint32_t dst_y, float inv_sf1, float filter_radius1, - uint32_t n_threads, + uint32_t n_hvx, size_t vtcm_budget, uint32_t * out_dst_chunk_rows ) { // 1. Try full plane uint32_t chunk_rows = dst_y; - htp_upscale_vtcm_layout_build(layout, src_x, src_y, dst_x, dst_y, chunk_rows, inv_sf1, filter_radius1, n_threads); + htp_upscale_vtcm_layout_build(layout, src_x, src_y, dst_x, dst_y, chunk_rows, inv_sf1, filter_radius1, n_hvx); if (layout->total_bytes <= vtcm_budget) { *out_dst_chunk_rows = chunk_rows; return true; @@ -179,7 +179,7 @@ static inline bool htp_upscale_solve_layout( chunk_rows = 64; while (chunk_rows > 1) { if (chunk_rows < dst_y) { - htp_upscale_vtcm_layout_build(layout, src_x, src_y, dst_x, dst_y, chunk_rows, inv_sf1, filter_radius1, n_threads); + htp_upscale_vtcm_layout_build(layout, src_x, src_y, dst_x, dst_y, chunk_rows, inv_sf1, filter_radius1, n_hvx); if (layout->total_bytes <= vtcm_budget) { *out_dst_chunk_rows = chunk_rows; return true; @@ -190,7 +190,7 @@ static inline bool htp_upscale_solve_layout( // 3. Fallback to 1 row chunk_rows = 1; - htp_upscale_vtcm_layout_build(layout, src_x, src_y, dst_x, dst_y, chunk_rows, inv_sf1, filter_radius1, n_threads); + htp_upscale_vtcm_layout_build(layout, src_x, src_y, dst_x, dst_y, chunk_rows, inv_sf1, filter_radius1, n_hvx); if (layout->total_bytes <= vtcm_budget) { *out_dst_chunk_rows = chunk_rows; return true; diff --git a/ggml/src/ggml-hexagon/htp/work-queue.c b/ggml/src/ggml-hexagon/htp/work-queue.c index 6c74445f756b..5d274274a5e7 100644 --- a/ggml/src/ggml-hexagon/htp/work-queue.c +++ b/ggml/src/ggml-hexagon/htp/work-queue.c @@ -16,8 +16,8 @@ // internal structure kept in thread-local storage per instance of work queue typedef struct { - work_queue_t queue; - unsigned int id; + htp_thread_t th; + work_queue_t queue; } worker_context_t; struct work_queue_task_s { @@ -35,15 +35,15 @@ struct work_queue_s { uint32_t idx_mask; uint32_t capacity; - qurt_thread_t thread[WORK_QUEUE_MAX_N_THREADS]; // thread ID's of the workers - worker_context_t context[WORK_QUEUE_MAX_N_THREADS]; // worker contexts - void * stack[WORK_QUEUE_MAX_N_THREADS]; // thread stack pointers - unsigned int n_threads; // total threads (workers + main) - unsigned int n_workers; // number of active threads (just workers) + qurt_thread_t thread[HTP_MAX_NTHREADS]; // thread ID's of the workers + worker_context_t context[HTP_MAX_NTHREADS]; // worker contexts (0 = main, 1..n_workers = workers) + void * stack[HTP_MAX_NTHREADS]; // thread stack pointers + unsigned int n_threads; // total threads (workers + main) + unsigned int n_workers; // number of active threads (just workers) - atomic_uint active; // workers are polling/active - atomic_bool killed; // threads need to exit - bool external_mem; // memory owned externally + atomic_uint active; // workers are polling/active + atomic_bool killed; // threads need to exit + bool external_mem; // memory owned externally struct work_queue_task_s queue[] __attribute__((aligned(HEX_L2_LINE_SIZE))); }; @@ -51,8 +51,9 @@ struct work_queue_s { static void work_queue_thread(void * context) { worker_context_t * me = (worker_context_t *) context; work_queue_t q = me->queue; + unsigned int id = (unsigned int) (me - q->context); - FARF(HIGH, "work-queue: thread %u started", me->id); + FARF(HIGH, "work-queue: thread %u (tid %u) started", id, me->th.tid); unsigned int prev_seqn = 0; @@ -62,6 +63,7 @@ static void work_queue_thread(void * context) { if (atomic_load_explicit(&q->active, memory_order_relaxed)) { hex_pause(); } else { + qurt_hvx_unlock(); qurt_futex_wait(&q->active, 0); } continue; @@ -77,9 +79,8 @@ static void work_queue_thread(void * context) { struct work_queue_task_s * task = &q->queue[ir]; unsigned int n = task->n_threads; - unsigned int i = me->id; - if (i < n) { - task->func(n, i, task->data); + if (id < n) { + task->func(&me->th, n, id, task->data); atomic_fetch_sub_explicit(&task->barrier, 1, memory_order_release); } else { @@ -92,10 +93,15 @@ static void work_queue_thread(void * context) { } } - FARF(HIGH, "work-queue: thread %u stopped", me->id); + FARF(HIGH, "work-queue: thread %u (tid %u) stopped", id, me->th.tid); } -bool work_queue_run_async(work_queue_t q, work_queue_func_t func, void * data, unsigned int n) { +bool work_queue_run(work_queue_t q, work_queue_func_t func, void * data, unsigned int n) { + if (n <= 1) { + func(&q->context[0].th, n, 0, data); + return true; + } + if (n > q->n_threads) { FARF(ERROR, "work-queue: invalid number of jobs %u for n-threads %u", n, q->n_threads); return false; @@ -121,7 +127,7 @@ bool work_queue_run_async(work_queue_t q, work_queue_func_t func, void * data, u atomic_fetch_add_explicit(&q->seqn, 1, memory_order_release); // main thread runs job #0 - func(n, 0, data); + func(&q->context[0].th, n, 0, data); atomic_fetch_sub_explicit(&task->barrier, 1, memory_order_release); @@ -148,7 +154,7 @@ size_t work_queue_alignof(void) { return 4096; } -work_queue_t work_queue_init(void * ptr, uint32_t n_threads, uint32_t capacity, uint32_t stack_size) { +work_queue_t work_queue_init(void * ptr, uint32_t n_threads, uint32_t capacity, uint32_t stack_size, dma_queue_t * dma, struct htp_thread_trace * trace) { capacity = hex_ceil_pow2(capacity); uint32_t n_workers = n_threads > 1 ? n_threads - 1 : 0; unsigned char * mem_blob = (unsigned char *) ptr; @@ -161,11 +167,14 @@ work_queue_t work_queue_init(void * ptr, uint32_t n_threads, uint32_t capacity, q->external_mem = true; q->capacity = capacity; + for (unsigned int i = 0; i < n_threads; i++) { + htp_thread_init(&q->context[i].th, dma, trace); + q->context[i].queue = q; + } + for (unsigned int i = 0; i < n_workers; i++) { q->stack[i] = mem_blob; mem_blob += stack_size; q->thread[i] = 0; - q->context[i].id = i + 1; - q->context[i].queue = q; } atomic_init(&q->idx_read, 0); @@ -190,7 +199,7 @@ work_queue_t work_queue_init(void * ptr, uint32_t n_threads, uint32_t capacity, qurt_thread_attr_set_stack_size(&attr, stack_size); char thread_name[32]; - snprintf(thread_name, sizeof(thread_name), "work-queue:%u", i); + snprintf(thread_name, sizeof(thread_name), "work-queue:%u", i + 1); qurt_thread_attr_set_name(&attr, thread_name); // set up priority - by default, match the creating thread's prio @@ -204,7 +213,7 @@ work_queue_t work_queue_init(void * ptr, uint32_t n_threads, uint32_t capacity, qurt_thread_attr_set_priority(&attr, prio); - int err = qurt_thread_create(&q->thread[i], &attr, work_queue_thread, (void *) &q->context[i]); + int err = qurt_thread_create(&q->thread[i], &attr, work_queue_thread, (void *) &q->context[i + 1]); if (err) { FARF(ERROR, "Could not launch worker threads!"); work_queue_free(q); diff --git a/ggml/src/ggml-hexagon/htp/work-queue.h b/ggml/src/ggml-hexagon/htp/work-queue.h index 09ca4b1f4392..446b72c46026 100644 --- a/ggml/src/ggml-hexagon/htp/work-queue.h +++ b/ggml/src/ggml-hexagon/htp/work-queue.h @@ -5,34 +5,21 @@ #include #include -typedef void (*work_queue_func_t)(unsigned int n, unsigned int i, void *); +#include "htp-thread.h" + +typedef void (*work_queue_func_t)(const htp_thread_t * th, unsigned int n, unsigned int i, void *); struct work_queue_s; typedef struct work_queue_s * work_queue_t; -#define WORK_QUEUE_MAX_N_THREADS 10 - size_t work_queue_sizeof(uint32_t n_threads, uint32_t capacity, uint32_t stack_size); size_t work_queue_alignof(void); -work_queue_t work_queue_init(void * ptr, uint32_t n_threads, uint32_t capacity, uint32_t stack_size); +work_queue_t work_queue_init(void * ptr, uint32_t n_threads, uint32_t capacity, uint32_t stack_size, dma_queue_t * dma, struct htp_thread_trace * trace); void work_queue_free(work_queue_t q); void work_queue_wakeup(work_queue_t q); void work_queue_suspend(work_queue_t q); -bool work_queue_run_async(work_queue_t q, work_queue_func_t func, void * data, unsigned int n); - -static inline bool work_queue_run(work_queue_t q, work_queue_func_t func, void * data, unsigned int n) { - if (n <= 1) { - func(n, 0, data); - return true; - } - return work_queue_run_async(q, func, data, n); -} - -// Legacy compatibility -typedef work_queue_func_t worker_callback_t; -#define worker_pool_run_func work_queue_run -#define worker_pool work_queue +bool work_queue_run(work_queue_t q, work_queue_func_t func, void * data, unsigned int n); #endif // #ifndef HTP_WORK_QUEUE_H diff --git a/ggml/src/ggml-impl.h b/ggml/src/ggml-impl.h index 4dc55e78e218..7c6a605afe61 100644 --- a/ggml/src/ggml-impl.h +++ b/ggml/src/ggml-impl.h @@ -107,6 +107,13 @@ static inline bool ggml_impl_is_view(const struct ggml_tensor * t) { static inline float ggml_compute_softplus_f32(float input) { return (input > 20.0f) ? input : logf(1 + expf(input)); } + +// compute correction dims for YaRN RoPE scaling +GGML_API void ggml_rope_yarn_corr_dims( + int n_dims, int n_ctx_orig, float freq_base, float beta_fast, float beta_slow, bool truncate, float dims[2]); + +GGML_API bool ggml_rope_get_truncate(const struct ggml_tensor * a); + // // logging // diff --git a/ggml/src/ggml-metal/ggml-metal-common.cpp b/ggml/src/ggml-metal/ggml-metal-common.cpp index 70fd551e34b2..f363122540cf 100644 --- a/ggml/src/ggml-metal/ggml-metal-common.cpp +++ b/ggml/src/ggml-metal/ggml-metal-common.cpp @@ -38,7 +38,7 @@ bool ggml_metal_op_mul_mat_use_mm(const struct ggml_tensor * op, bool has_simdgr return !ggml_is_transposed(op->src[0]) && !ggml_is_transposed(op->src[1]) && - has_simdgroup_mm && ne00 >= 64 && ne11 > 8; + has_simdgroup_mm && ne11 > 8 && ne00 >= 32; } bool ggml_metal_op_mul_mat_id_use_mm(const struct ggml_tensor * op, bool has_simdgroup_mm) { diff --git a/ggml/src/ggml-metal/ggml-metal-context.m b/ggml/src/ggml-metal/ggml-metal-context.m index 597c126ca3f7..e12c55a3909c 100644 --- a/ggml/src/ggml-metal/ggml-metal-context.m +++ b/ggml/src/ggml-metal/ggml-metal-context.m @@ -53,9 +53,9 @@ // command buffer state int n_cb; // number of extra threads used to submit the command buffers - int n_nodes_0; // number of nodes submitted by the main thread - int n_nodes_1; // remaining number of nodes submitted by the n_cb threads - int n_nodes_per_cb; + + // first node of each extra command buffer, then the end of the graph + int cb_start[GGML_METAL_MAX_COMMAND_BUFFERS + 1]; struct ggml_cgraph * gf; @@ -491,7 +491,7 @@ enum ggml_status ggml_metal_graph_compute(ggml_metal_t ctx, struct ggml_cgraph * ggml_metal_device_rsets_keep_alive(ctx->dev); // submit the ggml compute graph to the GPU by creating command buffers and encoding the ops in them - // the first n_nodes_0 are encoded and submitted for processing directly by the calling thread + // the first n_main nodes are encoded and submitted for processing directly by the calling thread // while these nodes are processing, we start n_cb threads to enqueue the rest of the nodes // each thread creates it's own command buffer and enqueues the ops in parallel // @@ -500,16 +500,22 @@ enum ggml_status ggml_metal_graph_compute(ggml_metal_t ctx, struct ggml_cgraph * @autoreleasepool { ctx->gf = gf; - if (ctx->n_cb == 0) { - // single-threaded encoding: the whole graph is encoded by one command buffer - ctx->n_nodes_0 = gf->n_nodes; - ctx->n_nodes_1 = 0; - ctx->n_nodes_per_cb = 0; - } else { - ctx->n_nodes_0 = MIN(n_main, gf->n_nodes); - ctx->n_nodes_1 = gf->n_nodes - ctx->n_nodes_0; - - ctx->n_nodes_per_cb = (ctx->n_nodes_1 + ctx->n_cb - 1) / ctx->n_cb; + // the main thread encodes [0, cb_start[0]) and the extra command buffer i encodes [cb_start[i], cb_start[i + 1]) + // a fusion cannot span two command buffers, so each boundary moves to the end of the fusable group it falls + // into and the encoders fuse the same nodes as with a single command buffer + { + const int n_rest = gf->n_nodes - MIN(n_main, gf->n_nodes); + const int n_per_cb = n_cb > 0 ? (n_rest + n_cb - 1) / n_cb : 0; + + int i = 0; + for (int cb_idx = 0; cb_idx < n_cb; ++cb_idx) { + const int target = MIN(n_main + cb_idx*n_per_cb, gf->n_nodes); + while (i < target) { + i += ggml_metal_fusion_max(gf, i); + } + ctx->cb_start[cb_idx] = MIN(i, gf->n_nodes); + } + ctx->cb_start[n_cb] = gf->n_nodes; } if (ctx->capture_compute >= 0) { @@ -744,17 +750,12 @@ void ggml_metal_set_n_cb(ggml_metal_t ctx, int n_cb) { const int cb_idx = iter; const int n_cb_l = ctx->n_cb; - const int n_nodes_0 = ctx->n_nodes_0; - const int n_nodes_1 = ctx->n_nodes_1; - - const int n_nodes_per_cb = ctx->n_nodes_per_cb; - int idx_start = 0; - int idx_end = n_nodes_0; + int idx_end = ctx->cb_start[0]; if (cb_idx < n_cb_l) { - idx_start = n_nodes_0 + ( (cb_idx + 0) * n_nodes_per_cb); - idx_end = n_nodes_0 + (MIN((cb_idx == n_cb_l - 1) ? n_nodes_1 : (cb_idx + 1) * n_nodes_per_cb, n_nodes_1)); + idx_start = ctx->cb_start[cb_idx]; + idx_end = ctx->cb_start[cb_idx + 1]; } id cmd_buf = ctx->cmd_bufs[cb_idx].obj; diff --git a/ggml/src/ggml-metal/ggml-metal-device.cpp b/ggml/src/ggml-metal/ggml-metal-device.cpp index 463d57c7b111..0d18a5f01e5c 100644 --- a/ggml/src/ggml-metal/ggml-metal-device.cpp +++ b/ggml/src/ggml-metal/ggml-metal-device.cpp @@ -1738,12 +1738,13 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_top_k_merge(ggml } ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_topk_moe( - ggml_metal_library_t lib, int32_t n_expert, int32_t top_k, bool with_norm) { + ggml_metal_library_t lib, int32_t n_expert, int32_t top_k, bool with_norm, int32_t gating, bool pbias) { char base[256]; char name[256]; snprintf(base, 256, "kernel_topk_moe_f32"); - snprintf(name, 256, "%s_n_expert=%d_top_k=%d_with_norm=%d", base, n_expert, top_k, with_norm ? 1 : 0); + snprintf(name, 256, "%s_n_expert=%d_top_k=%d_with_norm=%d_gating=%d_pbias=%d", + base, n_expert, top_k, with_norm ? 1 : 0, gating, pbias ? 1 : 0); ggml_metal_pipeline_with_params res = ggml_metal_library_get_pipeline(lib, name); if (!res.pipeline) { @@ -1751,6 +1752,8 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_topk_moe( ggml_metal_cv_set_bool (cv, with_norm, FC_TOPK_MOE + 0); ggml_metal_cv_set_int32(cv, n_expert, FC_TOPK_MOE + 1); ggml_metal_cv_set_int32(cv, top_k, FC_TOPK_MOE + 2); + ggml_metal_cv_set_int32(cv, gating, FC_TOPK_MOE + 3); + ggml_metal_cv_set_bool (cv, pbias, FC_TOPK_MOE + 4); res = ggml_metal_library_compile_pipeline(lib, base, name, cv); @@ -2238,17 +2241,34 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_l2_norm(ggml_met ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_group_norm(ggml_metal_library_t lib, const ggml_tensor * op) { assert(op->op == GGML_OP_GROUP_NORM); - GGML_ASSERT(ggml_is_contiguous(op->src[0])); + GGML_ASSERT(ggml_is_contiguous_rows(op->src[0])); char base[256]; char name[256]; snprintf(base, 256, "kernel_group_norm_f32"); - snprintf(name, 256, "%s", base); + + const ggml_tensor * src0 = op->src[0]; + + const bool flat = src0->nb[1] == src0->ne[0]*src0->nb[0] && + src0->nb[2] == src0->nb[1]*src0->ne[1] && + op->nb[1] == op->ne[0] *op->nb[0] && + op->nb[2] == op->nb[1] *op->ne[1]; + + if (flat) { + snprintf(name, 256, "%s", base); + } else { + snprintf(name, 256, "%s_strided", base); + } ggml_metal_pipeline_with_params res = ggml_metal_library_get_pipeline(lib, name); if (!res.pipeline) { - res = ggml_metal_library_compile_pipeline(lib, base, name, nullptr); + ggml_metal_cv_t cv = ggml_metal_cv_init(); + ggml_metal_cv_set_bool(cv, flat, FC_GROUP_NORM + 0); + + res = ggml_metal_library_compile_pipeline(lib, base, name, cv); + + ggml_metal_cv_free(cv); } res.smem = 32*sizeof(float); diff --git a/ggml/src/ggml-metal/ggml-metal-device.h b/ggml/src/ggml-metal/ggml-metal-device.h index e096d611be95..4053a624ea77 100644 --- a/ggml/src/ggml-metal/ggml-metal-device.h +++ b/ggml/src/ggml-metal/ggml-metal-device.h @@ -154,7 +154,7 @@ struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_fwht struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_top_k (ggml_metal_library_t lib, const struct ggml_tensor * op); struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_top_k_radix (ggml_metal_library_t lib, const struct ggml_tensor * op); struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_top_k_merge (ggml_metal_library_t lib, const struct ggml_tensor * op); -struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_topk_moe (ggml_metal_library_t lib, int32_t n_expert, int32_t top_k, bool with_norm); +struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_topk_moe (ggml_metal_library_t lib, int32_t n_expert, int32_t top_k, bool with_norm, int32_t gating, bool pbias); struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_moe_reduce (ggml_metal_library_t lib, int32_t n_expert_used); struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_bin (ggml_metal_library_t lib, const struct ggml_tensor * op, int32_t n_fuse); struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_bin_one (ggml_metal_library_t lib, enum ggml_op op); diff --git a/ggml/src/ggml-metal/ggml-metal-device.m b/ggml/src/ggml-metal/ggml-metal-device.m index 68a9addb42aa..0a07a76bf112 100644 --- a/ggml/src/ggml-metal/ggml-metal-device.m +++ b/ggml/src/ggml-metal/ggml-metal-device.m @@ -1510,6 +1510,18 @@ void ggml_metal_device_get_memory(ggml_metal_device_t dev, size_t * free, size_t } } +static bool ggml_metal_supports_mul_mat_src1_prec(const struct ggml_tensor * op, bool has_tensor) { + const enum ggml_prec prec = ggml_prec_get_src(op, 1); + + if (prec == GGML_PREC_F32) { + return false; + } + if (prec == GGML_PREC_BF16) { + return has_tensor && op->op == GGML_OP_MUL_MAT && op->src[0]->type != GGML_TYPE_F32; + } + return true; +} + static bool ggml_metal_supports_mul_mat_op( bool has_simdgroup_reduction, const struct ggml_tensor * op, @@ -1681,9 +1693,13 @@ bool ggml_metal_device_supports_op(ggml_metal_device_t dev, const struct ggml_te case GGML_OP_CUMSUM: case GGML_OP_MEAN: case GGML_OP_SOFT_MAX: - case GGML_OP_GROUP_NORM: case GGML_OP_L2_NORM: return has_simdgroup_reduction && ggml_is_contiguous_rows(op->src[0]); + case GGML_OP_GROUP_NORM: + return has_simdgroup_reduction && + op->src[0]->type == GGML_TYPE_F32 && + op->type == GGML_TYPE_F32 && + ggml_is_contiguous_rows(op->src[0]); case GGML_OP_COUNT_EQUAL: return has_simdgroup_reduction && op->src[0]->type == GGML_TYPE_I32 && @@ -1696,7 +1712,7 @@ bool ggml_metal_device_supports_op(ggml_metal_device_t dev, const struct ggml_te return has_simdgroup_reduction && (ggml_is_contiguous_rows(op->src[0])); case GGML_OP_ROPE: case GGML_OP_ROPE_BACK: - return true; + return ggml_rope_get_truncate(op); case GGML_OP_IM2COL: return ggml_is_contiguous(op->src[1]) && op->src[1]->type == GGML_TYPE_F32 && (op->type == GGML_TYPE_F16 || op->type == GGML_TYPE_F32); case GGML_OP_CONV_2D: @@ -1890,6 +1906,9 @@ bool ggml_metal_device_supports_op(ggml_metal_device_t dev, const struct ggml_te case GGML_OP_SOLVE_TRI: return has_simdgroup_reduction && op->src[0]->type == GGML_TYPE_F32; case GGML_OP_MUL_MAT: + if (!ggml_metal_supports_mul_mat_src1_prec(op, dev->props.has_tensor)) { + return false; + } // the FWHT kernels read an F16 source directly; every other F16 src1 path // still goes through ggml_metal_supports_mul_mat_op if (op->src[0]->type == GGML_TYPE_F32 && op->src[1]->type == GGML_TYPE_F16 && @@ -1900,6 +1919,9 @@ bool ggml_metal_device_supports_op(ggml_metal_device_t dev, const struct ggml_te has_simdgroup_reduction, op, true, ggml_metal_op_mul_mat_use_mm(op, has_simdgroup_mm)); case GGML_OP_MUL_MAT_ID: + if (!ggml_metal_supports_mul_mat_src1_prec(op, dev->props.has_tensor)) { + return false; + } return ggml_metal_supports_mul_mat_op( has_simdgroup_reduction, op, false, ggml_metal_op_mul_mat_id_use_mm(op, has_simdgroup_mm)); diff --git a/ggml/src/ggml-metal/ggml-metal-fusion.cpp b/ggml/src/ggml-metal/ggml-metal-fusion.cpp index 020cad29d198..6766b4bd998f 100644 --- a/ggml/src/ggml-metal/ggml-metal-fusion.cpp +++ b/ggml/src/ggml-metal/ggml-metal-fusion.cpp @@ -345,9 +345,6 @@ static bool ggml_metal_fusion_check_snake( #define GGML_METAL_TOPK_MOE_MAX_EXPERTS 1024 -// SOFT_MAX + ARGSORT + GET_ROWS (plus optional norm/scale) for MoE routing. -// This is a multi-output elision chain: the fused kernel writes both the selected -// expert ids and the gathered/normalized routing weights. static const std::vector ops_topk_moe = { GGML_OP_SOFT_MAX, GGML_OP_RESHAPE, GGML_OP_ARGSORT, GGML_OP_VIEW, GGML_OP_GET_ROWS }; @@ -362,6 +359,60 @@ static const std::vector ops_topk_moe_norm_scale = { GGML_OP_SOFT_MAX, GGML_OP_RESHAPE, GGML_OP_ARGSORT, GGML_OP_VIEW, GGML_OP_GET_ROWS, GGML_OP_RESHAPE, GGML_OP_SUM_ROWS, GGML_OP_CLAMP, GGML_OP_DIV, GGML_OP_RESHAPE, GGML_OP_SCALE }; +static const std::vector ops_topk_moe_norm_bias = { + GGML_OP_SOFT_MAX, GGML_OP_RESHAPE, GGML_OP_ARGSORT, GGML_OP_VIEW, GGML_OP_GET_ROWS, + GGML_OP_RESHAPE, GGML_OP_SUM_ROWS, GGML_OP_SCALE, GGML_OP_DIV, GGML_OP_RESHAPE +}; +static const std::vector ops_topk_moe_norm_scale_bias = { + GGML_OP_SOFT_MAX, GGML_OP_RESHAPE, GGML_OP_ARGSORT, GGML_OP_VIEW, GGML_OP_GET_ROWS, + GGML_OP_RESHAPE, GGML_OP_SUM_ROWS, GGML_OP_SCALE, GGML_OP_DIV, GGML_OP_RESHAPE, GGML_OP_SCALE +}; + +static const std::vector ops_topk_moe_sp = { + GGML_OP_UNARY, GGML_OP_SQRT, GGML_OP_RESHAPE, GGML_OP_ADD, GGML_OP_ARGSORT, GGML_OP_VIEW, GGML_OP_GET_ROWS +}; +static const std::vector ops_topk_moe_sp_scale = { + GGML_OP_UNARY, GGML_OP_SQRT, GGML_OP_RESHAPE, GGML_OP_ADD, GGML_OP_ARGSORT, GGML_OP_VIEW, GGML_OP_GET_ROWS, GGML_OP_SCALE +}; +static const std::vector ops_topk_moe_sp_norm = { + GGML_OP_UNARY, GGML_OP_SQRT, GGML_OP_RESHAPE, GGML_OP_ADD, GGML_OP_ARGSORT, GGML_OP_VIEW, GGML_OP_GET_ROWS, + GGML_OP_RESHAPE, GGML_OP_SUM_ROWS, GGML_OP_CLAMP, GGML_OP_DIV, GGML_OP_RESHAPE +}; +static const std::vector ops_topk_moe_sp_norm_scale = { + GGML_OP_UNARY, GGML_OP_SQRT, GGML_OP_RESHAPE, GGML_OP_ADD, GGML_OP_ARGSORT, GGML_OP_VIEW, GGML_OP_GET_ROWS, + GGML_OP_RESHAPE, GGML_OP_SUM_ROWS, GGML_OP_CLAMP, GGML_OP_DIV, GGML_OP_RESHAPE, GGML_OP_SCALE +}; +static const std::vector ops_topk_moe_sp_norm_bias = { + GGML_OP_UNARY, GGML_OP_SQRT, GGML_OP_RESHAPE, GGML_OP_ADD, GGML_OP_ARGSORT, GGML_OP_VIEW, GGML_OP_GET_ROWS, + GGML_OP_RESHAPE, GGML_OP_SUM_ROWS, GGML_OP_SCALE, GGML_OP_DIV, GGML_OP_RESHAPE +}; +static const std::vector ops_topk_moe_sp_norm_scale_bias = { + GGML_OP_UNARY, GGML_OP_SQRT, GGML_OP_RESHAPE, GGML_OP_ADD, GGML_OP_ARGSORT, GGML_OP_VIEW, GGML_OP_GET_ROWS, + GGML_OP_RESHAPE, GGML_OP_SUM_ROWS, GGML_OP_SCALE, GGML_OP_DIV, GGML_OP_RESHAPE, GGML_OP_SCALE +}; + +static const std::vector ops_topk_moe_sig = { + GGML_OP_UNARY, GGML_OP_RESHAPE, GGML_OP_ADD, GGML_OP_ARGSORT, GGML_OP_VIEW, GGML_OP_GET_ROWS +}; +static const std::vector ops_topk_moe_sig_scale = { + GGML_OP_UNARY, GGML_OP_RESHAPE, GGML_OP_ADD, GGML_OP_ARGSORT, GGML_OP_VIEW, GGML_OP_GET_ROWS, GGML_OP_SCALE +}; +static const std::vector ops_topk_moe_sig_norm = { + GGML_OP_UNARY, GGML_OP_RESHAPE, GGML_OP_ADD, GGML_OP_ARGSORT, GGML_OP_VIEW, GGML_OP_GET_ROWS, + GGML_OP_RESHAPE, GGML_OP_SUM_ROWS, GGML_OP_CLAMP, GGML_OP_DIV, GGML_OP_RESHAPE +}; +static const std::vector ops_topk_moe_sig_norm_scale = { + GGML_OP_UNARY, GGML_OP_RESHAPE, GGML_OP_ADD, GGML_OP_ARGSORT, GGML_OP_VIEW, GGML_OP_GET_ROWS, + GGML_OP_RESHAPE, GGML_OP_SUM_ROWS, GGML_OP_CLAMP, GGML_OP_DIV, GGML_OP_RESHAPE, GGML_OP_SCALE +}; +static const std::vector ops_topk_moe_sig_norm_bias = { + GGML_OP_UNARY, GGML_OP_RESHAPE, GGML_OP_ADD, GGML_OP_ARGSORT, GGML_OP_VIEW, GGML_OP_GET_ROWS, + GGML_OP_RESHAPE, GGML_OP_SUM_ROWS, GGML_OP_SCALE, GGML_OP_DIV, GGML_OP_RESHAPE +}; +static const std::vector ops_topk_moe_sig_norm_scale_bias = { + GGML_OP_UNARY, GGML_OP_RESHAPE, GGML_OP_ADD, GGML_OP_ARGSORT, GGML_OP_VIEW, GGML_OP_GET_ROWS, + GGML_OP_RESHAPE, GGML_OP_SUM_ROWS, GGML_OP_SCALE, GGML_OP_DIV, GGML_OP_RESHAPE, GGML_OP_SCALE +}; static bool ggml_metal_fusion_check_topk_moe( const ggml_metal_fusion * fusion, @@ -375,8 +426,11 @@ static bool ggml_metal_fusion_check_topk_moe( const int n_ops = (int) fusion->ops.size(); - const bool with_norm = n_ops >= 6; - const bool with_scale = n_ops == 4 || n_ops == 7; + bool with_norm = false; + for (int j = 0; j < n_ops; ++j) { + with_norm |= fusion->ops[j] == GGML_OP_SUM_ROWS; + } + const bool with_scale = fusion->ops[n_ops - 1] == GGML_OP_SCALE; // the fusion table operates on the non-empty node sequence; the raw graph also // contains the RESHAPE/VIEW nodes that the fused kernel elides. @@ -411,24 +465,70 @@ static bool ggml_metal_fusion_check_topk_moe( } } - const ggml_tensor * softmax = gf->nodes[raw_start]; - const ggml_tensor * probs_reshaped = gf->nodes[raw_start + 1]; - const ggml_tensor * argsort = gf->nodes[raw_start + 2]; - const ggml_tensor * ids = gf->nodes[raw_start + 3]; - const ggml_tensor * get_rows = gf->nodes[raw_start + 4]; - const ggml_tensor * out = gf->nodes[raw_end]; - const ggml_tensor * logits = softmax->src[0]; - - // the fused kernel implements plain softmax only - float scale = 1.0f; - float max_bias = 0.0f; - memcpy(&scale, ((const int32_t *) softmax->op_params) + 0, sizeof(scale)); - memcpy(&max_bias, ((const int32_t *) softmax->op_params) + 1, sizeof(max_bias)); - if (scale != 1.0f || max_bias != 0.0f || softmax->src[1] || softmax->src[2]) { + const ggml_tensor * first = gf->nodes[raw_start]; + const ggml_tensor * out = gf->nodes[raw_end]; + + // the unary chains start at the sigmoid or the softplus node and rank biased scores + const bool with_unary = first->op == GGML_OP_UNARY; + const bool with_sp = with_unary && ggml_get_unary_op(first) == GGML_UNARY_OP_SOFTPLUS; + const bool with_sig = with_unary && ggml_get_unary_op(first) == GGML_UNARY_OP_SIGMOID; + + if (with_unary && !with_sp && !with_sig) { return false; } - if (logits->type != GGML_TYPE_F32 || softmax->type != GGML_TYPE_F32 || + const ggml_tensor * probs; // the scoring output + const ggml_tensor * scores; // the node the argsort ranks (adds the bias for the unary chains) + const ggml_tensor * probs_reshaped; + const ggml_tensor * argsort; + const ggml_tensor * ids; + const ggml_tensor * get_rows; + const ggml_tensor * logits = first->src[0]; + int ids_off; + + if (with_unary) { + // the softplus chains add a SQRT node, the sigmoid chains start at the sigmoid node + const int base = with_sp ? 1 : 0; + + probs = gf->nodes[raw_start + base + 0]; + probs_reshaped = gf->nodes[raw_start + base + 1]; + scores = gf->nodes[raw_start + base + 2]; + argsort = gf->nodes[raw_start + base + 3]; + ids = gf->nodes[raw_start + base + 4]; + get_rows = gf->nodes[raw_start + base + 5]; + ids_off = base + 4; + + if (scores->op != GGML_OP_ADD || scores->src[0] != probs || + (with_sp && (probs->op != GGML_OP_SQRT || probs->src[0] != first))) { + return false; + } + + // the selection bias is a per-expert vector shared by all rows + const ggml_tensor * bias = scores->src[1]; + if (bias->type != GGML_TYPE_F32 || scores->type != GGML_TYPE_F32 || !ggml_is_contiguous(bias) || + bias->ne[0] != probs->ne[0] || bias->ne[1] != 1 || bias->ne[2] != 1 || bias->ne[3] != 1) { + return false; + } + } else { + probs = first; + scores = first; + probs_reshaped = gf->nodes[raw_start + 1]; + argsort = gf->nodes[raw_start + 2]; + ids = gf->nodes[raw_start + 3]; + get_rows = gf->nodes[raw_start + 4]; + ids_off = 3; + + // the fused kernel implements plain softmax only + float scale = 1.0f; + float max_bias = 0.0f; + memcpy(&scale, ((const int32_t *) first->op_params) + 0, sizeof(scale)); + memcpy(&max_bias, ((const int32_t *) first->op_params) + 1, sizeof(max_bias)); + if (scale != 1.0f || max_bias != 0.0f || first->src[1] || first->src[2]) { + return false; + } + } + + if (logits->type != GGML_TYPE_F32 || probs->type != GGML_TYPE_F32 || out->type != GGML_TYPE_F32 || ids->type != GGML_TYPE_I32) { return false; } @@ -455,38 +555,40 @@ static bool ggml_metal_fusion_check_topk_moe( return false; } - if (probs_reshaped->src[0] != softmax || argsort->src[0] != softmax || + // the top-k gathers the unbiased scores, the argsort ranks the biased ones + if (probs_reshaped->src[0] != probs || argsort->src[0] != scores || ids->src[0] != argsort || get_rows->src[0] != probs_reshaped || get_rows->src[1] != ids) { return false; } if (with_norm) { - const ggml_tensor * weights_reshaped = gf->nodes[raw_start + 5]; - const ggml_tensor * sum_rows = gf->nodes[raw_start + 6]; - const ggml_tensor * clamp = gf->nodes[raw_start + 7]; - const ggml_tensor * div = gf->nodes[raw_start + 8]; - const ggml_tensor * out_reshaped = gf->nodes[raw_start + 9]; + const int norm_off = with_unary ? 6 + (with_sp ? 1 : 0) : 5; + + const ggml_tensor * weights_reshaped = gf->nodes[raw_start + norm_off + 0]; + const ggml_tensor * sum_rows = gf->nodes[raw_start + norm_off + 1]; + const ggml_tensor * denom = gf->nodes[raw_start + norm_off + 2]; + const ggml_tensor * div = gf->nodes[raw_start + norm_off + 3]; + const ggml_tensor * out_reshaped = gf->nodes[raw_start + norm_off + 4]; if (weights_reshaped->src[0] != get_rows || sum_rows->src[0] != weights_reshaped || - clamp->src[0] != sum_rows || div->src[0] != weights_reshaped || div->src[1] != clamp || + denom->src[0] != sum_rows || div->src[0] != weights_reshaped || div->src[1] != denom || out_reshaped->src[0] != div) { return false; } - if (with_scale) { - const ggml_tensor * scale_node = gf->nodes[raw_start + 10]; - if (scale_node->src[0] != out_reshaped) { - return false; - } + // the kernel applies the sum adjustment as a lower clamp or an additive bias, never a scale + if (denom->op == GGML_OP_SCALE && ggml_get_op_params_f32(denom, 0) != 1.0f) { + return false; } - } else if (with_scale) { - const ggml_tensor * scale_node = gf->nodes[raw_start + 5]; - if (scale_node->src[0] != get_rows) { + + if (with_scale && out->src[0] != out_reshaped) { return false; } + } else if (with_scale && out->src[0] != get_rows) { + return false; } - const int outputs[2] = { raw_start + 3, raw_end }; + const int outputs[2] = { raw_start + ids_off, raw_end }; if (!ggml_can_fuse_subgraph_ext(gf, raw_idxs, raw_count, ops_all.data(), outputs, 2)) { return false; } @@ -763,33 +865,54 @@ static const std::vector ops_moe_reduce_8 = { static const std::vector ops_mul_mat_add = { GGML_OP_MUL_MAT, GGML_OP_ADD }; static const std::vector ggml_metal_fusions = { - { GGML_METAL_FUSION_NORM_MUL, ops_norm_mul, {}, false, ggml_metal_fusion_check_norm }, - { GGML_METAL_FUSION_NORM_MUL_ADD, ops_norm_mul_add, {}, false, ggml_metal_fusion_check_norm }, - { GGML_METAL_FUSION_NORM_SCALE, ops_norm_scale, {}, false, ggml_metal_fusion_check_norm }, - { GGML_METAL_FUSION_NORM_MUL, ops_rms_norm_mul, {}, false, ggml_metal_fusion_check_norm }, - { GGML_METAL_FUSION_NORM_MUL_ADD, ops_rms_norm_mul_add, {}, false, ggml_metal_fusion_check_norm }, - { GGML_METAL_FUSION_NORM_SCALE, ops_rms_norm_scale, {}, false, ggml_metal_fusion_check_norm }, - { GGML_METAL_FUSION_ADD_CHAIN, ops_add_2, {}, false, ggml_metal_fusion_check_add_chain }, - { GGML_METAL_FUSION_ADD_CHAIN, ops_add_3, {}, false, ggml_metal_fusion_check_add_chain }, - { GGML_METAL_FUSION_ADD_CHAIN, ops_add_4, {}, false, ggml_metal_fusion_check_add_chain }, - { GGML_METAL_FUSION_ADD_CHAIN, ops_add_5, {}, false, ggml_metal_fusion_check_add_chain }, - { GGML_METAL_FUSION_ADD_CHAIN, ops_add_6, {}, false, ggml_metal_fusion_check_add_chain }, - { GGML_METAL_FUSION_ADD_CHAIN, ops_add_7, {}, false, ggml_metal_fusion_check_add_chain }, - { GGML_METAL_FUSION_SNAKE, ops_snake, {}, false, ggml_metal_fusion_check_snake }, - { GGML_METAL_FUSION_GDN_CACHE, ops_gdn_cache, {}, true, ggml_metal_fusion_check_gdn_cache }, - { GGML_METAL_FUSION_TOPK_MOE, ops_topk_moe, {1}, true, ggml_metal_fusion_check_topk_moe }, - { GGML_METAL_FUSION_TOPK_MOE, ops_topk_moe_scale, {1}, true, ggml_metal_fusion_check_topk_moe }, - { GGML_METAL_FUSION_TOPK_MOE, ops_topk_moe_norm, {1}, true, ggml_metal_fusion_check_topk_moe }, - { GGML_METAL_FUSION_TOPK_MOE, ops_topk_moe_norm_scale, {1}, true, ggml_metal_fusion_check_topk_moe }, - { GGML_METAL_FUSION_MOE_REDUCE, ops_moe_reduce_2, {}, true, ggml_metal_fusion_check_moe_reduce }, - { GGML_METAL_FUSION_MOE_REDUCE, ops_moe_reduce_3, {}, true, ggml_metal_fusion_check_moe_reduce }, - { GGML_METAL_FUSION_MOE_REDUCE, ops_moe_reduce_4, {}, true, ggml_metal_fusion_check_moe_reduce }, - { GGML_METAL_FUSION_MOE_REDUCE, ops_moe_reduce_5, {}, true, ggml_metal_fusion_check_moe_reduce }, - { GGML_METAL_FUSION_MOE_REDUCE, ops_moe_reduce_6, {}, true, ggml_metal_fusion_check_moe_reduce }, - { GGML_METAL_FUSION_MOE_REDUCE, ops_moe_reduce_7, {}, true, ggml_metal_fusion_check_moe_reduce }, - { GGML_METAL_FUSION_MOE_REDUCE, ops_moe_reduce_8, {}, true, ggml_metal_fusion_check_moe_reduce }, - { GGML_METAL_FUSION_SSM_CONV_SILU, ops_ssm_conv_silu, {}, false, ggml_metal_fusion_check_ssm_conv_silu }, - { GGML_METAL_FUSION_MUL_MAT_ADD, ops_mul_mat_add, {}, false, ggml_metal_fusion_check_mul_mat_add }, + { GGML_METAL_FUSION_NORM_MUL, ops_norm_mul, {}, false, ggml_metal_fusion_check_norm }, + { GGML_METAL_FUSION_NORM_MUL_ADD, ops_norm_mul_add, {}, false, ggml_metal_fusion_check_norm }, + { GGML_METAL_FUSION_NORM_SCALE, ops_norm_scale, {}, false, ggml_metal_fusion_check_norm }, + { GGML_METAL_FUSION_NORM_MUL, ops_rms_norm_mul, {}, false, ggml_metal_fusion_check_norm }, + { GGML_METAL_FUSION_NORM_MUL_ADD, ops_rms_norm_mul_add, {}, false, ggml_metal_fusion_check_norm }, + { GGML_METAL_FUSION_NORM_SCALE, ops_rms_norm_scale, {}, false, ggml_metal_fusion_check_norm }, + + { GGML_METAL_FUSION_ADD_CHAIN, ops_add_2, {}, false, ggml_metal_fusion_check_add_chain }, + { GGML_METAL_FUSION_ADD_CHAIN, ops_add_3, {}, false, ggml_metal_fusion_check_add_chain }, + { GGML_METAL_FUSION_ADD_CHAIN, ops_add_4, {}, false, ggml_metal_fusion_check_add_chain }, + { GGML_METAL_FUSION_ADD_CHAIN, ops_add_5, {}, false, ggml_metal_fusion_check_add_chain }, + { GGML_METAL_FUSION_ADD_CHAIN, ops_add_6, {}, false, ggml_metal_fusion_check_add_chain }, + { GGML_METAL_FUSION_ADD_CHAIN, ops_add_7, {}, false, ggml_metal_fusion_check_add_chain }, + + { GGML_METAL_FUSION_SNAKE, ops_snake, {}, false, ggml_metal_fusion_check_snake }, + + { GGML_METAL_FUSION_GDN_CACHE, ops_gdn_cache, {}, true, ggml_metal_fusion_check_gdn_cache }, + + { GGML_METAL_FUSION_TOPK_MOE, ops_topk_moe, {1}, true, ggml_metal_fusion_check_topk_moe }, + { GGML_METAL_FUSION_TOPK_MOE, ops_topk_moe_scale, {1}, true, ggml_metal_fusion_check_topk_moe }, + { GGML_METAL_FUSION_TOPK_MOE, ops_topk_moe_norm, {1}, true, ggml_metal_fusion_check_topk_moe }, + { GGML_METAL_FUSION_TOPK_MOE, ops_topk_moe_norm_scale, {1}, true, ggml_metal_fusion_check_topk_moe }, + { GGML_METAL_FUSION_TOPK_MOE, ops_topk_moe_norm_bias, {1}, true, ggml_metal_fusion_check_topk_moe }, + { GGML_METAL_FUSION_TOPK_MOE, ops_topk_moe_norm_scale_bias, {1}, true, ggml_metal_fusion_check_topk_moe }, + { GGML_METAL_FUSION_TOPK_MOE, ops_topk_moe_sp, {3}, true, ggml_metal_fusion_check_topk_moe }, + { GGML_METAL_FUSION_TOPK_MOE, ops_topk_moe_sp_scale, {3}, true, ggml_metal_fusion_check_topk_moe }, + { GGML_METAL_FUSION_TOPK_MOE, ops_topk_moe_sp_norm, {3}, true, ggml_metal_fusion_check_topk_moe }, + { GGML_METAL_FUSION_TOPK_MOE, ops_topk_moe_sp_norm_scale, {3}, true, ggml_metal_fusion_check_topk_moe }, + { GGML_METAL_FUSION_TOPK_MOE, ops_topk_moe_sp_norm_bias, {3}, true, ggml_metal_fusion_check_topk_moe }, + { GGML_METAL_FUSION_TOPK_MOE, ops_topk_moe_sp_norm_scale_bias, {3}, true, ggml_metal_fusion_check_topk_moe }, + { GGML_METAL_FUSION_TOPK_MOE, ops_topk_moe_sig, {2}, true, ggml_metal_fusion_check_topk_moe }, + { GGML_METAL_FUSION_TOPK_MOE, ops_topk_moe_sig_scale, {2}, true, ggml_metal_fusion_check_topk_moe }, + { GGML_METAL_FUSION_TOPK_MOE, ops_topk_moe_sig_norm, {2}, true, ggml_metal_fusion_check_topk_moe }, + { GGML_METAL_FUSION_TOPK_MOE, ops_topk_moe_sig_norm_scale, {2}, true, ggml_metal_fusion_check_topk_moe }, + { GGML_METAL_FUSION_TOPK_MOE, ops_topk_moe_sig_norm_bias, {2}, true, ggml_metal_fusion_check_topk_moe }, + { GGML_METAL_FUSION_TOPK_MOE, ops_topk_moe_sig_norm_scale_bias, {2}, true, ggml_metal_fusion_check_topk_moe }, + + { GGML_METAL_FUSION_MOE_REDUCE, ops_moe_reduce_2, {}, true, ggml_metal_fusion_check_moe_reduce }, + { GGML_METAL_FUSION_MOE_REDUCE, ops_moe_reduce_3, {}, true, ggml_metal_fusion_check_moe_reduce }, + { GGML_METAL_FUSION_MOE_REDUCE, ops_moe_reduce_4, {}, true, ggml_metal_fusion_check_moe_reduce }, + { GGML_METAL_FUSION_MOE_REDUCE, ops_moe_reduce_5, {}, true, ggml_metal_fusion_check_moe_reduce }, + { GGML_METAL_FUSION_MOE_REDUCE, ops_moe_reduce_6, {}, true, ggml_metal_fusion_check_moe_reduce }, + { GGML_METAL_FUSION_MOE_REDUCE, ops_moe_reduce_7, {}, true, ggml_metal_fusion_check_moe_reduce }, + { GGML_METAL_FUSION_MOE_REDUCE, ops_moe_reduce_8, {}, true, ggml_metal_fusion_check_moe_reduce }, + + { GGML_METAL_FUSION_SSM_CONV_SILU, ops_ssm_conv_silu, {}, false, ggml_metal_fusion_check_ssm_conv_silu }, + + { GGML_METAL_FUSION_MUL_MAT_ADD, ops_mul_mat_add, {}, false, ggml_metal_fusion_check_mul_mat_add }, }; // ---- alloc deps ----------------------------------------------------------- diff --git a/ggml/src/ggml-metal/ggml-metal-impl.h b/ggml/src/ggml-metal/ggml-metal-impl.h index 11c15feffe06..7fd9d97cff22 100644 --- a/ggml/src/ggml-metal/ggml-metal-impl.h +++ b/ggml/src/ggml-metal/ggml-metal-impl.h @@ -126,6 +126,7 @@ #define FC_LIGHTNING_INDEXER 2200 #define FC_MUL_MV_MMA 2300 #define FC_TOP_K_MASK 2400 +#define FC_GROUP_NORM 2500 // op-specific constants #define OP_FLASH_ATTN_EXT_NQPSG 8 @@ -677,6 +678,10 @@ typedef struct { uint64_t nb00; uint64_t nb01; uint64_t nb02; + uint64_t nb03; + uint64_t nb1; + uint64_t nb2; + uint64_t nb3; int32_t ngrp; float eps; } ggml_metal_kargs_group_norm; @@ -1270,6 +1275,7 @@ typedef struct { uint64_t nb01; // logits row stride uint64_t nb1_ids; // ids row stride float clamp; + float bias; float scale; } ggml_metal_kargs_topk_moe; diff --git a/ggml/src/ggml-metal/ggml-metal-ops.cpp b/ggml/src/ggml-metal/ggml-metal-ops.cpp index cace34bd4578..5c93bc4f0741 100644 --- a/ggml/src/ggml-metal/ggml-metal-ops.cpp +++ b/ggml/src/ggml-metal/ggml-metal-ops.cpp @@ -821,6 +821,16 @@ int ggml_metal_op_acc(ggml_metal_op_t ctx, int idx) { int ggml_metal_op_unary(ggml_metal_op_t ctx, int idx) { ggml_tensor * op = ctx->node(idx); + // the sigmoid and sqrt(softplus) TOPK_MOE fusions anchor at the gating unary node + if (op->op == GGML_OP_UNARY && ctx->use_fusion() && + (ggml_get_unary_op(op) == GGML_UNARY_OP_SOFTPLUS || ggml_get_unary_op(op) == GGML_UNARY_OP_SIGMOID)) { + int n = 1; + const ggml_metal_fusion * fusion = ctx->can_fuse(idx, GGML_METAL_FUSION_FULL, &n); + if (fusion && ggml_metal_fusion_get_id(fusion) == GGML_METAL_FUSION_TOPK_MOE) { + return ggml_metal_op_topk_moe(ctx, idx); + } + } + ggml_metal_library_t lib = ctx->lib; ggml_metal_encoder_t enc = ctx->enc; @@ -4600,6 +4610,10 @@ int ggml_metal_op_group_norm(ggml_metal_op_t ctx, int idx) { /*.nb00 =*/ nb00, /*.nb01 =*/ nb01, /*.nb02 =*/ nb02, + /*.nb03 =*/ nb03, + /*.nb1 =*/ nb1, + /*.nb2 =*/ nb2, + /*.nb3 =*/ nb3, /*.ngrp =*/ ngrp, /*.eps =*/ eps, }; @@ -4623,7 +4637,7 @@ int ggml_metal_op_group_norm(ggml_metal_op_t ctx, int idx) { ggml_metal_encoder_set_threadgroup_memory_size(enc, smem, 0); - ggml_metal_encoder_dispatch_threadgroups(enc, ngrp, 1, 1, nth, 1, 1); + ggml_metal_encoder_dispatch_threadgroups(enc, ngrp, 1, ne03, nth, 1, 1); return 1; } @@ -5994,11 +6008,10 @@ static void ggml_metal_op_top_k_radix(ggml_metal_op_t ctx, int idx) { /*.top_k =*/ (int32_t) op->ne[0], }; - // shared memory: 256-entry histogram + bucket/above scalars + output counter + // shared memory: 256-entry histogram + bucket/above scalars const size_t smem_histo = GGML_PAD(256*sizeof(uint32_t), 16); const size_t smem_bucket = GGML_PAD( sizeof(uint32_t), 16); const size_t smem_above = GGML_PAD( sizeof(uint32_t), 16); - const size_t smem_out = GGML_PAD( sizeof(uint32_t), 16); ggml_metal_encoder_set_pipeline(enc, pipeline); ggml_metal_encoder_set_bytes (enc, &args, sizeof(args), 0); @@ -6008,7 +6021,6 @@ static void ggml_metal_op_top_k_radix(ggml_metal_op_t ctx, int idx) { ggml_metal_encoder_set_threadgroup_memory_size(enc, smem_histo, 0); ggml_metal_encoder_set_threadgroup_memory_size(enc, smem_bucket, 1); ggml_metal_encoder_set_threadgroup_memory_size(enc, smem_above, 2); - ggml_metal_encoder_set_threadgroup_memory_size(enc, smem_out, 3); ggml_metal_encoder_dispatch_threadgroups(enc, ne01, ne02, ne03, nth, 1, 1); } @@ -6023,29 +6035,48 @@ int ggml_metal_op_topk_moe(ggml_metal_op_t ctx, int idx) { return 1; } - ggml_tensor * softmax = ctx->node(idx); - ggml_tensor * logits = softmax->src[0]; - ggml_tensor * get_rows = ctx->node(idx + 2); - ggml_tensor * ids = get_rows->src[1]; - ggml_tensor * weights = ctx->node(idx + n_fuse - 1); + ggml_tensor * first = ctx->node(idx); + ggml_tensor * logits = first->src[0]; + + // the unary chains score with sigmoid or sqrt(softplus) and rank biased scores + const bool with_unary = first->op == GGML_OP_UNARY; + const int gating = !with_unary ? 0 : ggml_get_unary_op(first) == GGML_UNARY_OP_SOFTPLUS ? 1 : 2; + + ggml_tensor * get_rows = nullptr; + ggml_tensor * sum_rows = nullptr; + ggml_tensor * prob_bias = nullptr; + int sum_rows_pos = -1; + for (int j = 0; j < n_fuse; ++j) { + ggml_tensor * node = ctx->node(idx + j); + if (node->op == GGML_OP_GET_ROWS) { get_rows = node; } + if (node->op == GGML_OP_ADD) { prob_bias = node->src[1]; } + if (node->op == GGML_OP_SUM_ROWS) { sum_rows = node; sum_rows_pos = j; } + } + + ggml_tensor * ids = get_rows->src[1]; + ggml_tensor * weights = ctx->node(idx + n_fuse - 1); + + const bool with_norm = sum_rows != nullptr; + const bool with_scale = weights->op == GGML_OP_SCALE; const int64_t n_expert = logits->ne[0]; const int64_t n_tokens = logits->ne[1]; const int64_t n_expert_used = ids->ne[0]; - const bool with_norm = n_fuse >= 6; - const bool with_scale = n_fuse == 4 || n_fuse == 7; - float clamp = -INFINITY; + float bias = 0.0f; if (with_norm) { - ggml_tensor * clamp_node = ctx->node(idx + 4); - clamp = ggml_get_op_params_f32(clamp_node, 0); + ggml_tensor * denom_node = ctx->node(idx + sum_rows_pos + 1); + if (denom_node->op == GGML_OP_SCALE) { + bias = ggml_get_op_params_f32(denom_node, 1); + } else { + clamp = ggml_get_op_params_f32(denom_node, 0); + } } float scale = 1.0f; if (with_scale) { - ggml_tensor * scale_node = ctx->node(idx + n_fuse - 1); - scale = ggml_get_op_params_f32(scale_node, 0); + scale = ggml_get_op_params_f32(weights, 0); } ggml_metal_kargs_topk_moe args = { @@ -6053,23 +6084,27 @@ int ggml_metal_op_topk_moe(ggml_metal_op_t ctx, int idx) { /*.nb01 =*/ logits->nb[1], /*.nb1_ids =*/ ids->nb[1], /*.clamp =*/ clamp, + /*.bias =*/ bias, /*.scale =*/ scale, }; - auto pipeline = ggml_metal_library_get_pipeline_topk_moe(lib, (int32_t) n_expert, (int32_t) n_expert_used, with_norm); + auto pipeline = ggml_metal_library_get_pipeline_topk_moe(lib, (int32_t) n_expert, (int32_t) n_expert_used, + with_norm, gating, prob_bias != nullptr); ggml_metal_encoder_set_pipeline(enc, pipeline); ggml_metal_encoder_set_bytes (enc, &args, sizeof(args), 0); ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(logits), 1); ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(weights), 2); ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(ids), 3); + ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(prob_bias ? prob_bias : logits), 4); ggml_metal_encoder_dispatch_threadgroups(enc, (uint32_t) n_tokens, 1, 1, 32, 1, 1); ctx->count_fusions(fusion); if (ggml_metal_fusion_info_debug(ctx->finfo) > 1) { - GGML_LOG_DEBUG("%s: fuse: SOFT_MAX + ARGSORT + GET_ROWS\n", __func__); + GGML_LOG_DEBUG("%s: fuse: %s + ARGSORT + GET_ROWS\n", __func__, + gating == 1 ? "SOFTPLUS + SQRT" : gating == 2 ? "SIGMOID" : "SOFT_MAX"); } return n_fuse; diff --git a/ggml/src/ggml-metal/kernels/argsort.metal b/ggml/src/ggml-metal/kernels/argsort.metal index 5231b8395c9a..6c6fcb53425b 100644 --- a/ggml/src/ggml-metal/kernels/argsort.metal +++ b/ggml/src/ggml-metal/kernels/argsort.metal @@ -3,6 +3,8 @@ constant bool FC_topk_moe_with_norm [[function_constant(FC_TOPK_MOE + 0)]]; constant int FC_topk_moe_n_expert [[function_constant(FC_TOPK_MOE + 1)]]; constant int FC_topk_moe_top_k [[function_constant(FC_TOPK_MOE + 2)]]; +constant int FC_topk_moe_gating [[function_constant(FC_TOPK_MOE + 3)]]; +constant bool FC_topk_moe_pbias [[function_constant(FC_TOPK_MOE + 4)]]; constant int FC_moe_reduce_n_expert_used [[function_constant(FC_MOE_REDUCE + 0)]]; @@ -254,7 +256,6 @@ kernel void kernel_top_k_f32_i32( threadgroup atomic_uint * histo [[threadgroup(0)]], threadgroup uint * sh_bucket [[threadgroup(1)]], threadgroup uint * sh_above [[threadgroup(2)]], - threadgroup atomic_uint * out_count [[threadgroup(3)]], uint3 tgpig[[threadgroup_position_in_grid]], ushort3 tpitg[[thread_position_in_threadgroup]], ushort3 ntg[[threads_per_threadgroup]]) { @@ -292,20 +293,32 @@ kernel void kernel_top_k_f32_i32( } threadgroup_barrier(mem_flags::mem_threadgroup); - // top-down scan for the bucket holding the k-th value - if (tid == 0) { - uint acc = 0; - uint b = 0; - for (int bb = 255; bb >= 0; --bb) { - const uint c = atomic_load_explicit(&histo[bb], memory_order_relaxed); - if (acc + c >= desired) { - b = uint(bb); - break; + // top-down scan for the bucket holding the k-th value: each lane of the first + // simdgroup sums 8 bins, the lane whose range reaches desired scans its own bins + if (tid < 32) { + const uint top = 255 - 8*tid; + + uint c[8]; + uint sum = 0; + for (uint j = 0; j < 8; ++j) { + c[j] = atomic_load_explicit(&histo[top - j], memory_order_relaxed); + sum += c[j]; + } + uint acc = simd_prefix_exclusive_sum(sum); + + const simd_vote::vote_t hit = simd_vote::vote_t(simd_ballot(acc + sum >= desired)); + if (tid == (uint) ctz(hit)) { + uint b = top; + for (uint j = 0; j < 8; ++j) { + if (acc + c[j] >= desired) { + b = top - j; + break; + } + acc += c[j]; } - acc += c; + *sh_bucket = b; + *sh_above = acc; } - *sh_bucket = b; - *sh_above = acc; } threadgroup_barrier(mem_flags::mem_threadgroup); @@ -316,39 +329,113 @@ kernel void kernel_top_k_f32_i32( threadgroup_barrier(mem_flags::mem_threadgroup); } - if (tid == 0) { - atomic_store_explicit(out_count, 0u, memory_order_relaxed); + // deterministic compaction: rank the kept elements by column order, so the + // result does not depend on the thread scheduling + const uint threshold = prefix; + + // each warp scans a contiguous chunk of the row, and the lanes of the warp + // interleave within the chunk with stride 32 to keep the accesses coalesced + const uint lane = tid & 31u; + const uint wg = tid >> 5u; + const uint nw = ntg_x >> 5u; + const uint wlo = wg*((ncols + nw - 1) / nw); + const uint whi = MIN(wlo + (ncols + nw - 1) / nw, ncols); + const uint wlen = wlo < whi ? whi - wlo : 0; + const uint nstep = (wlen + 31) >> 5; + + uint ca = 0; // elements above the threshold in this lane's substream + uint ct = 0; // elements tied with the threshold in this lane's substream + for (uint i = wlo + lane; i < whi; i += 32u) { + const uint key = ggml_top_k_f2ui(src0_row[i]); + ca += (key > threshold) ? 1u : 0u; + ct += (key == threshold) ? 1u : 0u; + } + + // per-warp totals into the now unused histogram, so that the first warp can + // compute the exclusive base of each warp with a scan across its lanes + const uint wca = simd_sum(ca); + const uint wct = simd_sum(ct); + if (lane == 0) { + atomic_store_explicit(&histo[wg], wca, memory_order_relaxed); + atomic_store_explicit(&histo[32 + wg], wct, memory_order_relaxed); } threadgroup_barrier(mem_flags::mem_threadgroup); - // emit everything above the threshold, then fill the rest from ties - const uint threshold = prefix; - - for (uint i = tid; i < ncols; i += ntg_x) { - if (ggml_top_k_f2ui(src0_row[i]) > threshold) { - const uint pos = atomic_fetch_add_explicit(out_count, 1u, memory_order_relaxed); - dst_row[pos] = (int32_t) i; + if (tid < 32) { + const bool valid = tid < nw; + const uint sa = valid ? atomic_load_explicit(&histo[tid], memory_order_relaxed) : 0u; + const uint st = valid ? atomic_load_explicit(&histo[32 + tid], memory_order_relaxed) : 0u; + const uint ea = simd_prefix_exclusive_sum(sa); + const uint et = simd_prefix_exclusive_sum(st); + if (valid) { + atomic_store_explicit(&histo[tid], ea, memory_order_relaxed); + atomic_store_explicit(&histo[32 + tid], et, memory_order_relaxed); + } + const uint ta = simd_sum(sa); + const uint tt = simd_sum(st); + if (tid == 0) { + *sh_bucket = ta; // total above the threshold + *sh_above = tt; // total tied with the threshold } } threadgroup_barrier(mem_flags::mem_threadgroup); - for (uint i = tid; i < ncols; i += ntg_x) { - if (ggml_top_k_f2ui(src0_row[i]) == threshold) { - const uint pos = atomic_fetch_add_explicit(out_count, 1u, memory_order_relaxed); - if (pos < top_k) { + const uint above_total = *sh_bucket; + const uint tie_total = *sh_above; + const uint base_wa = atomic_load_explicit(&histo[wg], memory_order_relaxed); + const uint base_wt = above_total + atomic_load_explicit(&histo[32 + wg], memory_order_relaxed); + + // the lanes write their kept elements at lane-major ranks, which does not + // depend on the thread scheduling. with more tied candidates than remaining + // slots, the ties are instead ranked in column order with a per-step + // cross-lane scan, so that the lowest tied indices are the ones kept + const uint excl_a = simd_prefix_exclusive_sum(ca); + const uint excl_t = simd_prefix_exclusive_sum(ct); + + if (tie_total <= top_k - above_total) { + uint na = 0; + uint nt = 0; + for (uint i = wlo + lane; i < whi; i += 32u) { + const uint key = ggml_top_k_f2ui(src0_row[i]); + if (key > threshold) { + dst_row[base_wa + excl_a + na] = (int32_t) i; + ++na; + } else if (key == threshold) { + dst_row[base_wt + excl_t + nt] = (int32_t) i; + ++nt; + } + } + } else { + uint na = 0; + for (uint i = wlo + lane; i < whi; i += 32u) { + if (ggml_top_k_f2ui(src0_row[i]) > threshold) { + dst_row[base_wa + excl_a + na] = (int32_t) i; + ++na; + } + } + + uint tt = 0; // tied elements in the earlier steps of this warp + for (uint s = 0; s < nstep; ++s) { + const uint i = wlo + 32u*s + lane; + const bool ok = i < whi; + const bool t = ok && ggml_top_k_f2ui(src0_row[i]) == threshold; + const uint pos = base_wt + tt + simd_prefix_exclusive_sum((uint) t); + if (t && pos < top_k) { dst_row[pos] = (int32_t) i; } + tt += simd_sum((uint) t); } } } -// fused SOFT_MAX + top-k + GET_ROWS (+ optional norm/scale) for MoE routing. +// fused SOFT_MAX or sqrt(softplus) + top-k + GET_ROWS (+ optional norm/scale) for MoE routing. // One SIMDgroup handles one token row; n_expert is limited to 1024 by the host. kernel void kernel_topk_moe_f32( constant ggml_metal_kargs_topk_moe & args, device const char * src0, device float * weights, device int32_t * ids, + device const float * probs_bias, uint3 tgpig[[threadgroup_position_in_grid]], ushort tiisg[[thread_index_in_simdgroup]]) { const int row = (int) tgpig.x; @@ -377,50 +464,78 @@ kernel void kernel_topk_moe_f32( wt[i / 32] = isnan(v) ? -FLT_MAX : v; } - // softmax over the expert logits - float max_val = -INFINITY; - FOR_UNROLL (int i = 0; i < n_per_lane; ++i) { - max_val = max(max_val, wt[i]); - } - max_val = simd_max(max_val); + if (FC_topk_moe_gating == 1) { + FOR_UNROLL (int i = 0; i < n_per_lane; ++i) { + const float v = wt[i]; + wt[i] = sqrt(v > 20.0f ? v : log(1.0f + exp(v))); + } + } else if (FC_topk_moe_gating == 2) { + FOR_UNROLL (int i = 0; i < n_per_lane; ++i) { + wt[i] = 1.0f / (1.0f + exp(-wt[i])); + } + } else { + // softmax over the expert logits + float max_val = -INFINITY; + FOR_UNROLL (int i = 0; i < n_per_lane; ++i) { + max_val = max(max_val, wt[i]); + } + max_val = simd_max(max_val); - float sum_val = 0.0f; - FOR_UNROLL (int i = 0; i < n_per_lane; ++i) { - wt[i] = exp(wt[i] - max_val); - sum_val += wt[i]; + float sum_val = 0.0f; + FOR_UNROLL (int i = 0; i < n_per_lane; ++i) { + wt[i] = exp(wt[i] - max_val); + sum_val += wt[i]; + } + sum_val = simd_sum(sum_val); + + const float inv_sum = 1.0f / sum_val; + FOR_UNROLL (int i = 0; i < n_per_lane; ++i) { + wt[i] *= inv_sum; + } } - sum_val = simd_sum(sum_val); - const float inv_sum = 1.0f / sum_val; - FOR_UNROLL (int i = 0; i < n_per_lane; ++i) { - wt[i] *= inv_sum; + // the ranking keys add the selection bias, the output weights stay unbiased + float sel[32]; + if (FC_topk_moe_pbias) { + FOR_UNROLL (int i = 0; i < n_per_lane; ++i) { + const int expert = lane + i * 32; + sel[i] = expert < n_expert ? wt[i] + probs_bias[expert] : -INFINITY; + } } float wt_sum = 0.0f; for (int k = 0; k < top_k; ++k) { - float best_val = -INFINITY; + float best_val = -INFINITY; + float best_key = -INFINITY; int best_expert = -1; FOR_UNROLL (int i = 0; i < n_per_lane; ++i) { - const int expert = lane + i * 32; - if (expert < n_expert && (wt[i] > best_val || (wt[i] == best_val && expert < best_expert))) { + const int expert = lane + i * 32; + const float key = FC_topk_moe_pbias ? sel[i] : wt[i]; + if (expert < n_expert && (key > best_key || (key == best_key && expert < best_expert))) { best_val = wt[i]; + best_key = key; best_expert = expert; } } FOR_UNROLL (int mask = 16; mask > 0; mask >>= 1) { const float val = simd_shuffle_xor(best_val, mask); + const float key = simd_shuffle_xor(best_key, mask); const int expert = simd_shuffle_xor(best_expert, mask); - if (val > best_val || (val == best_val && expert < best_expert)) { + if (key > best_key || (key == best_key && expert < best_expert)) { best_val = val; + best_key = key; best_expert = expert; } } if ((best_expert & 31) == lane) { wt[best_expert / 32] = -INFINITY; + if (FC_topk_moe_pbias) { + sel[best_expert / 32] = -INFINITY; + } } if ((k & 31) == lane) { @@ -437,7 +552,7 @@ kernel void kernel_topk_moe_f32( if (FC_topk_moe_with_norm) { wt_sum = simd_sum(wt_sum); - wt_sum = max(wt_sum, args.clamp); + wt_sum = max(wt_sum, args.clamp) + args.bias; const float inv = 1.0f / wt_sum; FOR_UNROLL (int i = 0; i < n_per_lane; ++i) { output_weights[i] *= inv; diff --git a/ggml/src/ggml-metal/kernels/fa_vec_common.metal b/ggml/src/ggml-metal/kernels/fa_vec_common.metal index c81e4d09bb75..1a96c6f7ec8b 100644 --- a/ggml/src/ggml-metal/kernels/fa_vec_common.metal +++ b/ggml/src/ggml-metal/kernels/fa_vec_common.metal @@ -131,9 +131,11 @@ kernel void kernel_flash_attn_ext_vec( } // zero out so - for (short qq = 0; qq < Q; ++qq) { - for (short i = 0; i < DV4/NL; ++i) { - so4[qq*DV4 + i*NL] = (o4_t) 0.0f; + if ((DV4/NL % NW == 0) || tiisg < NL) { + for (short qq = 0; qq < Q; ++qq) { + for (short i = 0; i < DV4/NL; ++i) { + so4[qq*DV4 + i*NL] = (o4_t) 0.0f; + } } } diff --git a/ggml/src/ggml-metal/kernels/mul_mm.metal b/ggml/src/ggml-metal/kernels/mul_mm.metal index edcc061d516e..fab720097399 100644 --- a/ggml/src/ggml-metal/kernels/mul_mm.metal +++ b/ggml/src/ggml-metal/kernels/mul_mm.metal @@ -895,7 +895,8 @@ typedef decltype(kernel_mul_mm; template [[host_name("kernel_mul_mm_f16_f32")]] kernel mul_mm_t kernel_mul_mm; #if defined(GGML_METAL_HAS_BF16) -template [[host_name("kernel_mul_mm_bf16_f32")]] kernel mul_mm_t kernel_mul_mm; +template [[host_name("kernel_mul_mm_bf16_f32")]] kernel mul_mm_t kernel_mul_mm; +template [[host_name("kernel_mul_mm_bf16_bf16")]] kernel mul_mm_t kernel_mul_mm; #endif template [[host_name("kernel_mul_mm_q1_0_f32")]] kernel mul_mm_t kernel_mul_mm; template [[host_name("kernel_mul_mm_q2_0_f32")]] kernel mul_mm_t kernel_mul_mm; diff --git a/ggml/src/ggml-metal/kernels/norm.metal b/ggml/src/ggml-metal/kernels/norm.metal index c76d0c7ff169..983d9b5cbb49 100644 --- a/ggml/src/ggml-metal/kernels/norm.metal +++ b/ggml/src/ggml-metal/kernels/norm.metal @@ -236,37 +236,59 @@ typedef decltype(kernel_l2_norm_impl) kernel_l2_norm_t; template [[host_name("kernel_l2_norm_f32_f32")]] kernel kernel_l2_norm_t kernel_l2_norm_impl; template [[host_name("kernel_l2_norm_f32_f32_4")]] kernel kernel_l2_norm_t kernel_l2_norm_impl; +constant bool FC_group_norm_flat [[function_constant(FC_GROUP_NORM + 0)]]; + kernel void kernel_group_norm_f32( constant ggml_metal_kargs_group_norm & args, - device const float * src0, - device float * dst, + device const char * src0, + device char * dst, threadgroup float * buf [[threadgroup(0)]], - uint tgpig[[threadgroup_position_in_grid]], - uint tpitg[[thread_position_in_threadgroup]], - uint sgitg[[simdgroup_index_in_threadgroup]], - uint tiisg[[thread_index_in_simdgroup]], - uint ntg[[threads_per_threadgroup]]) { - const int64_t ne = args.ne00*args.ne01*args.ne02; - const int64_t gs = args.ne00*args.ne01*((args.ne02 + args.ngrp - 1) / args.ngrp); + uint3 tgpig[[threadgroup_position_in_grid]], + ushort3 tpitg[[thread_position_in_threadgroup]], + ushort sgitg[[simdgroup_index_in_threadgroup]], + ushort tiisg[[thread_index_in_simdgroup]], + ushort3 ntg[[threads_per_threadgroup]]) { + const int64_t ne00 = args.ne00; + const int64_t ne01 = args.ne01; + const int64_t ne02 = args.ne02; - int start = tgpig * gs; - int end = start + gs; + const int64_t ngrp = args.ngrp; - start += tpitg; + const int64_t i03 = tgpig.z; // batch index + const int64_t ig = tgpig.x; // group index - if (end >= ne) { - end = ne; - } + const int64_t cpg = (ne02 + ngrp - 1)/ngrp; + + const int64_t c0 = ig*cpg; + const int64_t c1 = (c0 + cpg < ne02) ? c0 + cpg : ne02; + + const int64_t npf = ne00*ne01; // elements per channel + const int64_t gs = npf*(c1 - c0); // elements in this group + + device const char * x03 = src0 + i03*args.nb03 + c0*args.nb02; + device char * y03 = dst + i03*args.nb3 + c0*args.nb2; float tmp = 0.0f; // partial sum for thread in warp - for (int j = start; j < end; j += ntg) { - tmp += src0[j]; + if (FC_group_norm_flat) { + device const float * x = (device const float *) x03; + for (int64_t e = tpitg.x; e < gs; e += ntg.x) { + tmp += x[e]; + } + } else { + for (int64_t e = tpitg.x; e < gs; e += ntg.x) { + const int64_t i02 = c0 + e/npf; + const int64_t ir = e - (i02 - c0)*npf; + const int64_t i01 = ir/ne00; + const int64_t i00 = ir - i01*ne00; + + tmp += *(device const float *)(x03 + (i02 - c0)*args.nb02 + i01*args.nb01 + i00*sizeof(float)); + } } threadgroup_barrier(mem_flags::mem_threadgroup); tmp = simd_sum(tmp); - if (ntg > N_SIMDWIDTH) { + if (ntg.x > N_SIMDWIDTH) { if (sgitg == 0) { buf[tiisg] = 0.0f; } @@ -286,14 +308,31 @@ kernel void kernel_group_norm_f32( const float mean = tmp / gs; tmp = 0.0f; - for (int j = start; j < end; j += ntg) { - float xi = src0[j] - mean; - dst[j] = xi; - tmp += xi * xi; + if (FC_group_norm_flat) { + device const float * x = (device const float *) x03; + device float * y = (device float *) y03; + for (int64_t e = tpitg.x; e < gs; e += ntg.x) { + const float xi = x[e] - mean; + y[e] = xi; + tmp += xi*xi; + } + } else { + for (int64_t e = tpitg.x; e < gs; e += ntg.x) { + const int64_t i02 = c0 + e/npf; + const int64_t ir = e - (i02 - c0)*npf; + const int64_t i01 = ir/ne00; + const int64_t i00 = ir - i01*ne00; + + const float xi = *(device const float *)(x03 + (i02 - c0)*args.nb02 + i01*args.nb01 + i00*sizeof(float)) - mean; + + *(device float *)(y03 + (i02 - c0)*args.nb2 + i01*args.nb1 + i00*sizeof(float)) = xi; + + tmp += xi*xi; + } } tmp = simd_sum(tmp); - if (ntg > N_SIMDWIDTH) { + if (ntg.x > N_SIMDWIDTH) { if (sgitg == 0) { buf[tiisg] = 0.0f; } @@ -312,7 +351,20 @@ kernel void kernel_group_norm_f32( const float variance = tmp / gs; const float scale = 1.0f/sqrt(variance + args.eps); - for (int j = start; j < end; j += ntg) { - dst[j] *= scale; + + if (FC_group_norm_flat) { + device float * y = (device float *) y03; + for (int64_t e = tpitg.x; e < gs; e += ntg.x) { + y[e] *= scale; + } + } else { + for (int64_t e = tpitg.x; e < gs; e += ntg.x) { + const int64_t i02 = c0 + e/npf; + const int64_t ir = e - (i02 - c0)*npf; + const int64_t i01 = ir/ne00; + const int64_t i00 = ir - i01*ne00; + + *(device float *)(y03 + (i02 - c0)*args.nb2 + i01*args.nb1 + i00*sizeof(float)) *= scale; + } } } diff --git a/ggml/src/ggml-metal/kernels/quantize.metal b/ggml/src/ggml-metal/kernels/quantize.metal index 73679b9ab76f..663e47d5fd2f 100644 --- a/ggml/src/ggml-metal/kernels/quantize.metal +++ b/ggml/src/ggml-metal/kernels/quantize.metal @@ -26,11 +26,11 @@ kernel void kernel_cpy_t_t( const int32_t i1 = (n - i3*args.ne2*args.ne1*args.ne0 - i2*args.ne1*args.ne0)/args.ne0; const int32_t i0 = (n - i3*args.ne2*args.ne1*args.ne0 - i2*args.ne1*args.ne0 - i1*args.ne0); - device T1 * dst_data = (device T1 *) (dst + i3*args.nb3 + i2*args.nb2 + i1*args.nb1 + i0*args.nb0); + device char * dst_data = dst + i3*args.nb3 + i2*args.nb2 + i1*args.nb1 + i0*args.nb0; for (int32_t i00 = iw0*ntg[0] + tpitg.x; i00 < args.ne00;) { device const T0 * src = (device T0 *)(src0 + i03*args.nb03 + i02*args.nb02 + i01*args.nb01 + i00*args.nb00); - dst_data[i00] = (T1) src[0]; + *(device T1 *)(dst_data + i00*args.nb0) = (T1) src[0]; break; } } diff --git a/ggml/src/ggml-metal/kernels/solve_tri.metal b/ggml/src/ggml-metal/kernels/solve_tri.metal index 50f16facbf2e..216f605fe29b 100644 --- a/ggml/src/ggml-metal/kernels/solve_tri.metal +++ b/ggml/src/ggml-metal/kernels/solve_tri.metal @@ -39,7 +39,9 @@ kernel void kernel_solve_tri_f32( for (short t = 0; t*NW < N; ++t) { const short idx = t*NW + tiisg; - sh0_cur[idx] = src0_ptr[idx]; + if (rr + sgitg < N && idx < N) { + sh0_cur[idx] = src0_ptr[idx]; + } } src0_ptr += NSG*N; @@ -60,7 +62,9 @@ kernel void kernel_solve_tri_f32( for (short t = 0; t*NW < r; ++t) { const short idx = t*NW + tiisg; - sum += sh0_cur[idx] * dst_ptr[idx*K] * (idx < r); + if (idx < r) { + sum += sh0_cur[idx] * dst_ptr[idx*K]; + } } sum = simd_sum(sum); diff --git a/ggml/src/ggml-metal/kernels/tri.metal b/ggml/src/ggml-metal/kernels/tri.metal index 862f78678b52..05c8c4682fba 100644 --- a/ggml/src/ggml-metal/kernels/tri.metal +++ b/ggml/src/ggml-metal/kernels/tri.metal @@ -46,8 +46,7 @@ kernel void kernel_tri( // threadgroup, so this will loop once for each index that this thread is // responsible for for (int64_t i0 = tpitg.x; i0 < args.ne00; i0 += ntg.x) { - // Use the comparison as a mask for branchless - dst_row[i0] = static_cast(_ggml_vec_tri_cmp(i0, i1)) * src_row[i0]; + dst_row[i0] = _ggml_vec_tri_cmp(i0, i1) ? src_row[i0] : static_cast(0); } } diff --git a/ggml/src/ggml-opencl/CMakeLists.txt b/ggml/src/ggml-opencl/CMakeLists.txt index 79fc37b60990..cdf4a5d54455 100644 --- a/ggml/src/ggml-opencl/CMakeLists.txt +++ b/ggml/src/ggml-opencl/CMakeLists.txt @@ -71,13 +71,14 @@ set(GGML_OPENCL_KERNELS lightning_indexer tri fill + arange fwht clamp cpy cvt - diag_mask_inf diag div + dsv4_hc gelu get_rows glu @@ -131,6 +132,9 @@ set(GGML_OPENCL_KERNELS gemm_noshuffle_iq3_xxs_f32 gemm_noshuffle_iq3_xxs_q8_1_dp4a gemv_noshuffle_iq3_xxs_f32 + gemm_noshuffle_iq4_xs_f32 + gemm_noshuffle_iq4_xs_q8_1_dp4a + gemv_noshuffle_iq4_xs_f32 gemm_noshuffle_q3_k_q8_1_dp4a gemv_noshuffle_q3_k_f32 mul_mv_q2_k_f32 @@ -165,6 +169,10 @@ set(GGML_OPENCL_KERNELS mul_mv_iq3_s_f32 mul_mv_iq3_s_f32_flat mul_mm_iq3_s_f32_l4_lm + iq4_xs_grid + mul_mv_iq4_xs_f32 + mul_mv_iq4_xs_f32_flat + mul_mm_iq4_xs_f32_l4_lm mul_mv_q3_k_f32_flat mul_mm_q3_k_f32_l4_lm mul_mv_q2_k_f32_flat @@ -186,6 +194,8 @@ set(GGML_OPENCL_KERNELS mul_mv_mxfp4_f32 mul_mv_mxfp4_f32_flat mul_mv_id_q4_0_f32_8x_flat + mul_mv_id_f32_f32 + mul_mm_id_f32_f32 mul_mv_id_q8_0_f32 mul_mv_id_q8_0_f32_flat mul_mv_id_mxfp4_f32 @@ -311,11 +321,14 @@ set(GGML_OPENCL_KERNELS repeat mul_mat_f16_f32 mul_mm_f16_f32_kq_kqv + mul_mm_f16_f32_kq_p8 + mul_mm_q8_f32_kqv conv2d conv2d_f16_f32 conv2_dw flash_attn_repack flash_attn_pre_f16 + flash_attn_pv_direct flash_attn_f32_f16 flash_attn_f32_q8_0 flash_attn_f32_q4_0 diff --git a/ggml/src/ggml-opencl/ggml-opencl.cpp b/ggml/src/ggml-opencl/ggml-opencl.cpp index 9999aac905ad..7fa1a8e35b6b 100644 --- a/ggml/src/ggml-opencl/ggml-opencl.cpp +++ b/ggml/src/ggml-opencl/ggml-opencl.cpp @@ -36,6 +36,7 @@ typedef const void * (*get_adreno_bin_kernel_func_t)( #include #include +#include #include #include #include @@ -78,6 +79,11 @@ bool ggml_cl_compute_forward(ggml_backend_t backend, struct ggml_tensor * tensor static bool ggml_cl_is_q4_0_soa(const ggml_tensor * tensor); static bool ggml_cl_is_q8_0_soa(const ggml_tensor * tensor); static void ggml_cl_mul_mat(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst); +static void ggml_cl_cpy(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst); +static void ggml_cl_fa_decompose_soft_max(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * src2, ggml_tensor * dst, cl_mem sums); +#ifdef GGML_OPENCL_USE_ADRENO_KERNELS +static void ggml_cl_mul_mat_kq_kqv_adreno(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst, bool is_kq); +#endif // See https://gmplib.org/~tege/divcnst-pldi94.pdf figure 4.1. // Precompute mp (m' in the paper) and L such that division @@ -570,6 +576,8 @@ struct ggml_opencl_fa_kernels { std::map, cl_kernel> kv_pad_f16; std::map, cl_kernel> mask_pad_f16; std::map, cl_kernel> blk_f16; + // decomposed prefill: V^T, built with the prepass + std::map, cl_kernel> v_transpose_f16; // generic prefill tile dims (f16 / f32 paths) std::map, int> bm; std::map, int> bn; @@ -759,6 +767,18 @@ struct ggml_backend_opencl_context { ggml_cl_buffer prealloc_src1; ggml_cl_buffer prealloc_top_k; + // scratch for the decomposed flash attention prefill, sized per query chunk + ggml_cl_buffer prealloc_fa_vt; + ggml_cl_buffer prealloc_fa_kq; + ggml_cl_buffer prealloc_fa_kqv; + ggml_cl_buffer prealloc_fa_sums; + ggml_cl_buffer prealloc_fa_vtq; + ggml_cl_buffer prealloc_fa_vtd; + ggml_cl_buffer prealloc_fa_pq; + ggml_cl_buffer prealloc_fa_pd; + ggml_cl_buffer prealloc_fa_bmax; + ggml_cl_buffer prealloc_fa_bsum; + #ifdef GGML_OPENCL_USE_ADRENO_KERNELS ggml_cl_buffer prealloc_adreno_xmem_const; bool adreno_xmem_gemm_enabled = false; @@ -778,7 +798,6 @@ struct ggml_backend_opencl_context { cl_program program_moe_add_id_glu; cl_program program_clamp; cl_program program_cvt; - cl_program program_diag_mask_inf; cl_program program_gelu; cl_program program_gemv_noshuffle_general; cl_program program_gemv_noshuffle; @@ -853,8 +872,10 @@ struct ggml_backend_opencl_context { cl_kernel kernel_gelu_quick, kernel_gelu_quick_4; cl_kernel kernel_relu; cl_kernel kernel_sigmoid_f32, kernel_sigmoid_f16; + cl_kernel kernel_sigmoid_f32_nc, kernel_sigmoid_f16_nc; cl_kernel kernel_tri; cl_kernel kernel_fill; + cl_kernel kernel_arange; cl_kernel kernel_clamp, kernel_clamp_nc; cl_kernel kernel_geglu, kernel_reglu, kernel_swiglu, kernel_swiglu_oai, kernel_swiglu_clamp, kernel_geglu_erf, kernel_geglu_quick, kernel_geglu_f16, kernel_reglu_f16, kernel_swiglu_f16, kernel_swiglu_clamp_f16, @@ -864,10 +885,16 @@ struct ggml_backend_opencl_context { cl_kernel kernel_rms_norm_mul_add = nullptr; // fused rms_norm(x)*w + b (residual) cl_kernel kernel_l2_norm_f32; cl_kernel kernel_group_norm, kernel_group_norm_mul_add; - cl_kernel kernel_diag_mask_inf, kernel_diag_mask_inf_8; cl_kernel kernel_diag_f32; cl_kernel kernel_soft_max, kernel_soft_max_4; cl_kernel kernel_soft_max_f16, kernel_soft_max_4_f16; + cl_kernel kernel_fa_decompose_soft_max_4_f16, kernel_fa_decompose_scale_rows_f32; + cl_kernel kernel_mul_mm_f16_f32_kq_p8 = nullptr; + cl_kernel kernel_mul_mm_q8_f32_kqv = nullptr; + cl_kernel kernel_fa_p8_fixup = nullptr; + cl_kernel kernel_fa_kqv_direct_f16 = nullptr; + cl_kernel kernel_fa_kqv_direct_reduce = nullptr; + cl_kernel kernel_fa_v_transpose_q8 = nullptr; ggml_opencl_fa_kernels fa; #ifdef GGML_OPENCL_USE_ADRENO_KERNELS ggml_cl_adreno_xmem_attn_state adreno_xmem_attn; @@ -887,6 +914,7 @@ struct ggml_backend_opencl_context { cl_kernel kernel_cpy_f32_f32_pack = nullptr; cl_kernel kernel_cpy_f32_f32_flat = nullptr; cl_kernel kernel_mul_mat_f32_f32; + cl_kernel kernel_mul_mv_f32_f32_r2 = nullptr; cl_kernel kernel_fwht[2] = {}; size_t fwht_workgroup_size[2] = {}; cl_kernel kernel_mul_mat_f16_f16; @@ -1027,10 +1055,13 @@ struct ggml_backend_opencl_context { cl_kernel kernel_mul_mv_q2_0_f32_flat; cl_kernel kernel_mul_mm_q2_0_f32_l4_lm; cl_kernel kernel_gemv_noshuffle_q2_0_f32 = nullptr; + cl_kernel kernel_gemv_noshuffle_q2_0_f32_wide = nullptr; + size_t q2_0_mv_workgroup_size = 512; + size_t q2_0_mv_workgroup_size_wide = 512; cl_kernel kernel_gemm_noshuffle_q2_0_f32 = nullptr; cl_kernel kernel_gemm_noshuffle_q2_0_q8_1_dp4a = nullptr; cl_kernel kernel_gemm_noshuffle_q2_0_q8_1_dp4a_narrow = nullptr; - int q2_0_mv_nsg = 8; + int q2_0_mv_nsg = 32; cl_kernel kernel_mul_mv_q2_0_f32; cl_kernel kernel_convert_block_iq1_m, kernel_restore_block_iq1_m; cl_kernel kernel_mul_mv_iq1_m_f32_flat; @@ -1074,6 +1105,16 @@ struct ggml_backend_opencl_context { ggml_cl_buffer iq3_s_grid_buffer; cl_mem iq3_s_grid_image = nullptr; cl_kernel kernel_mul_mv_iq3_s_f32; + cl_kernel kernel_convert_block_iq4_xs, kernel_restore_block_iq4_xs; + cl_kernel kernel_convert_block_iq4_xs_noshuffle = nullptr, kernel_restore_block_iq4_xs_noshuffle = nullptr; + cl_kernel kernel_mul_mv_iq4_xs_f32_flat; + cl_kernel kernel_mul_mm_iq4_xs_f32_l4_lm; + cl_kernel kernel_gemv_noshuffle_iq4_xs_f32 = nullptr; + cl_kernel kernel_gemm_noshuffle_iq4_xs_f32 = nullptr; + cl_kernel kernel_gemm_noshuffle_iq4_xs_q8_1_dp4a = nullptr; + cl_kernel kernel_gemm_noshuffle_iq4_xs_q8_1_dp4a_narrow = nullptr; + int iq4_xs_mv_nsg = 8; + cl_kernel kernel_mul_mv_iq4_xs_f32; cl_kernel kernel_mul_mv_q4_K_f32; cl_kernel kernel_mul_mv_q4_K_f32_flat; cl_kernel kernel_mul_mv_q5_K_f32; @@ -1090,8 +1131,12 @@ struct ggml_backend_opencl_context { cl_kernel kernel_argmax_f32_i32; cl_kernel kernel_argsort_f32_i32; cl_kernel kernel_sum_rows_f32, kernel_sum_rows_f32_4; + cl_kernel kernel_dsv4_hc_pre = nullptr; + cl_kernel kernel_dsv4_hc_post[2] = {}; + size_t local_dsv4_hc_pre = 0; + size_t local_dsv4_hc_post[2] = {}; cl_kernel kernel_cumsum_blk, kernel_cumsum_add; - cl_kernel kernel_repeat_f32; + cl_kernel kernel_repeat_f32, kernel_repeat_f16, kernel_repeat_i32; cl_kernel kernel_pad; cl_kernel kernel_tanh_f32, kernel_tanh_f32_4, kernel_tanh_f32_nc; cl_kernel kernel_tanh_f16, kernel_tanh_f16_4, kernel_tanh_f16_nc; @@ -1165,6 +1210,8 @@ struct ggml_backend_opencl_context { cl_kernel kernel_moe_zero_dst; cl_kernel kernel_moe_combine_f32 = nullptr; // fused router-weight mul + cross-expert sum cl_kernel kernel_moe_combine_bias_f32 = nullptr; // same, with the down-projection bias add folded in + cl_kernel kernel_mul_mv_id_f32_f32; + cl_kernel kernel_mul_mm_id_f32_f32; cl_kernel kernel_mul_mv_id_q4_0_f32_8x_flat; cl_kernel kernel_mul_mv_id_q8_0_f32, kernel_mul_mv_id_q8_0_f32_flat; cl_kernel kernel_mul_mv_id_mxfp4_f32; @@ -1351,6 +1398,7 @@ struct ggml_backend_opencl_context { cl_kernel kernel_gemv_noshuffle_q4_0_f32; cl_kernel kernel_gemv_noshuffle_q4_0_f32_mc3; // multi-column (N=3) verify GEMV (spec/MTP) cl_kernel kernel_gemm_noshuffle_q4_0_f32_32b_trans_ila_a8_bin; + cl_kernel kernel_gemm_noshuffle_q4_0_f32_32b_trans_ila_a7_bin; cl_kernel kernel_gemm_noshuffle_q4_0_q8_1_dp4a_ila_a8_bin; cl_kernel kernel_gemv_noshuffle_q4_0_f32_32b_trans; cl_kernel kernel_gemv_noshuffle_q4_0_f32_4096_1_11008; @@ -1367,6 +1415,7 @@ struct ggml_backend_opencl_context { cl_kernel kernel_gemm_noshuffle_q8_0_q8_1_dp4a_ila_a8_bin = nullptr; cl_kernel kernel_gemv_noshuffle_q8_0_f32; cl_kernel kernel_gemv_noshuffle_q8_0_f32_splitk; // split-K across WGs (small-M decode) + cl_kernel kernel_gemv_noshuffle_q8_0_f32_mc; // multi-column (N 2..4) verify GEMV cl_kernel kernel_gemm_noshuffle_q1_0_f32; cl_kernel kernel_gemv_noshuffle_q1_0_f32; cl_kernel kernel_gemv_noshuffle_q1_0_f32_nsg1 = nullptr; @@ -1404,7 +1453,7 @@ struct ggml_backend_opencl_context { cl_kernel kernel_gemv_noshuffle_q6_K_f32_o4; cl_kernel kernel_gemv_noshuffle_q6_K_f32_o4_global; // weights via __global (opt-in) cl_kernel kernel_gemv_noshuffle_q6_K_f32_tiled; // tiled-wide layout (opt-in) - cl_kernel kernel_gemv_noshuffle_q6_K_f32_tiled_mc3; // tiled multi-column (N=3) verify lm_head + cl_kernel kernel_gemv_noshuffle_q6_K_f32_tiled_mc3; // tiled multi-column (N 2..4) verify lm_head cl_kernel kernel_gemm_noshuffle_q6_K_f32_tiled; // batched (N>1) over the tiled layout cl_kernel kernel_convert_block_q6_k_tiled_ns; // tiled-wide convert (opt-in) cl_kernel kernel_gemv_noshuffle_q6_K_f32_mc3; // multi-column (N=3) verify GEMV @@ -1449,6 +1498,19 @@ struct ggml_backend_opencl_context { #endif // GGML_OPENCL_USE_ADRENO_KERNELS ~ggml_backend_opencl_context() { + for (cl_kernel kernel : {kernel_mul_mm_f16_f32_kq_p8, kernel_mul_mm_q8_f32_kqv, kernel_fa_p8_fixup, kernel_fa_v_transpose_q8, kernel_fa_kqv_direct_f16, kernel_fa_kqv_direct_reduce}) { + if (kernel != nullptr) { + CL_CHECK(clReleaseKernel(kernel)); + } + } + if (kernel_dsv4_hc_pre) { + CL_CHECK(clReleaseKernel(kernel_dsv4_hc_pre)); + } + for (int i = 0; i < 2; ++i) { + if (kernel_dsv4_hc_post[i]) { + CL_CHECK(clReleaseKernel(kernel_dsv4_hc_post[i])); + } + } if (iq1_s_grid_image) { CL_CHECK(clReleaseMemObject(iq1_s_grid_image)); } @@ -1968,6 +2030,26 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) { GGML_LOG_CONT("."); } + // moe_sort_by_expert + { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "moe_sort_by_expert.cl.h" + }; +#else + const std::string kernel_src = read_file("moe_sort_by_expert.cl"); +#endif + cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); + + CL_CHECK((backend_ctx->kernel_moe_histogram = clCreateKernel(prog, "kernel_moe_histogram", &err), err)); + CL_CHECK((backend_ctx->kernel_moe_scan = clCreateKernel(prog, "kernel_moe_scan", &err), err)); + CL_CHECK((backend_ctx->kernel_moe_fill = clCreateKernel(prog, "kernel_moe_fill", &err), err)); + CL_CHECK((backend_ctx->kernel_moe_scatter = clCreateKernel(prog, "kernel_moe_scatter", &err), err)); + CL_CHECK((backend_ctx->kernel_moe_scatter_stable = clCreateKernel(prog, "kernel_moe_scatter_stable", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); + } + // moe_add_id_glu { #ifdef GGML_OPENCL_EMBED_KERNELS @@ -2021,6 +2103,24 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) { CL_CHECK(clReleaseProgram(prog)); } + // arange + { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "arange.cl.h" + }; +#else + const std::string kernel_src = read_file("arange.cl"); +#endif + cl_program prog = + build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); + + CL_CHECK((backend_ctx->kernel_arange = clCreateKernel(prog, "kernel_arange_f32", &err), err)); + GGML_LOG_CONT("."); + + CL_CHECK(clReleaseProgram(prog)); + } + // clamp { #ifdef GGML_OPENCL_EMBED_KERNELS @@ -2163,6 +2263,10 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) { CL_CHECK((backend_ctx->kernel_restore_block_iq3_xxs = clCreateKernel(backend_ctx->program_cvt, "kernel_restore_block_iq3_xxs", &err), err)); CL_CHECK((backend_ctx->kernel_convert_block_iq3_s = clCreateKernel(backend_ctx->program_cvt, "kernel_convert_block_iq3_s", &err), err)); CL_CHECK((backend_ctx->kernel_restore_block_iq3_s = clCreateKernel(backend_ctx->program_cvt, "kernel_restore_block_iq3_s", &err), err)); + CL_CHECK((backend_ctx->kernel_convert_block_iq4_xs_noshuffle = clCreateKernel(backend_ctx->program_cvt, "kernel_convert_block_iq4_xs_noshuffle", &err), err)); + CL_CHECK((backend_ctx->kernel_restore_block_iq4_xs_noshuffle = clCreateKernel(backend_ctx->program_cvt, "kernel_restore_block_iq4_xs_noshuffle", &err), err)); + CL_CHECK((backend_ctx->kernel_convert_block_iq4_xs = clCreateKernel(backend_ctx->program_cvt, "kernel_convert_block_iq4_xs", &err), err)); + CL_CHECK((backend_ctx->kernel_restore_block_iq4_xs = clCreateKernel(backend_ctx->program_cvt, "kernel_restore_block_iq4_xs", &err), err)); CL_CHECK((backend_ctx->kernel_convert_block_q3_K = clCreateKernel(backend_ctx->program_cvt, "kernel_convert_block_q3_K", &err), err)); CL_CHECK((backend_ctx->kernel_restore_block_q3_K = clCreateKernel(backend_ctx->program_cvt, "kernel_restore_block_q3_K", &err), err)); CL_CHECK((backend_ctx->kernel_convert_block_q2_K = clCreateKernel(backend_ctx->program_cvt, "kernel_convert_block_q2_K", &err), err)); @@ -2185,23 +2289,6 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) { GGML_LOG_CONT("."); } - // diag_mask_inf - { -#ifdef GGML_OPENCL_EMBED_KERNELS - const std::string kernel_src { - #include "diag_mask_inf.cl.h" - }; -#else - const std::string kernel_src = read_file("diag_mask_inf.cl"); -#endif - backend_ctx->program_diag_mask_inf = - build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); - - CL_CHECK((backend_ctx->kernel_diag_mask_inf_8 = clCreateKernel(backend_ctx->program_diag_mask_inf, "kernel_diag_mask_inf_8", &err), err)); - CL_CHECK((backend_ctx->kernel_diag_mask_inf = clCreateKernel(backend_ctx->program_diag_mask_inf, "kernel_diag_mask_inf", &err), err)); - GGML_LOG_CONT("."); - } - // diag { #ifdef GGML_OPENCL_EMBED_KERNELS @@ -3054,6 +3141,59 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) { GGML_LOG_CONT("."); } +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string iq4_xs_grid_src { + #include "iq4_xs_grid.cl.h" + }; +#else + const std::string iq4_xs_grid_src = read_file("iq4_xs_grid.cl"); +#endif + + // mul_mv_iq4_xs_f32_flat + { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "mul_mv_iq4_xs_f32_flat.cl.h" + }; +#else + const std::string kernel_src = read_file("mul_mv_iq4_xs_f32_flat.cl"); +#endif + cl_program prog = build_program_from_source(backend_ctx, (iq4_xs_grid_src + kernel_src).c_str(), compile_opts); + CL_CHECK((backend_ctx->kernel_mul_mv_iq4_xs_f32_flat = clCreateKernel(prog, "kernel_mul_mv_iq4_xs_f32_flat", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); + } + + // mul_mm_iq4_xs_f32_l4_lm + { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "mul_mm_iq4_xs_f32_l4_lm.cl.h" + }; +#else + const std::string kernel_src = read_file("mul_mm_iq4_xs_f32_l4_lm.cl"); +#endif + cl_program prog = build_program_from_source(backend_ctx, (iq4_xs_grid_src + kernel_src).c_str(), compile_opts); + CL_CHECK((backend_ctx->kernel_mul_mm_iq4_xs_f32_l4_lm = clCreateKernel(prog, "kernel_mul_mm_iq4_xs_f32_l4_lm", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); + } + + // mul_mv_iq4_xs_f32 + { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "mul_mv_iq4_xs_f32.cl.h" + }; +#else + const std::string kernel_src = read_file("mul_mv_iq4_xs_f32.cl"); +#endif + cl_program prog = build_program_from_source(backend_ctx, (iq4_xs_grid_src + kernel_src).c_str(), compile_opts); + CL_CHECK((backend_ctx->kernel_mul_mv_iq4_xs_f32 = clCreateKernel(prog, "kernel_mul_mv_iq4_xs_f32", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); + } + // mul_mv_q3_k_f32 { #ifdef GGML_OPENCL_EMBED_KERNELS @@ -3524,6 +3664,17 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) { build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); CL_CHECK((backend_ctx->kernel_mul_mat_f32_f32 = clCreateKernel(backend_ctx->program_mul_mv_f32_f32, "kernel_mul_mat_f32_f32", &err), err)); + if (backend_ctx->adreno_gen == ADRENO_GPU_GEN::A8X) { + cl_kernel kernel = clCreateKernel(backend_ctx->program_mul_mv_f32_f32, "kernel_mul_mv_f32_f32_r2", &err); + CL_CHECK(err); + size_t max_workgroup; + CL_CHECK(clGetKernelWorkGroupInfo(kernel, backend_ctx->device, CL_KERNEL_WORK_GROUP_SIZE, sizeof(max_workgroup), &max_workgroup, nullptr)); + if (max_workgroup >= 64) { + backend_ctx->kernel_mul_mv_f32_f32_r2 = kernel; + } else { + CL_CHECK(clReleaseKernel(kernel)); + } + } GGML_LOG_CONT("."); } @@ -3834,6 +3985,26 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) { GGML_LOG_CONT("."); } +#ifdef GGML_OPENCL_USE_ADRENO_KERNELS + // mul_mm_f16_f32_kq_p8 + if (backend_ctx->gpu_family == ADRENO && backend_ctx->has_integer_dot) { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "mul_mm_f16_f32_kq_p8.cl.h" + }; +#else + const std::string kernel_src = read_file("mul_mm_f16_f32_kq_p8.cl"); +#endif + cl_program prog = build_program_from_source_ex(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), + compile_opts, /*fatal=*/false); + if (prog != nullptr) { + CL_CHECK((backend_ctx->kernel_mul_mm_f16_f32_kq_p8 = clCreateKernel(prog, "mul_mm_f16_f32_kq_p8", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + } + GGML_LOG_CONT("."); + } +#endif + // mul { #ifdef GGML_OPENCL_EMBED_KERNELS @@ -4040,9 +4211,51 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) { build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); CL_CHECK((backend_ctx->kernel_soft_max_4_f16 = clCreateKernel(backend_ctx->program_softmax_4_f16, "kernel_soft_max_4_f16", &err), err)); + CL_CHECK((backend_ctx->kernel_fa_decompose_soft_max_4_f16 = clCreateKernel(backend_ctx->program_softmax_4_f16, "kernel_fa_decompose_soft_max_4_f16", &err), err)); + CL_CHECK((backend_ctx->kernel_fa_decompose_scale_rows_f32 = clCreateKernel(backend_ctx->program_softmax_4_f16, "kernel_fa_decompose_scale_rows_f32", &err), err)); + CL_CHECK((backend_ctx->kernel_fa_p8_fixup = clCreateKernel(backend_ctx->program_softmax_4_f16, "kernel_fa_p8_fixup", &err), err)); GGML_LOG_CONT("."); } +#ifdef GGML_OPENCL_USE_ADRENO_KERNELS + if (backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E || + backend_ctx->adreno_gen == ADRENO_GPU_GEN::A8X || + backend_ctx->adreno_gen == ADRENO_GPU_GEN::X1E) { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "flash_attn_pv_direct.cl.h" + }; +#else + const std::string kernel_src = read_file("flash_attn_pv_direct.cl"); +#endif + cl_program prog = build_program_from_source_ex(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), + compile_opts, /*fatal=*/false); + if (prog != nullptr) { + CL_CHECK((backend_ctx->kernel_fa_kqv_direct_f16 = clCreateKernel(prog, "kernel_fa_kqv_direct_f16", &err), err)); + CL_CHECK((backend_ctx->kernel_fa_kqv_direct_reduce = clCreateKernel(prog, "kernel_fa_kqv_direct_reduce", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + } + GGML_LOG_CONT("."); + } + + if (backend_ctx->gpu_family == ADRENO && backend_ctx->has_integer_dot) { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "mul_mm_q8_f32_kqv.cl.h" + }; +#else + const std::string kernel_src = read_file("mul_mm_q8_f32_kqv.cl"); +#endif + cl_program prog = build_program_from_source_ex(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), + compile_opts, /*fatal=*/false); + if (prog != nullptr) { + CL_CHECK((backend_ctx->kernel_mul_mm_q8_f32_kqv = clCreateKernel(prog, "kernel_mul_mm_q8_f32_kqv", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + } + GGML_LOG_CONT("."); + } +#endif + // div { #ifdef GGML_OPENCL_EMBED_KERNELS @@ -4145,6 +4358,28 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) { GGML_LOG_CONT("."); } + // dsv4_hc + { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "dsv4_hc.cl.h" + }; +#else + const std::string kernel_src = read_file("dsv4_hc.cl"); +#endif + for (int i = 0; i < 2; ++i) { + cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts + (i == 0 ? " -DHC_VEC=1" : " -DHC_VEC=4")); + if (i == 0) { + CL_CHECK((backend_ctx->kernel_dsv4_hc_pre = clCreateKernel(prog, "kernel_dsv4_hc_pre", &err), err)); + backend_ctx->local_dsv4_hc_pre = MIN((size_t) 64, backend_ctx->get_kernel_workgroup_size(backend_ctx->kernel_dsv4_hc_pre)); + } + CL_CHECK((backend_ctx->kernel_dsv4_hc_post[i] = clCreateKernel(prog, "kernel_dsv4_hc_post", &err), err)); + backend_ctx->local_dsv4_hc_post[i] = MIN((size_t) 64, backend_ctx->get_kernel_workgroup_size(backend_ctx->kernel_dsv4_hc_post[i])); + CL_CHECK(clReleaseProgram(prog)); + } + GGML_LOG_CONT("."); + } + // sum_rows { #ifdef GGML_OPENCL_EMBED_KERNELS @@ -4194,6 +4429,8 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) { CL_CHECK((backend_ctx->kernel_sigmoid_f32 = clCreateKernel(backend_ctx->program_sigmoid, "kernel_sigmoid_f32", &err), err)); CL_CHECK((backend_ctx->kernel_sigmoid_f16 = clCreateKernel(backend_ctx->program_sigmoid, "kernel_sigmoid_f16", &err), err)); + CL_CHECK((backend_ctx->kernel_sigmoid_f32_nc = clCreateKernel(backend_ctx->program_sigmoid, "kernel_sigmoid_f32_nc", &err), err)); + CL_CHECK((backend_ctx->kernel_sigmoid_f16_nc = clCreateKernel(backend_ctx->program_sigmoid, "kernel_sigmoid_f16_nc", &err), err)); GGML_LOG_CONT("."); } @@ -4226,6 +4463,8 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) { cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); CL_CHECK((backend_ctx->kernel_repeat_f32 = clCreateKernel(prog, "kernel_repeat_f32", &err), err)); + CL_CHECK((backend_ctx->kernel_repeat_f16 = clCreateKernel(prog, "kernel_repeat_f16", &err), err)); + CL_CHECK((backend_ctx->kernel_repeat_i32 = clCreateKernel(prog, "kernel_repeat_i32", &err), err)); CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); } @@ -4728,6 +4967,36 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) { GGML_LOG_CONT("."); } + // mul_mv_id_f32_f32 + { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "mul_mv_id_f32_f32.cl.h" + }; +#else + const std::string kernel_src = read_file("mul_mv_id_f32_f32.cl"); +#endif + cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); + CL_CHECK((backend_ctx->kernel_mul_mv_id_f32_f32 = clCreateKernel(prog, "kernel_mul_mv_id_f32_f32", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); + } + + // mul_mm_id_f32_f32 + { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "mul_mm_id_f32_f32.cl.h" + }; +#else + const std::string kernel_src = read_file("mul_mm_id_f32_f32.cl"); +#endif + cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); + CL_CHECK((backend_ctx->kernel_mul_mm_id_f32_f32 = clCreateKernel(prog, "kernel_mul_mm_id_f32_f32", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); + } + // mul_mv_id_q4_0_f32_8x_flat { #ifdef GGML_OPENCL_EMBED_KERNELS @@ -4915,6 +5184,39 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) { GGML_LOG_CONT("."); } + // gemm_noshuffle_iq4_xs_q8_1_dp4a + { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "gemm_noshuffle_iq4_xs_q8_1_dp4a.cl.h" + }; +#else + const std::string kernel_src = read_file("gemm_noshuffle_iq4_xs_q8_1_dp4a.cl"); +#endif + cl_program prog = build_program_from_source(backend_ctx, (iq4_xs_grid_src + kernel_src).c_str(), compile_opts); + CL_CHECK((backend_ctx->kernel_gemm_noshuffle_iq4_xs_q8_1_dp4a = clCreateKernel(prog, "kernel_gemm_noshuffle_iq4_xs_q8_1_dp4a", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + prog = build_program_from_source(backend_ctx, (iq4_xs_grid_src + kernel_src).c_str(), compile_opts + " -DTILESIZE_N=8"); + CL_CHECK((backend_ctx->kernel_gemm_noshuffle_iq4_xs_q8_1_dp4a_narrow = clCreateKernel(prog, "kernel_gemm_noshuffle_iq4_xs_q8_1_dp4a", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); + } + + // gemm_noshuffle_iq4_xs_f32 + { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "gemm_noshuffle_iq4_xs_f32.cl.h" + }; +#else + const std::string kernel_src = read_file("gemm_noshuffle_iq4_xs_f32.cl"); +#endif + cl_program prog = build_program_from_source(backend_ctx, (iq4_xs_grid_src + kernel_src).c_str(), compile_opts); + CL_CHECK((backend_ctx->kernel_gemm_noshuffle_iq4_xs_f32 = clCreateKernel(prog, "kernel_gemm_noshuffle_iq4_xs_f32", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); + } + // gemm_noshuffle_q2_0_f32 { #ifdef GGML_OPENCL_EMBED_KERNELS @@ -4930,6 +5232,41 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) { GGML_LOG_CONT("."); } + // gemv_noshuffle_iq4_xs_f32 + { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "gemv_noshuffle_iq4_xs_f32.cl.h" + }; +#else + const std::string kernel_src = read_file("gemv_noshuffle_iq4_xs_f32.cl"); +#endif + if (const char * env = getenv("GGML_OPENCL_IQ4_XS_MV_NSG")) { + const int nsg = atoi(env); + if (nsg == 1 || nsg == 2 || nsg == 4 || nsg == 8) { + backend_ctx->iq4_xs_mv_nsg = nsg; + } + } + for (;;) { + const std::string opts = compile_opts + " -DIQ4_XS_MV_NSG=" + std::to_string(backend_ctx->iq4_xs_mv_nsg); + cl_program prog = build_program_from_source(backend_ctx, (iq4_xs_grid_src + kernel_src).c_str(), opts); + cl_kernel kernel = clCreateKernel(prog, "kernel_gemv_noshuffle_iq4_xs_f32", &err); + CL_CHECK(err); + size_t max_workgroup; + CL_CHECK(clGetKernelWorkGroupInfo(kernel, backend_ctx->device, CL_KERNEL_WORK_GROUP_SIZE, sizeof(max_workgroup), &max_workgroup, nullptr)); + if (64*(size_t)backend_ctx->iq4_xs_mv_nsg <= max_workgroup) { + backend_ctx->kernel_gemv_noshuffle_iq4_xs_f32 = kernel; + CL_CHECK(clReleaseProgram(prog)); + break; + } + CL_CHECK(clReleaseKernel(kernel)); + CL_CHECK(clReleaseProgram(prog)); + GGML_ASSERT(backend_ctx->iq4_xs_mv_nsg > 1); + backend_ctx->iq4_xs_mv_nsg /= 2; + } + GGML_LOG_CONT("."); + } + // gemv_noshuffle_q2_0_f32 { #ifdef GGML_OPENCL_EMBED_KERNELS @@ -4941,25 +5278,30 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) { #endif if (const char * env = getenv("GGML_OPENCL_Q2_0_MV_NSG")) { const int nsg = atoi(env); - if (nsg == 1 || nsg == 2 || nsg == 4 || nsg == 8) { + if (nsg == 1 || nsg == 2 || nsg == 4 || nsg == 8 || nsg == 16 || nsg == 32) { backend_ctx->q2_0_mv_nsg = nsg; } } - for (;;) { - const std::string opts = compile_opts + " -DQ2_0_MV_NSG=" + std::to_string(backend_ctx->q2_0_mv_nsg); + for (int wide = 0; wide < 2; ++wide) { + const std::string opts = compile_opts + " -DQ2_0_MV_WIDE=" + std::to_string(wide); cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), opts); cl_kernel kernel = clCreateKernel(prog, "kernel_gemv_noshuffle_q2_0_f32", &err); CL_CHECK(err); CL_CHECK(clReleaseProgram(prog)); size_t max_workgroup; CL_CHECK(clGetKernelWorkGroupInfo(kernel, backend_ctx->device, CL_KERNEL_WORK_GROUP_SIZE, sizeof(max_workgroup), &max_workgroup, nullptr)); - if (64*(size_t)backend_ctx->q2_0_mv_nsg <= max_workgroup) { + size_t workgroup = 512; + while (workgroup > max_workgroup) { + workgroup /= 2; + } + GGML_ASSERT(workgroup >= 64); + if (wide) { + backend_ctx->kernel_gemv_noshuffle_q2_0_f32_wide = kernel; + backend_ctx->q2_0_mv_workgroup_size_wide = workgroup; + } else { backend_ctx->kernel_gemv_noshuffle_q2_0_f32 = kernel; - break; + backend_ctx->q2_0_mv_workgroup_size = workgroup; } - CL_CHECK(clReleaseKernel(kernel)); - GGML_ASSERT(backend_ctx->q2_0_mv_nsg > 1); - backend_ctx->q2_0_mv_nsg /= 2; } GGML_LOG_CONT("."); } @@ -5806,6 +6148,7 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) { backend_ctx->kernel_gemv_noshuffle_q4_0_f32_32b_trans = nullptr; backend_ctx->kernel_gemm_noshuffle_q4_0_f32_32b_trans_ila_a8_bin = nullptr; + backend_ctx->kernel_gemm_noshuffle_q4_0_f32_32b_trans_ila_a7_bin = nullptr; backend_ctx->kernel_gemm_noshuffle_q4_0_q8_1_dp4a_ila_a8_bin = nullptr; { { @@ -5850,6 +6193,17 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) { CL_CHECK(clReleaseProgram(bin_prog)); GGML_LOG_CONT("."); } + + kernel_bin = (const char *)backend_ctx->get_adreno_bin_kernel("gemm_noshuffle_q4_0_f32_32b_trans_ila_a7", &bin_size); + if (kernel_bin && bin_size > 0) { + cl_program bin_prog = + build_program_from_binary(backend_ctx->context, backend_ctx->device, kernel_bin, "", bin_size); + + CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q4_0_f32_32b_trans_ila_a7_bin = + clCreateKernel(bin_prog, "kernel_gemm_noshuffle_q4_0_f32_32b_trans_ila_a7", &err), err)); + CL_CHECK(clReleaseProgram(bin_prog)); + GGML_LOG_CONT("."); + } } } @@ -6121,6 +6475,7 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) { CL_CHECK((backend_ctx->kernel_gemv_noshuffle_q8_0_f32 = clCreateKernel(prog, "kernel_gemv_noshuffle_q8_0_f32", &err), err)); CL_CHECK((backend_ctx->kernel_gemv_noshuffle_q8_0_f32_splitk = clCreateKernel(prog, "kernel_gemv_noshuffle_q8_0_f32_splitk", &err), err)); + CL_CHECK((backend_ctx->kernel_gemv_noshuffle_q8_0_f32_mc = clCreateKernel(prog, "kernel_gemv_noshuffle_q8_0_f32_mc", &err), err)); CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); } @@ -6256,16 +6611,13 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) { if (backend_ctx->adreno_gen == ADRENO_GPU_GEN::A6X) { CL_gemv_compile_opts += " -DGGML_CL_A6X_CONSTFOLD_FIX"; } - // Opt-in: dequant-once-per-block mc3 verify GEMV (factors q4_K dequant - // out of the 3-column loop; byte-identical, lower spill). A/B vs the - // shipped inline mc3 in the same binary. - if (getenv("GGML_OPENCL_Q4K_MC3_DQ")) { + // dequant-once-per-block mc3 verify GEMV + if (getenv("GGML_OPENCL_Q4_K_MC3_DQ")) { CL_gemv_compile_opts += " -DQ4K_MC3_DEQUANT_ONCE "; } - // Opt-in: LDS-staged dequant mc3 verify GEMV (stages the dequantized - // q4_K weights in __local instead of private regs that spill to slow - // global on Adreno; byte-identical). A/B vs inline + dequant-once. - if (getenv("GGML_OPENCL_Q4K_MC3_LDS")) { + // LM-staged dequant mc3 verify GEMV, stages the dequantized + // q4_K weights in __local instead of regs + if (getenv("GGML_OPENCL_Q4_K_MC3_LDS")) { CL_gemv_compile_opts += " -DQ4K_MC3_DEQUANT_LDS "; } @@ -7145,27 +7497,6 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) { GGML_LOG_CONT("."); } - // moe_sort_by_expert - { -#ifdef GGML_OPENCL_EMBED_KERNELS - const std::string kernel_src { - #include "moe_sort_by_expert.cl.h" - }; -#else - const std::string kernel_src = read_file("moe_sort_by_expert.cl"); -#endif - cl_program prog = - build_program_from_source(backend_ctx, kernel_src.c_str(), CL_moe_compile_opts); - - CL_CHECK((backend_ctx->kernel_moe_histogram = clCreateKernel(prog, "kernel_moe_histogram", &err), err)); - CL_CHECK((backend_ctx->kernel_moe_scan = clCreateKernel(prog, "kernel_moe_scan", &err), err)); - CL_CHECK((backend_ctx->kernel_moe_fill = clCreateKernel(prog, "kernel_moe_fill", &err), err)); - CL_CHECK((backend_ctx->kernel_moe_scatter = clCreateKernel(prog, "kernel_moe_scatter", &err), err)); - CL_CHECK((backend_ctx->kernel_moe_scatter_stable = clCreateKernel(prog, "kernel_moe_scatter_stable", &err), err)); - CL_CHECK(clReleaseProgram(prog)); - GGML_LOG_CONT("."); - } - // gemv_noshuffle_q6_k_f32 { #ifdef GGML_OPENCL_EMBED_KERNELS @@ -7187,17 +7518,6 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) { CL_CHECK((backend_ctx->kernel_gemv_noshuffle_q6_K_f32 = clCreateKernel(prog, "kernel_gemv_noshuffle_q6_K_f32", &err), err)); CL_CHECK((backend_ctx->kernel_gemv_noshuffle_q6_K_f32_mc3 = clCreateKernel(prog, "kernel_gemv_noshuffle_q6_K_f32_mc3", &err), err)); - if (getenv("GGML_OPENCL_MC3_PROBE")) { - cl_ulong pm6 = 0, pm4 = 0; size_t wg6 = 0, wg4 = 0, mult = 0; - clGetKernelWorkGroupInfo(backend_ctx->kernel_gemv_noshuffle_q6_K_f32_mc3, backend_ctx->device, CL_KERNEL_PRIVATE_MEM_SIZE, sizeof(pm6), &pm6, NULL); - clGetKernelWorkGroupInfo(backend_ctx->kernel_gemv_noshuffle_q6_K_f32_mc3, backend_ctx->device, CL_KERNEL_WORK_GROUP_SIZE, sizeof(wg6), &wg6, NULL); - clGetKernelWorkGroupInfo(backend_ctx->kernel_gemv_noshuffle_q4_k_f32_mc3, backend_ctx->device, CL_KERNEL_PRIVATE_MEM_SIZE, sizeof(pm4), &pm4, NULL); - clGetKernelWorkGroupInfo(backend_ctx->kernel_gemv_noshuffle_q4_k_f32_mc3, backend_ctx->device, CL_KERNEL_WORK_GROUP_SIZE, sizeof(wg4), &wg4, NULL); - clGetKernelWorkGroupInfo(backend_ctx->kernel_gemv_noshuffle_q6_K_f32_mc3, backend_ctx->device, CL_KERNEL_PREFERRED_WORK_GROUP_SIZE_MULTIPLE, sizeof(mult), &mult, NULL); - fprintf(stderr, "[MC3-PROBE] q4K_mc3 private=%llu wg_cap=%zu | q6K_mc3 private=%llu wg_cap=%zu | pref_mult=%zu\n", - (unsigned long long)pm4, wg4, (unsigned long long)pm6, wg6, mult); - fflush(stderr); - } GGML_LOG_CONT("."); } @@ -7592,6 +7912,17 @@ static void ggml_opencl_ensure_fa_pre_kernels(ggml_backend_opencl_context * back backend_ctx->fa.kv_pad_f16[{dk, dv}] = k_kv_pad_f16; backend_ctx->fa.mask_pad_f16[{dk, dv}] = k_mask_pad_f16; backend_ctx->fa.blk_f16[{dk, dv}] = k_blk_f16; + // optional, the decomposed prefill checks for it and falls back to the fused tile + cl_kernel k_v_transpose_f16 = clCreateKernel(prog_pre_f16, "flash_attn_v_transpose_f16", &err); + if (err == CL_SUCCESS) { + backend_ctx->fa.v_transpose_f16[{dk, dv}] = k_v_transpose_f16; + } + if (backend_ctx->kernel_fa_v_transpose_q8 == nullptr && backend_ctx->has_integer_dot) { + backend_ctx->kernel_fa_v_transpose_q8 = clCreateKernel(prog_pre_f16, "flash_attn_v_transpose_q8", &err); + if (err != CL_SUCCESS) { + backend_ctx->kernel_fa_v_transpose_q8 = nullptr; + } + } clReleaseProgram(prog_pre_f16); } @@ -7680,7 +8011,8 @@ static void ggml_opencl_ensure_fa_f32_f16_mq_decode(ggml_backend_opencl_context } const bool cluster = dk == 64 || dk == 128; - const int head_sub = cluster ? 2 : (gqa == 8 ? (dk == 512 ? 4 : 2) : 1); + const int head_sub = gqa == 16 ? (cluster ? 4 : 8) : + cluster ? 2 : (gqa == 8 ? (dk == 512 ? 4 : 2) : 1); const int nsg_max = dk == 64 ? 1 : (dk == 128 || (gqa == 8 && dk == 256) ? 2 : 4); const char * kernel_name = cluster ? "flash_attn_f32_f16_q1_vec_mq_split_c8" : "flash_attn_f32_f16_q1_vec_mq_split"; const std::string src = ggml_opencl_fa_kernel_src(FA_VARIANT_F32_F16); @@ -7688,7 +8020,7 @@ static void ggml_opencl_ensure_fa_f32_f16_mq_decode(ggml_backend_opencl_context " -D FA_MQ_ONLY -D MQ_GQA=" + std::to_string(gqa / head_sub) + " -D FA_HEAD_SUB=" + std::to_string(head_sub) + (cluster ? " -D MQ_NSG=" + std::to_string(nsg_max) + " -D FA_CL_C=16" : " -D FA_MQ_SPLIT_ONLY") + - (dk == 64 ? " -D FA_CL_MHRED -D FA_CL_MASK_BCAST" : "") + + (dk == 64 ? " -D FA_CL_MHRED -D FA_CL_MASK_BCAST" : (cluster && gqa == 16 ? " -D FA_CL_MHRED" : "")) + (gqa == 8 && dk == 256 ? " -D FA_Q1_Q_REG" : ""); for (int nsg = nsg_max; nsg >= 1; nsg /= 2) { const size_t wg = 64 * nsg; @@ -9848,6 +10180,27 @@ struct ggml_tensor_extra_cl_iq1_m { } }; +struct ggml_tensor_extra_cl_iq4_xs { + bool noshuffle = false; + cl_mem qs = nullptr; + cl_mem sl = nullptr; + cl_mem sh = nullptr; + cl_mem d = nullptr; + + size_t size_qs = 0, size_sl = 0, size_sh = 0, size_d = 0; + + ~ggml_tensor_extra_cl_iq4_xs() { reset(); } + + void reset() { + noshuffle = false; + if (qs != nullptr) { CL_CHECK(clReleaseMemObject(qs)); qs = nullptr; } + if (sl != nullptr) { CL_CHECK(clReleaseMemObject(sl)); sl = nullptr; } + if (sh != nullptr) { CL_CHECK(clReleaseMemObject(sh)); sh = nullptr; } + if (d != nullptr) { CL_CHECK(clReleaseMemObject(d)); d = nullptr; } + size_qs = size_sl = size_sh = size_d = 0; + } +}; + struct ggml_tensor_extra_cl_q6_K { // Lower 4 bits of quantized weights. cl_mem ql = nullptr; @@ -10663,7 +11016,8 @@ static void ggml_cl_mul_mat_q4_k_glu_fused(ggml_backend_t backend, ggml_tensor * // byte-identical to the per-op path. Keep the maxwg query as a further floor // for any driver that reports < 512. size_t maxwg = backend_ctx->get_kernel_workgroup_size(kernel); - size_t nsg_y = 8; + // X2E: 4 is faster than 8 on every q4_K FFN shape measured there + size_t nsg_y = backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E ? 4 : 8; while (nsg_y > 1 && 64 * nsg_y > maxwg) { nsg_y >>= 1; } size_t local_work_size[3] = { 64, nsg_y, 1 }; size_t global_work_size[3] = { (size_t)CEIL_DIV(M / 2, 64) * 64, nsg_y, 1 }; @@ -10895,6 +11249,13 @@ inline bool use_adreno_kernels(const ggml_backend_opencl_context *backend_ctx, c threshold_ne0 = 128; threshold_ne1 = 128; } + // K floor 256 on X2E: the gemma-4 E2B/E4B MTP drafters' weights are all K = 256. + // Default on for X2E, GGML_OPENCL_K_FLOOR_256=0/1 forces it. + static const char * k_floor_256_env = getenv("GGML_OPENCL_K_FLOOR_256"); + const bool k_floor_256 = k_floor_256_env ? atoi(k_floor_256_env) != 0 : backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E; + if (k_floor_256 && threshold_ne0 > 256) { + threshold_ne0 = 256; + } bool threashold_ok = tensor->ne[0] >= threshold_ne0 && tensor->ne[1] >= threshold_ne1 && tensor->ne[2] == 1 && tensor->ne[3] == 1; @@ -10966,6 +11327,24 @@ static bool ggml_cl_q3_k_use_noshuffle(const ggml_backend_opencl_context * backe #endif } +static bool ggml_cl_iq4_xs_use_noshuffle(const ggml_backend_opencl_context * backend_ctx, const ggml_tensor * tensor) { +#if defined(GGML_OPENCL_SOA_Q) && defined(GGML_OPENCL_USE_ADRENO_KERNELS) + if (const char * env = getenv("GGML_OPENCL_IQ4_XS_NOSHUFFLE")) { + if (env[0] == '0') { + return false; + } + } + return tensor->type == GGML_TYPE_IQ4_XS && + backend_ctx->gpu_family == ADRENO && + ggml_is_contiguous(tensor) && !tensor->view_src && + use_adreno_kernels(backend_ctx, tensor); +#else + GGML_UNUSED(backend_ctx); + GGML_UNUSED(tensor); + return false; +#endif +} + static bool ggml_cl_iq3_s_use_noshuffle(const ggml_backend_opencl_context * backend_ctx, const ggml_tensor * tensor) { #if defined(GGML_OPENCL_SOA_Q) && defined(GGML_OPENCL_USE_ADRENO_KERNELS) if (const char * env = getenv("GGML_OPENCL_IQ3_S_NOSHUFFLE")) { @@ -11230,7 +11609,8 @@ inline bool enable_adreno_trans_weight_q5_K(const ggml_backend_opencl_context *b inline bool use_q4_0_bin_kernels(const ggml_backend_opencl_context *backend_ctx, const ggml_tensor *tensor) { #ifdef GGML_OPENCL_USE_ADRENO_KERNELS if (!backend_ctx->kernel_gemv_noshuffle_q4_0_f32_32b_trans || - !backend_ctx->kernel_gemm_noshuffle_q4_0_f32_32b_trans_ila_a8_bin) { + !(backend_ctx->kernel_gemm_noshuffle_q4_0_f32_32b_trans_ila_a8_bin || + backend_ctx->kernel_gemm_noshuffle_q4_0_f32_32b_trans_ila_a7_bin)) { return false; } @@ -11257,6 +11637,13 @@ static bool use_fa_bin_kernels_prefill(const ggml_backend_opencl_context * backe const int dk = q->ne[0]; const int dv = v->ne[0]; + // do not choose bin FA kernels to avoid repacking the KV cache for small + // verify batches on X2 + if (backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E && is_mixed && + (dk == 256 || dk == 512) && n_q <= 4) { + return false; + } + constexpr bool prefill_only = true; return (backend_ctx->gpu_family == GPU_FAMILY::ADRENO && @@ -11529,7 +11916,8 @@ static bool ggml_opencl_supports_op(ggml_backend_dev_t dev, const struct ggml_te case GGML_UNARY_OP_GELU_QUICK: return ggml_is_contiguous(op->src[0]) && op->src[0]->type == GGML_TYPE_F32; case GGML_UNARY_OP_SIGMOID: - return ggml_is_contiguous(op->src[0]); + return op->src[0]->type == op->type && + (op->type == GGML_TYPE_F32 || op->type == GGML_TYPE_F16 || op->type == GGML_TYPE_BF16); case GGML_UNARY_OP_TANH: case GGML_UNARY_OP_NEG: case GGML_UNARY_OP_EXP: @@ -11570,6 +11958,8 @@ static bool ggml_opencl_supports_op(ggml_backend_dev_t dev, const struct ggml_te return op->type == GGML_TYPE_F32 && ggml_is_contiguous(op); case GGML_OP_FILL: return op->type == GGML_TYPE_F32 && ggml_is_contiguous(op); + case GGML_OP_ARANGE: + return op->type == GGML_TYPE_F32; case GGML_OP_CLAMP: return op->src[0]->type == GGML_TYPE_F32; case GGML_OP_SOFT_MAX: @@ -11580,7 +11970,16 @@ static bool ggml_opencl_supports_op(ggml_backend_dev_t dev, const struct ggml_te case GGML_OP_L2_NORM: return ggml_is_contiguous_rows(op->src[0]); case GGML_OP_REPEAT: - return op->src[0]->type == GGML_TYPE_F32 && op->type == GGML_TYPE_F32; // Assuming F32 for now, can be expanded + switch (op->type) { + case GGML_TYPE_F32: + case GGML_TYPE_F16: + case GGML_TYPE_I32: + case GGML_TYPE_I16: + case GGML_TYPE_BF16: + return op->src[0]->type == op->type; + default: + return false; + } case GGML_OP_PAD: // TODO: add circular padding support for opencl, see https://github.com/ggml-org/llama.cpp/pull/16985 if (ggml_get_op_params_i32(op, 8) != 0) { @@ -11637,6 +12036,9 @@ static bool ggml_opencl_supports_op(ggml_backend_dev_t dev, const struct ggml_te case GGML_OP_GROUP_NORM: return ggml_is_contiguous(op->src[0]); case GGML_OP_MUL_MAT: + if (ggml_prec_get_src(op, 1) == GGML_PREC_BF16 || ggml_prec_get_src(op, 1) == GGML_PREC_F32) { + return false; + } if (ggml_cl_mul_mat_can_use_fwht(backend_ctx, op)) { return true; } @@ -11699,6 +12101,12 @@ static bool ggml_opencl_supports_op(ggml_backend_dev_t dev, const struct ggml_te return false; } return op->src[1]->type == GGML_TYPE_F32 && ggml_is_contiguous_rows(op->src[1]); + } else if (op->src[0]->type == GGML_TYPE_IQ4_XS) { + const ggml_tensor * weight = op->src[0]->view_src; + if (weight && ggml_cl_iq4_xs_use_noshuffle(backend_ctx, weight)) { + return false; + } + return op->src[1]->type == GGML_TYPE_F32 && ggml_is_contiguous_rows(op->src[1]); } else if (op->src[0]->type == GGML_TYPE_Q3_K) { const ggml_tensor * weight = op->src[0]->view_src; if (weight && ggml_cl_q3_k_use_noshuffle(backend_ctx, weight)) { @@ -11743,8 +12151,14 @@ static bool ggml_opencl_supports_op(ggml_backend_dev_t dev, const struct ggml_te } return false; case GGML_OP_MUL_MAT_ID: + if (op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_F16 || op->src[0]->type == GGML_TYPE_BF16) { + return op->src[1]->type == GGML_TYPE_F32 && op->src[2]->type == GGML_TYPE_I32 && op->type == GGML_TYPE_F32 && + op->src[0]->nb[0] == ggml_type_size(op->src[0]->type) && op->src[1]->nb[0] == sizeof(float) && + op->nb[0] == sizeof(float) && op->src[0]->ne[0] <= INT_MAX - 256 && op->ne[0] <= INT_MAX - 64 && + op->src[1]->ne[1] <= INT_MAX && op->ne[1] <= INT_MAX && op->ne[2] <= INT_MAX - 64; + } #ifdef GGML_OPENCL_USE_ADRENO_KERNELS - if (ggml_get_op_params_i32(op, 3) == GGML_PREC_F32 && + if ((ggml_prec_get_src(op, 1) == GGML_PREC_BF16 || ggml_prec_get_src(op, 1) == GGML_PREC_F32) && use_adreno_moe_kernels(backend_ctx, op->src[0])) { return false; } @@ -11781,10 +12195,11 @@ static bool ggml_opencl_supports_op(ggml_backend_dev_t dev, const struct ggml_te return true; case GGML_OP_DIAG: return true; - case GGML_OP_DIAG_MASK_INF: - return op->ne[3] == 1; case GGML_OP_ROPE: { const int mode = ((const int32_t *) op->op_params)[2]; + if (!ggml_rope_get_truncate(op)) { + return false; + } const bool is_mrope = mode & GGML_ROPE_TYPE_MROPE; const bool is_vision = mode == GGML_ROPE_TYPE_VISION; if (is_mrope && !is_vision) { @@ -11846,6 +12261,14 @@ static bool ggml_opencl_supports_op(ggml_backend_dev_t dev, const struct ggml_te case GGML_OP_SUM_ROWS: case GGML_OP_CUMSUM: return op->src[0]->type == GGML_TYPE_F32 && ggml_is_contiguous(op->src[0]); + case GGML_OP_DSV4_HC_PRE: + case GGML_OP_DSV4_HC_POST: + for (int i = 0; i < (op->op == GGML_OP_DSV4_HC_PRE ? 2 : 4); ++i) { + if (op->src[i] && op->src[i]->type != GGML_TYPE_F32) { + return false; + } + } + return op->type == GGML_TYPE_F32 && ggml_is_contiguous(op); case GGML_OP_MEAN: return op->src[0]->type == GGML_TYPE_F32; case GGML_OP_FLASH_ATTN_EXT: { @@ -11948,9 +12371,13 @@ static bool ggml_opencl_supports_op(ggml_backend_dev_t dev, const struct ggml_te } if (q->ne[1] == 1) { const char * decode_env = getenv("GGML_OPENCL_FA_DK512_DECODE"); + const int64_t gqa = k->ne[2] > 0 ? q->ne[2] / k->ne[2] : 0; + // GQA 2 and 16 decode on the X2E MQ kernels (see mq_decode_x2e in ggml_cl_flash_attn) + const bool gqa_ok = gqa == 4 || gqa == 8 || + (backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E && (gqa == 2 || gqa == 16)); if ((decode_env && decode_env[0] == '0') || backend_ctx->gpu_family != ADRENO || k->ne[2] <= 0 || - (q->ne[2] / k->ne[2] != 4 && q->ne[2] / k->ne[2] != 8) || q->ne[2] % k->ne[2] != 0 || + !gqa_ok || q->ne[2] % k->ne[2] != 0 || !ggml_opencl_ensure_fa_f32_f16_vec_512(backend_ctx)) { return false; } @@ -12130,6 +12557,9 @@ struct ggml_backend_opencl_buffer_context { for (ggml_tensor_extra_cl_iq3_xxs * e : temp_tensor_extras_iq3_xxs) { delete e; } + for (ggml_tensor_extra_cl_iq4_xs * e : temp_tensor_extras_iq4_xs) { + delete e; + } for (ggml_tensor_extra_cl_q6_K * e : temp_tensor_extras_q6_K_in_use) { delete e; } @@ -12163,6 +12593,9 @@ struct ggml_backend_opencl_buffer_context { for (ggml_tensor_extra_cl_iq3_xxs * e : temp_tensor_extras_iq3_xxs_in_use) { delete e; } + for (ggml_tensor_extra_cl_iq4_xs * e : temp_tensor_extras_iq4_xs_in_use) { + delete e; + } for (ggml_tensor_extra_cl_q5_K * e : temp_tensor_extras_q5_K) { delete e; } @@ -12481,6 +12914,20 @@ struct ggml_backend_opencl_buffer_context { } temp_tensor_extras_iq3_xxs_in_use.push_back(extra); + extra->reset(); + return extra; + } + + ggml_tensor_extra_cl_iq4_xs * ggml_opencl_alloc_temp_tensor_extra_iq4_xs() { + ggml_tensor_extra_cl_iq4_xs * extra; + if (temp_tensor_extras_iq4_xs.empty()) { + extra = new ggml_tensor_extra_cl_iq4_xs(); + } else { + extra = temp_tensor_extras_iq4_xs.back(); + temp_tensor_extras_iq4_xs.pop_back(); + } + + temp_tensor_extras_iq4_xs_in_use.push_back(extra); extra->reset(); return extra; @@ -12611,6 +13058,10 @@ struct ggml_backend_opencl_buffer_context { temp_tensor_extras_iq3_xxs.push_back(e); } temp_tensor_extras_iq3_xxs_in_use.clear(); + for (ggml_tensor_extra_cl_iq4_xs * e : temp_tensor_extras_iq4_xs_in_use) { + temp_tensor_extras_iq4_xs.push_back(e); + } + temp_tensor_extras_iq4_xs_in_use.clear(); q8_0_soa_tensors.clear(); q4_0_soa_tensors.clear(); @@ -12663,6 +13114,8 @@ struct ggml_backend_opencl_buffer_context { std::vector temp_tensor_extras_iq3_s; std::vector temp_tensor_extras_iq3_s_in_use; std::vector temp_tensor_extras_iq3_xxs_in_use; + std::vector temp_tensor_extras_iq4_xs; + std::vector temp_tensor_extras_iq4_xs_in_use; std::vector temp_tensor_extras_q6_K; std::vector temp_tensor_extras_q6_K_in_use; @@ -14599,6 +15052,87 @@ static void ggml_backend_opencl_buffer_set_tensor(ggml_backend_buffer_t buffer, #endif return; } + if (tensor->type == GGML_TYPE_IQ4_XS) { + GGML_ASSERT(offset == 0 && size == ggml_nbytes(tensor)); + ggml_tensor_extra_cl * extra_orig = (ggml_tensor_extra_cl *)tensor->extra; + GGML_ASSERT(extra_orig && "Tensors in OpenCL backend should have been allocated and initialized"); + + ggml_backend_opencl_buffer_context * ctx = (ggml_backend_opencl_buffer_context *) buffer->context; + ggml_tensor_extra_cl_iq4_xs * extra = ctx->ggml_opencl_alloc_temp_tensor_extra_iq4_xs(); + extra->noshuffle = ggml_cl_iq4_xs_use_noshuffle(backend_ctx, tensor); + + const size_t blck = (size_t)ggml_blck_size(tensor->type); + const size_t n_blk = ggml_nelements(tensor) / blck; + + const size_t size_qs = n_blk * 128; + const size_t size_sl = n_blk * 4; + const size_t size_sh = n_blk * 2; + const size_t size_d = n_blk * 2; + GGML_ASSERT(size_qs + size_sl + size_sh + size_d == ggml_nbytes(tensor) && "Incorrect tensor size"); + + cl_int err; + cl_mem data_device = clCreateBuffer(context, CL_MEM_READ_WRITE, + ggml_nbytes(tensor), NULL, &err); + CL_CHECK(err); + CL_CHECK(clEnqueueWriteBuffer( + queue, data_device, CL_TRUE, 0, + ggml_nbytes(tensor), data, 0, NULL, NULL)); + + cl_buffer_region region; + region.origin = align_to(extra_orig->offset + tensor->view_offs + offset, backend_ctx->alignment); + region.size = size_qs; + CL_CHECK((extra->qs = clCreateSubBuffer(extra_orig->data_device, CL_MEM_READ_WRITE, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); + + region.origin = align_to(region.origin + size_qs, backend_ctx->alignment); + region.size = size_sl; + CL_CHECK((extra->sl = clCreateSubBuffer(extra_orig->data_device, CL_MEM_READ_WRITE, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); + + region.origin = align_to(region.origin + size_sl, backend_ctx->alignment); + region.size = size_sh; + CL_CHECK((extra->sh = clCreateSubBuffer(extra_orig->data_device, CL_MEM_READ_WRITE, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); + + region.origin = align_to(region.origin + size_sh, backend_ctx->alignment); + region.size = size_d; + CL_CHECK((extra->d = clCreateSubBuffer(extra_orig->data_device, CL_MEM_READ_WRITE, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); + + cl_kernel kernel = extra->noshuffle ? backend_ctx->kernel_convert_block_iq4_xs_noshuffle : backend_ctx->kernel_convert_block_iq4_xs; + cl_ulong nb_arg = (cl_ulong)n_blk; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->qs)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra->sl)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra->sh)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extra->d)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &nb_arg)); + + size_t global_work_size[] = { (size_t)CEIL_DIV(n_blk, 64)*64, 1, 1 }; + size_t local_work_size[] = { 64, 1, 1 }; + + cl_event evt; + CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, global_work_size, local_work_size, 0, NULL, &evt)); + CL_CHECK(clWaitForEvents(1, &evt)); + CL_CHECK(clReleaseEvent(evt)); + CL_CHECK(clReleaseMemObject(data_device)); + +#ifdef GGML_OPENCL_USE_ADRENO_KERNELS + if (extra->noshuffle) { + const int m = tensor->ne[1]; + const int k = tensor->ne[0]; + transpose_2d_as_16b(backend_ctx, extra->qs, extra->qs, size_qs, k/4, m, true, true); + transpose_2d_as_32b(backend_ctx, extra->sl, extra->sl, size_sl, k/256, m, true, true); + transpose_2d_as_16b(backend_ctx, extra->sh, extra->sh, size_sh, k/256, m, true, true); + transpose_2d_as_16b(backend_ctx, extra->d, extra->d, size_d, k/256, m, true, true); + } +#endif + + extra->size_qs = size_qs; + extra->size_sl = size_sl; + extra->size_sh = size_sh; + extra->size_d = size_d; + + tensor->extra = extra; + return; + } + if (tensor->type == GGML_TYPE_Q4_K) { ggml_tensor_extra_cl * extra_orig = (ggml_tensor_extra_cl *)tensor->extra; GGML_ASSERT(extra_orig && "Tesnors in OpenCL backend should have been allocated and initialized"); @@ -16480,6 +17014,61 @@ static void ggml_backend_opencl_buffer_get_tensor(ggml_backend_buffer_t buffer, CL_CHECK(clReleaseMemObject(data_device)); return; } + if (tensor->type == GGML_TYPE_IQ4_XS) { + if (tensor->view_src) { + offset += tensor->view_offs; + tensor = tensor->view_src; + } + ggml_tensor_extra_cl_iq4_xs * extra = (ggml_tensor_extra_cl_iq4_xs *)tensor->extra; + + const size_t blck = (size_t)ggml_blck_size(tensor->type); + + ggml_cl_buffer buf_unpacked; + buf_unpacked.allocate(backend_ctx->context, ggml_nbytes(tensor)); + + const size_t n_blk = (size_t)ggml_nelements(tensor) / blck; + cl_ulong nb_arg = (cl_ulong)n_blk; + + cl_mem qs = extra->qs, sl = extra->sl, sh = extra->sh, d = extra->d; +#ifdef GGML_OPENCL_USE_ADRENO_KERNELS + ggml_cl_buffer buf_qs, buf_sl, buf_sh, buf_d; + if (extra->noshuffle) { + buf_qs.allocate(context, extra->size_qs); + buf_sl.allocate(context, extra->size_sl); + buf_sh.allocate(context, extra->size_sh); + buf_d.allocate(context, extra->size_d); + + qs = buf_qs.buffer; + sl = buf_sl.buffer; + sh = buf_sh.buffer; + d = buf_d.buffer; + const int m = tensor->ne[1]; + const int k = tensor->ne[0]; + transpose_2d_as_16b(backend_ctx, extra->qs, qs, extra->size_qs, m, k/4, true, true); + transpose_2d_as_32b(backend_ctx, extra->sl, sl, extra->size_sl, m, k/256, true, true); + transpose_2d_as_16b(backend_ctx, extra->sh, sh, extra->size_sh, m, k/256, true, true); + transpose_2d_as_16b(backend_ctx, extra->d, d, extra->size_d, m, k/256, true, true); + } +#endif + + cl_kernel kernel = extra->noshuffle ? backend_ctx->kernel_restore_block_iq4_xs_noshuffle : backend_ctx->kernel_restore_block_iq4_xs; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &qs)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &sl)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &sh)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &d)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &buf_unpacked.buffer)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &nb_arg)); + + size_t gws[] = { (n_blk + 63) / 64 * 64 }; + size_t lws[] = { 64 }; + cl_event evt; + CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 1, NULL, gws, lws, 0, NULL, &evt)); + CL_CHECK(clWaitForEvents(1, &evt)); + CL_CHECK(clReleaseEvent(evt)); + CL_CHECK(clEnqueueReadBuffer(queue, buf_unpacked.buffer, CL_TRUE, offset, size, data, 0, NULL, NULL)); + return; + } + if (tensor->type == GGML_TYPE_Q4_K) { ggml_tensor_extra_cl_q4_K * extra = (ggml_tensor_extra_cl_q4_K *)tensor->extra; @@ -17906,7 +18495,7 @@ static void ggml_cl_add(ggml_backend_t backend, const ggml_tensor * src0, const cl_kernel kernel; - const bool bcast_row = ggml_nelements(src1) == ne10 && ggml_is_contiguous(src1) && ne00 % 4 == 0 && ne10 % 4 == 0; + const bool bcast_row = ggml_is_contiguous(src0) && ggml_nelements(src1) == ne10 && ggml_is_contiguous(src1) && ne00 % 4 == 0 && ne10 % 4 == 0; if (bcast_row) { GGML_ASSERT(ggml_is_contiguous(src0)); @@ -19054,6 +19643,33 @@ static void ggml_cl_sigmoid(ggml_backend_t backend, const ggml_tensor * src0, co cl_ulong offset0 = extra0->offset + src0->view_offs; cl_ulong offsetd = extrad->offset + dst->view_offs; + if (!ggml_is_contiguous(src0) || !ggml_is_contiguous(dst)) { + GGML_TENSOR_LOCALS(int, ne0, src0, ne); + GGML_TENSOR_LOCALS(cl_ulong, nb0, src0, nb); + GGML_TENSOR_LOCALS(cl_ulong, nb, dst, nb); + + cl_kernel kernel = src0->type == GGML_TYPE_F32 ? backend_ctx->kernel_sigmoid_f32_nc : backend_ctx->kernel_sigmoid_f16_nc; + + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &nb00)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_ulong), &nb01)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &nb02)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &nb03)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &nb0)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb1)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ulong), &nb2)); + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nb3)); + + size_t global_work_size[] = {(size_t)ne01 * 64, (size_t)ne02, (size_t)ne03}; + size_t local_work_size[] = {64, 1, 1}; + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); + return; + } + cl_kernel kernel; if (src0->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32) { kernel = backend_ctx->kernel_sigmoid_f32; @@ -19152,6 +19768,40 @@ static void ggml_cl_fill(ggml_backend_t backend, const ggml_tensor * src0, const backend_ctx->enqueue_ndrange_kernel(kernel, 1, global_work_size, local_work_size, dst); } +static void ggml_cl_arange(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { + GGML_ASSERT(dst); + GGML_ASSERT(dst->extra); + GGML_ASSERT(dst->type == GGML_TYPE_F32); + + UNUSED(src0); + UNUSED(src1); + + ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; + + ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; + cl_ulong offsetd = extrad->offset + dst->view_offs; + + float start; + float step; + memcpy(&start, (float *) dst->op_params + 0, sizeof(float)); + memcpy(&step, (float *) dst->op_params + 2, sizeof(float)); + + const int n = dst->ne[0]; + + cl_kernel kernel = backend_ctx->kernel_arange; + + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(int), &n)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(float), &start)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(float), &step)); + + size_t local_work_size[1] = { 64 }; + size_t global_work_size[1] = { ((size_t)n + local_work_size[0] - 1) / local_work_size[0] * local_work_size[0] }; + + backend_ctx->enqueue_ndrange_kernel(kernel, 1, global_work_size, local_work_size, dst); +} + static void ggml_cl_clamp(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { GGML_ASSERT(src0); GGML_ASSERT(src0->extra); @@ -20389,7 +21039,22 @@ static void ggml_cl_repeat(ggml_backend_t backend, const ggml_tensor * src0, con const cl_ulong nb2 = dst->nb[2]; const cl_ulong nb3 = dst->nb[3]; - cl_kernel kernel = backend_ctx->kernel_repeat_f32; + cl_kernel kernel; + switch (src0->type) { + case GGML_TYPE_F32: + kernel = backend_ctx->kernel_repeat_f32; + break; + case GGML_TYPE_F16: + case GGML_TYPE_I16: + case GGML_TYPE_BF16: + kernel = backend_ctx->kernel_repeat_f16; + break; + case GGML_TYPE_I32: + kernel = backend_ctx->kernel_repeat_i32; + break; + default: + GGML_ABORT("unsupported type"); + } CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); @@ -22131,6 +22796,610 @@ static void ggml_cl_flash_attn_prefill_bin(ggml_backend_t backend, const ggml_te } #endif // GGML_OPENCL_USE_ADRENO_KERNELS +static void ggml_cl_fa_kq_p8_dispatch( + ggml_backend_t backend, + const ggml_tensor * q, const ggml_tensor * k, + const ggml_tensor * mask, const ggml_tensor * sinks, + const ggml_tensor * dst, + int64_t q0, int nqc, float scale, float max_bias, int p_pitch +) { + auto * backend_ctx = (ggml_backend_opencl_context *) backend->context; + auto * extra_k = (ggml_tensor_extra_cl *) (k->view_src ? k->view_src->extra : k->extra); + auto * extra_q = (ggml_tensor_extra_cl *) q->extra; + auto * extra_m = (ggml_tensor_extra_cl *) mask->extra; + + const int M = k->ne[1]; + const int K = k->ne[0]; + const int N = nqc; + const int D_A = k->ne[2]; + const int D_B = q->ne[2]; + + cl_int err; + + cl_buffer_region region = {extra_k->offset + k->view_offs, k->nb[1] * k->ne[1]}; + cl_mem k_sub = clCreateSubBuffer(extra_k->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err); + CL_CHECK(err); + + cl_image_format fmt = {CL_RGBA, CL_FLOAT}; + cl_image_desc desc = {}; + desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + desc.image_width = region.size / 16; + desc.buffer = k_sub; + cl_mem k_img = clCreateImage(backend_ctx->context, CL_MEM_READ_ONLY, &fmt, &desc, nullptr, &err); + CL_CHECK(err); + + region.origin = extra_q->offset + q->view_offs + q0 * q->nb[1]; + region.size = (size_t) K * N * D_B * sizeof(float); + cl_mem q_sub = clCreateSubBuffer(extra_q->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err); + CL_CHECK(err); + + const cl_ulong off_mask = extra_m->offset + mask->view_offs + q0 * mask->nb[1]; + const cl_ulong mask_nb1 = mask->nb[1]; + const int n_head_log2 = 1u << (unsigned) floorf(log2f((float) D_B)); + const float m0 = powf(2.0f, -max_bias / n_head_log2); + const float m1 = powf(2.0f, -(max_bias / 2.0f) / n_head_log2); + + cl_uint a = 0; + + cl_kernel kernel = backend_ctx->kernel_mul_mm_f16_f32_kq_p8; + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &k_img)); + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &q_sub)); + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &backend_ctx->prealloc_fa_pq.buffer)); + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &backend_ctx->prealloc_fa_bmax.buffer)); + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &backend_ctx->prealloc_fa_bsum.buffer)); + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &extra_m->data_device)); + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_ulong), &off_mask)); + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_ulong), &mask_nb1)); + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(float), &scale)); + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(float), &max_bias)); + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(float), &m0)); + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(float), &m1)); + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &n_head_log2)); + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &M)); + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &K)); + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &N)); + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &D_A)); + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &D_B)); + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &p_pitch)); + + const size_t n_tiles = (N + 31) / 32; + size_t gws[3] = {64, n_tiles, (size_t)(M / 64) * D_B}; + + const bool two_tiles = n_tiles % 2 == 0 && backend_ctx->get_kernel_workgroup_size(kernel) >= 128; + size_t lws[3] = {64, two_tiles ? (size_t) 2 : (size_t) 1, 1}; + + backend_ctx->enqueue_ndrange_kernel(kernel, 3, gws, lws, dst); + CL_CHECK(clReleaseMemObject(k_img)); + CL_CHECK(clReleaseMemObject(k_sub)); + CL_CHECK(clReleaseMemObject(q_sub)); + + cl_mem sinks_buf = extra_q->data_device; + cl_ulong sinks_off = 0; + const int has_sinks = sinks != nullptr; + + if (sinks) { + auto * extra_s = (ggml_tensor_extra_cl *) sinks->extra; + sinks_buf = extra_s->data_device; + sinks_off = extra_s->offset + sinks->view_offs; + } + + const int nblk = M / 32; + const int row_blk = p_pitch / 32; + + kernel = backend_ctx->kernel_fa_p8_fixup; + + a = 0; + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &backend_ctx->prealloc_fa_bmax.buffer)); + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &backend_ctx->prealloc_fa_bsum.buffer)); + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &backend_ctx->prealloc_fa_pd.buffer)); + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &backend_ctx->prealloc_fa_sums.buffer)); + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &sinks_buf)); + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_ulong), &sinks_off)); + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &has_sinks)); + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &nblk)); + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &row_blk)); + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &N)); + + size_t gws_fixup[3] = {(size_t) 64 * N, (size_t) D_B, 1}; + size_t lws_fixup[3] = {64, 1, 1}; + backend_ctx->enqueue_ndrange_kernel(kernel, 3, gws_fixup, lws_fixup, dst); +} + +static void ggml_cl_fa_decompose_scratch_tensor( + ggml_tensor & t, ggml_tensor_extra_cl & extra, cl_mem buffer, ggml_type type, + int64_t ne0, int64_t ne1, int64_t ne2 +) { + memset(&t, 0, sizeof(t)); + t.type = type; + t.ne[0] = ne0; t.ne[1] = ne1; t.ne[2] = ne2; t.ne[3] = 1; + t.nb[0] = ggml_type_size(type); + t.nb[1] = t.nb[0]*ne0; + t.nb[2] = t.nb[1]*ne1; + t.nb[3] = t.nb[2]*ne2; + + extra.reset(); + extra.data_device = buffer; + t.extra = &extra; +} + +static void ggml_cl_fa_decompose_write_chunk( + ggml_backend_t backend, const ggml_tensor & kqv, int64_t q0, ggml_tensor * dst +) { + auto * backend_ctx = (ggml_backend_opencl_context *) backend->context; + const int64_t dv = kqv.ne[0]; + const int64_t nqc = kqv.ne[1]; + const int64_t n_head = kqv.ne[2]; + + // apply the deferred 1/sum to the KQV rows + { + cl_kernel kernel = backend_ctx->kernel_fa_decompose_scale_rows_f32; + const cl_ulong offset_kqv = 0; + const cl_ulong offset_sums = 0; + const int dv_i = (int) dv; + const int nqc_i = (int) nqc; + + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &backend_ctx->prealloc_fa_kqv.buffer)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset_kqv)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &backend_ctx->prealloc_fa_sums.buffer)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset_sums)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &dv_i)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(int), &nqc_i)); + + size_t gws[3] = { (size_t) 64 * nqc, (size_t) n_head, 1 }; + size_t lws[3] = { 64, 1, 1 }; + backend_ctx->enqueue_ndrange_kernel(kernel, 3, gws, lws, dst); + } + + // permute into dst: [dv, nqc, n_head] -> dst[dv, n_head, q0 .. q0+nqc) + { + ggml_tensor kqv_perm = kqv; + kqv_perm.ne[1] = n_head; + kqv_perm.ne[2] = nqc; + kqv_perm.nb[1] = kqv.nb[2]; + kqv_perm.nb[2] = kqv.nb[1]; + + ggml_tensor dst_chunk = *dst; + dst_chunk.ne[2] = nqc; + dst_chunk.view_offs = dst->view_offs + (size_t) q0 * dst->nb[2]; + + ggml_cl_cpy(backend, &kqv_perm, &dst_chunk, nullptr); + } +} + +static void ggml_cl_flash_attn_decompose( + ggml_backend_t backend, + const ggml_tensor * q, const ggml_tensor * k, ggml_tensor * dst, + int64_t n_q_chunk, float scale, float max_bias +) { + auto * backend_ctx = (ggml_backend_opencl_context *) backend->context; + const ggml_tensor * v = dst->src[2]; + const ggml_tensor * mask = dst->src[3]; + const ggml_tensor * sinks = dst->src[4]; + const int64_t n_q = q->ne[1]; + const int64_t n_kv = k->ne[1]; + const int64_t dv = v->ne[0]; + const int64_t n_head = q->ne[2]; + const int64_t n_head_kv = k->ne[2]; + + const std::pair dk_dv = {(int) q->ne[0], (int) dv}; + + // The image GEMMs work in 64 x 32 tiles and can touch past the exact extent, + // so size each scratch by the padded shape plus one extra head. + auto pad_up = [](int64_t x, int64_t a) { return ((x + a - 1) / a) * a; }; + const int64_t kq_m = pad_up(n_kv, 64); + const int64_t kqv_m = pad_up(dv, 64); + const int64_t tile_n = pad_up(n_q_chunk, 32); + + backend_ctx->prealloc_fa_vt.allocate(backend_ctx->context, (size_t)(n_kv * kqv_m * (n_head_kv + 1)) * sizeof(ggml_fp16_t)); + backend_ctx->prealloc_fa_kq.allocate(backend_ctx->context, (size_t)(kq_m * tile_n * (n_head + 1)) * sizeof(float)); + backend_ctx->prealloc_fa_kqv.allocate(backend_ctx->context, (size_t)(kqv_m * tile_n * (n_head + 1)) * sizeof(float)); + backend_ctx->prealloc_fa_sums.allocate(backend_ctx->context, (size_t)(n_q_chunk * n_head) * sizeof(float)); + + // V^T once per call: [dv, n_kv, n_head_kv] -> [n_kv, dv, n_head_kv] + { + ggml_tensor_extra_cl * extra_v = (ggml_tensor_extra_cl *)v->extra; + cl_kernel kernel = backend_ctx->fa.v_transpose_f16.at(dk_dv); + + const cl_ulong offset_v = extra_v->offset + v->view_offs; + const cl_ulong v_nb1 = v->nb[1]; + const cl_ulong v_nb2 = v->nb[2]; + const int n_kv_i = (int) n_kv; + const int dv_i = (int) dv; + const int n_head_kv_i = (int) n_head_kv; + + cl_uint a = 0; + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &extra_v->data_device)); + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_ulong), &offset_v)); + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &backend_ctx->prealloc_fa_vt.buffer)); + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &n_kv_i)); + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &dv_i)); + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &n_head_kv_i)); + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_ulong), &v_nb1)); + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_ulong), &v_nb2)); + + size_t gws[3] = { (size_t)((dv + 31) / 32) * 32, (size_t)((n_kv + 31) / 32) * 8, (size_t) n_head_kv }; + size_t lws[3] = { 32, 8, 1 }; + backend_ctx->enqueue_ndrange_kernel(kernel, 3, gws, lws, dst); + } + + ggml_tensor_extra_cl extra_vt; + ggml_tensor_extra_cl extra_kq; + ggml_tensor_extra_cl extra_kqv; + ggml_tensor vt; + ggml_tensor kq; + ggml_tensor kqv; + ggml_cl_fa_decompose_scratch_tensor(vt, extra_vt, backend_ctx->prealloc_fa_vt.buffer, GGML_TYPE_F16, n_kv, dv, n_head_kv); + + for (int64_t q0 = 0; q0 < n_q; q0 += n_q_chunk) { + const int64_t nqc = MIN(n_q_chunk, n_q - q0); + + // KQ: [dk, n_kv, n_head_kv] x [dk, nqc, n_head] -> [n_kv, nqc, n_head] + ggml_cl_fa_decompose_scratch_tensor(kq, extra_kq, backend_ctx->prealloc_fa_kq.buffer, GGML_TYPE_F32, n_kv, nqc, n_head); + + { + ggml_tensor q_chunk = *q; + q_chunk.ne[1] = nqc; + q_chunk.view_offs = q->view_offs + (size_t) q0 * q->nb[1]; + ggml_cl_mul_mat(backend, k, &q_chunk, &kq); + } + + // softmax in place, the row sums go to prealloc_fa_sums + { + memcpy((float *) kq.op_params + 0, &scale, sizeof(float)); + memcpy((float *) kq.op_params + 1, &max_bias, sizeof(float)); + + ggml_tensor mask_chunk = *mask; + mask_chunk.ne[1] = nqc; + mask_chunk.view_offs = mask->view_offs + (size_t) q0 * mask->nb[1]; + + ggml_cl_fa_decompose_soft_max(backend, &kq, &mask_chunk, sinks, &kq, backend_ctx->prealloc_fa_sums.buffer); + } + + // KQV: [n_kv, dv, n_head_kv] x [n_kv, nqc, n_head] -> [dv, nqc, n_head] + // V^T is contiguous, so mul_mat would not pick the KQV image kernel; call it here. + ggml_cl_fa_decompose_scratch_tensor(kqv, extra_kqv, backend_ctx->prealloc_fa_kqv.buffer, GGML_TYPE_F32, dv, nqc, n_head); +#ifdef GGML_OPENCL_USE_ADRENO_KERNELS + if (backend_ctx->kernel_mul_mm_f16_f32_kqv != nullptr && + (dv % 64) == 0 && (n_kv % 16) == 0 && nqc >= 32 && + ((size_t)(vt.nb[2] * n_head_kv / 4) / 4 <= backend_ctx->image_max_buffer_size) && + ((size_t)(dv * nqc * n_head) <= backend_ctx->image_max_buffer_size)) { + ggml_cl_mul_mat_kq_kqv_adreno(backend, &vt, &kq, &kqv, /*is_kq =*/ false); + } else +#endif + { + ggml_cl_mul_mat(backend, &vt, &kq, &kqv); + } + + ggml_cl_fa_decompose_write_chunk(backend, kqv, q0, dst); + } +} + +static void ggml_cl_fa_pv_direct(ggml_backend_t backend, ggml_tensor * dst, int kv_pitch) { + auto * backend_ctx = (ggml_backend_opencl_context *) backend->context; + const ggml_tensor * q = dst->src[0]; + const ggml_tensor * v = dst->src[2]; + auto * extra_v = (ggml_tensor_extra_cl *) v->extra; + + const int dv = v->ne[0]; + const int n_q = q->ne[1]; + const int n_head = q->ne[2]; + const int n_head_kv = v->ne[2]; + const int gqa = n_head / n_head_kv; + const int nblk = v->ne[1] / 32; + const int d_blocks = dv / 128; + // Split the KV range to give a small query batch about 256 workgroups. + const int want = MAX(1, 256 / (d_blocks * n_head_kv)); + const int blocks_per_split = (nblk + want - 1) / want; + const int n_split = (nblk + blocks_per_split - 1) / blocks_per_split; + const int n_elem = n_head * n_q * dv; + backend_ctx->prealloc_fa_kq.allocate(backend_ctx->context, (size_t) n_split * n_elem * sizeof(float)); + + const cl_ulong off_v = extra_v->offset + v->view_offs; + const cl_ulong v_nb1 = v->nb[1]; + const cl_ulong v_nb2 = v->nb[2]; + { + cl_kernel kernel = backend_ctx->kernel_fa_kqv_direct_f16; + cl_uint a = 0; + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &extra_v->data_device)); + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_ulong), &off_v)); + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_ulong), &v_nb1)); + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_ulong), &v_nb2)); + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &backend_ctx->prealloc_fa_pq.buffer)); + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &backend_ctx->prealloc_fa_pd.buffer)); + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &backend_ctx->prealloc_fa_kq.buffer)); + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &dv)); + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &n_q)); + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &gqa)); + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &n_head)); + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &nblk)); + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &blocks_per_split)); + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &kv_pitch)); + size_t gws[3] = {(size_t) d_blocks * 64, (size_t) n_head_kv, (size_t) n_split}; + size_t lws[3] = {64, 1, 1}; + backend_ctx->enqueue_ndrange_kernel(kernel, 3, gws, lws, dst); + } + { + cl_kernel kernel = backend_ctx->kernel_fa_kqv_direct_reduce; + cl_uint a = 0; + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &backend_ctx->prealloc_fa_kq.buffer)); + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &backend_ctx->prealloc_fa_kqv.buffer)); + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &n_elem)); + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &n_split)); + size_t gws[3] = {((size_t) n_elem + 63) / 64 * 64, 1, 1}; + size_t lws[3] = {64, 1, 1}; + backend_ctx->enqueue_ndrange_kernel(kernel, 3, gws, lws, dst); + } +} + +static void ggml_cl_flash_attn_decompose_kq_p8( + ggml_backend_t backend, + const ggml_tensor * q, const ggml_tensor * k, + ggml_tensor * dst, + int64_t n_q_chunk, float scale, float max_bias, bool kqv_direct +) { + auto * backend_ctx = (ggml_backend_opencl_context *) backend->context; + const ggml_tensor * v = dst->src[2]; + const ggml_tensor * mask = dst->src[3]; + const ggml_tensor * sinks = dst->src[4]; + const int64_t n_q = q->ne[1]; + const int64_t n_kv = k->ne[1]; + const int64_t dv = v->ne[0]; + const int64_t n_head = q->ne[2]; + const int64_t n_head_kv = k->ne[2]; + + auto pad_up = [](int64_t x, int64_t a) { return ((x + a - 1) / a) * a; }; + const int64_t kqv_m = pad_up(dv, 64); + const int64_t tile_n = pad_up(n_q_chunk, 32); + + // Pad the int8 row pitch by one cache line, as in the P8 KQV kernel. + const int kv_pitch = (int) n_kv + 64; + const size_t nblk = (size_t) kv_pitch / 32; + + backend_ctx->prealloc_fa_pq.allocate(backend_ctx->context, (size_t) kv_pitch * n_q_chunk * n_head); + backend_ctx->prealloc_fa_pd.allocate(backend_ctx->context, nblk * n_q_chunk * n_head * sizeof(float)); + backend_ctx->prealloc_fa_bmax.allocate(backend_ctx->context, nblk * n_q_chunk * n_head * sizeof(float)); + backend_ctx->prealloc_fa_bsum.allocate(backend_ctx->context, nblk * n_q_chunk * n_head * sizeof(float)); + backend_ctx->prealloc_fa_kqv.allocate(backend_ctx->context, (size_t)(kqv_m * tile_n * (n_head + 1)) * sizeof(float)); + backend_ctx->prealloc_fa_sums.allocate(backend_ctx->context, (size_t)(n_q_chunk * n_head) * sizeof(float)); + + // Direct PV reads the original V cache and needs no transposed copy. + if (!kqv_direct) { + backend_ctx->prealloc_fa_vtq.allocate(backend_ctx->context, (size_t) kv_pitch * dv * n_head_kv); + backend_ctx->prealloc_fa_vtd.allocate(backend_ctx->context, nblk * dv * n_head_kv * sizeof(cl_half)); + + ggml_tensor_extra_cl * extra_v = (ggml_tensor_extra_cl *)v->extra; + cl_kernel kernel = backend_ctx->kernel_fa_v_transpose_q8; + + const cl_ulong offset_v = extra_v->offset + v->view_offs; + const cl_ulong v_nb1 = v->nb[1]; + const cl_ulong v_nb2 = v->nb[2]; + const cl_ulong v_nb3 = v->nb[3]; + const int n_kv_i = (int) n_kv; + const int dv_i = (int) dv; + const int n_head_kv_i = (int) n_head_kv; + + cl_uint a = 0; + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &extra_v->data_device)); + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_ulong), &offset_v)); + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &backend_ctx->prealloc_fa_vtq.buffer)); + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &backend_ctx->prealloc_fa_vtd.buffer)); + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &n_kv_i)); + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &dv_i)); + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &n_head_kv_i)); + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_ulong), &v_nb1)); + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_ulong), &v_nb2)); + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_ulong), &v_nb3)); + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &kv_pitch)); + + size_t gws[3] = { (size_t)((dv + 31) / 32) * 32, (size_t)((n_kv + 31) / 32) * 8, (size_t) n_head_kv }; + size_t lws[3] = { 32, 8, 1 }; + backend_ctx->enqueue_ndrange_kernel(kernel, 3, gws, lws, dst); + } + + ggml_tensor_extra_cl extra_kqv; + ggml_tensor kqv; + + for (int64_t q0 = 0; q0 < n_q; q0 += n_q_chunk) { + const int64_t nqc = MIN(n_q_chunk, n_q - q0); + + ggml_cl_fa_kq_p8_dispatch(backend, q, k, mask, sinks, dst, q0, nqc, scale, max_bias, kv_pitch); + + ggml_cl_fa_decompose_scratch_tensor(kqv, extra_kqv, backend_ctx->prealloc_fa_kqv.buffer, GGML_TYPE_F32, dv, nqc, n_head); + + if (kqv_direct) { + GGML_ASSERT(q0 == 0 && nqc == n_q); + ggml_cl_fa_pv_direct(backend, dst, kv_pitch); + } else { + const cl_ulong zero = 0; + const int dv_i = dv; + const int nkv_i = n_kv; + const int nqc_i = nqc; + const int nh_i = n_head; + const int nhkv_i = n_head_kv; + + cl_kernel kernel = backend_ctx->kernel_mul_mm_q8_f32_kqv; + + cl_uint a = 0; + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &backend_ctx->prealloc_fa_vtq.buffer)); + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_ulong), &zero)); + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &backend_ctx->prealloc_fa_vtd.buffer)); + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_ulong), &zero)); + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &backend_ctx->prealloc_fa_pq.buffer)); + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_ulong), &zero)); + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &backend_ctx->prealloc_fa_pd.buffer)); + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_ulong), &zero)); + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &backend_ctx->prealloc_fa_kqv.buffer)); + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_ulong), &zero)); + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &dv_i)); + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &nkv_i)); + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &nqc_i)); + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &nh_i)); + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &nhkv_i)); + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &kv_pitch)); + size_t gws[3] = {(size_t)(dv / 64) * 64, (size_t)(n_head / n_head_kv) * ((nqc + 31) / 32), (size_t) n_head_kv}; + size_t lws[3] = {64, 1, 1}; + backend_ctx->enqueue_ndrange_kernel(kernel, 3, gws, lws, dst); + } + + ggml_cl_fa_decompose_write_chunk(backend, kqv, q0, dst); + } +} + +// Decomposed flash attention prefill. +// +// On Adreno the fused prefill tile is slow at large head sizes, while the +// KQ / KQV image GEMMs are fast. -fa 0 uses those GEMMs, but it keeps KQ for +// the whole ubatch in the graph. Here the FLASH_ATTN_EXT node stays, and the +// queries are split into chunks. For each chunk: KQ -> softmax -> KQV into +// backend scratch, so the scratch size depends on the chunk and not on the +// ubatch, and the KV cache keeps the flash attention layout. +// +// Returns false when the shape is not supported, then the fused tile runs. +// GGML_OPENCL_FA_DECOMPOSE=0 disables it, =1 enables it for any Adreno GPU and +// head size. GGML_OPENCL_FA_DECOMPOSE_MB sets the KQ scratch budget (default 256). +// GGML_OPENCL_FA_KQ_P8=0 disables U8 probabilities and int8 KQV. +// GGML_OPENCL_FA_KQV_DIRECT=0 disables direct PV for small query batches. +static bool ggml_cl_flash_attn_decompose_dispatch( + ggml_backend_t backend, + const ggml_tensor * q, const ggml_tensor * k, ggml_tensor * dst +) { + ggml_backend_opencl_context * backend_ctx = (ggml_backend_opencl_context *)backend->context; + + const ggml_tensor * v = dst->src[2]; + const ggml_tensor * mask = dst->src[3]; + + if (backend_ctx->gpu_family != ADRENO) { + return false; + } + + static const int env = []{ + const char * e = getenv("GGML_OPENCL_FA_DECOMPOSE"); + return (e && e[0]) ? (e[0] != '0' ? 1 : 0) : -1; + }(); + + if (env == 0) { + return false; + } + + const int64_t n_q = q->ne[1]; + const int64_t n_kv = k->ne[1]; + const int64_t dk = q->ne[0]; + const int64_t dv = v->ne[0]; + const int64_t n_head = q->ne[2]; + const int64_t n_head_kv = k->ne[2]; + + const bool measured_gen = backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E || + backend_ctx->adreno_gen == ADRENO_GPU_GEN::A8X || + backend_ctx->adreno_gen == ADRENO_GPU_GEN::X1E; + + if (n_q < 2) { + return false; + } + if (q->type != GGML_TYPE_F32 || dst->type != GGML_TYPE_F32 || + k->type != GGML_TYPE_F16 || v->type != GGML_TYPE_F16) { + return false; + } + if (q->ne[3] != 1 || k->ne[3] != 1 || v->ne[3] != 1 || dst->ne[3] != 1) { + return false; + } + if (n_head_kv <= 0 || n_head < n_head_kv || n_head % n_head_kv != 0) { + return false; + } + if (k->nb[0] != ggml_type_size(k->type) || v->nb[0] != ggml_type_size(v->type)) { + return false; + } + // K and Q must be packed [dk, head, row], as the KV cache and the permuted Q + // are. mul_mat sends the KQ to the image KQ kernel only for this layout; + // other K views can reach the KQV image kernel there and give wrong results. + if (k->nb[2] != (size_t) dk * sizeof(ggml_fp16_t) || k->nb[1] != (size_t) dk * n_head_kv * sizeof(ggml_fp16_t) || + q->nb[0] != sizeof(float) || q->nb[2] != (size_t) dk * sizeof(float) || q->nb[1] != (size_t) dk * n_head * sizeof(float)) { + return false; + } + + // soft_max applies scale, mask, ALiBi and sinks, but not the softcap. + float scale; + float max_bias; + float logit_softcap; + memcpy(&scale, (const float *) dst->op_params + 0, sizeof(float)); + memcpy(&max_bias, (const float *) dst->op_params + 1, sizeof(float)); + memcpy(&logit_softcap, (const float *) dst->op_params + 2, sizeof(float)); + if (logit_softcap != 0.0f) { + return false; + } + + // The deferred softmax kernel needs an f16 mask and rows that divide by 4. + if (mask == nullptr || mask->type != GGML_TYPE_F16 || mask->ne[0] < n_kv || (n_kv % 4) != 0) { + return false; + } + + const std::pair dk_dv = {(int) dk, (int) dv}; + if (backend_ctx->fa.v_transpose_f16.count(dk_dv) == 0) { + return false; + } + + static const char * p8_env = getenv("GGML_OPENCL_FA_KQ_P8"); + const bool kq_p8 = (!p8_env || p8_env[0] != '0') && measured_gen && + backend_ctx->kernel_mul_mm_f16_f32_kq_p8 != nullptr && + backend_ctx->kernel_mul_mm_q8_f32_kqv != nullptr && + backend_ctx->kernel_fa_v_transpose_q8 != nullptr && + n_kv <= INT_MAX - 64 && k->nb[1] <= INT_MAX && + (n_kv % 64) == 0 && (dk % 16) == 0 && (dv % 64) == 0 && + mask->ne[2] == 1 && mask->ne[3] == 1 && mask->nb[0] == sizeof(ggml_fp16_t) && + (mask->nb[1] % sizeof(ggml_fp16_t)) == 0 && + k->nb[1] * n_kv / 16 <= backend_ctx->image_max_buffer_size; + + // Keep DK64 with attention sinks opt-in until its accuracy is validated. + const bool dk64_ok = kq_p8 && dst->src[4] == nullptr && n_head / n_head_kv >= 2 && + n_kv >= 1024 && n_q <= n_kv / 2; + const int min_dk = backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E ? (dk64_ok ? 64 : 128) : 256; + if (env != 1 && !(measured_gen && dk >= min_dk)) { + return false; + } + + static const char * direct_env = getenv("GGML_OPENCL_FA_KQV_DIRECT"); + const bool kqv_direct = (!direct_env || direct_env[0] != '0') && kq_p8 && + backend_ctx->kernel_fa_kqv_direct_f16 != nullptr && backend_ctx->kernel_fa_kqv_direct_reduce != nullptr && + v->ne[1] == n_kv && v->ne[2] == n_head_kv && + n_q <= 32 / (n_head / n_head_kv) && dv >= 128 && dv <= INT_MAX && (dv % 128) == 0 && + n_head <= INT_MAX / (n_q * dv) && (v->nb[1] % 4) == 0 && (v->nb[2] % 4) == 0; + const bool small_p8 = backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E && + kq_p8 && dk >= 128 && n_kv >= 1024; + // Small batches without direct PV need P8 and a long KV cache on X2E. + if (n_q < 32 && !kqv_direct && !small_p8) { + return false; + } + + // Chunk size: a byte budget for KQ, at least 8 tiles of 32 queries so the + // GEMMs get usable shapes, and below the image1d_buffer limit of the KQ dst. + static const size_t budget_bytes = []{ + const char * e = getenv("GGML_OPENCL_FA_DECOMPOSE_MB"); + const int mb = (e && e[0]) ? atoi(e) : 256; + return (size_t)(mb > 0 ? mb : 256) * 1024 * 1024; + }(); + + const size_t kq_row_bytes = (size_t) n_kv * n_head * sizeof(float); + int64_t n_q_chunk = MIN((int64_t)(budget_bytes / kq_row_bytes), n_q); + n_q_chunk = MAX(n_q_chunk, MIN((int64_t) 8 * 32, n_q)); + + if (!kq_p8) { + n_q_chunk = MIN(n_q_chunk, (int64_t)(backend_ctx->image_max_buffer_size / (n_kv * n_head))); + } + if (n_q_chunk < 32 && n_q_chunk < n_q) { + return false; + } + + // even out the chunks, then align to the 32-wide n-tile + const int64_t n_chunks = (n_q + n_q_chunk - 1) / n_q_chunk; + n_q_chunk = MIN(n_q_chunk, ((n_q + n_chunks - 1) / n_chunks + 31) & ~(int64_t) 31); + + if (kq_p8) { + ggml_cl_flash_attn_decompose_kq_p8(backend, q, k, dst, n_q_chunk, scale, max_bias, kqv_direct); + } else { + ggml_cl_flash_attn_decompose(backend, q, k, dst, n_q_chunk, scale, max_bias); + } + return true; +} + static void ggml_cl_flash_attn(ggml_backend_t backend, const ggml_tensor * q, const ggml_tensor * k, ggml_tensor * dst) { const ggml_tensor * v = dst->src[2]; const ggml_tensor * mask = dst->src[3]; @@ -22234,14 +23503,29 @@ static void ggml_cl_flash_attn(ggml_backend_t backend, const ggml_tensor * q, co const std::pair dk_dv = {d_head_q, d_head_v}; const int mq_gqa = n_head_kv > 0 ? n_head / n_head_kv : 0; const std::tuple mq_decode_key = {d_head_q, d_head_v, mq_gqa}; - const bool mq_decode_shape = backend_ctx->gpu_family == ADRENO && is_mixed && n_q == 1 && - d_head_q == d_head_v && n_head_kv > 0 && n_head % n_head_kv == 0 && - ((((d_head_q == 64 && mq_gqa == 8) || (d_head_q == 128 && mq_gqa == 4)) && - backend_ctx->has_subgroup_shuffle) || - ((d_head_q == 256 || d_head_q == 512) && (mq_gqa == 4 || mq_gqa == 8))); - if (mq_decode_shape && n_kv >= 32) { + // X2E also takes GQA 2 and 6 at head size 256 (gemma-4 SWA layers, Qwen3.5-27B), + // GQA 16 at 512 and 128 (gemma-4-12b global layers, muse-glimmer) and GQA 2 at 512 (gemma-4-E4B MTP drafter) + const bool mq_decode_x2e = backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E && + ((d_head_q == 256 && (mq_gqa == 2 || mq_gqa == 6)) || + (d_head_q == 512 && (mq_gqa == 2 || mq_gqa == 16)) || + (d_head_q == 128 && mq_gqa == 16 && backend_ctx->has_subgroup_shuffle)); + const bool mq_decode_kernel = backend_ctx->gpu_family == ADRENO && is_mixed && + d_head_q == d_head_v && n_head_kv > 0 && n_head % n_head_kv == 0 && + ((((d_head_q == 64 && mq_gqa == 8) || (d_head_q == 128 && mq_gqa == 4)) && + backend_ctx->has_subgroup_shuffle) || + ((d_head_q == 256 || d_head_q == 512) && (mq_gqa == 4 || mq_gqa == 8)) || + mq_decode_x2e); + const bool mq_decode_shape = mq_decode_kernel && n_q == 1; + // compile on the first call for this shape (the warmup is a prefill), not on the first decode + if (mq_decode_kernel) { ggml_opencl_ensure_fa_f32_f16_mq_decode(backend_ctx, d_head_q, d_head_v, mq_gqa); } + + // after all lazy compiles, so the decode kernels are built during warmup as on the fused path + if (n_q > 1 && ggml_cl_flash_attn_decompose_dispatch(backend, q, k, dst)) { + return; + } + const bool use_native_q8_0_q1 = is_q8_0 && n_q == 1 && backend_ctx->fa.f32_q8_0_q1.count(dk_dv) > 0; // Native q8_0 prefill — reads q8_0 directly, wg_size = cfg->bm. @@ -23906,7 +25190,15 @@ static void ggml_cl_mul_mat_q2_0_f32_adreno(ggml_backend_t backend, const ggml_t const int k = src0->ne[0]; if (n == 1 || !ggml_is_contiguous(src1) || offset1 % backend_ctx->alignment != 0) { - cl_kernel gemv = backend_ctx->kernel_gemv_noshuffle_q2_0_f32; + const bool wide = m >= 16384 && m % 8 == 0; + cl_kernel gemv = wide ? backend_ctx->kernel_gemv_noshuffle_q2_0_f32_wide : backend_ctx->kernel_gemv_noshuffle_q2_0_f32; + const size_t workgroup = wide ? backend_ctx->q2_0_mv_workgroup_size_wide : backend_ctx->q2_0_mv_workgroup_size; + const size_t rows = wide ? 8 : 4; + size_t nsg = std::min((size_t)backend_ctx->q2_0_mv_nsg, wide ? size_t(8) : m >= 4096 ? size_t(16) : size_t(32)); + while (nsg > (size_t)k/64 || nsg*16 > workgroup) { + nsg /= 2; + } + const size_t lanes = workgroup/nsg; const int stride_d = dst->ne[0]; const int stride_b = src1->nb[1]/sizeof(float); CL_CHECK(clSetKernelArg(gemv, 0, sizeof(cl_mem), &extra0->qs)); @@ -23919,9 +25211,8 @@ static void ggml_cl_mul_mat_q2_0_f32_adreno(ggml_backend_t backend, const ggml_t CL_CHECK(clSetKernelArg(gemv, 7, sizeof(int), &m)); CL_CHECK(clSetKernelArg(gemv, 8, sizeof(int), &stride_b)); CL_CHECK(clSetKernelArg(gemv, 9, sizeof(int), &stride_d)); - const size_t nsg = backend_ctx->q2_0_mv_nsg; - size_t global[] = { (size_t)CEIL_DIV(m, 128)*64, (size_t)n*nsg, 1 }; - size_t local[] = { 64, nsg, 1 }; + size_t global[] = { (size_t)CEIL_DIV(m, rows*lanes)*lanes, (size_t)n*nsg, 1 }; + size_t local[] = { lanes, nsg, 1 }; backend_ctx->enqueue_ndrange_kernel(gemv, 3, global, local, dst); return; } @@ -24059,7 +25350,8 @@ static void ggml_cl_mul_mat_q4_0_f32_adreno_ila(ggml_backend_t backend, const gg CL_CHECK(clReleaseMemObject(b_sub_buf)); CL_CHECK(clReleaseMemObject(b_img)); } else { - const int gemm_tile_n = 64; + const bool have_a8 = backend_ctx->kernel_gemm_noshuffle_q4_0_f32_32b_trans_ila_a8_bin != nullptr; + const int gemm_tile_n = have_a8 ? 64 : 32; int N_pad = (N + gemm_tile_n - 1) & ~(gemm_tile_n - 1); cl_mem a_img = extra0_q4_0->q_img; @@ -24159,6 +25451,53 @@ static void ggml_cl_mul_mat_q4_0_f32_adreno_ila(ggml_backend_t backend, const gg cl_mem b_img; CL_CHECK((b_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err)); + int line_stride_matrix_A_in_bytes = M * 4; + int line_stride_matrix_S_in_bytes = M * 2; + int line_stride_matrix_B_in_bytes = K * 4; + int line_stride_matrix_C_in_bytes = M * 4; + + const int b_offset_for_kernel = 0; + + if (!have_a8) { + const int c_offset_for_kernel = (int)offsetd; + + kernel = backend_ctx->kernel_gemm_noshuffle_q4_0_f32_32b_trans_ila_a7_bin; + + cl_uint k_arg = 0; + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &a_img)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &extra0_q4_0->d)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &b_img)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &b_offset_for_kernel)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &c_offset_for_kernel)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &K)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &line_stride_matrix_A_in_bytes)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &line_stride_matrix_S_in_bytes)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &line_stride_matrix_B_in_bytes)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &line_stride_matrix_C_in_bytes)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &M)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &N)); + + size_t m_tiles = (size_t)CEIL_DIV(M, 64); + if ((m_tiles % 2) != 0) { + // keep the work-group count in y even so every work-group is full + m_tiles += 1; + } + + size_t local_work_size[3] = { 64, 2, 4 }; + size_t global_work_size[3] = { 64, m_tiles, (size_t)CEIL_DIV(N_pad, gemm_tile_n) }; + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); + + CL_CHECK(clReleaseMemObject(b_img)); + if (b_sub_buf) { + CL_CHECK(clReleaseMemObject(b_sub_buf)); + } + if (b_padded) { + CL_CHECK(clReleaseMemObject(b_padded)); + } + return; + } + region.origin = offsetd; region.size = (size_t)M * N * sizeof(float); cl_mem d_sub_buf; @@ -24172,13 +25511,7 @@ static void ggml_cl_mul_mat_q4_0_f32_adreno_ila(ggml_backend_t backend, const gg cl_mem d_img; CL_CHECK((d_img = clCreateImage(context, CL_MEM_WRITE_ONLY, &img_fmt, &img_desc, NULL, &err), err)); - int line_stride_matrix_A_in_bytes = M * 4; - int line_stride_matrix_S_in_bytes = M * 2; - int line_stride_matrix_B_in_bytes = K * 4; - int line_stride_matrix_C_in_bytes = M * 4; - - int c_offset_for_kernel = 0; - int b_offset_for_kernel = 0; + const int c_offset_for_kernel = 0; kernel = backend_ctx->kernel_gemm_noshuffle_q4_0_f32_32b_trans_ila_a8_bin; @@ -24198,19 +25531,19 @@ static void ggml_cl_mul_mat_q4_0_f32_adreno_ila(ggml_backend_t backend, const gg CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &N)); size_t local_work_size[3] = { 64, 2, 2 }; - size_t m_tiles = (size_t)CEIL_DIV(M, 64); - size_t global_work_size[3] = { 64, m_tiles, (size_t)CEIL_DIV(N_pad, gemm_tile_n) }; + size_t global_work_size[3] = { 64, (size_t)CEIL_DIV(M, 64), (size_t)CEIL_DIV(N_pad, gemm_tile_n) }; backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); + CL_CHECK(clReleaseMemObject(d_img)); + CL_CHECK(clReleaseMemObject(d_sub_buf)); CL_CHECK(clReleaseMemObject(b_img)); + if (b_sub_buf) { CL_CHECK(clReleaseMemObject(b_sub_buf)); } if (b_padded) { CL_CHECK(clReleaseMemObject(b_padded)); } - CL_CHECK(clReleaseMemObject(d_img)); - CL_CHECK(clReleaseMemObject(d_sub_buf)); } } #endif // GGML_OPENCL_USE_ADRENO_KERNELS @@ -24257,14 +25590,8 @@ static void ggml_cl_mul_mat_q4_0_f32_adreno(ggml_backend_t backend, const ggml_t int N = ne1; int K = ne00; - // Multi-column (N=3) verify GEMV for q4_0: route the spec/MTP verify batch - // (ne1==3) onto the efficient GEMV path instead of the transposed-GEMM dead- - // zone (gemm_noshuffle_q4_0 is ~50% of MTP decode on a Q4_0 model since q4_0 - // weights have no cok/mc3, unlike q4_K). Reuses the ne1==1 GEMV image setup - // (activation image already sized by N=ne1). Byte-identical. Opt-in via - // GGML_OPENCL_Q40_MC3=1. Per-layer only (ne01 < 32768); q4_0 lm_head doesn't - // occur (token_embd/output stay Q6_K), guard kept for parity with q4_K mc3. - static const bool q40_mc3 = (getenv("GGML_OPENCL_Q40_MC3") != nullptr); + // Multi-column (N=3) verify GEMV for q4_0 + static const bool q40_mc3 = (getenv("GGML_OPENCL_Q4_0_MC3") != nullptr); const bool use_q40_mc3 = q40_mc3 && (ne1 >= 2 && ne1 <= 4) && (ne01 < 32768); const bool use_bin = use_q4_0_bin_kernels(backend_ctx, src0); @@ -24273,7 +25600,7 @@ static void ggml_cl_mul_mat_q4_0_f32_adreno(ggml_backend_t backend, const ggml_t if (use_q40_mc3) { static bool warned = false; if (!warned) { - GGML_LOG_WARN("ggml_opencl: GGML_OPENCL_Q40_MC3 is bypassed by Q4_0 binary kernels\n"); + GGML_LOG_WARN("ggml_opencl: GGML_OPENCL_Q4_0_MC3 is bypassed by Q4_0 binary kernels\n"); warned = true; } } @@ -24651,11 +25978,8 @@ static void ggml_cl_mul_mat_q4_1_f32_adreno(ggml_backend_t backend, const ggml_t int N = ne1; int K = ne00; - // Multi-column (N=3) verify GEMV for q4_1: route the spec/MTP verify batch - // (ne1==3) onto the efficient GEMV path instead of the transposed-GEMM dead- - // zone (gemm_noshuffle_q4_1). Reuses the ne1==1 GEMV image setup. Opt-in via - // GGML_OPENCL_Q41_MC3=1. Per-layer only (ne01 < 32768). - static const bool q41_mc3 = (getenv("GGML_OPENCL_Q41_MC3") != nullptr); + // Multi-column (N=3) verify GEMV for q4_1 + static const bool q41_mc3 = (getenv("GGML_OPENCL_Q4_1_MC3") != nullptr); const bool use_q41_mc3 = q41_mc3 && (ne1 >= 2 && ne1 <= 4) && (ne01 < 32768); if (ne1 == 1 || use_q41_mc3) { @@ -25466,6 +26790,169 @@ static void ggml_cl_mul_mat_iq4_nl_f32_adreno(ggml_backend_t backend, const ggml #endif } +#ifdef GGML_OPENCL_USE_ADRENO_KERNELS +static bool ggml_cl_mul_mat_iq4_xs_dp4a(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { + ggml_backend_opencl_context * backend_ctx = (ggml_backend_opencl_context *)backend->context; + ggml_tensor_extra_cl_iq4_xs * extra0 = (ggml_tensor_extra_cl_iq4_xs *)src0->extra; + ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra; + ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; + + cl_ulong offset1 = extra1->offset + src1->view_offs; + cl_ulong offsetd = extrad->offset + dst->view_offs; + const int m = src0->ne[1]; + const int n = src1->ne[1]; + const int k = src0->ne[0]; + + const int tile_n = n <= 8 ? 8 : 32; + cl_kernel gemm = n <= 8 ? backend_ctx->kernel_gemm_noshuffle_iq4_xs_q8_1_dp4a_narrow + : backend_ctx->kernel_gemm_noshuffle_iq4_xs_q8_1_dp4a; + const char * env = getenv("GGML_OPENCL_IQ4_XS_DENSE_DP4A"); + if (!gemm || !backend_ctx->has_integer_dot || (env && env[0] == '0') || + n <= 1 || !ggml_is_contiguous(src1) || offset1 % backend_ctx->alignment != 0) { + return false; + } + + cl_int err; + cl_buffer_region region = { (size_t)offset1, (size_t)k*n*sizeof(float) }; + cl_mem activation = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err); + CL_CHECK(err); + const size_t n_blocks = (size_t)n*(k/32); + backend_ctx->prealloc_moe_qa.allocate(backend_ctx->context, (size_t)n*k); + backend_ctx->prealloc_moe_da.allocate(backend_ctx->context, n_blocks*sizeof(cl_half)); + backend_ctx->prealloc_moe_sa.allocate(backend_ctx->context, n_blocks*sizeof(cl_half)); + + cl_int nb = n_blocks; + cl_kernel quant = backend_ctx->kernel_quant_a_q8_1; + CL_CHECK(clSetKernelArg(quant, 0, sizeof(cl_mem), &activation)); + CL_CHECK(clSetKernelArg(quant, 1, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer)); + CL_CHECK(clSetKernelArg(quant, 2, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer)); + CL_CHECK(clSetKernelArg(quant, 3, sizeof(cl_mem), &backend_ctx->prealloc_moe_sa.buffer)); + CL_CHECK(clSetKernelArg(quant, 4, sizeof(cl_int), &nb)); + size_t q_global[] = { CEIL_DIV(n_blocks, 64)*64 }; + size_t q_local[] = { 64 }; + backend_ctx->enqueue_ndrange_kernel(quant, 1, q_global, q_local, dst); + + CL_CHECK(clSetKernelArg(gemm, 0, sizeof(cl_mem), &extra0->qs)); + CL_CHECK(clSetKernelArg(gemm, 1, sizeof(cl_mem), &extra0->sl)); + CL_CHECK(clSetKernelArg(gemm, 2, sizeof(cl_mem), &extra0->sh)); + CL_CHECK(clSetKernelArg(gemm, 3, sizeof(cl_mem), &extra0->d)); + CL_CHECK(clSetKernelArg(gemm, 4, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer)); + CL_CHECK(clSetKernelArg(gemm, 5, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer)); + CL_CHECK(clSetKernelArg(gemm, 6, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(gemm, 7, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(gemm, 8, sizeof(int), &m)); + CL_CHECK(clSetKernelArg(gemm, 9, sizeof(int), &n)); + CL_CHECK(clSetKernelArg(gemm, 10, sizeof(int), &k)); + size_t global[] = { 64, (size_t)CEIL_DIV(m, 64), (size_t)CEIL_DIV(n, tile_n) }; + size_t local[] = { 64, 1, 1 }; + backend_ctx->enqueue_ndrange_kernel(gemm, 3, global, local, dst); + CL_CHECK(clReleaseMemObject(activation)); + return true; +} + +static void ggml_cl_mul_mat_iq4_xs_f32_adreno(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { + ggml_backend_opencl_context * backend_ctx = (ggml_backend_opencl_context *)backend->context; + ggml_tensor_extra_cl_iq4_xs * extra0 = (ggml_tensor_extra_cl_iq4_xs *)src0->extra; + ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra; + ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; + + cl_ulong offset1 = extra1->offset + src1->view_offs; + cl_ulong offsetd = extrad->offset + dst->view_offs; + const int m = src0->ne[1]; + const int n = src1->ne[1]; + const int k = src0->ne[0]; + + if (n == 1 || !ggml_is_contiguous(src1) || offset1 % backend_ctx->alignment != 0) { + cl_kernel gemv = backend_ctx->kernel_gemv_noshuffle_iq4_xs_f32; + const int stride_d = dst->ne[0]; + const int stride_b = src1->nb[1]/sizeof(float); + CL_CHECK(clSetKernelArg(gemv, 0, sizeof(cl_mem), &extra0->qs)); + CL_CHECK(clSetKernelArg(gemv, 1, sizeof(cl_mem), &extra0->sl)); + CL_CHECK(clSetKernelArg(gemv, 2, sizeof(cl_mem), &extra0->sh)); + CL_CHECK(clSetKernelArg(gemv, 3, sizeof(cl_mem), &extra0->d)); + CL_CHECK(clSetKernelArg(gemv, 4, sizeof(cl_mem), &extra1->data_device)); + CL_CHECK(clSetKernelArg(gemv, 5, sizeof(cl_ulong), &offset1)); + CL_CHECK(clSetKernelArg(gemv, 6, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(gemv, 7, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(gemv, 8, sizeof(int), &k)); + CL_CHECK(clSetKernelArg(gemv, 9, sizeof(int), &m)); + CL_CHECK(clSetKernelArg(gemv, 10, sizeof(int), &stride_b)); + CL_CHECK(clSetKernelArg(gemv, 11, sizeof(int), &stride_d)); + const size_t nsg = backend_ctx->iq4_xs_mv_nsg; + size_t global[] = { (size_t)CEIL_DIV(m, 128)*64, (size_t)n*nsg, 1 }; + size_t local[] = { 64, nsg, 1 }; + backend_ctx->enqueue_ndrange_kernel(gemv, 3, global, local, dst); + return; + } + + if (ggml_cl_mul_mat_iq4_xs_dp4a(backend, src0, src1, dst)) { + return; + } + + cl_context context = backend_ctx->context; + cl_int err; + cl_buffer_region region = { (size_t)offset1, (size_t)k*n*sizeof(float) }; + cl_mem b_sub_buf = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err); + CL_CHECK(err); + + cl_image_format img_fmt = { CL_RGBA, CL_FLOAT }; + cl_image_desc img_desc = {}; + img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + img_desc.image_width = (size_t)k*n/4; + img_desc.buffer = b_sub_buf; + cl_mem b_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, nullptr, &err); + CL_CHECK(err); + + const int padded_n = CEIL_DIV(n, 8)*8; + region.origin = 0; + region.size = (size_t)k*padded_n*sizeof(cl_half); + backend_ctx->prealloc_act_trans.allocate(context, region.size); + cl_mem b_sub_buf_trans = clCreateSubBuffer(backend_ctx->prealloc_act_trans.buffer, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err); + CL_CHECK(err); + + img_fmt = { CL_RGBA, CL_HALF_FLOAT }; + img_desc.image_width = (size_t)k*padded_n/4; + img_desc.buffer = b_sub_buf_trans; + cl_mem b_img_trans = clCreateImage(context, 0, &img_fmt, &img_desc, nullptr, &err); + CL_CHECK(err); + + const int height_b = std::max(n/4, 1); + const int width_b = k/4; + const int padded_height_b = padded_n/4; + cl_kernel kernel = backend_ctx->kernel_transpose_32_16; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &b_img)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &b_img_trans)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(int), &height_b)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(int), &width_b)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &padded_height_b)); + size_t t_global[] = { (size_t)width_b, (size_t)padded_height_b }; + size_t t_local[] = { 1, 16 }; + backend_ctx->enqueue_ndrange_kernel(kernel, 2, t_global, t_local, dst); + + kernel = backend_ctx->kernel_gemm_noshuffle_iq4_xs_f32; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->qs)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0->sl)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0->sh)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra0->d)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &b_img_trans)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &m)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &padded_n)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &k)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &n)); + size_t global[] = { (size_t)CEIL_DIV(n, 8), (size_t)CEIL_DIV(m, 512)*128, 1 }; + size_t local[] = { 1, 128, 1 }; + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global, local, dst); + + CL_CHECK(clReleaseMemObject(b_img_trans)); + CL_CHECK(clReleaseMemObject(b_sub_buf_trans)); + CL_CHECK(clReleaseMemObject(b_img)); + CL_CHECK(clReleaseMemObject(b_sub_buf)); +} + +#endif + static void ggml_cl_mul_mat_q8_0_f32_adreno(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { #ifdef GGML_OPENCL_USE_ADRENO_KERNELS GGML_ASSERT(src0); @@ -25515,7 +27002,13 @@ static void ggml_cl_mul_mat_q8_0_f32_adreno(ggml_backend_t backend, const ggml_t int N = ne1; int K = ne00; - if (ne1 == 1) { + // Multi-column verify GEMV for ne1 2..4, reuses the ne1 == 1 image setup. + // Default on for X2E, GGML_OPENCL_Q8_0_MC=0/1 forces it. + static const char * q8_mc_env = getenv("GGML_OPENCL_Q8_0_MC"); + const bool q8_mc = q8_mc_env ? atoi(q8_mc_env) != 0 : backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E; + const bool use_q8_mc = q8_mc && ne1 >= 2 && ne1 <= 4; + + if (ne1 == 1 || use_q8_mc) { cl_mem q_img = nullptr; cl_mem b_sub_buf = nullptr; cl_mem b_img = nullptr; @@ -25541,6 +27034,30 @@ static void ggml_cl_mul_mat_q8_0_f32_adreno(ggml_backend_t backend, const ggml_t img_desc.buffer = b_sub_buf; CL_CHECK((b_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err)); + if (use_q8_mc) { + kernel = backend_ctx->kernel_gemv_noshuffle_q8_0_f32_mc; + + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &q_img)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q8_0->d)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &b_img)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne1)); + + size_t wavesize = backend_ctx->adreno_wave_size; + size_t local_work_size[] = { wavesize, 4, 1 }; + size_t global_work_size[] = { CEIL_DIV(M, wavesize)*wavesize, 4, 1 }; + + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); + + CL_CHECK(clReleaseMemObject(q_img)); + CL_CHECK(clReleaseMemObject(b_img)); + CL_CHECK(clReleaseMemObject(b_sub_buf)); + return; + } + // Split-K for small-M decode GEMVs. The base kernel puts one output row // per lane and splits K only inside one workgroup, so M is the sole source // of workgroup parallelism: gpt-oss's K and V projections are M=512 = 8 @@ -26309,15 +27826,8 @@ static void ggml_cl_mul_mat_q4_k_f32_adreno(ggml_backend_t backend, const ggml_t cl_uchar mask_d4 = 0x0F; cl_uchar mask_hi2 = 0xC0; - // Multi-column verify GEMV: route the spec/MTP verify batch (ne1==3 = 2 - // drafts + 1 bonus) onto the efficient GEMV path (subgroup-broadcast, no - // transpose) instead of the transposed-GEMM dead-zone. Reuses the ne1==1 - // GEMV setup (the activation image is already sized by N=ne1). Byte- - // identical. Opt-in via GGML_OPENCL_Q4K_MC3=1 while validating. - static const bool q4k_mc3 = (getenv("GGML_OPENCL_Q4K_MC3") != nullptr); - // Per-layer only (ne01 < 32768): the batched large-vocab lm_head at ne1==3 - // is left to the existing routing (corrupts on the Adreno GEMV path; x2- - // unified routes batched Q6_K lm_head to CPU). Per-layer mc3 is byte-identical. + // Multi-column verify GEMV + static const bool q4k_mc3 = (getenv("GGML_OPENCL_Q4_K_MC3") != nullptr); const bool use_mc3 = q4k_mc3 && (ne1 == 3) && (ne01 < 32768); const bool use_bin = use_q4_k_bin_kernels(backend_ctx, src0); @@ -26326,7 +27836,7 @@ static void ggml_cl_mul_mat_q4_k_f32_adreno(ggml_backend_t backend, const ggml_t if (use_mc3) { static bool warned = false; if (!warned) { - GGML_LOG_WARN("ggml_opencl: GGML_OPENCL_Q4K_MC3 is bypassed by Q4_K binary kernels\n"); + GGML_LOG_WARN("ggml_opencl: GGML_OPENCL_Q4_K_MC3 is bypassed by Q4_K binary kernels\n"); warned = true; } } @@ -27151,20 +28661,15 @@ static void ggml_cl_mul_mat_q6_K_f32_adreno(ggml_backend_t backend, const ggml_t cl_image_format img_fmt; cl_image_desc img_desc; - // subbuffer and image for activation - // Multi-column verify GEMV: route the spec/MTP verify q6_K matmuls (ne1==3) - // onto the efficient GEMV path instead of the transposed-GEMM dead-zone. - // Reuses the ne1==1 image setup (activation image sized by N=ne1). Byte- - // identical. Opt-in via GGML_OPENCL_Q6K_MC3=1 while validating. - static const bool q6k_mc3 = (getenv("GGML_OPENCL_Q6K_MC3") != nullptr); - // Per-layer only (ne01 < 32768): batched large-vocab lm_head stays on the - // existing path (x2-unified routes batched Q6_K lm_head to CPU; the Adreno - // GEMV corrupts it). Per-layer mc3 is byte-identical. + // Multi-column verify GEMV + static const bool q6k_mc3 = (getenv("GGML_OPENCL_Q6_K_MC3") != nullptr); const bool use_q6k_mc3 = q6k_mc3 && (ne1 == 3) && (ne01 < 32768); - // Batched verify lm_head/embed (ne1==3, tiled layout): multi-column tiled - // GEMV — streams the large lm_head weight once across the 3 verify columns - // (the #1 MTP bottleneck; mc3 above can't, it reads the noshuffle layout). - const bool use_q6k_tiled_mc = q6k_mc3 && (ne1 == 3) && (ne01 >= 32768) && use_q6k_tiled(backend_ctx, src0); + // Multi-column tiled vocab GEMV for verify + // streams the large lm_head weight once across the verify columns + static const char * q6k_tiled_mc_env = getenv("GGML_OPENCL_Q6_K_TILED_MC"); + const bool q6k_tiled_mc = q6k_tiled_mc_env ? atoi(q6k_tiled_mc_env) != 0 + : (q6k_mc3 || backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E); + const bool use_q6k_tiled_mc = q6k_tiled_mc && (ne1 >= 2 && ne1 <= 4) && (ne01 >= 32768) && use_q6k_tiled(backend_ctx, src0); const bool use_bin = use_q6_k_bin_kernels(backend_ctx, src0); @@ -27172,7 +28677,7 @@ static void ggml_cl_mul_mat_q6_K_f32_adreno(ggml_backend_t backend, const ggml_t if (use_q6k_mc3 || use_q6k_tiled_mc) { static bool warned = false; if (!warned) { - GGML_LOG_WARN("ggml_opencl: GGML_OPENCL_Q6K_MC3 is bypassed by Q6_K binary kernels\n"); + GGML_LOG_WARN("ggml_opencl: GGML_OPENCL_Q6_K_MC3 is bypassed by Q6_K binary kernels\n"); warned = true; } } @@ -27260,6 +28765,9 @@ static void ggml_cl_mul_mat_q6_K_f32_adreno(ggml_backend_t backend, const ggml_t CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_ulong), &offsetd)); CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_int), &ne00)); CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_int), &ne01)); + if (use_q6k_tiled_mc) { + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_int), &ne1)); + } const size_t gws_x = use_tiled ? (size_t) CEIL_DIV(ne01, 64) * 64 @@ -27827,8 +29335,9 @@ static void ggml_cl_mul_mat_q5_K_f32_adreno(ggml_backend_t backend, const ggml_t // (ne1==3) onto the efficient GEMV path instead of the transposed-GEMM dead- // zone (gemm_noshuffle_q5_k, the #2 chunk of MTP decode on a Q4_0-mix model // after q4_0 mc3). Reuses the ne1==1 GEMV image setup (q + qh + activations). - // Opt-in via GGML_OPENCL_Q5K_MC3=1. Per-layer only (ne01 < 32768). - static const bool q5k_mc3 = (getenv("GGML_OPENCL_Q5K_MC3") != nullptr); + // Default on for X2E, GGML_OPENCL_Q5_K_MC3=0/1 forces it. Per-layer only (ne01 < 32768). + static const char * q5k_mc3_env = getenv("GGML_OPENCL_Q5_K_MC3"); + const bool q5k_mc3 = q5k_mc3_env ? atoi(q5k_mc3_env) != 0 : backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E; const bool use_q5k_mc3 = q5k_mc3 && (ne1 >= 2 && ne1 <= 4) && (ne01 < 32768); const bool use_bin = use_q5_k_bin_kernels(backend_ctx, src0); @@ -27837,7 +29346,7 @@ static void ggml_cl_mul_mat_q5_K_f32_adreno(ggml_backend_t backend, const ggml_t if (use_q5k_mc3) { static bool warned = false; if (!warned) { - GGML_LOG_WARN("ggml_opencl: GGML_OPENCL_Q5K_MC3 is bypassed by Q5_K binary kernels\n"); + GGML_LOG_WARN("ggml_opencl: GGML_OPENCL_Q5_K_MC3 is bypassed by Q5_K binary kernels\n"); warned = true; } } @@ -29875,6 +31384,7 @@ static void ggml_cl_mul_mat(ggml_backend_t backend, const ggml_tensor * src0, co ggml_tensor_extra_cl_mxfp4 * extra0_mxfp4 = (ggml_tensor_extra_cl_mxfp4 *)soa0_src->extra; ggml_tensor_extra_cl_q8_0 * extra0_q8_0 = (ggml_tensor_extra_cl_q8_0 *)soa0_src->extra; ggml_tensor_extra_cl_iq4_nl * extra0_iq4_nl = (ggml_tensor_extra_cl_iq4_nl *)soa0_src->extra; + ggml_tensor_extra_cl_iq4_xs * extra0_iq4_xs = (ggml_tensor_extra_cl_iq4_xs *)soa0_src->extra; ggml_tensor_extra_cl_q2_K * extra0_q2_K = (ggml_tensor_extra_cl_q2_K *)soa0_src->extra; ggml_tensor_extra_cl_q3_K * extra0_q3_K = (ggml_tensor_extra_cl_q3_K *)soa0_src->extra; ggml_tensor_extra_cl_iq2_s * extra0_iq2_s = (ggml_tensor_extra_cl_iq2_s *)soa0_src->extra; @@ -30278,6 +31788,12 @@ static void ggml_cl_mul_mat(ggml_backend_t backend, const ggml_tensor * src0, co return; } + // iq4_xs x fp32 + if (src0t == GGML_TYPE_IQ4_XS && src1t == GGML_TYPE_F32 && extra0_iq4_xs->noshuffle) { + ggml_cl_mul_mat_iq4_xs_f32_adreno(backend, src0, src1, dst); + return; + } + // q8_0 x fp32 if (src0t == GGML_TYPE_Q8_0 && src1t == GGML_TYPE_F32 && enable_adreno_trans_weight(backend_ctx, src0)) { @@ -30369,9 +31885,15 @@ static void ggml_cl_mul_mat(ggml_backend_t backend, const ggml_tensor * src0, co // Override with GGML_OPENCL_A7X_F32_LM_BYPASS=0. static const char * a7x_f32lm_env = getenv("GGML_OPENCL_A7X_F32_LM_BYPASS"); static const bool a7x_f32lm_bypass = (a7x_f32lm_env == nullptr || a7x_f32lm_env[0] != '0'); + // The 64-column tile wastes work on small F32 matrices, including block rotations. + const bool f32_use_mv = (backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E || + (backend_ctx->kernel_mul_mv_f32_f32_r2 != nullptr && ne00 <= 512)) && + src0t == GGML_TYPE_F32 && ne00 >= 256 && ne00 <= 1024 && ne00 % 16 == 0 && ne01 <= 512 && ne11 <= 16 && + ne02 == 1 && ne12 == 1 && ne13 == 1 && + ggml_is_contiguous(src0) && ggml_is_contiguous(src1) && (offset0 | offset1) % 16 == 0; if (src1t == GGML_TYPE_F32 && ne00 % 16 == 0 && - ne11 > 1 && + ne11 > 1 && !f32_use_mv && !(a7x_f32lm_bypass && src0t == GGML_TYPE_F32 && ne11 > 8 && backend_ctx->adreno_gen == ADRENO_GPU_GEN::A7X)) { switch(src0t) { @@ -30381,8 +31903,9 @@ static void ggml_cl_mul_mat(ggml_backend_t backend, const ggml_tensor * src0, co // skinny f32 weight (GDN ssm_alpha/ssm_beta, M=32) it launches one // under-occupied WG at ~2.3% tile utilization. Route to a per-output // (m,n) GEMV (64-thread WG, K-split + __local reduce) instead. - // Opt-in GGML_OPENCL_F32_MC=1; 2D contiguous, small N + skinny M only. - static const bool f32_mc = (getenv("GGML_OPENCL_F32_MC") != nullptr); + // 2D contiguous, small N + skinny M only. Default on for X2E, GGML_OPENCL_F32_MC=0/1 forces it. + static const char * f32_mc_env = getenv("GGML_OPENCL_F32_MC"); + const bool f32_mc = f32_mc_env ? atoi(f32_mc_env) != 0 : backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E; if (f32_mc && ne11 >= 2 && ne11 <= 8 && ne01 <= 512 && (ne00 % 4 == 0) && ne02 == 1 && ne12 == 1 && ne13 == 1 && ggml_is_contiguous(src0) && ggml_is_contiguous(src1)) { @@ -30589,7 +32112,306 @@ static void ggml_cl_mul_mat(ggml_backend_t backend, const ggml_tensor * src0, co backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); return; } - case GGML_TYPE_Q2_0: { + case GGML_TYPE_Q2_0: { + if (ne11 < 32) { + break; + } + if (!ggml_is_contiguous(src0) || !ggml_is_contiguous(src1)) { + break; + } + + kernel = backend_ctx->kernel_mul_mm_q2_0_f32_l4_lm; + nth0 = (backend_ctx->gpu_family == INTEL) ? 64 : 128; + offset0 = src0->view_offs; + + int batch_stride_a = ne00*ne01; + int batch_stride_b = ne10*ne11; + int batch_stride_d = ne0*ne1; + + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q2_0->qs)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q2_0->d)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_ulong), &offset0)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra1->data_device)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_ulong), &offset1)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne02)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne11)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne12)); + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &ne10)); + CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &ne0)); + CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &batch_stride_a)); + CL_CHECK(clSetKernelArg(kernel, 16, sizeof(int), &batch_stride_b)); + CL_CHECK(clSetKernelArg(kernel, 17, sizeof(int), &batch_stride_d)); + CL_CHECK(clSetKernelArg(kernel, 18, sizeof(int), &r2)); + CL_CHECK(clSetKernelArg(kernel, 19, sizeof(int), &r3)); + + size_t global_work_size[] = {(size_t)(CEIL_DIV(ne01, 64)*nth0), (size_t)CEIL_DIV(ne11, 64), (size_t)ne12*ne13}; + size_t local_work_size[] = {(size_t)nth0, 1, 1}; + + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); + return; + } + case GGML_TYPE_Q4_0: { + if (ne11 < 32) { + break; + } + if (!ggml_is_contiguous(src0) || !ggml_is_contiguous(src1)) { + break; + } + + kernel = backend_ctx->kernel_mul_mm_q4_0_f32_l4_lm; + nth0 = 128; // calculated as (BM*BN)/(TM*TN) + + int batch_stride_a = ne00*ne01; + int batch_stride_b = ne10*ne11; + int batch_stride_d = ne0*ne1; + + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q4_0->q)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q4_0->d)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne02)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne11)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne12)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne10)); // stride_a + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne10)); // stride_b + CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &ne01)); // stride_d + CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &batch_stride_a)); + CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &batch_stride_b)); + CL_CHECK(clSetKernelArg(kernel, 16, sizeof(int), &batch_stride_d)); + CL_CHECK(clSetKernelArg(kernel, 17, sizeof(int), &r2)); + CL_CHECK(clSetKernelArg(kernel, 18, sizeof(int), &r3)); + + // 64 is block tile size BM and BN - change here when BM and BN in the kernel are changed. + size_t global_work_size[] = {(size_t)(CEIL_DIV(ne01, 64)*nth0), (size_t)(CEIL_DIV(ne11, 64)), (size_t)ne12*ne13}; + size_t local_work_size[] = {(size_t)nth0, 1, 1}; + + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); + return; + } + case GGML_TYPE_Q4_1: { + if (ne11 < 32) { + break; + } + if (!ggml_is_contiguous(src0) || !ggml_is_contiguous(src1)) { + break; + } + + kernel = backend_ctx->kernel_mul_mm_q4_1_f32_l4_lm; + nth0 = 128; // calculated as (BM*BN)/(TM*TN) + + int batch_stride_a = ne00*ne01; + int batch_stride_b = ne10*ne11; + int batch_stride_d = ne0*ne1; + + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q4_1->q)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q4_1->d)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0_q4_1->m)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra1->data_device)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_ulong), &offset1)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne02)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne11)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne12)); + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne10)); // stride_a + CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &ne10)); // stride_b + CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &ne01)); // stride_d + CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &batch_stride_a)); + CL_CHECK(clSetKernelArg(kernel, 16, sizeof(int), &batch_stride_b)); + CL_CHECK(clSetKernelArg(kernel, 17, sizeof(int), &batch_stride_d)); + CL_CHECK(clSetKernelArg(kernel, 18, sizeof(int), &r2)); + CL_CHECK(clSetKernelArg(kernel, 19, sizeof(int), &r3)); + + // 64 is block tile size BM and BN - change here when BM and BN in the kernel are changed. + size_t global_work_size[] = {(size_t)(CEIL_DIV(ne01, 64)*nth0), (size_t)(CEIL_DIV(ne11, 64)), (size_t)ne12*ne13}; + size_t local_work_size[] = {(size_t)nth0, 1, 1}; + + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); + return; + } + case GGML_TYPE_Q5_0: { + if (ne11 < 32) { + break; + } + if (!ggml_is_contiguous(src0) || !ggml_is_contiguous(src1)) { + break; + } + + kernel = backend_ctx->kernel_mul_mm_q5_0_f32_l4_lm; + nth0 = 128; // calculated as (BM*BN)/(TM*TN) + + int batch_stride_a = ne00*ne01; + int batch_stride_b = ne10*ne11; + int batch_stride_d = ne0*ne1; + + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q5_0->qs)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q5_0->qh)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0_q5_0->d)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra1->data_device)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_ulong), &offset1)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne02)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne11)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne12)); + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne10)); // stride_a + CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &ne10)); // stride_b + CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &ne01)); // stride_d + CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &batch_stride_a)); + CL_CHECK(clSetKernelArg(kernel, 16, sizeof(int), &batch_stride_b)); + CL_CHECK(clSetKernelArg(kernel, 17, sizeof(int), &batch_stride_d)); + CL_CHECK(clSetKernelArg(kernel, 18, sizeof(int), &r2)); + CL_CHECK(clSetKernelArg(kernel, 19, sizeof(int), &r3)); + + // 64 is block tile size BM and BN - change here when BM and BN in the kernel are changed. + size_t global_work_size[] = {(size_t)(CEIL_DIV(ne01, 64)*nth0), (size_t)(CEIL_DIV(ne11, 64)), (size_t)ne12*ne13}; + size_t local_work_size[] = {(size_t)nth0, 1, 1}; + + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); + return; + } + case GGML_TYPE_Q5_1: { + if (ne11 < 32) { + break; + } + if (!ggml_is_contiguous(src0) || !ggml_is_contiguous(src1)) { + break; + } + + kernel = backend_ctx->kernel_mul_mm_q5_1_f32_l4_lm; + nth0 = 128; // calculated as (BM*BN)/(TM*TN) + + int batch_stride_a = ne00*ne01; + int batch_stride_b = ne10*ne11; + int batch_stride_d = ne0*ne1; + + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q5_1->qs)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q5_1->qh)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0_q5_1->d)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra0_q5_1->m)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extra1->data_device)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offset1)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne02)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne11)); + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne12)); + CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &ne10)); // stride_a + CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &ne10)); // stride_b + CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &ne01)); // stride_d + CL_CHECK(clSetKernelArg(kernel, 16, sizeof(int), &batch_stride_a)); + CL_CHECK(clSetKernelArg(kernel, 17, sizeof(int), &batch_stride_b)); + CL_CHECK(clSetKernelArg(kernel, 18, sizeof(int), &batch_stride_d)); + CL_CHECK(clSetKernelArg(kernel, 19, sizeof(int), &r2)); + CL_CHECK(clSetKernelArg(kernel, 20, sizeof(int), &r3)); + + // 64 is block tile size BM and BN - change here when BM and BN in the kernel are changed. + size_t global_work_size[] = {(size_t)(CEIL_DIV(ne01, 64)*nth0), (size_t)(CEIL_DIV(ne11, 64)), (size_t)ne12*ne13}; + size_t local_work_size[] = {(size_t)nth0, 1, 1}; + + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); + return; + } + case GGML_TYPE_Q8_0: { + if (ne11 < 32) { + break; + } + if (!ggml_is_contiguous(src0) || !ggml_is_contiguous(src1)) { + break; + } + + kernel = backend_ctx->kernel_mul_mm_q8_0_f32_l4_lm; + nth0 = 128; // calculated as (BM*BN)/(TM*TN) + + int batch_stride_a = ne00*ne01; + int batch_stride_b = ne10*ne11; + int batch_stride_d = ne0*ne1; + + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q8_0->q)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q8_0->d)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne02)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne11)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne12)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne10)); // stride_a + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne10)); // stride_b + CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &ne01)); // stride_d + CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &batch_stride_a)); + CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &batch_stride_b)); + CL_CHECK(clSetKernelArg(kernel, 16, sizeof(int), &batch_stride_d)); + CL_CHECK(clSetKernelArg(kernel, 17, sizeof(int), &r2)); + CL_CHECK(clSetKernelArg(kernel, 18, sizeof(int), &r3)); + + // 64 is block tile size BM and BN - change here when BM and BN in the kernel are changed. + size_t global_work_size[] = {(size_t)(CEIL_DIV(ne01, 64)*nth0), (size_t)(CEIL_DIV(ne11, 64)), (size_t)ne12*ne13}; + size_t local_work_size[] = {(size_t)nth0, 1, 1}; + + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); + return; + } + case GGML_TYPE_IQ4_NL: { + if (ne11 < 32) { + break; + } + if (!ggml_is_contiguous(src0) || !ggml_is_contiguous(src1)) { + break; + } + + kernel = backend_ctx->kernel_mul_mm_iq4_nl_f32_l4_lm; + nth0 = 128; // calculated as (BM*BN)/(TM*TN) + + int batch_stride_a = ne00*ne01; + int batch_stride_b = ne10*ne11; + int batch_stride_d = ne0*ne1; + + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_iq4_nl->q)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_iq4_nl->d)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne02)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne11)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne12)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne10)); // stride_a + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne10)); // stride_b + CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &ne01)); // stride_d + CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &batch_stride_a)); + CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &batch_stride_b)); + CL_CHECK(clSetKernelArg(kernel, 16, sizeof(int), &batch_stride_d)); + CL_CHECK(clSetKernelArg(kernel, 17, sizeof(int), &r2)); + CL_CHECK(clSetKernelArg(kernel, 18, sizeof(int), &r3)); + + // 64 is block tile size BM and BN - change here when BM and BN in the kernel are changed. + size_t global_work_size[] = {(size_t)(CEIL_DIV(ne01, 64)*nth0), (size_t)(CEIL_DIV(ne11, 64)), (size_t)ne12*ne13}; + size_t local_work_size[] = {(size_t)nth0, 1, 1}; + + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); + return; + } + case GGML_TYPE_IQ4_XS: { if (ne11 < 32) { break; } @@ -30597,7 +32419,7 @@ static void ggml_cl_mul_mat(ggml_backend_t backend, const ggml_tensor * src0, co break; } - kernel = backend_ctx->kernel_mul_mm_q2_0_f32_l4_lm; + kernel = backend_ctx->kernel_mul_mm_iq4_xs_f32_l4_lm; nth0 = (backend_ctx->gpu_family == INTEL) ? 64 : 128; offset0 = src0->view_offs; @@ -30605,26 +32427,28 @@ static void ggml_cl_mul_mat(ggml_backend_t backend, const ggml_tensor * src0, co int batch_stride_b = ne10*ne11; int batch_stride_d = ne0*ne1; - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q2_0->qs)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q2_0->d)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_ulong), &offset0)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra1->data_device)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_ulong), &offset1)); - CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_mem), &extrad->data_device)); - CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_ulong), &offsetd)); - CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne00)); - CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne01)); - CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne02)); - CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne11)); - CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne12)); - CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne00)); - CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &ne10)); - CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &ne0)); - CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &batch_stride_a)); - CL_CHECK(clSetKernelArg(kernel, 16, sizeof(int), &batch_stride_b)); - CL_CHECK(clSetKernelArg(kernel, 17, sizeof(int), &batch_stride_d)); - CL_CHECK(clSetKernelArg(kernel, 18, sizeof(int), &r2)); - CL_CHECK(clSetKernelArg(kernel, 19, sizeof(int), &r3)); + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_iq4_xs->qs)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_iq4_xs->sl)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0_iq4_xs->sh)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra0_iq4_xs->d)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_ulong), &offset0)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_mem), &extra1->data_device)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_ulong), &offset1)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne02)); + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne11)); + CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &ne12)); + CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &ne10)); + CL_CHECK(clSetKernelArg(kernel, 16, sizeof(int), &ne0)); + CL_CHECK(clSetKernelArg(kernel, 17, sizeof(int), &batch_stride_a)); + CL_CHECK(clSetKernelArg(kernel, 18, sizeof(int), &batch_stride_b)); + CL_CHECK(clSetKernelArg(kernel, 19, sizeof(int), &batch_stride_d)); + CL_CHECK(clSetKernelArg(kernel, 20, sizeof(int), &r2)); + CL_CHECK(clSetKernelArg(kernel, 21, sizeof(int), &r3)); size_t global_work_size[] = {(size_t)(CEIL_DIV(ne01, 64)*nth0), (size_t)CEIL_DIV(ne11, 64), (size_t)ne12*ne13}; size_t local_work_size[] = {(size_t)nth0, 1, 1}; @@ -30632,262 +32456,6 @@ static void ggml_cl_mul_mat(ggml_backend_t backend, const ggml_tensor * src0, co backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); return; } - case GGML_TYPE_Q4_0: { - if (ne11 < 32) { - break; - } - if (!ggml_is_contiguous(src0) || !ggml_is_contiguous(src1)) { - break; - } - - kernel = backend_ctx->kernel_mul_mm_q4_0_f32_l4_lm; - nth0 = 128; // calculated as (BM*BN)/(TM*TN) - - int batch_stride_a = ne00*ne01; - int batch_stride_b = ne10*ne11; - int batch_stride_d = ne0*ne1; - - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q4_0->q)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q4_0->d)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device)); - CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd)); - CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne00)); - CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne01)); - CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne02)); - CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne11)); - CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne12)); - CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne10)); // stride_a - CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne10)); // stride_b - CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &ne01)); // stride_d - CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &batch_stride_a)); - CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &batch_stride_b)); - CL_CHECK(clSetKernelArg(kernel, 16, sizeof(int), &batch_stride_d)); - CL_CHECK(clSetKernelArg(kernel, 17, sizeof(int), &r2)); - CL_CHECK(clSetKernelArg(kernel, 18, sizeof(int), &r3)); - - // 64 is block tile size BM and BN - change here when BM and BN in the kernel are changed. - size_t global_work_size[] = {(size_t)(CEIL_DIV(ne01, 64)*nth0), (size_t)(CEIL_DIV(ne11, 64)), (size_t)ne12*ne13}; - size_t local_work_size[] = {(size_t)nth0, 1, 1}; - - backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); - return; - } - case GGML_TYPE_Q4_1: { - if (ne11 < 32) { - break; - } - if (!ggml_is_contiguous(src0) || !ggml_is_contiguous(src1)) { - break; - } - - kernel = backend_ctx->kernel_mul_mm_q4_1_f32_l4_lm; - nth0 = 128; // calculated as (BM*BN)/(TM*TN) - - int batch_stride_a = ne00*ne01; - int batch_stride_b = ne10*ne11; - int batch_stride_d = ne0*ne1; - - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q4_1->q)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q4_1->d)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0_q4_1->m)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra1->data_device)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_ulong), &offset1)); - CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_mem), &extrad->data_device)); - CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_ulong), &offsetd)); - CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne00)); - CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne01)); - CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne02)); - CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne11)); - CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne12)); - CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne10)); // stride_a - CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &ne10)); // stride_b - CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &ne01)); // stride_d - CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &batch_stride_a)); - CL_CHECK(clSetKernelArg(kernel, 16, sizeof(int), &batch_stride_b)); - CL_CHECK(clSetKernelArg(kernel, 17, sizeof(int), &batch_stride_d)); - CL_CHECK(clSetKernelArg(kernel, 18, sizeof(int), &r2)); - CL_CHECK(clSetKernelArg(kernel, 19, sizeof(int), &r3)); - - // 64 is block tile size BM and BN - change here when BM and BN in the kernel are changed. - size_t global_work_size[] = {(size_t)(CEIL_DIV(ne01, 64)*nth0), (size_t)(CEIL_DIV(ne11, 64)), (size_t)ne12*ne13}; - size_t local_work_size[] = {(size_t)nth0, 1, 1}; - - backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); - return; - } - case GGML_TYPE_Q5_0: { - if (ne11 < 32) { - break; - } - if (!ggml_is_contiguous(src0) || !ggml_is_contiguous(src1)) { - break; - } - - kernel = backend_ctx->kernel_mul_mm_q5_0_f32_l4_lm; - nth0 = 128; // calculated as (BM*BN)/(TM*TN) - - int batch_stride_a = ne00*ne01; - int batch_stride_b = ne10*ne11; - int batch_stride_d = ne0*ne1; - - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q5_0->qs)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q5_0->qh)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0_q5_0->d)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra1->data_device)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_ulong), &offset1)); - CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_mem), &extrad->data_device)); - CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_ulong), &offsetd)); - CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne00)); - CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne01)); - CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne02)); - CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne11)); - CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne12)); - CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne10)); // stride_a - CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &ne10)); // stride_b - CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &ne01)); // stride_d - CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &batch_stride_a)); - CL_CHECK(clSetKernelArg(kernel, 16, sizeof(int), &batch_stride_b)); - CL_CHECK(clSetKernelArg(kernel, 17, sizeof(int), &batch_stride_d)); - CL_CHECK(clSetKernelArg(kernel, 18, sizeof(int), &r2)); - CL_CHECK(clSetKernelArg(kernel, 19, sizeof(int), &r3)); - - // 64 is block tile size BM and BN - change here when BM and BN in the kernel are changed. - size_t global_work_size[] = {(size_t)(CEIL_DIV(ne01, 64)*nth0), (size_t)(CEIL_DIV(ne11, 64)), (size_t)ne12*ne13}; - size_t local_work_size[] = {(size_t)nth0, 1, 1}; - - backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); - return; - } - case GGML_TYPE_Q5_1: { - if (ne11 < 32) { - break; - } - if (!ggml_is_contiguous(src0) || !ggml_is_contiguous(src1)) { - break; - } - - kernel = backend_ctx->kernel_mul_mm_q5_1_f32_l4_lm; - nth0 = 128; // calculated as (BM*BN)/(TM*TN) - - int batch_stride_a = ne00*ne01; - int batch_stride_b = ne10*ne11; - int batch_stride_d = ne0*ne1; - - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q5_1->qs)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q5_1->qh)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0_q5_1->d)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra0_q5_1->m)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extra1->data_device)); - CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offset1)); - CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_mem), &extrad->data_device)); - CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &offsetd)); - CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne00)); - CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne01)); - CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne02)); - CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne11)); - CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne12)); - CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &ne10)); // stride_a - CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &ne10)); // stride_b - CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &ne01)); // stride_d - CL_CHECK(clSetKernelArg(kernel, 16, sizeof(int), &batch_stride_a)); - CL_CHECK(clSetKernelArg(kernel, 17, sizeof(int), &batch_stride_b)); - CL_CHECK(clSetKernelArg(kernel, 18, sizeof(int), &batch_stride_d)); - CL_CHECK(clSetKernelArg(kernel, 19, sizeof(int), &r2)); - CL_CHECK(clSetKernelArg(kernel, 20, sizeof(int), &r3)); - - // 64 is block tile size BM and BN - change here when BM and BN in the kernel are changed. - size_t global_work_size[] = {(size_t)(CEIL_DIV(ne01, 64)*nth0), (size_t)(CEIL_DIV(ne11, 64)), (size_t)ne12*ne13}; - size_t local_work_size[] = {(size_t)nth0, 1, 1}; - - backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); - return; - } - case GGML_TYPE_Q8_0: { - if (ne11 < 32) { - break; - } - if (!ggml_is_contiguous(src0) || !ggml_is_contiguous(src1)) { - break; - } - - kernel = backend_ctx->kernel_mul_mm_q8_0_f32_l4_lm; - nth0 = 128; // calculated as (BM*BN)/(TM*TN) - - int batch_stride_a = ne00*ne01; - int batch_stride_b = ne10*ne11; - int batch_stride_d = ne0*ne1; - - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q8_0->q)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q8_0->d)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device)); - CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd)); - CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne00)); - CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne01)); - CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne02)); - CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne11)); - CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne12)); - CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne10)); // stride_a - CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne10)); // stride_b - CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &ne01)); // stride_d - CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &batch_stride_a)); - CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &batch_stride_b)); - CL_CHECK(clSetKernelArg(kernel, 16, sizeof(int), &batch_stride_d)); - CL_CHECK(clSetKernelArg(kernel, 17, sizeof(int), &r2)); - CL_CHECK(clSetKernelArg(kernel, 18, sizeof(int), &r3)); - - // 64 is block tile size BM and BN - change here when BM and BN in the kernel are changed. - size_t global_work_size[] = {(size_t)(CEIL_DIV(ne01, 64)*nth0), (size_t)(CEIL_DIV(ne11, 64)), (size_t)ne12*ne13}; - size_t local_work_size[] = {(size_t)nth0, 1, 1}; - - backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); - return; - } - case GGML_TYPE_IQ4_NL: { - if (ne11 < 32) { - break; - } - if (!ggml_is_contiguous(src0) || !ggml_is_contiguous(src1)) { - break; - } - - kernel = backend_ctx->kernel_mul_mm_iq4_nl_f32_l4_lm; - nth0 = 128; // calculated as (BM*BN)/(TM*TN) - - int batch_stride_a = ne00*ne01; - int batch_stride_b = ne10*ne11; - int batch_stride_d = ne0*ne1; - - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_iq4_nl->q)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_iq4_nl->d)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device)); - CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd)); - CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne00)); - CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne01)); - CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne02)); - CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne11)); - CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne12)); - CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne10)); // stride_a - CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne10)); // stride_b - CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &ne01)); // stride_d - CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &batch_stride_a)); - CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &batch_stride_b)); - CL_CHECK(clSetKernelArg(kernel, 16, sizeof(int), &batch_stride_d)); - CL_CHECK(clSetKernelArg(kernel, 17, sizeof(int), &r2)); - CL_CHECK(clSetKernelArg(kernel, 18, sizeof(int), &r3)); - - // 64 is block tile size BM and BN - change here when BM and BN in the kernel are changed. - size_t global_work_size[] = {(size_t)(CEIL_DIV(ne01, 64)*nth0), (size_t)(CEIL_DIV(ne11, 64)), (size_t)ne12*ne13}; - size_t local_work_size[] = {(size_t)nth0, 1, 1}; - - backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); - return; - } case GGML_TYPE_Q2_K: { if (ne11 < 32) { break; @@ -31515,6 +33083,9 @@ static void ggml_cl_mul_mat(ggml_backend_t backend, const ggml_tensor * src0, co GGML_ASSERT(src1t == GGML_TYPE_F32); kernel = backend_ctx->kernel_mul_mat_f32_f32; nrows = 4; + if (f32_use_mv && ne11 > 1 && backend_ctx->kernel_mul_mv_f32_f32_r2 != nullptr) { + kernel = backend_ctx->kernel_mul_mv_f32_f32_r2; + } if (backend_ctx->gpu_family == INTEL) { nth0 = 32; @@ -32264,6 +33835,70 @@ static void ggml_cl_mul_mat(ggml_backend_t backend, const ggml_tensor * src0, co CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne1)); CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &r2)); CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &r3)); +#endif // GGML_OPENCL_SOA_Q + break; + } + case GGML_TYPE_IQ4_XS: { + if (backend_ctx->gpu_family == INTEL) { + nth0 = 16; + nth1 = 1; + ndst = 4; + } else if (backend_ctx->gpu_family == ADRENO) { + nth0 = 64; + nth1 = 1; + ndst = 4; + } else { + GGML_ASSERT(false && "TODO: Unknown GPU"); + } + +#ifdef GGML_OPENCL_SOA_Q + kernel = backend_ctx->kernel_mul_mv_iq4_xs_f32_flat; + offset0 = src0->view_offs; + + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_iq4_xs->qs)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_iq4_xs->sl)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0_iq4_xs->sh)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra0_iq4_xs->d)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_ulong), &offset0)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_mem), &extra1->data_device)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_ulong), &offset1)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ulong), &nb01)); + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nb02)); + CL_CHECK(clSetKernelArg(kernel, 13, sizeof(cl_ulong), &nb03)); + CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &ne12)); + CL_CHECK(clSetKernelArg(kernel, 15, sizeof(cl_ulong), &nb11)); + CL_CHECK(clSetKernelArg(kernel, 16, sizeof(cl_ulong), &nb12)); + CL_CHECK(clSetKernelArg(kernel, 17, sizeof(cl_ulong), &nb13)); + CL_CHECK(clSetKernelArg(kernel, 18, sizeof(int), &ne0)); + CL_CHECK(clSetKernelArg(kernel, 19, sizeof(int), &ne1)); + CL_CHECK(clSetKernelArg(kernel, 20, sizeof(int), &r2)); + CL_CHECK(clSetKernelArg(kernel, 21, sizeof(int), &r3)); +#else + kernel = backend_ctx->kernel_mul_mv_iq4_xs_f32; + + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &nb01)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &nb02)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb03)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne12)); + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nb11)); + CL_CHECK(clSetKernelArg(kernel, 13, sizeof(cl_ulong), &nb12)); + CL_CHECK(clSetKernelArg(kernel, 14, sizeof(cl_ulong), &nb13)); + CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &ne0)); + CL_CHECK(clSetKernelArg(kernel, 16, sizeof(int), &ne1)); + CL_CHECK(clSetKernelArg(kernel, 17, sizeof(int), &r2)); + CL_CHECK(clSetKernelArg(kernel, 18, sizeof(int), &r3)); #endif // GGML_OPENCL_SOA_Q break; } @@ -33147,7 +34782,7 @@ static void ggml_cl_mul_mat(ggml_backend_t backend, const ggml_tensor * src0, co src0t == GGML_TYPE_Q1_0 || src0t == GGML_TYPE_Q2_0 || src0t == GGML_TYPE_IQ4_NL || - src0t == GGML_TYPE_Q2_K || src0t == GGML_TYPE_Q3_K || src0t == GGML_TYPE_IQ2_S || src0t == GGML_TYPE_IQ2_XS || src0t == GGML_TYPE_IQ2_XXS || src0t == GGML_TYPE_IQ1_S || src0t == GGML_TYPE_IQ1_M || src0t == GGML_TYPE_IQ3_S || src0t == GGML_TYPE_IQ3_XXS) { + src0t == GGML_TYPE_Q2_K || src0t == GGML_TYPE_Q3_K || src0t == GGML_TYPE_IQ2_S || src0t == GGML_TYPE_IQ2_XS || src0t == GGML_TYPE_IQ2_XXS || src0t == GGML_TYPE_IQ1_S || src0t == GGML_TYPE_IQ1_M || src0t == GGML_TYPE_IQ3_S || src0t == GGML_TYPE_IQ3_XXS || src0t == GGML_TYPE_IQ4_XS) { // Each SIMD group produces N_DST values in the result. Assuming each // workgroup has N_SIMDGROUP SIMD groups, then each workgroup will // produce N_DST*N_SIMDGROUP values in the result. Hence, the grid size @@ -33212,8 +34847,9 @@ static void ggml_cl_mul_mat(ggml_backend_t backend, const ggml_tensor * src0, co backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); } else { int64_t ny = (ne11 + nrows - 1)/nrows; + int64_t nx = kernel == backend_ctx->kernel_mul_mv_f32_f32_r2 ? CEIL_DIV(ne01, 2) : ne01; - size_t global_work_size[] = {(size_t)ne01*nth0, (size_t)ny*nth1, (size_t)ne12*ne13}; + size_t global_work_size[] = {(size_t)nx*nth0, (size_t)ny*nth1, (size_t)ne12*ne13}; size_t local_work_size[] = {(size_t)nth0, (size_t)nth1, 1}; backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); @@ -33248,7 +34884,11 @@ static void moe_router_reoerder(ggml_backend_t backend, const ggml_tensor * src, cl_mem emap_buf = clCreateSubBuffer(backend_ctx->prealloc_emap.buffer, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err); CL_CHECK(err); - backend_ctx->prealloc_hist.allocate(backend_ctx->context, sizeof(int) * ne02); + if (sizeof(int) * ne02 > backend_ctx->prealloc_hist.size) { + backend_ctx->prealloc_hist.allocate(backend_ctx->context, sizeof(int) * ne02); + const int zero = 0; + CL_CHECK(clEnqueueFillBuffer(backend_ctx->queue, backend_ctx->prealloc_hist.buffer, &zero, sizeof(zero), 0, sizeof(int) * ne02, 0, nullptr, nullptr)); + } region.origin = 0; region.size = sizeof(int) * ne02; cl_mem hist_buf = clCreateSubBuffer(backend_ctx->prealloc_hist.buffer, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err); @@ -33372,7 +35012,125 @@ static void moe_router_reoerder(ggml_backend_t backend, const ggml_tensor * src, CL_CHECK(clReleaseMemObject(emap_buf)); } +// Use the per-token MoE GEMV instead of the tiled expert GEMM for small batches. +// The GEMV costs one weight pass per token, the GEMM pads every expert to a +// 32-token tile, so the GEMV wins while tokens rarely share an expert. +// Default on for X2E up to 4 tokens while 2 * tokens * used < experts. +// GGML_OPENCL_MOE_GEMV_MAX_TOK overrides with a plain token limit (1 = decode only). +static bool use_adreno_moe_gemv_small_batch(const ggml_backend_opencl_context * backend_ctx, int n_tokens, int n_used, int n_expert) { + if (n_tokens <= 1) { + return true; + } + static const char * env = getenv("GGML_OPENCL_MOE_GEMV_MAX_TOK"); + if (env) { + return n_tokens <= std::max(1, atoi(env)); + } + return backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E && n_tokens <= 4 && + 2 * (int64_t) n_tokens * n_used < (int64_t) n_expert; +} + +static void ggml_cl_mul_mat_id_float(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { + ggml_backend_opencl_context * backend_ctx = (ggml_backend_opencl_context *) backend->context; + cl_kernel kernel = backend_ctx->kernel_mul_mv_id_f32_f32; + const ggml_tensor * src2 = dst->src[2]; + ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *) src0->extra; + ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *) src1->extra; + ggml_tensor_extra_cl * extra2 = (ggml_tensor_extra_cl *) src2->extra; + ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *) dst->extra; + + const cl_ulong offset0 = extra0->offset + src0->view_offs; + const cl_ulong offset1 = extra1->offset + src1->view_offs; + const cl_ulong offset2 = extra2->offset + src2->view_offs; + const cl_ulong offsetd = extrad->offset + dst->view_offs; + + const int ncols = src0->ne[0]; + const int ne11 = src1->ne[1]; + const int nrows = dst->ne[0]; + // BF16 tensors are stored as F16 on the device. Activations remain F32. + const int src0_f16 = src0->type == GGML_TYPE_F32 ? 0 : 1; + + const cl_ulong nb01 = src0->nb[1]; + const cl_ulong nb02 = src0->nb[2]; + const cl_ulong nb11 = src1->nb[1]; + const cl_ulong nb12 = src1->nb[2]; + const cl_ulong nb20 = src2->nb[0]; + const cl_ulong nb21 = src2->nb[1]; + const cl_ulong nb1 = dst->nb[1]; + const cl_ulong nb2 = dst->nb[2]; + + // The router uses the IDs row stride as its expert count. + const size_t router_experts = src2->nb[1]/sizeof(int32_t); + if (src1->ne[2] >= 16 && ncols >= 64 && nrows >= 32 && ggml_is_contiguous(dst) && + src2->nb[0] == sizeof(int32_t) && src2->nb[1] % sizeof(int32_t) == 0 && + router_experts >= (size_t) src0->ne[2] && router_experts <= 1024 && + dst->ne[1]*dst->ne[2] >= (int64_t) router_experts*4 && + dst->ne[1]*dst->ne[2] >= 16384/nrows && + dst->ne[1]*dst->ne[2] <= INT_MAX - 32*(int64_t) router_experts) { + const int nused = dst->ne[1]; + const size_t max_tiles = (size_t) nused*dst->ne[2]/32 + router_experts; + moe_router_reoerder(backend, src2, nused); + backend_ctx->toggle_reorder = true; + const float zero = 0.0f; + CL_CHECK(clEnqueueFillBuffer(backend_ctx->queue, extrad->data_device, &zero, sizeof(zero), offsetd, ggml_nbytes(dst), 0, nullptr, nullptr)); + kernel = backend_ctx->kernel_mul_mm_id_f32_f32; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &backend_ctx->prealloc_post_router.buffer)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_mem), &backend_ctx->prealloc_emap.buffer)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_mem), &backend_ctx->prealloc_total_tiles.buffer)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ncols)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &nrows)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne11)); + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &nused)); + CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &src0_f16)); + CL_CHECK(clSetKernelArg(kernel, 14, sizeof(cl_ulong), &nb01)); + CL_CHECK(clSetKernelArg(kernel, 15, sizeof(cl_ulong), &nb02)); + CL_CHECK(clSetKernelArg(kernel, 16, sizeof(cl_ulong), &nb11)); + CL_CHECK(clSetKernelArg(kernel, 17, sizeof(cl_ulong), &nb12)); + CL_CHECK(clSetKernelArg(kernel, 18, sizeof(cl_ulong), &nb1)); + CL_CHECK(clSetKernelArg(kernel, 19, sizeof(cl_ulong), &nb2)); + size_t global_work_size[] = {(size_t) ((nrows + 63)/64)*128, max_tiles}; + size_t local_work_size[] = {128, 1}; + backend_ctx->enqueue_ndrange_kernel(kernel, 2, global_work_size, local_work_size, dst); + return; + } + + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extra2->data_device)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offset2)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ncols)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne11)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &src0_f16)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ulong), &nb01)); + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nb02)); + CL_CHECK(clSetKernelArg(kernel, 13, sizeof(cl_ulong), &nb11)); + CL_CHECK(clSetKernelArg(kernel, 14, sizeof(cl_ulong), &nb12)); + CL_CHECK(clSetKernelArg(kernel, 15, sizeof(cl_ulong), &nb20)); + CL_CHECK(clSetKernelArg(kernel, 16, sizeof(cl_ulong), &nb21)); + CL_CHECK(clSetKernelArg(kernel, 17, sizeof(cl_ulong), &nb1)); + CL_CHECK(clSetKernelArg(kernel, 18, sizeof(cl_ulong), &nb2)); + CL_CHECK(clSetKernelArg(kernel, 19, sizeof(int), &nrows)); + + size_t global_work_size[] = {(size_t) ((dst->ne[0] + 3)/4)*64, (size_t) dst->ne[1], (size_t) dst->ne[2]}; + size_t local_work_size[] = {64, 1, 1}; + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); +} + static void ggml_cl_mul_mat_id(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { + if (src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16 || src0->type == GGML_TYPE_BF16) { + ggml_cl_mul_mat_id_float(backend, src0, src1, dst); + return; + } + GGML_ASSERT(src0); GGML_ASSERT(src0->extra); GGML_ASSERT(src1); @@ -33501,7 +35259,7 @@ static void ggml_cl_mul_mat_id(ggml_backend_t backend, const ggml_tensor * src0, size_t local_size[3] = {64, 2, 1}; size_t global_size[3] = {64, 2, 1}; - if (ne12 == 1) { // for gemv + if (use_adreno_moe_gemv_small_batch(backend_ctx, ne12, ne20, ne02)) { // for gemv kernel = backend_ctx->kernel_gemv_moe_q4_0_f32_ns; cl_mem src1_sub_buffer, buf_src1_image, buf_src2; @@ -33509,14 +35267,14 @@ static void ggml_cl_mul_mat_id(ggml_backend_t backend, const ggml_tensor * src0, // create a sub_buffer for src2 cl_buffer_region region; region.origin = offset2; - region.size = ne20 * ne21 * sizeof(int); + region.size = (size_t) nb21 * (ne21 - 1) + ne20 * sizeof(int); buf_src2 = clCreateSubBuffer(extra2->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &status); CL_CHECK(status); // set thread grid global_size[0] = static_cast(((ne01 + 63) / 64) * 64); global_size[1] = 4; - global_size[2] = static_cast(ne20); + global_size[2] = static_cast(ne20 * ne12); local_size[1] = 4; // create a sub_buffer for src1 @@ -33533,6 +35291,7 @@ static void ggml_cl_mul_mat_id(ggml_backend_t backend, const ggml_tensor * src0, // Set kernel args int arg_idx = 0; + const int ids_stride = (int) (nb21 / sizeof(int)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &extra0_q4_0->q)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &extra0_q4_0->d)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &buf_src1_image)); @@ -33542,6 +35301,8 @@ static void ggml_cl_mul_mat_id(ggml_backend_t backend, const ggml_tensor * src0, CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(int), &ne00)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(int), &ne01)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(int), &ne11)); + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(int), &ne20)); + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(int), &ids_stride)); // launch kernel backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_size, local_size, dst); @@ -33810,7 +35571,7 @@ static void ggml_cl_mul_mat_id(ggml_backend_t backend, const ggml_tensor * src0, size_t local_size[3] = {64, 2, 1}; size_t global_size[3] = {64, 2, 1}; - if (ne12 == 1) { // for gemv + if (use_adreno_moe_gemv_small_batch(backend_ctx, ne12, ne20, ne02)) { // for gemv kernel = backend_ctx->kernel_gemv_moe_q4_1_f32_ns; cl_mem src1_sub_buffer, buf_src1_image, buf_src2; @@ -33818,14 +35579,14 @@ static void ggml_cl_mul_mat_id(ggml_backend_t backend, const ggml_tensor * src0, // create a sub_buffer for src2 cl_buffer_region region; region.origin = offset2; - region.size = ne20 * ne21 * sizeof(int); + region.size = (size_t) nb21 * (ne21 - 1) + ne20 * sizeof(int); buf_src2 = clCreateSubBuffer(extra2->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &status); CL_CHECK(status); // set thread grid global_size[0] = static_cast(((ne01 + 63) / 64) * 64); global_size[1] = 4; - global_size[2] = static_cast(ne20); + global_size[2] = static_cast(ne20 * ne12); local_size[1] = 4; // create a sub_buffer for src1 @@ -33842,6 +35603,7 @@ static void ggml_cl_mul_mat_id(ggml_backend_t backend, const ggml_tensor * src0, // Set kernel args int arg_idx = 0; + const int ids_stride = (int) (nb21 / sizeof(int)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &extra0_q4_1->q)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &extra0_q4_1->d)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &extra0_q4_1->m)); @@ -33852,6 +35614,8 @@ static void ggml_cl_mul_mat_id(ggml_backend_t backend, const ggml_tensor * src0, CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(int), &ne00)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(int), &ne01)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(int), &ne11)); + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(int), &ne20)); + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(int), &ids_stride)); // launch kernel backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_size, local_size, dst); @@ -33996,7 +35760,7 @@ static void ggml_cl_mul_mat_id(ggml_backend_t backend, const ggml_tensor * src0, size_t local_size[3] = {64, 2, 1}; size_t global_size[3] = {64, 2, 1}; - if (ne12 == 1) { // for gemv + if (use_adreno_moe_gemv_small_batch(backend_ctx, ne12, ne20, ne02)) { // for gemv kernel = backend_ctx->kernel_gemv_moe_q5_0_f32_ns; cl_mem src1_sub_buffer, buf_src1_image, buf_src2; @@ -34004,14 +35768,14 @@ static void ggml_cl_mul_mat_id(ggml_backend_t backend, const ggml_tensor * src0, // create a sub_buffer for src2 cl_buffer_region region; region.origin = offset2; - region.size = ne20 * ne21 * sizeof(int); + region.size = (size_t) nb21 * (ne21 - 1) + ne20 * sizeof(int); buf_src2 = clCreateSubBuffer(extra2->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &status); CL_CHECK(status); // set thread grid global_size[0] = static_cast(((ne01 + 63) / 64) * 64); global_size[1] = 4; - global_size[2] = static_cast(ne20); + global_size[2] = static_cast(ne20 * ne12); local_size[1] = 4; // create a sub_buffer for src1 @@ -34028,6 +35792,7 @@ static void ggml_cl_mul_mat_id(ggml_backend_t backend, const ggml_tensor * src0, // Set kernel args int arg_idx = 0; + const int ids_stride = (int) (nb21 / sizeof(int)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &extra0_q5_0->qs)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &extra0_q5_0->qh)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &extra0_q5_0->d)); @@ -34038,6 +35803,8 @@ static void ggml_cl_mul_mat_id(ggml_backend_t backend, const ggml_tensor * src0, CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(int), &ne00)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(int), &ne01)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(int), &ne11)); + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(int), &ne20)); + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(int), &ids_stride)); // launch kernel backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_size, local_size, dst); @@ -34249,7 +36016,7 @@ static void ggml_cl_mul_mat_id(ggml_backend_t backend, const ggml_tensor * src0, size_t local_size[3] = {64, 2, 1}; size_t global_size[3] = {64, 2, 1}; - if (ne12 == 1) { // for gemv + if (use_adreno_moe_gemv_small_batch(backend_ctx, ne12, ne20, ne02)) { // for gemv kernel = backend_ctx->kernel_gemv_moe_q5_1_f32_ns; cl_mem src1_sub_buffer, buf_src1_image, buf_src2; @@ -34257,14 +36024,14 @@ static void ggml_cl_mul_mat_id(ggml_backend_t backend, const ggml_tensor * src0, // create a sub_buffer for src2 cl_buffer_region region; region.origin = offset2; - region.size = ne20 * ne21 * sizeof(int); + region.size = (size_t) nb21 * (ne21 - 1) + ne20 * sizeof(int); buf_src2 = clCreateSubBuffer(extra2->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &status); CL_CHECK(status); // set thread grid global_size[0] = static_cast(((ne01 + 63) / 64) * 64); global_size[1] = 4; - global_size[2] = static_cast(ne20); + global_size[2] = static_cast(ne20 * ne12); local_size[1] = 4; // create a sub_buffer for src1 @@ -34281,6 +36048,7 @@ static void ggml_cl_mul_mat_id(ggml_backend_t backend, const ggml_tensor * src0, // Set kernel args int arg_idx = 0; + const int ids_stride = (int) (nb21 / sizeof(int)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &extra0_q5_1->qs)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &extra0_q5_1->qh)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &extra0_q5_1->d)); @@ -34292,6 +36060,8 @@ static void ggml_cl_mul_mat_id(ggml_backend_t backend, const ggml_tensor * src0, CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(int), &ne00)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(int), &ne01)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(int), &ne11)); + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(int), &ne20)); + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(int), &ids_stride)); // launch kernel backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_size, local_size, dst); @@ -34681,7 +36451,7 @@ static void ggml_cl_mul_mat_id(ggml_backend_t backend, const ggml_tensor * src0, size_t local_size[3] = {64, 2, 1}; size_t global_size[3] = {64, 2, 1}; - if (ne12 == 1) { // for gemv + if (use_adreno_moe_gemv_small_batch(backend_ctx, ne12, ne20, ne02)) { // for gemv kernel = backend_ctx->kernel_gemv_moe_q4_k_f32_ns; // Weight-as-texture MoE decode GEMV @@ -34701,14 +36471,14 @@ static void ggml_cl_mul_mat_id(ggml_backend_t backend, const ggml_tensor * src0, // create a sub_buffer for src2 cl_buffer_region region; region.origin = offset2; - region.size = ne20 * ne21 * sizeof(int); + region.size = (size_t) nb21 * (ne21 - 1) + ne20 * sizeof(int); buf_src2 = clCreateSubBuffer(extra2->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &status); CL_CHECK(status); // set thread grid global_size[0] = static_cast(((ne01 + 63) / 64) * 64); global_size[1] = 4; - global_size[2] = static_cast(ne20); + global_size[2] = static_cast(ne20 * ne12); local_size[1] = 4; // create a sub_buffer for src1 @@ -34725,6 +36495,7 @@ static void ggml_cl_mul_mat_id(ggml_backend_t backend, const ggml_tensor * src0, // Set kernel args int arg_idx = 0; + const int ids_stride = (int) (nb21 / sizeof(int)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), use_moe_decode_wimg ? &extra0_q4_K->q_img : &extra0_q4_K->q)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &extra0_q4_K->d)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &extra0_q4_K->dm)); @@ -34736,6 +36507,8 @@ static void ggml_cl_mul_mat_id(ggml_backend_t backend, const ggml_tensor * src0, CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(int), &ne00)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(int), &ne01)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(int), &ne11)); + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(int), &ne20)); + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(int), &ids_stride)); // launch kernel backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_size, local_size, dst); @@ -34951,7 +36724,7 @@ static void ggml_cl_mul_mat_id(ggml_backend_t backend, const ggml_tensor * src0, size_t local_size[3] = {64, 2, 1}; size_t global_size[3] = {64, 2, 1}; - if (ne12 == 1) { // for gemv + if (use_adreno_moe_gemv_small_batch(backend_ctx, ne12, ne20, ne02)) { // for gemv kernel = backend_ctx->kernel_gemv_moe_q5_k_f32_ns; cl_mem src1_sub_buffer, buf_src1_image, buf_src2; @@ -34959,14 +36732,14 @@ static void ggml_cl_mul_mat_id(ggml_backend_t backend, const ggml_tensor * src0, // create a sub_buffer for src2 cl_buffer_region region; region.origin = offset2; - region.size = ne20 * ne21 * sizeof(int); + region.size = (size_t) nb21 * (ne21 - 1) + ne20 * sizeof(int); buf_src2 = clCreateSubBuffer(extra2->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &status); CL_CHECK(status); // set thread grid global_size[0] = static_cast(((ne01 + 63) / 64) * 64); global_size[1] = 4; - global_size[2] = static_cast(ne20); + global_size[2] = static_cast(ne20 * ne12); local_size[1] = 4; // create a sub_buffer for src1 @@ -34983,6 +36756,7 @@ static void ggml_cl_mul_mat_id(ggml_backend_t backend, const ggml_tensor * src0, // Set kernel args int arg_idx = 0; + const int ids_stride = (int) (nb21 / sizeof(int)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &extra0_q5_K->q)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &extra0_q5_K->qh)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &extra0_q5_K->d)); @@ -34995,6 +36769,8 @@ static void ggml_cl_mul_mat_id(ggml_backend_t backend, const ggml_tensor * src0, CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(int), &ne00)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(int), &ne01)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(int), &ne11)); + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(int), &ne20)); + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(int), &ids_stride)); // launch kernel backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_size, local_size, dst); @@ -35208,7 +36984,7 @@ static void ggml_cl_mul_mat_id(ggml_backend_t backend, const ggml_tensor * src0, size_t local_size[3] = {64, 2, 1}; size_t global_size[3] = {64, 2, 1}; - if (ne12 == 1) { // for gemv + if (use_adreno_moe_gemv_small_batch(backend_ctx, ne12, ne20, ne02)) { // for gemv kernel = backend_ctx->kernel_gemv_moe_q6_k_f32_ns; cl_mem src1_sub_buffer, buf_src1_image, buf_src2; @@ -35216,14 +36992,14 @@ static void ggml_cl_mul_mat_id(ggml_backend_t backend, const ggml_tensor * src0, // create a sub_buffer for src2 cl_buffer_region region; region.origin = offset2; - region.size = ne20 * ne21 * sizeof(int); + region.size = (size_t) nb21 * (ne21 - 1) + ne20 * sizeof(int); buf_src2 = clCreateSubBuffer(extra2->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &status); CL_CHECK(status); // set thread grid global_size[0] = static_cast(((ne01 + 63) / 64) * 64); global_size[1] = 4; - global_size[2] = static_cast(ne20); + global_size[2] = static_cast(ne20 * ne12); local_size[1] = 4; // create a sub_buffer for src1 @@ -35240,6 +37016,7 @@ static void ggml_cl_mul_mat_id(ggml_backend_t backend, const ggml_tensor * src0, // Set kernel args int arg_idx = 0; + const int ids_stride = (int) (nb21 / sizeof(int)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &extra0_q6_K->ql)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &extra0_q6_K->qh)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &extra0_q6_K->s)); @@ -35251,6 +37028,8 @@ static void ggml_cl_mul_mat_id(ggml_backend_t backend, const ggml_tensor * src0, CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(int), &ne00)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(int), &ne01)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(int), &ne11)); + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(int), &ne20)); + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(int), &ids_stride)); // launch kernel backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_size, local_size, dst); @@ -36148,68 +37927,6 @@ static void ggml_cl_set(ggml_backend_t backend, const ggml_tensor * src0, const backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); } -static void ggml_cl_diag_mask_inf(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { - GGML_ASSERT(src0); - GGML_ASSERT(src0->extra); - GGML_ASSERT(dst); - GGML_ASSERT(dst->extra); - - UNUSED(src1); - - int n_past = ((int32_t *)(dst->op_params))[0]; - - const int ne00 = src0 ? src0->ne[0] : 0; - const int ne01 = src0 ? src0->ne[1] : 0; - const int ne02 = src0 ? src0->ne[2] : 0; - - ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; - - ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra; - ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; - - cl_ulong offset0 = extra0->offset + src0->view_offs; - cl_ulong offsetd = extrad->offset + dst->view_offs; - - cl_kernel kernel; - - if (ne00%8 == 0) { - kernel = backend_ctx->kernel_diag_mask_inf_8; - - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extrad->data_device)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offsetd)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &ne00)); - CL_CHECK(clSetKernelArg(kernel, 5, sizeof(int), &ne01)); - CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &n_past)); - - size_t global_work_size[] = {(size_t)ne00*ne01*ne02/8, 1, 1}; - size_t local_work_size[] = {64, 1, 1}; - - backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); - } else { - kernel = backend_ctx->kernel_diag_mask_inf; - - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extrad->data_device)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offsetd)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &ne00)); - CL_CHECK(clSetKernelArg(kernel, 5, sizeof(int), &ne01)); - CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &n_past)); - - size_t global_work_size[] = {(size_t)ne00, (size_t)ne01, (size_t)ne02}; - size_t local_work_size[] = {64, 1, 1}; - - size_t * local_work_size_ptr = local_work_size; - if (ne00 % 64 != 0 && !backend_ctx->non_uniform_workgroups) { - local_work_size_ptr = nullptr; // Let driver choose the work-group sizes. - } - - backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size_ptr, dst); - } -} - static void ggml_cl_diag(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { GGML_ASSERT(src0); GGML_ASSERT(src0->extra); @@ -36381,6 +38098,102 @@ static void ggml_cl_soft_max(ggml_backend_t backend, const ggml_tensor * src0, c backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); } +// Softmax for the decomposed flash attention: like ggml_cl_soft_max, but dst keeps +// exp(x - max) and the row sums go to `sums`, one float per (row, head). src2 is the +// optional sinks tensor. +static void ggml_cl_fa_decompose_soft_max(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * src2, ggml_tensor * dst, cl_mem sums) { + GGML_ASSERT(src0 && src0->extra); + GGML_ASSERT(src1 && src1->extra && src1->type == GGML_TYPE_F16); + GGML_ASSERT(dst && dst->extra); + GGML_ASSERT(sums != nullptr); + GGML_ASSERT(src0->ne[0] % 4 == 0); + + if (src2) { + GGML_ASSERT(src2->extra); + } + + ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; + + ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra; + ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra; + ggml_tensor_extra_cl * extra2 = src2 ? (ggml_tensor_extra_cl *)src2->extra : nullptr; + ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; + + cl_ulong offset0 = extra0->offset + src0->view_offs; + cl_ulong offset1 = extra1->offset + src1->view_offs; + cl_ulong offset2 = extra2 ? extra2->offset + src2->view_offs : offset0; + cl_ulong offsetd = extrad->offset + dst->view_offs; + cl_ulong offsets = 0; + + const int ne00 = src0->ne[0]; + const int ne01 = src0->ne[1]; + const int ne02 = src0->ne[2]; + const int ne03 = src0->ne[3]; + + const cl_ulong nb01 = src0->nb[1]; + const cl_ulong nb02 = src0->nb[2]; + const cl_ulong nb03 = src0->nb[3]; + + const int ne12 = src1->ne[2]; + const int ne13 = src1->ne[3]; + + const cl_ulong nb11 = src1->nb[1]; + const cl_ulong nb12 = src1->nb[2]; + const cl_ulong nb13 = src1->nb[3]; + + const cl_ulong nb1 = dst->nb[1]; + const cl_ulong nb2 = dst->nb[2]; + const cl_ulong nb3 = dst->nb[3]; + + float scale, max_bias; + memcpy(&scale, dst->op_params + 0, sizeof(float)); + memcpy(&max_bias, dst->op_params + 1, sizeof(float)); + + const int n_head = src0->ne[2]; + const int n_head_log2 = 1u << (uint32_t) floorf(log2f((float) n_head)); + + const float m0 = powf(2.0f, -(max_bias ) / n_head_log2); + const float m1 = powf(2.0f, -(max_bias / 2.0f) / n_head_log2); + + // one wave per row, same as ggml_cl_soft_max on Adreno + const int nth = 64; + + cl_kernel kernel = backend_ctx->kernel_fa_decompose_soft_max_4_f16; + + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), extra2 ? &extra2->data_device : &extra0->data_device)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offset2)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_mem), &sums)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &offsets)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ulong), &nb01)); + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nb02)); + CL_CHECK(clSetKernelArg(kernel, 13, sizeof(cl_ulong), &nb03)); + CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &ne12)); + CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &ne13)); + CL_CHECK(clSetKernelArg(kernel, 16, sizeof(cl_ulong), &nb11)); + CL_CHECK(clSetKernelArg(kernel, 17, sizeof(cl_ulong), &nb12)); + CL_CHECK(clSetKernelArg(kernel, 18, sizeof(cl_ulong), &nb13)); + CL_CHECK(clSetKernelArg(kernel, 19, sizeof(cl_ulong), &nb1)); + CL_CHECK(clSetKernelArg(kernel, 20, sizeof(cl_ulong), &nb2)); + CL_CHECK(clSetKernelArg(kernel, 21, sizeof(cl_ulong), &nb3)); + CL_CHECK(clSetKernelArg(kernel, 22, sizeof(float), &scale)); + CL_CHECK(clSetKernelArg(kernel, 23, sizeof(float), &max_bias)); + CL_CHECK(clSetKernelArg(kernel, 24, sizeof(float), &m0)); + CL_CHECK(clSetKernelArg(kernel, 25, sizeof(float), &m1)); + CL_CHECK(clSetKernelArg(kernel, 26, sizeof(int), &n_head_log2)); + + size_t global_work_size[] = {(size_t)ne01*nth, (size_t)ne02, (size_t)ne03}; + size_t local_work_size[] = {(size_t)nth, 1, 1}; + + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); +} + static void ggml_cl_rope(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { GGML_ASSERT(src0); GGML_ASSERT(src0->extra); @@ -37168,6 +38981,47 @@ static void ggml_cl_argsort(ggml_backend_t backend, const ggml_tensor * src0, co #endif // GGML_OPENCL_USE_ADRENO_KERNELS } +static void ggml_cl_dsv4_hc(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { + auto * backend_ctx = (ggml_backend_opencl_context *) backend->context; + const bool pre = dst->op == GGML_OP_DSV4_HC_PRE; + const bool vec = !pre && src0->ne[0] % 4 == 0 && src0->nb[0] == sizeof(float) && src1->nb[0] == sizeof(float); + cl_kernel kernel = pre ? backend_ctx->kernel_dsv4_hc_pre : backend_ctx->kernel_dsv4_hc_post[vec]; + + cl_uint arg = 0; + for (int i = 0; i < (pre ? 2 : 4); ++i) { + const ggml_tensor * src = dst->src[i]; + auto * extra = src ? (ggml_tensor_extra_cl *) src->extra : nullptr; + const cl_mem buffer = extra ? extra->data_device : nullptr; + const cl_ulong offset = src ? extra->offset + src->view_offs : 0; + CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(cl_mem), &buffer)); + CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(cl_ulong), &offset)); + for (int j = 0; j < 3; ++j) { + const cl_ulong stride = src ? src->nb[j] : 0; + CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(cl_ulong), &stride)); + } + } + + auto * extra = (ggml_tensor_extra_cl *) dst->extra; + const cl_ulong offset = extra->offset + dst->view_offs; + const cl_long n_embd = src0->ne[0]; + const cl_long n_hc = pre ? src0->ne[1] : src1->ne[1]; + CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(cl_mem), &extra->data_device)); + CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(cl_ulong), &offset)); + CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(cl_long), &n_embd)); + CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(cl_long), &n_hc)); + if (pre) { + const float scale = ggml_get_op_params_f32(dst, 0); + const cl_int gated = ggml_get_op_params_i32(dst, 1); + CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(float), &scale)); + CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(cl_int), &gated)); + } + + const size_t n_work = (size_t) n_embd / (vec ? 4 : 1); + size_t local_work_size[] = { pre ? backend_ctx->local_dsv4_hc_pre : backend_ctx->local_dsv4_hc_post[vec], 1, 1 }; + size_t global_work_size[] = { CEIL_DIV(n_work, local_work_size[0]) * local_work_size[0], (size_t) dst->ne[1], (size_t) dst->ne[2] }; + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); +} + static void ggml_cl_sum_rows(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { GGML_ASSERT(src0); GGML_ASSERT(src0->extra); @@ -37865,6 +39719,12 @@ bool ggml_cl_compute_forward(ggml_backend_t backend, struct ggml_tensor * tensor } func = ggml_cl_fill; break; + case GGML_OP_ARANGE: + if (!any_on_device) { + return false; + } + func = ggml_cl_arange; + break; case GGML_OP_CLAMP: if (!any_on_device) { return false; @@ -37992,12 +39852,6 @@ bool ggml_cl_compute_forward(ggml_backend_t backend, struct ggml_tensor * tensor } func = ggml_cl_diag; break; - case GGML_OP_DIAG_MASK_INF: - if (!any_on_device) { - return false; - } - func = ggml_cl_diag_mask_inf; - break; case GGML_OP_SOFT_MAX: if (!any_on_device) { return false; @@ -38055,6 +39909,10 @@ bool ggml_cl_compute_forward(ggml_backend_t backend, struct ggml_tensor * tensor } func = ggml_cl_sum_rows; break; + case GGML_OP_DSV4_HC_PRE: + case GGML_OP_DSV4_HC_POST: + func = ggml_cl_dsv4_hc; + break; case GGML_OP_CUMSUM: if (!any_on_device) { return false; diff --git a/ggml/src/ggml-opencl/kernels/arange.cl b/ggml/src/ggml-opencl/kernels/arange.cl new file mode 100644 index 000000000000..53dc383e6305 --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/arange.cl @@ -0,0 +1,14 @@ +__kernel void kernel_arange_f32( + __global float * dst, + ulong offsetd, + int n, + float start, + float step +) { + dst = (global float*)((global char*)dst + offsetd); + + const int i = get_global_id(0); + if (i < n) { + dst[i] = start + step * i; + } +} diff --git a/ggml/src/ggml-opencl/kernels/cvt.cl b/ggml/src/ggml-opencl/kernels/cvt.cl index 143f7c24d1df..1a50ec4da678 100644 --- a/ggml/src/ggml-opencl/kernels/cvt.cl +++ b/ggml/src/ggml-opencl/kernels/cvt.cl @@ -3283,3 +3283,98 @@ kernel void kernel_restore_block_iq3_xxs( q[1] = (ushort)(sas >> 16); } } + +struct block_iq4_xs { + half d; + ushort scales_h; + uchar scales_l[QK_K/64]; + uchar qs[QK_K/2]; +}; + +kernel void kernel_convert_block_iq4_xs( + global struct block_iq4_xs * src0, + global uchar * dst_qs, + global uchar * dst_sl, + global ushort * dst_sh, + global half * dst_d, + ulong n_blocks +) { + const ulong i = get_global_id(0); + if (i >= n_blocks) { return; } + dst_d[i] = src0[i].d; + dst_sh[i] = src0[i].scales_h; + for (int j = 0; j < 128; ++j) { + dst_qs[128*i + j] = src0[i].qs[j]; + } + for (int j = 0; j < 4; ++j) { + dst_sl[4*i + j] = src0[i].scales_l[j]; + } +} + +kernel void kernel_restore_block_iq4_xs( + global uchar * src_qs, + global uchar * src_sl, + global ushort * src_sh, + global half * src_d, + global struct block_iq4_xs * dst, + ulong n_blocks +) { + const ulong i = get_global_id(0); + if (i >= n_blocks) { return; } + dst[i].d = src_d[i]; + dst[i].scales_h = src_sh[i]; + for (int j = 0; j < 128; ++j) { + dst[i].qs[j] = src_qs[128*i + j]; + } + for (int j = 0; j < 4; ++j) { + dst[i].scales_l[j] = src_sl[4*i + j]; + } +} + +kernel void kernel_convert_block_iq4_xs_noshuffle( + global struct block_iq4_xs * src0, + global ushort * dst_qs, + global uint * dst_sl, + global ushort * dst_sh, + global half * dst_d, + ulong n_blocks +) { + const ulong i = get_global_id(0); + if (i >= n_blocks) { return; } + dst_d[i] = src0[i].d; + dst_sh[i] = src0[i].scales_h; + dst_sl[i] = as_uint(vload4(0, src0[i].scales_l)); + for (int sb = 0; sb < 8; ++sb) { + for (int u = 0; u < 8; ++u) { + ushort q = 0; + for (int j = 0; j < 4; ++j) { + uchar v = src0[i].qs[16*sb + 4*(u & 3) + j]; + q |= (ushort)(((v >> (4*(u >> 2))) & 15) << (4*j)); + } + dst_qs[64*i + 8*sb + u] = q; + } + } +} + +kernel void kernel_restore_block_iq4_xs_noshuffle( + global ushort * src_qs, + global uint * src_sl, + global ushort * src_sh, + global half * src_d, + global struct block_iq4_xs * dst, + ulong n_blocks +) { + const ulong i = get_global_id(0); + if (i >= n_blocks) { return; } + dst[i].d = src_d[i]; + dst[i].scales_h = src_sh[i]; + vstore4(as_uchar4(src_sl[i]), 0, dst[i].scales_l); + for (int sb = 0; sb < 8; ++sb) { + for (int j = 0; j < 16; ++j) { + const int shift = 4*(j & 3); + const ushort lo = src_qs[64*i + 8*sb + j/4]; + const ushort hi = src_qs[64*i + 8*sb + j/4 + 4]; + dst[i].qs[16*sb + j] = (uchar)(((lo >> shift) & 15) | (((hi >> shift) & 15) << 4)); + } + } +} diff --git a/ggml/src/ggml-opencl/kernels/diag_mask_inf.cl b/ggml/src/ggml-opencl/kernels/diag_mask_inf.cl deleted file mode 100644 index 36eff0439fa7..000000000000 --- a/ggml/src/ggml-opencl/kernels/diag_mask_inf.cl +++ /dev/null @@ -1,58 +0,0 @@ -#pragma OPENCL EXTENSION cl_khr_fp16 : enable - -//------------------------------------------------------------------------------ -// diag_mask_inf kernels -//------------------------------------------------------------------------------ -kernel void kernel_diag_mask_inf( - global float * src0, - ulong offset0, - global float * dst, - ulong offsetd, - int ne00, - int ne01, - int n_past -) { - src0 = (global float*)((global char*)src0 + offset0); - dst = (global float*)((global char*)dst + offsetd); - - int i02 = get_global_id(2); - int i01 = get_global_id(1); - int i00 = get_global_id(0); - - if (i00 > n_past + i01) { - dst[i02*ne01*ne00 + i01*ne00 + i00] = -INFINITY; - } else { - dst[i02*ne01*ne00 + i01*ne00 + i00] = src0[i02*ne01*ne00 + i01*ne00 + i00]; - } -} - -kernel void kernel_diag_mask_inf_8( - global float4 * src0, - ulong offset0, - global float4 * dst, - ulong offsetd, - int ne00, - int ne01, - int n_past -) { - src0 = (global float4*)((global char*)src0 + offset0); - dst = (global float4*)((global char*)dst + offsetd); - - int i = 2*get_global_id(0); - - dst[i+0] = src0[i+0]; - dst[i+1] = src0[i+1]; - int i4 = 4*i; - int i02 = i4/(ne00*ne01); i4 -= i02*ne00*ne01; - int i01 = i4/(ne00); i4 -= i01*ne00; - int i00 = i4; - for (int k = 3; k >= 0; --k) { - if (i00 + 4 + k <= n_past + i01) { - break; - } - (&dst[i+1])[k] = -INFINITY; - if (i00 + k > n_past + i01) { - (&dst[i])[k] = -INFINITY; - } - } -} diff --git a/ggml/src/ggml-opencl/kernels/dsv4_hc.cl b/ggml/src/ggml-opencl/kernels/dsv4_hc.cl new file mode 100644 index 000000000000..164ab1dae9dc --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/dsv4_hc.cl @@ -0,0 +1,101 @@ +#if HC_VEC == 4 +typedef float4 hc_float; +#define HC_LOAD(p) vload4(0, (global const float *) (p)) +#define HC_STORE(v, p) vstore4(v, 0, (global float *) (p)) +#else +typedef float hc_float; +#define HC_LOAD(p) (*(global const float *) (p)) +#define HC_STORE(v, p) (*(global float *) (p) = (v)) +#endif + +#if HC_VEC == 1 +kernel void kernel_dsv4_hc_pre( + global const char * x, + ulong offset_x, + ulong nbx0, + ulong nbx1, + ulong nbx2, + global const char * weights, + ulong offset_w, + ulong nbw0, + ulong nbw1, + ulong nbw2, + global char * dst, + ulong offset_dst, + long n_embd, + long n_hc, + float scale, + int gated +) { + const size_t i = get_global_id(0); + const size_t t = get_global_id(1); + if (i >= n_embd) { + return; + } + + x += offset_x + i*nbx0 + t*nbx2; + weights += offset_w + (gated ? i*nbw0 + t*nbw2 : t*nbw1); + + float sum = 0.0f; + for (long h = 0; h < n_hc; ++h) { + const float xv = *(global const float *) (x + h*nbx1); + float wv = *(global const float *) (weights + h*(gated ? nbw1 : nbw0)); + if (gated) { + wv = 1.0f / (1.0f + exp(-wv)); + } + sum += xv * wv; + } + ((global float *) (dst + offset_dst))[i + t*n_embd] = scale * sum; +} +#endif + +kernel void kernel_dsv4_hc_post( + global const char * x, + ulong offset_x, + ulong nbx0, + ulong nbx1, + ulong nbx2, + global const char * residual, + ulong offset_r, + ulong nbr0, + ulong nbr1, + ulong nbr2, + global const char * post, + ulong offset_p, + ulong nbp0, + ulong nbp1, + ulong nbp2, + global const char * comb, + ulong offset_c, + ulong nbc0, + ulong nbc1, + ulong nbc2, + global char * dst, + ulong offset_dst, + long n_embd, + long n_hc +) { + const size_t i = get_global_id(0) * HC_VEC; + const size_t h = get_global_id(1); + const size_t t = get_global_id(2); + if (i >= n_embd) { + return; + } + + const hc_float xv = HC_LOAD(x + offset_x + i*nbx0 + t*nbx1); + const float pv = *(global const float *) (post + offset_p + h*nbp0 + t*nbp1); + residual += offset_r + i*nbr0 + t*nbr2; + + hc_float sum = xv * pv; + if (comb) { + comb += offset_c + h*nbc0 + t*nbc2; + for (long s = 0; s < n_hc; ++s) { + const hc_float rv = HC_LOAD(residual + s*nbr1); + const float cv = *(global const float *) (comb + s*nbc1); + sum += rv * cv; + } + } else { + sum += HC_LOAD(residual + h*nbr1); + } + HC_STORE(sum, dst + offset_dst + (i + (h + t*n_hc)*n_embd)*sizeof(float)); +} diff --git a/ggml/src/ggml-opencl/kernels/flash_attn_pre_f16.cl b/ggml/src/ggml-opencl/kernels/flash_attn_pre_f16.cl index 88ead4bcb513..74aa67b08367 100644 --- a/ggml/src/ggml-opencl/kernels/flash_attn_pre_f16.cl +++ b/ggml/src/ggml-opencl/kernels/flash_attn_pre_f16.cl @@ -154,3 +154,123 @@ __kernel void flash_attn_blk_f16( blk[((ulong) mask_slice * (ulong) n_q_blocks + (ulong) q_block_idx) * (ulong) n_kv_blocks + (ulong) kv_block_idx] = res; } + +// V^T for the decomposed flash attention: [dv, n_kv, n_head_kv] (strided) -> [n_kv, dv, n_head_kv] +// (contiguous). This is the layout the Adreno KQV image GEMM reads. A local tile keeps both the +// read and the write coalesced, and the +1 column avoids bank conflicts. +#define FA_VT_TILE 32 +#define FA_VT_ROWS 8 +__kernel void flash_attn_v_transpose_f16( + const global void * v_void, ulong v_offset, + global void * vt_void, + const int n_kv, + const int dv, + const int n_head_kv, + const ulong v_nb1, + const ulong v_nb2 +) { + __local half tile[FA_VT_TILE][FA_VT_TILE + 1]; + + const int d0 = get_group_id(0) * FA_VT_TILE; + const int k0 = get_group_id(1) * FA_VT_TILE; + const int head_kv_idx = get_group_id(2); + + const int lx = get_local_id(0); + const int ly = get_local_id(1); + + const global char * v_head = (const global char *) v_void + v_offset + (ulong) head_kv_idx * v_nb2; + + for (int r = 0; r < FA_VT_TILE; r += FA_VT_ROWS) { + const int d = d0 + lx; + const int kv = k0 + ly + r; + half val = (half) 0.0f; + if (d < dv && kv < n_kv) { + val = ((const global half *) (v_head + (ulong) kv * v_nb1))[d]; + } + tile[ly + r][lx] = val; + } + + barrier(CLK_LOCAL_MEM_FENCE); + + global half * vt_head = (global half *) vt_void + (ulong) head_kv_idx * (ulong) n_kv * (ulong) dv; + + for (int r = 0; r < FA_VT_TILE; r += FA_VT_ROWS) { + const int kv = k0 + lx; + const int d = d0 + ly + r; + if (d < dv && kv < n_kv) { + vt_head[(ulong) d * (ulong) n_kv + (ulong) kv] = tile[lx][ly + r]; + } + } +} + +// Transpose V and quantize each block of 32 KV rows for the int8 KQV GEMM. +__kernel void flash_attn_v_transpose_q8( + const global void * v_void, ulong v_offset, + global void * vt_q_void, // char, [n_head_kv][dv][kv_pitch] + global void * vt_d_void, // half, [n_head_kv][dv][kv_pitch/32] + const int n_kv, + const int dv, + const int n_head_kv, + const ulong v_nb1, + const ulong v_nb2, + const ulong v_nb3, + const int kv_pitch // bytes per V^T row, >= n_kv, multiple of 32 +) { + __local half tile[FA_VT_TILE][FA_VT_TILE + 1]; + + const int d0 = get_group_id(0) * FA_VT_TILE; + const int k0 = get_group_id(1) * FA_VT_TILE; + const int head_kv_idx = get_group_id(2) % n_head_kv; + const int batch_idx = get_group_id(2) / n_head_kv; + + const int lx = get_local_id(0); + const int ly = get_local_id(1); + + const global char * v_head = (const global char *) v_void + v_offset + + (ulong) batch_idx * v_nb3 + (ulong) head_kv_idx * v_nb2; + + for (int r = 0; r < FA_VT_TILE; r += FA_VT_ROWS) { + const int d = d0 + lx; + const int kv = k0 + ly + r; + half val = (half) 0.0f; + if (d < dv && kv < n_kv) { + val = ((const global half *) (v_head + (ulong) kv * v_nb1))[d]; + } + tile[ly + r][lx] = val; + } + + barrier(CLK_LOCAL_MEM_FENCE); + + const int lid = ly * FA_VT_TILE + lx; + const int nblk = kv_pitch / FA_VT_TILE; + + if (lid < FA_VT_TILE) { + const int d = d0 + lid; + if (d < dv) { + const ulong hbase = ((ulong) batch_idx * (ulong) n_head_kv + (ulong) head_kv_idx); + + global char * qrow = (global char *) vt_q_void + hbase*(ulong)kv_pitch*(ulong)dv + (ulong) d * (ulong) kv_pitch; + global half * drow = (global half *) vt_d_void + hbase*(ulong)nblk*(ulong)dv + (ulong) d * (ulong) nblk; + + float v[FA_VT_TILE]; + float amax = 0.0f; + + for (int i = 0; i < FA_VT_TILE; ++i) { + v[i] = (float) tile[i][lid]; + amax = fmax(amax, fabs(v[i])); + } + + const float dq = amax / 127.0f; + const float id = amax > 0.0f ? 127.0f / amax : 0.0f; + + drow[k0 / FA_VT_TILE] = (half) dq; + + for (int i = 0; i < FA_VT_TILE; ++i) { + const int kv = k0 + i; + if (kv < n_kv) { + qrow[kv] = (char) clamp((int) rint(v[i] * id), -127, 127); + } + } + } + } +} diff --git a/ggml/src/ggml-opencl/kernels/flash_attn_pv_direct.cl b/ggml/src/ggml-opencl/kernels/flash_attn_pv_direct.cl new file mode 100644 index 000000000000..201fe444359d --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/flash_attn_pv_direct.cl @@ -0,0 +1,106 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable +#pragma OPENCL EXTENSION cl_khr_subgroups : enable +#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable + +#define FA_KQVD_MAXC 32 + +// Read F16 V in its cache layout. Share each V row across the queries and heads of one KV group. +// P8 and its block scales are expanded in local memory; KV splits write separate partial sums. +__attribute__((qcom_reqd_sub_group_size("half"))) +kernel void kernel_fa_kqv_direct_f16( + global const char * v, + ulong off_v, + ulong v_nb1, + ulong v_nb2, + global const uchar * pq, + global const float * pd, + global float * part, // [split][head][n_q][dv] + int dv, + int n_q, + int gqa, + int n_head, + int nblk, + int blk_per_split, + int p_pitch +) { + const int lid = get_local_id(0); + const int d = get_group_id(0)*128 + lid*2; + const int head_kv = get_group_id(1); + const int split = get_group_id(2); + const int ncol = n_q*gqa; + const int b0 = split*blk_per_split; + const int b1 = min(b0 + blk_per_split, nblk); + const int nbp = p_pitch / 32; + + __local float sh_p[32][FA_KQVD_MAXC]; + + float2 acc[FA_KQVD_MAXC]; + #pragma unroll + for (int c = 0; c < FA_KQVD_MAXC; ++c) { + acc[c] = (float2)(0.0f); + } + + global const char * vbase = v + off_v + (size_t)head_kv*v_nb2 + (size_t)d*sizeof(half); + + for (int b = b0; b < b1; ++b) { + barrier(CLK_LOCAL_MEM_FENCE); + // Each lane expands 16 KV entries for one column. + const int c = lid % FA_KQVD_MAXC; + const int r0 = (lid / FA_KQVD_MAXC)*16; + float16 f = (float16)(0.0f); + if (c < ncol) { + const int n = c / gqa; + const int h = head_kv*gqa + c % gqa; + const size_t row = (size_t)h*n_q + n; + const float sc = pd[row*nbp + b]; + const uchar16 u = vload16(0, pq + row*p_pitch + (size_t)b*32 + r0); + f = convert_float16(u)*sc; + } + sh_p[r0+ 0][c] = f.s0; sh_p[r0+ 1][c] = f.s1; sh_p[r0+ 2][c] = f.s2; sh_p[r0+ 3][c] = f.s3; + sh_p[r0+ 4][c] = f.s4; sh_p[r0+ 5][c] = f.s5; sh_p[r0+ 6][c] = f.s6; sh_p[r0+ 7][c] = f.s7; + sh_p[r0+ 8][c] = f.s8; sh_p[r0+ 9][c] = f.s9; sh_p[r0+10][c] = f.sa; sh_p[r0+11][c] = f.sb; + sh_p[r0+12][c] = f.sc; sh_p[r0+13][c] = f.sd; sh_p[r0+14][c] = f.se; sh_p[r0+15][c] = f.sf; + barrier(CLK_LOCAL_MEM_FENCE); + + global const char * vr = vbase + (size_t)b*32*v_nb1; + for (int r = 0; r < 32; ++r) { + const float2 x = vload_half2(0, (global const half *)(vr + (size_t)r*v_nb1)); + #pragma unroll + for (int c = 0; c < FA_KQVD_MAXC; c += 4) { + if (c < ncol) { + const float4 pv = vload4(0, &sh_p[r][c]); + acc[c+0] = mad((float2)(pv.x), x, acc[c+0]); + acc[c+1] = mad((float2)(pv.y), x, acc[c+1]); + acc[c+2] = mad((float2)(pv.z), x, acc[c+2]); + acc[c+3] = mad((float2)(pv.w), x, acc[c+3]); + } + } + } + } + + #pragma unroll + for (int c = 0; c < FA_KQVD_MAXC; ++c) { + if (c < ncol) { + const int n = c / gqa; + const int h = head_kv*gqa + c % gqa; + vstore2(acc[c], 0, part + (((size_t)split*n_head + h)*n_q + n)*dv + d); + } + } +} + +kernel void kernel_fa_kqv_direct_reduce( + global const float * part, + global float * dst, + int n_elem, + int n_split +) { + const int i = get_global_id(0); + if (i >= n_elem) { + return; + } + float s = 0.0f; + for (int k = 0; k < n_split; ++k) { + s += part[(size_t)k*n_elem + i]; + } + dst[i] = s; +} diff --git a/ggml/src/ggml-opencl/kernels/gemm_noshuffle_iq4_xs_f32.cl b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_iq4_xs_f32.cl new file mode 100644 index 000000000000..284868008d95 --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_iq4_xs_f32.cl @@ -0,0 +1,88 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable + +#ifdef cl_qcom_reqd_sub_group_size +#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable +__attribute__((qcom_reqd_sub_group_size("full"))) +#endif +kernel void kernel_gemm_noshuffle_iq4_xs_f32( + global const ushort * src0_qs, + global const uint * src0_sl, + global const ushort * src0_sh, + global const half * src0_d, + read_only image1d_buffer_t src1, + global float * dst, + ulong offsetd, + int m, + int n, + int k, + int n_no_padding +) { + dst = (global float *)((global char *)dst + offsetd); + const int gy = get_global_id(0); + const int row = get_global_id(1) * 4; + const int n_4 = n / 4; + if (row >= m) { + return; + } + + half8 c0 = 0, c1 = 0, c2 = 0, c3 = 0; + for (int i = 0; i < k; i += 16) { + const uint4 sl = vload4(0, src0_sl + row + (i / 256) * m); + const uint4 sh = convert_uint4(vload4(0, src0_sh + row + (i / 256) * m)); + const half4 d = vload4(0, src0_d + row + (i / 256) * m); + const int sb = (i / 32) & 7; + const int4 ls = convert_int4(((sl >> (uint4)(4*sb)) & (uint4)(15)) | (((sh >> (uint4)(2*sb)) & (uint4)(3)) << (uint4)(4))) - (int4)(32); + const half4 scale = convert_half4(convert_float4(d) * convert_float4(ls)); + + for (int l = 0; l < 16; l += 4) { + const int ki = i + l; + const ushort4 q = vload4(0, src0_qs + row + (ki / 4) * m); + #pragma unroll + for (int j = 0; j < 4; ++j) { + half8 b; + b.s0123 = read_imageh(src1, gy * 2 + (ki + j) * n_4); + b.s4567 = read_imageh(src1, gy * 2 + 1 + (ki + j) * n_4); + const ushort4 indices = (q >> (ushort4)(4*j)) & (ushort4)(15); + const half4 w = (half4)(iq4_xs_values[indices.s0], iq4_xs_values[indices.s1], + iq4_xs_values[indices.s2], iq4_xs_values[indices.s3]) * scale; + c0 += b * w.s0; + c1 += b * w.s1; + c2 += b * w.s2; + c3 += b * w.s3; + } + } + } + + int idx = gy * 8 * m + row; + if (idx + 3 < m * n_no_padding) { + vstore4((float4)(c0.s0, c1.s0, c2.s0, c3.s0), 0, dst + idx); + idx += m; + } + if (idx + 3 < m * n_no_padding) { + vstore4((float4)(c0.s1, c1.s1, c2.s1, c3.s1), 0, dst + idx); + idx += m; + } + if (idx + 3 < m * n_no_padding) { + vstore4((float4)(c0.s2, c1.s2, c2.s2, c3.s2), 0, dst + idx); + idx += m; + } + if (idx + 3 < m * n_no_padding) { + vstore4((float4)(c0.s3, c1.s3, c2.s3, c3.s3), 0, dst + idx); + idx += m; + } + if (idx + 3 < m * n_no_padding) { + vstore4((float4)(c0.s4, c1.s4, c2.s4, c3.s4), 0, dst + idx); + idx += m; + } + if (idx + 3 < m * n_no_padding) { + vstore4((float4)(c0.s5, c1.s5, c2.s5, c3.s5), 0, dst + idx); + idx += m; + } + if (idx + 3 < m * n_no_padding) { + vstore4((float4)(c0.s6, c1.s6, c2.s6, c3.s6), 0, dst + idx); + idx += m; + } + if (idx + 3 < m * n_no_padding) { + vstore4((float4)(c0.s7, c1.s7, c2.s7, c3.s7), 0, dst + idx); + } +} diff --git a/ggml/src/ggml-opencl/kernels/gemm_noshuffle_iq4_xs_q8_1_dp4a.cl b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_iq4_xs_q8_1_dp4a.cl new file mode 100644 index 000000000000..f23fb1dd5785 --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_iq4_xs_q8_1_dp4a.cl @@ -0,0 +1,135 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable +#pragma OPENCL EXTENSION cl_khr_subgroups : enable +#ifdef cl_khr_integer_dot_product +#pragma OPENCL EXTENSION cl_khr_integer_dot_product : enable +#endif + +#define QK_K 256 + +#ifndef TILESIZE_N +#define TILESIZE_N 32 +#endif + +inline uint iq4_xs_pack(ushort q) { + return iq4_xs_code(q & 15) + | (iq4_xs_code((q >> 4) & 15) << 8) + | (iq4_xs_code((q >> 8) & 15) << 16) + | (iq4_xs_code(q >> 12) << 24); +} + +// Pass staged operands by value; local dp4a operands miscompile on X2. +inline int dot8_q8a_v(uint8 qw, uint4 a0, uint4 a1) { + int r = 0; + r = dot_acc_sat_4x8packed_ss_int(qw.s0, a0.x, r); + r = dot_acc_sat_4x8packed_ss_int(qw.s1, a0.y, r); + r = dot_acc_sat_4x8packed_ss_int(qw.s2, a0.z, r); + r = dot_acc_sat_4x8packed_ss_int(qw.s3, a0.w, r); + r = dot_acc_sat_4x8packed_ss_int(qw.s4, a1.x, r); + r = dot_acc_sat_4x8packed_ss_int(qw.s5, a1.y, r); + r = dot_acc_sat_4x8packed_ss_int(qw.s6, a1.z, r); + r = dot_acc_sat_4x8packed_ss_int(qw.s7, a1.w, r); + return r; +} + +__attribute__((qcom_wave_pair_mode(1))) +kernel void kernel_gemm_noshuffle_iq4_xs_q8_1_dp4a( + __global const ushort * src0_qs, + __global const uint * src0_sl, + __global const ushort * src0_sh, + __global const half * src0_d, + __global const uint * src1_qa, + __global const half * src1_da, + __global float * dst, + ulong offsetd, + int m, + int n_no_padding, + int k +) { + dst = (global float *)((global char *)dst + offsetd); + + const uint lid = get_local_id(0); + const uint block_id_m = get_global_id(1); + const uint block_id_n = get_global_id(2); + + const uint row = block_id_m * 64 + lid; + const uint col_base = block_id_n * TILESIZE_N; + const bool row_valid = row < (uint)m; + const uint rrow = row_valid ? row : 0; + + const uint k_u = (uint)k >> 2; + const uint k_b = (uint)k >> 5; + + __local uint4 sh_qa4[TILESIZE_N][2]; + __local half sh_d[TILESIZE_N]; + +#define NGROUPS (TILESIZE_N / 4) + float4 acc[NGROUPS]; + #pragma unroll + for (int g = 0; g < NGROUPS; ++g) acc[g] = (float4)(0.0f); + + for (uint step = 0; step < (uint)k; step += 32) { + const uint sub = step >> 5; + + const uint ib = sub >> 3; + const uint sb = sub & 7u; + const uint sl = src0_sl[rrow + ib*(uint)m]; + const uint sh = src0_sh[rrow + ib*(uint)m]; + const int ls = (int)(((sl >> (4*sb)) & 15u) | (((sh >> (2*sb)) & 3u) << 4)) - 32; + const float d_w = (float)src0_d[rrow + ib*(uint)m] * (float)ls; + const uint qsb = rrow + sub*8u*(uint)m; + + uint8 qw; + qw.s0 = iq4_xs_pack(src0_qs[qsb + 0u*(uint)m]); + qw.s1 = iq4_xs_pack(src0_qs[qsb + 1u*(uint)m]); + qw.s2 = iq4_xs_pack(src0_qs[qsb + 2u*(uint)m]); + qw.s3 = iq4_xs_pack(src0_qs[qsb + 3u*(uint)m]); + qw.s4 = iq4_xs_pack(src0_qs[qsb + 4u*(uint)m]); + qw.s5 = iq4_xs_pack(src0_qs[qsb + 5u*(uint)m]); + qw.s6 = iq4_xs_pack(src0_qs[qsb + 6u*(uint)m]); + qw.s7 = iq4_xs_pack(src0_qs[qsb + 7u*(uint)m]); + + for (uint idx = lid; idx < TILESIZE_N * 2; idx += 64) { + const uint t = idx >> 1; + const uint v = idx & 1; + const uint c = col_base + t; + sh_qa4[t][v] = (c < (uint)n_no_padding) + ? vload4(0, src1_qa + c * k_u + (step >> 2) + (v << 2)) + : (uint4)(0u); + } + if (lid < TILESIZE_N) { + const uint c = col_base + lid; + sh_d[lid] = (c < (uint)n_no_padding) ? src1_da[c * k_b + sub] : (half)0; + } + barrier(CLK_LOCAL_MEM_FENCE); + +#define LD4(arr, b) ((float4)((float)arr[(b)+0], (float)arr[(b)+1], (float)arr[(b)+2], (float)arr[(b)+3])) + #pragma unroll + for (int g = 0; g < NGROUPS; ++g) { + const int b = g * 4; + float4 rf; +#define TDOT4(T) dot8_q8a_v(qw, (uint4)(sh_qa4[T][0]), (uint4)(sh_qa4[T][1])) + rf.s0 = (float)TDOT4(b+0); rf.s1 = (float)TDOT4(b+1); + rf.s2 = (float)TDOT4(b+2); rf.s3 = (float)TDOT4(b+3); + acc[g] += d_w * LD4(sh_d, b) * rf; + } +#undef TDOT4 +#undef LD4 + barrier(CLK_LOCAL_MEM_FENCE); + } + + if (!row_valid) { + return; + } + + #pragma unroll + for (int g = 0; g < NGROUPS; ++g) { + const uint b = (uint)(g * 4); + const float4 a = acc[g]; + const uint c0 = col_base + b; + if (c0 + 0 < (uint)n_no_padding) dst[(c0 + 0) * (uint)m + row] = a.s0; + if (c0 + 1 < (uint)n_no_padding) dst[(c0 + 1) * (uint)m + row] = a.s1; + if (c0 + 2 < (uint)n_no_padding) dst[(c0 + 2) * (uint)m + row] = a.s2; + if (c0 + 3 < (uint)n_no_padding) dst[(c0 + 3) * (uint)m + row] = a.s3; + } +#undef NGROUPS +} diff --git a/ggml/src/ggml-opencl/kernels/gemv_moe_q4_0_f32_ns.cl b/ggml/src/ggml-opencl/kernels/gemv_moe_q4_0_f32_ns.cl index f90970b21b27..47ffed1125ff 100644 --- a/ggml/src/ggml-opencl/kernels/gemv_moe_q4_0_f32_ns.cl +++ b/ggml/src/ggml-opencl/kernels/gemv_moe_q4_0_f32_ns.cl @@ -30,7 +30,9 @@ __kernel void kernel_gemv_moe_q4_0_f32_ns( ulong offsetd, int ne00, int ne01, - int ne11 + int ne11, + int ne20, + int ids_stride ) { uint i01 = get_global_id(0); uint i20 = get_global_id(2); @@ -41,9 +43,10 @@ __kernel void kernel_gemv_moe_q4_0_f32_ns( return; } - uint i11 = i20 % ne11; + // i20 runs over n_tokens x n_expert_used; src1 has ne11 = 1 (shared) or ne20 rows per token + uint i11 = (i20 / ne20) * ne11 + (i20 % ne20) % ne11; - uint expert_id = src2[i20]; + uint expert_id = src2[(i20 / ne20) * ids_stride + i20 % ne20]; if ((int)expert_id == -1) { // skipped slot, the dst row is zeroed by kernel_moe_zero_dst return; } diff --git a/ggml/src/ggml-opencl/kernels/gemv_moe_q4_1_f32_ns.cl b/ggml/src/ggml-opencl/kernels/gemv_moe_q4_1_f32_ns.cl index 44bc15ff9497..4041887f0277 100644 --- a/ggml/src/ggml-opencl/kernels/gemv_moe_q4_1_f32_ns.cl +++ b/ggml/src/ggml-opencl/kernels/gemv_moe_q4_1_f32_ns.cl @@ -31,7 +31,9 @@ __kernel void kernel_gemv_moe_q4_1_f32_ns( ulong offsetd, int ne00, int ne01, - int ne11 + int ne11, + int ne20, + int ids_stride ) { uint i01 = get_global_id(0); uint i20 = get_global_id(2); @@ -42,9 +44,10 @@ __kernel void kernel_gemv_moe_q4_1_f32_ns( return; } - uint i11 = i20 % ne11; + // i20 runs over n_tokens x n_expert_used; src1 has ne11 = 1 (shared) or ne20 rows per token + uint i11 = (i20 / ne20) * ne11 + (i20 % ne20) % ne11; - uint expert_id = src2[i20]; + uint expert_id = src2[(i20 / ne20) * ids_stride + i20 % ne20]; if ((int)expert_id == -1) { // skipped slot, the dst row is zeroed by kernel_moe_zero_dst return; } diff --git a/ggml/src/ggml-opencl/kernels/gemv_moe_q4_k_f32_ns.cl b/ggml/src/ggml-opencl/kernels/gemv_moe_q4_k_f32_ns.cl index 290a315933aa..13787568eac4 100644 --- a/ggml/src/ggml-opencl/kernels/gemv_moe_q4_k_f32_ns.cl +++ b/ggml/src/ggml-opencl/kernels/gemv_moe_q4_k_f32_ns.cl @@ -47,7 +47,9 @@ __kernel void kernel_gemv_moe_q4_k_f32_ns( ulong offsetd, int ne00, int ne01, - int ne11 + int ne11, + int ne20, + int ids_stride ) { uint i01 = get_global_id(0); uint i20 = get_global_id(2); @@ -58,9 +60,10 @@ __kernel void kernel_gemv_moe_q4_k_f32_ns( return; } - uint i11 = i20 % ne11; + // i20 runs over n_tokens x n_expert_used; src1 has ne11 = 1 (shared) or ne20 rows per token + uint i11 = (i20 / ne20) * ne11 + (i20 % ne20) % ne11; - uint expert_id = src2[i20]; + uint expert_id = src2[(i20 / ne20) * ids_stride + i20 % ne20]; if ((int)expert_id == -1) { // skipped slot, the dst row is zeroed by kernel_moe_zero_dst return; } @@ -169,7 +172,9 @@ __kernel void kernel_gemv_moe_q4_k_f32_ns_wimg( ulong offsetd, int ne00, int ne01, - int ne11 + int ne11, + int ne20, + int ids_stride ) { uint i01 = get_global_id(0); uint i20 = get_global_id(2); @@ -180,9 +185,10 @@ __kernel void kernel_gemv_moe_q4_k_f32_ns_wimg( return; } - uint i11 = i20 % ne11; + // i20 runs over n_tokens x n_expert_used; src1 has ne11 = 1 (shared) or ne20 rows per token + uint i11 = (i20 / ne20) * ne11 + (i20 % ne20) % ne11; - uint expert_id = src2[i20]; + uint expert_id = src2[(i20 / ne20) * ids_stride + i20 % ne20]; if ((int)expert_id == -1) { // skipped slot, the dst row is zeroed by kernel_moe_zero_dst return; } diff --git a/ggml/src/ggml-opencl/kernels/gemv_moe_q5_0_f32_ns.cl b/ggml/src/ggml-opencl/kernels/gemv_moe_q5_0_f32_ns.cl index 83b186b44c24..f98e7bc39e98 100644 --- a/ggml/src/ggml-opencl/kernels/gemv_moe_q5_0_f32_ns.cl +++ b/ggml/src/ggml-opencl/kernels/gemv_moe_q5_0_f32_ns.cl @@ -31,7 +31,9 @@ __kernel void kernel_gemv_moe_q5_0_f32_ns( ulong offsetd, uint ne00, uint ne01, - uint ne11 + uint ne11, + uint ne20, + uint ids_stride ) { uint i01 = get_global_id(0); uint i20 = get_global_id(2); @@ -42,9 +44,10 @@ __kernel void kernel_gemv_moe_q5_0_f32_ns( return; } - uint i11 = i20 % ne11; + // i20 runs over n_tokens x n_expert_used; src1 has ne11 = 1 (shared) or ne20 rows per token + uint i11 = (i20 / ne20) * ne11 + (i20 % ne20) % ne11; - uint expert_id = src2[i20]; + uint expert_id = src2[(i20 / ne20) * ids_stride + i20 % ne20]; if ((int)expert_id == -1) { // skipped slot, the dst row is zeroed by kernel_moe_zero_dst return; } diff --git a/ggml/src/ggml-opencl/kernels/gemv_moe_q5_1_f32_ns.cl b/ggml/src/ggml-opencl/kernels/gemv_moe_q5_1_f32_ns.cl index 2eba34e68557..d5ec3c34dd74 100644 --- a/ggml/src/ggml-opencl/kernels/gemv_moe_q5_1_f32_ns.cl +++ b/ggml/src/ggml-opencl/kernels/gemv_moe_q5_1_f32_ns.cl @@ -32,7 +32,9 @@ __kernel void kernel_gemv_moe_q5_1_f32_ns( ulong offsetd, uint ne00, uint ne01, - uint ne11 + uint ne11, + uint ne20, + uint ids_stride ) { uint i01 = get_global_id(0); uint i20 = get_global_id(2); @@ -43,9 +45,10 @@ __kernel void kernel_gemv_moe_q5_1_f32_ns( return; } - uint i11 = i20 % ne11; + // i20 runs over n_tokens x n_expert_used; src1 has ne11 = 1 (shared) or ne20 rows per token + uint i11 = (i20 / ne20) * ne11 + (i20 % ne20) % ne11; - uint expert_id = src2[i20]; + uint expert_id = src2[(i20 / ne20) * ids_stride + i20 % ne20]; if ((int)expert_id == -1) { // skipped slot, the dst row is zeroed by kernel_moe_zero_dst return; } diff --git a/ggml/src/ggml-opencl/kernels/gemv_moe_q5_k_f32_ns.cl b/ggml/src/ggml-opencl/kernels/gemv_moe_q5_k_f32_ns.cl index a959f72c397e..cb41cd1d09e6 100644 --- a/ggml/src/ggml-opencl/kernels/gemv_moe_q5_k_f32_ns.cl +++ b/ggml/src/ggml-opencl/kernels/gemv_moe_q5_k_f32_ns.cl @@ -48,7 +48,9 @@ __kernel void kernel_gemv_moe_q5_k_f32_ns( ulong offsetd, int ne00, int ne01, - int ne11 + int ne11, + int ne20, + int ids_stride ) { uint i01 = get_global_id(0); uint i20 = get_global_id(2); @@ -59,9 +61,10 @@ __kernel void kernel_gemv_moe_q5_k_f32_ns( return; } - uint i11 = i20 % ne11; + // i20 runs over n_tokens x n_expert_used; src1 has ne11 = 1 (shared) or ne20 rows per token + uint i11 = (i20 / ne20) * ne11 + (i20 % ne20) % ne11; - uint expert_id = src2[i20]; + uint expert_id = src2[(i20 / ne20) * ids_stride + i20 % ne20]; if ((int)expert_id == -1) { // skipped slot, the dst row is zeroed by kernel_moe_zero_dst return; } diff --git a/ggml/src/ggml-opencl/kernels/gemv_moe_q6_k_f32_ns.cl b/ggml/src/ggml-opencl/kernels/gemv_moe_q6_k_f32_ns.cl index dcd34fbc2ab3..66cd9a6cc19d 100644 --- a/ggml/src/ggml-opencl/kernels/gemv_moe_q6_k_f32_ns.cl +++ b/ggml/src/ggml-opencl/kernels/gemv_moe_q6_k_f32_ns.cl @@ -31,7 +31,9 @@ __kernel void kernel_gemv_moe_q6_k_f32_ns( ulong offsetd, int ne00, int ne01, - int ne11 + int ne11, + int ne20, + int ids_stride ) { uint i01 = get_global_id(0); uint i20 = get_global_id(2); @@ -42,9 +44,10 @@ __kernel void kernel_gemv_moe_q6_k_f32_ns( return; } - uint i11 = i20 % ne11; + // i20 runs over n_tokens x n_expert_used; src1 has ne11 = 1 (shared) or ne20 rows per token + uint i11 = (i20 / ne20) * ne11 + (i20 % ne20) % ne11; - uint expert_id = src2[i20]; + uint expert_id = src2[(i20 / ne20) * ids_stride + i20 % ne20]; if ((int)expert_id == -1) { // skipped slot, the dst row is zeroed by kernel_moe_zero_dst return; } diff --git a/ggml/src/ggml-opencl/kernels/gemv_noshuffle_iq4_xs_f32.cl b/ggml/src/ggml-opencl/kernels/gemv_noshuffle_iq4_xs_f32.cl new file mode 100644 index 000000000000..3f71add46ca8 --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/gemv_noshuffle_iq4_xs_f32.cl @@ -0,0 +1,85 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable + +#define QK_K 256 + +#ifndef IQ4_XS_MV_NSG +#define IQ4_XS_MV_NSG 8 +#endif + +inline float4 iq4_xs_vals(ushort q) { + return (float4)((char)iq4_xs_code(q & 15), (char)iq4_xs_code((q >> 4) & 15), + (char)iq4_xs_code((q >> 8) & 15), (char)iq4_xs_code(q >> 12)); +} + +kernel void kernel_gemv_noshuffle_iq4_xs_f32( + global const ushort * src0_qs, + global const uint * src0_sl, + global const ushort * src0_sh, + global const half * src0_d, + global const float * src1, + ulong offset1, + global float * dst, + ulong offsetd, + int ne00, + int ne01, + int ne10, + int ne0 +) { + src1 = (global const float *)((global const char *)src1 + offset1); + dst = (global float *)((global char *)dst + offsetd); + + const uint m = (uint)ne01; + const uint mh = m >> 1; + const uint lid = get_local_id(0); + const uint sgi = get_local_id(1); + const uint col = get_group_id(1); + const uint j = get_group_id(0)*64u + lid; + const uint row = j << 1; + global const float * y = src1 + (ulong)col * (uint)ne10; + + float sum0 = 0.0f, sum1 = 0.0f; + if (j < mh) { + global const uint * qs = (global const uint *)src0_qs; + global const uint * sh = (global const uint *)src0_sh; + for (uint ib = sgi; ib < (uint)ne00/QK_K; ib += IQ4_XS_MV_NSG) { + const uint base = j + ib*mh; + const uint2 sl = vload2(base, src0_sl); + const uint shp = sh[base]; + const half2 d = vload2(base, src0_d); + float acc0 = 0.0f, acc1 = 0.0f; + for (uint sb = 0; sb < 8; ++sb) { + const int ls0 = (int)(((sl.s0 >> (4*sb)) & 15u) | (((shp >> (2*sb)) & 3u) << 4)) - 32; + const int ls1 = (int)(((sl.s1 >> (4*sb)) & 15u) | (((shp >> (16 + 2*sb)) & 3u) << 4)) - 32; + float a0 = 0.0f, a1 = 0.0f; + for (uint u = 0; u < 8; ++u) { + const uint group = ib*64 + sb*8 + u; + const uint q = qs[j + group*mh]; + const float4 yv = vload4(group, y); + a0 += dot(yv, iq4_xs_vals((ushort)q)); + a1 += dot(yv, iq4_xs_vals((ushort)(q >> 16))); + } + acc0 += (float)ls0*a0; + acc1 += (float)ls1*a1; + } + sum0 += (float)d.s0*acc0; + sum1 += (float)d.s1*acc1; + } + } + +#if IQ4_XS_MV_NSG > 1 + local float2 part[IQ4_XS_MV_NSG][64]; + part[sgi][lid] = (float2)(sum0, sum1); + barrier(CLK_LOCAL_MEM_FENCE); + if (sgi != 0) { + return; + } + for (uint s = 1; s < IQ4_XS_MV_NSG; ++s) { + const float2 v = part[s][lid]; + sum0 += v.s0; + sum1 += v.s1; + } +#endif + if (j < mh) { + vstore2((float2)(sum0, sum1), 0, dst + (ulong)col*(uint)ne0 + row); + } +} diff --git a/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q2_0_f32.cl b/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q2_0_f32.cl index 27ff7edd34fc..a46293c248aa 100644 --- a/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q2_0_f32.cl +++ b/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q2_0_f32.cl @@ -1,7 +1,19 @@ #pragma OPENCL EXTENSION cl_khr_fp16 : enable -#ifndef Q2_0_MV_NSG -#define Q2_0_MV_NSG 8 +#if Q2_0_MV_WIDE +#define Q2_0_MV_ROWS 8 +#define Q2_0_MV_LOAD_D vload8 +#define Q2_0_MV_STORE vstore8 +#define Q2_0_MV_FLOAT convert_float8 +typedef float8 q2_0_mv_acc_t; +typedef uchar8 q2_0_mv_quant_t; +#else +#define Q2_0_MV_ROWS 4 +#define Q2_0_MV_LOAD_D vload4 +#define Q2_0_MV_STORE vstore4 +#define Q2_0_MV_FLOAT convert_float4 +typedef float4 q2_0_mv_acc_t; +typedef uchar4 q2_0_mv_quant_t; #endif kernel void kernel_gemv_noshuffle_q2_0_f32( @@ -19,46 +31,47 @@ kernel void kernel_gemv_noshuffle_q2_0_f32( src1 = (global const float *)((global const char *)src1 + offset1); dst = (global float *)((global char *)dst + offsetd); - const uint m = (uint)ne01; - const uint mh = m >> 1; + const uint mr = (uint)ne01/Q2_0_MV_ROWS; const uint lid = get_local_id(0); const uint sgi = get_local_id(1); + const uint lanes = get_local_size(0); + const uint nsg = get_local_size(1); const uint col = get_group_id(1); - const uint j = get_group_id(0)*64u + lid; - const uint row = j << 1; + const uint j = get_group_id(0)*lanes + lid; global const float * y = src1 + (ulong)col * (uint)ne10; - float2 sum = 0.0f; - if (j < mh) { - global const ushort * qs = (global const ushort *)src0_qs; - for (uint ib = sgi; ib < (uint)ne00/64; ib += Q2_0_MV_NSG) { - const float2 d = convert_float2(vload2(j + ib*mh, src0_d)); - float2 acc = 0.0f; + q2_0_mv_acc_t sum = 0.0f; + if (j < mr) { + for (uint ib = sgi; ib < (uint)ne00/64; ib += nsg) { + const q2_0_mv_acc_t d = Q2_0_MV_FLOAT(Q2_0_MV_LOAD_D(j + ib*mr, src0_d)); + q2_0_mv_acc_t acc = 0.0f; for (uint u = 0; u < 16; ++u) { const uint group = ib*16 + u; - const uint q = qs[j + group*mh]; +#if Q2_0_MV_WIDE + const uchar8 q = as_uchar8(((global const uint2 *)src0_qs)[j + group*mr]); +#else + const uchar4 q = as_uchar4(((global const uint *)src0_qs)[j + group*mr]); +#endif const float4 yv = vload4(group, y); - const int4 v0 = convert_int4(((uint4)(q) >> (uint4)(0, 2, 4, 6)) & (uint4)(3)) - (int4)(1); - const int4 v1 = convert_int4(((uint4)(q) >> (uint4)(8, 10, 12, 14)) & (uint4)(3)) - (int4)(1); - acc.s0 += dot(yv, convert_float4(v0)); - acc.s1 += dot(yv, convert_float4(v1)); + acc += (Q2_0_MV_FLOAT( q & (q2_0_mv_quant_t)(3)) - 1.0f)*yv.s0; + acc += (Q2_0_MV_FLOAT((q >> (q2_0_mv_quant_t)(2)) & (q2_0_mv_quant_t)(3)) - 1.0f)*yv.s1; + acc += (Q2_0_MV_FLOAT((q >> (q2_0_mv_quant_t)(4)) & (q2_0_mv_quant_t)(3)) - 1.0f)*yv.s2; + acc += (Q2_0_MV_FLOAT( q >> (q2_0_mv_quant_t)(6)) - 1.0f)*yv.s3; } sum += d*acc; } } -#if Q2_0_MV_NSG > 1 - local float2 part[Q2_0_MV_NSG][64]; - part[sgi][lid] = sum; + local q2_0_mv_acc_t part[512]; + part[sgi*lanes + lid] = sum; barrier(CLK_LOCAL_MEM_FENCE); if (sgi != 0) { return; } - for (uint s = 1; s < Q2_0_MV_NSG; ++s) { - sum += part[s][lid]; + for (uint s = 1; s < nsg; ++s) { + sum += part[s*lanes + lid]; } -#endif - if (j < mh) { - vstore2(sum, 0, dst + (ulong)col*(uint)ne0 + row); + if (j < mr) { + Q2_0_MV_STORE(sum, 0, dst + (ulong)col*(uint)ne0 + j*Q2_0_MV_ROWS); } } diff --git a/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q6_k_f32_tiled.cl b/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q6_k_f32_tiled.cl index c5049f3964ea..a9ddc1130e57 100644 --- a/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q6_k_f32_tiled.cl +++ b/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q6_k_f32_tiled.cl @@ -104,11 +104,11 @@ kernel void kernel_gemv_noshuffle_q6_K_f32_tiled( } } -// Multi-column (N=3) variant of the tiled q6_K decode GEMV, for the speculative/ -// MTP VERIFY lm_head/embed (ne1=3 = 2 drafts + 1 bonus). Identical tiled weight -// layout + unpack as the ne1=1 kernel above; each WI computes 3 output columns, +// Multi-column (ne1 in [2..4]) variant of the tiled q6_K decode GEMV, for the +// speculative/MTP VERIFY lm_head/embed (drafts + 1 bonus). Identical tiled weight +// layout + unpack as the ne1=1 kernel above; each WI computes up to 4 output columns, // streaming the (large) lm_head weight ONCE per superblock and reusing it across -// the 3 verify activation columns (dequant once per code, MAC into 3 accs). This +// the verify activation columns (dequant once per code, MAC into 4 accs). This // is the lm_head analogue of the per-layer mc3 GEMV; the multiply order matches // the ne1=1 kernel, so each column is byte-identical to a standalone tiled GEMV. #if defined(ADRENO_GPU) @@ -123,7 +123,8 @@ kernel void kernel_gemv_noshuffle_q6_K_f32_tiled_mc3( global float * dst, ulong offsetd, int ne00, - int ne01 + int ne01, + int ne1 ) { int grp = get_local_id(1); int row = get_global_id(0); @@ -133,7 +134,12 @@ kernel void kernel_gemv_noshuffle_q6_K_f32_tiled_mc3( int nb = ne00 / 256; int col_stride = ne00 / 4; // activation float4 pixels per column - float acc0 = 0.0f, acc1 = 0.0f, acc2 = 0.0f; + // columns past ne1 re-read the last column and are not stored + int col1 = min(1, ne1 - 1) * col_stride; + int col2 = min(2, ne1 - 1) * col_stride; + int col3 = min(3, ne1 - 1) * col_stride; + + float acc0 = 0.0f, acc1 = 0.0f, acc2 = 0.0f, acc3 = 0.0f; for (int sb = grp; sb < nb; sb += NSUBGROUPS) { int tile_blk = rt * nb + sb; @@ -157,9 +163,10 @@ kernel void kernel_gemv_noshuffle_q6_K_f32_tiled_mc3( int act_base = sb * 64; #pragma unroll for (int e4 = 0; e4 < 64; ++e4) { - float4 a0 = read_imagef(src1, 0*col_stride + act_base + e4); - float4 a1 = read_imagef(src1, 1*col_stride + act_base + e4); - float4 a2 = read_imagef(src1, 2*col_stride + act_base + e4); + float4 a0 = read_imagef(src1, act_base + e4); + float4 a1 = read_imagef(src1, col1 + act_base + e4); + float4 a2 = read_imagef(src1, col2 + act_base + e4); + float4 a3 = read_imagef(src1, col3 + act_base + e4); #pragma unroll for (int t = 0; t < 4; ++t) { int e = e4 * 4 + t; @@ -171,15 +178,17 @@ kernel void kernel_gemv_noshuffle_q6_K_f32_tiled_mc3( float av0 = (t == 0) ? a0.x : (t == 1) ? a0.y : (t == 2) ? a0.z : a0.w; float av1 = (t == 0) ? a1.x : (t == 1) ? a1.y : (t == 2) ? a1.z : a1.w; float av2 = (t == 0) ? a2.x : (t == 1) ? a2.y : (t == 2) ? a2.z : a2.w; + float av3 = (t == 0) ? a3.x : (t == 1) ? a3.y : (t == 2) ? a3.z : a3.w; acc0 += w * av0; acc1 += w * av1; acc2 += w * av2; + acc3 += w * av3; } } } local float4 reduce_lm[NSUBGROUPS * TILE_ROWS]; - reduce_lm[grp * TILE_ROWS + rit] = (float4)(acc0, acc1, acc2, 0.0f); + reduce_lm[grp * TILE_ROWS + rit] = (float4)(acc0, acc1, acc2, acc3); barrier(CLK_LOCAL_MEM_FENCE); if (grp == 0) { @@ -188,9 +197,10 @@ kernel void kernel_gemv_noshuffle_q6_K_f32_tiled_mc3( + reduce_lm[2 * TILE_ROWS + rit] + reduce_lm[3 * TILE_ROWS + rit]; dst = (global float*)((global char*)dst + offsetd); - // dst column-major [ne01 rows x 3 cols]: (row, col) at col*ne01 + row + // dst column-major [ne01 rows x ne1 cols]: (row, col) at col*ne01 + row dst[0*ne01 + row] = total.x; dst[1*ne01 + row] = total.y; - dst[2*ne01 + row] = total.z; + if (ne1 > 2) dst[2*ne01 + row] = total.z; + if (ne1 > 3) dst[3*ne01 + row] = total.w; } } diff --git a/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q8_0_f32.cl b/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q8_0_f32.cl index 6109c967aac1..3c854c32a7ea 100644 --- a/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q8_0_f32.cl +++ b/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q8_0_f32.cl @@ -201,6 +201,88 @@ __kernel void kernel_gemv_noshuffle_q8_0_f32_splitk( } } +// Multi-column (n_cols in [2..4]) variant of the decode GEMV below, for the spec/MTP +// verify batch: each block's weights are loaded once and reused across the columns. +// Per column the operand order matches the decode GEMV. +#define MC_COL_Q8_0(ts, c) \ + { if (slid < 4) { regB.s0123 = read_imagef(src1, (c)*COL_STRIDE + slid*2 + k*8); \ + regB.s4567 = read_imagef(src1, (c)*COL_STRIDE + 1 + slid*2 + k*8); } \ + dequantizeBlockAccum_ns_sgbroadcast_1(ts, regA, scale, regB); } +#ifdef ADRENO_GPU +REQD_SUBGROUP_SIZE_64 +#endif +__kernel void kernel_gemv_noshuffle_q8_0_f32_mc( + __read_only image1d_buffer_t src0_q, // quantized A + global half * src0_d, // A scales + __read_only image1d_buffer_t src1, // B, n_cols columns of K + global float * dst, // C, column-major [M x n_cols] + ulong offsetd, + int ne00, // K + int ne01, // M + int n_cols) +{ + uint groupId = get_local_id(1); + uint gid = get_global_id(0); + ushort slid = get_sub_group_local_id(); + + uint K = ne00; + uint M = ne01; + + uint LINE_STRIDE_A = M; + // Keep tail lanes active for subgroup ops, but read a valid weight row. + uint gid_s = min(gid, LINE_STRIDE_A - 1); + uint BLOCK_STRIDE_A = 8 * M; + uint COL_STRIDE = K / 4; // float4 pixels per activation column + + __private uint8 regA; + __private float8 regB; + + __private float ts0 = 0.0f; + __private float ts1 = 0.0f; + __private float ts2 = 0.0f; + __private float ts3 = 0.0f; + + #pragma unroll 1 + for (uint k = groupId; k < (K / QK8_0); k += N_SIMDGROUP) { + float scale = convert_float(src0_d[gid_s + k * LINE_STRIDE_A]); + + regA.s0 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 0)).x; + regA.s1 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 1)).x; + regA.s2 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 2)).x; + regA.s3 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 3)).x; + regA.s4 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 4)).x; + regA.s5 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 5)).x; + regA.s6 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 6)).x; + regA.s7 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 7)).x; + + MC_COL_Q8_0(ts0, 0); + MC_COL_Q8_0(ts1, 1); + if (n_cols > 2) MC_COL_Q8_0(ts2, 2); + if (n_cols > 3) MC_COL_Q8_0(ts3, 3); + } + + // reduction in local memory, assumes #wave=4 + __local float4 reduceLM[SIMDGROUP_WIDTH * 3]; + float4 acc = (float4)(ts0, ts1, ts2, ts3); + if (groupId == 1) reduceLM[SIMDGROUP_WIDTH * 0 + slid] = acc; + if (groupId == 2) reduceLM[SIMDGROUP_WIDTH * 1 + slid] = acc; + if (groupId == 3) reduceLM[SIMDGROUP_WIDTH * 2 + slid] = acc; + barrier(CLK_LOCAL_MEM_FENCE); + if (groupId == 0) { + acc += reduceLM[SIMDGROUP_WIDTH * 0 + slid]; + acc += reduceLM[SIMDGROUP_WIDTH * 1 + slid]; + acc += reduceLM[SIMDGROUP_WIDTH * 2 + slid]; + dst = (global float*)((global char*)dst + offsetd); + if (gid < M) { + dst[0 * M + gid] = acc.s0; + dst[1 * M + gid] = acc.s1; + if (n_cols > 2) dst[2 * M + gid] = acc.s2; + if (n_cols > 3) dst[3 * M + gid] = acc.s3; + } + } +} +#undef MC_COL_Q8_0 + #ifdef ADRENO_GPU REQD_SUBGROUP_SIZE_64 #endif diff --git a/ggml/src/ggml-opencl/kernels/iq4_xs_grid.cl b/ggml/src/ggml-opencl/kernels/iq4_xs_grid.cl new file mode 100644 index 000000000000..26fb42291ff6 --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/iq4_xs_grid.cl @@ -0,0 +1,10 @@ +constant char iq4_xs_values[16] = { + -127, -104, -83, -65, -49, -35, -22, -10, 1, 13, 25, 38, 53, 69, 89, 113, +}; + +inline uint iq4_xs_code(uint n) { + const uint lo = (n & 4u) ? 0xF6EADDCFu : 0xBFAD9881u; + const uint hi = (n & 4u) ? 0x71594535u : 0x26190D01u; + const uint t = (n & 8u) ? hi : lo; + return (t >> (8*(n & 3u))) & 255u; +} diff --git a/ggml/src/ggml-opencl/kernels/mul_mm_f16_f32_kq_p8.cl b/ggml/src/ggml-opencl/kernels/mul_mm_f16_f32_kq_p8.cl new file mode 100644 index 000000000000..2c0a56c855c7 --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/mul_mm_f16_f32_kq_p8.cl @@ -0,0 +1,250 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable +#pragma OPENCL EXTENSION cl_khr_subgroups : enable + +#define LM_FIRST_256B 0 +#define LM_SECOND_256B 64 +#define LM_THIRD_256B 128 +#define LM_FOURTH_256B 192 + + +inline float16 mm_load_a( + image1d_buffer_t matrix_A, + uint subMatrixAStartInElements, + int line_stride_matrix_A_in_bytes +) { + __private float8 regA; + size_t sub_block_id_m = get_local_id(0); + + uint a_texCoord = subMatrixAStartInElements/2 + (sub_block_id_m * line_stride_matrix_A_in_bytes/4); + + regA.s0123 = read_imagef(matrix_A, a_texCoord/4); + regA.s4567 = read_imagef(matrix_A, (a_texCoord+4)/4); + + return convert_float16(as_half16(regA)); +} + +inline float4 alu_32( + float16 regA, + __local float4 * matrix_B_vec +) { + + __private float4 rC = 0; + int i = get_sub_group_id() * 64; + + rC += regA.s0 * matrix_B_vec[i]; + rC += regA.s1 * matrix_B_vec[i + 16]; + rC += regA.s4 * matrix_B_vec[i + 1]; + rC += regA.s5 * matrix_B_vec[i + 17]; + rC += regA.s8 * matrix_B_vec[i + 2]; + rC += regA.s9 * matrix_B_vec[i + 18]; + rC += regA.sc * matrix_B_vec[i + 3]; + rC += regA.sd * matrix_B_vec[i + 19]; + + i += 32; + + rC += regA.s2 * matrix_B_vec[i]; + rC += regA.s3 * matrix_B_vec[i + 16]; + rC += regA.s6 * matrix_B_vec[i + 1]; + rC += regA.s7 * matrix_B_vec[i + 17]; + rC += regA.sa * matrix_B_vec[i + 2]; + rC += regA.sb * matrix_B_vec[i + 18]; + rC += regA.se * matrix_B_vec[i + 3]; + rC += regA.sf * matrix_B_vec[i + 19]; + + return rC; +} + +inline float16 alu_16( + float16 regA, + __local float * matrix_B_local +) { + float16 out; + __local float4 * matrix_B_vec = (__local float4*)matrix_B_local; + + out.s0123 = alu_32(regA, matrix_B_vec); + out.s4567 = alu_32(regA, matrix_B_vec + 4); + out.s89ab = alu_32(regA, matrix_B_vec + 8); + out.scdef = alu_32(regA, matrix_B_vec + 12); + + return out; +} + +inline void mm_mad( + __local float * matrix_B_local, + float16 regA, + float8 regB, + uint b_localOffsetInWords, + float16 * regC0_ptr, + float16 * regC1_ptr +) { + int offset = b_localOffsetInWords + get_sub_group_id() * 256; + + matrix_B_local[offset + LM_FIRST_256B] = regB.s0; + matrix_B_local[offset + LM_SECOND_256B] = regB.s1; + matrix_B_local[offset + LM_THIRD_256B] = regB.s2; + matrix_B_local[offset + LM_FOURTH_256B] = regB.s3; + + float16 add0 = alu_16(regA, matrix_B_local); + *regC0_ptr += add0; + + matrix_B_local[offset + LM_FIRST_256B] = regB.s4; + matrix_B_local[offset + LM_SECOND_256B] = regB.s5; + matrix_B_local[offset + LM_THIRD_256B] = regB.s6; + matrix_B_local[offset + LM_FOURTH_256B] = regB.s7; + + float16 add1 = alu_16(regA, matrix_B_local); + *regC1_ptr += add1; +} + +#define TILESIZE_K 16 +#define TILESIZE_M 64 +#define TILESIZE_N 32 +// Fuse KQ and block softmax into U8 P, block maxima and sums. No F32 score matrix is stored. +__kernel void mul_mm_f16_f32_kq_p8( + __read_only image1d_buffer_t matrix_A, + __global float * matrix_B, + __global uchar * qp, // u8 P, [D_B][N][p_pitch] + __global float * bmax, // per 32-row block max, [D_B][N][p_pitch/32] + __global float * bsum, // per block sum of exp(s - bmax), [D_B][N][p_pitch/32] + __global char * mask, // f16 [>=M][N] view (chunk rows), row stride mask_nb1 bytes + ulong offset_mask, + ulong mask_nb1, + float scale, + float max_bias, + float m0, + float m1, + int n_head_log2, + int M, + int K, + int N, + int D_A, + int D_B, + int p_pitch // bytes per P row, >= M, multiple of 32 +) { + + // Match the n-tile-first grid in ggml_cl_fa_kq_p8_dispatch. + const uint m_blocks = (M+TILESIZE_M-1)/TILESIZE_M; + uint block_id_n = get_global_id(1); + uint block_id_m = get_global_id(2) % m_blocks; + uint block_id_d = get_global_id(2) / m_blocks; + + __private float16 regA; + __private float8 regB; + __private float16 regC0 = (float16)(0.0f); + __private float16 regC1 = (float16)(0.0f); + + const uint col = block_id_m * TILESIZE_M; + const uint row = block_id_n * TILESIZE_N; + const uint depth_A = block_id_d / (D_B/D_A); + const uint depth_B = block_id_d; + + int line_stride_matrix_A_in_bytes = K * D_A * 2; + int line_stride_matrix_B_in_bytes = K * D_B * 4; + + const uint strideAinElements = line_stride_matrix_A_in_bytes / 2; + const uint strideBinElements = line_stride_matrix_B_in_bytes / 4; + + size_t sub_block_id_m = get_local_id(0); + + uint b_localOffsetInWords = (sub_block_id_m/16)*16 + + ((((sub_block_id_m)>>0)&1)<<2) + + ((((sub_block_id_m)>>1)&1)<<3) + + ((((sub_block_id_m)>>2)&1)<<0) + + ((((sub_block_id_m)>>3)&1)<<1); + + uint2 b_globalOffsetInWords_xy = {((sub_block_id_m%4)*4), (sub_block_id_m>>2)}; + uint subMatrixAStartInElements = col * strideAinElements + depth_A * K; + uint subMatrixBStartInElements = 0; + + __local float matrix_B_local[1024]; + + // Clamp Q loads in the last tile; columns past N are not stored. + const uint c0 = min(row + b_globalOffsetInWords_xy.y, (uint)N - 1); + const uint c1 = min(row + b_globalOffsetInWords_xy.y + 16, (uint)N - 1); + const uint b_globalOffsetInWords00 = c0*strideBinElements + depth_B*K + b_globalOffsetInWords_xy.x; + const uint b_globalOffsetInWords16 = c1*strideBinElements + depth_B*K + b_globalOffsetInWords_xy.x; + + for (uint step=0; step < K; step+=TILESIZE_K) { + regA = mm_load_a(matrix_A, subMatrixAStartInElements, line_stride_matrix_A_in_bytes); + + uint b_coordInWords00 = subMatrixBStartInElements + b_globalOffsetInWords00; + uint b_coordInWords16 = subMatrixBStartInElements + b_globalOffsetInWords16; + + regB.s0123 = vload4(b_coordInWords00/4, matrix_B); + regB.s4567 = vload4(b_coordInWords16/4, matrix_B); + + mm_mad(matrix_B_local, regA, regB, b_localOffsetInWords, ®C0, ®C1); + + subMatrixAStartInElements += TILESIZE_K; + subMatrixBStartInElements += TILESIZE_K; + } + + // Transpose each score tile in local memory so one lane owns a 32-row softmax block. + __local float p8_lds[2048]; + __local float * lds = p8_lds + get_sub_group_id() * 1024; + const uint lane = get_local_id(0); + __global const half * mrow = (__global const half *)(mask + offset_mask); + const uint mstride = (uint)(mask_nb1 / 2); + const int nblk = p_pitch / 32; + + #pragma unroll + for (int half_ = 0; half_ < 2; ++half_) { + barrier(CLK_LOCAL_MEM_FENCE); + const float16 cc = half_ == 0 ? regC0 : regC1; + lds[ 0*64 + lane] = cc.s0; lds[ 1*64 + lane] = cc.s1; lds[ 2*64 + lane] = cc.s2; lds[ 3*64 + lane] = cc.s3; + lds[ 4*64 + lane] = cc.s4; lds[ 5*64 + lane] = cc.s5; lds[ 6*64 + lane] = cc.s6; lds[ 7*64 + lane] = cc.s7; + lds[ 8*64 + lane] = cc.s8; lds[ 9*64 + lane] = cc.s9; lds[10*64 + lane] = cc.sa; lds[11*64 + lane] = cc.sb; + lds[12*64 + lane] = cc.sc; lds[13*64 + lane] = cc.sd; lds[14*64 + lane] = cc.se; lds[15*64 + lane] = cc.sf; + barrier(CLK_LOCAL_MEM_FENCE); + if (lane < 32) { + const int j = lane >> 1; + const int b = lane & 1; + const uint n = row + half_*16 + j; + const uint h = depth_B; + if (n < (uint)N) { + float slope = 1.0f; + if (max_bias > 0.0f) { + const float base = (int)h < n_head_log2 ? m0 : m1; + const int ex = (int)h < n_head_log2 ? (int)h + 1 : 2*((int)h - n_head_log2) + 1; + slope = pow(base, ex); + } + const uint kvb = col + b*32; // first kv row of this block + const __local float * src = lds + j*64 + b*32; + float16 s0 = vload16(0, src); + float16 s1 = vload16(1, src); + const __global half * mp = mrow + (size_t)n*mstride + kvb; + const float16 m0v = convert_float16(vload16(0, mp)); + const float16 m1v = convert_float16(vload16(1, mp)); + s0 = s0*scale + slope*m0v; + s1 = s1*scale + slope*m1v; + const float16 mx16 = fmax(s0, s1); + const float8 mx8 = fmax(mx16.lo, mx16.hi); + const float4 mx4 = fmax(mx8.lo, mx8.hi); + const float2 mx2 = fmax(mx4.lo, mx4.hi); + const float amax = fmax(mx2.x, mx2.y); + const uint prow = h*(uint)N + n; + const uint blk = kvb >> 5; + float sum = 0.0f; + uchar16 q0 = (uchar16)(0); + uchar16 q1 = (uchar16)(0); + // Use a finite sentinel: -cl-finite-math-only can remove infinity checks. + if (amax > -1.0e30f) { + const float16 e0 = exp(s0 - amax); + const float16 e1 = exp(s1 - amax); + const float16 a16 = e0 + e1; + const float8 a8 = a16.lo + a16.hi; + const float4 a4 = a8.lo + a8.hi; + const float2 a2 = a4.lo + a4.hi; + sum = a2.x + a2.y; + q0 = convert_uchar16_sat_rte(e0*255.0f); + q1 = convert_uchar16_sat_rte(e1*255.0f); + } + __global uchar * qdst = qp + (size_t)prow*(size_t)p_pitch + (size_t)kvb; + vstore16(q0, 0, qdst); + vstore16(q1, 1, qdst); + bmax[(size_t)prow*nblk + blk] = amax; + bsum[(size_t)prow*nblk + blk] = sum; + } + } + } +} diff --git a/ggml/src/ggml-opencl/kernels/mul_mm_id_f32_f32.cl b/ggml/src/ggml-opencl/kernels/mul_mm_id_f32_f32.cl new file mode 100644 index 000000000000..fdcbd6113834 --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/mul_mm_id_f32_f32.cl @@ -0,0 +1,98 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable + +#define BM 64 +#define BN 32 +#define BK 16 +#define TM 4 +#define TN 4 + +kernel void kernel_mul_mm_id_f32_f32( + global const char * src0, ulong offset0, + global const char * src1, ulong offset1, + global const int * routes, + global const ushort * experts, + global const int * total_tiles, + global char * dst, ulong offsetd, + int ncols, int nrows, int ne11, int nused, int src0_f16, + ulong nb01, ulong nb02, ulong nb11, ulong nb12, ulong nb1, ulong nb2) { + const int tile = get_group_id(1); + if (tile >= total_tiles[0]) { + return; + } + const int tid = get_local_id(0); + const int row = get_group_id(0)*BM; + const int mr = tid % (BM/TM)*TM; + const int nr = tid / (BM/TM)*TN; + src0 += offset0 + experts[tile]*nb02; + src1 += offset1; + dst += offsetd; + local float a[BM*BK]; + local float b[BN*BK]; + float sums[TM*TN] = {0.0f}; + + for (int base = 0; base < ncols; base += BK) { + for (int i = tid; i < BM*BK/4; i += 128) { + const int r = i/(BK/4); + const int k = base + i%(BK/4)*4; + float4 value = 0.0f; + if (row + r < nrows && k + 3 < ncols) { + global const char * arow = src0 + (row + r)*nb01; + value = src0_f16 ? convert_float4(vload4(0, (global const half *) arow + k)) : vload4(0, (global const float *) arow + k); + } else if (row + r < nrows) { + float tail[4] = {0.0f}; + for (int j = 0; j < 4 && k + j < ncols; ++j) { + global const char * arow = src0 + (row + r)*nb01; + tail[j] = src0_f16 ? (float) ((global const half *) arow)[k + j] : ((global const float *) arow)[k + j]; + } + value = (float4) (tail[0], tail[1], tail[2], tail[3]); + } + a[(i%(BK/4)*4 + 0)*BM + r] = value.s0; + a[(i%(BK/4)*4 + 1)*BM + r] = value.s1; + a[(i%(BK/4)*4 + 2)*BM + r] = value.s2; + a[(i%(BK/4)*4 + 3)*BM + r] = value.s3; + } + for (int i = tid; i < BN*BK/4; i += 128) { + const int route = routes[tile*BN + i/(BK/4)]; + const int k = base + i%(BK/4)*4; + float4 value = 0.0f; + if (route >= 0) { + global const float * brow = (global const float *) (src1 + (route%nused%ne11)*nb11 + (route/nused)*nb12); + if (k + 3 < ncols) { + value = vload4(0, brow + k); + } else { + float tail[4] = {0.0f}; + for (int j = 0; j < 4 && k + j < ncols; ++j) { + tail[j] = brow[k + j]; + } + value = (float4) (tail[0], tail[1], tail[2], tail[3]); + } + } + b[(i%(BK/4)*4 + 0)*BN + i/(BK/4)] = value.s0; + b[(i%(BK/4)*4 + 1)*BN + i/(BK/4)] = value.s1; + b[(i%(BK/4)*4 + 2)*BN + i/(BK/4)] = value.s2; + b[(i%(BK/4)*4 + 3)*BN + i/(BK/4)] = value.s3; + } + barrier(CLK_LOCAL_MEM_FENCE); + #pragma unroll + for (int k = 0; k < BK; ++k) { + #pragma unroll + for (int n = 0; n < TN; ++n) { + const float bv = b[k*BN + nr + n]; + #pragma unroll + for (int m = 0; m < TM; ++m) { + sums[n*TM + m] = mad(a[k*BM + mr + m], bv, sums[n*TM + m]); + } + } + } + barrier(CLK_LOCAL_MEM_FENCE); + } + for (int n = 0; n < TN; ++n) { + const int route = routes[tile*BN + nr + n]; + if (route >= 0) { + global float * out = (global float *) (dst + (route%nused)*nb1 + (route/nused)*nb2); + for (int m = 0; m < TM && row + mr + m < nrows; ++m) { + out[row + mr + m] = sums[n*TM + m]; + } + } + } +} diff --git a/ggml/src/ggml-opencl/kernels/mul_mm_iq4_xs_f32_l4_lm.cl b/ggml/src/ggml-opencl/kernels/mul_mm_iq4_xs_f32_l4_lm.cl new file mode 100644 index 000000000000..f127bd9f8a38 --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/mul_mm_iq4_xs_f32_l4_lm.cl @@ -0,0 +1,186 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable + +#ifdef cl_intel_required_subgroup_size +#define INTEL_GPU 1 +#endif + +#define QK_K 256 + +struct block_iq4_xs { + half d; + ushort scales_h; + uchar scales_l[QK_K/64]; + uchar qs[QK_K/2]; +}; + +#define LOAD_VEC_A 4 +#define LOAD_VEC_B 4 + +#define BM 64 +#define BN 64 +#define BK 32 +#ifdef INTEL_GPU +#define TM 8 +#define TN 8 +#else +#define TM 4 +#define TN 8 +#endif + +kernel void kernel_mul_mm_iq4_xs_f32_l4_lm( + global uchar * src0_qs, + global uchar * src0_sl, + global ushort * src0_sh, + global half * src0_d, + ulong offset0, + global float4 * src1, + ulong offset1, + global float * dst, + ulong offsetd, + + int ne00, + int ne01, + int ne02, + int ne11, + int ne12, + + int stride_a, + int stride_b, + int stride_d, + + int batch_stride_a, + int batch_stride_b, + int batch_stride_d, + + int r2, + int r3 +) { + src1 = (global float4*)((global char*)src1 + offset1); + dst = (global float *)((global char*)dst + offsetd); + + local float buf_a[BM * BK]; + local float buf_b[BN * BK]; + + const int batch_idx = get_global_id(2); + + const int i13 = batch_idx / ne12; + const int i12 = batch_idx % ne12; + + const int i03 = i13 / r3; + const int i02 = i12 / r2; + + const int batch_idx_a = i03 * ne02 + i02; + + const int ir = get_group_id(0); + const int ic = get_group_id(1); + + const int tid = get_local_id(0); + const int th_r = tid % (BM / TM); + const int th_c = tid / (BM / TM); + + const int loadr_a = get_local_id(0) % (BK / LOAD_VEC_A); + const int loadc_a = get_local_id(0) / (BK / LOAD_VEC_A); + const int loadr_b = get_local_id(0) % (BK / LOAD_VEC_B); + const int loadc_b = get_local_id(0) / (BK / LOAD_VEC_B); + + const int loadstride_a = get_local_size(0) * LOAD_VEC_A / BK; + const int loadstride_b = get_local_size(0) * LOAD_VEC_B / BK; + + // pos_a counts elements, not blocks - one iq4_xs super block holds QK_K of them + int pos_a = batch_idx_a * batch_stride_a + ir * BM * stride_a; + int pos_b = (batch_idx * batch_stride_b + ic * BN * stride_b) / LOAD_VEC_B; + + float sums[TM * TN]; + float cache_a[TM]; + float cache_b[TN]; + + for (int i = 0; i < TM * TN; i++) { + sums[i] = 0.0f; + } + + for (int block = 0; block < ne00; block += BK) { + for (int l = 0; l < BM; l += loadstride_a) { + if (ir*BM + loadc_a + l < ne01) { + int idx = pos_a + (loadc_a + l) * stride_a + loadr_a * LOAD_VEC_A; + int ib = idx / QK_K; + ib += offset0/sizeof(struct block_iq4_xs); + int e = idx % QK_K; + + int ib32 = e >> 5; + int j = e & 15; + int shift = (e & 16) >> 2; + int ls = ((src0_sl[4*ib + ib32/2] >> (4*(ib32 & 1))) & 15) | (((src0_sh[ib] >> (2*ib32)) & 3) << 4); + float db = (float)src0_d[ib] * (float)(ls - 32); + uchar4 q = vload4(0, src0_qs + 128*ib + 16*ib32 + j); + q = (q >> (uchar4)(shift)) & (uchar4)(15); + + float4 v1; + v1.s0 = db * (float)iq4_xs_values[q.s0]; + v1.s1 = db * (float)iq4_xs_values[q.s1]; + v1.s2 = db * (float)iq4_xs_values[q.s2]; + v1.s3 = db * (float)iq4_xs_values[q.s3]; + + buf_a[(loadr_a * LOAD_VEC_A + 0) * BM + loadc_a + l] = v1.s0; + buf_a[(loadr_a * LOAD_VEC_A + 1) * BM + loadc_a + l] = v1.s1; + buf_a[(loadr_a * LOAD_VEC_A + 2) * BM + loadc_a + l] = v1.s2; + buf_a[(loadr_a * LOAD_VEC_A + 3) * BM + loadc_a + l] = v1.s3; + } else { + buf_a[(loadr_a * LOAD_VEC_A + 0) * BM + loadc_a + l] = 0.0f; + buf_a[(loadr_a * LOAD_VEC_A + 1) * BM + loadc_a + l] = 0.0f; + buf_a[(loadr_a * LOAD_VEC_A + 2) * BM + loadc_a + l] = 0.0f; + buf_a[(loadr_a * LOAD_VEC_A + 3) * BM + loadc_a + l] = 0.0f; + } + } + + for (int l = 0; l < BN; l += loadstride_b) { + if (ic*BN + loadc_b + l < ne11) { + int idx = pos_b + (loadc_b + l) * stride_b / LOAD_VEC_B + loadr_b; + buf_b[(loadr_b * LOAD_VEC_B + 0) * BN + loadc_b + l] = src1[idx].s0; + buf_b[(loadr_b * LOAD_VEC_B + 1) * BN + loadc_b + l] = src1[idx].s1; + buf_b[(loadr_b * LOAD_VEC_B + 2) * BN + loadc_b + l] = src1[idx].s2; + buf_b[(loadr_b * LOAD_VEC_B + 3) * BN + loadc_b + l] = src1[idx].s3; + } else { + buf_b[(loadr_b * LOAD_VEC_B + 0) * BN + loadc_b + l] = 0.0f; + buf_b[(loadr_b * LOAD_VEC_B + 1) * BN + loadc_b + l] = 0.0f; + buf_b[(loadr_b * LOAD_VEC_B + 2) * BN + loadc_b + l] = 0.0f; + buf_b[(loadr_b * LOAD_VEC_B + 3) * BN + loadc_b + l] = 0.0f; + } + } + + barrier(CLK_LOCAL_MEM_FENCE); + + pos_a += BK; + pos_b += BK / LOAD_VEC_B; + + for (int i = 0; i < BK; i++) { + for (int j = 0; j < TM; j++) { + cache_a[j] = buf_a[(i) * BM + th_r * TM + j]; + } + + for (int j = 0; j < TN; j++) { + cache_b[j] = buf_b[(i) * BN + th_c * TN + j]; + } + + for (int cc = 0; cc < TN; cc++) { + for (int cr = 0; cr < TM; cr++) { + const int sums_idx = cc*TM + cr; + sums[sums_idx] = mad(cache_a[cr], cache_b[cc], sums[sums_idx]); + } + } + } + barrier(CLK_LOCAL_MEM_FENCE); + } + + const int dr = ir * BM + th_r * TM; + const int dc = ic * BN + th_c * TN; + + const ulong offsets = batch_idx * batch_stride_d; + + for (int cc = 0; cc < TN; cc++) { + for (int cr = 0; cr < TM; cr++) { + if (dr + cr < ne01 && dc + cc < ne11) { + dst[offsets + (dc + cc) * stride_d + dr + cr] = sums[cc * TM + cr]; + } + } + } +} diff --git a/ggml/src/ggml-opencl/kernels/mul_mm_q8_f32_kqv.cl b/ggml/src/ggml-opencl/kernels/mul_mm_q8_f32_kqv.cl new file mode 100644 index 000000000000..08d80961ca88 --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/mul_mm_q8_f32_kqv.cl @@ -0,0 +1,131 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable +#pragma OPENCL EXTENSION cl_khr_subgroups : enable +#ifdef cl_khr_integer_dot_product +#pragma OPENCL EXTENSION cl_khr_integer_dot_product : enable +#endif + +// Multiply U8 P by S8 V^T, with one scale per 32 KV entries. +// Group query heads by KV head to share V^T in L2. +// The packed dot takes the V word (registers) first: swapped operands miscompile on Adreno. + +#define KQV_TM 64 // d rows per workgroup, one per lane +#ifndef KQV_TN +#define KQV_TN 32 // queries per workgroup, all owned by every lane +#endif +#ifndef KQV_NB +#define KQV_NB 8 // 32-blocks of P staged per barrier +#endif +#define KQV_WG 64 + +__attribute__((qcom_wave_pair_mode(1))) +kernel void kernel_mul_mm_q8_f32_kqv( + global const uint * vq, // V^T int8, [head_kv][dv][kv_pitch] as packed uints + ulong off_vq, + global const half * vd, // V^T scales, [head_kv][dv][kv_pitch/32] + ulong off_vd, + global const uint * pq, // P u8, [head][n_q][kv_pitch] as packed uints + ulong off_pq, + global const float * pd, // P scales f32, [head][n_q][kv_pitch/32] + ulong off_pd, + global float * dst, // [head][n_q][dv] + ulong off_dst, + int dv, + int n_kv, + int n_q, + int n_head, + int n_head_kv, + int kv_pitch // bytes per V^T row and per P row, >= n_kv, multiple of 32 +) { + vq = (global const uint *)((global const char *)vq + off_vq); + vd = (global const half *)((global const char *)vd + off_vd); + pq = (global const uint *)((global const char *)pq + off_pq); + pd = (global const float *)((global const char *)pd + off_pd); + dst = (global float *)((global const char *)dst + off_dst); + + const int lid = get_local_id(0); + const int gsz = n_head / n_head_kv; // query heads per KV head + const int g1 = get_group_id(1); + const int head_kv = get_group_id(2); + const int head = head_kv*gsz + (g1 % gsz); + const int qn0 = (g1 / gsz) * KQV_TN; // first query of this tile + const int d = get_group_id(0)*KQV_TM + lid; // this lane's d row + + const int nblk = n_kv / 32; // dispatch guarantees n_kv % 32 == 0 + const int nu = kv_pitch / 4; // uints per row + const int nbp = kv_pitch / 32; // scales per row + + __local uint sh_pq[KQV_TN][KQV_NB][8]; + __local float sh_pd[KQV_TN][KQV_NB]; + + float acc[KQV_TN]; + #pragma unroll + for (int t = 0; t < KQV_TN; ++t) { + acc[t] = 0.0f; + } + + const int dl = min(d, dv - 1); // clamp so a tail lane loads in bounds + const size_t vbase = ((size_t)head_kv*dv + dl)*nu; + const size_t vdbas = ((size_t)head_kv*dv + dl)*nbp; + const size_t pbase = (size_t)head*n_q; + + for (int bg = 0; bg < nblk; bg += KQV_NB) { + const int nb_here = min(KQV_NB, nblk - bg); + + // stage P: each (query, block) is 8 uints = two 16-byte loads + for (int i = lid; i < KQV_TN*KQV_NB*2; i += KQV_WG) { + const int t = i / (KQV_NB*2); + const int bb = (i / 2) % KQV_NB; + const int h = (i & 1) * 4; + const int qi = qn0 + t; + uint4 v = (uint4)(0u); + if (qi < n_q && bb < nb_here) { + v = vload4(0, &pq[(pbase + qi)*nu + (size_t)(bg + bb)*8 + h]); + } + vstore4(v, 0, &sh_pq[t][bb][h]); + } + for (int i = lid; i < KQV_TN*KQV_NB; i += KQV_WG) { + const int t = i / KQV_NB; + const int bb = i % KQV_NB; + const int qi = qn0 + t; + sh_pd[t][bb] = (qi < n_q && bb < nb_here) + ? pd[(pbase + qi)*nbp + bg + bb] : 0.0f; + } + barrier(CLK_LOCAL_MEM_FENCE); + + for (int bb = 0; bb < nb_here; ++bb) { + const uint4 w0 = vload4(0, &vq[vbase + (size_t)(bg + bb)*8]); + const uint4 w1 = vload4(0, &vq[vbase + (size_t)(bg + bb)*8 + 4]); + const float dvs = (float)vd[vdbas + bg + bb]; + + #pragma unroll + for (int t = 0; t < KQV_TN; ++t) { + const uint4 a0 = vload4(0, &sh_pq[t][bb][0]); + const uint4 a1 = vload4(0, &sh_pq[t][bb][4]); + int raw = 0; + raw = dot_acc_sat_4x8packed_su_int(w0.s0, a0.s0, raw); + raw = dot_acc_sat_4x8packed_su_int(w0.s1, a0.s1, raw); + raw = dot_acc_sat_4x8packed_su_int(w0.s2, a0.s2, raw); + raw = dot_acc_sat_4x8packed_su_int(w0.s3, a0.s3, raw); + raw = dot_acc_sat_4x8packed_su_int(w1.s0, a1.s0, raw); + raw = dot_acc_sat_4x8packed_su_int(w1.s1, a1.s1, raw); + raw = dot_acc_sat_4x8packed_su_int(w1.s2, a1.s2, raw); + raw = dot_acc_sat_4x8packed_su_int(w1.s3, a1.s3, raw); + acc[t] += dvs * sh_pd[t][bb] * (float)raw; + } + } + barrier(CLK_LOCAL_MEM_FENCE); + } + + if (d >= dv) { + return; + } + + // dst is [head][n_q][dv] with dv contiguous, so the 64 lanes of a d-block write contiguously + #pragma unroll + for (int t = 0; t < KQV_TN; ++t) { + const int qi = qn0 + t; + if (qi < n_q) { + dst[(pbase + qi)*dv + d] = acc[t]; + } + } +} diff --git a/ggml/src/ggml-opencl/kernels/mul_mv_f32_f32.cl b/ggml/src/ggml-opencl/kernels/mul_mv_f32_f32.cl index ec71b8756523..1ac250a84b9b 100644 --- a/ggml/src/ggml-opencl/kernels/mul_mv_f32_f32.cl +++ b/ggml/src/ggml-opencl/kernels/mul_mv_f32_f32.cl @@ -116,3 +116,70 @@ kernel void kernel_mul_mat_f32_f32( } } } + +// Each group computes two rows and four columns, with 32 lanes per row. +kernel void kernel_mul_mv_f32_f32_r2( + global const char * src0, + ulong offset0, + global const char * src1, + ulong offset1, + global float * dst, + ulong offsetd, + int ne00, + int ne01, + int ne02, + ulong nb00, + ulong nb01, + ulong nb02, + ulong nb03, + int ne10, + int ne11, + int ne12, + ulong nb10, + ulong nb11, + ulong nb12, + ulong nb13, + int ne0, + int ne1, + int r2, + int r3 +) { + src0 += offset0; + src1 += offset1; + dst = (global float *)((global char *)dst + offsetd); + + const int lid = get_local_id(0); + const int lane = lid % 32; + const int row = get_group_id(0)*2 + lid/32; + const int col = get_group_id(1)*4; + const int im = get_group_id(2); + const int i12 = im % ne12; + const int i13 = im / ne12; + global const float4 * x = (global const float4 *)(src0 + min(row, ne01 - 1)*nb01 + (i12/r2)*nb02 + (i13/r3)*nb03); + global const char * y = src1 + col*nb11 + i12*nb12 + i13*nb13; + + float4 sum = 0.0f; + for (int i = lane; i < ne00/4; i += 32) { + const float4 xv = x[i]; + if (col + 0 < ne11) sum.s0 += dot(xv, ((global const float4 *)(y + 0*nb11))[i]); + if (col + 1 < ne11) sum.s1 += dot(xv, ((global const float4 *)(y + 1*nb11))[i]); + if (col + 2 < ne11) sum.s2 += dot(xv, ((global const float4 *)(y + 2*nb11))[i]); + if (col + 3 < ne11) sum.s3 += dot(xv, ((global const float4 *)(y + 3*nb11))[i]); + } + + local float4 part[64]; + part[lid] = sum; + barrier(CLK_LOCAL_MEM_FENCE); + for (int s = 16; s > 0; s /= 2) { + if (lane < s) { + part[lid] += part[lid + s]; + } + barrier(CLK_LOCAL_MEM_FENCE); + } + if (lane == 0 && row < ne01) { + if (col + 0 < ne11) dst[im*ne1*ne0 + (col + 0)*ne0 + row] = part[lid].s0; + if (col + 1 < ne11) dst[im*ne1*ne0 + (col + 1)*ne0 + row] = part[lid].s1; + if (col + 2 < ne11) dst[im*ne1*ne0 + (col + 2)*ne0 + row] = part[lid].s2; + if (col + 3 < ne11) dst[im*ne1*ne0 + (col + 3)*ne0 + row] = part[lid].s3; + } +} diff --git a/ggml/src/ggml-opencl/kernels/mul_mv_id_f32_f32.cl b/ggml/src/ggml-opencl/kernels/mul_mv_id_f32_f32.cl new file mode 100644 index 000000000000..b82ca90c8e8a --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/mul_mv_id_f32_f32.cl @@ -0,0 +1,89 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable + +#ifdef cl_qcom_reqd_sub_group_size +#pragma OPENCL EXTENSION cl_khr_subgroups : enable +#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable +__attribute__((qcom_reqd_sub_group_size("half"))) +#endif +kernel void kernel_mul_mv_id_f32_f32( + global const char * src0, + ulong offset0, + global const char * src1, + ulong offset1, + global const char * ids, + ulong offset2, + global char * dst, + ulong offsetd, + int ncols, + int ne11, + int src0_f16, + ulong nb01, + ulong nb02, + ulong nb11, + ulong nb12, + ulong nb20, + ulong nb21, + ulong nb1, + ulong nb2, + int nrows) { + const int row = get_group_id(0)*4; + const int slot = get_group_id(1); + const int token = get_group_id(2); + const int tid = get_local_id(0); + const int expert = *(global const int *) (ids + offset2 + slot*nb20 + token*nb21); + dst += offsetd + slot*nb1 + token*nb2; + if (expert == -1) { + if (tid == 0) { + for (int r = 0; r < 4 && row + r < nrows; ++r) { + ((global float *) dst)[row + r] = 0.0f; + } + } + return; + } + src0 += offset0 + row*nb01 + expert*nb02; + src1 += offset1 + (slot % ne11)*nb11 + token*nb12; + float sum[4] = {0.0f, 0.0f, 0.0f, 0.0f}; + for (int i = tid*4; i + 3 < ncols; i += 64*4) { + const float4 b = vload4(0, (global const float *) src1 + i); + for (int r = 0; r < 4 && row + r < nrows; ++r) { + global const char * arow = src0 + r*nb01; + const float4 a = src0_f16 ? convert_float4(vload4(0, (global const half *) arow + i)) : vload4(0, (global const float *) arow + i); + sum[r] += dot(a, b); + } + } + for (int i = ncols/4*4 + tid; i < ncols; i += 64) { + const float b = ((global const float *) src1)[i]; + for (int r = 0; r < 4 && row + r < nrows; ++r) { + global const char * arow = src0 + r*nb01; + const float a = src0_f16 ? (float) ((global const half *) arow)[i] : ((global const float *) arow)[i]; + sum[r] = mad(a, b, sum[r]); + } + } +#ifdef cl_qcom_reqd_sub_group_size + for (int r = 0; r < 4; ++r) { + sum[r] = sub_group_reduce_add(sum[r]); + } +#else + local float sums[4][64]; + for (int r = 0; r < 4; ++r) { + sums[r][tid] = sum[r]; + } + barrier(CLK_LOCAL_MEM_FENCE); + for (int stride = 32; stride > 0; stride /= 2) { + for (int r = 0; r < 4; ++r) { + if (tid < stride) { + sums[r][tid] += sums[r][tid + stride]; + } + } + barrier(CLK_LOCAL_MEM_FENCE); + } + for (int r = 0; r < 4; ++r) { + sum[r] = sums[r][0]; + } +#endif + if (tid == 0) { + for (int r = 0; r < 4 && row + r < nrows; ++r) { + ((global float *) dst)[row + r] = sum[r]; + } + } +} diff --git a/ggml/src/ggml-opencl/kernels/mul_mv_iq4_xs_f32.cl b/ggml/src/ggml-opencl/kernels/mul_mv_iq4_xs_f32.cl new file mode 100644 index 000000000000..2fe0044ebdc1 --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/mul_mv_iq4_xs_f32.cl @@ -0,0 +1,134 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable + +#ifdef cl_intel_required_subgroup_size +#pragma OPENCL EXTENSION cl_intel_required_subgroup_size : enable +#define INTEL_GPU 1 +#define REQD_SUBGROUP_SIZE_16 __attribute__((intel_reqd_sub_group_size(16))) +#define REQD_SUBGROUP_SIZE_32 __attribute__((intel_reqd_sub_group_size(32))) +#elif defined(cl_qcom_reqd_sub_group_size) +#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable +#define ADRENO_GPU 1 +#define REQD_SUBGROUP_SIZE_64 __attribute__((qcom_reqd_sub_group_size("half"))) +#define REQD_SUBGROUP_SIZE_128 __attribute__((qcom_reqd_sub_group_size("full"))) +#endif + +#define QK_K 256 + +typedef struct { + half d; + ushort scales_h; + uchar scales_l[QK_K/64]; + uchar qs[QK_K/2]; +} block_iq4_xs; + +#undef N_DST +#undef N_SIMDGROUP +#undef N_SIMDWIDTH + +#ifdef INTEL_GPU +#define N_DST 4 +#define N_SIMDGROUP 1 +#define N_SIMDWIDTH 16 +#elif defined (ADRENO_GPU) +#define N_DST 4 +#define N_SIMDGROUP 1 +#define N_SIMDWIDTH 64 +#endif + +#undef BLOCK_STRIDE +// 8 threads cover one super block, one 32 element sub block each +#define BLOCK_STRIDE (N_SIMDWIDTH/8) + +#ifdef INTEL_GPU +REQD_SUBGROUP_SIZE_16 +#elif defined (ADRENO_GPU) +REQD_SUBGROUP_SIZE_64 +#endif +kernel void kernel_mul_mv_iq4_xs_f32( + global char * src0, + ulong offset0, + global char * src1, + ulong offset1, + global char * dst, + ulong offsetd, + int ne00, + int ne01, + ulong nb01, + ulong nb02, + ulong nb03, + int ne12, + ulong nb11, + ulong nb12, + ulong nb13, + int ne0, + int ne1, + int r2, + int r3 +) { + src0 = src0 + offset0; + src1 = src1 + offset1; + dst = dst + offsetd; + + int ix = get_sub_group_local_id()/8; // super block index + int it = get_sub_group_local_id()%8; // sub block inside the super block + + int nb = ne00/QK_K; + + int r0 = get_group_id(0); + int r1 = get_group_id(1); + int im = get_group_id(2); + int first_row = (r0 * N_SIMDGROUP + get_sub_group_id()) * N_DST; + + int i12 = im%ne12; + int i13 = im/ne12; + + ulong offset_src0 = first_row*nb01 + (i12/r2)*nb02 + (i13/r3)*nb03; + ulong offset_src1 = r1*nb11 + (i12 )*nb12 + (i13 )*nb13; + + global block_iq4_xs * x = (global block_iq4_xs *) (src0 + offset_src0); + global float * y = (global float *) (src1 + offset_src1); + + float yl[32]; + float sumf[N_DST] = {0.f}; + float all_sum; + + global float * y4 = y + ix * QK_K + 32 * it; + + for (int ib = ix; ib < nb; ib += BLOCK_STRIDE) { + for (int i = 0; i < 32; ++i) { + yl[i] = y4[i]; + } + + global char * xrow = (global char *)(x + ib); + + // only ne01 output rows exist; reading past them can pull in an inf scale + for (int row = 0; row < N_DST && first_row + row < ne01; row++) { + global block_iq4_xs * xb = (global block_iq4_xs *)(xrow + row*nb01); + + int ls = ((xb->scales_l[it/2] >> (4*(it%2))) & 15) | (((xb->scales_h >> (2*it)) & 3) << 4); + float db = (float)xb->d * (float)(ls - 32); + global uchar * qs = xb->qs + 16*it; + + float acc = 0.f; + for (int j = 0; j < 16; ++j) { + acc += yl[j] * (float)iq4_xs_values[qs[j] & 15]; + acc += yl[j+16] * (float)iq4_xs_values[qs[j] >> 4]; + } + + sumf[row] += db * acc; + } + + y4 += BLOCK_STRIDE * QK_K; + } + + global float * dst_f32 = (global float *) dst + im*ne0*ne1 + r1*ne0; + + for (int row = 0; row < N_DST; ++row) { + all_sum = sub_group_reduce_add(sumf[row]); + if (first_row + row < ne01) { + if (get_sub_group_local_id() == 0) { + dst_f32[first_row + row] = all_sum; + } + } + } +} diff --git a/ggml/src/ggml-opencl/kernels/mul_mv_iq4_xs_f32_flat.cl b/ggml/src/ggml-opencl/kernels/mul_mv_iq4_xs_f32_flat.cl new file mode 100644 index 000000000000..4426137f0b48 --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/mul_mv_iq4_xs_f32_flat.cl @@ -0,0 +1,134 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable + +#ifdef cl_intel_required_subgroup_size +#pragma OPENCL EXTENSION cl_intel_required_subgroup_size : enable +#define INTEL_GPU 1 +#define REQD_SUBGROUP_SIZE_16 __attribute__((intel_reqd_sub_group_size(16))) +#define REQD_SUBGROUP_SIZE_32 __attribute__((intel_reqd_sub_group_size(32))) +#elif defined(cl_qcom_reqd_sub_group_size) +#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable +#define ADRENO_GPU 1 +#define REQD_SUBGROUP_SIZE_64 __attribute__((qcom_reqd_sub_group_size("half"))) +#define REQD_SUBGROUP_SIZE_128 __attribute__((qcom_reqd_sub_group_size("full"))) +#endif + +#define QK_K 256 + +typedef struct { + half d; + ushort scales_h; + uchar scales_l[QK_K/64]; + uchar qs[QK_K/2]; +} block_iq4_xs; + +#undef N_DST +#undef N_SIMDGROUP +#undef N_SIMDWIDTH + +#ifdef INTEL_GPU +#define N_DST 4 +#define N_SIMDGROUP 1 +#define N_SIMDWIDTH 16 +#elif defined (ADRENO_GPU) +#define N_DST 4 +#define N_SIMDGROUP 1 +#define N_SIMDWIDTH 64 +#endif + +#undef BLOCK_STRIDE +// 8 threads cover one super block, one 32 element sub block each +#define BLOCK_STRIDE (N_SIMDWIDTH/8) + +#ifdef INTEL_GPU +REQD_SUBGROUP_SIZE_16 +#elif defined (ADRENO_GPU) +REQD_SUBGROUP_SIZE_64 +#endif +kernel void kernel_mul_mv_iq4_xs_f32_flat( + global uchar * src0_qs, + global uchar * src0_sl, + global ushort * src0_sh, + global half * src0_d, + ulong offset0, + global char * src1, + ulong offset1, + global char * dst, + ulong offsetd, + int ne00, + int ne01, + ulong nb01, + ulong nb02, + ulong nb03, + int ne12, + ulong nb11, + ulong nb12, + ulong nb13, + int ne0, + int ne1, + int r2, + int r3 +) { + src1 = src1 + offset1; + dst = dst + offsetd; + + int ix = get_sub_group_local_id()/8; // super block index + int it = get_sub_group_local_id()%8; // sub block inside the super block + + int nb = ne00/QK_K; + + int r0 = get_group_id(0); + int r1 = get_group_id(1); + int im = get_group_id(2); + int first_row = (r0 * N_SIMDGROUP + get_sub_group_id()) * N_DST; + + int i12 = im%ne12; + int i13 = im/ne12; + + ulong offset_src0 = (offset0 + first_row*nb01 + (i12/r2)*nb02 + (i13/r3)*nb03)/sizeof(block_iq4_xs); + ulong row_stride = nb01/sizeof(block_iq4_xs); + ulong offset_src1 = r1*nb11 + (i12 )*nb12 + (i13 )*nb13; + + global float * y = (global float *) (src1 + offset_src1); + + float yl[32]; + float sumf[N_DST] = {0.f}; + float all_sum; + + global float * y4 = y + ix * QK_K + 32 * it; + + for (int ib = ix; ib < nb; ib += BLOCK_STRIDE) { + for (int i = 0; i < 32; ++i) { + yl[i] = y4[i]; + } + + // only ne01 output rows exist; reading past them can pull in an inf scale + for (int row = 0; row < N_DST && first_row + row < ne01; row++) { + const ulong b = offset_src0 + ib + row*row_stride; + + int ls = ((src0_sl[4*b + it/2] >> (4*(it%2))) & 15) | (((src0_sh[b] >> (2*it)) & 3) << 4); + float db = (float)src0_d[b] * (float)(ls - 32); + global uchar * qs = src0_qs + 128*b + 16*it; + + float acc = 0.f; + for (int j = 0; j < 16; ++j) { + acc += yl[j] * (float)iq4_xs_values[qs[j] & 15]; + acc += yl[j+16] * (float)iq4_xs_values[qs[j] >> 4]; + } + + sumf[row] += db * acc; + } + + y4 += BLOCK_STRIDE * QK_K; + } + + global float * dst_f32 = (global float *) dst + im*ne0*ne1 + r1*ne0; + + for (int row = 0; row < N_DST; ++row) { + all_sum = sub_group_reduce_add(sumf[row]); + if (first_row + row < ne01) { + if (get_sub_group_local_id() == 0) { + dst_f32[first_row + row] = all_sum; + } + } + } +} diff --git a/ggml/src/ggml-opencl/kernels/repeat.cl b/ggml/src/ggml-opencl/kernels/repeat.cl index 53951a554343..88dee31cc683 100644 --- a/ggml/src/ggml-opencl/kernels/repeat.cl +++ b/ggml/src/ggml-opencl/kernels/repeat.cl @@ -1,38 +1,43 @@ -kernel void kernel_repeat_f32( - global const char * src0, - ulong offset0, - global char * dst, - ulong offsetd, - int ne00, - int ne01, - int ne02, - int ne03, - ulong nb00, - ulong nb01, - ulong nb02, - ulong nb03, - int ne0, - ulong nb0, - ulong nb1, - ulong nb2, - ulong nb3 -) { - src0 = src0 + offset0; - dst = dst + offsetd; - - const int i3 = get_group_id(2); - const int i2 = get_group_id(1); - const int i1 = get_group_id(0); - - const int i03 = i3%ne03; - const int i02 = i2%ne02; - const int i01 = i1%ne01; - - global const char * src0_ptr = src0 + i03*nb03 + i02*nb02 + i01*nb01; - global char * dst_ptr = dst + i3*nb3 + i2*nb2 + i1*nb1; - - for (int i0 = get_local_id(0); i0 < ne0; i0 += get_local_size(0)) { - const int i00 = i0%ne00; - *((global float *)(dst_ptr + i0*nb0)) = *((global float *)(src0_ptr + i00*nb00)); - } +#define KERNEL_REPEAT(SUFFIX, T) \ +kernel void kernel_repeat_##SUFFIX( \ + global const char * src0, \ + ulong offset0, \ + global char * dst, \ + ulong offsetd, \ + int ne00, \ + int ne01, \ + int ne02, \ + int ne03, \ + ulong nb00, \ + ulong nb01, \ + ulong nb02, \ + ulong nb03, \ + int ne0, \ + ulong nb0, \ + ulong nb1, \ + ulong nb2, \ + ulong nb3 \ +) { \ + src0 = src0 + offset0; \ + dst = dst + offsetd; \ + \ + const int i3 = get_group_id(2); \ + const int i2 = get_group_id(1); \ + const int i1 = get_group_id(0); \ + \ + const int i03 = i3%ne03; \ + const int i02 = i2%ne02; \ + const int i01 = i1%ne01; \ + \ + global const char * src0_ptr = src0 + i03*nb03 + i02*nb02 + i01*nb01; \ + global char * dst_ptr = dst + i3*nb3 + i2*nb2 + i1*nb1; \ + \ + for (int i0 = get_local_id(0); i0 < ne0; i0 += get_local_size(0)) { \ + const int i00 = i0%ne00; \ + *((global T *)(dst_ptr + i0*nb0)) = *((global T *)(src0_ptr + i00*nb00)); \ + } \ } + +KERNEL_REPEAT(f32, float) +KERNEL_REPEAT(f16, ushort) +KERNEL_REPEAT(i32, int) diff --git a/ggml/src/ggml-opencl/kernels/sigmoid.cl b/ggml/src/ggml-opencl/kernels/sigmoid.cl index e3f669dde830..cfe9f7295c7d 100644 --- a/ggml/src/ggml-opencl/kernels/sigmoid.cl +++ b/ggml/src/ggml-opencl/kernels/sigmoid.cl @@ -27,3 +27,63 @@ kernel void kernel_sigmoid_f16( dst[get_global_id(0)] = 1.0f / (1.0f + exp(-src0[get_global_id(0)])); } + +kernel void kernel_sigmoid_f32_nc( + global const char * src0, + ulong offset0, + global char * dst, + ulong offsetd, + int ne00, + ulong nb00, + ulong nb01, + ulong nb02, + ulong nb03, + ulong nb0, + ulong nb1, + ulong nb2, + ulong nb3 +) { + src0 = src0 + offset0; + dst = dst + offsetd; + + const int i3 = get_group_id(2); + const int i2 = get_group_id(1); + const int i1 = get_group_id(0); + + for (int i0 = get_local_id(0); i0 < ne00; i0 += get_local_size(0)) { + global const float * x = (global const float *)(src0 + i3*nb03 + i2*nb02 + i1*nb01 + i0*nb00); + global float * y = (global float *)(dst + i3*nb3 + i2*nb2 + i1*nb1 + i0*nb0); + + *y = 1.0f / (1.0f + exp(-*x)); + } +} + +kernel void kernel_sigmoid_f16_nc( + global const char * src0, + ulong offset0, + global char * dst, + ulong offsetd, + int ne00, + ulong nb00, + ulong nb01, + ulong nb02, + ulong nb03, + ulong nb0, + ulong nb1, + ulong nb2, + ulong nb3 +) { + src0 = src0 + offset0; + dst = dst + offsetd; + + const int i3 = get_group_id(2); + const int i2 = get_group_id(1); + const int i1 = get_group_id(0); + + for (int i0 = get_local_id(0); i0 < ne00; i0 += get_local_size(0)) { + global const half * x = (global const half *)(src0 + i3*nb03 + i2*nb02 + i1*nb01 + i0*nb00); + global half * y = (global half *)(dst + i3*nb3 + i2*nb2 + i1*nb1 + i0*nb0); + + *y = 1.0f / (1.0f + exp(-(float)*x)); + } +} diff --git a/ggml/src/ggml-opencl/kernels/softmax_4_f16.cl b/ggml/src/ggml-opencl/kernels/softmax_4_f16.cl index 571d16507c6f..8dfc5519de52 100644 --- a/ggml/src/ggml-opencl/kernels/softmax_4_f16.cl +++ b/ggml/src/ggml-opencl/kernels/softmax_4_f16.cl @@ -106,3 +106,176 @@ kernel void kernel_soft_max_4_f16( pdst4[i00] /= sum; } } + +// Same as kernel_soft_max_4_f16, but leaves exp(x - max) in dst and writes the row sum to psums. +// The decomposed flash attention applies 1/sum to the KQV result, which is much smaller than P. +#ifdef ADRENO_GPU +REQD_SUBGROUP_SIZE_64 +#endif +kernel void kernel_fa_decompose_soft_max_4_f16( + global char * src0, + ulong offset0, + global char * src1, + ulong offset1, + global char * src2, + ulong offset2, + global char * dst, + ulong offsetd, + global char * psums, + ulong offset_sums, + int ne00, + ulong nb01, + ulong nb02, + ulong nb03, + int ne12, + int ne13, + ulong nb11, + ulong nb12, + ulong nb13, + ulong nb1, + ulong nb2, + ulong nb3, + float scale, + float max_bias, + float m0, + float m1, + int n_head_log2 +) { + src0 = src0 + offset0; + src1 = src1 + offset1; + src2 = src2 + offset2; + dst = dst + offsetd; + + int i03 = get_group_id(2); + int i02 = get_group_id(1); + int i01 = get_group_id(0); + + int i13 = i03%ne13; + int i12 = i02%ne12; + int i11 = i01; + + global float4 * psrc4 = (global float4 *)(src0 + i01*nb01 + i02*nb02 + i03*nb03); + global half4 * pmask = src1 != src0 ? (global half4 *)(src1 + i11*nb11 + i12*nb12 + i13*nb13) : 0; + global float * psrc2 = src2 != src0 ? (global float *)(src2) : 0; + global float4 * pdst4 = (global float4 *)(dst + i01*nb1 + i02*nb2 + i03*nb3); + + float slope = 1.0f; + + // ALiBi + if (max_bias > 0.0f) { + int h = i02; + + float base = h < n_head_log2 ? m0 : m1; + int exp = h < n_head_log2 ? h + 1 : 2*(h - n_head_log2) + 1; + + slope = pow(base, exp); + } + + // parallel max + float4 lmax4 = psrc2 ? psrc2[i02] : -INFINITY; + for (int i00 = get_local_id(0); i00 < ne00/4; i00 += get_local_size(0)) { + lmax4 = fmax(lmax4, psrc4[i00]*scale + slope*(pmask ? convert_float4(pmask[i00]) : 0.0f)); + } + float lmax = fmax(fmax(lmax4.s0, lmax4.s1), fmax(lmax4.s2, lmax4.s3)); + + float max = sub_group_reduce_max(lmax); + // Avoid -inf - (-inf) for fully masked rows. + if (max == -INFINITY) { + max = 0.0f; + } + + // parallel sum + float4 lsum4 = 0.0f; + for (int i00 = get_local_id(0); i00 < ne00/4; i00 += get_local_size(0)) { + const float4 exp_psrc4 = exp((psrc4[i00]*scale + slope*(pmask ? convert_float4(pmask[i00]) : 0.0f)) - max); + lsum4 += exp_psrc4; + pdst4[i00] = exp_psrc4; + } + float lsum = lsum4.s0 + lsum4.s1 + lsum4.s2 + lsum4.s3; + + float sum = sub_group_reduce_add(lsum); + + if (psrc2) { + sum += exp(psrc2[i02] - max); + } + + if (get_local_id(0) == 0) { + global float * sums = (global float *)(psums + offset_sums); + sums[(i03*get_num_groups(1) + i02)*get_num_groups(0) + i01] = sum; + } +} + +// out[:, n, h] *= 1/sums[h*ne1 + n], out is [dv, ne1, n_head] contiguous. +kernel void kernel_fa_decompose_scale_rows_f32( + global char * dst, + ulong offsetd, + global char * psums, + ulong offset_sums, + int dv, + int ne1 +) { + global float * out = (global float *)(dst + offsetd); + global float * sums = (global float *)(psums + offset_sums); + + const int n = get_group_id(0); + const int h = get_group_id(1); + + const float s = sums[h*ne1 + n]; + const float r = s > 0.0f ? 1.0f/s : 0.0f; + + global float * row = out + ((size_t)h*ne1 + n)*dv; + for (int i = get_local_id(0); i < dv; i += get_local_size(0)) { + row[i] *= r; + } +} + +// Merge the P8 block maxima and sums into F32 block scales and row sums. +#ifdef ADRENO_GPU +REQD_SUBGROUP_SIZE_64 +#endif +kernel void kernel_fa_p8_fixup( + global float * bmax, + global float * bsum, + global float * dp, + global float * psums, + global char * sinks, + ulong offset_sinks, + int has_sinks, + int nblk, + int row_blk, // blocks per row of bmax/bsum/dp (the padded pitch), >= nblk + int N +) { + const int n = get_group_id(0); + const int head = get_group_id(1); + const uint row = (uint)head*(uint)N + (uint)n; + global float * bm = bmax + (size_t)row*row_blk; + global float * bs = bsum + (size_t)row*row_blk; + global float * dr = dp + (size_t)row*row_blk; + + float lmax = -INFINITY; + for (int b = get_local_id(0); b < nblk; b += get_local_size(0)) { + lmax = fmax(lmax, bm[b]); + } + float rowmax = sub_group_reduce_max(lmax); + float sink = 0.0f; + if (has_sinks) { + sink = ((global float *)(sinks + offset_sinks))[head]; + rowmax = fmax(rowmax, sink); + } + + float lsum = 0.0f; + for (int b = get_local_id(0); b < nblk; b += get_local_size(0)) { + const float m = bm[b]; + // Use a finite sentinel: -cl-finite-math-only can remove infinity checks. + const float f = (m > -1.0e30f && rowmax > -1.0e30f) ? exp(m - rowmax) : 0.0f; + dr[b] = f / 255.0f; + lsum += bs[b]*f; + } + float sum = sub_group_reduce_add(lsum); + if (has_sinks && rowmax > -1.0e30f) { + sum += exp(sink - rowmax); + } + if (get_local_id(0) == 0) { + psums[row] = sum; + } +} diff --git a/ggml/src/ggml-openvino/ggml-openvino.cpp b/ggml/src/ggml-openvino/ggml-openvino.cpp index 88634557cd58..3aff66724711 100644 --- a/ggml/src/ggml-openvino/ggml-openvino.cpp +++ b/ggml/src/ggml-openvino/ggml-openvino.cpp @@ -1581,6 +1581,9 @@ static ggml_openvino_op_support is_op_supported_case(const ggml_tensor * op) { break; } case GGML_OP_MUL_MAT: { + if (ggml_prec_get_src(op, 1) == GGML_PREC_BF16 || ggml_prec_get_src(op, 1) == GGML_PREC_F32) { + return {false, "MUL_MAT with a BF16 or F32 src1 precision is not supported"}; + } if (ggml_openvino_is_gpu() && op->src[0] != nullptr && op->src[1] != nullptr && ggml_is_quantized(op->src[0]->type) && strcmp(op->src[0]->name, "a") == 0 && strcmp(op->src[1]->name, "b") == 0 && op->src[0]->ne[1] == 1 && op->src[1]->ne[1] == 64 && diff --git a/ggml/src/ggml-openvino/openvino/op_table.cpp b/ggml/src/ggml-openvino/openvino/op_table.cpp index 23b3a2cff3ea..857fbf175ebc 100644 --- a/ggml/src/ggml-openvino/openvino/op_table.cpp +++ b/ggml/src/ggml-openvino/openvino/op_table.cpp @@ -33,7 +33,6 @@ std::unordered_map get_supported_ops() { using namespace ov::op; return { {"GGML_OP_ADD", op::translate_add }, - {"GGML_OP_ADD1", op::translate_1to1_match_2_inputs }, {"GGML_OP_ADD_ID", op::translate_add_id }, {"GGML_OP_CONCAT", op::translate_concat }, {"GGML_OP_CONT", op::translate_cont }, diff --git a/ggml/src/ggml-openvino/openvino/utils.cpp b/ggml/src/ggml-openvino/openvino/utils.cpp index 40ba938a62ce..e47f976956ed 100644 --- a/ggml/src/ggml-openvino/openvino/utils.cpp +++ b/ggml/src/ggml-openvino/openvino/utils.cpp @@ -1,5 +1,7 @@ #include "utils.h" +#include "ggml-impl.h" + #include #include #include @@ -90,24 +92,6 @@ ov::Output rope_yarn_ramp_mix(int num_elements, const float corr_dims[ return ramp_mix; } -float ggml_rope_yarn_corr_dim(int n_dims, int n_ctx_orig, float n_rot, float base) { -#ifndef M_PI -# define M_PI 3.14159265358979323846 -#endif - return n_dims * logf(n_ctx_orig / (n_rot * 2 * (float) M_PI)) / (2 * logf(base)); -} - -void ggml_rope_yarn_corr_dims(int n_dims, - int n_ctx_orig, - float freq_base, - float beta_fast, - float beta_slow, - float dims[2]) { - float start = floorf(ggml_rope_yarn_corr_dim(n_dims, n_ctx_orig, beta_fast, freq_base)); - float end = ceilf(ggml_rope_yarn_corr_dim(n_dims, n_ctx_orig, beta_slow, freq_base)); - dims[0] = std::max(0.0f, start); - dims[1] = std::min(static_cast(n_dims - 1), end); -} } // namespace std::pair, ov::Output> make_sin_cos(int32_t * rope_params, @@ -154,6 +138,7 @@ std::pair, ov::Output> make_sin_cos(int32_t * rope_params float beta_slow; const int n_dims = rope_params[1]; const size_t n_dims_half = n_dims >> 1; + const bool truncate = rope_params[3] == 0; const int n_ctx_orig = rope_params[4]; memcpy(&freq_base, rope_params + 5, sizeof(float)); memcpy(&freq_scale, rope_params + 6, sizeof(float)); @@ -248,7 +233,7 @@ std::pair, ov::Output> make_sin_cos(int32_t * rope_params theta = theta_interp; } else { float corr_dims[2]; - ggml_rope_yarn_corr_dims(n_dims, n_ctx_orig, freq_base, beta_fast, beta_slow, corr_dims); + ggml_rope_yarn_corr_dims(n_dims, n_ctx_orig, freq_base, beta_fast, beta_slow, truncate, corr_dims); auto ramp_mix = rope_yarn_ramp_mix(num_pairs, corr_dims, ext_factor); Output one = ov::op::v0::Constant::create(ov::element::f32, Shape{1, 1, 1, 1}, {1.0f}); auto one_minus_ramp = std::make_shared(one, ramp_mix); @@ -261,7 +246,7 @@ std::pair, ov::Output> make_sin_cos(int32_t * rope_params std::vector factor(n_dims_half); Output freq_factors; float corr_dims[2]; - ggml_rope_yarn_corr_dims(n_dims, n_ctx_orig, freq_base, beta_fast, beta_slow, corr_dims); + ggml_rope_yarn_corr_dims(n_dims, n_ctx_orig, freq_base, beta_fast, beta_slow, truncate, corr_dims); factor[0] = 1.0f; for (size_t i = 1; i < factor.size(); i++) { factor[i] = theta_scale * factor[i - 1]; diff --git a/ggml/src/ggml-rpc/ggml-rpc.cpp b/ggml/src/ggml-rpc/ggml-rpc.cpp index 5cf9339bd4bb..e4a84cb0caf0 100644 --- a/ggml/src/ggml-rpc/ggml-rpc.cpp +++ b/ggml/src/ggml-rpc/ggml-rpc.cpp @@ -125,11 +125,16 @@ struct rpc_msg_hello_req { uint8_t conn_caps[RPC_CONN_CAPS_SIZE]; }; +// server feature flags in rpc_msg_hello_rsp::flags +enum rpc_dispatcher_flags : uint8_t { + RPC_DISPATCHER_FLAGS_SEND_TENSOR_HASH = 1 << 0, // client should try SET_TENSOR_HASH before sending tensor data +}; + struct rpc_msg_hello_rsp { uint8_t major; uint8_t minor; uint8_t patch; - uint8_t padding; + uint8_t flags; uint8_t conn_caps[RPC_CONN_CAPS_SIZE]; }; @@ -373,21 +378,30 @@ static bool parse_endpoint(const std::string & endpoint, std::string & host, int } // RPC request : | rpc_cmd (1 byte) | request_size (8 bytes) | request_data (request_size bytes) | +// request_data is input followed by an optional payload, so large data does not need to be copied into input // No response -static bool send_rpc_cmd(socket_ptr sock, enum rpc_cmd cmd, const void * input, size_t input_size) { +static bool send_rpc_cmd_payload(socket_ptr sock, enum rpc_cmd cmd, const void * input, size_t input_size, const void * payload, size_t payload_size) { uint8_t cmd_byte = cmd; if (!sock->send_data(&cmd_byte, sizeof(cmd_byte))) { return false; } - if (!sock->send_data(&input_size, sizeof(input_size))) { + const size_t request_size = input_size + payload_size; + if (!sock->send_data(&request_size, sizeof(request_size))) { return false; } if (!sock->send_data(input, input_size)) { return false; } + if (payload_size > 0 && !sock->send_data(payload, payload_size)) { + return false; + } return sock->flush(); } +static bool send_rpc_cmd(socket_ptr sock, enum rpc_cmd cmd, const void * input, size_t input_size) { + return send_rpc_cmd_payload(sock, cmd, input, input_size, nullptr, 0); +} + // RPC request : | rpc_cmd (1 byte) | request_size (8 bytes) | request_data (request_size bytes) | // RPC response: | response_size (8 bytes) | response_data (response_size bytes) | static bool send_rpc_cmd(socket_ptr sock, enum rpc_cmd cmd, const void * input, size_t input_size, void * output, size_t output_size) { @@ -423,14 +437,17 @@ static inline void rpc_cpu_relax() { // Performs HELLO handshake with transport auto-negotiation. // Advertises local capabilities via conn_caps; if the server responds with // matching capabilities, the socket is upgraded transparently. -static bool negotiate_hello(const std::shared_ptr & sock) { +static bool negotiate_hello(const std::shared_ptr & sock, rpc_dispatcher_flags & flags) { rpc_msg_hello_req request = {}; rpc_msg_hello_rsp response = {}; sock->get_caps(request.conn_caps); bool status = send_rpc_cmd(sock, RPC_CMD_HELLO, &request, sizeof(request), &response, sizeof(response)); - RPC_STATUS_ASSERT(status); + if (!status) { + LOG_ERROR("RPC hello exchange with the server failed\n"); + return false; + } if (response.major != RPC_PROTO_MAJOR_VERSION || response.minor > RPC_PROTO_MINOR_VERSION) { LOG_ERROR("RPC server version mismatch: %d.%d.%d\n", @@ -439,6 +456,7 @@ static bool negotiate_hello(const std::shared_ptr & sock) { } sock->update_caps(response.conn_caps); + flags = (rpc_dispatcher_flags) response.flags; LOG_DBG("[%s] handshake ok: server version %d.%d.%d, transport: %s\n", __func__, response.major, response.minor, response.patch, sock->transport_name()); return true; @@ -499,10 +517,8 @@ class rpc_dispatcher { rpc_dispatcher() { } - void send(enum rpc_cmd cmd, std::shared_ptr input, size_t input_size); - void send(enum rpc_cmd cmd, std::shared_ptr input, size_t input_size, void * output, size_t output_size); - void send_async(enum rpc_cmd cmd, std::shared_ptr input, size_t input_size); - void send_async(enum rpc_cmd cmd, std::shared_ptr input, size_t input_size, void * output, size_t output_size); + void send(enum rpc_cmd cmd, std::shared_ptr input, size_t input_size, const void * payload, size_t payload_size, void * output, size_t output_size); + void send_async(enum rpc_cmd cmd, std::shared_ptr input, size_t input_size, void * output = nullptr, size_t output_size = 0); ggml_backend_event_t event_new(ggml_backend_dev_t dev); void event_free(ggml_backend_event_t event); @@ -513,9 +529,11 @@ class rpc_dispatcher { void busy_spin_release(); void graph_compute(uint32_t device, const ggml_cgraph * cgraph); - void start(const std::string & endpoint); + bool start(const std::string & endpoint); void work(); + bool get_send_tensor_hash() const { return flags & RPC_DISPATCHER_FLAGS_SEND_TENSOR_HASH; } + ~rpc_dispatcher(); private: @@ -525,18 +543,26 @@ class rpc_dispatcher { size_t input_size; void * output; size_t output_size; + const void * payload = nullptr; + size_t payload_size = 0; std::promise completion; }; using rpc_msg_ptr = std::shared_ptr; using rpc_msg_queue = message_queue; + struct rpc_event { rpc_msg_ptr msg; std::shared_future sf; }; + std::mutex graph_mutex; std::unordered_map> graph_uids; - rpc_msg_queue queue; - socket_ptr sock; + + rpc_msg_queue queue; + socket_ptr sock; + + rpc_dispatcher_flags flags = {}; + std::atomic_uint busy_spin_users = 0; std::atomic_bool running; std::thread thread; @@ -547,35 +573,15 @@ static void rpc_dispatcher_trampoline(rpc_dispatcher * dispatcher) dispatcher->work(); } -void rpc_dispatcher::send(enum rpc_cmd cmd, std::shared_ptr input, size_t input_size) { +void rpc_dispatcher::send(enum rpc_cmd cmd, std::shared_ptr input, size_t input_size, const void * payload, size_t payload_size, void * output, size_t output_size) { auto msg = std::make_shared(); msg->cmd = cmd; - msg->input = input; - msg->input_size = input_size; - msg->output = nullptr; - msg->output_size = 0; - GGML_ASSERT(queue.push(msg)); - auto future = msg->completion.get_future(); - future.wait(); -} - -void rpc_dispatcher::send_async(enum rpc_cmd cmd, std::shared_ptr input, size_t input_size) { - auto msg = std::make_shared(); - msg->cmd = cmd; - msg->input = input; - msg->input_size = input_size; - msg->output = nullptr; - msg->output_size = 0; - GGML_ASSERT(queue.push(msg)); -} - -void rpc_dispatcher::send(enum rpc_cmd cmd, std::shared_ptr input, size_t input_size, void * output, size_t output_size) { - auto msg = std::make_shared(); - msg->cmd = cmd; - msg->input = input; + msg->input = std::move(input); msg->input_size = input_size; msg->output = output; msg->output_size = output_size; + msg->payload = payload; + msg->payload_size = payload_size; GGML_ASSERT(queue.push(msg)); auto future = msg->completion.get_future(); future.wait(); @@ -584,7 +590,7 @@ void rpc_dispatcher::send(enum rpc_cmd cmd, std::shared_ptr input, s void rpc_dispatcher::send_async(enum rpc_cmd cmd, std::shared_ptr input, size_t input_size, void * output, size_t output_size) { auto msg = std::make_shared(); msg->cmd = cmd; - msg->input = input; + msg->input = std::move(input); msg->input_size = input_size; msg->output = output; msg->output_size = output_size; @@ -638,26 +644,31 @@ void rpc_dispatcher::busy_spin_release() { GGML_ASSERT(previous > 0); } -void rpc_dispatcher::start(const std::string & endpoint) { +bool rpc_dispatcher::start(const std::string & endpoint) { std::string host; int port; if (!parse_endpoint(endpoint, host, port)) { - GGML_ABORT("Failed to parse endpoint: %s\n", endpoint.c_str()); + LOG_ERROR("Failed to parse endpoint: %s\n", endpoint.c_str()); + return false; } if (!rpc_transport_init()) { - GGML_ABORT("RPC transport initialization failed\n"); + LOG_ERROR("RPC transport initialization failed\n"); + return false; } sock = socket_t::connect(host.c_str(), port); if (sock == nullptr) { - GGML_ABORT("Failed to connect to %s\n", endpoint.c_str()); + LOG_ERROR("Failed to connect to %s\n", endpoint.c_str()); + return false; } - if (!negotiate_hello(sock)) { - GGML_ABORT("RPC handshake failed for %s\n", endpoint.c_str()); + if (!negotiate_hello(sock, flags)) { + LOG_ERROR("RPC handshake failed for %s\n", endpoint.c_str()); + return false; } LOG_DBG("[%s] connected to %s\n", __func__, endpoint.c_str()); running = true; thread = std::thread(rpc_dispatcher_trampoline, this); + return true; } void rpc_dispatcher::work() { @@ -684,7 +695,7 @@ void rpc_dispatcher::work() { bool status = send_rpc_cmd(sock, msg_ptr->cmd, msg_ptr->input.get(), msg_ptr->input_size, msg_ptr->output, msg_ptr->output_size); RPC_STATUS_ASSERT(status); } else { - bool status = send_rpc_cmd(sock, msg_ptr->cmd, msg_ptr->input.get(), msg_ptr->input_size); + bool status = send_rpc_cmd_payload(sock, msg_ptr->cmd, msg_ptr->input.get(), msg_ptr->input_size, msg_ptr->payload, msg_ptr->payload_size); RPC_STATUS_ASSERT(status); } if (trace) { @@ -721,7 +732,9 @@ static std::shared_ptr get_dispatcher(const std::string & endpoi } auto dispatcher = std::make_shared(); - dispatcher->start(endpoint); + if (!dispatcher->start(endpoint)) { + return nullptr; + } dispatchers[endpoint] = dispatcher; return dispatcher; } @@ -731,7 +744,7 @@ static void ggml_backend_rpc_buffer_free_buffer(ggml_backend_buffer_t buffer) { auto request = std::make_shared(); request->remote_ptr = ctx->remote_ptr; LOG_DBG2("[%s] remote_ptr: 0x%" PRIx64 "\n", __func__, request->remote_ptr); - ctx->dispatcher->send(RPC_CMD_FREE_BUFFER, request, sizeof(*request)); + ctx->dispatcher->send(RPC_CMD_FREE_BUFFER, request, sizeof(*request), nullptr, 0, nullptr, 0); delete ctx; } @@ -744,7 +757,7 @@ static void * ggml_backend_rpc_buffer_get_base(ggml_backend_buffer_t buffer) { request->remote_ptr = ctx->remote_ptr; LOG_DBG2("[%s] remote_ptr: 0x%" PRIx64 "\n", __func__, ctx->remote_ptr); rpc_msg_buffer_get_base_rsp response; - ctx->dispatcher->send(RPC_CMD_BUFFER_GET_BASE, request, sizeof(*request), &response, sizeof(response)); + ctx->dispatcher->send(RPC_CMD_BUFFER_GET_BASE, request, sizeof(*request), nullptr, 0, &response, sizeof(response)); ctx->base_ptr = reinterpret_cast(response.base_ptr); return ctx->base_ptr; } @@ -810,7 +823,7 @@ static enum ggml_status ggml_backend_rpc_buffer_init_tensor(ggml_backend_buffer_ LOG_DBG2("[%s] tensor: %s\n", __func__, rpc_log_format_tensor(tensor).c_str()); auto request = std::make_shared(); request->tensor = serialize_tensor(tensor); - ctx->dispatcher->send(RPC_CMD_INIT_TENSOR, request, sizeof(*request)); + ctx->dispatcher->send(RPC_CMD_INIT_TENSOR, request, sizeof(*request), nullptr, 0, nullptr, 0); } return GGML_STATUS_SUCCESS; } @@ -825,18 +838,21 @@ static void ggml_backend_rpc_buffer_memset_tensor( request->value = value; LOG_DBG2("[%s] tensor: %s, offset: %zu, size: %s, value: %u\n", __func__, rpc_log_format_tensor(tensor).c_str(), offset, rpc_log_format_size(size).c_str(), value); - ctx->dispatcher->send(RPC_CMD_MEMSET_TENSOR, request, sizeof(*request)); + ctx->dispatcher->send(RPC_CMD_MEMSET_TENSOR, request, sizeof(*request), nullptr, 0, nullptr, 0); } // input serialization format: | rpc_tensor | cache_flag (1 byte) | offset (8 bytes) | data (size bytes) +// with size == 0 only the header is serialized and data is sent separately as payload static std::shared_ptr serialize_set_tensor(const rpc_tensor & rpc_tensor, uint8_t cache_flag, uint64_t offset, const void * data, size_t size, size_t & input_size) { input_size = sizeof(rpc_tensor) + sizeof(cache_flag) + sizeof(offset) + size; - uint8_t * input = new uint8_t[input_size](); + uint8_t * input = new uint8_t[input_size]; uint8_t * p = input; memcpy(p, &rpc_tensor, sizeof(rpc_tensor)); p += sizeof(rpc_tensor); memcpy(p, &cache_flag, sizeof(cache_flag)); p += sizeof(cache_flag); memcpy(p, &offset, sizeof(offset)); p += sizeof(offset); - memcpy(p, data, size); + if (size > 0) { + memcpy(p, data, size); + } return std::shared_ptr(input, std::default_delete()); } @@ -844,8 +860,9 @@ static std::shared_ptr serialize_set_tensor(const rpc_tensor & rpc_tens // compute-buffer inputs (the activations ggml_backend_sched copies between backends) must not // take this path, otherwise with `rpc-server -c` every ubatch above the threshold is written // to the cache directory and later served from there. -static bool rpc_use_hash_cache(const ggml_tensor * tensor, size_t size) { - return size > HASH_THRESHOLD && tensor->buffer->usage == GGML_BACKEND_BUFFER_USAGE_WEIGHTS; +// hashing is slow (~1 GB/s), so only do it when the server asks for tensor hashes. +static bool rpc_use_hash_cache(const rpc_dispatcher & dispatcher, const ggml_tensor * tensor, size_t size) { + return dispatcher.get_send_tensor_hash() && size > HASH_THRESHOLD && tensor->buffer->usage == GGML_BACKEND_BUFFER_USAGE_WEIGHTS; } static void ggml_backend_rpc_buffer_set_tensor(ggml_backend_buffer_t buffer, ggml_tensor * tensor, const void * data, size_t offset, size_t size) { @@ -854,13 +871,13 @@ static void ggml_backend_rpc_buffer_set_tensor(ggml_backend_buffer_t buffer, ggm uint8_t cache_flag = 0; LOG_DBG2("[%s] tensor: %s, offset: %zu, size: %s\n", __func__, rpc_log_format_tensor(tensor).c_str(), offset, rpc_log_format_size(size).c_str()); - if (rpc_use_hash_cache(tensor, size)) { + if (rpc_use_hash_cache(*ctx->dispatcher, tensor, size)) { auto request = std::make_shared(); request->tensor = rpc_tensor; request->offset = offset; request->hash = fnv_hash((const uint8_t*)data, size); rpc_msg_set_tensor_hash_rsp response; - ctx->dispatcher->send(RPC_CMD_SET_TENSOR_HASH, request, sizeof(*request), &response, sizeof(response)); + ctx->dispatcher->send(RPC_CMD_SET_TENSOR_HASH, request, sizeof(*request), nullptr, 0, &response, sizeof(response)); LOG_DBG2("[%s] tensor: %s, hash: 0x%" PRIx64 ", cache: %s\n", __func__, rpc_log_format_tensor(tensor).c_str(), request->hash, response.result ? "hit" : "miss"); if (response.result) { @@ -870,9 +887,9 @@ static void ggml_backend_rpc_buffer_set_tensor(ggml_backend_buffer_t buffer, ggm // the server has no cache entry for this tensor - ask it to save one cache_flag = 1; } - size_t input_size; - auto input = serialize_set_tensor(rpc_tensor, cache_flag, offset, data, size, input_size); - ctx->dispatcher->send(RPC_CMD_SET_TENSOR, input, input_size); + size_t header_size; + auto header = serialize_set_tensor(rpc_tensor, cache_flag, offset, nullptr, 0, header_size); + ctx->dispatcher->send(RPC_CMD_SET_TENSOR, header, header_size, data, size, nullptr, 0); } static void ggml_backend_rpc_buffer_set_tensor_2d(ggml_backend_buffer_t buffer, ggml_tensor * tensor, const void * data, @@ -894,7 +911,7 @@ static void ggml_backend_rpc_buffer_set_tensor_2d(ggml_backend_buffer_t buffer, memcpy(dest + i*size, (const char *)data + i*stride_data, size); } std::shared_ptr input_ptr(input, std::default_delete()); - ctx->dispatcher->send(RPC_CMD_SET_TENSOR_2D, input_ptr, input_size); + ctx->dispatcher->send(RPC_CMD_SET_TENSOR_2D, input_ptr, input_size, nullptr, 0, nullptr, 0); } static void ggml_backend_rpc_buffer_get_tensor_2d(ggml_backend_buffer_t buffer, const ggml_tensor * tensor, void * data, @@ -909,10 +926,10 @@ static void ggml_backend_rpc_buffer_get_tensor_2d(ggml_backend_buffer_t buffer, request->n_copies = n_copies; request->stride = stride_tensor; if (stride_data == size) { - ctx->dispatcher->send(RPC_CMD_GET_TENSOR_2D, request, sizeof(*request), data, size*n_copies); + ctx->dispatcher->send(RPC_CMD_GET_TENSOR_2D, request, sizeof(*request), nullptr, 0, data, size*n_copies); } else { std::vector packed(size*n_copies); - ctx->dispatcher->send(RPC_CMD_GET_TENSOR_2D, request, sizeof(*request), packed.data(), packed.size()); + ctx->dispatcher->send(RPC_CMD_GET_TENSOR_2D, request, sizeof(*request), nullptr, 0, packed.data(), packed.size()); for (size_t i = 0; i < n_copies; i++) { memcpy((char *)data + i*stride_data, packed.data() + i*size, size); } @@ -927,7 +944,7 @@ static void ggml_backend_rpc_buffer_get_tensor(ggml_backend_buffer_t buffer, con request->tensor = serialize_tensor(tensor); request->offset = offset; request->size = size; - ctx->dispatcher->send(RPC_CMD_GET_TENSOR, request, sizeof(*request), data, size); + ctx->dispatcher->send(RPC_CMD_GET_TENSOR, request, sizeof(*request), nullptr, 0, data, size); } static bool ggml_backend_rpc_buffer_cpy_tensor(ggml_backend_buffer_t buffer, const ggml_tensor * src, ggml_tensor * dst) { @@ -947,7 +964,7 @@ static bool ggml_backend_rpc_buffer_cpy_tensor(ggml_backend_buffer_t buffer, con request->src = serialize_tensor(src); request->dst = serialize_tensor(dst); rpc_msg_copy_tensor_rsp response; - ctx->dispatcher->send(RPC_CMD_COPY_TENSOR, request, sizeof(*request), &response, sizeof(response)); + ctx->dispatcher->send(RPC_CMD_COPY_TENSOR, request, sizeof(*request), nullptr, 0, &response, sizeof(response)); return response.result; } return false; @@ -959,7 +976,7 @@ static void ggml_backend_rpc_buffer_clear(ggml_backend_buffer_t buffer, uint8_t request->remote_ptr = ctx->remote_ptr; request->value = value; LOG_DBG2("[%s] remote_ptr: 0x%" PRIx64 ", value: %u\n", __func__, request->remote_ptr, request->value); - ctx->dispatcher->send(RPC_CMD_BUFFER_CLEAR, request, sizeof(*request)); + ctx->dispatcher->send(RPC_CMD_BUFFER_CLEAR, request, sizeof(*request), nullptr, 0, nullptr, 0); } static ggml_backend_buffer_i ggml_backend_rpc_buffer_interface = { @@ -992,7 +1009,10 @@ static ggml_backend_buffer_t ggml_backend_rpc_buffer_type_alloc_buffer(ggml_back rpc_msg_alloc_buffer_rsp response; auto dispatcher = get_dispatcher(buft_ctx->endpoint); - dispatcher->send(RPC_CMD_ALLOC_BUFFER, request, sizeof(*request), &response, sizeof(response)); + if (dispatcher == nullptr) { + return nullptr; + } + dispatcher->send(RPC_CMD_ALLOC_BUFFER, request, sizeof(*request), nullptr, 0, &response, sizeof(response)); if (response.remote_ptr != 0) { LOG_DBG("[%s] endpoint: %s, device: %u, size: %s -> remote_ptr: 0x%" PRIx64 ", remote_size: %s\n", __func__, buft_ctx->endpoint.c_str(), buft_ctx->device, rpc_log_format_size(request->size).c_str(), @@ -1013,7 +1033,7 @@ static size_t get_alignment(const std::shared_ptr & dispatcher, auto request = std::make_shared(); request->device = device; rpc_msg_get_alignment_rsp response; - dispatcher->send(RPC_CMD_GET_ALIGNMENT, request, sizeof(*request), &response, sizeof(response)); + dispatcher->send(RPC_CMD_GET_ALIGNMENT, request, sizeof(*request), nullptr, 0, &response, sizeof(response)); LOG_DBG2("[%s] device: %u, alignment: %s\n", __func__, device, rpc_log_format_size(response.alignment).c_str()); return response.alignment; } @@ -1027,7 +1047,7 @@ static size_t get_max_size(const std::shared_ptr & dispatcher, u auto request = std::make_shared(); request->device = device; rpc_msg_get_max_size_rsp response; - dispatcher->send(RPC_CMD_GET_MAX_SIZE, request, sizeof(*request), &response, sizeof(response)); + dispatcher->send(RPC_CMD_GET_MAX_SIZE, request, sizeof(*request), nullptr, 0, &response, sizeof(response)); LOG_DBG2("[%s] device: %u, max_size: %s\n", __func__, device, rpc_log_format_size(response.max_size).c_str()); return response.max_size; } @@ -1112,7 +1132,10 @@ static size_t ggml_backend_rpc_buffer_type_get_alloc_size(ggml_backend_buffer_ty LOG_DBG2("[%s] cache miss: %s, querying server\n", __func__, rpc_log_format_tensor(tensor).c_str()); rpc_msg_get_alloc_size_rsp response; auto dispatcher = get_dispatcher(buft_ctx->endpoint); - dispatcher->send(RPC_CMD_GET_ALLOC_SIZE, request, sizeof(*request), &response, sizeof(response)); + if (dispatcher == nullptr) { + return min_size; + } + dispatcher->send(RPC_CMD_GET_ALLOC_SIZE, request, sizeof(*request), nullptr, 0, &response, sizeof(response)); { std::lock_guard lock(cache_mutex); @@ -1157,14 +1180,14 @@ static void ggml_backend_rpc_set_tensor_async(ggml_backend_t backend, ggml_tenso uint8_t cache_flag = 0; LOG_DBG2("[%s] tensor: %s, offset: %zu, size: %s\n", __func__, rpc_log_format_tensor(tensor).c_str(), offset, rpc_log_format_size(size).c_str()); - if (rpc_use_hash_cache(tensor, size)) { + if (rpc_use_hash_cache(*ctx->dispatcher, tensor, size)) { auto request = std::make_shared(); request->tensor = rpc_tensor; request->offset = offset; request->hash = fnv_hash((const uint8_t*)data, size); rpc_msg_set_tensor_hash_rsp response; // TODO: make this async - ctx->dispatcher->send(RPC_CMD_SET_TENSOR_HASH, request, sizeof(*request), &response, sizeof(response)); + ctx->dispatcher->send(RPC_CMD_SET_TENSOR_HASH, request, sizeof(*request), nullptr, 0, &response, sizeof(response)); LOG_DBG2("[%s] tensor: %s, hash: 0x%" PRIx64 ", cache: %s\n", __func__, rpc_log_format_tensor(tensor).c_str(), request->hash, response.result ? "hit" : "miss"); if (response.result) { @@ -1332,6 +1355,9 @@ ggml_backend_buffer_type_t ggml_backend_rpc_buffer_type(const char * endpoint, u return it->second; } auto dispatcher = get_dispatcher(endpoint); + if (dispatcher == nullptr) { + return nullptr; + } size_t alignment = get_alignment(dispatcher, device); size_t max_size = get_max_size(dispatcher, device); ggml_backend_rpc_buffer_type_context * buft_ctx = new ggml_backend_rpc_buffer_type_context { @@ -1356,6 +1382,9 @@ ggml_backend_buffer_type_t ggml_backend_rpc_buffer_type(const char * endpoint, u ggml_backend_t ggml_backend_rpc_init(const char * endpoint, uint32_t device) { std::string dev_name = "RPC" + std::to_string(device) + "[" + std::string(endpoint) + "]"; auto dispatcher = get_dispatcher(endpoint); + if (dispatcher == nullptr) { + return nullptr; + } ggml_backend_rpc_context * ctx = new ggml_backend_rpc_context { /* .dispatcher = */ dispatcher, /* .endpoint = */ endpoint, @@ -1379,10 +1408,15 @@ bool ggml_backend_is_rpc(ggml_backend_t backend) { void ggml_backend_rpc_get_device_memory(const char * endpoint, uint32_t device, size_t * free, size_t * total) { auto dispatcher = get_dispatcher(endpoint); + if (dispatcher == nullptr) { + *free = 0; + *total = 0; + return; + } auto request = std::make_shared(); request->device = device; rpc_msg_get_device_memory_rsp response; - dispatcher->send(RPC_CMD_GET_DEVICE_MEMORY, request, sizeof(*request), &response, sizeof(response)); + dispatcher->send(RPC_CMD_GET_DEVICE_MEMORY, request, sizeof(*request), nullptr, 0, &response, sizeof(response)); LOG_DBG2("[%s] device: %u, free: %s, total: %s\n", __func__, device, rpc_log_format_size(response.free_mem).c_str(), rpc_log_format_size(response.total_mem).c_str()); *free = response.free_mem; @@ -1408,7 +1442,7 @@ class rpc_server { bool free_buffer(const rpc_msg_free_buffer_req & request); bool buffer_clear(const rpc_msg_buffer_clear_req & request); bool memset_tensor(const rpc_msg_memset_tensor_req & request); - bool set_tensor(const std::vector & input); + bool set_tensor(socket_ptr sock, uint64_t input_size); bool set_tensor_2d(const std::vector & input); bool set_tensor_hash(const rpc_msg_set_tensor_hash_req & request, rpc_msg_set_tensor_hash_rsp & response); bool get_tensor(const rpc_msg_get_tensor_req & request, std::vector & response); @@ -1430,6 +1464,9 @@ class rpc_server { private: void sync_all_backends(); + void comm_free_device(uint32_t dev_id); + static bool comm_allreduce_pairwise(void * comm_ctx_v, ggml_tensor ** tensors); + static bool comm_bootstrap_exchange(void * user_data, const void * send_buf, void * recv_buf, size_t size); bool get_cached_file(uint64_t hash, std::vector & data); ggml_tensor * deserialize_tensor(struct ggml_context * ctx, const rpc_tensor * tensor); ggml_tensor * create_node(uint64_t id, @@ -1438,15 +1475,26 @@ class rpc_server { std::unordered_map & tensor_map); - // pairwise allreduce over a direct connection to the peer server + // Per-device state for one communicator. struct comm_state { - socket_ptr peer; + // Sockets to the other servers. Rank 0 holds one per peer in rank order, so peers[i] is + // rank i + 1. Every other rank holds one to rank 0. The pairwise path reduces over peers[0]. + std::vector peers; uint32_t rank = 0; uint32_t world = 0; + ggml_backend_t pairwise_backend = nullptr; ggml_backend_buffer_ptr scratch; size_t scratch_size = 0; std::vector send_buf; std::vector recv_buf; + + // The backend's communicator. Null on the pairwise path, which has none. + void * backend_comm_ctx = nullptr; + + // The chosen reduction, set once a path is agreed. Receives backend_comm_ctx, or + // comm_state on the pairwise path. A communicator covers one device, so the caller + // passes a one-element tensor array. + ggml_backend_comm_allreduce_tensor_t allreduce_fn = nullptr; }; std::vector backends; @@ -1455,12 +1503,15 @@ class rpc_server { // computed graphs cached per backend, keyed by uid std::vector> stored_graphs; std::vector comm_states; + // SET_TENSOR data is received in chunks through this buffer + std::vector set_tensor_buf; }; void rpc_server::hello(rpc_msg_hello_rsp & response) { response.major = RPC_PROTO_MAJOR_VERSION; response.minor = RPC_PROTO_MINOR_VERSION; response.patch = RPC_PROTO_PATCH_VERSION; + response.flags = cache_dir ? RPC_DISPATCHER_FLAGS_SEND_TENSOR_HASH : 0; LOG_DBG("[%s] version: %d.%d.%d\n", __func__, response.major, response.minor, response.patch); } @@ -1701,19 +1752,21 @@ ggml_tensor * rpc_server::deserialize_tensor(struct ggml_context * ctx, const rp } -bool rpc_server::set_tensor(const std::vector & input) { +bool rpc_server::set_tensor(socket_ptr sock, uint64_t input_size) { sync_all_backends(); // serialization format: | rpc_tensor | cache_flag (1 byte) | offset (8 bytes) | data (size bytes) | + // only the header is read here, the data is streamed from the socket below uint8_t cache_flag; uint64_t offset; - const size_t header_size = sizeof(rpc_tensor) + sizeof(cache_flag) + sizeof(offset); - if (input.size() < header_size) { + uint8_t header[sizeof(rpc_tensor) + sizeof(cache_flag) + sizeof(offset)]; + const size_t header_size = sizeof(header); + if (input_size < header_size || !sock->recv_data(header, header_size)) { return false; } - const rpc_tensor * in_tensor = (const rpc_tensor *)input.data(); - memcpy(&cache_flag, input.data() + sizeof(rpc_tensor), sizeof(cache_flag)); - memcpy(&offset, input.data() + sizeof(rpc_tensor) + sizeof(cache_flag), sizeof(offset)); - const size_t size = input.size() - header_size; + const rpc_tensor * in_tensor = (const rpc_tensor *)header; + memcpy(&cache_flag, header + sizeof(rpc_tensor), sizeof(cache_flag)); + memcpy(&offset, header + sizeof(rpc_tensor) + sizeof(cache_flag), sizeof(offset)); + const size_t size = input_size - header_size; struct ggml_init_params params { /*.mem_size =*/ ggml_tensor_overhead(), @@ -1744,18 +1797,37 @@ bool rpc_server::set_tensor(const std::vector & input) { } } - const void * data = input.data() + header_size; if (cache_dir && cache_flag) { - uint64_t hash = fnv_hash((const uint8_t*)data, size); + // the whole data is needed to compute the hash + std::vector data(size); + if (!sock->recv_data(data.data(), size)) { + return false; + } + uint64_t hash = fnv_hash(data.data(), size); char hash_str[17]; snprintf(hash_str, sizeof(hash_str), "%016" PRIx64, hash); // save to cache_dir/hash_str fs::path cache_file = fs::path(cache_dir) / hash_str; std::ofstream ofs(cache_file, std::ios::binary); - ofs.write((const char *)data, size); + ofs.write((const char *)data.data(), size); LOG_INFO("[%s] saved to '%s'\n", __func__, cache_file.string().c_str()); + ggml_backend_tensor_set(tensor, data.data(), offset, size); + return true; + } + + // a buffer of the tensor size would be allocated and page faulted for every tensor, which is slow + const size_t chunk_size = 16*1024*1024; + if (set_tensor_buf.size() < std::min(size, chunk_size)) { + set_tensor_buf.resize(std::min(size, chunk_size)); + } + for (size_t done = 0; done < size; ) { + const size_t n = std::min(size - done, chunk_size); + if (!sock->recv_data(set_tensor_buf.data(), n)) { + return false; + } + ggml_backend_tensor_set(tensor, set_tensor_buf.data(), offset + done, n); + done += n; } - ggml_backend_tensor_set(tensor, data, offset, size); return true; } @@ -2225,78 +2297,188 @@ void rpc_server::sync_all_backends() { } } -// The comm link between two servers uses the same caps negotiation as the client HELLO, +void rpc_server::comm_free_device(uint32_t dev_id) { + comm_state & state = comm_states[dev_id]; + if (state.backend_comm_ctx != nullptr) { + auto comm_free = (ggml_backend_comm_free_t) ggml_backend_reg_get_proc_address( + ggml_backend_dev_backend_reg(ggml_backend_get_device(backends[dev_id])), "ggml_backend_comm_free"); + if (comm_free != nullptr) { + comm_free(state.backend_comm_ctx); + } + } + state = comm_state(); +} + +// Exchange RPC transport capabilities over a peer connection. recv_first selects which side +// receives first. Both sides receiving first would deadlock. +static bool comm_caps_handshake(const socket_ptr & peer, bool recv_first) { + uint8_t local_caps[RPC_CONN_CAPS_SIZE] = {}; + uint8_t remote_caps[RPC_CONN_CAPS_SIZE] = {}; + peer->get_caps(local_caps); + if (recv_first) { + if (!peer->recv_data(remote_caps, sizeof(remote_caps)) || + !peer->send_data(local_caps, sizeof(local_caps))) { + return false; + } + } else { + if (!peer->send_data(local_caps, sizeof(local_caps)) || + !peer->recv_data(remote_caps, sizeof(remote_caps))) { + return false; + } + } + peer->update_caps(remote_caps); + return true; +} + +// The comm link between servers uses the same caps negotiation as the client HELLO, // so it gets the same transport upgrades (e.g. RDMA). bool rpc_server::comm_init(const rpc_msg_comm_init_req & request, rpc_msg_comm_init_rsp & response) { response.ok = 0; - if (request.device >= backends.size() || request.world != 2 || request.rank >= request.world) { + if (request.device >= backends.size() || request.world < 2 || request.rank >= request.world) { return true; } - comm_state & state = comm_states[request.device]; - if (state.peer != nullptr) { + const uint32_t dev_id = request.device; + comm_state & state = comm_states[dev_id]; + if (state.allreduce_fn != nullptr) { response.ok = 1; return true; } - uint8_t local_caps[RPC_CONN_CAPS_SIZE] = {}; - uint8_t remote_caps[RPC_CONN_CAPS_SIZE] = {}; + + // Joining a communicator blocks until every rank arrives, so drain outstanding work first. + sync_all_backends(); + + // Rank 0 listens and accepts world-1 peers; every other rank connects back to it. if (request.rank == 0) { socket_ptr srv = socket_t::create_server("0.0.0.0", request.port); if (srv == nullptr) { LOG_ERROR("[%s] failed to listen on comm port %u\n", __func__, request.port); return true; } - state.peer = srv->accept(); - if (state.peer == nullptr) { - return true; - } - if (!state.peer->recv_data(remote_caps, sizeof(remote_caps))) { - state.peer = nullptr; - return true; - } - state.peer->get_caps(local_caps); - if (!state.peer->send_data(local_caps, sizeof(local_caps))) { - state.peer = nullptr; - return true; + // Store peers[i] as rank i + 1, as comm_bootstrap_exchange needs. + // Each peer sends its rank because peers may connect in any order. + state.peers.resize(request.world - 1); + for (uint32_t i = 1; i < request.world; i++) { + socket_ptr peer = srv->accept(); + uint32_t peer_rank = 0; + if (peer == nullptr || !comm_caps_handshake(peer, /*recv_first =*/ true) || + !peer->recv_data(&peer_rank, sizeof(peer_rank)) || + peer_rank == 0 || peer_rank >= request.world || state.peers[peer_rank - 1] != nullptr) { + LOG_ERROR("[%s] bad peer connection (rank %u)\n", __func__, peer_rank); + state.peers.clear(); + return true; + } + state.peers[peer_rank - 1] = std::move(peer); } - state.peer->update_caps(remote_caps); } else { const std::string host(request.host, strnlen(request.host, sizeof(request.host))); + socket_ptr peer; // rank 0 may not be listening yet, retry for a few seconds - for (int i = 0; i < 100 && state.peer == nullptr; i++) { - state.peer = socket_t::connect(host.c_str(), request.port); - if (state.peer == nullptr) { + for (int i = 0; i < 100 && peer == nullptr; i++) { + peer = socket_t::connect(host.c_str(), request.port); + if (peer == nullptr) { std::this_thread::sleep_for(std::chrono::milliseconds(50)); } } - if (state.peer == nullptr) { + if (peer == nullptr) { LOG_ERROR("[%s] failed to connect to peer %s:%u\n", __func__, host.c_str(), request.port); return true; } - state.peer->get_caps(local_caps); - if (!state.peer->send_data(local_caps, sizeof(local_caps)) || - !state.peer->recv_data(remote_caps, sizeof(remote_caps))) { - state.peer = nullptr; + if (!comm_caps_handshake(peer, /*recv_first =*/ false) || + !peer->send_data(&request.rank, sizeof(request.rank)) || !peer->flush()) { return true; } - state.peer->update_caps(remote_caps); + state.peers.push_back(std::move(peer)); } + state.rank = request.rank; state.world = request.world; - LOG_INFO("[%s] device %u joined pairwise comm as rank %u\n", __func__, request.device, request.rank); + + // Forces the pairwise path, for comparing the two on one build. Setting it on any one rank + // is enough, because the negotiation spreads the decision. + const bool no_backend_communicator = std::getenv("GGML_RPC_NO_BACKEND_COMMUNICATOR") != nullptr; + + ggml_backend_reg_t reg = ggml_backend_dev_backend_reg(ggml_backend_get_device(backends[dev_id])); + auto init_rank = (ggml_backend_comm_init_rank_t) + ggml_backend_reg_get_proc_address(reg, "ggml_backend_comm_init_rank"); + auto allreduce_fn = (ggml_backend_comm_allreduce_tensor_t) + ggml_backend_reg_get_proc_address(reg, "ggml_backend_comm_allreduce_tensor"); + + // The ranks must agree on backend communicator or pairwise. Every rank shares whether its + // backend provides a communicator, and all ranks use it only if every rank can. + const uint8_t self_can_use_backend_comm = !no_backend_communicator && init_rank != nullptr && allreduce_fn != nullptr; + std::vector all_can_use_backend_comm(request.world); + if (!comm_bootstrap_exchange(&state, &self_can_use_backend_comm, all_can_use_backend_comm.data(), + sizeof(self_can_use_backend_comm))) { + state.peers.clear(); + return true; + } + const bool use_backend_communicator = std::all_of(all_can_use_backend_comm.begin(), all_can_use_backend_comm.end(), + [](uint8_t c) { return c != 0; }); + + if (use_backend_communicator) { + void * backend_ctx = init_rank(backends[dev_id], (int) request.rank, (int) request.world, comm_bootstrap_exchange, &state); + if (backend_ctx == nullptr) { + LOG_ERROR("[%s] failed to join communicator as rank %u/%u\n", __func__, request.rank, request.world); + state.peers.clear(); + return true; + } + state.backend_comm_ctx = backend_ctx; + state.allreduce_fn = allreduce_fn; + // The sockets have done their job; the collective runs between the backends from here on. + state.peers.clear(); + LOG_INFO("[%s] device %u using backend communicator, rank %u/%u\n", __func__, dev_id, request.rank, request.world); + response.ok = 1; + return true; + } + + // No backend communicator, so fall back to the pairwise exchange, which only covers two ranks. + if (request.world != 2) { + LOG_ERROR("[%s] world size %u requires a backend communicator\n", __func__, request.world); + state.peers.clear(); + return true; + } + state.pairwise_backend = backends[dev_id]; + state.allreduce_fn = comm_allreduce_pairwise; + LOG_INFO("[%s] device %u using pairwise communicator, rank %u\n", __func__, dev_id, request.rank); response.ok = 1; return true; } -bool rpc_server::comm_allreduce(const rpc_msg_comm_allreduce_req & request) { - if (request.device >= backends.size()) { - return false; - } - comm_state & state = comm_states[request.device]; - if (state.peer == nullptr) { - LOG_ERROR("[%s] no communicator for device %u\n", __func__, request.device); - return false; +// Bootstrap callback for ggml_backend_comm_init_rank_t, over the peer sockets. Rank 0 collects +// size bytes from every rank in rank order, then sends the whole array back to every rank. +// user_data is the comm_state. +bool rpc_server::comm_bootstrap_exchange(void * user_data, const void * send_buf, void * recv_buf, size_t size) { + comm_state & state = *(comm_state *) user_data; + uint8_t * recv = (uint8_t *) recv_buf; + const size_t total = (size_t) state.world * size; + + if (state.rank == 0) { + memcpy(recv, send_buf, size); + for (size_t i = 0; i < state.peers.size(); i++) { + if (!state.peers[i]->recv_data(recv + (i + 1) * size, size)) { + return false; + } + } + for (const socket_ptr & peer : state.peers) { + if (!peer->send_data(recv, total) || !peer->flush()) { + return false; + } + } + return true; } - ggml_backend_t backend = backends[request.device]; + return state.peers[0]->send_data(send_buf, size) && state.peers[0]->flush() && + state.peers[0]->recv_data(recv, total); +} + +// Reduces this rank's contribution in place, exchanging partials with the peer over peers[0]. +// Used when the backend provides no communicator. comm_ctx_v is the comm_state. +bool rpc_server::comm_allreduce_pairwise(void * comm_ctx_v, ggml_tensor ** tensors) { + comm_state & state = *(comm_state *) comm_ctx_v; + ggml_tensor * t_dst = tensors[0]; + ggml_backend_t backend = state.pairwise_backend; + + const size_t nbytes = ggml_nbytes(t_dst); + const int64_t ne = ggml_nelements(t_dst); size_t ctx_size = 16*ggml_tensor_overhead() + 2*ggml_graph_overhead_custom(8, false); struct ggml_init_params params = { @@ -2307,16 +2489,7 @@ bool rpc_server::comm_allreduce(const rpc_msg_comm_allreduce_req & request) { ggml_context_ptr ctx_ptr { ggml_init(params) }; GGML_ASSERT(ctx_ptr != nullptr); ggml_context * ctx = ctx_ptr.get(); - ggml_tensor * t_dst = deserialize_tensor(ctx, &request.tensor); - if (t_dst == nullptr || t_dst->buffer == nullptr) { - LOG_ERROR("[%s] error deserializing tensor\n", __func__); - return false; - } - const size_t nbytes = ggml_nbytes(t_dst); - const int64_t ne = ggml_nelements(t_dst); - if (nbytes == 0) { - return true; - } + // reduce large partials in bf16 to halve the wire bytes; small (decode-sized) ones // stay f32 since the extra casts and sync cost more than the bytes saved const bool wire_bf16 = t_dst->type == GGML_TYPE_F32 && ne >= 32768; @@ -2372,13 +2545,13 @@ bool rpc_server::comm_allreduce(const rpc_msg_comm_allreduce_req & request) { // rank 0 sends first, rank 1 receives first, so large payloads cannot deadlock if (state.rank == 0) { - if (!state.peer->send_data(state.send_buf.data(), wire_bytes) || !state.peer->flush() || - !state.peer->recv_data(state.recv_buf.data(), wire_bytes)) { + if (!state.peers[0]->send_data(state.send_buf.data(), wire_bytes) || !state.peers[0]->flush() || + !state.peers[0]->recv_data(state.recv_buf.data(), wire_bytes)) { return false; } } else { - if (!state.peer->recv_data(state.recv_buf.data(), wire_bytes) || - !state.peer->send_data(state.send_buf.data(), wire_bytes) || !state.peer->flush()) { + if (!state.peers[0]->recv_data(state.recv_buf.data(), wire_bytes) || + !state.peers[0]->send_data(state.send_buf.data(), wire_bytes) || !state.peers[0]->flush()) { return false; } } @@ -2408,11 +2581,50 @@ bool rpc_server::comm_allreduce(const rpc_msg_comm_allreduce_req & request) { return true; } +bool rpc_server::comm_allreduce(const rpc_msg_comm_allreduce_req & request) { + if (request.device >= backends.size()) { + return false; + } + comm_state & state = comm_states[request.device]; + if (state.allreduce_fn == nullptr) { + LOG_ERROR("[%s] no communicator for device %u\n", __func__, request.device); + return false; + } + + struct ggml_init_params params = { + /*.mem_size =*/ ggml_tensor_overhead(), + /*.mem_buffer =*/ NULL, + /*.no_alloc =*/ true, + }; + ggml_context_ptr ctx_ptr { ggml_init(params) }; + GGML_ASSERT(ctx_ptr != nullptr); + ggml_tensor * t_dst = deserialize_tensor(ctx_ptr.get(), &request.tensor); + if (t_dst == nullptr || t_dst->buffer == nullptr) { + LOG_ERROR("[%s] error deserializing tensor\n", __func__); + return false; + } + if (ggml_nbytes(t_dst) == 0) { + return true; + } + + // Neither path needs a wait here. The pairwise path waits internally, and the backend orders + // its reduction after the work that produced the partial sum. + void * comm_ctx = state.backend_comm_ctx != nullptr ? state.backend_comm_ctx : &state; + if (!state.allreduce_fn(comm_ctx, &t_dst)) { + LOG_ERROR("[%s] allreduce failed for tensor %s\n", __func__, t_dst->name); + return false; + } + return true; +} + bool rpc_server::comm_free(const rpc_msg_comm_free_req & request) { if (request.device >= backends.size()) { return false; } - comm_states[request.device] = comm_state(); + // Wait for outstanding work. Destroying a communicator while its kernels are still running + // is unsafe. + sync_all_backends(); + comm_free_device(request.device); return true; } @@ -2432,6 +2644,14 @@ bool rpc_server::get_device_memory(const rpc_msg_get_device_memory_req & request } rpc_server::~rpc_server() { + // Wait for outstanding work. Destroying a communicator while its kernels are still running + // is unsafe. + sync_all_backends(); + + // Free any communicator still open. A client can disconnect without sending COMM_FREE. + for (uint32_t dev_id = 0; dev_id < comm_states.size(); dev_id++) { + comm_free_device(dev_id); + } for (auto buffer : buffers) { ggml_backend_buffer_free(buffer); } @@ -2604,11 +2824,11 @@ static void rpc_serve_client(const std::vector & backends, const break; } case RPC_CMD_SET_TENSOR: { - std::vector input; - if (!recv_msg(sock, input)) { + uint64_t input_size; + if (!sock->recv_data(&input_size, sizeof(input_size))) { return; } - if (!server.set_tensor(input)) { + if (!server.set_tensor(sock, input_size)) { return; } break; @@ -2910,18 +3130,27 @@ static bool ggml_backend_rpc_device_supports_buft(ggml_backend_dev_t dev, ggml_b static ggml_backend_event_t ggml_backend_rpc_device_event_new(ggml_backend_dev_t dev) { ggml_backend_rpc_device_context * ctx = (ggml_backend_rpc_device_context *)dev->context; auto dispatcher = get_dispatcher(ctx->endpoint); + if (dispatcher == nullptr) { + return nullptr; + } return dispatcher->event_new(dev); } static void ggml_backend_rpc_device_event_free(ggml_backend_dev_t dev, ggml_backend_event_t event) { ggml_backend_rpc_device_context * ctx = (ggml_backend_rpc_device_context *)dev->context; auto dispatcher = get_dispatcher(ctx->endpoint); + if (dispatcher == nullptr) { + return; + } dispatcher->event_free(event); } static void ggml_backend_rpc_device_event_synchronize(ggml_backend_dev_t dev, ggml_backend_event_t event) { ggml_backend_rpc_device_context * ctx = (ggml_backend_rpc_device_context *)dev->context; auto dispatcher = get_dispatcher(ctx->endpoint); + if (dispatcher == nullptr) { + return; + } dispatcher->event_synchronize(event); } @@ -2990,14 +3219,14 @@ static void ggml_backend_rpc_comm_free(void * comm_ctx_v) { for (const auto & rank : comm_ctx->ranks) { auto request = std::make_shared(); request->device = rank.device; - rank.dispatcher->send(RPC_CMD_COMM_FREE, request, sizeof(*request)); + rank.dispatcher->send(RPC_CMD_COMM_FREE, request, sizeof(*request), nullptr, 0, nullptr, 0); rank.dispatcher->busy_spin_release(); } delete comm_ctx; } static void * ggml_backend_rpc_comm_init(ggml_backend_t * backends, size_t n_backends) { - if (n_backends != 2 || std::getenv("GGML_RPC_NO_COMM") != nullptr) { + if (n_backends < 2 || std::getenv("GGML_RPC_NO_COMM") != nullptr) { return nullptr; } std::vector ranks; @@ -3141,8 +3370,11 @@ ggml_backend_reg_t ggml_backend_rpc_reg(void) { static uint32_t ggml_backend_rpc_get_device_count(const char * endpoint) { auto dispatcher = get_dispatcher(endpoint); + if (dispatcher == nullptr) { + return 0; + } rpc_msg_device_count_rsp response; - dispatcher->send(RPC_CMD_DEVICE_COUNT, nullptr, 0, &response, sizeof(response)); + dispatcher->send(RPC_CMD_DEVICE_COUNT, nullptr, 0, nullptr, 0, &response, sizeof(response)); return response.device_count; } diff --git a/ggml/src/ggml-rpc/transport.cpp b/ggml/src/ggml-rpc/transport.cpp index d5670f34d904..c7d9f532117c 100644 --- a/ggml/src/ggml-rpc/transport.cpp +++ b/ggml/src/ggml-rpc/transport.cpp @@ -71,6 +71,11 @@ struct rdma_conn { struct ibv_mr * rx_mr = nullptr; int rx_head = 0; + // received slot that is only partly consumed, so recv sizes need not match send sizes + int rx_cur = -1; + size_t rx_off = 0; + size_t rx_len = 0; + uint32_t max_inline = 0; uint8_t * rx_slot(int i) const { @@ -512,19 +517,24 @@ bool socket_t::impl::rdma_recv(void * data, size_t size) { uint8_t * dst = (uint8_t *)data; size_t rem = size; while (rem > 0) { - struct ibv_wc wc; - if (!rdma_poll(c->rcq, &wc)) return false; + if (c->rx_cur < 0) { + struct ibv_wc wc; + if (!rdma_poll(c->rcq, &wc)) return false; - int slot = (int)wc.wr_id; - size_t got = wc.byte_len; - if (got > rem) { - LOG_ERROR("RDMA recv overrun: got=%zu rem=%zu\n", got, rem); - c->post_rx(slot); - return false; + c->rx_cur = (int)wc.wr_id; + c->rx_off = 0; + c->rx_len = wc.byte_len; } - memcpy(dst, c->rx_slot(slot), got); - if (!c->post_rx(slot)) return false; + size_t got = std::min(rem, c->rx_len - c->rx_off); + memcpy(dst, c->rx_slot(c->rx_cur) + c->rx_off, got); + c->rx_off += got; + + // repost the slot only once it is fully consumed + if (c->rx_off == c->rx_len) { + if (!c->post_rx(c->rx_cur)) return false; + c->rx_cur = -1; + } dst += got; rem -= got; diff --git a/ggml/src/ggml-sycl/CMakeLists.txt b/ggml/src/ggml-sycl/CMakeLists.txt index 35d36e583734..8312a83052ca 100644 --- a/ggml/src/ggml-sycl/CMakeLists.txt +++ b/ggml/src/ggml-sycl/CMakeLists.txt @@ -221,7 +221,7 @@ if (GGML_SYCL_TARGET STREQUAL "INTEL") if (NOT GGML_SYCL_DEVICE_ARCH) target_link_options(ggml-sycl PRIVATE -Xs -ze-intel-greater-than-4GB-buffer-required) else() - message(STATUS "Skipping -ze-intel-greater-than-4GB-buffer-required for spir64_gen AOT") + message(STATUS "spir64_gen AOT: -ze-intel-greater-than-4GB-buffer-required passed to ocloc") endif() # Link against Intel oneMKL @@ -251,7 +251,7 @@ if (GGML_SYCL_DEVICE_ARCH) target_compile_options( ggml-sycl PRIVATE -fsycl-targets=spir64_gen - "SHELL:-Xsycl-target-backend=spir64_gen \"-device ${GGML_SYCL_DEVICE_ARCH}\"" + "SHELL:-Xsycl-target-backend=spir64_gen \"-device ${GGML_SYCL_DEVICE_ARCH} -options -ze-intel-greater-than-4GB-buffer-required\"" ) # Pass through parallel job (process) count for parallelising the @@ -266,7 +266,7 @@ if (GGML_SYCL_DEVICE_ARCH) target_link_options( ggml-sycl PRIVATE -fsycl-targets=spir64_gen - "SHELL:-Xsycl-target-backend=spir64_gen \"-device ${GGML_SYCL_DEVICE_ARCH}\"" + "SHELL:-Xsycl-target-backend=spir64_gen \"-device ${GGML_SYCL_DEVICE_ARCH} -options -ze-intel-greater-than-4GB-buffer-required\"" -fsycl-max-parallel-link-jobs=${GGML_SYCL_MAX_PARALLEL_LINK_JOBS} ) diff --git a/ggml/src/ggml-sycl/common.cpp b/ggml/src/ggml-sycl/common.cpp index f01d0ff955e5..44cfede0b90a 100644 --- a/ggml/src/ggml-sycl/common.cpp +++ b/ggml/src/ggml-sycl/common.cpp @@ -11,6 +11,8 @@ // #include "common.hpp" +#include +#include #include #ifdef GGML_SYCL_SUPPORT_LEVEL_ZERO_API #include @@ -19,8 +21,14 @@ #include "ggml-backend-impl.h" #include "ggml-impl.h" +thread_local int g_sycl_current_device_id = -1; + int get_current_device_id() { - return dpct::dev_mgr::instance().current_device_id(); + if (g_sycl_current_device_id >= 0) { + return g_sycl_current_device_id; + } + g_sycl_current_device_id = static_cast(dpct::dev_mgr::instance().current_device_id()); + return g_sycl_current_device_id; } void* ggml_sycl_host_malloc(size_t size) try { @@ -92,9 +100,16 @@ static bool ggml_sycl_use_level_zero_device_alloc(sycl::queue &q) { } #endif +// Device bytes allocated through ggml_sycl_malloc_device, per device and per pointer +static std::mutex g_ggml_sycl_dev_alloc_mutex; +static std::unordered_map> g_ggml_sycl_dev_allocs; +static size_t g_ggml_sycl_dev_alloc_bytes[GGML_SYCL_MAX_DEVICES] = {}; +// free memory reported when we had nothing allocated (0 = not taken yet) +static size_t g_ggml_sycl_dev_free_baseline[GGML_SYCL_MAX_DEVICES] = {}; + // Use Level Zero zeMemAllocDevice to avoid sycl::malloc_device triggering // DMA-buf/TTM system RAM staging in the xe kernel driver during multi-GPU inference. -void * ggml_sycl_malloc_device(size_t size, sycl::queue &q, ggml_sycl_mem_type type) { +static void * ggml_sycl_malloc_device_impl(size_t size, sycl::queue &q, ggml_sycl_mem_type type) { #ifdef GGML_SYCL_SUPPORT_LEVEL_ZERO_API if (ggml_sycl_use_level_zero_device_alloc(q)) { void *ptr = nullptr; @@ -134,8 +149,78 @@ void * ggml_sycl_malloc_device(size_t size, sycl::queue &q, ggml_sycl_mem_type t return ptr; } +// caller holds g_ggml_sycl_dev_alloc_mutex +static bool ggml_sycl_free_mem_enough(int device, size_t size, ggml_sycl_mem_type type) { + const size_t allocated = g_ggml_sycl_dev_alloc_bytes[device]; + + // TODO: remove once zeMemAllocDevice refuses allocations beyond VRAM on BMG + // (https://github.com/intel/compute-runtime/issues/1047). Only model/split buffers are checked. + const gpu_arch arch = ggml_sycl_info().devices[device].hw_info.arch; + const bool is_bmg = arch == gpu_arch::intel_gpu_bmg_g21 || arch == gpu_arch::intel_gpu_bmg_g31; + const bool check = g_ggml_sycl_bmg_vram_check && is_bmg && type == GGML_SYCL_MEM_BUFFER; + const bool trace = type == GGML_SYCL_MEM_BUFFER && ggml_sycl_memtrace_enabled(); + size_t free_mem = 0, total_mem = 0; + if (check || trace) { + sycl_get_mem_info(device, &free_mem, &total_mem); + } + if (trace) { + ggml_sycl_memtrace_report_device("before buffer alloc", device, free_mem, total_mem); + } + if (check) { + size_t & baseline = g_ggml_sycl_dev_free_baseline[device]; + if (allocated == 0) { + baseline = free_mem; + } + // the drop in free memory since the baseline is ours once written, the rest of ours is not counted yet + const size_t written = baseline > free_mem ? baseline - free_mem : 0; + const size_t unwritten = baseline && allocated > written ? allocated - written : 0; + const size_t usable = free_mem > unwritten ? free_mem - unwritten : 0; + if (allocated + size > total_mem || size > usable) { + GGML_LOG_ERROR("%s: SYCL%d: cannot allocate %zu MiB: %zu MiB already allocated, %zu MiB usable " + "(%zu MiB free of %zu MiB), set GGML_SYCL_BMG_VRAM_CHECK=0 to disable this check\n", + __func__, device, size >> 20, allocated >> 20, usable >> 20, free_mem >> 20, total_mem >> 20); + return false; + } + } + return true; +} + +static void ggml_sycl_dev_alloc_record(void * ptr, int device, size_t size) { + g_ggml_sycl_dev_allocs[ptr] = { device, size }; + g_ggml_sycl_dev_alloc_bytes[device] += size; +} + +void * ggml_sycl_malloc_device(size_t size, sycl::queue &q, ggml_sycl_mem_type type) { + const int device = (int) dpct::dev_mgr::instance().get_device_id(q.get_device()); + GGML_ASSERT(device >= 0 && device < GGML_SYCL_MAX_DEVICES); + + std::lock_guard lock(g_ggml_sycl_dev_alloc_mutex); + if (!ggml_sycl_free_mem_enough(device, size, type)) { + ggml_sycl_memtrace_fail(type, size); + return nullptr; + } + + void * ptr = ggml_sycl_malloc_device_impl(size, q, type); + if (ptr) { + ggml_sycl_dev_alloc_record(ptr, device, size); + } + return ptr; +} + void ggml_sycl_free_device(void *ptr, sycl::queue &q) { if (!ptr) return; + { + std::lock_guard lock(g_ggml_sycl_dev_alloc_mutex); + auto it = g_ggml_sycl_dev_allocs.find(ptr); + if (it != g_ggml_sycl_dev_allocs.end()) { + const int device = it->second.first; + g_ggml_sycl_dev_alloc_bytes[device] -= it->second.second; + if (g_ggml_sycl_dev_alloc_bytes[device] == 0) { + g_ggml_sycl_dev_free_baseline[device] = 0; + } + g_ggml_sycl_dev_allocs.erase(it); + } + } ggml_sycl_memtrace_del(ptr); #ifdef GGML_SYCL_SUPPORT_LEVEL_ZERO_API if (ggml_sycl_use_level_zero_device_alloc(q)) { diff --git a/ggml/src/ggml-sycl/common.hpp b/ggml/src/ggml-sycl/common.hpp index 668e9befb9c5..f19b0631d32a 100644 --- a/ggml/src/ggml-sycl/common.hpp +++ b/ggml/src/ggml-sycl/common.hpp @@ -47,6 +47,7 @@ namespace syclexp = sycl::ext::oneapi::experimental; #if GGML_SYCL_DNNL #include "dnnl.hpp" #include "dnnl_sycl.hpp" +#include "oneapi/dnnl/dnnl_graph.hpp" #endif #define GGML_COMMON_DECL_SYCL @@ -92,9 +93,17 @@ enum ggml_sycl_xmx_gather_type { GGML_SYCL_XMX_GATHER_Q4_K = 1 << 10, GGML_SYCL_XMX_GATHER_Q5_K = 1 << 11, GGML_SYCL_XMX_GATHER_Q6_K = 1 << 12, + GGML_SYCL_XMX_GATHER_Q2_K = 1 << 13, + GGML_SYCL_XMX_GATHER_Q3_K = 1 << 14, }; static constexpr int GGML_SYCL_XMX_GATHER_TYPES_DEFAULT = ~0; extern int g_ggml_sycl_xmx_gather_types; + +enum ggml_sycl_mmid_sched_bit { + GGML_SYCL_MMID_SCHED_DEVICE = 1 << 0, + GGML_SYCL_MMID_SCHED_UNBOUNDED = 1 << 1, +}; +extern int g_ggml_sycl_mmid_sched; // Which joint_matrix combinations the XMX dequant-GEMM paths may use, one bit each (see fused-gemm.cpp). // GGML_SYCL_DYNAMIC_PRECISION picks the operand type, this mask the combinations of that type. static constexpr int GGML_SYCL_XMX_GATHER_SHAPES_DEFAULT = 0xff; @@ -121,7 +130,7 @@ extern int g_ggml_sycl_dynamic_required_precision; // [TAG_GGML_PREC] src1 precision request of the MUL_MAT/MUL_MAT_ID op dst static inline int32_t ggml_sycl_src1_prec(const ggml_tensor * dst) { return g_ggml_sycl_dynamic_required_precision == GGML_SYCL_DYNAMIC_PRECISION_F16 ? GGML_PREC_UNDEFINED : - dst->op_params[3]; + ggml_prec_get_src(dst, 1); } // [TAG_GGML_PREC] the library GEMM and dmmv may convert src1 of the MUL_MAT/MUL_MAT_ID op dst to f16 @@ -229,9 +238,9 @@ typedef sycl::float2 dfloat2; #define MMVQ_MAX_BATCH_SIZE 8 -#if defined(__INTEL_LLVM_COMPILER) && __has_include() +#if defined(GGML_SYCL_SUPPORT_XMX) && defined(__INTEL_LLVM_COMPILER) && __has_include() # define GGML_SYCL_MMVQ_HAS_XMX -#endif // __INTEL_LLVM_COMPILER +#endif // GGML_SYCL_SUPPORT_XMX && __INTEL_LLVM_COMPILER // default of the most columns the XMX mul_mat_vec_q handles (GGML_SYCL_XMX_MAX_COLS), wider batches use other kernels #define GGML_SYCL_XMX_MAX_COLS_DEFAULT 80 @@ -283,6 +292,10 @@ static size_t g_scratch_offset = 0; (void)bad_arch; // suppress unused function warning } +// thread-local copy of dpct's per-thread selected device, -1 until first read. +// Select devices only through ggml_sycl_set_device so the copy stays in sync. +extern thread_local int g_sycl_current_device_id; + int get_current_device_id(); inline int ggml_sycl_get_device() { @@ -290,16 +303,14 @@ inline int ggml_sycl_get_device() { } inline dpct::err0 ggml_sycl_set_device(const int device) try { - int current_device_id; - SYCL_CHECK(CHECK_TRY_ERROR(current_device_id = get_current_device_id())); - - // GGML_SYCL_DEBUG("ggml_sycl_set_device device_id=%d, - // current_device_id=%d\n", device, current_device); - if (device == current_device_id) { + if (device == g_sycl_current_device_id) { return 0; } - - return CHECK_TRY_ERROR(dpct::select_device(device)); + const dpct::err0 err = CHECK_TRY_ERROR(dpct::select_device(device)); + if (err == dpct::success) { + g_sycl_current_device_id = device; + } + return err; } catch (sycl::exception const& exc) { std::cerr << exc.what() << "Exception caught at file:" << __FILE__ << ", line:" << __LINE__ << std::endl; @@ -428,6 +439,8 @@ struct ggml_tensor_extra_gpu { }; extern int g_ggml_sycl_use_level_zero_api; +extern int g_ggml_sycl_bmg_vram_check; +bool sycl_get_mem_info(int device, size_t * free, size_t * total); void * ggml_sycl_malloc_device(size_t size, sycl::queue &q, ggml_sycl_mem_type type = GGML_SYCL_MEM_DIRECT); void ggml_sycl_free_device(void *ptr, sycl::queue &q); @@ -439,7 +452,7 @@ struct mmid_row_mapping { int32_t i2; }; -struct ggml_sycl_gg_tile { +struct ggml_sycl_grouped_gemm_tile { int32_t expert; int32_t n0; int32_t n1; @@ -472,6 +485,16 @@ struct ggml_sycl_graph { static_assert(std::is_trivial::value, "node_properties must be trivial"); #endif +#if GGML_SYCL_DNNL +struct sdpa_partition { + dnnl::graph::compiled_partition cp; + std::vector ins; + dnnl::graph::logical_tensor out; + size_t id_q = 0, id_k = 0, id_v = 0, id_scale = 0, id_mask = 0; + bool ok = false; +}; +#endif + struct ggml_backend_sycl_context { int device; std::string name; @@ -540,6 +563,8 @@ struct ggml_backend_sycl_context { dnnl::stream stream_dnnl() { return stream_dnnl(device, 0); } + + std::unordered_map sdpa_partitions; #endif // pool @@ -548,7 +573,7 @@ struct ggml_backend_sycl_context { std::unique_ptr host_pools[GGML_SYCL_MAX_DEVICES]; std::vector mmid_row_mapping_host; - std::vector mmid_tile_schedule_host; + std::vector mmid_tile_schedule_host; static std::unique_ptr new_pool_for_device(queue_ptr qptr, int device); diff --git a/ggml/src/ggml-sycl/cpy.hpp b/ggml/src/ggml-sycl/cpy.hpp index 9d63960810bb..d0e20f46ccb8 100644 --- a/ggml/src/ggml-sycl/cpy.hpp +++ b/ggml/src/ggml-sycl/cpy.hpp @@ -25,6 +25,24 @@ __dpct_inline__ int best_index_int8(int n, const int8_t * val, float x) { return x - val[mu - 1] < val[mu] - x ? mu - 1 : mu; } +// exact roundf (half away from zero); x + 0.5f carries for x = 0.5 - 2^-25, and sycl::round is not exact under fast fp +inline int round_half_away_int(float x) { + const float t = sycl::trunc(x); + return (int) (sycl::fabs(x - t) >= 0.5f ? t + sycl::copysign(1.0f, x) : t); +} + +// correctly rounded a / b to match the CPU: fast fp gives an approximate reciprocal (~12% off by an ulp on B580), +// so pick the approximation or a neighbour by smallest exact FMA residual +inline float div_correctly_rounded(float a, float b) { + const float y = a / b; + const float lo = sycl::nextafter(y, -INFINITY); + const float hi = sycl::nextafter(y, INFINITY); + const float e = sycl::fabs(sycl::fma(-b, y, a)); + const float el = sycl::fabs(sycl::fma(-b, lo, a)); + const float eh = sycl::fabs(sycl::fma(-b, hi, a)); + return el < e && el <= eh ? lo : (eh < e ? hi : y); +} + inline void cpy_blck_f32_q8_0(const char * cxi, char * cdsti) { const float * xi = (const float *) cxi; block_q8_0 * dsti = (block_q8_0 *) cdsti; @@ -36,15 +54,16 @@ inline void cpy_blck_f32_q8_0(const char * cxi, char * cdsti) { amax = sycl::fmax(amax, sycl::fabs((float) v)); } - const float d = amax / ((1 << 7) - 1); - const float id = d ? 1.0f / d : 0.0f; + // matches the x86 AVX/AVX2 CPU path (ggml-cpu/arch/x86/quants.c): id = 127/amax, round half to even + const float d = div_correctly_rounded(amax, 127.0f); + const float id = amax != 0.0f ? div_correctly_rounded(127.0f, amax) : 0.0f; dsti->d = d; for (int j = 0; j < QK8_0; ++j) { const float x0 = xi[j] * id; - dsti->qs[j] = sycl::round((float) x0); + dsti->qs[j] = (int) sycl::rint(x0); } } @@ -70,24 +89,6 @@ inline void cpy_blck_f32_q1_0(const char * cxi, char * cdsti) { } } -// exact roundf (half away from zero); x + 0.5f carries for x = 0.5 - 2^-25, and sycl::round is not exact under fast fp -inline int round_half_away_int(float x) { - const float t = sycl::trunc(x); - return (int) (sycl::fabs(x - t) >= 0.5f ? t + sycl::copysign(1.0f, x) : t); -} - -// correctly rounded a / b to match the CPU: fast fp gives an approximate reciprocal (~12% off by an ulp on B580), -// so pick the approximation or a neighbour by smallest exact FMA residual -inline float div_correctly_rounded(float a, float b) { - const float y = a / b; - const float lo = sycl::nextafter(y, -INFINITY); - const float hi = sycl::nextafter(y, INFINITY); - const float e = sycl::fabs(sycl::fma(-b, y, a)); - const float el = sycl::fabs(sycl::fma(-b, lo, a)); - const float eh = sycl::fabs(sycl::fma(-b, hi, a)); - return el < e && el <= eh ? lo : (eh < e ? hi : y); -} - inline void cpy_blck_f32_q2_0(const char * cxi, char * cdsti) { const float * xi = (const float *) cxi; block_q2_0 * dsti = (block_q2_0 *) cdsti; @@ -423,8 +424,8 @@ inline void cpy_blck_f32_q4_1(const char * cxi, char * cdsti) { vmax = sycl::max(v, vmax); } - const float d = (vmax - vmin) / ((1 << 4) - 1); - const float id = d ? 1.0f / d : 0.0f; + const float d = div_correctly_rounded(vmax - vmin, (1 << 4) - 1); + const float id = d ? div_correctly_rounded(1.0f, d) : 0.0f; dsti->dm.x() = d; dsti->dm.y() = vmin; @@ -433,8 +434,9 @@ inline void cpy_blck_f32_q4_1(const char * cxi, char * cdsti) { const float x0 = (xi[0 + j] - vmin) * id; const float x1 = (xi[QK4_1 / 2 + j] - vmin) * id; - const uint8_t xi0 = dpct::min(15, (int8_t) (x0 + 0.5f)); - const uint8_t xi1 = dpct::min(15, (int8_t) (x1 + 0.5f)); + // fmin: no FMA, see cpy_blck_f32_q4_0 + const uint8_t xi0 = dpct::min(15, (int8_t) (sycl::fmin(x0, 15.5f) + 0.5f)); + const uint8_t xi1 = dpct::min(15, (int8_t) (sycl::fmin(x1, 15.5f) + 0.5f)); dsti->qs[j] = xi0; dsti->qs[j] |= xi1 << 4; @@ -490,8 +492,8 @@ inline void cpy_blck_f32_q5_1(const char * cxi, char * cdsti) { max = v > max ? v : max; } - const float d = (max - min) / 31; - const float id = d ? 1.0f / d : 0.0f; + const float d = div_correctly_rounded(max - min, 31); + const float id = d ? div_correctly_rounded(1.0f, d) : 0.0f; dsti->dm.x() = d; dsti->dm.y() = min; @@ -501,8 +503,9 @@ inline void cpy_blck_f32_q5_1(const char * cxi, char * cdsti) { const float x0 = (xi[0 + j] - min) * id; const float x1 = (xi[QK5_1 / 2 + j] - min) * id; - const uint8_t xi0 = (uint8_t) (x0 + 0.5f); - const uint8_t xi1 = (uint8_t) (x1 + 0.5f); + // fmin: no FMA, see cpy_blck_f32_q4_0 + const uint8_t xi0 = (uint8_t) (sycl::fmin(x0, 31.5f) + 0.5f); + const uint8_t xi1 = (uint8_t) (sycl::fmin(x1, 31.5f) + 0.5f); dsti->qs[j] = (xi0 & 0xf) | ((xi1 & 0xf) << 4); qh |= ((xi0 & 0x10u) >> 4) << (j + 0); diff --git a/ggml/src/ggml-sycl/element_wise.cpp b/ggml/src/ggml-sycl/element_wise.cpp index c7fe70ac33db..247590f7c656 100644 --- a/ggml/src/ggml-sycl/element_wise.cpp +++ b/ggml/src/ggml-sycl/element_wise.cpp @@ -361,13 +361,6 @@ static void unary_op_sqr_kernel(const T * x, T * dst, const int k, const sycl::n } } -template -static void unary_op_clamp_kernel(const T * x, T * dst, const int k, const sycl::nd_item<1> &item_ct1, float min_val, float max_val) { - SYCL_GLOBAL_ID_LOOP(k, item_ct1) { - dst[i] = op_clamp(x[i], min_val, max_val); - } -} - template static void unary_op_ceil_kernel(const T * x, T * dst, const int k, const sycl::nd_item<1> &item_ct1) { SYCL_GLOBAL_ID_LOOP(k, item_ct1) { @@ -375,14 +368,6 @@ static void unary_op_ceil_kernel(const T * x, T * dst, const int k, const sycl:: } } -template -static void clamp(const T * x, T * dst, const float min, const float max, const int k, - const sycl::nd_item<1> &item_ct1) { - SYCL_GLOBAL_ID_LOOP(k, item_ct1) { - dst[i] = x[i] < static_cast(min) ? static_cast(min) : (x[i] > static_cast(max) ? static_cast(max) : x[i]); - } -} - template static void unary_gated_op_flat_kernel(const T * x, const T * g, T * dst, const uint64_t k, const sycl::nd_item<1> & item_ct1, F func) { SYCL_GLOBAL_ID_LOOP(k, item_ct1) { @@ -929,20 +914,11 @@ static inline void ggml_sycl_op_sqr(ggml_backend_sycl_context & ctx, ggml_tensor } static inline void ggml_sycl_op_clamp(ggml_backend_sycl_context & ctx, ggml_tensor * dst) { - float min_val; - float max_val; - memcpy(&min_val, dst->op_params, sizeof(float)); - memcpy(&max_val, (float *) dst->op_params + 1, sizeof(float)); - ggml_sycl_detail::dispatch_ggml_sycl_op_unary(ctx, dst, - [](const auto* src, auto* dst_ptr, int k_elements, queue_ptr stream, float min_arg, float max_arg) { - const int num_blocks = ceil_div(k_elements, SYCL_CLAMP_BLOCK_SIZE); - stream->parallel_for( - sycl::nd_range<1>(sycl::range<1>(num_blocks) * sycl::range<1>(SYCL_CLAMP_BLOCK_SIZE), - sycl::range<1>(SYCL_CLAMP_BLOCK_SIZE)), - [=](sycl::nd_item<1> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { - clamp(src, dst_ptr, min_arg, max_arg, k_elements, item_ct1); - }); - }, min_val, max_val); + const float min_val = ggml_get_op_params_f32(dst, 0); + const float max_val = ggml_get_op_params_f32(dst, 1); + ggml_sycl_detail::ggml_sycl_op_unary(ctx, dst, [min_val, max_val](auto x) { + return op_clamp(x, min_val, max_val); + }); } static inline void ggml_sycl_op_xielu(ggml_backend_sycl_context & ctx, ggml_tensor * dst) { @@ -1183,9 +1159,7 @@ static void ggml_sycl_op_swiglu_oai(ggml_backend_sycl_context & ctx, ggml_tensor GGML_ASSERT(ggml_is_contiguous_1(src0)); GGML_ASSERT(src0->nb[0] == ggml_element_size(src0)); GGML_ASSERT(ggml_is_contiguous(dst)); - - GGML_ASSERT(src0->type == GGML_TYPE_F32); - GGML_ASSERT( dst->type == GGML_TYPE_F32); + GGML_ASSERT(src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16); GGML_ASSERT(src0->type == dst->type); GGML_ASSERT(dst->ne[0] == nc); GGML_ASSERT(ggml_nrows(dst) == ggml_nrows(src0)); @@ -1197,20 +1171,33 @@ static void ggml_sycl_op_swiglu_oai(ggml_backend_sycl_context & ctx, ggml_tensor GGML_ASSERT(src0->type == src1->type); } - //const int32_t swapped = ((const int32_t *) dst->op_params)[1]; const int32_t swapped = ggml_get_op_params_i32(dst, 1); - const float alpha = ggml_get_op_params_f32(dst, 2); - const float limit = ggml_get_op_params_f32(dst, 3); + const float alpha = ggml_get_op_params_f32(dst, 2); + const float limit = ggml_get_op_params_f32(dst, 3); - float * src0_p = (float *) src0_d; - float * src1_p = (float *) src1_d; + if (src0->type == GGML_TYPE_F16) { + sycl::half * src0_p = (sycl::half *) src0_d; + sycl::half * src1_p = (sycl::half *) src1_d; - if (!src1) { - src0_p += swapped ? nc : 0; - src1_p += swapped ? 0 : nc; - } + if (!src1) { + src0_p += swapped ? nc : 0; + src1_p += swapped ? 0 : nc; + } - swiglu_oai_sycl(src0_p, src1_p, (float *)dst_d, ggml_nelements(dst), nc, src0_o / sizeof(float), src1_o / sizeof(float), alpha, limit, stream); + swiglu_oai_sycl(src0_p, src1_p, (sycl::half *) dst_d, ggml_nelements(dst), nc, src0_o / sizeof(sycl::half), + src1_o / sizeof(sycl::half), alpha, limit, stream); + } else { + float * src0_p = (float *) src0_d; + float * src1_p = (float *) src1_d; + + if (!src1) { + src0_p += swapped ? nc : 0; + src1_p += swapped ? 0 : nc; + } + + swiglu_oai_sycl(src0_p, src1_p, (float *) dst_d, ggml_nelements(dst), nc, src0_o / sizeof(float), + src1_o / sizeof(float), alpha, limit, stream); + } } template diff --git a/ggml/src/ggml-sycl/esimd.hpp b/ggml/src/ggml-sycl/esimd.hpp index 85cc93e21ef1..568173e7a9df 100644 --- a/ggml/src/ggml-sycl/esimd.hpp +++ b/ggml/src/ggml-sycl/esimd.hpp @@ -2,11 +2,14 @@ #define GGML_SYCL_ESIMD_HPP #include -#include #include #include "common.hpp" +#ifdef GGML_SYCL_MMVQ_HAS_XMX +#include +#endif + namespace ggml_sycl_esimd { constexpr int GGML_SYCL_DMMV_ESIMD_WG_SIZE = 4; @@ -586,6 +589,7 @@ template <> struct esimd_reorder_q_traits { } }; + // // Multi-column mul_mat_vec for reordered K-quant weights on the XMX engines (int8 DPAS). // A thread owns 16 weight rows (B operand, one dword is 4 weights of a row), the q8_1 activation @@ -1024,6 +1028,7 @@ template constexpr int xmx_grf() { return FUSED || NC >= T::nc_wide ? 256 : 128; } +#ifdef GGML_SYCL_MMVQ_HAS_XMX // One DPAS of a block of RC columns: the int32 products of the weights B and the activations A, as float and scaled // per weight row. Every quant type sums one or two of these per group of 32. template @@ -1394,6 +1399,8 @@ static void xmx_mul_mat_ncols(const void * vx, } } +#endif // GGML_SYCL_MMVQ_HAS_XMX + } // namespace ggml_sycl_esimd #endif // GGML_SYCL_ESIMD_HPP diff --git a/ggml/src/ggml-sycl/fattn-onednn.cpp b/ggml/src/ggml-sycl/fattn-onednn.cpp index 4691c8600f18..66ff32f15eaf 100644 --- a/ggml/src/ggml-sycl/fattn-onednn.cpp +++ b/ggml/src/ggml-sycl/fattn-onednn.cpp @@ -1,4 +1,5 @@ #include +#include #include #include #include @@ -25,6 +26,20 @@ bool ggml_sycl_fattn_onednn_binds_kv(const ggml_tensor * K, const ggml_tensor * return bindable(K) && bindable(V); } +#if GGML_SYCL_DNNL +// per device: set once a oneDNN SDPA execute throws, so dispatch stops picking oneDNN there +static std::atomic g_ggml_sycl_fa_onednn_failed[GGML_SYCL_MAX_DEVICES] = {}; +#endif + +bool ggml_sycl_flash_attn_ext_onednn_failed(int device) { +#if GGML_SYCL_DNNL + return g_ggml_sycl_fa_onednn_failed[device].load(std::memory_order_relaxed); +#else + GGML_UNUSED(device); + return false; +#endif +} + bool ggml_sycl_flash_attn_ext_onednn_supported(const ggml_tensor * dst, bool use_shape_limit) { #if !GGML_SYCL_DNNL GGML_UNUSED(dst); @@ -69,17 +84,6 @@ bool ggml_sycl_flash_attn_ext_onednn_supported(const ggml_tensor * dst, bool use } } } - // This is the improved SPDA gate. Rather than gating Alchemist GPUs from all SPDA features, we instead target only the failing shapes. - // If the GPU being assessed isn't in the grouping below, it has full access to all SPDA shapes. Otherwise, if it's an Alchemist GPU, we block only the shapes with head sizes that fail. - // It is much easier to compare the device to a small list of failing cases than to define all the passing ones. - const gpu_arch arch = ggml_sycl_info().devices[ggml_sycl_get_device()].hw_info.arch; - bool support_spda = !(arch == gpu_arch::intel_gpu_dg2_g10 || - arch == gpu_arch::intel_gpu_dg2_g11 || - arch == gpu_arch::intel_gpu_dg2_g12); - - if (!support_spda && K->ne[0] == 64) { - return false; - } // Optional KV-length ceiling (GGML_SYCL_FA_ONEDNN_MAX_KV, 0 = unlimited). Escape hatch: // very long sequences make the fused SDPA slow enough to risk the xe driver watchdog on // some stacks; past the cap we fall back to the native FA kernel instead. @@ -157,14 +161,6 @@ static void permute_sdpa_out_sycl(const sycl::half * out, float * dst, }); } -struct sdpa_partition { - compiled_partition cp; - std::vector ins; - logical_tensor out; - size_t id_q = 0, id_k = 0, id_v = 0, id_scale = 0, id_mask = 0; - bool ok = false; -}; - // Build + compile the contiguous-input GQA SDPA graph (MatMul->Divide->Add->SoftMax->MatMul), f32 out. // Mirrors the hardware-verified scratch/onednn_sdpa_probe.cpp build_gqa (partitions=1, sdp_primitive_kernel_t). static sdpa_partition build_sdpa(const engine & eng, int H, int Hkv, int q, int seq, int d, @@ -372,17 +368,16 @@ void ggml_sycl_flash_attn_ext_onednn(ggml_backend_sycl_context & ctx, ggml_tenso outf_ptr = outf_pool->get(); } - // compile once per (device, shape, KV strides), reuse across layers/calls. Stride 2 always - // repeats stride 1 and stride 4 is always 1, so the key covers every entry that can differ. - static std::unordered_map cache; + // compile once per (shape, KV strides) in this device's context, reuse across layers/calls. Stride 2 + // always repeats stride 1 and stride 4 is always 1, so the key covers every entry that can differ. char keyb[256]; - snprintf(keyb, sizeof(keyb), "%d:%lld:%lld:%lld:%lld:%lld:%lld:%lld:%lld:%lld:%lld:%lld", ggml_sycl_get_device(), + snprintf(keyb, sizeof(keyb), "%lld:%lld:%lld:%lld:%lld:%lld:%lld:%lld:%lld:%lld:%lld", (long long) H, (long long) Hkv, (long long) q, (long long) seq, (long long) d, (long long) k_str[0], (long long) k_str[1], (long long) k_str[3], (long long) v_str[0], (long long) v_str[1], (long long) v_str[3]); - auto it = cache.find(keyb); - if (it == cache.end()) { - it = cache.emplace(keyb, build_sdpa(eng, (int) H, (int) Hkv, (int) q, (int) seq, (int) d, k_str, v_str)).first; + auto it = ctx.sdpa_partitions.find(keyb); + if (it == ctx.sdpa_partitions.end()) { + it = ctx.sdpa_partitions.emplace(keyb, build_sdpa(eng, (int) H, (int) Hkv, (int) q, (int) seq, (int) d, k_str, v_str)).first; } sdpa_partition & E = it->second; if (!E.ok) { @@ -418,8 +413,10 @@ void ggml_sycl_flash_attn_ext_onednn(ggml_backend_sycl_context & ctx, ggml_tenso } } catch (const std::exception & e) { - // any oneDNN/SYCL failure is non-fatal: fall back to the existing kernel (strictly additive). - GGML_LOG_WARN("%s: oneDNN SDPA failed (%s); falling back to TILE kernel\n", __func__, e.what()); + // any oneDNN/SYCL failure is non-fatal: warn once, stop picking oneDNN on this device, run TILE instead + if (!g_ggml_sycl_fa_onednn_failed[ctx.device].exchange(true)) { + GGML_LOG_WARN("%s: oneDNN SDPA failed (%s); falling back to TILE kernel\n", __func__, e.what()); + } ggml_sycl_flash_attn_ext_tile(ctx, dst); } diff --git a/ggml/src/ggml-sycl/fattn-onednn.hpp b/ggml/src/ggml-sycl/fattn-onednn.hpp index 9669d1bd27a6..4756b240a76a 100644 --- a/ggml/src/ggml-sycl/fattn-onednn.hpp +++ b/ggml/src/ggml-sycl/fattn-onednn.hpp @@ -7,6 +7,10 @@ // (f16 KV, no softcap/ALiBi, single stream, tuned head_dim, prefill-sized q.) bool ggml_sycl_flash_attn_ext_onednn_supported(const ggml_tensor * dst, bool use_shape_limit = true); +// True once a oneDNN SDPA execute has failed on this device. Not part of _supported(): that also sizes the +// FA scratch buffer, and a size that depends on runtime state can come out smaller than a later call needs. +bool ggml_sycl_flash_attn_ext_onednn_failed(int device); + // True when the oneDNN path binds an F16 KV cache in place instead of staging a dense copy of // it. Depends only on the types and strides of K and V, so the answer holds for every call. bool ggml_sycl_fattn_onednn_binds_kv(const ggml_tensor * K, const ggml_tensor * V); diff --git a/ggml/src/ggml-sycl/fattn.cpp b/ggml/src/ggml-sycl/fattn.cpp index 6f5f8a832951..525e6fd8c095 100644 --- a/ggml/src/ggml-sycl/fattn.cpp +++ b/ggml/src/ggml-sycl/fattn.cpp @@ -134,7 +134,7 @@ static best_fattn_kernel ggml_sycl_get_best_fattn_kernel(const int device, const // XMX-accelerated path: oneDNN SDPA (native F16 and dequant+non-F16). // ONEDNN requires min 32 query tokens — short-circuit decode to avoid // calling _supported() on every decode FA call. - if (Q->ne[1] >= 32 + if (Q->ne[1] >= 32 && !ggml_sycl_flash_attn_ext_onednn_failed(device) && ggml_sycl_flash_attn_ext_onednn_supported(dst)) { return BEST_FATTN_KERNEL_ONEDNN; } @@ -259,7 +259,7 @@ static best_fattn_kernel ggml_sycl_get_best_fattn_kernel(const int device, const // Fused-XMX path: oneDNN Graph SDPA (flash attention). Strictly // additive -- taken only when statically supported, otherwise falls through to VEC/TILE below. - if (ggml_sycl_flash_attn_ext_onednn_supported(dst)) { + if (!ggml_sycl_flash_attn_ext_onednn_failed(device) && ggml_sycl_flash_attn_ext_onednn_supported(dst)) { return BEST_FATTN_KERNEL_ONEDNN; } diff --git a/ggml/src/ggml-sycl/fused-gemm.cpp b/ggml/src/ggml-sycl/fused-gemm.cpp index 3024f139d5dc..803a994c04bb 100644 --- a/ggml/src/ggml-sycl/fused-gemm.cpp +++ b/ggml/src/ggml-sycl/fused-gemm.cpp @@ -411,6 +411,8 @@ template <> struct fg_block_traits { static constexpr int qk = QK template <> struct fg_block_traits { static constexpr int qk = QK_K; }; template <> struct fg_block_traits { static constexpr int qk = QK_K; }; template <> struct fg_block_traits { static constexpr int qk = QK8_0; }; +template <> struct fg_block_traits { static constexpr int qk = QK_K; }; +template <> struct fg_block_traits { static constexpr int qk = QK_K; }; template <> struct fg_block_traits { static constexpr int qk = QK_K; }; template <> struct fg_block_traits { static constexpr int qk = QK_K; }; template <> struct fg_block_traits { static constexpr int qk = QK_K; }; @@ -595,6 +597,49 @@ static __dpct_inline__ void fg_scale_min_k4(const int j, const uint8_t * __restr } } +template +static __dpct_inline__ void fg_decode_q2_K(const uint8_t * __restrict__ qs, const uint8_t * __restrict__ scales, + const float d, const float dmin, const int ib, typename E::pair * a) { + const uint8_t * q = qs + 32 * (ib / 4); + const int shift = 2 * (ib % 4); +#pragma unroll + for (int j = 0; j < 16; ++j) { + const uint8_t sc = scales[2 * ib + j / 8]; + const float dl = d * (sc & 0xF); + const float ml = dmin * (sc >> 4); + a[j] = E::make(dl * ((q[2 * j] >> shift) & 3) - ml, + dl * ((q[2 * j + 1] >> shift) & 3) - ml); + } +} + +// scales of the two 16-value halves of sub-block ib +static __dpct_inline__ void fg_q3_K_dl(const uint8_t * __restrict__ scales, const float d, const int ib, float * dl) { +#pragma unroll + for (int i = 0; i < 2; ++i) { + const int s = 2 * ib + i; + const int sc = ((scales[s % 8] >> (4 * (s / 8))) & 0xF) | (((scales[8 + s % 4] >> (2 * (s / 4))) & 3) << 4); + dl[i] = d * (sc - 32); + } +} + +template +static __dpct_inline__ void fg_decode_q3_K(const uint8_t * __restrict__ qs, const uint8_t * __restrict__ hmask, + const uint8_t * __restrict__ scales, const float d, const int ib, + typename E::pair * a) { + // block_q3_K is 110 bytes, so qs and hmask are only 2-byte aligned + const uint16_t * q = (const uint16_t *) (qs + 32 * (ib / 4)); + const uint16_t * h = (const uint16_t *) hmask; + const int shift = 2 * (ib % 4); + float dl[2]; + fg_q3_K_dl(scales, d, ib, dl); +#pragma unroll + for (int j = 0; j < 16; ++j) { + // per byte: 2 low bits, plus 4 if the hmask bit is set, minus 4 + const uint32_t v = ((q[j] >> shift) & 0x0303u) | (((h[j] >> ib) & 0x0101u) << 2); + a[j] = E::make(dl[j / 8] * ((int) (v & 0xFF) - 4), dl[j / 8] * ((int) (v >> 8) - 4)); + } +} + // q4_K sub-block ib (0..7) uses scale/min pair ib and the low (even ib) or high (odd ib) nibbles of // qs[32 * (ib / 2) ...], as in dequantize_row_q4_K template @@ -658,6 +703,18 @@ static __dpct_inline__ void fg_stage_a(const block_q8_0 * __restrict__ xrow, con fg_decode_q8_0(xrow[kb].qs, (float) xrow[kb].d, a); } +template +static __dpct_inline__ void fg_stage_a(const block_q2_K * __restrict__ xrow, const int kb, typename E::pair * a) { + FG_SUPERBLOCK(block_q2_K); + fg_decode_q2_K(blk->qs, blk->scales, (float) blk->dm[0], (float) blk->dm[1], ib, a); +} + +template +static __dpct_inline__ void fg_stage_a(const block_q3_K * __restrict__ xrow, const int kb, typename E::pair * a) { + FG_SUPERBLOCK(block_q3_K); + fg_decode_q3_K(blk->qs, blk->hmask, blk->scales, (float) blk->d, ib, a); +} + template static __dpct_inline__ void fg_stage_a(const block_q4_K * __restrict__ xrow, const int kb, typename E::pair * a) { FG_SUPERBLOCK(block_q4_K); @@ -693,6 +750,50 @@ template <> struct fg_soa { } }; +template <> struct fg_soa { + static constexpr bool supported = true; + + // [qs][scales][dm] + template + static __dpct_inline__ void stage(const uint8_t * x, const size_t nblocks, const size_t ib, const int kb, + typename E::pair * a) { + const uint8_t * scales = x + nblocks * (QK_K / 4); + const sycl::half2 dm = ((const sycl::half2 *) (scales + nblocks * (QK_K / 16)))[ib]; + fg_decode_q2_K(x + ib * (QK_K / 4), scales + ib * (QK_K / 16), (float) dm[0], (float) dm[1], + kb % (QK_K / 32), a); + } +}; + +template <> struct fg_soa { + static constexpr bool supported = true; + + // [qs][hmask][scales][d] + template + static __dpct_inline__ void stage(const uint8_t * x, const size_t nblocks, const size_t ib, const int kb, + typename E::pair * a) { + const uint8_t * hmask = x + nblocks * (QK_K / 4); + const uint8_t * scales = hmask + nblocks * (QK_K / 8); + const float d = (float) ((const sycl::half *) (scales + nblocks * 12))[ib]; + const int sub = kb % (QK_K / 32); + // 16-byte aligned, see opt_for_reorder_id + using v4 = sycl::vec; + const v4 * q = (const v4 *) (x + ib * (QK_K / 4) + 32 * (sub / 4)); + const v4 * h = (const v4 *) (hmask + ib * (QK_K / 8)); + const v4 qv[2] = { q[0], q[1] }; + const v4 hv[2] = { h[0], h[1] }; + const int shift = 2 * (sub % 4); + float dl[2]; + fg_q3_K_dl(scales + ib * 12, d, sub, dl); +#pragma unroll + for (int w = 0; w < 8; ++w) { + const uint32_t v = ((qv[w / 4][w % 4] >> shift) & 0x03030303u) | (((hv[w / 4][w % 4] >> sub) & 0x01010101u) << 2); + const float f = dl[w / 4]; + a[2 * w] = E::make(f * ((int) (v & 0xFF) - 4), f * ((int) ((v >> 8) & 0xFF) - 4)); + a[2 * w + 1] = E::make(f * ((int) ((v >> 16) & 0xFF) - 4), f * ((int) (v >> 24) - 4)); + } + } +}; + template <> struct fg_soa { static constexpr bool supported = true; // [qs][scales][dm] @@ -745,7 +846,7 @@ static void fused_dequant_gemm_tile( const int b0, const int n0, const int n1, sycl::local_accessor tile_a, sycl::local_accessor tile_c, - const sycl::nd_item<2> & item) { + const sycl::nd_item<2> & item, ggml_sycl_grouped_gemm_rows dst_rows = {}) { if constexpr (fg_built()) { using EA = typename S::EA; using TA = typename S::ta; @@ -854,7 +955,13 @@ static void fused_dequant_gemm_tile( for (int s = 0; s < FG_KSPLIT; ++s) { sum += tile_c[s * S::SG_ROWS * S::BN + r * S::BN + c]; } - dst[(size_t) n * ldd + m] = sum; + if (dst_rows.map) { + const mmid_row_mapping row = dst_rows.map[n]; + float * out = (float *) (dst_rows.base + (row.i1 % dst_rows.ne1) * dst_rows.nb1 + row.i2 * dst_rows.nb2); + out[m] = sum; + } else { + dst[(size_t) n * ldd + m] = sum; + } } } } @@ -880,20 +987,24 @@ static void fused_dequant_gemm_launch(const void * src0, const typename S::tsb * // grouped: work-group (t, mt) is tile t of the schedule; its B columns sit at t * BN template static void grouped_dequant_gemm_launch(const char * src0_dd, const size_t expert_stride, - const ggml_sycl_gg_tile * tiles_ptr, const typename S::tsb * packed, float * dst, - const int M, const int Npad, const int K, const int64_t n_tiles, - const int64_t groups_m, dpct::queue_ptr stream) { + const ggml_sycl_grouped_gemm_tile * tiles_ptr, const typename S::tsb * packed, + float * dst, const int M, const int Npad, const int K, const int64_t n_tiles, + const int64_t groups_m, dpct::queue_ptr stream, + ggml_sycl_grouped_gemm_rows dst_rows = {}) { stream->submit([&](sycl::handler & cgh) { sycl::local_accessor tile_a(FG_KSPLIT * S::SG_ROWS * FG_BK, cgh); sycl::local_accessor tile_c(FG_KSPLIT * S::SG_ROWS * S::BN, cgh); cgh.parallel_for( sycl::nd_range<2>(sycl::range<2>(n_tiles, groups_m * S::WG_SIZE), sycl::range<2>(1, S::WG_SIZE)), [=](sycl::nd_item<2> item) [[sycl::reqd_sub_group_size(S::SG)]] { - const int t = item.get_group(0); - const ggml_sycl_gg_tile tile = tiles_ptr[t]; + const int t = item.get_group(0); + const ggml_sycl_grouped_gemm_tile tile = tiles_ptr[t]; + if (tile.n0 == tile.n1) { + return; + } const block_q_t * x = (const block_q_t *) (src0_dd + (size_t) tile.expert * expert_stride); fused_dequant_gemm_tile(x, packed, dst, M, Npad, K, M, t * S::BN, tile.n0, - tile.n1, tile_a, tile_c, item); + tile.n1, tile_a, tile_c, item, dst_rows); }); }); } @@ -901,17 +1012,17 @@ static void grouped_dequant_gemm_launch(const char * src0_dd, const size_t exper // src1 f32 rows -> packed [K/V][n_tiles*BN][V], tile t holds its rows [n0, n1) at columns t*BN.., // zero past n1. The column runs fastest so a sub-group writes one contiguous run. template -static void grouped_gemm_pack_b(const float * y, typename S::tsb * packed, const ggml_sycl_gg_tile * tiles, int Npad, - int K, dpct::queue_ptr stream) { +static void grouped_gemm_pack_b(const float * y, typename S::tsb * packed, const ggml_sycl_grouped_gemm_tile * tiles, + int Npad, int K, dpct::queue_ptr stream) { using E = typename S::EB; constexpr int V = S::VNNI; const int kqs = K / V; stream->parallel_for(sycl::range<1>((size_t) Npad * kqs), [=](sycl::id<1> id) { - const size_t idx = id[0]; - const int kq = idx / Npad; - const int n = idx - (size_t) kq * Npad; - const ggml_sycl_gg_tile tile = tiles[n / S::BN]; - const int row = tile.n0 + n % S::BN; + const size_t idx = id[0]; + const int kq = idx / Npad; + const int n = idx - (size_t) kq * Npad; + const ggml_sycl_grouped_gemm_tile tile = tiles[n / S::BN]; + const int row = tile.n0 + n % S::BN; // one guarded load run per work-item, as a per-element select costs ~1.5% prefill typename S::tsb vals[V] = {}; if (row < tile.n1) { @@ -932,7 +1043,8 @@ static void grouped_gemm_pack_b(const float * y, typename S::tsb * packed, const // q8_0 and the k-quants take only the grouped path. The plain kernel decodes A again for every BN columns // of a dense batch, and for these formats that costs more than the one dequantization of the library GEMM. template static constexpr bool fg_plain_ok() { - return !std::is_same_v && !std::is_same_v && !std::is_same_v && + return !std::is_same_v && !std::is_same_v && !std::is_same_v && + !std::is_same_v && !std::is_same_v && !std::is_same_v; } @@ -966,6 +1078,8 @@ template static bool fg_visit_type(ggml_type type, bool reordered, case GGML_TYPE_IQ1_S: return fg_visit_layout(reordered, f); case GGML_TYPE_IQ1_M: return fg_visit_layout(reordered, f); case GGML_TYPE_Q8_0: return fg_visit_layout(reordered, f); + case GGML_TYPE_Q2_K: return fg_visit_layout(reordered, f); + case GGML_TYPE_Q3_K: return fg_visit_layout(reordered, f); case GGML_TYPE_Q4_K: return fg_visit_layout(reordered, f); case GGML_TYPE_Q5_K: return fg_visit_layout(reordered, f); case GGML_TYPE_Q6_K: return fg_visit_layout(reordered, f); @@ -1037,7 +1151,7 @@ bool ggml_sycl_fused_dequant_gemm(ggml_type src0_type, bool src0_reordered, cons template static bool fg_grouped_run(ggml_type src0_type, bool reordered, const void * src0_base, size_t expert_stride, const float * src1, float * dst, const int64_t * expert_row_offsets, int64_t n_as, int64_t M, - int64_t K, std::vector & tiles, ggml_sycl_pool & pool, + int64_t K, std::vector & tiles, ggml_sycl_pool & pool, dpct::queue_ptr stream) { // the host knows every slice, so it lays out the work-groups: no search on the device tiles.clear(); @@ -1053,15 +1167,16 @@ static bool fg_grouped_run(ggml_type src0_type, bool reordered, const void * src const int64_t max_tiles = grouped_gemm_max_tiles(expert_row_offsets[n_as], n_as, S::BN); GGML_ASSERT(n_tiles <= max_tiles); - ggml_sycl_pool_alloc tiles_dev(pool, max_tiles); - SYCL_CHECK(CHECK_TRY_ERROR(stream->memcpy(tiles_dev.get(), tiles.data(), n_tiles * sizeof(ggml_sycl_gg_tile)))); + ggml_sycl_pool_alloc tiles_dev(pool, max_tiles); + SYCL_CHECK(CHECK_TRY_ERROR( + stream->memcpy(tiles_dev.get(), tiles.data(), n_tiles * sizeof(ggml_sycl_grouped_gemm_tile)))); ggml_sycl_pool_alloc packed_b(pool, (size_t) K * max_tiles * S::BN); grouped_gemm_pack_b(src1, packed_b.get(), tiles_dev.get(), Npad, (int) K, stream); - const typename S::tsb * packed = packed_b.get(); - const ggml_sycl_gg_tile * tiles_ptr = tiles_dev.get(); - const char * src0_dd = (const char *) src0_base; + const typename S::tsb * packed = packed_b.get(); + const ggml_sycl_grouped_gemm_tile * tiles_ptr = tiles_dev.get(); + const char * src0_dd = (const char *) src0_base; return fg_visit_type(src0_type, reordered, [&](auto tag) { using T = decltype(tag); grouped_dequant_gemm_launch(src0_dd, expert_stride, tiles_ptr, packed, dst, @@ -1073,7 +1188,7 @@ static bool fg_grouped_run(ggml_type src0_type, bool reordered, const void * src bool ggml_sycl_grouped_dequant_gemm(ggml_type src0_type, bool reordered, const void * src0_base, size_t expert_stride, const float * src1, int32_t src1_prec, float * dst, const int64_t * expert_row_offsets, int64_t n_as, int64_t M, int64_t K, - int64_t total_rows, std::vector & tiles, + int64_t total_rows, std::vector & tiles, ggml_sycl_pool & pool, dpct::queue_ptr stream) { int64_t n_active = 0; for (int64_t e = 0; e < n_as; ++e) { @@ -1097,3 +1212,227 @@ bool ggml_sycl_grouped_dequant_gemm(ggml_type src0_type, bool reordered, const v }); return launched; } + +// One work-group sorts routes by expert and builds the tile table. +static constexpr int MMID_SCHED_WG = 256; + +using mmid_sched_atomic = sycl::atomic_ref; + +bool ggml_sycl_build_mmid_schedule(const int32_t * ids_dev, size_t ids_token_stride, int64_t n_as, int64_t n_ids, + int64_t n_tokens, int64_t n_tiles_max, mmid_row_mapping * row_mapping, + ggml_sycl_grouped_gemm_tile * tiles, uint32_t * expert_offsets, + dpct::queue_ptr stream) { + if (n_as <= 0 || n_as > GGML_SYCL_MMID_SCHED_MAX_EXPERTS || n_ids <= 0 || n_tokens <= 0 || n_ids > INT32_MAX / n_tokens) { + return false; + } + const int64_t n_routes = n_tokens * n_ids; + if (n_tiles_max < grouped_gemm_max_tiles(n_routes, n_as, GGML_SYCL_FG_BN)) { + return false; + } + + stream->submit([&](sycl::handler & cgh) { + // l_cnt is reused: route counts, then the write cursors, then the tile count per expert + sycl::local_accessor l_cnt(n_as + 1, cgh); + sycl::local_accessor l_off(n_as + 1, cgh); + sycl::local_accessor l_toff(n_as + 1, cgh); + cgh.parallel_for( + sycl::nd_range<1>(MMID_SCHED_WG, MMID_SCHED_WG), + [=](sycl::nd_item<1> item) { + const auto grp = item.get_group(); + const int64_t tid = item.get_local_id(0); + uint32_t * cnt = l_cnt.get_multi_ptr().get(); + uint32_t * off = l_off.get_multi_ptr().get(); + uint32_t * toff = l_toff.get_multi_ptr().get(); + + for (int64_t e = tid; e < n_as; e += MMID_SCHED_WG) { + cnt[e] = 0; + } + // a dropped route must leave a row that names (0, 0) rather than stale memory + for (int64_t r = tid; r < n_routes; r += MMID_SCHED_WG) { + row_mapping[r] = { 0, 0 }; + } + sycl::group_barrier(grp); + + for (int64_t r = tid; r < n_routes; r += MMID_SCHED_WG) { + const int64_t token = r / n_ids; + const int64_t slot = r - token * n_ids; + const int32_t * id_row = (const int32_t *) ((const char *) ids_dev + token * ids_token_stride); + const int32_t e = id_row[slot]; + if (e >= 0 && e < n_as) { + mmid_sched_atomic(cnt[e]).fetch_add(1u); + } + } + sycl::group_barrier(grp); + + sycl::joint_exclusive_scan(grp, cnt, cnt + n_as, off, 0u, sycl::plus()); + sycl::group_barrier(grp); + if (tid == 0) { + off[n_as] = off[n_as - 1] + cnt[n_as - 1]; + } + sycl::group_barrier(grp); + + for (int64_t e = tid; e < n_as; e += MMID_SCHED_WG) { + cnt[e] = off[e]; + } + sycl::group_barrier(grp); + + // Atomic arrival order can differ from the host sort; each expert gets the same rows. + for (int64_t r = tid; r < n_routes; r += MMID_SCHED_WG) { + const int64_t token = r / n_ids; + const int64_t slot = r - token * n_ids; + const int32_t * id_row = (const int32_t *) ((const char *) ids_dev + token * ids_token_stride); + const int32_t e = id_row[slot]; + if (e >= 0 && e < n_as) { + const uint32_t pos = mmid_sched_atomic(cnt[e]).fetch_add(1u); + row_mapping[pos] = { (int32_t) slot, (int32_t) token }; + } + } + sycl::group_barrier(grp); + + for (int64_t e = tid; e < n_as; e += MMID_SCHED_WG) { + cnt[e] = (off[e + 1] - off[e] + GGML_SYCL_FG_BN - 1) / GGML_SYCL_FG_BN; + } + sycl::group_barrier(grp); + + sycl::joint_exclusive_scan(grp, cnt, cnt + n_as, toff, 0u, sycl::plus()); + sycl::group_barrier(grp); + if (tid == 0) { + toff[n_as] = toff[n_as - 1] + cnt[n_as - 1]; + } + sycl::group_barrier(grp); + + const uint32_t n_tiles = toff[n_as]; + for (int64_t e = tid; e < n_as; e += MMID_SCHED_WG) { + const uint32_t base = off[e]; + const uint32_t end = off[e + 1]; + for (uint32_t j = 0; j < cnt[e]; ++j) { + const uint32_t n0 = base + j * GGML_SYCL_FG_BN; + const uint32_t n1 = n0 + GGML_SYCL_FG_BN < end ? n0 + GGML_SYCL_FG_BN : end; + tiles[toff[e] + j] = { (int32_t) e, (int32_t) n0, (int32_t) n1 }; + } + } + // the launch is bounded, so the unused tail must be tiles the GEMM skips + for (int64_t t = n_tiles + tid; t < n_tiles_max; t += MMID_SCHED_WG) { + tiles[t] = { 0, 0, 0 }; + } + for (int64_t e = tid; e <= n_as; e += MMID_SCHED_WG) { + expert_offsets[e] = off[e]; + } + }); + }); + return true; +} + +template +static bool fg_grouped_device_run(ggml_type src0_type, bool reordered, const void * src0_base, size_t expert_stride, + ggml_sycl_grouped_gemm_rows src1, ggml_sycl_grouped_gemm_rows dst, + const int32_t * ids_dev, size_t ids_token_stride, int64_t n_as, int64_t n_ids, + int64_t n_tokens, int64_t M, int64_t K, ggml_sycl_pool & pool, + dpct::queue_ptr stream) { + if constexpr (S::BN != GGML_SYCL_FG_BN) { + return false; + } else { + const int64_t total_rows = n_tokens * n_ids; + const int64_t max_tiles = grouped_gemm_max_tiles(total_rows, n_as, S::BN); + if (max_tiles > INT32_MAX / S::BN) { + return false; + } + const int Npad = (int) (max_tiles * S::BN); + ggml_sycl_pool_alloc rows(pool, total_rows); + ggml_sycl_pool_alloc tiles(pool, max_tiles); + ggml_sycl_pool_alloc offsets(pool, n_as + 1); + if (!ggml_sycl_build_mmid_schedule(ids_dev, ids_token_stride, n_as, n_ids, n_tokens, max_tiles, + rows.get(), tiles.get(), offsets.get(), stream)) { + return false; + } + src1.map = rows.get(); + dst.map = rows.get(); + ggml_sycl_pool_alloc packed_b(pool, (size_t) K * Npad); + const ggml_sycl_grouped_gemm_tile * tile_ptr = tiles.get(); + typename S::tsb * packed = packed_b.get(); + using E = typename S::EB; + constexpr int V = S::VNNI; + stream->parallel_for(sycl::range<1>((size_t) Npad * (K / V)), [=](sycl::id<1> id) { + const size_t idx = id[0]; + const int kq = idx / Npad; + const int n = idx - (size_t) kq * Npad; + const ggml_sycl_grouped_gemm_tile tile = tile_ptr[n / S::BN]; + const int row = tile.n0 + n % S::BN; + typename S::tsb vals[V] = {}; + if (row < tile.n1) { + const mmid_row_mapping mapping = src1.map[row]; + const float * in = (const float *) (src1.base + (mapping.i1 % src1.ne1) * src1.nb1 + mapping.i2 * src1.nb2) + V * kq; +#pragma unroll + for (int v = 0; v < V; ++v) { + vals[v] = E::cvt(in[v]); + } + } + typename S::tsb * out = packed + ((size_t) kq * Npad + n) * V; +#pragma unroll + for (int v = 0; v < V; ++v) { + out[v] = vals[v]; + } + }); + const int64_t groups_m = (M + S::SG_ROWS - 1) / S::SG_ROWS; + return fg_visit_type(src0_type, reordered, [&](auto tag) { + using T = decltype(tag); + grouped_dequant_gemm_launch((const char *) src0_base, expert_stride, + tile_ptr, packed, (float *) dst.base, (int) M, Npad, (int) K, max_tiles, groups_m, stream, dst); + }); + } +} + +bool ggml_sycl_grouped_dequant_gemm_device_supported(ggml_type src0_type, bool reordered, int32_t src1_prec, + int64_t n_as, int64_t n_ids, int64_t n_tokens, + int64_t M, int64_t K, dpct::queue_ptr stream) { + if (n_as <= 0 || n_as > GGML_SYCL_MMID_SCHED_MAX_EXPERTS || n_ids <= 0 || n_tokens <= 0 || + n_ids > INT32_MAX / n_tokens) { + return false; + } + const int64_t total_rows = n_ids * n_tokens; + // The active expert count is unknown until the GPU builds the schedule. + if (g_ggml_sycl_dynamic_precision == GGML_SYCL_DYNAMIC_PRECISION_F32 || + !ggml_sycl_xmx_gather_type_enabled(src0_type) || !fg_visit_type(src0_type, reordered, [](auto) {}) || + !ggml_sycl_grouped_dequant_gemm_shape_ok(src0_type, M, K, total_rows, std::min(n_as, total_rows), + (g_ggml_sycl_mmid_sched & GGML_SYCL_MMID_SCHED_UNBOUNDED) != 0)) { + return false; + } + const sycl::device device = stream->get_device(); + if (device.get_info() < MMID_SCHED_WG || + device.get_info() < 3 * (n_as + 1) * sizeof(uint32_t)) { + return false; + } + if (grouped_gemm_max_tiles(total_rows, n_as, GGML_SYCL_FG_BN) > INT32_MAX / GGML_SYCL_FG_BN) { + return false; + } + const int combo = fg_pick_combo(stream, GGML_TYPE_F32, src1_prec); + if (combo < 0) { + return false; + } + // the device schedule is built for GGML_SYCL_FG_BN-wide tiles, see fg_grouped_device_run() + bool bn_ok = false; + fg_visit_combo(combo, [&](auto s) { bn_ok = decltype(s)::BN == GGML_SYCL_FG_BN; }); + return bn_ok; +} + +bool ggml_sycl_grouped_dequant_gemm_device(ggml_type src0_type, bool reordered, const void * src0_base, + size_t expert_stride, const ggml_sycl_grouped_gemm_rows & src1, + int32_t src1_prec, const ggml_sycl_grouped_gemm_rows & dst, + const int32_t * ids_dev, size_t ids_token_stride, int64_t n_as, + int64_t n_ids, int64_t n_tokens, int64_t M, int64_t K, ggml_sycl_pool & pool, + dpct::queue_ptr stream) { + if (src1.ne1 <= 0 || dst.ne1 <= 0 || + !ggml_sycl_grouped_dequant_gemm_device_supported(src0_type, reordered, src1_prec, n_as, n_ids, + n_tokens, M, K, stream)) { + return false; + } + const int combo = fg_pick_combo(stream, GGML_TYPE_F32, src1_prec); + bool launched = false; + fg_visit_combo(combo, [&](auto s) { + launched = fg_grouped_device_run(src0_type, reordered, src0_base, expert_stride, src1, dst, + ids_dev, ids_token_stride, n_as, n_ids, n_tokens, M, K, pool, stream); + }); + return launched; +} diff --git a/ggml/src/ggml-sycl/fused-gemm.hpp b/ggml/src/ggml-sycl/fused-gemm.hpp index 226676d12d50..0e807a2c0ab9 100644 --- a/ggml/src/ggml-sycl/fused-gemm.hpp +++ b/ggml/src/ggml-sycl/fused-gemm.hpp @@ -4,6 +4,18 @@ #include "common.hpp" +// Routed row n is at base + (map[n].i1 % ne1)*nb1 + map[n].i2*nb2. +struct ggml_sycl_grouped_gemm_rows { + char * base; + const mmid_row_mapping * map; + int64_t ne1; + size_t nb1; + size_t nb2; +}; + +static constexpr int GGML_SYCL_FG_BN = 32; +static constexpr int64_t GGML_SYCL_MMID_SCHED_MAX_EXPERTS = 4096; + // Shape and type gates for the kernels below. Device capability is separate: it needs a queue to ask. static constexpr int GGML_SYCL_FG_MAX_N = 64; // widest N taken; each shape covers it in BN-wide tiles @@ -15,6 +27,8 @@ constexpr bool ggml_sycl_fused_dequant_gemm_type_ok(ggml_type src0_type, int64_t return K % 32 == 0; } const bool superblock = + src0_type == GGML_TYPE_Q2_K || + src0_type == GGML_TYPE_Q3_K || src0_type == GGML_TYPE_Q4_K || src0_type == GGML_TYPE_Q5_K || src0_type == GGML_TYPE_Q6_K || @@ -39,9 +53,9 @@ constexpr bool ggml_sycl_fused_dequant_gemm_shape_ok(ggml_type src0_type, int64_ // grouped variant: the per-expert fused kernel is only worth it while each expert is narrow, // so wider average slices are left to the per-expert library GEMM loop constexpr bool ggml_sycl_grouped_dequant_gemm_shape_ok(ggml_type src0_type, int64_t M, int64_t K, - int64_t total_rows, int64_t n_active) { + int64_t total_rows, int64_t n_active, bool unbounded = false) { return ggml_sycl_fused_dequant_gemm_shape_ok(src0_type, M, 1, K, M) && total_rows > 0 && - total_rows <= INT32_MAX && total_rows <= n_active * GGML_SYCL_FG_MAX_N; + total_rows <= INT32_MAX && (unbounded || total_rows <= n_active * GGML_SYCL_FG_MAX_N); } // Runtime type gate, kept out of the constexpr predicates above so those stay pure. @@ -57,6 +71,8 @@ inline bool ggml_sycl_xmx_gather_type_enabled(ggml_type src0_type) { case GGML_TYPE_IQ1_S: return (g_ggml_sycl_xmx_gather_types & GGML_SYCL_XMX_GATHER_IQ1_S ) != 0; case GGML_TYPE_IQ1_M: return (g_ggml_sycl_xmx_gather_types & GGML_SYCL_XMX_GATHER_IQ1_M ) != 0; case GGML_TYPE_Q8_0: return (g_ggml_sycl_xmx_gather_types & GGML_SYCL_XMX_GATHER_Q8_0 ) != 0; + case GGML_TYPE_Q2_K: return (g_ggml_sycl_xmx_gather_types & GGML_SYCL_XMX_GATHER_Q2_K ) != 0; + case GGML_TYPE_Q3_K: return (g_ggml_sycl_xmx_gather_types & GGML_SYCL_XMX_GATHER_Q3_K ) != 0; case GGML_TYPE_Q4_K: return (g_ggml_sycl_xmx_gather_types & GGML_SYCL_XMX_GATHER_Q4_K ) != 0; case GGML_TYPE_Q5_K: return (g_ggml_sycl_xmx_gather_types & GGML_SYCL_XMX_GATHER_Q5_K ) != 0; case GGML_TYPE_Q6_K: return (g_ggml_sycl_xmx_gather_types & GGML_SYCL_XMX_GATHER_Q6_K ) != 0; @@ -82,7 +98,25 @@ bool ggml_sycl_fused_dequant_gemm(ggml_type src0_type, bool src0_reordered, cons bool ggml_sycl_grouped_dequant_gemm(ggml_type src0_type, bool reordered, const void * src0_base, size_t expert_stride, const float * src1, int32_t src1_prec, float * dst, const int64_t * expert_row_offsets, int64_t n_as, int64_t M, int64_t K, - int64_t total_rows, std::vector & tiles, + int64_t total_rows, std::vector & tiles, ggml_sycl_pool & pool, dpct::queue_ptr stream); +// Build routing and tile tables on the device; unused tile slots are empty. +bool ggml_sycl_build_mmid_schedule(const int32_t * ids_dev, size_t ids_token_stride, int64_t n_as, int64_t n_ids, + int64_t n_tokens, int64_t n_tiles_max, mmid_row_mapping * row_mapping, + ggml_sycl_grouped_gemm_tile * tiles, uint32_t * expert_offsets, + dpct::queue_ptr stream); + +bool ggml_sycl_grouped_dequant_gemm_device_supported(ggml_type src0_type, bool reordered, int32_t src1_prec, + int64_t n_as, int64_t n_ids, int64_t n_tokens, + int64_t M, int64_t K, dpct::queue_ptr stream); + +// Returns false before queuing work when the type, shape, layout or device is unsupported. +bool ggml_sycl_grouped_dequant_gemm_device(ggml_type src0_type, bool reordered, const void * src0_base, + size_t expert_stride, const ggml_sycl_grouped_gemm_rows & src1, + int32_t src1_prec, const ggml_sycl_grouped_gemm_rows & dst, + const int32_t * ids_dev, size_t ids_token_stride, int64_t n_as, + int64_t n_ids, int64_t n_tokens, int64_t M, int64_t K, ggml_sycl_pool & pool, + dpct::queue_ptr stream); + #endif // GGML_SYCL_FUSED_GEMM_HPP diff --git a/ggml/src/ggml-sycl/ggml-sycl.cpp b/ggml/src/ggml-sycl/ggml-sycl.cpp index 9ee16423aa7c..fdd526425c40 100644 --- a/ggml/src/ggml-sycl/ggml-sycl.cpp +++ b/ggml/src/ggml-sycl/ggml-sycl.cpp @@ -118,6 +118,7 @@ int g_ggml_sycl_xmx_glu_max_cols = GGML_SYCL_XMX_GLU_MAX_COLS; int g_ggml_sycl_prioritize_dmmv = 0; int g_ggml_sycl_xmx_gather_types = GGML_SYCL_XMX_GATHER_TYPES_DEFAULT; int g_ggml_sycl_xmx_gather_shapes = GGML_SYCL_XMX_GATHER_SHAPES_DEFAULT; +int g_ggml_sycl_mmid_sched = GGML_SYCL_MMID_SCHED_DEVICE; int g_ggml_sycl_dynamic_precision = GGML_SYCL_DYNAMIC_PRECISION_DEFAULT; int g_ggml_sycl_dynamic_required_precision = GGML_SYCL_DYNAMIC_PRECISION_F32; static const char * ggml_sycl_dynamic_precision_names[] = { "F16", "BF16", "TF32", "F32" }; @@ -146,6 +147,7 @@ int g_ggml_sycl_use_level_zero_api = 0; int g_ggml_sycl_enable_flash_attention = 1; int g_ggml_sycl_dev2dev_memcpy = DEV2DEV_MEMCPY_SYCL; int g_ggml_sycl_usm_system = 0; +int g_ggml_sycl_bmg_vram_check = 1; int g_ggml_sycl_enable_host_pinned_mem = 1; int g_ggml_sycl_host_pinned_mem_2g = 0; int g_ggml_sycl_upload_staging_slots = 4; @@ -450,6 +452,7 @@ static void ggml_check_sycl() try { g_ggml_sycl_prioritize_dmmv = ggml_sycl_get_env("GGML_SYCL_PRIORITIZE_DMMV", 0); g_ggml_sycl_graph_eviction_timeout = ggml_sycl_get_env("GGML_SYCL_GRAPH_EVICTION_TIMEOUT", 10); g_ggml_sycl_xmx_gather_types = ggml_sycl_get_env("GGML_SYCL_XMX_GATHER_TYPES", GGML_SYCL_XMX_GATHER_TYPES_DEFAULT); + g_ggml_sycl_mmid_sched = ggml_sycl_get_env("GGML_SYCL_MMID_SCHED", GGML_SYCL_MMID_SCHED_DEVICE) & 3; g_ggml_sycl_xmx_gather_shapes = ggml_sycl_get_env("GGML_SYCL_XMX_GATHER_SHAPES", GGML_SYCL_XMX_GATHER_SHAPES_DEFAULT); g_ggml_sycl_dynamic_precision = ggml_sycl_get_env_precision("GGML_SYCL_DYNAMIC_PRECISION", GGML_SYCL_DYNAMIC_PRECISION_DEFAULT); @@ -475,6 +478,7 @@ static void ggml_check_sycl() try { #endif g_ggml_sycl_usm_system = ggml_sycl_get_env("GGML_SYCL_USM_SYSTEM", 0); + g_ggml_sycl_bmg_vram_check = ggml_sycl_get_env("GGML_SYCL_BMG_VRAM_CHECK", 1); g_ggml_sycl_enable_host_pinned_mem = ggml_sycl_get_env("GGML_SYCL_ENABLE_HOST_PINNED_MEM", 1); @@ -573,6 +577,7 @@ static void ggml_check_sycl() try { GGML_LOG_INFO(" GGML_SYCL_ENABLE_OPT: %d\n", g_ggml_sycl_enable_optimize); GGML_LOG_INFO(" GGML_SYCL_XMX_GATHER_TYPES: %d\n", g_ggml_sycl_xmx_gather_types); + GGML_LOG_INFO(" GGML_SYCL_MMID_SCHED: %d\n", g_ggml_sycl_mmid_sched); GGML_LOG_INFO(" GGML_SYCL_XMX_GATHER_SHAPES: %d\n", g_ggml_sycl_xmx_gather_shapes); GGML_LOG_INFO(" GGML_SYCL_DYNAMIC_PRECISION: %s\n", ggml_sycl_dynamic_precision_names[g_ggml_sycl_dynamic_precision]); @@ -612,6 +617,7 @@ static void ggml_check_sycl() try { #endif GGML_LOG_INFO(" GGML_SYCL_USM_SYSTEM: %d\n", g_ggml_sycl_usm_system); + GGML_LOG_INFO(" GGML_SYCL_BMG_VRAM_CHECK: %d\n", g_ggml_sycl_bmg_vram_check); GGML_LOG_INFO(" GGML_SYCL_ENABLE_HOST_PINNED_MEM: %d\n", g_ggml_sycl_enable_host_pinned_mem); GGML_LOG_INFO(" GGML_SYCL_HOST_PINNED_MEM_2G: %d\n", g_ggml_sycl_host_pinned_mem_2g); @@ -809,7 +815,9 @@ ggml_backend_sycl_buffer_init_tensor(ggml_backend_buffer_t buffer, return GGML_STATUS_SUCCESS; } - if (g_ggml_sycl_enable_optimize) { + // a compute buffer copy of a host weight (e.g. -ncmoe experts offloaded for prompt processing) is + // refilled in the normal layout before every run, so reordering it is wasted work + if (g_ggml_sycl_enable_optimize && ggml_backend_buffer_get_usage(buffer) != GGML_BACKEND_BUFFER_USAGE_COMPUTE) { // set reorder extra buffer based on supported type switch (tensor->type) { case GGML_TYPE_Q4_0: @@ -2509,23 +2517,6 @@ k_argsort_f32_i32(const float *x, int *dst, const int ncols, int ncols_pad, } } -static void diag_mask_inf_f32(const float * x, float * dst, const int ncols, const int rows_per_channel, const int n_past, - const sycl::nd_item<3> &item_ct1) { - const int col = item_ct1.get_local_range(1) * item_ct1.get_group(1) + - item_ct1.get_local_id(1); - const int row = item_ct1.get_local_range(2) * item_ct1.get_group(2) + - item_ct1.get_local_id(2); - - if (col >= ncols) { - return; - } - - const int i = row*ncols + col; - //dst[i] = col > (n_past + row % rows_per_channel) ? -INFINITY : x[i]; - //dst[i] = x[i] - (col > n_past + row % rows_per_channel) * INT_MAX; // equivalent within rounding error but slightly faster on GPU - dst[i] = x[i] - (col > n_past + row % rows_per_channel) * FLT_MAX; -} - static void scale_f32(const float * x, float * dst, const float scale, const float bias, const int k, const sycl::nd_item<3> &item_ct1) { const int i = item_ct1.get_local_range(2) * item_ct1.get_group(2) + @@ -3131,20 +3122,6 @@ static void argmax_f32_i32_sycl(const float *x, int *dst, const int ncols, }); }); } -static void diag_mask_inf_f32_sycl(const float *x, float *dst, - const int ncols_x, const int nrows_x, - const int rows_per_channel, const int n_past, - queue_ptr stream) { - const sycl::range<3> block_dims(1, SYCL_DIAG_MASK_INF_BLOCK_SIZE, 1); - const int block_num_x = (ncols_x + SYCL_DIAG_MASK_INF_BLOCK_SIZE - 1) / SYCL_DIAG_MASK_INF_BLOCK_SIZE; - const sycl::range<3> block_nums(1, block_num_x, nrows_x); - stream->parallel_for(sycl::nd_range<3>(block_nums * block_dims, block_dims), - [=](sycl::nd_item<3> item_ct1) { - diag_mask_inf_f32(x, dst, ncols_x, - rows_per_channel, n_past, - item_ct1); - }); -} static dpct::err0 ggml_sycl_cpy_tensor_2d(void *dst, const struct ggml_tensor *src, @@ -3307,7 +3284,7 @@ inline void ggml_sycl_op_mul_mat_sycl( // the f16 route converts src1 to f16 [TAG_GGML_PREC] use_fp16 = use_fp16 && ggml_sycl_src1_f16_ok(dst); if ((src0->type == GGML_TYPE_F16 || ggml_is_quantized(src0->type)) && use_fp16 && ggml_is_contiguous(src0) && - row_diff == src0->ne[1] && dst->op_params[0] == GGML_PREC_DEFAULT) { + row_diff == src0->ne[1] && ggml_prec_get_acc(dst) == GGML_PREC_UNDEFINED) { ggml_sycl_pool_alloc src1_as_f16(ctx.pool()); if (src1->type != GGML_TYPE_F16) { scope_op_debug_print scope_dbg_print(__func__, "/to_fp16_sycl", dst, /*num_src=*/2, @@ -3528,25 +3505,6 @@ inline void ggml_sycl_op_argmax(ggml_backend_sycl_context & ctx, ggml_tensor * d argmax_f32_i32_sycl(src0_dd, dst_dd, ncols, nrows, main_stream); } -inline void ggml_sycl_op_diag_mask_inf(ggml_backend_sycl_context & ctx, ggml_tensor * dst) { - GGML_ASSERT(dst->src[0]->type == GGML_TYPE_F32); - GGML_ASSERT( dst->type == GGML_TYPE_F32); - dpct::queue_ptr main_stream = ctx.stream(); - SYCL_CHECK(ggml_sycl_set_device(ctx.device)); - const float * src0_dd = static_cast(dst->src[0]->data); - float * dst_dd = static_cast(dst->data); - - const int64_t ne00 = dst->src[0]->ne[0]; - const int64_t ne01 = dst->src[0]->ne[1]; - const int nrows0 = ggml_nrows(dst->src[0]); - // the kernel indexes in 32-bit - GGML_ASSERT(ne00 * nrows0 <= INT32_MAX); - - const int n_past = ((int32_t *) dst->op_params)[0]; - - diag_mask_inf_f32_sycl(src0_dd, dst_dd, ne00, nrows0, ne01, n_past, main_stream); -} - static void tri_f32_sycl( const float * src, float * dst, @@ -4471,36 +4429,6 @@ static bool ggml_sycl_supports_dmmv(enum ggml_type type) { } } -static bool ggml_sycl_supports_mmvq_id(enum ggml_type type) { - switch (type) { - case GGML_TYPE_Q4_0: - case GGML_TYPE_Q4_1: - case GGML_TYPE_Q5_0: - case GGML_TYPE_Q5_1: - case GGML_TYPE_Q8_0: - case GGML_TYPE_Q2_0: - case GGML_TYPE_Q2_K: - case GGML_TYPE_Q3_K: - case GGML_TYPE_Q4_K: - case GGML_TYPE_Q5_K: - case GGML_TYPE_Q6_K: - case GGML_TYPE_MXFP4: - case GGML_TYPE_NVFP4: - case GGML_TYPE_IQ2_XXS: - case GGML_TYPE_IQ2_XS: - case GGML_TYPE_IQ2_S: - case GGML_TYPE_IQ3_XXS: - case GGML_TYPE_IQ3_S: - case GGML_TYPE_IQ1_S: - case GGML_TYPE_IQ1_M: - case GGML_TYPE_IQ4_NL: - case GGML_TYPE_IQ4_XS: - return true; - default: - return false; - } -} - // Helper functions to unify device memory allocation for both async and sync paths static inline void * sycl_ext_malloc_device(dpct::queue_ptr stream, size_t size) { bool use_async = g_ggml_sycl_use_async_mem_op; @@ -4976,6 +4904,16 @@ static bool reorder_qw_q3_k(uint8_t * data_device, size_t size, size_t offset, d }, stream); } +// Reorder each Q3_K expert slice on its own. +static bool reorder_qw_q3_k_moe(uint8_t * data_device, size_t expert_bytes, int64_t n_expert, dpct::queue_ptr stream) { + for (int64_t e = 0; e < n_expert; ++e) { + if (!reorder_qw_q3_k(data_device + e * expert_bytes, expert_bytes, 0, stream)) { + return false; + } + } + return true; +} + static bool reorder_qw_q5_k(uint8_t * data_device, size_t size, size_t offset, dpct::queue_ptr stream) { GGML_ASSERT(offset == 0); return reorder_qw_planar(__func__, data_device, size, offsetof(block_q5_K, qs), QK_K / 2, @@ -5047,6 +4985,8 @@ static bool reorder_qw(const ggml_tensor * src0, dpct::queue_ptr stream) { switch (src0->type) { case GGML_TYPE_Q8_0: return reorder_qw_q8_0_moe(data_device, src0->nb[2], src0->ne[2], stream); + case GGML_TYPE_Q3_K: + return reorder_qw_q3_k_moe(data_device, src0->nb[2], src0->ne[2], stream); case GGML_TYPE_Q4_K: return reorder_qw_q4_k_moe(data_device, src0->nb[2], src0->ne[2], stream); case GGML_TYPE_Q5_K: @@ -5132,7 +5072,7 @@ static void opt_for_reorder_id(ggml_backend_sycl_context * ctx, const ggml_tenso if (!g_ggml_sycl_enable_optimize || !ctx->opt_feature.reorder) { return; } - if (src0->type != GGML_TYPE_Q8_0 && src0->type != GGML_TYPE_Q4_K && + if (src0->type != GGML_TYPE_Q8_0 && src0->type != GGML_TYPE_Q3_K && src0->type != GGML_TYPE_Q4_K && src0->type != GGML_TYPE_Q5_K && src0->type != GGML_TYPE_Q6_K && src0->type != GGML_TYPE_MXFP4) { return; } @@ -5140,6 +5080,10 @@ static void opt_for_reorder_id(ggml_backend_sycl_context * ctx, const ggml_tenso if (src0->type == GGML_TYPE_MXFP4 && (src0->nb[2] % 16 != 0 || (uintptr_t) src0->data % 16 != 0)) { return; } + // The grouped GEMM loads reordered Q3_K qs and hmask as 16-byte vectors. + if (src0->type == GGML_TYPE_Q3_K && ((src0->nb[2] / sizeof(block_q3_K)) % 16 != 0 || (uintptr_t) src0->data % 16 != 0)) { + return; + } ggml_tensor_extra_gpu * extra = static_cast(src0->extra); if (!extra || extra->optimized_feature.reorder) { return; @@ -5638,26 +5582,31 @@ __dpct_inline__ static void k_zero_dst_rows( } } -// returns true when ggml_sycl_mul_mat_id takes the fallback path that requires stream synchronization -static bool ggml_sycl_mul_mat_id_needs_sync(const ggml_tensor * dst) { - const ggml_tensor * src0 = dst->src[0]; - const ggml_tensor * src1 = dst->src[1]; - const ggml_tensor * ids = dst->src[2]; - +// layout checks for the fused MMVQ path; the type check is ggml_sycl_mul_mat_vec_q_id_supports_type +static bool ggml_sycl_mul_mat_id_mmvq_eligible( + const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * ids, const ggml_tensor * dst) { const int64_t ne10 = src1->ne[0]; const int64_t ne11 = src1->ne[1]; const int64_t ne12 = src1->ne[2]; - if (ne12 != 1) return true; - if (src1->type != GGML_TYPE_F32 || dst->type != GGML_TYPE_F32) return true; - if (ne10 != src0->ne[0] || ne10 % QK8_1 != 0) return true; - if (!ggml_is_contiguous(src1)) return true; + if (ne12 != 1) return false; + if (src1->type != GGML_TYPE_F32 || dst->type != GGML_TYPE_F32) return false; + if (ne10 != src0->ne[0] || ne10 % QK8_1 != 0) return false; + if (!ggml_is_contiguous(src1)) return false; const int64_t n_ids_per_group = ids->ne[0]; - if (ids->ne[1] != 1) return true; - if (ne11 != 1 && ne11 != n_ids_per_group) return true; + if (ids->type != GGML_TYPE_I32 || ids->nb[0] != sizeof(int32_t)) return false; + if (ids->ne[1] != 1) return false; + if (ne11 != 1 && ne11 != n_ids_per_group) return false; - return !ggml_sycl_supports_mmvq_id(src0->type); + return true; +} + +// returns true when ggml_sycl_mul_mat_id takes the fused MMVQ path +static bool ggml_sycl_mul_mat_id_should_use_mmvq(const ggml_tensor * dst) { + const ggml_tensor * src0 = dst->src[0]; + return ggml_sycl_mul_mat_id_mmvq_eligible(src0, dst->src[1], dst->src[2], dst) && + ggml_sycl_mul_mat_vec_q_id_supports_type(src0->type); } // Fused MoE TG fast path. Returns false to fall back to the per-expert loop below. @@ -5665,13 +5614,11 @@ static bool ggml_sycl_mul_mat_id_mmvq_fused( ggml_backend_sycl_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * ids, ggml_tensor * dst) { - if (ggml_sycl_mul_mat_id_needs_sync(dst)) return false; + if (!ggml_sycl_mul_mat_id_should_use_mmvq(dst)) return false; const int64_t ne10 = src1->ne[0]; const int64_t ne11 = src1->ne[1]; - const int64_t n_ids_per_group = ids->ne[0]; - const queue_ptr stream = ctx.stream(); const int src1_padded_cols = GGML_PAD((int) ne10, MATRIX_ROW_PADDING); const int n_experts_used = (int) n_ids_per_group; @@ -5769,6 +5716,18 @@ static void mmid_counting_sort_rows( routed_row_src.resize(expert_row_offsets[n_as]); } +// true when the device MoE schedule may handle this MUL_MAT_ID: enabled, prefill, and an +// f32/i32 layout it accepts; ggml_sycl_grouped_dequant_gemm_device decides type and width +static bool ggml_sycl_mul_mat_id_device_sched_eligible(const ggml_tensor * dst) { + const ggml_tensor * src0 = dst->src[0]; + const ggml_tensor * src1 = dst->src[1]; + const ggml_tensor * ids = dst->src[2]; + return (g_ggml_sycl_mmid_sched & GGML_SYCL_MMID_SCHED_DEVICE) && src1->ne[2] != 1 && ggml_is_contiguous(src0) && + ids->type == GGML_TYPE_I32 && ids->nb[0] == sizeof(int32_t) && + src1->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32 && + src1->nb[1] == sizeof(float)*src1->ne[0] && dst->nb[1] == sizeof(float)*dst->ne[0]; +} + static void ggml_sycl_mul_mat_id(ggml_backend_sycl_context & ctx, ggml_tensor *dst) try { scope_op_debug_print scope_dbg_print(__func__, dst, /*num_src=*/3); @@ -5789,13 +5748,46 @@ static void ggml_sycl_mul_mat_id(ggml_backend_sycl_context & ctx, return; } } - GGML_ASSERT(ggml_sycl_mul_mat_id_needs_sync(dst)); + GGML_ASSERT(!ggml_sycl_mul_mat_id_should_use_mmvq(dst)); // The per-expert loop below reads the experts in whatever layout they have: reorder MXFP4 here as well, so prompt processing does not depend on a single-token decode having run first. if (src0->type == GGML_TYPE_MXFP4) { opt_for_reorder_id(&ctx, src0); } + if (ggml_sycl_mul_mat_id_device_sched_eligible(dst)) { + if (g_ggml_sycl_dynamic_precision != GGML_SYCL_DYNAMIC_PRECISION_F32 && + ggml_sycl_xmx_gather_type_enabled(src0->type)) { + opt_for_reorder_id(&ctx, src0); + } + const bool reordered = src0->extra && ((const ggml_tensor_extra_gpu *) src0->extra)->optimized_feature.reorder; + const ggml_sycl_grouped_gemm_rows src1_rows = { (char *) src1->data, nullptr, ne11, nb11, nb12 }; + const ggml_sycl_grouped_gemm_rows dst_rows = { (char *) dst->data, nullptr, ne1, nb1, nb2 }; + if (ggml_sycl_grouped_dequant_gemm_device(src0->type, reordered, src0->data, nb02, src1_rows, + ggml_sycl_src1_prec(dst), dst_rows, (const int32_t *) ids->data, ids->nb[1], n_as, n_ids, + ids->ne[1], ne01, ne10, ctx.pool(), stream)) { + // the device schedule drops skipped slots (id == -1), so their dst rows are zeroed here + const char * ids_data = (const char *) ids->data; + const size_t ids_nb1 = ids->nb[1]; + char * dst_data = (char *) dst->data; + const int nth = (int) std::min(ne0, 256); + stream->parallel_for( + sycl::nd_range<2>(sycl::range<2>(ids->ne[1], n_ids * nth), sycl::range<2>(1, nth)), + [=](sycl::nd_item<2> item) { + const int64_t iid1 = item.get_group(0); + const int64_t id = item.get_group(1); + if (*(const int32_t *) (ids_data + iid1*ids_nb1 + id*sizeof(int32_t)) != -1) { + return; + } + float * row = (float *) (dst_data + id*nb1 + iid1*nb2); + for (int64_t j = item.get_local_id(1); j < ne0; j += nth) { + row[j] = 0.0f; + } + }); + return; + } + } + std::vector ids_host(ggml_nbytes(ids)); const char * ids_dev = (const char *) ids->data; @@ -6003,11 +5995,6 @@ static void ggml_sycl_scale(ggml_backend_sycl_context & ctx, ggml_tensor * dst) ggml_sycl_op_scale(ctx, dst); } -static void ggml_sycl_diag_mask_inf(ggml_backend_sycl_context & ctx, ggml_tensor * dst) { - scope_op_debug_print scope_dbg_print(__func__, dst, /*num_src=*/1); - ggml_sycl_op_diag_mask_inf(ctx, dst); -} - static void ggml_sycl_pool2d(ggml_backend_sycl_context & ctx, ggml_tensor * dst) { scope_op_debug_print scope_dbg_print(__func__, dst, /*num_src=*/1); ggml_sycl_op_pool2d(ctx, dst); @@ -6070,11 +6057,11 @@ static void ggml_sycl_argmax(ggml_backend_sycl_context & ctx, ggml_tensor * dst) static void ggml_sycl_set_main_device(const int main_device) try { - if (dpct::get_current_device_id() == static_cast (main_device)) { + if (get_current_device_id() == main_device) { return; } check_allow_gpu_index(main_device); - dpct::select_device(main_device); + SYCL_CHECK(ggml_sycl_set_device(main_device)); if (g_ggml_sycl_debug) { dpct::device_info prop; @@ -6151,7 +6138,6 @@ static bool ggml_sycl_compute_forward(ggml_backend_sycl_context & ctx, struct gg ggml_sycl_dup(ctx, dst); break; case GGML_OP_ADD: - case GGML_OP_ADD1: // TODO: more efficient implementation ggml_sycl_add(ctx, dst); break; case GGML_OP_ADD_ID: @@ -6354,9 +6340,6 @@ static bool ggml_sycl_compute_forward(ggml_backend_sycl_context & ctx, struct gg case GGML_OP_TRI: ggml_sycl_op_tri(ctx, dst); break; - case GGML_OP_DIAG_MASK_INF: - ggml_sycl_diag_mask_inf(ctx, dst); - break; case GGML_OP_SOFT_MAX: ggml_sycl_op_soft_max(ctx, dst); break; @@ -6478,7 +6461,7 @@ catch (sycl::exception const &exc) { std::exit(1); } -static bool sycl_get_mem_info(int device, size_t * free, size_t * total) { +bool sycl_get_mem_info(int device, size_t * free, size_t * total) { GGML_SYCL_DEBUG("[SYCL] [%s] g_ggml_sycl_get_mem_api=%d\n", __func__, g_ggml_sycl_get_mem_api); @@ -6887,7 +6870,7 @@ static bool mul_mat_uses_onednn(int device, ggml_tensor * dst) { #endif #ifdef GGML_SYCL_F16 if ((src0->type == GGML_TYPE_F16 || ggml_is_quantized(src0->type)) && ggml_is_contiguous(src0) && - dst->op_params[0] == GGML_PREC_DEFAULT) { + ggml_prec_get_acc(dst) == GGML_PREC_UNDEFINED) { return true; } #endif @@ -6901,6 +6884,54 @@ static bool mul_mat_uses_onednn(int device, ggml_tensor * dst) { #endif } +// returns true when ggml_sycl_mul_mat_id takes the fallback path that requires stream synchronization: +// neither the fused MMVQ path nor the device-scheduled grouped GEMM (GGML_SYCL_MMID_SCHED) handles dst +static bool ggml_sycl_mul_mat_id_needs_sync(ggml_backend_sycl_context * ctx, const ggml_tensor * dst) { + const ggml_tensor * src0 = dst->src[0]; + const ggml_tensor * src1 = dst->src[1]; + const ggml_tensor * ids = dst->src[2]; + if (!src0 || !src1 || !ids || ggml_backend_buffer_is_sycl_split(src0->buffer)) { + return true; + } + const bool reordered = src0->extra && ((const ggml_tensor_extra_gpu *) src0->extra)->optimized_feature.reorder; + if (ggml_sycl_mul_mat_id_mmvq_eligible(src0, src1, ids, dst)) { + return reordered ? !ggml_sycl_mul_mat_vec_q_id_reorder_supports_type(src0->type) : + !ggml_sycl_mul_mat_vec_q_id_supports_type(src0->type); + } + return !ggml_sycl_mul_mat_id_device_sched_eligible(dst) || + !ggml_sycl_grouped_dequant_gemm_device_supported(src0->type, reordered, ggml_sycl_src1_prec(dst), + src0->ne[2], ids->ne[0], ids->ne[1], src0->ne[1], src0->ne[0], ctx->stream()); +} + +// mirrors the paths in ggml_sycl_mul_mat_id() that call opt_for_reorder_id() +static bool mul_mat_id_reorders_src0(const ggml_tensor * dst) { + const ggml_tensor * src0 = dst->src[0]; + if (ggml_sycl_mul_mat_id_should_use_mmvq(dst) || src0->type == GGML_TYPE_MXFP4) { + return true; + } + return ggml_sycl_mul_mat_id_device_sched_eligible(dst) && + g_ggml_sycl_dynamic_precision != GGML_SYCL_DYNAMIC_PRECISION_F32 && + ggml_sycl_xmx_gather_type_enabled(src0->type); +} + +// true when the node and its SYCL sources live in this device's buffers (no split or other-device buffers) +static bool graph_node_is_in_local_device(ggml_backend_buffer_type_t device_buft, const ggml_tensor * node) { + if (!node->buffer || node->buffer->buft != device_buft) { + return false; + } + for (int j = 0; j < GGML_MAX_SRC; ++j) { + const ggml_tensor * src = node->src[j]; + if (!src || !src->buffer) { + continue; + } + if (ggml_backend_buffer_is_sycl_split(src->buffer) || + (ggml_backend_buffer_is_sycl(src->buffer) && src->buffer->buft != device_buft)) { + return false; + } + } + return true; +} + static bool check_graph_compatibility(ggml_backend_sycl_context * ctx, ggml_cgraph * cgraph) { if (ggml_sycl_info().active_device_count > 1) { // A sycl_ex::command_graph object can only be created for a single device @@ -6908,6 +6939,8 @@ static bool check_graph_compatibility(ggml_backend_sycl_context * ctx, ggml_cgra return false; } + const ggml_backend_buffer_type_t device_buft = ggml_backend_sycl_buffer_type(ctx->device); + for (int i = 0; i < cgraph->n_nodes; i++) { ggml_tensor * node = cgraph->nodes[i]; @@ -6918,6 +6951,18 @@ static bool check_graph_compatibility(ggml_backend_sycl_context * ctx, ggml_cgra const ggml_op node_op = node->op; + if (!graph_node_is_in_local_device(device_buft, node)) { + GGML_LOG_DEBUG("%s: disabling SYCL graphs due to node %s not being device local\n", __func__, + ggml_op_name(node_op)); + return false; + } + + // Reorder expert weights before recording, so the reorder state that + // ggml_sycl_mul_mat_id_needs_sync() sees is the one the recorded graph runs with. + if (node_op == GGML_OP_MUL_MAT_ID && mul_mat_id_reorders_src0(node)) { + opt_for_reorder_id(ctx, node->src[0]); + } + switch (node_op) { default: break; @@ -6925,7 +6970,7 @@ static bool check_graph_compatibility(ggml_backend_sycl_context * ctx, ggml_cgra // The ggml_sycl_mul_mat_id() fallback path does a blocking host wait on the sycl // queue after submitting a memcpy operation, but wait() can't be called on a queue // that is recording to a graph. - if (ggml_sycl_mul_mat_id_needs_sync(node)) { + if (ggml_sycl_mul_mat_id_needs_sync(ctx, node)) { GGML_LOG_DEBUG("%s: disabling SYCL graphs due to host wait in %s\n", __func__, ggml_op_name(node_op)); return false; @@ -6965,6 +7010,13 @@ static bool check_graph_compatibility(ggml_backend_sycl_context * ctx, ggml_cgra } #endif break; + case GGML_OP_OUT_PROD: + case GGML_OP_CONV_3D: + case GGML_OP_SOLVE_TRI: + // these call oneMKL and wait on the host + GGML_LOG_DEBUG("%s: disabling SYCL graphs due to unsupported node type %s\n", __func__, + ggml_op_name(node_op)); + return false; } } return true; @@ -7318,6 +7370,11 @@ static bool do_ggml_backend_sycl_device_supports_op(ggml_backend_dev_t dev, cons return false; } + if (op->op == GGML_OP_MUL_MAT && + (ggml_prec_get_src(op, 1) == GGML_PREC_BF16 || ggml_prec_get_src(op, 1) == GGML_PREC_F32)) { + return false; + } + return true; } case GGML_OP_OUT_PROD: @@ -7523,7 +7580,6 @@ static bool do_ggml_backend_sycl_device_supports_op(ggml_backend_dev_t dev, cons case GGML_OP_PERMUTE: case GGML_OP_TRANSPOSE: case GGML_OP_ADD: - case GGML_OP_ADD1: case GGML_OP_ADD_ID: case GGML_OP_SUB: case GGML_OP_COUNT_EQUAL: @@ -7558,8 +7614,6 @@ static bool do_ggml_backend_sycl_device_supports_op(ggml_backend_dev_t dev, cons op->type == GGML_TYPE_F32 && ggml_is_contiguous(src0); } - case GGML_OP_DIAG_MASK_INF: - return true; case GGML_OP_SOFT_MAX: return true; case GGML_OP_SOFT_MAX_BACK: { diff --git a/ggml/src/ggml-sycl/mmvq.cpp b/ggml/src/ggml-sycl/mmvq.cpp index c3853814aa29..03464bfaed8f 100644 --- a/ggml/src/ggml-sycl/mmvq.cpp +++ b/ggml/src/ggml-sycl/mmvq.cpp @@ -2706,6 +2706,50 @@ static void launch_mul_mat_vec_q_moe( }); } +bool ggml_sycl_mul_mat_vec_q_id_supports_type(enum ggml_type src0_type) { + switch (src0_type) { + case GGML_TYPE_Q4_0: + case GGML_TYPE_Q4_1: + case GGML_TYPE_Q5_0: + case GGML_TYPE_Q5_1: + case GGML_TYPE_Q8_0: + case GGML_TYPE_Q2_0: + case GGML_TYPE_Q2_K: + case GGML_TYPE_Q3_K: + case GGML_TYPE_Q4_K: + case GGML_TYPE_Q5_K: + case GGML_TYPE_Q6_K: + case GGML_TYPE_MXFP4: + case GGML_TYPE_NVFP4: + case GGML_TYPE_IQ2_XXS: + case GGML_TYPE_IQ2_XS: + case GGML_TYPE_IQ2_S: + case GGML_TYPE_IQ3_XXS: + case GGML_TYPE_IQ3_S: + case GGML_TYPE_IQ1_S: + case GGML_TYPE_IQ1_M: + case GGML_TYPE_IQ4_NL: + case GGML_TYPE_IQ4_XS: + return true; + default: + return false; + } +} + +bool ggml_sycl_mul_mat_vec_q_id_reorder_supports_type(enum ggml_type src0_type) { + switch (src0_type) { + case GGML_TYPE_Q8_0: + case GGML_TYPE_Q3_K: + case GGML_TYPE_Q4_K: + case GGML_TYPE_Q5_K: + case GGML_TYPE_Q6_K: + case GGML_TYPE_MXFP4: + return true; + default: + return false; + } +} + bool ggml_sycl_mul_mat_vec_q_id( enum ggml_type src0_type, const void * vx_base, @@ -2719,7 +2763,9 @@ bool ggml_sycl_mul_mat_vec_q_id( size_t dst_row_stride, size_t src1_row_stride, dpct::queue_ptr stream) { - // if adding a new type, ensure it is also added to ggml_sycl_supports_mmvq_id + if (!ggml_sycl_mul_mat_vec_q_id_supports_type(src0_type)) { + return false; + } switch (src0_type) { case GGML_TYPE_Q4_0: launch_mul_mat_vec_q_moe( @@ -2952,12 +2998,20 @@ bool ggml_sycl_mul_mat_vec_q_id_reorder( size_t dst_row_stride, size_t src1_row_stride, dpct::queue_ptr stream) { + if (!ggml_sycl_mul_mat_vec_q_id_reorder_supports_type(src0_type)) { + return false; + } switch (src0_type) { case GGML_TYPE_Q8_0: launch_mul_mat_vec_q_moe_reorder>( vx_base, vy, ids_dev, dst_base, ncols, nrows, n_experts_used, expert_weight_stride, dst_row_stride, src1_row_stride, stream); return true; + case GGML_TYPE_Q3_K: + launch_mul_mat_vec_q_moe_reorder>( + vx_base, vy, ids_dev, dst_base, ncols, nrows, n_experts_used, + expert_weight_stride, dst_row_stride, src1_row_stride, stream); + return true; case GGML_TYPE_Q4_K: launch_mul_mat_vec_q_moe_reorder>( vx_base, vy, ids_dev, dst_base, ncols, nrows, n_experts_used, diff --git a/ggml/src/ggml-sycl/mmvq.hpp b/ggml/src/ggml-sycl/mmvq.hpp index 5649cbcd267e..e372d50ba11c 100644 --- a/ggml/src/ggml-sycl/mmvq.hpp +++ b/ggml/src/ggml-sycl/mmvq.hpp @@ -117,4 +117,7 @@ bool ggml_sycl_mul_mat_vec_q_glu_plain( int stride_col_dst, // floats between output columns in dst dpct::queue_ptr stream); +bool ggml_sycl_mul_mat_vec_q_id_supports_type(enum ggml_type src0_type); +bool ggml_sycl_mul_mat_vec_q_id_reorder_supports_type(enum ggml_type src0_type); + #endif // GGML_SYCL_MMVQ_HPP diff --git a/ggml/src/ggml-sycl/presets.hpp b/ggml/src/ggml-sycl/presets.hpp index 789f3ef0f20f..e9fa6c8a6b12 100644 --- a/ggml/src/ggml-sycl/presets.hpp +++ b/ggml/src/ggml-sycl/presets.hpp @@ -33,10 +33,8 @@ #define SYCL_SET_BLOCK_SIZE 256 #define SYCL_CPY_BLOCK_SIZE 32 #define SYCL_SCALE_BLOCK_SIZE 256 -#define SYCL_CLAMP_BLOCK_SIZE 256 #define SYCL_ROPE_BLOCK_SIZE 256 #define SYCL_ALIBI_BLOCK_SIZE 32 -#define SYCL_DIAG_MASK_INF_BLOCK_SIZE 32 #define SYCL_QUANTIZE_BLOCK_SIZE 256 #define SYCL_DEQUANTIZE_BLOCK_SIZE 256 #define SYCL_GET_ROWS_BLOCK_SIZE 256 diff --git a/ggml/src/ggml-sycl/rope.cpp b/ggml/src/ggml-sycl/rope.cpp index b6d22559d18c..b41c1a80e56a 100644 --- a/ggml/src/ggml-sycl/rope.cpp +++ b/ggml/src/ggml-sycl/rope.cpp @@ -547,7 +547,7 @@ void ggml_sycl_op_rope_impl(ggml_backend_sycl_context &ctx, ggml_tensor *dst, rope_corr_dims corr_dims; ggml_rope_yarn_corr_dims(n_dims, n_ctx_orig, freq_base, beta_fast, - beta_slow, corr_dims.v); + beta_slow, ggml_rope_get_truncate(dst), corr_dims.v); // compute if (is_neox) { diff --git a/ggml/src/ggml-virtgpu/ggml-backend-device.cpp b/ggml/src/ggml-virtgpu/ggml-backend-device.cpp index 987ce9dd110c..86df4cc5f3b2 100644 --- a/ggml/src/ggml-virtgpu/ggml-backend-device.cpp +++ b/ggml/src/ggml-virtgpu/ggml-backend-device.cpp @@ -30,11 +30,14 @@ static void ggml_backend_remoting_device_get_memory(ggml_backend_dev_t dev, size } static bool ggml_backend_remoting_device_supports_op(ggml_backend_dev_t dev, const ggml_tensor * op) { + if ((op->op == GGML_OP_ROPE || op->op == GGML_OP_ROPE_BACK) && + !ggml_rope_get_truncate(op)) { + return false; + } #if USE_ALWAYS_TRUE_SUPPORTS_OP == 1 /* ggml-rpc cheats it like this */ /* with the current implementation of serialize_tensor, the src/view aren't properly passed */ UNUSED(dev); - UNUSED(op); return true; #else diff --git a/ggml/src/ggml-vulkan/ggml-vulkan-buffers.cpp b/ggml/src/ggml-vulkan/ggml-vulkan-buffers.cpp index 4e60df41b2c4..9c6edb569cbf 100644 --- a/ggml/src/ggml-vulkan/ggml-vulkan-buffers.cpp +++ b/ggml/src/ggml-vulkan/ggml-vulkan-buffers.cpp @@ -229,7 +229,7 @@ vk_buffer ggml_vk_create_buffer_device(vk_device& device, size_t size) { buf = ggml_vk_create_buffer(device, size, {vk::MemoryPropertyFlagBits::eDeviceLocal | vk::MemoryPropertyFlagBits::eHostVisible | vk::MemoryPropertyFlagBits::eHostCoherent, vk::MemoryPropertyFlagBits::eDeviceLocal, vk::MemoryPropertyFlagBits::eHostVisible | vk::MemoryPropertyFlagBits::eHostCoherent}); - } else if (device->disable_host_visible_vidmem) { + } else if (!device->rebar || device->disable_host_visible_vidmem) { if (device->allow_sysmem_fallback) { buf = ggml_vk_create_buffer(device, size, {vk::MemoryPropertyFlagBits::eDeviceLocal, vk::MemoryPropertyFlagBits::eHostVisible | vk::MemoryPropertyFlagBits::eHostCoherent}); @@ -404,6 +404,33 @@ void ggml_vk_ensure_sync_staging_buffer(ggml_backend_vk_context * ctx, size_t si } } +static vk_context ggml_vk_begin_sync_ctx(vk_device & device, vk_command_pool & pool) { + vk_context subctx = ggml_vk_create_temporary_context(pool); + ggml_vk_ctx_begin(device, subctx); + return subctx; +} + +static void ggml_vk_submit_sync(vk_device & device, vk_context & subctx) { + ggml_vk_ctx_end(subctx); + + for (auto & cpy : subctx->in_memcpys) { + memcpy(cpy.dst, cpy.src, cpy.n); + } + + for (auto & mset : subctx->memsets) { + memset(mset.dst, mset.val, mset.n); + } + + ggml_vk_submit(subctx, device->fence); + VK_CHECK(device->device.waitForFences({ device->fence }, true, UINT64_MAX), "vk_submit_sync waitForFences", device); + device->device.resetFences({ device->fence }); + ggml_vk_queue_command_pools_cleanup(device); + + for (auto & cpy : subctx->out_memcpys) { + memcpy(cpy.dst, cpy.src, cpy.n); + } +} + static void ggml_vk_buffer_write_nc_async(ggml_backend_vk_context * ctx, vk_context& subctx, vk_buffer& dst, size_t offset, const ggml_tensor * tensor, bool sync_staging = false) { VK_LOG_DEBUG("ggml_vk_buffer_write_nc_async(" << tensor << ")"); GGML_ASSERT(!ggml_is_contiguous(tensor)); @@ -590,24 +617,10 @@ void ggml_vk_buffer_write_2d(vk_buffer& dst, size_t offset, const void * src, si } else { std::lock_guard guard(dst->device->mutex); - vk_context subctx = ggml_vk_create_temporary_context(dst->device->transfer_queue->cmd_pool); - ggml_vk_ctx_begin(dst->device, subctx); + vk_context subctx = ggml_vk_begin_sync_ctx(dst->device, dst->device->transfer_queue->cmd_pool); bool ret = ggml_vk_buffer_write_2d_async(subctx, dst, offset, src, spitch, dpitch, width, height, true); GGML_ASSERT(ret); - ggml_vk_ctx_end(subctx); - - for (auto& cpy : subctx->in_memcpys) { - memcpy(cpy.dst, cpy.src, cpy.n); - } - - for (auto& mset : subctx->memsets) { - memset(mset.dst, mset.val, mset.n); - } - - ggml_vk_submit(subctx, dst->device->fence); - VK_CHECK(dst->device->device.waitForFences({ dst->device->fence }, true, UINT64_MAX), "vk_buffer_write_2d waitForFences", dst->device); - dst->device->device.resetFences({ dst->device->fence }); - ggml_vk_queue_command_pools_cleanup(dst->device); + ggml_vk_submit_sync(dst->device, subctx); } } @@ -695,21 +708,24 @@ static bool ggml_vk_buffer_read_async(vk_context subctx, vk_buffer& src, size_t return ggml_vk_buffer_read_2d_async(subctx, src, offset, dst, size, size, size, 1, sync_staging); } -void ggml_vk_buffer_read_2d(vk_buffer& src, size_t offset, void * dst, size_t spitch, size_t dpitch, size_t width, size_t height) { - VK_LOG_DEBUG("ggml_vk_buffer_read_2d(" << src->buffer << ", " << offset << ", " << width << ", " << height << ")"); - +static bool ggml_vk_buffer_host_read_direct(const vk_buffer & buf) { // If the device is not an UMA device the memory is host-accessible through rebar. While writing // through PCIe is sufficient fast reading back data from PCIe is slower than going through // the HW device to host copy path. // AMD UMA: uncached host-visible memory is write-combined, CPU reads are slow - const bool slow_host_read = src->device->vendor_id == VK_VENDOR_ID_AMD && - !(src->memory_property_flags & vk::MemoryPropertyFlagBits::eHostCached); - if(src->memory_property_flags & vk::MemoryPropertyFlagBits::eHostVisible && src->device->uma && !slow_host_read) { + const bool slow_host_read = buf->device->vendor_id == VK_VENDOR_ID_AMD && + !(buf->memory_property_flags & vk::MemoryPropertyFlagBits::eHostCached); + return (buf->memory_property_flags & vk::MemoryPropertyFlagBits::eHostVisible) && buf->device->uma && !slow_host_read; +} + +void ggml_vk_buffer_read_2d(vk_buffer& src, size_t offset, void * dst, size_t spitch, size_t dpitch, size_t width, size_t height) { + VK_LOG_DEBUG("ggml_vk_buffer_read_2d(" << src->buffer << ", " << offset << ", " << width << ", " << height << ")"); + + if (ggml_vk_buffer_host_read_direct(src)) { GGML_ASSERT(src->memory_property_flags & vk::MemoryPropertyFlagBits::eHostCoherent); std::lock_guard guard(src->device->mutex); - vk_context subctx = ggml_vk_create_temporary_context(src->device->compute_queue->cmd_pool); - ggml_vk_ctx_begin(src->device, subctx); + vk_context subctx = ggml_vk_begin_sync_ctx(src->device, src->device->compute_queue->cmd_pool); subctx->s->buffer->buf.pipelineBarrier( vk::PipelineStageFlagBits::eComputeShader | vk::PipelineStageFlagBits::eTransfer, vk::PipelineStageFlagBits::eHost, @@ -717,12 +733,7 @@ void ggml_vk_buffer_read_2d(vk_buffer& src, size_t offset, void * dst, size_t sp { { vk::AccessFlagBits::eShaderWrite | vk::AccessFlagBits::eTransferWrite, vk::AccessFlagBits::eHostRead } }, {}, {}); - ggml_vk_ctx_end(subctx); - ggml_vk_submit(subctx, src->device->fence); - VK_CHECK(src->device->device.waitForFences({ src->device->fence }, true, UINT64_MAX), - "vk_buffer_read_2d uma waitForFences", src->device); - src->device->device.resetFences({ src->device->fence }); - ggml_vk_queue_command_pools_cleanup(src->device); + ggml_vk_submit_sync(src->device, subctx); if (width == spitch && width == dpitch) { memcpy(dst, (const uint8_t *) src->ptr + offset, width * height); @@ -734,20 +745,10 @@ void ggml_vk_buffer_read_2d(vk_buffer& src, size_t offset, void * dst, size_t sp } else { std::lock_guard guard(src->device->mutex); - vk_context subctx = ggml_vk_create_temporary_context(src->device->transfer_queue->cmd_pool); - ggml_vk_ctx_begin(src->device, subctx); + vk_context subctx = ggml_vk_begin_sync_ctx(src->device, src->device->transfer_queue->cmd_pool); bool ret = ggml_vk_buffer_read_2d_async(subctx, src, offset, dst, spitch, dpitch, width, height, true); GGML_ASSERT(ret); - ggml_vk_ctx_end(subctx); - - ggml_vk_submit(subctx, src->device->fence); - VK_CHECK(src->device->device.waitForFences({ src->device->fence }, true, UINT64_MAX), "vk_buffer_read_2d waitForFences", src->device); - src->device->device.resetFences({ src->device->fence }); - ggml_vk_queue_command_pools_cleanup(src->device); - - for (auto& cpy : subctx->out_memcpys) { - memcpy(cpy.dst, cpy.src, cpy.n); - } + ggml_vk_submit_sync(src->device, subctx); } } @@ -771,14 +772,9 @@ void ggml_vk_buffer_copy(vk_buffer& dst, size_t dst_offset, vk_buffer& src, size std::lock_guard guard(src->device->mutex); VK_LOG_DEBUG("ggml_vk_buffer_copy(SINGLE_DEVICE, " << size << ")"); // Copy within the device - vk_context subctx = ggml_vk_create_temporary_context(src->device->transfer_queue->cmd_pool); - ggml_vk_ctx_begin(src->device, subctx); + vk_context subctx = ggml_vk_begin_sync_ctx(src->device, src->device->transfer_queue->cmd_pool); ggml_vk_buffer_copy_async(subctx, dst, dst_offset, src, src_offset, size); - ggml_vk_ctx_end(subctx); - ggml_vk_submit(subctx, src->device->fence); - VK_CHECK(src->device->device.waitForFences({ src->device->fence }, true, UINT64_MAX), "vk_buffer_copy waitForFences", src->device); - src->device->device.resetFences({ src->device->fence }); - ggml_vk_queue_command_pools_cleanup(src->device); + ggml_vk_submit_sync(src->device, subctx); } else { VK_LOG_DEBUG("ggml_vk_buffer_copy(MULTI_DEVICE, " << size << ")"); // Copy device to device @@ -814,15 +810,222 @@ void ggml_vk_buffer_memset(vk_buffer& dst, size_t offset, uint32_t c, size_t siz } std::lock_guard guard(dst->device->mutex); - vk_context subctx = ggml_vk_create_temporary_context(dst->device->transfer_queue->cmd_pool); - ggml_vk_ctx_begin(dst->device, subctx); + vk_context subctx = ggml_vk_begin_sync_ctx(dst->device, dst->device->transfer_queue->cmd_pool); subctx->s->buffer->buf.fillBuffer(dst->buffer, offset, size, c); - ggml_vk_ctx_end(subctx); + ggml_vk_submit_sync(dst->device, subctx); +} + +static size_t ggml_vk_staging_size_n(const vk_device & device, const std::vector & regions) { + size_t total = 0; + size_t max = 0; + for (const auto & r : regions) { + vk_buffer pinned; + size_t pinned_offset; + ggml_vk_host_get(device, r.host, pinned, pinned_offset); + if (pinned != nullptr) { + continue; + } + total += r.size; + max = std::max(max, r.size); + } + return std::max(max, std::min(total, (size_t) device->suballocation_block_size)); +} + +// returns number of regions recorded, stops when staging is full +size_t ggml_vk_buffer_write_n_async(vk_context& subctx, const vk_buffer_region * regions, size_t n, vk_buffer * staging, size_t * staging_offset) { + VK_LOG_DEBUG("ggml_vk_buffer_write_n_async(" << n << ")"); + ggml_vk_sync_buffers(nullptr, subctx); + + size_t i = 0; + for (; i < n; i++) { + const vk_buffer_region & r = regions[i]; + + vk_buffer pinned = nullptr; + size_t pinned_offset = 0; + ggml_vk_host_get(r.buf->device, r.host, pinned, pinned_offset); + + if (pinned != nullptr) { + subctx->s->buffer->buf.copyBuffer(pinned->buffer, r.buf->buffer, { vk::BufferCopy{ pinned_offset, r.offset, r.size } }); + continue; + } + + if (staging == nullptr || *staging_offset + r.size > (*staging)->size) { + break; + } + + subctx->s->buffer->buf.copyBuffer((*staging)->buffer, r.buf->buffer, { vk::BufferCopy{ *staging_offset, r.offset, r.size } }); + deferred_memcpy((uint8_t *) (*staging)->ptr + *staging_offset, r.host, r.size, &subctx->in_memcpys); + *staging_offset += r.size; + } + return i; +} + +// returns number of regions recorded, stops when staging is full +size_t ggml_vk_buffer_read_n_async(vk_context& subctx, const vk_buffer_region * regions, size_t n, vk_buffer * staging, size_t * staging_offset) { + VK_LOG_DEBUG("ggml_vk_buffer_read_n_async(" << n << ")"); + ggml_vk_sync_buffers(nullptr, subctx); + + size_t i = 0; + for (; i < n; i++) { + const vk_buffer_region & r = regions[i]; + + vk_buffer pinned = nullptr; + size_t pinned_offset = 0; + ggml_vk_host_get(r.buf->device, r.host, pinned, pinned_offset); + + if (pinned != nullptr) { + subctx->s->buffer->buf.copyBuffer(r.buf->buffer, pinned->buffer, { vk::BufferCopy{ r.offset, pinned_offset, r.size } }); + continue; + } + + if (staging == nullptr || *staging_offset + r.size > (*staging)->size) { + break; + } + + subctx->s->buffer->buf.copyBuffer(r.buf->buffer, (*staging)->buffer, { vk::BufferCopy{ r.offset, *staging_offset, r.size } }); + deferred_memcpy(r.host, (const uint8_t *) (*staging)->ptr + *staging_offset, r.size, &subctx->out_memcpys); + *staging_offset += r.size; + } + return i; +} + +void ggml_vk_buffer_copy_n_async(vk_context& subctx, const vk_buffer_copy_region * regions, size_t n) { + VK_LOG_DEBUG("ggml_vk_buffer_copy_n_async(" << n << ")"); + ggml_vk_sync_buffers(nullptr, subctx); + + for (size_t i = 0; i < n; i++) { + const vk_buffer_copy_region & r = regions[i]; + GGML_ASSERT(r.src->device == r.dst->device); + subctx->s->buffer->buf.copyBuffer(r.src->buffer, r.dst->buffer, { vk::BufferCopy{ r.src_offset, r.dst_offset, r.size } }); + } +} + +void ggml_vk_buffer_write_n(const vk_buffer_region * regions, size_t n) { + VK_LOG_DEBUG("ggml_vk_buffer_write_n(" << n << ")"); + std::vector pending; + + for (size_t i = 0; i < n; i++) { + const vk_buffer_region & r = regions[i]; + if (r.size == 0) { + continue; + } + if (r.buf->memory_property_flags & vk::MemoryPropertyFlagBits::eHostVisible) { + GGML_ASSERT(r.buf->memory_property_flags & vk::MemoryPropertyFlagBits::eHostCoherent); + memcpy((uint8_t *) r.buf->ptr + r.offset, r.host, r.size); + continue; + } + GGML_ASSERT(pending.empty() || r.buf->device == pending[0].buf->device); + pending.push_back(r); + } + + if (pending.empty()) { + return; + } + + vk_device device = pending[0].buf->device; + std::lock_guard guard(device->mutex); + const size_t staging_size = ggml_vk_staging_size_n(device, pending); + if (staging_size > 0) { + ggml_vk_ensure_sync_staging_buffer(device, staging_size); + } + + for (size_t i = 0; i < pending.size();) { + vk_context subctx = ggml_vk_begin_sync_ctx(device, device->transfer_queue->cmd_pool); + size_t staging_offset = 0; + const size_t recorded = ggml_vk_buffer_write_n_async(subctx, pending.data() + i, pending.size() - i, &device->sync_staging, &staging_offset); + GGML_ASSERT(recorded > 0); + ggml_vk_submit_sync(device, subctx); + i += recorded; + } +} + +void ggml_vk_buffer_read_n(const vk_buffer_region * regions, size_t n) { + VK_LOG_DEBUG("ggml_vk_buffer_read_n(" << n << ")"); + std::vector direct; + std::vector pending; + + for (size_t i = 0; i < n; i++) { + const vk_buffer_region & r = regions[i]; + if (r.size == 0) { + continue; + } + GGML_ASSERT((direct.empty() && pending.empty()) || r.buf->device == (direct.empty() ? pending[0] : direct[0]).buf->device); + if (ggml_vk_buffer_host_read_direct(r.buf)) { + GGML_ASSERT(r.buf->memory_property_flags & vk::MemoryPropertyFlagBits::eHostCoherent); + direct.push_back(r); + } else { + pending.push_back(r); + } + } + + if (direct.empty() && pending.empty()) { + return; + } + + vk_device device = (direct.empty() ? pending[0] : direct[0]).buf->device; + std::lock_guard guard(device->mutex); + + if (!direct.empty()) { + vk_context subctx = ggml_vk_begin_sync_ctx(device, device->compute_queue->cmd_pool); + subctx->s->buffer->buf.pipelineBarrier( + vk::PipelineStageFlagBits::eComputeShader | vk::PipelineStageFlagBits::eTransfer, + vk::PipelineStageFlagBits::eHost, + {}, + { { vk::AccessFlagBits::eShaderWrite | vk::AccessFlagBits::eTransferWrite, + vk::AccessFlagBits::eHostRead } }, + {}, {}); + ggml_vk_submit_sync(device, subctx); + + for (const auto & r : direct) { + memcpy(r.host, (const uint8_t *) r.buf->ptr + r.offset, r.size); + } + } + + if (pending.empty()) { + return; + } + + const size_t staging_size = ggml_vk_staging_size_n(device, pending); + if (staging_size > 0) { + ggml_vk_ensure_sync_staging_buffer(device, staging_size); + } + + for (size_t i = 0; i < pending.size();) { + vk_context subctx = ggml_vk_begin_sync_ctx(device, device->transfer_queue->cmd_pool); + size_t staging_offset = 0; + const size_t recorded = ggml_vk_buffer_read_n_async(subctx, pending.data() + i, pending.size() - i, &device->sync_staging, &staging_offset); + GGML_ASSERT(recorded > 0); + ggml_vk_submit_sync(device, subctx); + i += recorded; + } +} + +void ggml_vk_buffer_copy_n(const vk_buffer_copy_region * regions, size_t n) { + VK_LOG_DEBUG("ggml_vk_buffer_copy_n(" << n << ")"); + std::vector same_device; + + for (size_t i = 0; i < n; i++) { + vk_buffer_copy_region r = regions[i]; + if (r.size == 0) { + continue; + } + if (r.src->device != r.dst->device) { + ggml_vk_buffer_copy(r.dst, r.dst_offset, r.src, r.src_offset, r.size); + continue; + } + GGML_ASSERT(same_device.empty() || r.dst->device == same_device[0].dst->device); + same_device.push_back(std::move(r)); + } + + if (same_device.empty()) { + return; + } - ggml_vk_submit(subctx, dst->device->fence); - VK_CHECK(dst->device->device.waitForFences({ dst->device->fence }, true, UINT64_MAX), "vk_memset waitForFences", dst->device); - dst->device->device.resetFences({ dst->device->fence }); - ggml_vk_queue_command_pools_cleanup(dst->device); + vk_device device = same_device[0].dst->device; + std::lock_guard guard(device->mutex); + vk_context subctx = ggml_vk_begin_sync_ctx(device, device->transfer_queue->cmd_pool); + ggml_vk_buffer_copy_n_async(subctx, same_device.data(), same_device.size()); + ggml_vk_submit_sync(device, subctx); } ggml_backend_buffer_i ggml_backend_vk_buffer_interface = { @@ -837,9 +1040,9 @@ ggml_backend_buffer_i ggml_backend_vk_buffer_interface = { /* .cpy_tensor = */ ggml_backend_vk_buffer_cpy_tensor, /* .clear = */ ggml_backend_vk_buffer_clear, /* .reset = */ NULL, - /* .set_tensor_n = */ NULL, - /* .get_tensor_n = */ NULL, - /* .cpy_tensor_n = */ NULL, + /* .set_tensor_n = */ ggml_backend_vk_buffer_set_tensor_n, + /* .get_tensor_n = */ ggml_backend_vk_buffer_get_tensor_n, + /* .cpy_tensor_n = */ ggml_backend_vk_buffer_cpy_tensor_n, }; vk_buffer ggml_vk_buffer_from_host_ptr(vk_device & device, void * ptr, size_t size) { diff --git a/ggml/src/ggml-vulkan/ggml-vulkan-comm.cpp b/ggml/src/ggml-vulkan/ggml-vulkan-comm.cpp index a2370b8cd1b1..84af0661a38f 100644 --- a/ggml/src/ggml-vulkan/ggml-vulkan-comm.cpp +++ b/ggml/src/ggml-vulkan/ggml-vulkan-comm.cpp @@ -18,7 +18,7 @@ static bool ggml_vk_comm_needs_f32(const ggml_tensor * t, int depth = 0) { switch (t->op) { case GGML_OP_MUL_MAT: case GGML_OP_MUL_MAT_ID: - return ggml_get_op_params_i32(t, 0) == GGML_PREC_F32; + return ggml_prec_get_acc(t) == GGML_PREC_BF16 || ggml_prec_get_acc(t) == GGML_PREC_F32; case GGML_OP_ADD: return ggml_vk_comm_needs_f32(t->src[0], depth + 1) || ggml_vk_comm_needs_f32(t->src[1], depth + 1); case GGML_OP_MUL: diff --git a/ggml/src/ggml-vulkan/ggml-vulkan-common.h b/ggml/src/ggml-vulkan/ggml-vulkan-common.h index 0ac0f8b835c2..4e05fb81839b 100644 --- a/ggml/src/ggml-vulkan/ggml-vulkan-common.h +++ b/ggml/src/ggml-vulkan/ggml-vulkan-common.h @@ -59,6 +59,12 @@ void ggml_vk_buffer_read_2d(vk_buffer& src, size_t offset, void * dst, size_t sp void ggml_vk_buffer_read(vk_buffer& src, size_t offset, void * dst, size_t size); void ggml_vk_buffer_copy_async(vk_context& ctx, vk_buffer& dst, size_t dst_offset, vk_buffer& src, size_t src_offset, size_t size); void ggml_vk_buffer_copy(vk_buffer& dst, size_t dst_offset, vk_buffer& src, size_t src_offset, size_t size); +size_t ggml_vk_buffer_write_n_async(vk_context& subctx, const vk_buffer_region * regions, size_t n, vk_buffer * staging = nullptr, size_t * staging_offset = nullptr); +size_t ggml_vk_buffer_read_n_async(vk_context& subctx, const vk_buffer_region * regions, size_t n, vk_buffer * staging = nullptr, size_t * staging_offset = nullptr); +void ggml_vk_buffer_copy_n_async(vk_context& subctx, const vk_buffer_copy_region * regions, size_t n); +void ggml_vk_buffer_write_n(const vk_buffer_region * regions, size_t n); +void ggml_vk_buffer_read_n(const vk_buffer_region * regions, size_t n); +void ggml_vk_buffer_copy_n(const vk_buffer_copy_region * regions, size_t n); void ggml_vk_buffer_memset_async(vk_context& ctx, vk_buffer& dst, size_t offset, uint32_t c, size_t size); void ggml_vk_buffer_memset(vk_buffer& dst, size_t offset, uint32_t c, size_t size); vk_buffer ggml_vk_buffer_from_host_ptr(vk_device & device, void * ptr, size_t size); @@ -166,7 +172,6 @@ void ggml_vk_l2_norm(ggml_backend_vk_context * ctx, vk_context& subctx, const gg void ggml_vk_unary(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst); void ggml_vk_xielu(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst); void ggml_vk_glu(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst); -void ggml_vk_diag_mask_inf(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst); void ggml_vk_soft_max(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * src2, ggml_tensor * dst); void ggml_vk_soft_max_back(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst); void ggml_vk_topk_moe(ggml_backend_vk_context * ctx, vk_context& subctx, ggml_cgraph * cgraph, int node_idx); @@ -227,6 +232,9 @@ void ggml_backend_vk_buffer_set_tensor_2d(ggml_backend_buffer_t buffer, ggml_ten void ggml_backend_vk_buffer_get_tensor(ggml_backend_buffer_t buffer, const ggml_tensor * tensor, void * data, size_t offset, size_t size); void ggml_backend_vk_buffer_get_tensor_2d(ggml_backend_buffer_t buffer, const ggml_tensor * tensor, void * data, size_t offset, size_t size, size_t n_copies, size_t stride_tensor, size_t stride_data); bool ggml_backend_vk_buffer_cpy_tensor(ggml_backend_buffer_t buffer, const ggml_tensor * src, ggml_tensor * dst); +void ggml_backend_vk_buffer_set_tensor_n(ggml_backend_buffer_t buffer, ggml_tensor * const * tensors, const void * const * data, const size_t * offsets, const size_t * sizes, size_t n_tensors); +void ggml_backend_vk_buffer_get_tensor_n(ggml_backend_buffer_t buffer, const ggml_tensor * const * tensors, void * const * data, const size_t * offsets, const size_t * sizes, size_t n_tensors); +bool ggml_backend_vk_buffer_cpy_tensor_n(ggml_backend_buffer_t buffer, const ggml_tensor * const * srcs, ggml_tensor * const * dsts, size_t n_tensors); void ggml_backend_vk_buffer_clear(ggml_backend_buffer_t buffer, uint8_t value); const char * ggml_backend_vk_buffer_type_name(ggml_backend_buffer_type_t buft); ggml_backend_buffer_t ggml_backend_vk_buffer_type_alloc_buffer(ggml_backend_buffer_type_t buft, size_t size); @@ -245,6 +253,12 @@ bool ggml_backend_vk_comm_allreduce_tensor(void * comm_ctx, ggml_tensor ** tenso #endif // debug +void ggml_vk_count_fusion(const vk_device & device, const char * label); +void * ggml_backend_vk_fusion_get(ggml_backend_dev_t dev); +void ggml_backend_vk_fusion_stats_init(void * finfo); +void ggml_backend_vk_fusion_stats_reset(void * finfo); +int ggml_backend_vk_fusion_stats_get(void * finfo, const char ** labels, uint64_t * counts, int n); +void ggml_backend_vk_fusion_set_enabled(void * finfo, bool enabled); int64_t ggml_vk_get_op_batch_size(const ggml_tensor * op); #ifdef GGML_VULKAN_RUN_TESTS template diff --git a/ggml/src/ggml-vulkan/ggml-vulkan-debug.cpp b/ggml/src/ggml-vulkan/ggml-vulkan-debug.cpp index 97747e866df3..2c9c09c06be7 100644 --- a/ggml/src/ggml-vulkan/ggml-vulkan-debug.cpp +++ b/ggml/src/ggml-vulkan/ggml-vulkan-debug.cpp @@ -12,6 +12,58 @@ uint32_t vk_perf_logger_frequency = 1; std::string vk_pipeline_stats_filter; +void ggml_vk_count_fusion(const vk_device & device, const char * label) { + if (!device->fusion_stats_enabled.load(std::memory_order_relaxed)) { + return; + } + std::lock_guard lock(device->fusion_mutex); + device->fusion_counts[label]++; +} + +void * ggml_backend_vk_fusion_get(ggml_backend_dev_t dev) { + auto * ctx = (ggml_backend_vk_device_context *) dev->context; + return ggml_vk_get_device(ctx->device).get(); +} + +void ggml_backend_vk_fusion_stats_init(void * finfo) { + auto * device = (vk_device_struct *) finfo; + device->fusion_stats_enabled.store(true, std::memory_order_relaxed); +} + +void ggml_backend_vk_fusion_stats_reset(void * finfo) { + auto * device = (vk_device_struct *) finfo; + std::lock_guard lock(device->fusion_mutex); + // Keep labels and their order stable between fused and unfused runs. + for (auto & entry : device->fusion_counts) { + entry.second = 0; + } +} + +int ggml_backend_vk_fusion_stats_get(void * finfo, const char ** labels, uint64_t * counts, int n) { + auto * device = (vk_device_struct *) finfo; + std::lock_guard lock(device->fusion_mutex); + if (labels == nullptr) { + return (int) device->fusion_counts.size(); + } + int i = 0; + for (const auto & entry : device->fusion_counts) { + if (i >= n) { + break; + } + labels[i] = entry.first.c_str(); + if (counts) { + counts[i] = entry.second; + } + i++; + } + return i; +} + +void ggml_backend_vk_fusion_set_enabled(void * finfo, bool enabled) { + auto * device = (vk_device_struct *) finfo; + device->disable_fusion = !enabled; +} + void vk_memory_logger::log_allocation(vk_buffer_ref buf_ref, size_t size) { if (!vk_memory_logger_enabled) { return; @@ -983,8 +1035,6 @@ void ggml_vk_check_results_0(ggml_backend_vk_context * ctx, ggml_cgraph * cgraph } else if (tensor->op == GGML_OP_SCALE) { const float * params = (const float *)tensor->op_params; tensor_clone = ggml_scale_bias(ggml_ctx, src_clone[0], params[0], params[1]); - } else if (tensor->op == GGML_OP_ADD1) { - tensor_clone = ggml_add1(ggml_ctx, src_clone[0], src_clone[1]); } else if (tensor->op == GGML_OP_ARANGE) { const float start = ggml_get_op_params_f32(tensor, 0); const float stop = ggml_get_op_params_f32(tensor, 1); @@ -1049,12 +1099,9 @@ void ggml_vk_check_results_0(ggml_backend_vk_context * ctx, ggml_cgraph * cgraph } } else if (tensor->op == GGML_OP_SOFT_MAX_BACK) { tensor_clone = ggml_soft_max_ext_back(ggml_ctx, src_clone[0], src_clone[1], ((float *)tensor->op_params)[0], ((float *)tensor->op_params)[1]); - } else if (tensor->op == GGML_OP_DIAG_MASK_INF) { - tensor_clone = ggml_diag_mask_inf(ggml_ctx, src_clone[0], tensor->op_params[0]); } else if (tensor->op == GGML_OP_ROPE || tensor->op == GGML_OP_ROPE_BACK) { const int n_dims = ((int32_t *) tensor->op_params)[1]; const int mode = ((int32_t *) tensor->op_params)[2]; - //const int n_ctx_ggml = ((int32_t *) tensor->op_params)[3]; const int n_ctx_orig_ggml = ((int32_t *) tensor->op_params)[4]; const float freq_base = ((float *) tensor->op_params)[5]; const float freq_scale = ((float *) tensor->op_params)[6]; @@ -1076,6 +1123,7 @@ void ggml_vk_check_results_0(ggml_backend_vk_context * ctx, ggml_cgraph * cgraph tensor_clone = ggml_rope_ext_back(ggml_ctx, src_clone[0], src_clone[1], src_clone[2], n_dims, mode, n_ctx_orig_ggml, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow); } } + ggml_rope_set_truncate(tensor_clone, ggml_rope_get_truncate(tensor)); const int n_offs = ((int32_t *) tensor->op_params)[15]; if (n_offs != 0) { tensor_clone = ggml_rope_set_offset(tensor_clone, n_offs); diff --git a/ggml/src/ggml-vulkan/ggml-vulkan-push-constants.h b/ggml/src/ggml-vulkan/ggml-vulkan-push-constants.h index 219c477c6170..1ef00477bfef 100644 --- a/ggml/src/ggml-vulkan/ggml-vulkan-push-constants.h +++ b/ggml/src/ggml-vulkan/ggml-vulkan-push-constants.h @@ -124,22 +124,30 @@ struct vk_flash_attn_push_constants { static_assert(sizeof(vk_flash_attn_push_constants) <= 128, "sizeof(vk_flash_attn_push_constants) must be <= 128"); -struct vk_fa_xe_opt_push_constants { - uint32_t kv_seq_len; - uint32_t activation_length; - uint32_t q_head; - uint32_t kv_head; - uint32_t qk_ratio; - uint32_t qk_sub_groups; +struct vk_xe_fa_push_constants { + uint32_t neq1; + uint32_t neq2; + uint32_t neq3; + uint32_t nek1; + uint32_t nek2; + uint32_t nek3; + uint32_t nev2; + uint32_t nev3; + uint32_t nem1; + uint32_t nem2; + uint32_t nem3; + uint32_t nbq1; + uint32_t nbq2; + uint32_t nbk1; + uint32_t nbk2; + uint32_t nbv1; + uint32_t nbv2; + uint32_t nbq3; + uint32_t nbk3; + uint32_t nbv3; + uint32_t nb3; uint32_t flag; - uint32_t nbkv_tok; - uint32_t nbkv_head; - uint32_t batch_stride_q; - uint32_t batch_stride_k; - uint32_t batch_stride_v; - uint32_t batch_stride_m; - uint32_t batch_stride_o; - float softmax_scale; + float scale; }; struct vk_op_push_constants { @@ -454,12 +462,6 @@ struct vk_op_add_id_push_constants { uint32_t s21; }; -struct vk_op_diag_mask_push_constants { - uint32_t ncols; - uint32_t rows_per_channel; - int32_t n_past; -}; - struct vk_op_rope_push_constants { uint32_t rope_mode; uint32_t nrows; @@ -829,7 +831,7 @@ struct vk_op_upscale_push_constants { struct vk_op_sum_rows_push_constants { uint32_t n_cols; - uint32_t ne01, ne02; + uint32_t ne01, ne02, ne03; uint32_t nb01, nb02, nb03; uint32_t nb11, nb12, nb13; float weight; @@ -844,6 +846,7 @@ static vk_op_sum_rows_push_constants vk_op_sum_rows_push_constants_init(const gg p.n_cols = (uint32_t)n_cols; p.ne01 = (uint32_t)src->ne[1]; p.ne02 = (uint32_t)src->ne[2]; + p.ne03 = (uint32_t)src->ne[3]; p.nb01 = (uint32_t)src->nb[1] / type_size; p.nb02 = (uint32_t)src->nb[2] / type_size; p.nb03 = (uint32_t)src->nb[3] / type_size; diff --git a/ggml/src/ggml-vulkan/ggml-vulkan-types.h b/ggml/src/ggml-vulkan/ggml-vulkan-types.h index d3b1a8dd210b..81fd5b8a84ba 100644 --- a/ggml/src/ggml-vulkan/ggml-vulkan-types.h +++ b/ggml/src/ggml-vulkan/ggml-vulkan-types.h @@ -433,6 +433,21 @@ struct vk_fa_pipeline_state { } }; +struct vk_xe_fa_pipeline_state { + uint32_t HSK, HSV; + uint32_t qk_ratio, pv_ratio; + uint32_t decode_n; + bool unaligned_seq_len; + bool unaligned_q; + bool partial_row_mask; + bool multi_head_mask; + + bool operator<(const vk_xe_fa_pipeline_state &b) const { + return std::tie(HSK, HSV, qk_ratio, pv_ratio, decode_n, unaligned_seq_len, unaligned_q, partial_row_mask, multi_head_mask) < + std::tie(b.HSK, b.HSV, b.qk_ratio, b.pv_ratio, b.decode_n, b.unaligned_seq_len, b.unaligned_q, b.partial_row_mask, b.multi_head_mask); + } +}; + struct vk_conv2d_pipeline_state { vk_conv2d_pipeline_state(uint32_t s0, uint32_t s1, uint32_t p0, uint32_t p1, uint32_t d0, uint32_t d1, uint32_t KW, uint32_t KH, uint32_t aligned) : s0(s0), s1(s1), p0(p0), p1(p1), d0(d0), d1(d1), KW(KW), KH(KH), aligned(aligned) {} @@ -486,6 +501,15 @@ static constexpr uint32_t num_topk_moe_pipelines = 10; static constexpr uint32_t num_topk_pipelines = 11; +// FWHT block widths with a dedicated pipeline. The subgroup shader keeps n/subgroup_size +// values per invocation, so widths above fwht_max_subgroup_n, or a subgroup narrow enough to +// exceed fwht_max_subgroup_el_w values, use the shared-memory shader with one row per +// workgroup instead. +static constexpr uint32_t num_fwht_pipelines = 8; +static constexpr uint32_t fwht_max_subgroup_n = 512; +static constexpr uint32_t fwht_max_subgroup_el_w = 64; +static constexpr uint32_t fwht_rows = 4; + static constexpr std::initializer_list topk_moe_early_softmax_norm{ GGML_OP_SOFT_MAX, GGML_OP_RESHAPE, GGML_OP_ARGSORT, GGML_OP_VIEW, GGML_OP_GET_ROWS, GGML_OP_RESHAPE, GGML_OP_SUM_ROWS, GGML_OP_CLAMP, GGML_OP_DIV, @@ -698,6 +722,7 @@ struct vk_device_struct { uint32_t subgroup_size_log2; uint32_t shader_core_count; bool uma; + bool rebar; bool prefer_host_memory; bool float_controls_rte_fp16; bool float_controls_denorm_preserve_fp16; @@ -753,6 +778,9 @@ struct vk_device_struct { bool dot2_f16 {}; bool ocp_fp4 {}; + bool workgroup_memory_explicit_layout {}; + bool workgroup_memory_explicit_layout_16bit {}; + bool pipeline_executable_properties_support {}; bool device_fault {}; @@ -827,6 +855,7 @@ struct vk_device_struct { vk_pipeline pipeline_add_id_f32; vk_pipeline pipeline_concat_i8, pipeline_concat_i16, pipeline_concat_i32, pipeline_concat_i64; + vk_pipeline pipeline_concat_transpose_i8, pipeline_concat_transpose_i16, pipeline_concat_transpose_i32; vk_pipeline pipeline_upscale_nearest_f32, pipeline_upscale_bilinear_f32, pipeline_upscale_bicubic_f32, pipeline_upscale_bilinear_antialias_f32; vk_pipeline pipeline_scale_f32; vk_pipeline pipeline_log[2]; @@ -914,7 +943,6 @@ struct vk_device_struct { vk_pipeline pipeline_leaky_relu[2]; vk_pipeline pipeline_silu_back_f32; - vk_pipeline pipeline_diag_mask_inf_f32; vk_pipeline pipeline_soft_max_f32, pipeline_soft_max_f32_f16; vk_pipeline pipeline_soft_max_f32_wg512, pipeline_soft_max_f32_f16_wg512; vk_pipeline pipeline_soft_max_back_f32; @@ -935,7 +963,9 @@ struct vk_device_struct { vk_pipeline pipeline_sum_rows_f32; vk_pipeline pipeline_cross_entropy_loss_f32, pipeline_cross_entropy_loss_f32_wg512; vk_pipeline pipeline_cross_entropy_loss_back_f32, pipeline_cross_entropy_loss_back_f32_wg512; - vk_pipeline pipeline_fwht_f32[4]; + vk_pipeline pipeline_fwht_f32[num_fwht_pipelines]; + vk_pipeline pipeline_fwht_f16[num_fwht_pipelines]; + uint32_t fwht_rows_per_wg[num_fwht_pipelines] = {}; vk_pipeline pipeline_cumsum_f32; vk_pipeline pipeline_cumsum_small_f32; vk_pipeline pipeline_cumsum_multipass1_f32; @@ -993,7 +1023,8 @@ struct vk_device_struct { bool fa_sparse_compact_use_subgroups; vk_pipeline pipeline_flash_attn_split_k_reduce; - std::map, std::pair> pipeline_xe_fa_decode_dual_phases; + std::map> pipeline_xe_fa_decode_dual_phases; + std::map pipeline_xe_fa_prefill; vk_pipeline pipeline_count_experts; // [2] is for whether to take n_experts from spec constant (0) or push constant (1) @@ -1007,11 +1038,15 @@ struct vk_device_struct { ggml_backend_buffer_type buffer_type; bool disable_fusion; + std::atomic fusion_stats_enabled {false}; + std::mutex fusion_mutex; + std::map fusion_counts; bool disable_descriptor_reuse; std::atomic buffer_destroy_count {}; bool disable_host_visible_vidmem; bool allow_sysmem_fallback; bool disable_graph_optimize; + bool xe_fa_support {}; std::unique_ptr memory_logger; @@ -1119,6 +1154,21 @@ struct vk_staging_memset { size_t n; }; +struct vk_buffer_region { + vk_buffer buf; + size_t offset; + void * host; + size_t size; +}; + +struct vk_buffer_copy_region { + vk_buffer dst; + size_t dst_offset; + vk_buffer src; + size_t src_offset; + size_t size; +}; + struct vk_context_struct { vk_submission * s; std::vector seqs; @@ -1376,6 +1426,34 @@ struct vk_fa_tuning_params { } }; +struct vk_xe_fa_tuning_params { + size_t shmem_sz; + std::array workgroup_size; + std::array subgroup_size; + uint32_t tm; + uint32_t tn; + uint32_t tk; + + struct { + uint32_t wg_row; + uint32_t gqa_fold; + uint32_t q_in_reg; + uint32_t p_per_iter; + uint32_t sg_col; + uint32_t acc_count; + uint32_t pipeline_flag; + } prefill_params; + + struct { + uint32_t p_chunk_ph1; + uint32_t out_per_wg_ph1; + uint32_t split_p_per_iter_ph2; + uint32_t out_row_per_wg_ph2; + uint32_t out_col_per_wg_ph2; + uint32_t pipeline_flag_ph2; + } decode_params; +}; + struct GpuPipelineConfig { // GPU architecture identifier. // Example: vk_device_architecture::AMD_GCN diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vulkan.cpp index ba8578716ff1..22641f5fbdb4 100644 --- a/ggml/src/ggml-vulkan/ggml-vulkan.cpp +++ b/ggml/src/ggml-vulkan/ggml-vulkan.cpp @@ -161,11 +161,13 @@ void ggml_vk_print_device_fault_info(const vk_device& device) { std::vector address_infos(fault_counts.addressInfoCount); std::vector vendor_infos(fault_counts.vendorInfoCount); + std::vector vendor_binary(fault_counts.vendorBinarySize); VkDeviceFaultInfoEXT fault_info {}; fault_info.sType = VK_STRUCTURE_TYPE_DEVICE_FAULT_INFO_EXT; fault_info.pAddressInfos = address_infos.data(); fault_info.pVendorInfos = vendor_infos.data(); + fault_info.pVendorBinaryData = vendor_binary.empty() ? nullptr : vendor_binary.data(); res = device->pfn_vkGetDeviceFaultInfoEXT(device->device, &fault_counts, &fault_info); if (res != VK_SUCCESS) { @@ -1260,6 +1262,12 @@ static vk_fa_tuning_params get_fa_tuning_params_scalar(const vk_device& device, result.block_rows /= 2; } + // With head sizes above 128, 16 rows per workgroup spill registers on Qualcomm and run about + // 40x slower; at large KV a single dispatch can then exceed the Windows GPU timeout. + if (device->vendor_id == VK_VENDOR_ID_QUALCOMM && (hsk > 128 || hsv > 128)) { + result.block_rows = std::min(result.block_rows, 8u); + } + // On AMD RDNA, for small head sizes and big batch size the shader uses few registers, so too many subgroups get scheduled // at once and end up thrashing the cache. Fix this by setting a large (unused) shmem buffer that reduces occupancy. // This targets an occupancy of 4 subgroups per SIMD. @@ -1426,6 +1434,186 @@ static std::vector get_fa_spec_constants(const vk_fa_pipeline_state& s }; } +static vk_xe_fa_tuning_params get_xe_fa_tuning_params(const vk_device_struct * device, const vk_xe_fa_pipeline_state& pipeline_state, bool use_prefill) { + vk_xe_fa_tuning_params params {}; + size_t shmem_sz = 0; + uint32_t sg_sz = 32u; + std::array matrix_mnk = { 16u, 16u, 16u }; + bool intel_gpu = device->vendor_id == VK_VENDOR_ID_INTEL; + + if (device->architecture == INTEL_XE1) { + sg_sz = 8u; + matrix_mnk = { 8u, 8u, 16u }; + } else if (device->architecture == INTEL_XE2) { + sg_sz = 16u; + matrix_mnk = { 8u, 16u, 16u }; + } + + uint32_t tm = matrix_mnk[0]; + uint32_t tn = matrix_mnk[1]; + uint32_t tk = matrix_mnk[2]; + uint32_t hsk = pipeline_state.HSK; + uint32_t hsv = pipeline_state.HSV; + uint32_t qk_ratio = pipeline_state.qk_ratio; + uint32_t pv_ratio = pipeline_state.pv_ratio; + uint32_t sg_per_gw; + + params.tm = tm; + params.tn = tn; + params.tk = tk; + GGML_ASSERT(hsv % tm == 0); + + if (use_prefill) { + sg_per_gw = sg_sz == 8 ? 32 : 16; + uint32_t wg_sz = sg_per_gw * sg_sz; + uint32_t output_div = hsk > 256 ? 8 : hsk > 160 ? 4 : hsk > 64 ? 2 : 1; + uint32_t p_per_iter = intel_gpu ? (hsk > 256 ? 16 : 32) : 16; + uint32_t q_in_reg = hsk > 512 ? hsk - 384 : std::min(128u, hsk); + uint32_t q_in_shmem = hsk - q_in_reg; + uint32_t wg_row = 256u / output_div; + uint32_t gqa_fold = (pipeline_state.multi_head_mask || qk_ratio != pv_ratio) ? 1 : 16; + uint32_t max_fold_rows = sg_sz <= 16 ? 16 : 8; + + while (gqa_fold > 1 && (qk_ratio % gqa_fold != 0 || gqa_fold * output_div > max_fold_rows)) { + gqa_fold = gqa_fold / 2; + } + + uint32_t mask_row = wg_row / gqa_fold; + uint32_t k_col = (hsk % 64 == 0 && hsk <= 512 && wg_sz <= 256) ? 8 : 4; + uint32_t k_loops = CEIL_DIV(hsk * p_per_iter, k_col * wg_sz); + + bool k_in_shmem = hsk <= 576u; + bool v_in_shmem = hsk <= 128u && hsv <= 128u; + bool emf_in_shmem = wg_row / matrix_mnk[1] != sg_per_gw; + bool mask_in_shmem = mask_row <= 64; + size_t shmem_q_sz = q_in_shmem * wg_row * sizeof(ggml_fp16_t); + size_t shmem_oob_staging_sz = sg_per_gw * tm * tk * sizeof(ggml_fp16_t); + size_t shmem_mask_sz = mask_in_shmem ? (mask_row * p_per_iter * sizeof(ggml_fp16_t)) : 0; + size_t shmem_k_sz = k_in_shmem ? (k_loops * k_col * wg_sz * sizeof(ggml_fp16_t)) : shmem_oob_staging_sz; + size_t shmem_v_sz = v_in_shmem ? (hsv * p_per_iter * sizeof(ggml_fp16_t)) : 0; + size_t shmem_p_sz = wg_row * p_per_iter * sizeof(float); + size_t shmem_emf_sz = emf_in_shmem ? (wg_row * sizeof(float)) : 0; + size_t shmem_pv_sz = intel_gpu ? 0 : (sg_per_gw * tm * tn * sizeof(float)); + + // Reuse section sits at tail of shmem, reused with k + mask/kv oob staging/pv staging + size_t shmem_reuse_sz = shmem_k_sz + shmem_mask_sz; + // When v is not in shmem, its staging reuses the k + mask region; size the reuse section to the larger of the two + shmem_reuse_sz = v_in_shmem ? shmem_reuse_sz : std::max(shmem_reuse_sz, shmem_oob_staging_sz); + shmem_reuse_sz = std::max(shmem_pv_sz, shmem_reuse_sz); + + // Q shmem size check for prologue staging is safe to bypass: + // 1. Prologue q staging shmem is released right after prologue, so it never overlaps other shmem sections + // 2. Extreme large hsk one-time q staging is capped within shader (max_q_reg_staging = shmem_size / 2 / Wr) + shmem_sz = shmem_q_sz + shmem_reuse_sz + shmem_v_sz + shmem_p_sz + shmem_emf_sz; + + uint32_t sg_col = hsv / tm; + uint32_t pipeline_flag = 0; + sg_col = (sg_col + output_div - 1) / output_div; + sg_col = sg_col * tm; + if (pipeline_state.unaligned_seq_len) { + pipeline_flag |= 1 << 0; + } + + if (pipeline_state.unaligned_q) { + pipeline_flag |= 1 << 1; + } + + if (pipeline_state.partial_row_mask) { + pipeline_flag |= 1 << 2; + } + + if (intel_gpu) { + pipeline_flag |= 1 << 3; + } + + params.workgroup_size = { wg_sz }; + params.subgroup_size = { sg_sz }; + params.shmem_sz = shmem_sz; + params.prefill_params.pipeline_flag = pipeline_flag; + params.prefill_params.wg_row = wg_row; + params.prefill_params.gqa_fold = gqa_fold; + params.prefill_params.q_in_reg = q_in_reg; + params.prefill_params.p_per_iter = p_per_iter; + params.prefill_params.sg_col = sg_col; + params.prefill_params.acc_count = tm == 8 ? (sg_col % 16 == 0 ? 2 : 1) : 1; + } else { + auto upper_power_of_2 = [&](uint32_t in) { + GGML_ASSERT(in != 0); + if (in <= 1) return 1u; + uint32_t ret = in - 1; + ret |= ret >> 1; + ret |= ret >> 2; + ret |= ret >> 4; + ret |= ret >> 8; + ret |= ret >> 16; + return ret + 1; + }; + + size_t shmem_ph1_sz = 0; + size_t shmem_ph2_sz = 0; + + // Phase 1 shmem size calculation + const uint32_t p_chunk_ph1 = 32; + // Phase 1 k shmem staging for extreme large hsk is capped within shader, bypass that check + uint32_t pmat_row = CEIL_DIV(qk_ratio, tn); + uint32_t aligned_q_len = upper_power_of_2(pipeline_state.decode_n); + uint32_t group_sz_ph1 = std::min(std::max(aligned_q_len * sg_sz, 64u), 256u); + uint32_t out_per_wg_ph1 = std::min(pipeline_state.decode_n, 256u / sg_sz); + size_t shmem_k_sz = group_sz_ph1 / sg_sz * tm * tk * sizeof(ggml_fp16_t); + size_t shmem_mask_sz = out_per_wg_ph1 * p_chunk_ph1 * sizeof(float); + size_t shmem_p_sz = p_chunk_ph1 * pmat_row * tn * sizeof(float); + // K shmem is reused with mask/p, calculate the max size + shmem_ph1_sz = std::max(shmem_k_sz, shmem_mask_sz + shmem_p_sz); + + // Phase 2 shmem size calculation + uint32_t aligned_pv_ratio = upper_power_of_2(pv_ratio); + uint32_t split_p_per_iter_ph2 = (pv_ratio <= 32) ? 256 : 128; + uint32_t out_col_per_wg_ph2 = intel_gpu ? (pv_ratio > 16 ? 8 : 16) : 16; + uint32_t split_p_per_warp = 16; + uint32_t group_sz_ph2 = (split_p_per_iter_ph2 / split_p_per_warp) * sg_sz; + sg_per_gw = group_sz_ph2 / sg_sz; + uint32_t out_row_per_wg_ph2 = std::min(std::max(16u / aligned_pv_ratio, 1u), pipeline_state.decode_n); + uint32_t o_row = pv_ratio * out_row_per_wg_ph2; + uint32_t omat_row = CEIL_DIV(o_row, tn) * tn; + uint32_t global_max_loops = CEIL_DIV(o_row, sg_per_gw); + size_t shmem_global_max_sz = CEIL_DIV(global_max_loops * sg_per_gw, sg_sz) * sg_sz * sizeof(float); + + uint32_t size_max = o_row * split_p_per_iter_ph2 / p_chunk_ph1; + size_t shmem_group_max_sz = CEIL_DIV(size_max, group_sz_ph2) * group_sz_ph2 * sizeof(float); + + uint32_t p_loop_col = group_sz_ph2 * 4; + uint32_t p_loops = CEIL_DIV(o_row * split_p_per_iter_ph2, p_loop_col); + shmem_p_sz = std::max(p_loop_col * p_loops * sizeof(ggml_fp16_t), omat_row * split_p_per_iter_ph2 * sizeof(ggml_fp16_t)); + + size_t shmem_lf_sz = CEIL_DIV(sg_per_gw * p_loops, sg_sz) * sg_sz * sizeof(float); + size_t shmem_oob_staging_sz = sg_per_gw * tm * tk * sizeof(ggml_fp16_t); + size_t shmem_o_sz = sg_per_gw * out_col_per_wg_ph2 * omat_row * sizeof(float); + size_t shmem_reuse_sz = 2 * shmem_group_max_sz + std::max(shmem_oob_staging_sz, 2 * shmem_p_sz); + + shmem_ph2_sz = shmem_lf_sz + shmem_global_max_sz; + // Shmem of p/group max is reused with output accumulation, calculate the max size + shmem_ph2_sz += std::max(shmem_reuse_sz, shmem_o_sz); + shmem_sz = std::max(shmem_ph1_sz, shmem_ph2_sz); + + params.workgroup_size = { group_sz_ph1, group_sz_ph2 }; + params.subgroup_size = { sg_sz, sg_sz }; + params.shmem_sz = shmem_sz; + params.decode_params.p_chunk_ph1 = p_chunk_ph1; + params.decode_params.out_per_wg_ph1 = out_per_wg_ph1; + params.decode_params.split_p_per_iter_ph2 = split_p_per_iter_ph2; + params.decode_params.out_row_per_wg_ph2 = out_row_per_wg_ph2; + params.decode_params.out_col_per_wg_ph2 = out_col_per_wg_ph2; + params.decode_params.pipeline_flag_ph2 = pipeline_state.unaligned_seq_len ? (1u << 0) : 0; + } + + if (shmem_sz > device->properties.limits.maxComputeSharedMemorySize) { + VK_LOG_DEBUG("get_xe_fa_tuning_params(HSK=" << hsk << ", HSV=" << hsv << ", qk_ratio=" << qk_ratio << ", pv_ratio=" << pv_ratio << "), " + "shmem_sz = " << shmem_sz << ", exceeding max shmem size = " << device->properties.limits.maxComputeSharedMemorySize); + } + + return params; +} + static bool ggml_vk_matmul_shmem_support(const vk_device& device, const std::vector& warptile, bool mul_mat_id, ggml_type src0_type) { uint32_t lut_size = 0; @@ -1457,8 +1645,8 @@ static bool ggml_vk_matmul_shmem_support(const vk_device& device, const std::vec lut_size = 4*16; break; case GGML_TYPE_NVFP4: - // Same kvalues budget as MXFP4 plus ue4m3_fp32_lut[128] (types.glsl, DATA_A_NVFP4). - lut_size = 4*16 + 128u * (uint32_t)sizeof(float); + // Same kvalues budget as MXFP4 plus ue4m3_fp32_lut[256] (types.glsl, DATA_A_NVFP4). + lut_size = 4*16 + 256u * (uint32_t)sizeof(float); break; default: break; @@ -1592,7 +1780,7 @@ static bool ggml_vk_matmul_cm1_int_shmem_support(const vk_device& device, const total += 16u * (uint32_t)sizeof(int8_t); // cm1_kvalues[16] } if (src0_type == GGML_TYPE_NVFP4 && !device->ocp_fp4) { - total += 128u * (uint32_t)sizeof(float); // ue4m3_fp32_lut[128] + total += 256u * (uint32_t)sizeof(float); // ue4m3_fp32_lut[256] } if (mul_mat_id) { total += BN * 2u * (uint32_t)sizeof(uint16_t); // row_ids[BN] (u16vec2) @@ -1678,6 +1866,17 @@ static bool ggml_vk_fa_scalar_uses_mmq(const vk_device& device, ggml_type k_type #endif } +// AMD proprietary drivers before 2.0.416 (Adrenalin 26.10.1) run the large coopmat matmul tile much slower than the medium one. +static bool ggml_vk_amd_large_tile_supported(const vk_device& device) { + if (device->driver_id != vk::DriverId::eAmdProprietary) { + return true; + } + // AMD encodes driverVersion as major [31:22], build [21:0], not VK_MAKE_API_VERSION. + const uint32_t major = device->properties.driverVersion >> 22; + const uint32_t build = device->properties.driverVersion & 0x3fffff; + return major > 2 || (major == 2 && build >= 416); +} + void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) { VK_LOG_DEBUG("ggml_vk_load_shaders(" << device->name << ")"); @@ -1850,7 +2049,7 @@ void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) { if ((device->architecture == AMD_GCN) && (device->driver_id != vk::DriverId::eAmdProprietary)) { m_warptile_mmq = m_warptile_mmq_int = { 256, 64, 64, 32, 16, 16, 2, 2, 2, 1, 16 }; m_warptile_mmqid = m_warptile_mmqid_int = { 256, 64, 64, 32, 16, 16, 2, 2, 2, 1, 16 }; - } else if (device->vendor_id == VK_VENDOR_ID_AMD && device->coopmat_support && device->driver_id != vk::DriverId::eAmdProprietary) { + } else if (device->vendor_id == VK_VENDOR_ID_AMD && device->coopmat_support && ggml_vk_amd_large_tile_supported(device)) { // This is intentionally using tx_m values, slight performance increase l_warptile = { 256, 128, 128, 16, mm_warp_8, 64, 2, tm_m, tn_m, tk_m, mm_warp_8 }; l_warptile_mmq = l_warptile_mmq_int = { 256, 128, 128, 32, mm_warp_8, 64, 2, tm_m, tn_m, tk_m, mm_warp_8 }; @@ -3183,43 +3382,36 @@ void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) { ggml_vk_create_pipeline(device, device->pipeline_flash_attn_split_k_reduce, "fa_split_k_reduce", fa_split_k_reduce_len, fa_split_k_reduce_data, "main", 3, sizeof(vk_op_flash_attn_split_k_reduce_push_constants), {1, device->subgroup_size, 1}, {device->subgroup_size}, 1, true); #if defined(VK_KHR_cooperative_matrix) && defined(GGML_VULKAN_COOPMAT_GLSLC_SUPPORT) - if (device->vendor_id == VK_VENDOR_ID_INTEL && (device->architecture == INTEL_XE2 || (device->architecture == INTEL_XE1 && device->coopmat_support && device->uma))) { - auto upper_power_of_2 = [&](uint32_t in) { - GGML_ASSERT(in != 0); - if (in <= 1) return 1u; - uint32_t ret = in - 1; - ret |= ret >> 1; - ret |= ret >> 2; - ret |= ret >> 4; - ret |= ret >> 8; - ret |= ret >> 16; - return ret + 1; - }; - - uint32_t xe_native_sub_group_size = 16; - if (device->architecture == INTEL_XE1) { - xe_native_sub_group_size = 8; - } + if (device->xe_fa_support) { + uint32_t shmem_sz = (uint32_t)device->properties.limits.maxComputeSharedMemorySize; + vk_xe_fa_tuning_params xe_fa_params {}; for (auto& it : device->pipeline_xe_fa_decode_dual_phases) { - const uint32_t split_p_chunk = 32; - auto HdQk = it.first; + auto pipeline_state = it.first; auto& pipelines = it.second; - uint32_t head_dim_qk = std::get<0>(HdQk); - uint32_t head_dim_pv = std::get<1>(HdQk); - uint32_t gqa_ratio = std::get<2>(HdQk); - uint32_t q_len = std::get<3>(HdQk); - const uint32_t out_dim_per_wg = gqa_ratio > 16 ? 8 : 16; - uint32_t aligned_q_len = upper_power_of_2(q_len); - uint32_t group_sz_ph1 = std::min(std::max(aligned_q_len * xe_native_sub_group_size, 64u), 256u); - uint32_t out_per_wg_ph1 = std::min(q_len, 256u / xe_native_sub_group_size); - uint32_t aligned_gqa_ratio = upper_power_of_2(gqa_ratio); - uint32_t split_p_per_iter_ph2 = 256; - uint32_t split_p_per_warp = 16; - uint32_t group_sz_ph2 = (split_p_per_iter_ph2 / split_p_per_warp) * xe_native_sub_group_size; - uint32_t out_per_wg_ph2 = std::min(std::max(16u / aligned_gqa_ratio, 1u), q_len); - ggml_vk_create_pipeline(device, pipelines.first, "xe_fa_decode_ph1", fa_decode_ph1_cm1_len, fa_decode_ph1_cm1_data, "main", 5, sizeof(vk_fa_xe_opt_push_constants), { 1, 32, 1 }, { group_sz_ph1, gqa_ratio, head_dim_qk, xe_native_sub_group_size, split_p_chunk, out_per_wg_ph1 }, 1, false, true, xe_native_sub_group_size); - ggml_vk_create_pipeline(device, pipelines.second, "xe_fa_decode_ph2", fa_decode_ph2_cm1_len, fa_decode_ph2_cm1_data, "main", 5, sizeof(vk_fa_xe_opt_push_constants), { 1, 1, 1 }, { group_sz_ph2, gqa_ratio, head_dim_pv, out_per_wg_ph2, xe_native_sub_group_size, split_p_per_iter_ph2, split_p_chunk, out_dim_per_wg }, 1, false, true, xe_native_sub_group_size); + xe_fa_params = get_xe_fa_tuning_params(device.get(), pipeline_state, false); + std::vector sc_ph1 = { xe_fa_params.workgroup_size[0], pipeline_state.qk_ratio, pipeline_state.HSK, + xe_fa_params.subgroup_size[0], xe_fa_params.decode_params.p_chunk_ph1, + xe_fa_params.decode_params.out_per_wg_ph1, xe_fa_params.decode_params.p_chunk_ph1 / xe_fa_params.tm, + xe_fa_params.tm, xe_fa_params.tn, shmem_sz }; + std::vector sc_ph2 = { xe_fa_params.workgroup_size[1], pipeline_state.pv_ratio, pipeline_state.HSV, + xe_fa_params.decode_params.out_row_per_wg_ph2, xe_fa_params.subgroup_size[1], xe_fa_params.decode_params.split_p_per_iter_ph2, + xe_fa_params.decode_params.p_chunk_ph1, xe_fa_params.decode_params.out_col_per_wg_ph2, + xe_fa_params.tm, xe_fa_params.tn, xe_fa_params.decode_params.pipeline_flag_ph2 }; + ggml_vk_create_pipeline(device, pipelines.first, "xe_fa_decode_ph1", fa_decode_ph1_cm1_len, fa_decode_ph1_cm1_data, "main", 5, sizeof(vk_xe_fa_push_constants), { 1, xe_fa_params.decode_params.p_chunk_ph1, 1 }, sc_ph1, 1, false, true, xe_fa_params.subgroup_size[0]); + ggml_vk_create_pipeline(device, pipelines.second, "xe_fa_decode_ph2", fa_decode_ph2_cm1_len, fa_decode_ph2_cm1_data, "main", 5, sizeof(vk_xe_fa_push_constants), { 1, 1, 1 }, sc_ph2, 1, false, true, xe_fa_params.subgroup_size[0]); + } + + for (auto& it : device->pipeline_xe_fa_prefill) { + auto pipeline_state = it.first; + auto& pipelines = it.second; + xe_fa_params = get_xe_fa_tuning_params(device.get(), pipeline_state, true); + + std::vector sc = { xe_fa_params.workgroup_size[0], xe_fa_params.prefill_params.wg_row, xe_fa_params.prefill_params.sg_col, + pipeline_state.HSK, pipeline_state.HSV, pipeline_state.qk_ratio, xe_fa_params.prefill_params.p_per_iter, + xe_fa_params.tm, xe_fa_params.tn, xe_fa_params.prefill_params.q_in_reg, shmem_sz, xe_fa_params.prefill_params.acc_count, + xe_fa_params.prefill_params.pipeline_flag, xe_fa_params.subgroup_size[0], xe_fa_params.prefill_params.gqa_fold }; + ggml_vk_create_pipeline(device, pipelines, "xe_fa_prefill", fa_prefill_cm1_len, fa_prefill_cm1_data, "main", 6, sizeof(vk_xe_fa_push_constants), { xe_fa_params.prefill_params.gqa_fold, xe_fa_params.prefill_params.wg_row / xe_fa_params.prefill_params.gqa_fold, 1 }, sc, 1, false, true, xe_fa_params.subgroup_size[0]); } } #endif @@ -3260,7 +3452,7 @@ void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) { } ggml_vk_create_pipeline(device, device->pipeline_mul_mat_vec_nc_f16_f32, "mul_mat_vec_nc_f16_f32", mul_mat_vec_nc_f16_f32_len, mul_mat_vec_nc_f16_f32_data, "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_nc_push_constants), {1, 1, 1}, {}, 1); - ggml_vk_create_pipeline(device, device->pipeline_norm_f32, "norm_f32", norm_f32_len, norm_f32_data, "main", 2, sizeof(vk_op_unary_push_constants), {1, 1, 1}, {}, 1); + ggml_vk_create_pipeline(device, device->pipeline_norm_f32, "norm_f32", norm_f32_len, norm_f32_data, "main", 2, sizeof(vk_op_unary_push_constants), {1, 1, 1}, {}, 1, true); ggml_vk_create_pipeline(device, device->pipeline_group_norm_f32, "group_norm_f32", group_norm_f32_len, group_norm_f32_data, "main", 2, sizeof(vk_op_push_constants), {1, 1, 1}, {}, 1); const bool rms_subgroups = (device->vendor_id == VK_VENDOR_ID_INTEL || device->vendor_id == VK_VENDOR_ID_NVIDIA || device->vendor_id == VK_VENDOR_ID_AMD) && device->subgroup_arithmetic && device->subgroup_require_full_support; @@ -3293,7 +3485,7 @@ void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) { #undef RMS_SHADER ggml_vk_create_pipeline(device, device->pipeline_rms_norm_back_f32, "rms_norm_back_f32", rms_norm_back_f32_len, rms_norm_back_f32_data, "main", 3, sizeof(vk_op_push_constants), {1, 1, 1}, {}, 1); - ggml_vk_create_pipeline(device, device->pipeline_l2_norm_f32, "l2_norm_f32", l2_norm_f32_len, l2_norm_f32_data, "main", 2, sizeof(vk_op_unary_push_constants), {1, 1, 1}, {}, 1); + ggml_vk_create_pipeline(device, device->pipeline_l2_norm_f32, "l2_norm_f32", l2_norm_f32_len, l2_norm_f32_data, "main", 2, sizeof(vk_op_unary_push_constants), {1, 1, 1}, {}, 1, true); ggml_vk_create_pipeline(device, device->pipeline_cpy_32, "cpy_32", cpy_32_len, cpy_32_data, "main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1); ggml_vk_create_pipeline(device, device->pipeline_cpy_f32_f16, "cpy_f32_f16", cpy_f32_f16_len, cpy_f32_f16_data, "main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1); @@ -3399,6 +3591,9 @@ void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) { ggml_vk_create_pipeline(device, device->pipeline_concat_i8, "concat_i8", concat_i8_len, concat_i8_data, "main", 3, sizeof(vk_op_binary_push_constants), {512, 1, 1}, {}, 1); ggml_vk_create_pipeline(device, device->pipeline_concat_i16, "concat_i16", concat_i16_len, concat_i16_data, "main", 3, sizeof(vk_op_binary_push_constants), {512, 1, 1}, {}, 1); ggml_vk_create_pipeline(device, device->pipeline_concat_i32, "concat_i32", concat_i32_len, concat_i32_data, "main", 3, sizeof(vk_op_binary_push_constants), {512, 1, 1}, {}, 1); + ggml_vk_create_pipeline(device, device->pipeline_concat_transpose_i8, "concat_transpose_i8", concat_transpose_i8_len, concat_transpose_i8_data, "main", 3, sizeof(vk_op_binary_push_constants), {32, 32, 1}, {}, 1); + ggml_vk_create_pipeline(device, device->pipeline_concat_transpose_i16, "concat_transpose_i16", concat_transpose_i16_len, concat_transpose_i16_data, "main", 3, sizeof(vk_op_binary_push_constants), {32, 32, 1}, {}, 1); + ggml_vk_create_pipeline(device, device->pipeline_concat_transpose_i32, "concat_transpose_i32", concat_transpose_i32_len, concat_transpose_i32_data, "main", 3, sizeof(vk_op_binary_push_constants), {32, 32, 1}, {}, 1); ggml_vk_create_pipeline(device, device->pipeline_concat_i64, "concat_i64", concat_i64_len, concat_i64_data, "main", 3, sizeof(vk_op_binary_push_constants), {512, 1, 1}, {}, 1); ggml_vk_create_pipeline(device, device->pipeline_upscale_nearest_f32, "upscale_f32", upscale_f32_len, upscale_f32_data, "main", 2, sizeof(vk_op_upscale_push_constants), {512, 1, 1}, {GGML_SCALE_MODE_NEAREST}, 1); @@ -3481,10 +3676,6 @@ void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) { CREATE_UNARY_MUL(softplus, 3) #undef CREATE_UNARY_MUL - ggml_vk_create_pipeline(device, device->pipeline_add1_f16_f16, "add1_f16_f16", add1_f16_f16_len, add1_f16_f16_data, "main", 3, sizeof(vk_op_binary_push_constants), {512, 1, 1}, {}, 1); - ggml_vk_create_pipeline(device, device->pipeline_add1_f16_f32, "add1_f16_f32", add1_f16_f32_len, add1_f16_f32_data, "main", 3, sizeof(vk_op_binary_push_constants), {512, 1, 1}, {}, 1); - ggml_vk_create_pipeline(device, device->pipeline_add1_f32_f32, "add1_f32_f32", add1_f32_f32_len, add1_f32_f32_data, "main", 3, sizeof(vk_op_binary_push_constants), {512, 1, 1}, {}, 1); - ggml_vk_create_pipeline(device, device->pipeline_arange_f32, "arange_f32", arange_f32_len, arange_f32_data, "main", 1, sizeof(vk_op_push_constants), {512, 1, 1}, {}, 1); ggml_vk_create_pipeline(device, device->pipeline_fill_f32, "fill_f32", fill_f32_len, fill_f32_data, "main", 1, sizeof(vk_op_push_constants), {512, 1, 1}, {}, 1); @@ -3505,8 +3696,6 @@ void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) { ggml_vk_create_pipeline(device, device->pipeline_silu_back_f32, "silu_back_f32", silu_back_f32_len, silu_back_f32_data, "main", 3, sizeof(vk_op_push_constants), {512, 1, 1}, {}, 1); - ggml_vk_create_pipeline(device, device->pipeline_diag_mask_inf_f32, "diag_mask_inf_f32", diag_mask_inf_f32_len, diag_mask_inf_f32_data, "main", 2, sizeof(vk_op_diag_mask_push_constants), {1, 512, 1}, {}, 1, true); - ggml_vk_create_pipeline(device, device->pipeline_soft_max_f32, "soft_max_f32", soft_max_f32_len, soft_max_f32_data, "main", 4, sizeof(vk_op_soft_max_push_constants), {1, 1, 1}, { device->subgroup_size }, 1); ggml_vk_create_pipeline(device, device->pipeline_soft_max_f32_wg512, "soft_max_f32_wg512", soft_max_f32_len, soft_max_f32_data, "main", 4, sizeof(vk_op_soft_max_push_constants), {1, 1, 1}, { 512 }, 1); ggml_vk_create_pipeline(device, device->pipeline_soft_max_f32_f16, "soft_max_f32_f16", soft_max_f32_f16_len, soft_max_f32_f16_data, "main", 4, sizeof(vk_op_soft_max_push_constants), {1, 1, 1}, { device->subgroup_size }, 1); @@ -3573,7 +3762,7 @@ void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) { ggml_vk_create_pipeline(device, device->pipeline_argmax_f32, "argmax_f32", argmax_f32_len, argmax_f32_data, "main", 2, sizeof(vk_op_push_constants), {1, 1, 1}, { device->subgroup_size }, 1); - ggml_vk_create_pipeline(device, device->pipeline_sum_rows_f32, "sum_rows_f32", sum_rows_f32_len, sum_rows_f32_data, "main", 2, sizeof(vk_op_sum_rows_push_constants), {1, 1, 1}, { device->subgroup_size }, 1); + ggml_vk_create_pipeline(device, device->pipeline_sum_rows_f32, "sum_rows_f32", sum_rows_f32_len, sum_rows_f32_data, "main", 2, sizeof(vk_op_sum_rows_push_constants), {1, 1, 1}, { device->subgroup_size }, 1, true); ggml_vk_create_pipeline(device, device->pipeline_cross_entropy_loss_f32, "cross_entropy_loss_f32", cross_entropy_loss_f32_len, cross_entropy_loss_f32_data, "main", 3, sizeof(vk_op_push_constants), {1, 1, 1}, { device->subgroup_size }, 1); ggml_vk_create_pipeline(device, device->pipeline_cross_entropy_loss_f32_wg512, "cross_entropy_loss_f32_wg512", cross_entropy_loss_f32_len, cross_entropy_loss_f32_data, "main", 3, sizeof(vk_op_push_constants), {1, 1, 1}, { 512 }, 1); ggml_vk_create_pipeline(device, device->pipeline_cross_entropy_loss_back_f32, "cross_entropy_loss_back_f32", cross_entropy_loss_back_f32_len, cross_entropy_loss_back_f32_data, "main", 4, sizeof(vk_op_push_constants), {1, 1, 1}, { device->subgroup_size }, 1); @@ -3581,21 +3770,31 @@ void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) { // Intel Windows driver in range [32.0.101.8509, 32.0.101.8860) will crash when using fwht kernels so we gate that here const bool can_use_fwht = device->driver_id != vk::DriverId::eIntelProprietaryWindows || !ggml_vk_intel_windows_driver_in_range(device->properties.driverVersion, 101, 8509, 101, 8860); - if (can_use_fwht && device->subgroup_basic && device->subgroup_shuffle) { + if (can_use_fwht) { + const bool use_subgroup = device->subgroup_basic && device->subgroup_shuffle; + const uint32_t sg = std::max(device->subgroup_size, 1u); int idx = 0; - for (uint32_t n : {64, 128, 256, 512}) { - if (device->subgroup_size <= n) { - ggml_vk_create_pipeline(device, device->pipeline_fwht_f32[idx], "fwht_f32", fwht_f32_len, fwht_f32_data, "main", 2, sizeof(vk_op_fwht_push_constants), {1, 1, 1}, { device->subgroup_size, n }, 1, true, true, device->subgroup_size); + for (uint32_t n : {64, 128, 256, 512, 1024, 2048, 4096, 8192}) { + const bool wide = n > fwht_max_subgroup_n || n / sg > fwht_max_subgroup_el_w; + if (use_subgroup && !wide) { + if (device->subgroup_size <= n) { + ggml_vk_create_pipeline(device, device->pipeline_fwht_f32[idx], "fwht_f32", fwht_f32_len, fwht_f32_data, "main", 2, sizeof(vk_op_fwht_push_constants), {1, 1, 1}, { device->subgroup_size, n, fwht_rows }, 1, true, true, device->subgroup_size); + ggml_vk_create_pipeline(device, device->pipeline_fwht_f16[idx], "fwht_f16", fwht_f16_len, fwht_f16_data, "main", 2, sizeof(vk_op_fwht_push_constants), {1, 1, 1}, { device->subgroup_size, n, fwht_rows }, 1, true, true, device->subgroup_size); + device->fwht_rows_per_wg[idx] = fwht_rows; + } + } else { + // wide blocks trade rows per workgroup for a smaller register block + const uint32_t block_size = wide ? std::min(n, 256u) : std::min(sg, n); + const uint32_t rows = wide ? 1u : fwht_rows; + if ((uint64_t)rows * n * sizeof(float) <= device->properties.limits.maxComputeSharedMemorySize && + block_size * rows <= device->properties.limits.maxComputeWorkGroupInvocations) { + ggml_vk_create_pipeline(device, device->pipeline_fwht_f32[idx], "fwht_shmem_f32", fwht_shmem_f32_len, fwht_shmem_f32_data, "main", 2, sizeof(vk_op_fwht_push_constants), {1, 1, 1}, { block_size, n, rows }, 1); + ggml_vk_create_pipeline(device, device->pipeline_fwht_f16[idx], "fwht_shmem_f16", fwht_shmem_f16_len, fwht_shmem_f16_data, "main", 2, sizeof(vk_op_fwht_push_constants), {1, 1, 1}, { block_size, n, rows }, 1); + device->fwht_rows_per_wg[idx] = rows; + } } ++idx; } - } else if (can_use_fwht) { - int idx = 0; - for (uint32_t n : {64, 128, 256, 512}) { - const uint32_t block_size = std::min(device->subgroup_size, n); - ggml_vk_create_pipeline(device, device->pipeline_fwht_f32[idx], "fwht_shmem_f32", fwht_shmem_f32_len, fwht_shmem_f32_data, "main", 2, sizeof(vk_op_fwht_push_constants), {1, 1, 1}, { block_size, n }, 1); - ++idx; - } } const uint32_t cumsum_elem_per_thread = (device->vendor_id == VK_VENDOR_ID_AMD || device->vendor_id == VK_VENDOR_ID_INTEL) ? 2 : 4; @@ -4020,6 +4219,24 @@ static bool ggml_vk_host_buffer_import_usable(uint32_t vendor_id) { return vendor_id != VK_VENDOR_ID_AMD; } +static bool ggml_vk_has_rebar(const vk::PhysicalDevice & physical_device) { + const vk::PhysicalDeviceMemoryProperties mem_props = physical_device.getMemoryProperties(); + const vk::MemoryPropertyFlags hvv = vk::MemoryPropertyFlagBits::eDeviceLocal | + vk::MemoryPropertyFlagBits::eHostVisible | + vk::MemoryPropertyFlagBits::eHostCoherent; + uint32_t vram_heap = UINT32_MAX; + for (uint32_t i = 0; i < mem_props.memoryTypeCount; i++) { + const vk::MemoryType & t = mem_props.memoryTypes[i]; + if (vram_heap == UINT32_MAX && (t.propertyFlags & vk::MemoryPropertyFlagBits::eDeviceLocal)) { + vram_heap = t.heapIndex; + } + if ((t.propertyFlags & hvv) == hvv && t.heapIndex == vram_heap) { + return true; + } + } + return false; +} + vk_device ggml_vk_get_device(size_t idx) { VK_LOG_DEBUG("ggml_vk_get_device(" << idx << ")"); @@ -4075,6 +4292,7 @@ vk_device ggml_vk_get_device(size_t idx) { bool dot2_f16_support = false; bool ocp_microscaling_extension = false; bool shader_float8_extension = false; + bool workgroup_memory_explicit_layout_support = false; for (const auto& properties : ext_props) { if (strcmp("VK_KHR_maintenance4", properties.extensionName) == 0) { @@ -4130,6 +4348,10 @@ vk_device ggml_vk_get_device(size_t idx) { dot2_f16_support = true; } else if (strcmp("VK_KHR_pipeline_executable_properties", properties.extensionName) == 0) { pipeline_executable_properties_support = true; +#if defined(VK_KHR_workgroup_memory_explicit_layout) + } else if (strcmp("VK_KHR_workgroup_memory_explicit_layout", properties.extensionName) == 0) { + workgroup_memory_explicit_layout_support = true; +#endif } else if (strcmp("VK_EXT_memory_priority", properties.extensionName) == 0 && getenv("GGML_VK_ENABLE_MEMORY_PRIORITY")) { device->memory_priority = true; @@ -4254,6 +4476,7 @@ vk_device ggml_vk_get_device(size_t idx) { device->subgroup_size = subgroup_props.subgroupSize; device->subgroup_size_log2 = uint32_t(log2f(float(device->subgroup_size))); device->uma = device->properties.deviceType == vk::PhysicalDeviceType::eIntegratedGpu; + device->rebar = ggml_vk_has_rebar(device->physical_device); if (sm_builtins) { device->shader_core_count = sm_props.shaderSMCount; } else if (amd_shader_core_properties2) { @@ -4481,6 +4704,16 @@ vk_device ggml_vk_get_device(size_t idx) { device_extensions.push_back("VK_KHR_pipeline_executable_properties"); } +#if defined(VK_KHR_workgroup_memory_explicit_layout) + VkPhysicalDeviceWorkgroupMemoryExplicitLayoutFeaturesKHR wg_explicit_layout_features {}; + wg_explicit_layout_features.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_WORKGROUP_MEMORY_EXPLICIT_LAYOUT_FEATURES_KHR; + if (workgroup_memory_explicit_layout_support) { + last_struct->pNext = (VkBaseOutStructure *)&wg_explicit_layout_features; + last_struct = (VkBaseOutStructure *)&wg_explicit_layout_features; + device_extensions.push_back("VK_KHR_workgroup_memory_explicit_layout"); + } +#endif + if (device->external_memory_host) { device_extensions.push_back("VK_EXT_external_memory_host"); } @@ -4542,6 +4775,16 @@ vk_device ggml_vk_get_device(size_t idx) { shader_float8_extension && shader_float8_features.shaderFloat8 && !getenv("GGML_VK_DISABLE_OCP_FP4"); +#if defined(VK_KHR_workgroup_memory_explicit_layout) + device->workgroup_memory_explicit_layout = workgroup_memory_explicit_layout_support && + wg_explicit_layout_features.workgroupMemoryExplicitLayout; + device->workgroup_memory_explicit_layout_16bit = device->workgroup_memory_explicit_layout && + wg_explicit_layout_features.workgroupMemoryExplicitLayout16BitAccess; +#else + device->workgroup_memory_explicit_layout = false; + device->workgroup_memory_explicit_layout_16bit = false; +#endif + device->pipeline_robustness = pl_robustness_features.pipelineRobustness; device->multi_add = vk12_props.shaderRoundingModeRTEFloat16 && @@ -4816,7 +5059,7 @@ vk_device ggml_vk_get_device(size_t idx) { switch (device->vendor_id) { #ifndef GGML_VULKAN_RUN_TESTS case VK_VENDOR_ID_AMD: - device->mul_mat_l[i] = device->coopmat_support && device->driver_id != vk::DriverId::eAmdProprietary; + device->mul_mat_l[i] = device->coopmat_support && ggml_vk_amd_large_tile_supported(device); device->mul_mat_m[i] = true; device->mul_mat_s[i] = true; device->mul_mat_id_l[i] = false; @@ -4907,6 +5150,13 @@ vk_device ggml_vk_get_device(size_t idx) { descriptor_set_layout_create_info.setPNext(&dslbfci); device->dsl = device->device.createDescriptorSetLayout(descriptor_set_layout_create_info); +#if defined(VK_KHR_cooperative_matrix) && defined(GGML_VULKAN_COOPMAT_GLSLC_SUPPORT) + device->xe_fa_support = device->coopmat_support && device->properties.limits.maxComputeSharedMemorySize >= 48 * 1024 && + device->workgroup_memory_explicit_layout && device->workgroup_memory_explicit_layout_16bit && + ((device->vendor_id == VK_VENDOR_ID_INTEL && (device->architecture == INTEL_XE2 || (device->architecture == INTEL_XE1 && device->uma))) || + (device->subgroup_size == 32 && device->coopmat_support_16x16x16_f32acc && (getenv("GGML_VK_FORCE_XE_FA") != nullptr))); +#endif + ggml_vk_load_shaders(device); // Prefer a dedicated transfer queue on AMD dGPUs (non-GCN) when graphics queue use is disabled. @@ -4943,8 +5193,7 @@ vk_device ggml_vk_get_device(size_t idx) { device->disable_descriptor_reuse = getenv("GGML_VK_DISABLE_DESCRIPTOR_REUSE") != nullptr; - device->add_rms_fusion = !device->disable_fusion && - device->subgroup_arithmetic && + device->add_rms_fusion = device->subgroup_arithmetic && device->vendor_id != VK_VENDOR_ID_INTEL; device->partials_binding_alignment = std::max(4u, (uint32_t)device->properties.limits.minStorageBufferOffsetAlignment); @@ -5156,6 +5405,7 @@ static void ggml_vk_print_gpu_info(size_t idx) { uint32_t default_subgroup_size = get_subgroup_size("", device_architecture); const size_t subgroup_size = (default_subgroup_size != 0) ? default_subgroup_size : subgroup_props.subgroupSize; const bool uma = props2.properties.deviceType == vk::PhysicalDeviceType::eIntegratedGpu; + const bool rebar = ggml_vk_has_rebar(physical_device); integer_dot_product = integer_dot_product && shader_integer_dot_product_props.integerDotProduct4x8BitPackedSignedAccelerated @@ -5202,8 +5452,8 @@ static void ggml_vk_print_gpu_info(size_t idx) { #endif std::string device_name = props2.properties.deviceName.data(); - GGML_LOG_DEBUG("ggml_vulkan: %zu = %s (%s) | uma: %d | fp16: %s | bf16: %d | fp4: %d | warp size: %zu | shared memory: %d | int dot: %d | matrix cores: %s\n", - idx, device_name.c_str(), driver_props.driverName.data(), uma, fp16_str, bf16, fp4, subgroup_size, + GGML_LOG_DEBUG("ggml_vulkan: %zu = %s (%s) | uma: %d | rebar: %d | fp16: %s | bf16: %d | fp4: %d | warp size: %zu | shared memory: %d | int dot: %d | matrix cores: %s\n", + idx, device_name.c_str(), driver_props.driverName.data(), uma, rebar, fp16_str, bf16, fp4, subgroup_size, props2.properties.limits.maxComputeSharedMemorySize, integer_dot_product, matrix_cores.c_str()); if (props2.properties.deviceType == vk::PhysicalDeviceType::eCpu) { @@ -5955,6 +6205,11 @@ static uint32_t ggml_vk_guess_split_k(ggml_backend_vk_context * ctx, uint32_t m, split_k = ctx->device->shader_core_count / (m_tiles * n_tiles); } else if (m_tiles * n_tiles <= ctx->device->shader_core_count * 2 / 3) { split_k = 3; + } else if ((ctx->device->architecture == AMD_RDNA3 || ctx->device->architecture == AMD_RDNA4) && + m_tiles * n_tiles > ctx->device->shader_core_count && + m_tiles * n_tiles <= ctx->device->shader_core_count * 4 / 3) { + // two waves, the second one at most 1/3 full + split_k = 3; } // Cap the split at 8x. Unless k is huge this is a lot of overhead. split_k = std::min(split_k, 8u); @@ -7200,6 +7455,10 @@ static int ggml_vk_fwht_pipeline_idx(int64_t n) { case 128: return 1; case 256: return 2; case 512: return 3; + case 1024: return 4; + case 2048: return 5; + case 4096: return 6; + case 8192: return 7; default: return -1; } } @@ -7213,12 +7472,18 @@ bool ggml_vk_can_use_fwht(const ggml_backend_vk_context * ctx, const ggml_tensor return false; } - const int idx = ggml_vk_fwht_pipeline_idx(src1->ne[0]); - if (idx < 0 || ctx->device->pipeline_fwht_f32[idx] == nullptr) { + if (dst->type != GGML_TYPE_F32) { return false; } - if (src1->type != GGML_TYPE_F32 || dst->type != GGML_TYPE_F32) { + if (src1->type != GGML_TYPE_F32 && src1->type != GGML_TYPE_F16) { + return false; + } + + const bool src_f16 = src1->type == GGML_TYPE_F16; + + const int idx = ggml_vk_fwht_pipeline_idx(src1->ne[0]); + if (idx < 0 || (src_f16 ? ctx->device->pipeline_fwht_f16[idx] : ctx->device->pipeline_fwht_f32[idx]) == nullptr) { return false; } @@ -7232,9 +7497,10 @@ bool ggml_vk_can_use_fwht(const ggml_backend_vk_context * ctx, const ggml_tensor void ggml_vk_fwht(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src, ggml_tensor * dst) { const int idx = ggml_vk_fwht_pipeline_idx(src->ne[0]); - vk_pipeline pipeline = ctx->device->pipeline_fwht_f32[idx]; + vk_pipeline pipeline = src->type == GGML_TYPE_F16 ? ctx->device->pipeline_fwht_f16[idx] : ctx->device->pipeline_fwht_f32[idx]; - const uint32_t rows_per_workgroup = 4; + const uint32_t rows_per_workgroup = ctx->device->fwht_rows_per_wg[idx]; + GGML_ASSERT(rows_per_workgroup > 0); const uint32_t n_rows = (uint32_t)ggml_nrows(src); const uint32_t max_workgroups_x = ctx->device->properties.limits.maxComputeWorkGroupCount[0]; @@ -8221,18 +8487,25 @@ void ggml_vk_flash_attn(ggml_backend_vk_context * ctx, vk_context& subctx, const v->nb[1] != (uint64_t)HSV * ggml_type_size(v->type)) && (HSK % 8) == 0 && (HSV % 8) == 0; const size_t kv_scratch_ts = kv_raw_strided ? ggml_type_size(k->type) : sizeof(ggml_fp16_t); + // K and V share one scratch buffer, so the sum must fit in one allocation + const uint64_t dequant_kv_sz = ((uint64_t)ggml_nelements(k) + (uint64_t)ggml_nelements(v)) * kv_scratch_ts; + const uint32_t cm2_min_queries = KV >= 8192 ? 512 : 1024; + const uint32_t cm2_min_reuse = KV >= 8192 ? 4096 : 8192; + // Query reuse must cover the conversion and synchronization cost. + const bool cm2_dequant_kv = KV >= 4096 && neq1 >= cm2_min_queries && (uint64_t) neq1 * qk_ratio >= cm2_min_reuse; const bool use_dequant_kv = ((k_quant && v_quant) || kv_raw_strided) && neq1 >= 64 && is_dense_kv_cache(k) && is_dense_kv_cache(v) && (uint64_t)ggml_nelements(k) * kv_scratch_ts <= ctx->device->properties.limits.maxStorageBufferRange && (uint64_t)ggml_nelements(v) * kv_scratch_ts <= ctx->device->properties.limits.maxStorageBufferRange && + dequant_kv_sz <= ctx->device->max_buffer_size && + dequant_kv_sz <= ctx->device->max_memory_allocation_size && (kv_raw_strided || (ctx->device->pipeline_dequant_transpose[k->type] != nullptr && ctx->device->pipeline_dequant_transpose[v->type] != nullptr)) && - // coopmat2 path does not benefit from the f16 scratch - !ctx->device->coopmat2 && // Intel Xe1 regresses, see PR 25494 (ctx->device->vendor_id != VK_VENDOR_ID_INTEL || - (ctx->device->coopmat_support && ctx->device->architecture != vk_device_architecture::INTEL_XE1)); + (ctx->device->coopmat_support && ctx->device->architecture != vk_device_architecture::INTEL_XE1)) && + (!ctx->device->coopmat2 || cm2_dequant_kv); const ggml_type k_type_eff = (use_dequant_kv && !kv_raw_strided) ? GGML_TYPE_F16 : k->type; const ggml_type v_type_eff = (use_dequant_kv && !kv_raw_strided) ? GGML_TYPE_F16 : v->type; @@ -8285,6 +8558,7 @@ void ggml_vk_flash_attn(ggml_backend_vk_context * ctx, vk_context& subctx, const (gqa_ratio > 1 || (tuning_params.path == FA_SCALAR && N == 1)); const uint32_t q_stride = (uint32_t)(nbq1 / ggml_type_size(q->type)); + const uint32_t q_stride_eff = gqa_ratio > 1 ? (uint32_t)(nbq2 / ggml_type_size(q->type)) : q_stride; uint32_t k_stride = (uint32_t)(nbk1 / ggml_type_size(k->type)); uint32_t v_stride = (uint32_t)(nbv1 / ggml_type_size(v->type)); @@ -8307,9 +8581,13 @@ void ggml_vk_flash_attn(ggml_backend_vk_context * ctx, vk_context& subctx, const nbv3_eff = (uint32_t)((uint64_t)HSV * KV * nev2 * sizeof(ggml_fp16_t)); } const uint32_t alignment = tuning_params.block_cols; - bool aligned = (KV % alignment) == 0 && - // the "aligned" shader variant will forcibly align strides, for performance - (q_stride & 7) == 0 && (k_stride & 7) == 0 && (v_stride & 7) == 0; + // CM2 decoded K/V strides count whole blocks and do not need to be multiples of eight blocks. + const bool cm2_decode_k = tuning_params.path == FA_COOPMAT2 && + k_type_eff != GGML_TYPE_F16 && k_type_eff != GGML_TYPE_BF16; + const bool cm2_decode_v = tuning_params.path == FA_COOPMAT2 && + v_type_eff != GGML_TYPE_F16 && v_type_eff != GGML_TYPE_BF16; + bool aligned = (KV % alignment) == 0 && (q_stride_eff & 7) == 0 && + (cm2_decode_k || (k_stride & 7) == 0) && (cm2_decode_v || (v_stride & 7) == 0); // Need to use the coopmat2 variant that clamps loads when HSK/HSV aren't sufficiently aligned. if (((HSK | HSV) % 16) != 0 && tuning_params.path == FA_COOPMAT2) { @@ -8324,32 +8602,98 @@ void ggml_vk_flash_attn(ggml_backend_vk_context * ctx, vk_context& subctx, const vk_pipeline pipeline = nullptr; + auto tensor_aligned_with = [&](const ggml_tensor * t, uint32_t in) { + GGML_ASSERT(t != nullptr && in != 0); + return t->nb[1] % in == 0 && t->nb[2] % in == 0 && t->nb[3] % in == 0 && ggml_vk_tensor_buffer_offset(ctx, t) % in == 0; + }; + auto vk_dev = ctx->device.get(); bool xe_fa_opt = false; - bool fa_copy_qstate = false; - bool xe_fa_supported_platform = - (ctx->device.get()->architecture == INTEL_XE2 && ctx->device.get()->properties.deviceID != 0xFD80 && ctx->device.get()->properties.deviceID != 0xFD81) || - (ctx->device.get()->architecture == INTEL_XE1 && ctx->device.get()->coopmat_support && ctx->device.get()->uma); - bool xe_fa_supported_usage = neq0 % 32 == 0 && nev0 % 16 == 0 && q->nb[1] > q->nb[2] && k->nb[1] > k->nb[2] && v->nb[1] > v->nb[2] && mask != nullptr; - bool xe_fa_supported_dtype = q->type == GGML_TYPE_F32 && k->type == GGML_TYPE_F16 && v->type == GGML_TYPE_F16 && (mask != nullptr && mask->type == GGML_TYPE_F16); + bool xe_fa_use_2phases = false; + bool xe_fa_supported_platform = vk_dev->xe_fa_support && vk_dev->properties.deviceID != 0xFD80 && vk_dev->properties.deviceID != 0xFD81; + bool xe_fa_supported_usage = neq0 % 32 == 0 && nev0 % 16 == 0 && mask != nullptr && !use_sparse && max_bias == 0.0f && logit_softcap == 0.0f && + tensor_aligned_with(k, 16u) && tensor_aligned_with(v, 16u); + bool xe_fa_supported_dtype = q->type == GGML_TYPE_F32 && k->type == GGML_TYPE_F16 && v->type == GGML_TYPE_F16 && + mask != nullptr && mask->type == GGML_TYPE_F16; + xe_fa_opt = xe_fa_supported_platform && xe_fa_supported_usage && xe_fa_supported_dtype; std::pair xe_fa_pipeline_dual_phases = { nullptr , nullptr }; vk_pipeline xe_fa_pipeline = nullptr; + size_t size_q_staging = 0; size_t size_p = 0; size_t size_group_max = 0; + size_t size_decode_temp = 0; + uint32_t pv_ratio = uint32_t(neq2 / nev2); + vk_xe_fa_tuning_params xe_fa_params {}; { std::lock_guard guard(ctx->device->compile_mutex); - auto &pipelines = ctx->device->pipeline_flash_attn_f32_f16; - auto it = pipelines.find(fa_pipeline_state); - if (it != pipelines.end()) { - pipeline = it->second; - } else { - pipelines[fa_pipeline_state] = pipeline = std::make_shared(); + if (xe_fa_opt) { + if (neq1 > 8 || pv_ratio > 64) { + uint32_t head_stride_q = uint32_t(nbq2 / ggml_type_size(q->type)); + uint32_t batch_stride_q = uint32_t(nbq3 / ggml_type_size(q->type)); + vk_xe_fa_pipeline_state prefill_state = { (uint32_t)neq0, (uint32_t)nev0, qk_ratio, pv_ratio, 0, false, false, false, false }; + prefill_state.unaligned_seq_len = nek1 % 32 != 0; + prefill_state.unaligned_q = head_stride_q % 8 != 0 || q_stride % 8 != 0 || batch_stride_q % 8 != 0; + prefill_state.partial_row_mask = neq1 > nem1; + prefill_state.multi_head_mask = nem2 > 1; + xe_fa_params = get_xe_fa_tuning_params(vk_dev, prefill_state, true); + if (xe_fa_params.shmem_sz <= vk_dev->properties.limits.maxComputeSharedMemorySize) { + auto& xe_fa_pipelines = ctx->device->pipeline_xe_fa_prefill; + auto it = xe_fa_pipelines.find(prefill_state); + if (it != xe_fa_pipelines.end()) { + xe_fa_pipeline = it->second; + } else { + xe_fa_pipelines[prefill_state] = xe_fa_pipeline = std::make_shared(); + } + } else { + xe_fa_opt = false; + } + } else { + vk_xe_fa_pipeline_state decode_state = { (uint32_t)neq0, (uint32_t)nev0, qk_ratio, pv_ratio, (uint32_t)neq1, false, false, false, false }; + decode_state.unaligned_seq_len = nek1 % 256 != 0; + xe_fa_params = get_xe_fa_tuning_params(vk_dev, decode_state, false); + const uint32_t split_p_size = xe_fa_params.decode_params.p_chunk_ph1; + const size_t max_dim = std::max(CEIL_DIV(nek1, split_p_size), 1); + const size_t p_dim = max_dim * split_p_size; + const size_t q_head_padding = 16; + size_q_staging = neq0 * neq1 * (neq2 * neq3 + q_head_padding) * sizeof(ggml_fp16_t); + size_p = neq1 * neq2 * p_dim * neq3 * sizeof(ggml_fp16_t); + size_group_max = neq1 * neq2 * max_dim * neq3 * sizeof(float); + size_decode_temp = size_q_staging + size_p + size_group_max; + uint32_t ph1_wg1 = CEIL_DIV(nek1, xe_fa_params.decode_params.p_chunk_ph1); + if (xe_fa_params.shmem_sz <= vk_dev->properties.limits.maxComputeSharedMemorySize && + ph1_wg1 <= ctx->device->properties.limits.maxComputeWorkGroupCount[1] && + size_decode_temp <= ctx->device->properties.limits.maxStorageBufferRange) { + auto& pipelines = ctx->device->pipeline_xe_fa_decode_dual_phases; + auto it = pipelines.find(decode_state); + if (it != pipelines.end()) { + xe_fa_pipeline_dual_phases = it->second; + } else { + pipelines[decode_state] = xe_fa_pipeline_dual_phases = + std::make_pair(std::make_shared(), std::make_shared()); + } + xe_fa_use_2phases = true; + } else { + xe_fa_opt = false; + } + } + } + + if (!xe_fa_opt) { + auto &pipelines = ctx->device->pipeline_flash_attn_f32_f16; + auto it = pipelines.find(fa_pipeline_state); + if (it != pipelines.end()) { + pipeline = it->second; + } else { + pipelines[fa_pipeline_state] = pipeline = std::make_shared(); + } } } - assert(pipeline); - // Compile early to initialize wg_denoms. - ggml_pipeline_request_descriptor_sets(ctx, pipeline, 1); + if (!xe_fa_opt) { + assert(pipeline); + // Compile early to initialize wg_denoms. + ggml_pipeline_request_descriptor_sets(ctx, pipeline, 1); + } uint32_t split_kv = KV; uint32_t split_k = 1; @@ -8363,7 +8707,9 @@ void ggml_vk_flash_attn(ggml_backend_vk_context * ctx, vk_context& subctx, const const uint32_t Br = fa_pipeline_state.Br; const uint32_t Bc = fa_pipeline_state.Bc; - GGML_ASSERT(Br == pipeline->wg_denoms[0]); + if (!xe_fa_opt) { + GGML_ASSERT(Br == pipeline->wg_denoms[0]); + } const uint32_t Tr = CEIL_DIV(N, Br); // Try to use split_k when KV is large enough to be worth the overhead. @@ -8393,45 +8739,24 @@ void ggml_vk_flash_attn(ggml_backend_vk_context * ctx, vk_context& subctx, const // of "align", so recompute split_k based on that. split_kv = ROUNDUP_POW2(std::max(1u, KV / split_k), alignment); split_k = CEIL_DIV(KV, split_kv); - xe_fa_opt = xe_fa_supported_platform && xe_fa_supported_usage && xe_fa_supported_dtype; - if (xe_fa_opt) { - const uint32_t split_p_size = 32; - const size_t max_dim = (nek1 + split_p_size - 1) / split_p_size; - const size_t p_dim = max_dim * split_p_size; -#if defined(VK_KHR_cooperative_matrix) && defined(GGML_VULKAN_COOPMAT_GLSLC_SUPPORT) - std::lock_guard guard(ctx->device->compile_mutex); - auto& pipelines = ctx->device->pipeline_xe_fa_decode_dual_phases; - auto it = pipelines.find({ (uint32_t)neq0, (uint32_t)nev0, qk_ratio, (uint32_t)neq1 }); - if (it != pipelines.end()) { - xe_fa_pipeline_dual_phases = it->second; - } else { - pipelines[{(uint32_t)neq0, (uint32_t)nev0, qk_ratio, (uint32_t)neq1}] = xe_fa_pipeline_dual_phases = std::make_pair(std::make_shared(), std::make_shared()); + } + + if (xe_fa_opt) { + // Bypass mask_opt and split_k buffer grow when they are not in use + use_mask_opt = false; + split_k = 1; + if (xe_fa_use_2phases) { + if (ctx->prealloc_size_x < size_decode_temp) { + ctx->prealloc_size_x = size_decode_temp; + ggml_vk_preallocate_buffers(ctx, subctx); } -#endif - if (xe_fa_pipeline_dual_phases.first == nullptr || xe_fa_pipeline_dual_phases.second == nullptr) { - xe_fa_opt = false; - fa_copy_qstate = false; - } else { - size_p = neq1 * neq2 * p_dim * neq3 * sizeof(ggml_fp16_t); - size_group_max = neq1 * neq2 * max_dim * neq3 * sizeof(float); - size_t temp_size = ggml_nelements(q) * sizeof(ggml_fp16_t) + size_p + size_group_max; - fa_copy_qstate = true; - if (ctx->prealloc_size_x < temp_size) { - ctx->prealloc_size_x = temp_size; - ggml_vk_preallocate_buffers(ctx, subctx); - } - if (ctx->prealloc_x_need_sync) { - ggml_vk_sync_buffers(ctx, subctx); - } + if (ctx->prealloc_x_need_sync) { + ggml_vk_sync_buffers(ctx, subctx); } } } - if (xe_fa_opt == true) { - use_mask_opt = false; - } - // Reserve space for split_k temporaries. For each split x batch, we need to store the O matrix (D x ne1) // and the per-row m and L values (ne1 rows). We store all the matrices first, followed by the rows. // For matrices, the order is (inner to outer) [HSV, ne1, k, ne2, ne3]. @@ -8618,67 +8943,59 @@ void ggml_vk_flash_attn(ggml_backend_vk_context * ctx, vk_context& subctx, const mask_n_head_log2, m0, m1, gqa_ratio, split_kv, split_k }; - if (xe_fa_opt && split_k > 1) { - auto upper_power_of_2 = [&](uint32_t in) { - GGML_ASSERT(in != 0); - if (in <= 1) return 1u; - uint32_t ret = in - 1; - ret |= ret >> 1; - ret |= ret >> 2; - ret |= ret >> 4; - ret |= ret >> 8; - ret |= ret >> 16; - return ret + 1; - }; - auto to_fp16_vk_0 = ggml_vk_get_to_fp16(ctx, q->type); - const uint32_t out_dim_per_wg = qk_ratio > 16 ? 8 : 16; - size_t x_ne = ggml_nelements(q); - size_t temp_buf_offset = 0; + if (xe_fa_opt) { + uint32_t head_stride_q = uint32_t(nbq2 / ggml_type_size(q->type)); uint32_t head_stride_k = uint32_t(nbk2 / ggml_type_size(k->type)); uint32_t head_stride_v = uint32_t(nbv2 / ggml_type_size(v->type)); uint32_t batch_stride_q = uint32_t(nbq3 / ggml_type_size(q->type)); uint32_t batch_stride_k = uint32_t(nbk3 / ggml_type_size(k->type)); uint32_t batch_stride_v = uint32_t(nbv3 / ggml_type_size(v->type)); - uint32_t batch_stride_m = mask ? uint32_t(mask->nb[3] / ggml_type_size(mask->type)) : 0u; uint32_t batch_stride_o = uint32_t(nb3 / ggml_type_size(dst->type)); - vk_fa_xe_opt_push_constants pc_ph1 = { (uint32_t)nek1, (uint32_t)neq1, (uint32_t)neq2, (uint32_t)nek2, qk_ratio, 1, (sinks != nullptr) ? 1u : 0u, (uint32_t)k_stride, head_stride_k, - batch_stride_q, batch_stride_k, batch_stride_v, batch_stride_m, batch_stride_o, scale }; - vk_fa_xe_opt_push_constants pc_ph2 = pc_ph1; - pc_ph2.nbkv_tok = v_stride; - pc_ph2.nbkv_head = head_stride_v; - vk_subbuffer q_temp_buf = fa_copy_qstate ? ggml_vk_subbuffer(ctx, ctx->prealloc_x, temp_buf_offset) : q_buf; - temp_buf_offset += fa_copy_qstate ? x_ne * sizeof(ggml_fp16_t) : 0; - vk_subbuffer p_temp_buf = ggml_vk_subbuffer(ctx, ctx->prealloc_x, temp_buf_offset); - temp_buf_offset += size_p; - vk_subbuffer max_temp_buf = ggml_vk_subbuffer(ctx, ctx->prealloc_x, temp_buf_offset); - temp_buf_offset += size_group_max; - uint32_t xe_native_sub_group_size = ctx->device.get()->architecture == INTEL_XE1 ? 8 : 16; - uint32_t aligned_gqa_ratio = upper_power_of_2(qk_ratio); - uint32_t out_per_wg_ph1 = std::min(256u / xe_native_sub_group_size, (uint32_t)neq1); - uint32_t out_per_wg_ph2 = std::min(std::max(16u / aligned_gqa_ratio, 1u), (uint32_t)neq1); - uint32_t ph1_wg = ((neq1 + out_per_wg_ph1 - 1) / out_per_wg_ph1) * nek2; - uint32_t ph2_wg = ((neq1 + out_per_wg_ph2 - 1) / out_per_wg_ph2) * ne0 / out_dim_per_wg; - if (fa_copy_qstate) { - const std::vector pc_cpy_fp16 = - { (uint32_t)q->ne[0], (uint32_t)q->ne[1], (uint32_t)q->ne[2], (uint32_t)q->ne[3], (uint32_t)(x_ne) }; - ggml_vk_sync_buffers(ctx, subctx); + + vk_xe_fa_push_constants pc_xe_fa = { (uint32_t)neq1, (uint32_t)neq2, (uint32_t)neq3, + (uint32_t)nek1, (uint32_t)nek2, (uint32_t)nek3, (uint32_t)nev2, (uint32_t)nev3, + (uint32_t)nem1, (uint32_t)nem2, (uint32_t)nem3, + q_stride, head_stride_q, k_stride, head_stride_k, v_stride, head_stride_v, + batch_stride_q, batch_stride_k, batch_stride_v, batch_stride_o, + (sinks != nullptr) ? (1u << 16) : 0u, scale }; + + if (xe_fa_use_2phases) { + auto to_fp16_vk_0 = ggml_vk_get_cpy_pipeline(ctx, q, nullptr, GGML_TYPE_F16); + size_t temp_buf_offset = 0; + vk_subbuffer q_temp_buf = ggml_vk_subbuffer(ctx, ctx->prealloc_x, temp_buf_offset); + temp_buf_offset += size_q_staging; + vk_subbuffer p_temp_buf = ggml_vk_subbuffer(ctx, ctx->prealloc_x, temp_buf_offset); + temp_buf_offset += size_p; + vk_subbuffer max_temp_buf = ggml_vk_subbuffer(ctx, ctx->prealloc_x, temp_buf_offset); + temp_buf_offset += size_group_max; + uint32_t ph1_wg = CEIL_DIV(neq1, xe_fa_params.decode_params.out_per_wg_ph1) * nek2; + uint32_t ph2_wg = CEIL_DIV(neq1, xe_fa_params.decode_params.out_row_per_wg_ph2) * ne0 / xe_fa_params.decode_params.out_col_per_wg_ph2; + + pc_xe_fa.nbq1 = uint32_t(neq0); + pc_xe_fa.nbq2 = uint32_t(neq0 * neq1); + pc_xe_fa.nbq3 = uint32_t(neq0 * neq1 * neq2); + ggml_pipeline_request_descriptor_sets(ctx, to_fp16_vk_0, 1); - ggml_vk_dispatch_pipeline(ctx, subctx, to_fp16_vk_0, { q_buf, q_temp_buf }, pc_cpy_fp16, { (uint32_t)(x_ne), 1, 1 }); - } + ggml_vk_cpy_to_contiguous(ctx, subctx, to_fp16_vk_0, q, q_buf, q_temp_buf); - ggml_vk_sync_buffers(ctx, subctx); - ggml_pipeline_request_descriptor_sets(ctx, xe_fa_pipeline_dual_phases.first, 1); - ggml_vk_dispatch_pipeline(ctx, subctx, xe_fa_pipeline_dual_phases.first, - { q_temp_buf, k_buf, mask_buf, p_temp_buf, max_temp_buf }, - pc_ph1, { (uint32_t)ph1_wg, (uint32_t)nek1, (uint32_t)neq3 }); + ggml_pipeline_request_descriptor_sets(ctx, xe_fa_pipeline_dual_phases.first, 1); + ggml_vk_dispatch_pipeline(ctx, subctx, xe_fa_pipeline_dual_phases.first, + { q_temp_buf, k_buf, mask_buf, p_temp_buf, max_temp_buf }, + pc_xe_fa, { (uint32_t)ph1_wg, (uint32_t)nek1, (uint32_t)neq3 }); - ggml_vk_sync_buffers(ctx, subctx); - ggml_pipeline_request_descriptor_sets(ctx, xe_fa_pipeline_dual_phases.second, 1); - ggml_vk_dispatch_pipeline(ctx, subctx, xe_fa_pipeline_dual_phases.second, - { p_temp_buf, v_buf, max_temp_buf, sinks_buf, dst_buf }, - pc_ph2, { (uint32_t)ph2_wg, (uint32_t)nev2, (uint32_t)neq3 }); + ggml_vk_sync_buffers(ctx, subctx); + ggml_pipeline_request_descriptor_sets(ctx, xe_fa_pipeline_dual_phases.second, 1); + ggml_vk_dispatch_pipeline(ctx, subctx, xe_fa_pipeline_dual_phases.second, + { p_temp_buf, v_buf, max_temp_buf, sinks_buf, dst_buf }, + pc_xe_fa, { (uint32_t)ph2_wg, (uint32_t)nev2, (uint32_t)neq3 }); - ctx->prealloc_x_need_sync = true; + ctx->prealloc_x_need_sync = true; + } else { + ggml_pipeline_request_descriptor_sets(ctx, xe_fa_pipeline, 1); + ggml_vk_dispatch_pipeline(ctx, subctx, xe_fa_pipeline, + { q_buf, k_buf, v_buf, mask_buf, sinks_buf, dst_buf }, + pc_xe_fa, { (uint32_t)neq2, (uint32_t)neq1, (uint32_t)neq3 }); + } } else if (split_k > 1) { ggml_pipeline_request_descriptor_sets(ctx, ctx->device->pipeline_flash_attn_split_k_reduce, 1); @@ -8787,6 +9104,26 @@ static vk_pipeline ggml_vk_rms_norm_pipeline(const std::array & return pipelines[2]; } +static bool ggml_vk_concat_is_transposed(const vk_device & device, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * dst) { + if (ggml_get_op_params_i32(dst, 0) != 0) { // dim 0 only + return false; + } + if (src0->ne[2] != 1 || src0->ne[3] != 1 || src1->ne[2] != 1 || src1->ne[3] != 1) { + return false; + } + const size_t ts = ggml_type_size(src0->type); + if (src0->nb[0] != ts || dst->nb[0] != ts) { // src0 and dst rows must be contiguous + return false; + } + if (src1->nb[0] <= src1->nb[1]) { // src1 must actually be transposed + return false; + } + if (CEIL_DIV(src1->ne[1], 32) > device->properties.limits.maxComputeWorkGroupCount[0]) { + return false; + } + return src0->ne[1] == src1->ne[1] && dst->ne[1] == src1->ne[1]; +} + static vk_pipeline ggml_vk_op_get_pipeline(ggml_backend_vk_context * ctx, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * src2, const ggml_tensor * dst, ggml_op op) { switch (op) { case GGML_OP_GET_ROWS: @@ -8879,6 +9216,18 @@ static vk_pipeline ggml_vk_op_get_pipeline(ggml_backend_vk_context * ctx, const if (!ggml_vk_concat_supported(src0, src1, dst)) { return nullptr; } + if (!ggml_is_quantized(src0->type) && ggml_vk_concat_is_transposed(ctx->device, src0, src1, dst)) { + switch (ggml_vk_concat_unit_size(src0->type)) { + case 1: + return ctx->device->pipeline_concat_transpose_i8; + case 2: + return ctx->device->pipeline_concat_transpose_i16; + case 4: + return ctx->device->pipeline_concat_transpose_i32; + default: + break; + } + } switch (ggml_vk_concat_unit_size(src0->type)) { case 1: return ctx->device->pipeline_concat_i8; @@ -9122,11 +9471,6 @@ static vk_pipeline ggml_vk_op_get_pipeline(ggml_backend_vk_context * ctx, const break; } return nullptr; - case GGML_OP_DIAG_MASK_INF: - if (src0->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32) { - return ctx->device->pipeline_diag_mask_inf_f32; - } - return nullptr; case GGML_OP_SOFT_MAX: GGML_ASSERT(!src1 || src1->type == GGML_TYPE_F32 || src1->type == GGML_TYPE_F16); GGML_ASSERT(!src2 || src2->type == GGML_TYPE_F32); @@ -9505,17 +9849,6 @@ static vk_pipeline ggml_vk_op_get_pipeline(ggml_backend_vk_context * ctx, const return pipeline; } return nullptr; - case GGML_OP_ADD1: - if (src0->type == GGML_TYPE_F16 && src1->type == GGML_TYPE_F16 && dst->type == GGML_TYPE_F16) { - return ctx->device->pipeline_add1_f16_f16; - } - if (src0->type == GGML_TYPE_F16 && src1->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F16) { - return ctx->device->pipeline_add1_f16_f32; - } - if (src0->type == GGML_TYPE_F32 && src1->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32) { - return ctx->device->pipeline_add1_f32_f32; - } - return nullptr; case GGML_OP_ARANGE: if (dst->type == GGML_TYPE_F32) { return ctx->device->pipeline_arange_f32; @@ -9646,12 +9979,14 @@ static void ggml_vk_op_f32(ggml_backend_vk_context * ctx, vk_context& subctx, co break; case GGML_OP_GROUP_NORM: { - const uint32_t num_groups = dst->op_params[0]; - elements = { num_groups * (uint32_t)src0->ne[3], 1, 1 }; + const uint32_t total_workgroups = dst->op_params[0] * (uint32_t)src0->ne[3]; + const uint32_t groups_x = std::min(total_workgroups, ctx->device->properties.limits.maxComputeWorkGroupCount[0]); + const uint64_t groups_y_needed = CEIL_DIV(uint64_t(total_workgroups), groups_x); + const uint32_t groups_y = std::min(groups_y_needed, ctx->device->properties.limits.maxComputeWorkGroupCount[1]); + const uint64_t groups_xy = uint64_t(groups_x) * groups_y; + elements = { groups_x, groups_y, uint32_t(CEIL_DIV(uint64_t(total_workgroups), groups_xy)) }; + GGML_ASSERT(elements[2] <= ctx->device->properties.limits.maxComputeWorkGroupCount[2]); } break; - case GGML_OP_DIAG_MASK_INF: - elements = { (uint32_t)ggml_nrows(src0), (uint32_t)ne00, 1 }; - break; case GGML_OP_ROPE: case GGML_OP_ROPE_BACK: { @@ -9785,7 +10120,6 @@ static void ggml_vk_op_f32(ggml_backend_vk_context * ctx, vk_context& subctx, co case GGML_OP_SUB: case GGML_OP_DIV: case GGML_OP_MUL: - case GGML_OP_ADD1: case GGML_OP_OUT_PROD: case GGML_OP_ARANGE: case GGML_OP_FILL: @@ -9811,6 +10145,16 @@ static void ggml_vk_op_f32(ggml_backend_vk_context * ctx, vk_context& subctx, co case GGML_OP_GLU: case GGML_OP_CONV_2D_DW: { + // dispatched per 32x32 tile, not per element + if (op == GGML_OP_CONCAT && + (pipeline == ctx->device->pipeline_concat_transpose_i8 || + pipeline == ctx->device->pipeline_concat_transpose_i16 || + pipeline == ctx->device->pipeline_concat_transpose_i32)) { + const uint32_t tiles_y = CEIL_DIV(dst->ne[0], 32); + const uint32_t groups_y = std::min(tiles_y, ctx->device->properties.limits.maxComputeWorkGroupCount[1]); + elements = { (uint32_t)src1->ne[1], groups_y * 32, CEIL_DIV(tiles_y, groups_y) }; + break; + } uint32_t ne = ggml_nelements(dst); if (op == GGML_OP_CPY && ggml_is_quantized(src0->type) && ggml_is_quantized(dst->type)) { // Convert from number of logical elements to 2- or 4-byte units. @@ -10670,21 +11014,6 @@ void ggml_vk_sqrt(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_ ggml_vk_op_f32(ctx, subctx, src0, nullptr, nullptr, nullptr, dst, GGML_OP_SQRT, vk_op_unary_push_constants_init(src0, dst)); } -void ggml_vk_add1(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { - const uint32_t src0_type_size = ggml_type_size(src0->type); - const uint32_t src1_type_size = ggml_type_size(src1->type); - const uint32_t dst_type_size = ggml_type_size(dst->type); - - ggml_vk_op_f32(ctx, subctx, src0, src1, nullptr, nullptr, dst, GGML_OP_ADD1, { - (uint32_t)ggml_nelements(src0), - (uint32_t)src0->ne[0], (uint32_t)src0->ne[1], (uint32_t)src0->ne[2],(uint32_t)src0->ne[3], (uint32_t)src0->nb[0] / src0_type_size, (uint32_t)src0->nb[1] / src0_type_size, (uint32_t)src0->nb[2] / src0_type_size, (uint32_t)src0->nb[3] / src0_type_size, - (uint32_t)src1->ne[0], (uint32_t)src1->ne[1], (uint32_t)src1->ne[2],(uint32_t)src1->ne[3], (uint32_t)src1->nb[0] / src1_type_size, (uint32_t)src1->nb[1] / src1_type_size, (uint32_t)src1->nb[2] / src1_type_size, (uint32_t)src1->nb[3] / src1_type_size, - (uint32_t) dst->ne[0], (uint32_t) dst->ne[1], (uint32_t) dst->ne[2],(uint32_t) dst->ne[3], (uint32_t) dst->nb[0] / dst_type_size, (uint32_t) dst->nb[1] / dst_type_size, (uint32_t) dst->nb[2] / dst_type_size, (uint32_t) dst->nb[3] / dst_type_size, - 0, - 0.0f, 0.0f, 0, - }); -} - void ggml_vk_arange(ggml_backend_vk_context * ctx, vk_context& subctx, ggml_tensor * dst) { VK_LOG_DEBUG("ggml_vk_arange(dst=" << dst << ", ne=" << ggml_nelements(dst) << ")"); @@ -10867,8 +11196,9 @@ void ggml_vk_group_norm(ggml_backend_vk_context * ctx, vk_context& subctx, const const uint32_t num_groups = int_op_params[0]; const float eps = float_op_params[1]; const uint32_t group_size = src0->ne[0] * src0->ne[1] * ((src0->ne[2] + num_groups - 1) / num_groups); + const uint32_t num_workgroups = num_groups * src0->ne[3]; - ggml_vk_op_f32(ctx, subctx, src0, nullptr, nullptr, nullptr, dst, GGML_OP_GROUP_NORM, { group_size, 0, eps, 0.0f, 0.0f, 0.0f }); + ggml_vk_op_f32(ctx, subctx, src0, nullptr, nullptr, nullptr, dst, GGML_OP_GROUP_NORM, { group_size, num_workgroups, eps, 0.0f, 0.0f, 0.0f }); } static uint32_t ggml_vk_rms_num_partials(ggml_backend_vk_context * ctx, const ggml_tensor *node) { @@ -10888,7 +11218,6 @@ static vk_op_rope_push_constants ggml_vk_make_rope_constants(const ggml_tensor * const int n_dims = ((const int32_t *) dst->op_params)[1]; const int mode = ((const int32_t *) dst->op_params)[2]; const int n_offs = ((const int32_t *) dst->op_params)[15]; - // const int n_ctx = ((const int32_t *) dst->op_params)[3]; const int n_ctx_orig = ((const int32_t *) dst->op_params)[4]; const float freq_base = ((const float *) dst->op_params)[5]; const float freq_scale = ((const float *) dst->op_params)[6]; @@ -10904,7 +11233,7 @@ static vk_op_rope_push_constants ggml_vk_make_rope_constants(const ggml_tensor * const bool is_imrope = mode == GGML_ROPE_TYPE_IMROPE; float corr_dims[2]; - ggml_rope_yarn_corr_dims(n_dims, n_ctx_orig, freq_base, beta_fast, beta_slow, corr_dims); + ggml_rope_yarn_corr_dims(n_dims, n_ctx_orig, freq_base, beta_fast, beta_slow, ggml_rope_get_truncate(dst), corr_dims); const float theta_scale = powf(freq_base, -2.0f/n_dims); @@ -11127,7 +11456,7 @@ void ggml_vk_rms_norm(ggml_backend_vk_context * ctx, vk_context& subctx, const s void ggml_vk_rms_norm_back(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { float * op_params = (float *)dst->op_params; - ggml_vk_op_f32(ctx, subctx, src0, src1, nullptr, nullptr, dst, GGML_OP_RMS_NORM_BACK, { (uint32_t)src0->ne[0], (uint32_t)src0->ne[1], op_params[0], 0.0f, 0.0f, 0.0f }); + ggml_vk_op_f32(ctx, subctx, src0, src1, nullptr, nullptr, dst, GGML_OP_RMS_NORM_BACK, { (uint32_t)src0->ne[0], (uint32_t)ggml_nrows(src0), op_params[0], 0.0f, 0.0f, 0.0f }); } void ggml_vk_l2_norm(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) { @@ -11199,11 +11528,6 @@ void ggml_vk_glu(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_t }); } -void ggml_vk_diag_mask_inf(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) { - int32_t * op_params = (int32_t *)dst->op_params; - ggml_vk_op_f32(ctx, subctx, src0, nullptr, nullptr, nullptr, dst, GGML_OP_DIAG_MASK_INF, { (uint32_t)src0->ne[0], (uint32_t)src0->ne[1], op_params[0] }); -} - void ggml_vk_soft_max(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * src2, ggml_tensor * dst) { float * op_params = (float *)dst->op_params; @@ -11389,20 +11713,6 @@ void ggml_vk_rope(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_ const ggml_tensor * src1 = dst->src[1]; const ggml_tensor * src2 = dst->src[2]; const ggml_tensor * src3 = nullptr; - const int n_dims = ((int32_t *) dst->op_params)[1]; - const int mode = ((int32_t *) dst->op_params)[2]; - // const int n_ctx = ((int32_t *) dst->op_params)[3]; - const int n_ctx_orig = ((int32_t *) dst->op_params)[4]; - const float freq_base = ((float *) dst->op_params)[5]; - const float beta_fast = ((float *) dst->op_params)[9]; - const float beta_slow = ((float *) dst->op_params)[10]; - int sections[4] {}; - if (mode & GGML_ROPE_TYPE_MROPE) { - memcpy(sections, (int32_t *) dst->op_params + 11, sizeof(int)*4); - } - - float corr_dims[2]; - ggml_rope_yarn_corr_dims(n_dims, n_ctx_orig, freq_base, beta_fast, beta_slow, corr_dims); uint32_t set_rows_stride = 0; // Fused rope + view + set_rows passes the set_rows destination stride in set_rows_stride @@ -11805,6 +12115,7 @@ void ggml_vk_cross_entropy_loss(ggml_backend_vk_context * ctx, vk_context& subct sp.n_cols = nrows; sp.ne01 = 1; sp.ne02 = 1; + sp.ne03 = 1; sp.weight = 1.0f; init_pushconst_fastdiv(sp); sp.misalign_offsets = get_misalign_bytes(ctx, dst) / ggml_type_size(dst->type); @@ -12407,11 +12718,12 @@ bool ggml_vk_build_graph(ggml_backend_vk_context * ctx, ggml_cgraph * cgraph, in cgraph->nodes[next_node_idx]->op == GGML_OP_RMS_NORM && cgraph->nodes[next_node_idx]->src[0] == cgraph->nodes[next_node_idx - 1] && ggml_nrows(cgraph->nodes[next_node_idx]) == 1 && - ctx->device->add_rms_fusion) { + ctx->device->add_rms_fusion && !ctx->device->disable_fusion) { uint32_t size = ggml_vk_rms_partials_size(ctx, cgraph->nodes[node_idx]); ctx->do_add_rms_partials_offset_calculation = true; if (ctx->prealloc_size_add_rms_partials_offset + size <= ctx->prealloc_size_add_rms_partials) { ctx->do_add_rms_partials = true; + ggml_vk_count_fusion(ctx->device, "ADD_RMS_NORM_PARTIALS"); } } } @@ -12587,10 +12899,6 @@ bool ggml_vk_build_graph(ggml_backend_vk_context * ctx, ggml_cgraph * cgraph, in case GGML_OP_UPSCALE: ggml_vk_upscale(ctx, compute_ctx, src0, node); - break; - case GGML_OP_ADD1: - ggml_vk_add1(ctx, compute_ctx, src0, src1, node); - break; case GGML_OP_ARANGE: ggml_vk_arange(ctx, compute_ctx, node); @@ -12741,10 +13049,6 @@ bool ggml_vk_build_graph(ggml_backend_vk_context * ctx, ggml_cgraph * cgraph, in default: return false; } - break; - case GGML_OP_DIAG_MASK_INF: - ggml_vk_diag_mask_inf(ctx, compute_ctx, src0, node); - break; case GGML_OP_SOFT_MAX: if (ctx->fused_topk_moe_mode != TOPK_MOE_COUNT) { @@ -13231,6 +13535,54 @@ bool ggml_backend_vk_buffer_cpy_tensor(ggml_backend_buffer_t buffer, const ggml_ UNUSED(buffer); } +static vk_buffer & ggml_vk_tensor_dev_buffer(const ggml_tensor * tensor) { + ggml_backend_buffer_t buffer = tensor->view_src ? tensor->view_src->buffer : tensor->buffer; + return ((ggml_backend_vk_buffer_context *) buffer->context)->dev_buffer; +} + +void ggml_backend_vk_buffer_set_tensor_n(ggml_backend_buffer_t buffer, ggml_tensor * const * tensors, const void * const * data, + const size_t * offsets, const size_t * sizes, size_t n_tensors) { + VK_LOG_DEBUG("ggml_backend_vk_buffer_set_tensor_n(" << buffer << ", " << n_tensors << ")"); + std::vector regions(n_tensors); + for (size_t i = 0; i < n_tensors; i++) { + regions[i] = { ggml_vk_tensor_dev_buffer(tensors[i]), vk_tensor_offset(tensors[i]) + tensors[i]->view_offs + offsets[i], const_cast(data[i]), sizes[i] }; + } + ggml_vk_buffer_write_n(regions.data(), n_tensors); + + UNUSED(buffer); +} + +void ggml_backend_vk_buffer_get_tensor_n(ggml_backend_buffer_t buffer, const ggml_tensor * const * tensors, void * const * data, + const size_t * offsets, const size_t * sizes, size_t n_tensors) { + VK_LOG_DEBUG("ggml_backend_vk_buffer_get_tensor_n(" << buffer << ", " << n_tensors << ")"); + std::vector regions(n_tensors); + for (size_t i = 0; i < n_tensors; i++) { + regions[i] = { ggml_vk_tensor_dev_buffer(tensors[i]), vk_tensor_offset(tensors[i]) + tensors[i]->view_offs + offsets[i], data[i], sizes[i] }; + } + ggml_vk_buffer_read_n(regions.data(), n_tensors); + + UNUSED(buffer); +} + +bool ggml_backend_vk_buffer_cpy_tensor_n(ggml_backend_buffer_t buffer, const ggml_tensor * const * srcs, ggml_tensor * const * dsts, size_t n_tensors) { + VK_LOG_DEBUG("ggml_backend_vk_buffer_cpy_tensor_n(" << buffer << ", " << n_tensors << ")"); + for (size_t i = 0; i < n_tensors; i++) { + if (!ggml_backend_buffer_is_vk(srcs[i]->view_src ? srcs[i]->view_src->buffer : srcs[i]->buffer)) { + return false; + } + } + + std::vector regions(n_tensors); + for (size_t i = 0; i < n_tensors; i++) { + regions[i] = { ggml_vk_tensor_dev_buffer(dsts[i]), vk_tensor_offset(dsts[i]) + dsts[i]->view_offs, + ggml_vk_tensor_dev_buffer(srcs[i]), vk_tensor_offset(srcs[i]) + srcs[i]->view_offs, ggml_nbytes(srcs[i]) }; + } + ggml_vk_buffer_copy_n(regions.data(), n_tensors); + return true; + + UNUSED(buffer); +} + void ggml_backend_vk_buffer_clear(ggml_backend_buffer_t buffer, uint8_t value) { ggml_backend_vk_buffer_context * ctx = (ggml_backend_vk_buffer_context *)buffer->context; @@ -14819,6 +15171,16 @@ static ggml_status ggml_backend_vk_graph_compute(ggml_backend_t backend, ggml_cg } if (enqueued) { + if (fusion_string && ctx->device->fusion_stats_enabled.load(std::memory_order_relaxed)) { + std::string label = fusion_string; + if (cgraph->nodes[i]->op == GGML_OP_ADD) { + label += "_" + std::to_string(ctx->num_additional_fused_ops + 1); + } + if (ctx->fused_topk_moe_scale) { + label += "_SCALE"; + } + ggml_vk_count_fusion(ctx->device, label.c_str()); + } ++submitted_nodes; #ifndef GGML_VULKAN_CHECK_RESULTS @@ -15550,13 +15912,14 @@ static bool ggml_backend_vk_device_supports_op(ggml_backend_dev_t dev, const ggm } return true; }; - // reject any tensors larger than the max buffer size + // non-compute ops are never executed here; their size is bounded by the buffer allocation limit + const bool non_compute = ggml_op_is_empty(op->op); for (int i = 0; i < GGML_MAX_SRC; i++) { - if (op->src[i] && !tensor_size_supported(ggml_nbytes(op->src[i]))) { + if (op->src[i] && !non_compute && !tensor_size_supported(ggml_nbytes(op->src[i]))) { return false; } } - if (!tensor_size_supported(ggml_nbytes(op))) { + if (!non_compute && !tensor_size_supported(ggml_nbytes(op))) { return false; } @@ -15624,7 +15987,7 @@ static bool ggml_backend_vk_device_supports_op(ggml_backend_dev_t dev, const ggm return false; } } - if (ggml_prec_get_src(op, 1) == GGML_PREC_F32) { + if (ggml_prec_get_src(op, 1) == GGML_PREC_BF16 || ggml_prec_get_src(op, 1) == GGML_PREC_F32) { return false; } switch (src0_type) { @@ -15952,10 +16315,6 @@ static bool ggml_backend_vk_device_supports_op(ggml_backend_dev_t dev, const ggm case GGML_OP_CONCAT: { return ggml_vk_concat_supported(op->src[0], op->src[1], op); } - case GGML_OP_ADD1: - return (op->src[0]->type == GGML_TYPE_F32 && op->src[1]->type == GGML_TYPE_F32) - || (op->src[0]->type == GGML_TYPE_F16 && op->src[1]->type == GGML_TYPE_F32) - || (op->src[0]->type == GGML_TYPE_F16 && op->src[1]->type == GGML_TYPE_F16); case GGML_OP_ARANGE: return op->type == GGML_TYPE_F32; case GGML_OP_FILL: @@ -15966,8 +16325,6 @@ static bool ggml_backend_vk_device_supports_op(ggml_backend_dev_t dev, const ggm case GGML_OP_PAD_REFLECT_1D: case GGML_OP_ROLL: return op->src[0]->type == GGML_TYPE_F32; - case GGML_OP_DIAG_MASK_INF: - return ggml_is_contiguous(op->src[0]) && op->src[0]->type == GGML_TYPE_F32; case GGML_OP_SOFT_MAX: return ggml_is_contiguous(op->src[0]) && op->src[0]->type == GGML_TYPE_F32 && (!op->src[1] || (op->src[1]->type == GGML_TYPE_F32 || op->src[1]->type == GGML_TYPE_F16)); @@ -16382,9 +16739,22 @@ static void * ggml_backend_vk_reg_get_proc_address(ggml_backend_reg_t reg, const if (strcmp(name, "ggml_backend_comm_allreduce_tensor") == 0) { return (void *) ggml_backend_vk_comm_allreduce_tensor; } -#else - UNUSED(name); #endif + if (strcmp(name, "ggml_backend_fusion_get") == 0) { + return (void *) ggml_backend_vk_fusion_get; + } + if (strcmp(name, "ggml_backend_fusion_stats_init") == 0) { + return (void *) ggml_backend_vk_fusion_stats_init; + } + if (strcmp(name, "ggml_backend_fusion_stats_reset") == 0) { + return (void *) ggml_backend_vk_fusion_stats_reset; + } + if (strcmp(name, "ggml_backend_fusion_stats_get") == 0) { + return (void *) ggml_backend_vk_fusion_stats_get; + } + if (strcmp(name, "ggml_backend_fusion_set_enabled") == 0) { + return (void *) ggml_backend_vk_fusion_set_enabled; + } return nullptr; } diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/concat_transpose.comp b/ggml/src/ggml-vulkan/vulkan-shaders/concat_transpose.comp new file mode 100644 index 000000000000..e598ad818626 --- /dev/null +++ b/ggml/src/ggml-vulkan/vulkan-shaders/concat_transpose.comp @@ -0,0 +1,39 @@ +#version 450 + +#include "types.glsl" +#include "generic_binary_head.glsl" + +#define TILE_DIM 32 +layout(local_size_x = TILE_DIM, local_size_y = 8, local_size_z = 1) in; + +// 33 columns (not 32) pads away shared-memory bank conflicts. +shared A_TYPE tile[TILE_DIM][TILE_DIM + 1]; + +void main() { + const uint tid_x = gl_LocalInvocationID.x; + const uint tid_y = gl_LocalInvocationID.y; + + // workgroup x walks rows (src1's contiguous dim1) so consecutive workgroups stream src1, y walks dst columns + const uint row0 = gl_WorkGroupID.x * TILE_DIM; + const uint col0 = (gl_WorkGroupID.z * gl_NumWorkGroups.y + gl_WorkGroupID.y) * TILE_DIM; + + const uint i1 = row0 + tid_x; + [[unroll]] for (uint j = 0; j < TILE_DIM; j += gl_WorkGroupSize.y) { + const uint i0 = col0 + tid_y + j; + if (i0 >= p.ne00 && i0 - p.ne00 < p.ne10 && i1 < p.ne11) { + tile[tid_y + j][tid_x] = A_TYPE(data_b[get_boffset() + (i0 - p.ne00)*p.nb10 + i1*p.nb11]); + } + } + + barrier(); + + const uint d0 = col0 + tid_x; + [[unroll]] for (uint j = 0; j < TILE_DIM; j += gl_WorkGroupSize.y) { + const uint d1 = row0 + tid_y + j; + if (d0 < p.ne20 && d1 < p.ne21) { + data_d[get_doffset() + d1*p.nb21 + d0*p.nb20] = d0 < p.ne00 + ? D_TYPE(data_a[get_aoffset() + d1*p.nb01 + d0*p.nb00]) + : D_TYPE(tile[tid_x][tid_y + j]); + } + } +} diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/cumsum.comp b/ggml/src/ggml-vulkan/vulkan-shaders/cumsum.comp index 75e3c3b0eb44..08c785ad3be1 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/cumsum.comp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/cumsum.comp @@ -25,6 +25,8 @@ void main() { const uint row = gl_WorkGroupID.z * 262144 + gl_WorkGroupID.y * 512 + gl_WorkGroupID.x; const uint tid = gl_LocalInvocationID.x; + if (row >= p.ne01 * p.ne02 * p.ne03) { return; } + const uint i03 = fastdiv(row, p.ne0_12mp, p.ne0_12L); const uint i03_offset = i03 * p.ne01*p.ne02; const uint i02 = fastdiv(row - i03_offset, p.ne0_1mp, p.ne0_1L); diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/dequant_funcs_cm2.glsl b/ggml/src/ggml-vulkan/vulkan-shaders/dequant_funcs_cm2.glsl index cc6e242a90d5..ecea26672384 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/dequant_funcs_cm2.glsl +++ b/ggml/src/ggml-vulkan/vulkan-shaders/dequant_funcs_cm2.glsl @@ -10,7 +10,7 @@ #include "types.glsl" layout(buffer_reference, std430, buffer_reference_align = 16) buffer decodeBufF32 { - vec4 block; + readonly vec4 block; }; float16_t dequantFuncF32(const in decodeBufF32 bl, const in uint blockCoords[2], const in uint coordInBlock[2]) @@ -22,7 +22,7 @@ float16_t dequantFuncF32(const in decodeBufF32 bl, const in uint blockCoords[2], } layout(buffer_reference, std430, buffer_reference_align = 2) buffer decodeBufQ1_0 { - block_q1_0 block; + readonly block_q1_0 block; }; float16_t dequantFuncQ1_0(const in decodeBufQ1_0 bl, const in uint blockCoords[2], const in uint coordInBlock[2]) @@ -47,7 +47,7 @@ f16vec4 dequantFuncQ1_0_v(const in decodeBufQ1_0 bl, const in uint blockCoords[2 } layout(buffer_reference, std430, buffer_reference_align = 2) buffer decodeBufQ2_0 { - block_q2_0 block; + readonly block_q2_0 block; }; float16_t dequantFuncQ2_0(const in decodeBufQ2_0 bl, const in uint blockCoords[2], const in uint coordInBlock[2]) @@ -67,7 +67,7 @@ f16vec4 dequantFuncQ2_0_v(const in decodeBufQ2_0 bl, const in uint blockCoords[2 } layout(buffer_reference, std430, buffer_reference_align = 2) buffer decodeBufQ4_0 { - block_q4_0_packed16 block; + readonly block_q4_0_packed16 block; }; float16_t dequantFuncQ4_0(const in decodeBufQ4_0 bl, const in uint blockCoords[2], const in uint coordInBlock[2]) @@ -98,11 +98,11 @@ f16vec4 dequantFuncQ4_0_v(const in decodeBufQ4_0 bl, const in uint blockCoords[2 } layout(buffer_reference, std430, buffer_reference_align = 4) buffer decodeBufQ4_1 { - block_q4_1 block; + readonly block_q4_1 block; }; layout(buffer_reference, std430, buffer_reference_align = 4) buffer decodeBufQ4_1_packed32 { - block_q4_1_packed32 block; + readonly block_q4_1_packed32 block; }; float16_t dequantFuncQ4_1(const in decodeBufQ4_1 bl, const in uint blockCoords[2], const in uint coordInBlock[2]) @@ -133,11 +133,11 @@ f16vec4 dequantFuncQ4_1_v(const in decodeBufQ4_1 bl, const in uint blockCoords[2 } layout(buffer_reference, std430, buffer_reference_align = 2) buffer decodeBufQ5_0 { - block_q5_0 block; + readonly block_q5_0 block; }; layout(buffer_reference, std430, buffer_reference_align = 2) buffer decodeBufQ5_0_packed16 { - block_q5_0_packed16 block; + readonly block_q5_0_packed16 block; }; float16_t dequantFuncQ5_0(const in decodeBufQ5_0 bl, const in uint blockCoords[2], const in uint coordInBlock[2]) @@ -177,11 +177,11 @@ f16vec4 dequantFuncQ5_0_v(const in decodeBufQ5_0 bl, const in uint blockCoords[2 } layout(buffer_reference, std430, buffer_reference_align = 8) buffer decodeBufQ5_1 { - block_q5_1 block; + readonly block_q5_1 block; }; layout(buffer_reference, std430, buffer_reference_align = 8) buffer decodeBufQ5_1_packed32 { - block_q5_1_packed32 block; + readonly block_q5_1_packed32 block; }; float16_t dequantFuncQ5_1(const in decodeBufQ5_1 bl, const in uint blockCoords[2], const in uint coordInBlock[2]) @@ -221,7 +221,7 @@ f16vec4 dequantFuncQ5_1_v(const in decodeBufQ5_1 bl, const in uint blockCoords[2 } layout(buffer_reference, std430, buffer_reference_align = 2) buffer decodeBufQ8_0 { - block_q8_0_packed16 block; + readonly block_q8_0_packed16 block; }; float16_t dequantFuncQ8_0(const in decodeBufQ8_0 bl, const in uint blockCoords[2], const in uint coordInBlock[2]) @@ -248,7 +248,7 @@ f16vec4 dequantFuncQ8_0_v(const in decodeBufQ8_0 bl, const in uint blockCoords[2 } layout(buffer_reference, std430, buffer_reference_align = 2) buffer decodeBufTQ1_0 { - block_tq1_0 block; + readonly block_tq1_0 block; }; float16_t dequantFuncTQ1_0(const in decodeBufTQ1_0 bl, const in uint blockCoords[2], const in uint coordInBlock[2]) @@ -275,11 +275,11 @@ f16vec4 dequantFuncTQ1_0_v(const in decodeBufTQ1_0 bl, const in uint blockCoords } layout(buffer_reference, std430, buffer_reference_align = 2) buffer decodeBufTQ2_0 { - block_tq2_0 block; + readonly block_tq2_0 block; }; layout(buffer_reference, std430, buffer_reference_align = 2) buffer decodeBufTQ2_0_packed16 { - block_tq2_0_packed16 block; + readonly block_tq2_0_packed16 block; }; float16_t dequantFuncTQ2_0(const in decodeBufTQ2_0 bl, const in uint blockCoords[2], const in uint coordInBlock[2]) @@ -313,15 +313,15 @@ f16vec4 dequantFuncTQ2_0_v(const in decodeBufTQ2_0 bl, const in uint blockCoords } layout(buffer_reference, std430, buffer_reference_align = 4) buffer decodeBufQ2_K { - block_q2_K block; + readonly block_q2_K block; }; layout(buffer_reference, std430, buffer_reference_align = 16) buffer decodeBufQ2_K_packed16 { - block_q2_K_packed16 block; + readonly block_q2_K_packed16 block; }; layout(buffer_reference, std430, buffer_reference_align = 4) buffer decodeBufQ2_K_packed32 { - block_q2_K_packed32 block; + readonly block_q2_K_packed32 block; }; float16_t dequantFuncQ2_K(const in decodeBufQ2_K bl, const in uint blockCoords[2], const in uint coordInBlock[2]) @@ -365,11 +365,11 @@ f16vec4 dequantFuncQ2_K_v(const in decodeBufQ2_K bl, const in uint blockCoords[2 } layout(buffer_reference, std430, buffer_reference_align = 2) buffer decodeBufQ3_K { - block_q3_K block; + readonly block_q3_K block; }; layout(buffer_reference, std430, buffer_reference_align = 2) buffer decodeBufQ3_K_packed16 { - block_q3_K_packed16 block; + readonly block_q3_K_packed16 block; }; float16_t dequantFuncQ3_K(const in decodeBufQ3_K bl, const in uint blockCoords[2], const in uint coordInBlock[2]) @@ -442,19 +442,19 @@ f16vec4 dequantFuncQ3_K_v(const in decodeBufQ3_K bl, const in uint blockCoords[2 } layout(buffer_reference, std430, buffer_reference_align = 16) buffer decodeBufQ4_K { - block_q4_K block; + readonly block_q4_K block; }; layout(buffer_reference, std430, buffer_reference_align = 16) buffer decodeBufQ4_K_packed16 { - block_q4_K_packed16 block; + readonly block_q4_K_packed16 block; }; layout(buffer_reference, std430, buffer_reference_align = 16) buffer decodeBufQ4_K_packed32 { - block_q4_K_packed32 block; + readonly block_q4_K_packed32 block; }; layout(buffer_reference, std430, buffer_reference_align = 16) buffer decodeBufQ4_K_packed128 { - block_q4_K_packed128 block; + readonly block_q4_K_packed128 block; }; #if defined(IS_MUL_MM2) @@ -650,19 +650,19 @@ f16vec4 dequantFuncQ4_K_v(const in decodeBufQ4_K bl, const in uint blockCoords[2 } layout(buffer_reference, std430, buffer_reference_align = 16) buffer decodeBufQ5_K { - block_q5_K block; + readonly block_q5_K block; }; layout(buffer_reference, std430, buffer_reference_align = 16) buffer decodeBufQ5_K_packed16 { - block_q5_K_packed16 block; + readonly block_q5_K_packed16 block; }; layout(buffer_reference, std430, buffer_reference_align = 16) buffer decodeBufQ5_K_packed128 { - block_q5_K_packed128 block; + readonly block_q5_K_packed128 block; }; layout(buffer_reference, std430, buffer_reference_align = 16) buffer decodeBufQ5_K_packed32 { - block_q5_K_packed32 block; + readonly block_q5_K_packed32 block; }; float16_t dequantFuncQ5_K(const in decodeBufQ5_K bl, const in uint blockCoords[2], const in uint coordInBlock[2]) @@ -771,11 +771,11 @@ f16vec4 dequantFuncQ5_K_v(const in decodeBufQ5_K bl, const in uint blockCoords[2 } layout(buffer_reference, std430, buffer_reference_align = 2) buffer decodeBufQ6_K { - block_q6_K block; + readonly block_q6_K block; }; layout(buffer_reference, std430, buffer_reference_align = 16) buffer decodeBufQ6_K_packed16 { - block_q6_K_packed16 block; + readonly block_q6_K_packed16 block; }; float16_t dequantFuncQ6_K(const in decodeBufQ6_K bl, const in uint blockCoords[2], const in uint coordInBlock[2]) @@ -833,7 +833,7 @@ f16vec4 dequantFuncQ6_K_v(const in decodeBufQ6_K bl, const in uint blockCoords[2 #if defined(DATA_A_IQ1_S) layout(buffer_reference, std430, buffer_reference_align = 2) buffer decodeBufIQ1_S { - block_iq1_s block; + readonly block_iq1_s block; }; float16_t dequantFuncIQ1_S(const in decodeBufIQ1_S bl, const in uint blockCoords[2], const in uint coordInBlock[2]) @@ -880,11 +880,11 @@ f16vec4 dequantFuncIQ1_S_v(const in decodeBufIQ1_S bl, const in uint blockCoords #if defined(DATA_A_IQ1_M) layout(buffer_reference, std430, buffer_reference_align = 2) buffer decodeBufIQ1_M { - block_iq1_m block; + readonly block_iq1_m block; }; layout(buffer_reference, std430, buffer_reference_align = 8) buffer decodeBufIQ1_M_packed64 { - block_iq1_m_packed64 block; + readonly block_iq1_m_packed64 block; }; float16_t dequantFuncIQ1_M(const in decodeBufIQ1_M bl, const in uint blockCoords[2], const in uint coordInBlock[2]) @@ -939,11 +939,11 @@ f16vec4 dequantFuncIQ1_M_v(const in decodeBufIQ1_M bl, const in uint blockCoords #if defined(DATA_A_IQ2_XXS) layout(buffer_reference, std430, buffer_reference_align = 2) buffer decodeBufIQ2_XXS { - block_iq2_xxs block; + readonly block_iq2_xxs block; }; layout(buffer_reference, std430, buffer_reference_align = 2) buffer decodeBufIQ2_XXS_packed16 { - block_iq2_xxs_packed16 block; + readonly block_iq2_xxs_packed16 block; }; float16_t dequantFuncIQ2_XXS(const in decodeBufIQ2_XXS bl, const in uint blockCoords[2], const in uint coordInBlock[2]) @@ -1001,7 +1001,7 @@ f16vec4 dequantFuncIQ2_XXS_v(const in decodeBufIQ2_XXS bl, const in uint blockCo #if defined(DATA_A_IQ2_XS) layout(buffer_reference, std430, buffer_reference_align = 2) buffer decodeBufIQ2_XS { - block_iq2_xs block; + readonly block_iq2_xs block; }; float16_t dequantFuncIQ2_XS(const in decodeBufIQ2_XS bl, const in uint blockCoords[2], const in uint coordInBlock[2]) @@ -1054,7 +1054,7 @@ f16vec4 dequantFuncIQ2_XS_v(const in decodeBufIQ2_XS bl, const in uint blockCoor #if defined(DATA_A_IQ2_S) layout(buffer_reference, std430, buffer_reference_align = 2) buffer decodeBufIQ2_S { - block_iq2_s block; + readonly block_iq2_s block; }; float16_t dequantFuncIQ2_S(const in decodeBufIQ2_S bl, const in uint blockCoords[2], const in uint coordInBlock[2]) @@ -1108,11 +1108,11 @@ f16vec4 dequantFuncIQ2_S_v(const in decodeBufIQ2_S bl, const in uint blockCoords #if defined(DATA_A_IQ3_XXS) layout(buffer_reference, std430, buffer_reference_align = 2) buffer decodeBufIQ3_XXS { - block_iq3_xxs block; + readonly block_iq3_xxs block; }; layout(buffer_reference, std430, buffer_reference_align = 2) buffer decodeBufIQ3_XXS_packed16 { - block_iq3_xxs_packed16 block; + readonly block_iq3_xxs_packed16 block; }; float16_t dequantFuncIQ3_XXS(const in decodeBufIQ3_XXS bl, const in uint blockCoords[2], const in uint coordInBlock[2]) @@ -1167,7 +1167,7 @@ f16vec4 dequantFuncIQ3_XXS_v(const in decodeBufIQ3_XXS bl, const in uint blockCo #if defined(DATA_A_IQ3_S) layout(buffer_reference, std430, buffer_reference_align = 2) buffer decodeBufIQ3_S { - block_iq3_s block; + readonly block_iq3_s block; }; float16_t dequantFuncIQ3_S(const in decodeBufIQ3_S bl, const in uint blockCoords[2], const in uint coordInBlock[2]) @@ -1217,11 +1217,11 @@ f16vec4 dequantFuncIQ3_S_v(const in decodeBufIQ3_S bl, const in uint blockCoords #if defined(DATA_A_IQ4_XS) layout(buffer_reference, std430, buffer_reference_align = 2) buffer decodeBufIQ4_XS { - block_iq4_xs block; + readonly block_iq4_xs block; }; layout(buffer_reference, std430, buffer_reference_align = 4) buffer decodeBufIQ4_XS_packed32 { - block_iq4_xs_packed32 block; + readonly block_iq4_xs_packed32 block; }; float16_t dequantFuncIQ4_XS(const in decodeBufIQ4_XS bl, const in uint blockCoords[2], const in uint coordInBlock[2]) @@ -1267,11 +1267,11 @@ f16vec4 dequantFuncIQ4_XS_v(const in decodeBufIQ4_XS bl, const in uint blockCoor #if defined(DATA_A_IQ4_NL) layout(buffer_reference, std430, buffer_reference_align = 2) buffer decodeBufIQ4_NL { - block_iq4_nl block; + readonly block_iq4_nl block; }; layout(buffer_reference, std430, buffer_reference_align = 2) buffer decodeBufIQ4_NL_packed16 { - block_iq4_nl_packed16 block; + readonly block_iq4_nl_packed16 block; }; float16_t dequantFuncIQ4_NL(const in decodeBufIQ4_NL bl, const in uint blockCoords[2], const in uint coordInBlock[2]) @@ -1308,7 +1308,7 @@ f16vec4 dequantFuncIQ4_NL_v(const in decodeBufIQ4_NL bl, const in uint blockCoor #if defined(DATA_A_MXFP4) layout(buffer_reference, std430, buffer_reference_align = 2) buffer decodeBufMXFP4 { - block_mxfp4 block; + readonly block_mxfp4 block; }; float16_t dequantFuncMXFP4(const in decodeBufMXFP4 bl, const in uint blockCoords[2], const in uint coordInBlock[2]) @@ -1362,11 +1362,11 @@ f16vec4 dequantFuncMXFP4_v(const in decodeBufMXFP4 bl, const in uint blockCoords #if defined(DATA_A_NVFP4) layout(buffer_reference, std430, buffer_reference_align = 4) buffer decodeBufNVFP4 { - block_nvfp4 block; + readonly block_nvfp4 block; }; layout(buffer_reference, std430, buffer_reference_align = 4) buffer decodeBufNVFP4_packed32 { - block_nvfp4_packed32 block; + readonly block_nvfp4_packed32 block; }; float16_t dequantFuncNVFP4(const in decodeBufNVFP4 bl, const in uint blockCoords[2], const in uint coordInBlock[2]) diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/diag_mask_inf.comp b/ggml/src/ggml-vulkan/vulkan-shaders/diag_mask_inf.comp deleted file mode 100644 index 9cef8a8ec3d2..000000000000 --- a/ggml/src/ggml-vulkan/vulkan-shaders/diag_mask_inf.comp +++ /dev/null @@ -1,34 +0,0 @@ -#version 450 - -#extension GL_EXT_shader_16bit_storage : require -#extension GL_EXT_control_flow_attributes : enable - -layout (push_constant) uniform parameter -{ - uint ncols; - uint rows_per_channel; - uint n_past; -} p; - -#include "types.glsl" - -layout(local_size_x = 1, local_size_y = 512, local_size_z = 1) in; - -layout (binding = 0) readonly buffer X {A_TYPE data_a[];}; -layout (binding = 1) writeonly buffer D {D_TYPE data_d[];}; - -void main() { - const uint col = gl_GlobalInvocationID.y; - const uint row = gl_GlobalInvocationID.x; - - if (col >= p.ncols) { - return; - } - - const uint i = row*p.ncols + col; - if (col > p.n_past + row % p.rows_per_channel) { - data_d[i] = D_TYPE(uintBitsToFloat(0xFF800000)); - } else { - data_d[i] = D_TYPE(data_a[i]); - } -} diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_decode_phase_1.comp b/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_decode_phase_1.comp index b5f95aaa0d5f..53beab149963 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_decode_phase_1.comp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_decode_phase_1.comp @@ -3,261 +3,327 @@ #extension GL_EXT_control_flow_attributes : enable #extension GL_EXT_shader_16bit_storage : require #extension GL_EXT_shader_explicit_arithmetic_types_float16 : require -#extension GL_EXT_shader_explicit_arithmetic_types_int16 : require #extension GL_KHR_memory_scope_semantics : enable #extension GL_KHR_shader_subgroup_basic : enable -#extension GL_KHR_shader_subgroup_ballot : enable #extension GL_KHR_shader_subgroup_arithmetic : enable #extension GL_KHR_cooperative_matrix : enable #extension GL_EXT_shared_memory_block : enable -layout(local_size_x_id = 0, local_size_y = 1, local_size_z = 1) in; +layout (local_size_x_id = 0, local_size_y = 1, local_size_z = 1) in; -layout (binding = 0) readonly buffer Q {float16_t qState[];}; -layout (binding = 1) readonly buffer K_VEC4 {f16vec4 kStateVec4[];}; -layout (binding = 2) buffer MASK_F16 {float16_t mState_f16[];}; -layout (binding = 3) buffer P_FP16 {float16_t matP_f16[];}; -layout (binding = 4) buffer OUT_MAX {float out_max_f32[];}; +layout (binding = 0) readonly buffer Q {float16_t data_q[];}; +layout (binding = 1) readonly buffer KV4 {f16vec4 data_kv4[];}; +layout (binding = 2) readonly buffer M {float16_t data_m[];}; +layout (binding = 3) buffer P {float16_t data_p[];}; +layout (binding = 4) buffer MAX {float data_max[];}; layout (push_constant) uniform parameter { - uint kvSeqLen; - uint activationLength; - uint qHead; - uint kvHead; - uint qkRatio; - uint qkSubGroups; - uint flag; - uint kvStride1; - uint kvStride2; - uint batchStrideQ; - uint batchStrideK; - uint batchStrideV; - uint batchStrideM; - uint batchStrideO; - float softMaxScale; + uint neq1; + uint neq2; + uint neq3; + uint nek1; + uint nek2; + uint nek3; + uint nev2; + uint nev3; + uint nem1; + uint nem2; + uint nem3; + uint nbq1; + uint nbq2; + uint nbk1; + uint nbk2; + uint nbv1; + uint nbv2; + uint nbq3; + uint nbk3; + uint nbv3; + uint nb3; + uint flag; + float scale; } p; -layout (constant_id = 0) const uint GROUPSIZE = 128; -layout (constant_id = 1) const uint GQA_RATIO = 8; -layout (constant_id = 2) const uint HEAD_DIM = 128; -layout (constant_id = 3) const uint WARPSIZE = 16; -layout (constant_id = 4) const uint MATP_REDUCE = 32; -layout (constant_id = 5) const uint N_TOK = 1; -layout (constant_id = 6) const uint COOP_MAT_P_PER_LOOP = 4; - -#define MAX_HEADS 8 - -#define TN WARPSIZE -#define TM 8 -#define TK 16 -#define SUBGROUP_COUNT (GROUPSIZE / WARPSIZE) -#define MATP_PER_LOOP (COOP_MAT_P_PER_LOOP * TM) -#define P_LOOP_COUNT (MATP_REDUCE / MATP_PER_LOOP) - -#define COOP_MAT_Q_PER_TOKEN ((GQA_RATIO + TN - 1) / TN) -#define COOP_MAT_P_M COOP_MAT_Q_PER_TOKEN -#define COOP_MAT_P_N (MATP_REDUCE / TM) -#define SLM_PV_SIZE (MATP_REDUCE * COOP_MAT_P_M * TN) -#define SLM_MASK_SIZE (N_TOK * MATP_REDUCE) -#define SLM_POOL_SIZE_K (MATP_PER_LOOP * HEAD_DIM) -#define K_LOAD_PER_LOOP (GROUPSIZE * 4) -#define HEAD_DIM_VEC4 (HEAD_DIM / 4) -#define SLM_CHUNK_SIZE (TK / 4) -#define K_LOAD_LOOPS ((SLM_POOL_SIZE_K + K_LOAD_PER_LOOP - 1) / K_LOAD_PER_LOOP) -#define O_COUNT ((GQA_RATIO + SUBGROUP_COUNT - 1) / SUBGROUP_COUNT) - -shared slm_pool_block { - float slm_pool_pv[SLM_PV_SIZE + SLM_MASK_SIZE]; -} slm_pool_f32; - -shared slm_pool_alias_block { - float16_t slm_pool_k[SLM_POOL_SIZE_K]; -} slm_pool_f16; +layout (constant_id = 0) const uint WorkGroupSize = 128; +layout (constant_id = 1) const uint gqa_ratio = 8; +layout (constant_id = 2) const uint HSK = 128; +layout (constant_id = 3) const uint SubGroupSize = 16; +layout (constant_id = 4) const uint p_reduce = 32; +layout (constant_id = 5) const uint Br = 1; +layout (constant_id = 6) const uint pmat_count = 4; +layout (constant_id = 7) const uint TM = 8; +layout (constant_id = 8) const uint TN = 16; +layout (constant_id = 9) const uint shmem_size = 49152; + +const uint TK = 16; +const uint sg_count = WorkGroupSize / SubGroupSize; +const uint p_per_loop = pmat_count * TM; +const uint p_loops = p_reduce / p_per_loop; + +const uint pmat_row = (gqa_ratio + TN - 1) / TN; +const uint pmat_col = p_reduce / TM; + +const uint k_per_loop = WorkGroupSize * 4; +const uint hsk_v4 = HSK / 4; +const uint slm_col_chunk = TK / 4; +const uint k_loops = (p_per_loop * HSK + k_per_loop - 1) / k_per_loop; +const uint warp_o_row = (gqa_ratio + sg_count - 1) / sg_count; + +#define SLM_PV_SIZE (p_reduce * pmat_row * TN) +#define SLM_MASK_SIZE (Br * p_reduce) + +const bool k_in_shmem = k_loops * k_per_loop <= shmem_size / 2; +const uint ksh_size = k_in_shmem ? (k_loops * k_per_loop) : (sg_count * TM * TK); + +shared sh_block { + float F32sh[SLM_PV_SIZE + SLM_MASK_SIZE]; +} F32Poolsh; + +shared shf16_alias { + float16_t F16sh[ksh_size]; +} F16Poolsh; + +shared shf16v4_alias { + f16vec4 F16V4sh[ksh_size / 4]; +} F16V4Poolsh; void main() { - const uint lane = gl_SubgroupInvocationID; - const uint kHeadIdx = gl_WorkGroupID.x % p.kvHead; - const uint outGroupIdx = gl_WorkGroupID.x / p.kvHead; - const uint v = gl_WorkGroupID.y; - const uint d = gl_WorkGroupID.z; - const uint localLinearId = gl_SubgroupID; - const uint wgLane = localLinearId * WARPSIZE + lane; - const uint qDim = p.qHead * HEAD_DIM; - const uint kvDim = p.kvStride1; - const uint maskDim = p.kvSeqLen; - const uint maxDim = (p.kvSeqLen + MATP_REDUCE - 1) / MATP_REDUCE; - const uint pDim = maxDim * MATP_REDUCE; - const uint tokFlatIdx = localLinearId + outGroupIdx * N_TOK; - uint offsetBaseQ = min(tokFlatIdx, p.activationLength - 1) * qDim; - offsetBaseQ = offsetBaseQ + d * p.batchStrideQ + kHeadIdx * HEAD_DIM * GQA_RATIO; - const uint offsetBaseK = (d * p.batchStrideK + (v * MATP_REDUCE) * kvDim + kHeadIdx * p.kvStride2) / 4; - uint offsetOut = d * p.qHead * p.activationLength * pDim + v * MATP_REDUCE + kHeadIdx * GQA_RATIO * pDim + (localLinearId * O_COUNT + outGroupIdx * N_TOK * p.qHead) * pDim + lane; - uint offsetMax = d * p.qHead * p.activationLength * maxDim + v + kHeadIdx * GQA_RATIO * maxDim + (localLinearId * O_COUNT + outGroupIdx * N_TOK * p.qHead) * maxDim; - const uint offsetSlmLoadPv = (localLinearId * O_COUNT * MATP_REDUCE + lane); - const uint offsetBaseM = v * MATP_REDUCE + lane; - const float fp32Min = uintBitsToFloat(0xFEFFFFFF); - - const uint loopCount = HEAD_DIM / TK; - float maskFp32[MATP_REDUCE / WARPSIZE]; - - if (tokFlatIdx < p.activationLength) { - [[unroll]] for (uint mk = 0; mk < MATP_REDUCE / WARPSIZE; mk++) { - const uint maskOffset = mk * WARPSIZE + offsetBaseM; - if (maskOffset < maskDim) { - maskFp32[mk] = float(mState_f16[d * p.batchStrideM + tokFlatIdx * maskDim + maskOffset]); - } else { - maskFp32[mk] = fp32Min; - } + const uint lane = gl_SubgroupInvocationID; + const uint head_k_idx = gl_WorkGroupID.x % p.nek2; + const uint out_idx = gl_WorkGroupID.x / p.nek2; + const uint p_idx = gl_WorkGroupID.y; + const uint batch_k_ratio = p.neq3 / p.nek3; + const uint batch_idx = gl_WorkGroupID.z; + const uint batch_k_idx = batch_idx / batch_k_ratio; + const uint batch_mask_idx = batch_idx % p.nem3; + const uint batch_mask_stride = p.nek1 * p.nem1 * p.nem2; + const uint sg = gl_SubgroupID; + const uint wg_lane = sg * SubGroupSize + lane; + const uint k_dim = p.nbk1; + const uint max_dim = (p.nek1 + p_reduce - 1) / p_reduce; + const uint p_dim = max_dim * p_reduce; + const uint out_row_idx = sg + out_idx * Br; + const bool single_head_mask = p.nem2 == 1; + uint q_offset = min(out_row_idx, p.neq1 - 1) * p.nbq1; + q_offset = q_offset + batch_idx * p.nbq3 + head_k_idx * p.nbq2 * gqa_ratio; + const uint k_base = (batch_k_idx * p.nbk3 + (p_idx * p_reduce) * k_dim + head_k_idx * p.nbk2) / 4; + uint o_offset = batch_idx * p.neq2 * p.neq1 * p_dim + p_idx * p_reduce + head_k_idx * gqa_ratio * p_dim + + (sg * warp_o_row + out_idx * Br * p.neq2) * p_dim + lane; + uint max_offset = batch_idx * p.neq2 * p.neq1 * max_dim + p_idx + head_k_idx * gqa_ratio * max_dim + + (sg * warp_o_row + out_idx * Br * p.neq2) * max_dim; + const uint pvsh_load_offset = sg * warp_o_row * p_reduce + lane; + const uint mask_base = p_idx * p_reduce + lane; + const uint k_seq_boundary = ((p.nek1 - 1) * k_dim + batch_k_idx * p.nbk3 + head_k_idx * p.nbk2 + HSK) / 4 - 1; + const uint k_head_base = (batch_k_idx * p.nbk3 + head_k_idx * p.nbk2) / 4; + const float NEG_FLT_MAX_OVER_2 = uintBitsToFloat(0xFEFFFFFF); + + float Maskf[p_reduce / SubGroupSize]; + + if (out_row_idx < p.neq1 && single_head_mask) { + if (out_row_idx < p.nem1) { + [[unroll]] for (uint idx = 0; idx < p_reduce / SubGroupSize; ++idx) { + const uint mask_offset = idx * SubGroupSize + mask_base; + if (mask_offset < p.nek1) { + Maskf[idx] = float(data_m[batch_mask_idx * batch_mask_stride + out_row_idx * p.nek1 + mask_offset]); + } else { + Maskf[idx] = NEG_FLT_MAX_OVER_2; + } + } + } else { + [[unroll]] for (uint idx = 0; idx < p_reduce / SubGroupSize; ++idx) { + const uint mask_offset = idx * SubGroupSize + mask_base; + Maskf[idx] = (mask_offset < p.nek1) ? 0.0f : NEG_FLT_MAX_OVER_2; + } + } } - } - coopmat matP[COOP_MAT_P_M][COOP_MAT_P_N]; + coopmat PMat[pmat_row][pmat_col]; - [[unroll]] for (uint mp = 0; mp < COOP_MAT_P_M; mp++) { - [[unroll]] for (uint np = 0; np < COOP_MAT_P_N; np++) { - matP[mp][np] = coopmat(0.0f); + [[unroll]] for (uint r = 0; r < pmat_row; ++r) { + [[unroll]] for (uint c = 0; c < pmat_col; ++c) { + PMat[r][c] = coopmat(0.0f); + } } - } - - [[unroll]] for (uint kLoad = 0; kLoad < K_LOAD_LOOPS; kLoad++) { - const uint flatOffset = kLoad * GROUPSIZE + wgLane; - const uint kRowIdx = flatOffset / HEAD_DIM_VEC4; - const uint kColIdx = flatOffset % HEAD_DIM_VEC4; - const uint slmChunkCol = kColIdx % SLM_CHUNK_SIZE; - const uint slmChunkRow = kColIdx / SLM_CHUNK_SIZE; - const uint offsetK = offsetBaseK + kRowIdx * kvDim / 4 + kColIdx; - const uint offsetSlmK = kRowIdx * TK + slmChunkRow * TK * MATP_PER_LOOP + slmChunkCol * 4; - slm_pool_f16.slm_pool_k[offsetSlmK + 0] = kStateVec4[offsetK].x; - slm_pool_f16.slm_pool_k[offsetSlmK + 1] = kStateVec4[offsetK].y; - slm_pool_f16.slm_pool_k[offsetSlmK + 2] = kStateVec4[offsetK].z; - slm_pool_f16.slm_pool_k[offsetSlmK + 3] = kStateVec4[offsetK].w; - } - - [[unroll]] for (uint pLoop = 0; pLoop < P_LOOP_COUNT; pLoop++) { - f16vec4 kTemp[K_LOAD_LOOPS]; - - if (pLoop + 1 < P_LOOP_COUNT) { - [[unroll]] for (uint kLoad = 0; kLoad < K_LOAD_LOOPS; kLoad++) { - const uint flatOffset = kLoad * GROUPSIZE + wgLane; - const uint kRowIdx = flatOffset / HEAD_DIM_VEC4 + (pLoop + 1) * MATP_PER_LOOP; - const uint kColIdx = flatOffset % HEAD_DIM_VEC4; - const uint offsetK = offsetBaseK + kRowIdx * kvDim / 4 + kColIdx; - kTemp[kLoad] = kStateVec4[offsetK]; - } + + if (k_in_shmem) { + [[unroll]] for (uint idx = 0; idx < k_loops; ++idx) { + const uint flat_offset = idx * WorkGroupSize + wg_lane; + const uint row_idx = flat_offset / hsk_v4; + const uint col_idx = flat_offset % hsk_v4; + const uint sh_chunk_c = col_idx % slm_col_chunk; + const uint sh_chunk_r = col_idx / slm_col_chunk; + uint k_offset = k_base + row_idx * k_dim / 4 + col_idx; + k_offset = min(k_offset, k_seq_boundary); + const uint ksh_offset = (row_idx * TK + sh_chunk_r * TK * p_per_loop) / 4 + sh_chunk_c; + F16V4Poolsh.F16V4sh[ksh_offset] = data_kv4[k_offset]; + } } - barrier(); - if (localLinearId < N_TOK) { - [[unroll]] for (uint loop = 0; loop < loopCount; loop++) { - coopmat matQ[COOP_MAT_P_M]; - coopmat matK[COOP_MAT_P_PER_LOOP]; - - [[unroll]] for (uint mq = 0; mq < COOP_MAT_P_M; mq++) { - coopMatLoad( - matQ[mq], - qState, - offsetBaseQ + mq * TN * HEAD_DIM + loop * TK, - HEAD_DIM, - gl_CooperativeMatrixLayoutColumnMajor); + [[unroll]] for (uint pl = 0; pl < p_loops; ++pl) { + f16vec4 kv4_temp[k_loops]; + + if (k_in_shmem) { + if (pl + 1 < p_loops) { + [[unroll]] for (uint idx = 0; idx < k_loops; ++idx) { + const uint flat_offset = idx * WorkGroupSize + wg_lane; + const uint row_idx = flat_offset / hsk_v4 + (pl + 1) * p_per_loop; + const uint col_idx = flat_offset % hsk_v4; + uint k_offset = k_base + row_idx * k_dim / 4 + col_idx; + k_offset = min(k_offset, k_seq_boundary); + kv4_temp[idx] = data_kv4[k_offset]; + } + } } - [[unroll]] for (uint np = 0; np < COOP_MAT_P_PER_LOOP; np++) { - coopMatLoad( - matK[np], - slm_pool_f16.slm_pool_k, - loop * TK * MATP_PER_LOOP + np * TM * TK, - TK, - gl_CooperativeMatrixLayoutRowMajor); + barrier(); + if (sg < Br) { + [[unroll]] for (uint loop = 0; loop < HSK / TK; ++loop) { + coopmat QMat[pmat_row]; + coopmat KMat[pmat_count]; + + [[unroll]] for (uint r = 0; r < pmat_row; ++r) { + const uint coord = q_offset + r * TN * p.nbq2 + loop * TK; + coopMatLoad(QMat[r], data_q, coord, p.nbq2, gl_CooperativeMatrixLayoutColumnMajor); + } + + [[unroll]] for (uint c = 0; c < pmat_count; ++c) { + if (k_in_shmem) { + const uint coord = loop * TK * p_per_loop + c * TM * TK; + coopMatLoad(KMat[c], F16Poolsh.F16sh, coord, TK, gl_CooperativeMatrixLayoutRowMajor); + } else { + const uint k_base_row = (c * TM + pl * pmat_count * TM + p_idx * p_reduce); + const uint kv4_base_offset = k_head_base + k_base_row * p.nbk1 / 4 + loop * TK / 4; + [[unroll]] for (uint idx = 0; idx < TM * TK / 4 / SubGroupSize; ++idx) { + const uint flat_offset = idx * SubGroupSize + lane; + const uint row_idx = flat_offset / (TK / 4); + const uint col_idx = flat_offset % (TK / 4); + const uint k_temp_offset = kv4_base_offset + row_idx * p.nbk1 / 4 + col_idx; + const uint ksh_temp_offset = sg * TM * TK / 4 + idx * SubGroupSize + lane; + f16vec4 kv4_temp = (k_base_row + row_idx) < p.nek1 ? data_kv4[k_temp_offset] : f16vec4(0.0f, 0.0f, 0.0f, 0.0f); + F16V4Poolsh.F16V4sh[ksh_temp_offset] = kv4_temp; + } + + subgroupBarrier(); + coopMatLoad(KMat[c], F16Poolsh.F16sh, sg * TM * TK, TK, gl_CooperativeMatrixLayoutRowMajor); + subgroupBarrier(); + } + } + + [[unroll]] for (uint r = 0; r < pmat_row; ++r) { + [[unroll]] for (uint c = 0; c < pmat_count; ++c) { + PMat[r][pl * pmat_count + c] = coopMatMulAdd(KMat[c], QMat[r], PMat[r][pl * pmat_count + c]); + } + } + } } - [[unroll]] for (uint mp = 0; mp < COOP_MAT_P_M; mp++) { - [[unroll]] for (uint np = 0; np < COOP_MAT_P_PER_LOOP; np++) { - matP[mp][pLoop * COOP_MAT_P_PER_LOOP + np] = coopMatMulAdd(matK[np], matQ[mp], matP[mp][pLoop * COOP_MAT_P_PER_LOOP + np]); - } + barrier(); + + if (k_in_shmem) { + if (pl + 1 < p_loops) { + [[unroll]] for (uint idx = 0; idx < k_loops; ++idx) { + const uint flat_offset = idx * WorkGroupSize + wg_lane; + const uint row_idx = flat_offset / hsk_v4; + const uint col_idx = flat_offset % hsk_v4; + const uint sh_chunk_c = col_idx % slm_col_chunk; + const uint sh_chunk_r = col_idx / slm_col_chunk; + const uint ksh_offset = (row_idx * TK + sh_chunk_r * TK * p_per_loop) / 4 + sh_chunk_c; + F16V4Poolsh.F16V4sh[ksh_offset] = kv4_temp[idx]; + } + } } - } } barrier(); - if (pLoop + 1 < P_LOOP_COUNT) { - [[unroll]] for (uint kLoad = 0; kLoad < K_LOAD_LOOPS; kLoad++) { - const uint flatOffset = kLoad * GROUPSIZE + wgLane; - const uint kRowIdx = flatOffset / HEAD_DIM_VEC4; - const uint kColIdx = flatOffset % HEAD_DIM_VEC4; - const uint slmChunkCol = kColIdx % SLM_CHUNK_SIZE; - const uint slmChunkRow = kColIdx / SLM_CHUNK_SIZE; - const uint offsetSlmK = kRowIdx * TK + slmChunkRow * TK * MATP_PER_LOOP + slmChunkCol * 4; - slm_pool_f16.slm_pool_k[offsetSlmK + 0] = kTemp[kLoad].x; - slm_pool_f16.slm_pool_k[offsetSlmK + 1] = kTemp[kLoad].y; - slm_pool_f16.slm_pool_k[offsetSlmK + 2] = kTemp[kLoad].z; - slm_pool_f16.slm_pool_k[offsetSlmK + 3] = kTemp[kLoad].w; - } - } - } - - barrier(); - - if (tokFlatIdx < p.activationLength) { - [[unroll]] for (uint mk = 0; mk < MATP_REDUCE / WARPSIZE; mk++) { - slm_pool_f32.slm_pool_pv[SLM_PV_SIZE + localLinearId * MATP_REDUCE + mk * WARPSIZE + lane] = maskFp32[mk]; - } - } - - [[unroll]] for (uint oLoop = 0; oLoop < N_TOK; oLoop++) { - if (oLoop + outGroupIdx * N_TOK < p.activationLength) { - if (localLinearId == oLoop) { - [[unroll]] for (uint mp = 0; mp < COOP_MAT_P_M; mp++) { - [[unroll]] for (uint np = 0; np < COOP_MAT_P_N; np++) { - coopMatStore(matP[mp][np], slm_pool_f32.slm_pool_pv, mp * MATP_REDUCE * TN + np * TM, MATP_REDUCE, gl_CooperativeMatrixLayoutColumnMajor); - } - } - } - - barrier(); - - [[unroll]] for (uint maskIdx = 0; maskIdx < MATP_REDUCE / WARPSIZE; maskIdx++) { - maskFp32[maskIdx] = slm_pool_f32.slm_pool_pv[SLM_PV_SIZE + oLoop * MATP_REDUCE + maskIdx * WARPSIZE + lane]; - } - - float fp32O[O_COUNT][MATP_REDUCE / WARPSIZE]; - float maxOut[O_COUNT]; - - [[unroll]] for (uint oc = 0; oc < O_COUNT; oc++) { - [[unroll]] for (uint os = 0; os < MATP_REDUCE / WARPSIZE; os++) { - fp32O[oc][os] = slm_pool_f32.slm_pool_pv[offsetSlmLoadPv + os * WARPSIZE + oc * MATP_REDUCE] * p.softMaxScale; + if (sg < Br && out_row_idx < p.neq1 && single_head_mask) { + [[unroll]] for (uint idx = 0; idx < p_reduce / SubGroupSize; ++idx) { + F32Poolsh.F32sh[SLM_PV_SIZE + sg * p_reduce + idx * SubGroupSize + lane] = Maskf[idx]; } + } - [[unroll]] for (uint os = 0; os < MATP_REDUCE / WARPSIZE; os++) { - fp32O[oc][os] = fp32O[oc][os] + maskFp32[os]; - } - - float maxTemp = fp32Min; - [[unroll]] for (uint os = 0; os < MATP_REDUCE / WARPSIZE; os++) { - maxTemp = max(maxTemp, fp32O[oc][os]); - } - maxOut[oc] = subgroupMax(maxTemp); - [[unroll]] for (uint os = 0; os < MATP_REDUCE / WARPSIZE; os++) { - fp32O[oc][os] = exp(fp32O[oc][os] - maxOut[oc]); - } - } - - [[unroll]] for (uint oc = 0; oc < O_COUNT; oc++) { - if (localLinearId * O_COUNT + oc < GQA_RATIO) { - [[unroll]] for (uint os = 0; os < MATP_REDUCE / WARPSIZE; os++) { - matP_f16[offsetOut + oc * pDim + os * WARPSIZE] = float16_t(fp32O[oc][os]); - } - - if (lane == 0) { - out_max_f32[offsetMax + oc * maxDim] = maxOut[oc]; - } + [[unroll]] for (uint out_r = 0; out_r < Br; ++out_r) { + if (out_r + out_idx * Br < p.neq1) { + const uint mask_global_base = batch_mask_idx * batch_mask_stride + (out_r + out_idx * Br) * p.nek1; + if (sg == out_r) { + [[unroll]] for (uint r = 0; r < pmat_row; ++r) { + [[unroll]] for (uint c = 0; c < pmat_col; ++c) { + const uint coord = r * p_reduce * TN + c * TM; + coopMatStore(PMat[r][c], F32Poolsh.F32sh, coord, p_reduce, gl_CooperativeMatrixLayoutColumnMajor); + } + } + } + + barrier(); + + if (single_head_mask) { + [[unroll]] for (uint idx = 0; idx < p_reduce / SubGroupSize; ++idx) { + Maskf[idx] = F32Poolsh.F32sh[SLM_PV_SIZE + out_r * p_reduce + idx * SubGroupSize + lane]; + } + } + + float Of[warp_o_row][p_reduce / SubGroupSize]; + float Mf[warp_o_row]; + + [[unroll]] for (uint r = 0; r < warp_o_row; ++r) { + const uint head_q_idx = head_k_idx * gqa_ratio + sg * warp_o_row + r; + const uint mask_head_offset = (head_q_idx % p.nem2) * p.nek1 * p.nem1; + + [[unroll]] for (uint idx = 0; idx < p_reduce / SubGroupSize; ++idx) { + Of[r][idx] = F32Poolsh.F32sh[pvsh_load_offset + idx * SubGroupSize + r * p_reduce] * p.scale; + } + + if (!single_head_mask) { + if (out_r + out_idx * Br < p.nem1) { + [[unroll]] for (uint idx = 0; idx < p_reduce / SubGroupSize; ++idx) { + const uint mask_offset = idx * SubGroupSize + mask_base; + if (mask_offset < p.nek1) { + Maskf[idx] = float(data_m[mask_global_base + mask_head_offset + mask_offset]); + } else { + Maskf[idx] = NEG_FLT_MAX_OVER_2; + } + } + } else { + [[unroll]] for (uint idx = 0; idx < p_reduce / SubGroupSize; ++idx) { + const uint mask_offset = idx * SubGroupSize + mask_base; + Maskf[idx] = (mask_offset < p.nek1) ? 0.0f : NEG_FLT_MAX_OVER_2; + } + } + } + + [[unroll]] for (uint idx = 0; idx < p_reduce / SubGroupSize; ++idx) { + Of[r][idx] = Of[r][idx] + Maskf[idx]; + } + + float Mtempf = NEG_FLT_MAX_OVER_2; + [[unroll]] for (uint idx = 0; idx < p_reduce / SubGroupSize; ++idx) { + const uint p_col_offset = idx * SubGroupSize + mask_base; + if (p_col_offset < p.nek1) { + Mtempf = max(Mtempf, Of[r][idx]); + } + } + Mf[r] = subgroupMax(Mtempf); + [[unroll]] for (uint idx = 0; idx < p_reduce / SubGroupSize; ++idx) { + const uint p_col_offset = idx * SubGroupSize + mask_base; + Of[r][idx] = (p_col_offset < p.nek1) ? exp(Of[r][idx] - Mf[r]) : 0.0f; + } + } + + [[unroll]] for (uint r = 0; r < warp_o_row; ++r) { + if (sg * warp_o_row + r < gqa_ratio) { + [[unroll]] for (uint idx = 0; idx < p_reduce / SubGroupSize; ++idx) { + data_p[o_offset + r * p_dim + idx * SubGroupSize] = float16_t(Of[r][idx]); + } + + if (lane == 0) { + data_max[max_offset + r * max_dim] = Mf[r]; + } + } + } + + o_offset = o_offset + p.neq2 * p_dim; + max_offset = max_offset + p.neq2 * max_dim; + barrier(); } - } - - offsetOut = offsetOut + p.qHead * pDim; - offsetMax = offsetMax + p.qHead * maxDim; - barrier(); } - } } diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_decode_phase_2.comp b/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_decode_phase_2.comp index 60a1c2ce7362..42a5101d8542 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_decode_phase_2.comp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_decode_phase_2.comp @@ -3,406 +3,470 @@ #extension GL_EXT_control_flow_attributes : enable #extension GL_EXT_shader_16bit_storage : require #extension GL_EXT_shader_explicit_arithmetic_types_float16 : require -#extension GL_EXT_shader_explicit_arithmetic_types_int16 : require #extension GL_KHR_memory_scope_semantics : enable #extension GL_KHR_shader_subgroup_basic : enable -#extension GL_KHR_shader_subgroup_ballot : enable #extension GL_KHR_shader_subgroup_arithmetic : enable #extension GL_KHR_cooperative_matrix : enable #extension GL_EXT_shared_memory_block : enable -layout(local_size_x_id = 0, local_size_y = 1, local_size_z = 1) in; +layout (local_size_x_id = 0, local_size_y = 1, local_size_z = 1) in; -layout (binding = 0) readonly buffer P {f16vec4 pStateVec4[];}; -layout (binding = 1) readonly buffer V {float16_t vState[];}; -layout (binding = 1) readonly buffer V_VEC4 {f16vec4 vStateVec4[];}; -layout (binding = 2) buffer MAX_FP32 {float max_f32[];}; -layout (binding = 3) buffer SINK_FP32 {float sink_f32[];}; -layout (binding = 4) buffer OUT_FP32 {float out_f32[];}; -layout (binding = 4) buffer OUT_VEC4 {vec4 out_f32_vec4[];}; -layout (binding = 4) buffer OUT_F16 {float16_t out_f16[];}; +layout (binding = 0) readonly buffer PV4 {f16vec4 data_pv4[];}; +layout (binding = 1) readonly buffer V {float16_t data_v[];}; +layout (binding = 1) readonly buffer VV4 {f16vec4 data_vv4[];}; +layout (binding = 2) readonly buffer MAX {float data_max[];}; +layout (binding = 3) readonly buffer S {float data_s[];}; +layout (binding = 4) buffer OV4 {vec4 data_ov4[];}; layout (push_constant) uniform parameter { - uint kvSeqLen; - uint activationLength; - uint qHead; - uint kvHead; - uint qkRatio; - uint qkSubGroups; - uint flag; - uint kvStride1; - uint kvStride2; - uint batchStrideQ; - uint batchStrideK; - uint batchStrideV; - uint batchStrideM; - uint batchStrideO; - float softMaxScale; + uint neq1; + uint neq2; + uint neq3; + uint nek1; + uint nek2; + uint nek3; + uint nev2; + uint nev3; + uint nem1; + uint nem2; + uint nem3; + uint nbq1; + uint nbq2; + uint nbk1; + uint nbk2; + uint nbv1; + uint nbv2; + uint nbq3; + uint nbk3; + uint nbv3; + uint nb3; + uint flag; + float scale; } p; -layout (constant_id = 0) const uint GROUPSIZE = 256; -layout (constant_id = 1) const uint GQA_RATIO = 8; -layout (constant_id = 2) const uint HEAD_DIM = 128; -layout (constant_id = 3) const uint N_TOKS_PER_GROUP = 1; -layout (constant_id = 4) const uint WARPSIZE = 16; -layout (constant_id = 5) const uint MATP_PER_LOOP = 64; -layout (constant_id = 6) const uint MATP_REDUCE = 32; -layout (constant_id = 7) const uint WARP_V_DIM = 16; - -#define TN WARPSIZE -#define TM 8 -#define TK 16 -#define MAT_O_N (WARP_V_DIM / TM) -#define MAT_P_M (GQA_RATIO * N_TOKS_PER_GROUP) -#define ALIGNED_P_M ((MAT_P_M + WARPSIZE - 1) / WARPSIZE) -#define V_HEAD_GROUPS (HEAD_DIM / WARP_V_DIM) - -#define SUBGROUP_COUNT (GROUPSIZE / WARPSIZE) -#define SPLIT_P_GROUPS (MATP_PER_LOOP / TK) - -#define SLM_POOL_SIZE_O (SUBGROUP_COUNT * ALIGNED_P_M * TN * MAT_O_N * TM) - -#define P_LOAD_PER_LOOP (GROUPSIZE * 4) -#define P_LOAD_LOOPS ((MAT_P_M * MATP_PER_LOOP + P_LOAD_PER_LOOP - 1) / P_LOAD_PER_LOOP) -#define SLM_POOL_SIZE_P (P_LOAD_LOOPS * P_LOAD_PER_LOOP) -#define SIZE_LOCAL_MAX (MAT_P_M * MATP_PER_LOOP / MATP_REDUCE) -#define MAX_LOAD_LOOPS ((SIZE_LOCAL_MAX + GROUPSIZE - 1) / GROUPSIZE) -#define SLM_POOL_SIZE_LOCAL_MAX (MAX_LOAD_LOOPS * GROUPSIZE) -#define MAX_REDUCE_COUNT ((MAT_P_M + SUBGROUP_COUNT - 1) / SUBGROUP_COUNT) -#define GLOBAL_MAX_SIZE (MAX_REDUCE_COUNT * SUBGROUP_COUNT) - -#define SLM_POOL_SIZE_SOFTMAX_SUM (SUBGROUP_COUNT * P_LOAD_LOOPS) - -#define SLM_OFFSET_P (GLOBAL_MAX_SIZE * 2 + SLM_POOL_SIZE_SOFTMAX_SUM * 2 + SLM_POOL_SIZE_LOCAL_MAX * 2 * 2) - -#define SLM_OFFSET_GLOBAL_MAX 0 -#define SLM_OFFSET_SOFTMAX_SUM (SLM_OFFSET_GLOBAL_MAX + GLOBAL_MAX_SIZE) -#define SLM_OFFSET_O (SLM_OFFSET_SOFTMAX_SUM + SLM_POOL_SIZE_SOFTMAX_SUM) -#define SLM_OFFSET_LOCAL_MAX (GLOBAL_MAX_SIZE + SLM_POOL_SIZE_SOFTMAX_SUM) - -#define P_REDUCE_VEC4 (MATP_PER_LOOP / 4) -#define MAX_PER_LOOP (MATP_PER_LOOP / MATP_REDUCE) -#define SLM_MAX_STRIDE (MATP_REDUCE / 4) -#define SUB_GROUPS_PER_LINE (MATP_PER_LOOP / WARPSIZE / 4) - -shared slm_pool_block { - float slm_pool_o[GLOBAL_MAX_SIZE + SLM_POOL_SIZE_SOFTMAX_SUM + SLM_POOL_SIZE_O]; -} slm_pool_f32; - -shared slm_pool_alias_block { - float16_t slm_pool_pv[GLOBAL_MAX_SIZE * 2 + SLM_POOL_SIZE_SOFTMAX_SUM * 2 + SLM_POOL_SIZE_LOCAL_MAX * 2 * 2 + SLM_POOL_SIZE_P * 2]; -} slm_pool_alias_f16; +layout (constant_id = 0) const uint WorkGroupSize = 256; +layout (constant_id = 1) const uint gqa_ratio = 8; +layout (constant_id = 2) const uint HSV = 128; +layout (constant_id = 3) const uint Wr = 1; +layout (constant_id = 4) const uint SubGroupSize = 16; +layout (constant_id = 5) const uint p_per_loop = 64; +layout (constant_id = 6) const uint p_reduce = 32; +layout (constant_id = 7) const uint Bc = 16; +layout (constant_id = 8) const uint TM = 8; +layout (constant_id = 9) const uint TN = 16; +layout (constant_id = 10) const uint Flags = 0; + +#define UNALIGNED_SEQ_LEN_BIT (1 << 0) +#define SINK_ENABLE_BIT (1 << 16) + +const uint TK = 16; +const uint o_row = gqa_ratio * Wr; +const uint omat_row = (o_row + TN - 1) / TN; +const uint o_scalar_row = (o_row + SubGroupSize - 1) / SubGroupSize; + +const uint omat_col = Bc / TM; +const uint v_head_wgs = HSV / Bc; + +const uint sg_count = WorkGroupSize / SubGroupSize; +const uint split_p_sg = p_per_loop / TK; +const uint p_loop_col = WorkGroupSize * 4; +const uint p_loops = (o_row * p_per_loop + p_loop_col - 1) / p_loop_col; + +const uint size_max = o_row * p_per_loop / p_reduce; +const uint max_loops = (size_max + WorkGroupSize - 1) / WorkGroupSize; +const uint global_max_loops = (o_row + sg_count - 1) / sg_count; + +#define SLM_POOL_SIZE_GLOBAL_MAX ((global_max_loops * sg_count + SubGroupSize - 1) / SubGroupSize * SubGroupSize) +#define SLM_POOL_SIZE_O (sg_count * omat_row * TN * omat_col * TM) +#define SLM_POOL_SIZE_MAX (max_loops * WorkGroupSize) +#define SLM_POOL_SIZE_SOFTMAX_SUM ((sg_count * p_loops + SubGroupSize - 1) / SubGroupSize * SubGroupSize) +#define SLM_POOL_SIZE_OOB_STAGING (sg_count * TM * TK / 4) + +const uint psh_size = (p_loops * p_loop_col < omat_row * TN * p_per_loop) ? (omat_row * TN * p_per_loop) : (p_loops * p_loop_col); +const uint psh_base = SLM_POOL_SIZE_GLOBAL_MAX * 2 + SLM_POOL_SIZE_SOFTMAX_SUM * 2 + SLM_POOL_SIZE_MAX * 2 * 2; + +const uint global_maxsh_offset = 0; +const uint lfsh_offset = global_maxsh_offset + SLM_POOL_SIZE_GLOBAL_MAX; +const uint osh_offset = lfsh_offset + SLM_POOL_SIZE_SOFTMAX_SUM; +const uint maxsh_offset = SLM_POOL_SIZE_GLOBAL_MAX + SLM_POOL_SIZE_SOFTMAX_SUM; + +const uint p_v4 = p_per_loop / 4; +const uint max_per_loop = p_per_loop / p_reduce; +const uint maxsh_stride = p_reduce / 4; +const uint sg_per_line = p_per_loop / SubGroupSize / 4; +const bool seq_boundary_check = (Flags & UNALIGNED_SEQ_LEN_BIT) != 0; + +shared sh_block { + float F32sh[SLM_POOL_SIZE_GLOBAL_MAX + SLM_POOL_SIZE_SOFTMAX_SUM + SLM_POOL_SIZE_O]; +} F32Poolsh; + +shared shf16_alias { + float16_t F16sh[SLM_POOL_SIZE_GLOBAL_MAX * 2 + SLM_POOL_SIZE_SOFTMAX_SUM * 2 + SLM_POOL_SIZE_MAX * 2 * 2 + psh_size * 2]; +} F16Poolsh; + +shared shf16v4_alias { + f16vec4 F16V4sh[(SLM_POOL_SIZE_GLOBAL_MAX + SLM_POOL_SIZE_SOFTMAX_SUM + SLM_POOL_SIZE_MAX * 2) / 2 + SLM_POOL_SIZE_OOB_STAGING]; +} F16V4Poolsh; void main() { - const uint lane = gl_SubgroupInvocationID; - const uint v = gl_WorkGroupID.y; - const uint d = gl_WorkGroupID.z; - const uint vWarpIdx = gl_WorkGroupID.x % V_HEAD_GROUPS; - const uint outTokIdx = gl_WorkGroupID.x / V_HEAD_GROUPS; - const uint localLinearId = gl_SubgroupID; - const uint wgLane = localLinearId * WARPSIZE + lane; - const uint splitIdx = localLinearId; - const uint maxDim = (p.kvSeqLen + MATP_REDUCE - 1) / MATP_REDUCE; - const uint pDim = maxDim * MATP_REDUCE; - const uint kvDim = p.kvStride1; - const uint oDim = p.qHead * HEAD_DIM; - const uint offsetBaseP = (d * p.activationLength * p.qHead + v * GQA_RATIO + outTokIdx * N_TOKS_PER_GROUP * p.qHead) * pDim / 4; - const uint offsetBaseMax = (d * p.activationLength * p.qHead + v * GQA_RATIO + outTokIdx * N_TOKS_PER_GROUP * p.qHead) * maxDim; - const uint offsetBaseV = (d * p.batchStrideV + v * p.kvStride2 + vWarpIdx * WARP_V_DIM + splitIdx * TK * kvDim); - const uint offsetSlmP = (SLM_OFFSET_P + wgLane * 4); - const float fp32Min = uintBitsToFloat(0xFEFFFFFF); - const float fp32Max = uintBitsToFloat(0x7EFFFFFF); - uint offsetV = offsetBaseV; - - coopmat sums[ALIGNED_P_M][MAT_O_N]; - f16vec4 pStateTemp[P_LOAD_LOOPS]; - - float fp32CompensationP[P_LOAD_LOOPS]; - - uint loadRowBase[P_LOAD_LOOPS]; - uint loadColBase[P_LOAD_LOOPS]; - float fp32SoftMaxSum[P_LOAD_LOOPS]; - float fp32GlobalMaxP[P_LOAD_LOOPS]; - uint maxRowBase[MAX_LOAD_LOOPS]; - uint maxColBase[MAX_LOAD_LOOPS]; - uint outOffsets[ALIGNED_P_M]; - bool outputMask[ALIGNED_P_M]; - float fp32SinkCoeff[ALIGNED_P_M]; - - [[unroll]] for (uint pm = 0; pm < ALIGNED_P_M; pm++) { - const uint flatOffset = pm * WARPSIZE + lane; - const uint inGroupTokIdx = flatOffset / GQA_RATIO; - const uint inGroupHeadIdx = flatOffset % GQA_RATIO; - outputMask[pm] = (N_TOKS_PER_GROUP * outTokIdx + inGroupTokIdx < p.activationLength) && (inGroupHeadIdx < GQA_RATIO) && (inGroupTokIdx < N_TOKS_PER_GROUP); - outOffsets[pm] = (inGroupTokIdx * oDim + inGroupHeadIdx * HEAD_DIM) / 4; - if ((0x1 & p.flag) != 0) { - fp32SinkCoeff[pm] = sink_f32[inGroupHeadIdx + v * GQA_RATIO]; - } - } - - [[unroll]] for (uint maxCount = 0; maxCount < MAX_REDUCE_COUNT; maxCount++) { - const uint flatIdx = maxCount * SUBGROUP_COUNT + localLinearId; - const uint rowIdx = flatIdx % GQA_RATIO; - const uint tokIdx = flatIdx / GQA_RATIO; - - if (tokIdx < N_TOKS_PER_GROUP) { - float fp32MaxReduce = fp32Min; - const uint maxOffset = offsetBaseMax + (tokIdx * p.qHead + rowIdx) * maxDim; - [[unroll]] for (uint maxReduce = 0; maxReduce < (maxDim + WARPSIZE - 1) / WARPSIZE; maxReduce++) { - if (maxReduce * WARPSIZE + lane < maxDim) { - fp32MaxReduce = max(fp32MaxReduce, max_f32[maxOffset + maxReduce * WARPSIZE + lane]); + const uint lane = gl_SubgroupInvocationID; + const uint head_v_idx = gl_WorkGroupID.y; + const uint batch_v_ratio = p.neq3 / p.nev3; + const uint batch_idx = gl_WorkGroupID.z; + const uint batch_v_idx = batch_idx / batch_v_ratio; + const uint v_idx = gl_WorkGroupID.x % v_head_wgs; + const uint out_row_idx = gl_WorkGroupID.x / v_head_wgs; + const uint sg = gl_SubgroupID; + const uint wg_lane = sg * SubGroupSize + lane; + const uint max_dim = (p.nek1 + p_reduce - 1) / p_reduce; + const uint p_dim = max_dim * p_reduce; + const uint v_dim = p.nbv1; + const uint o_dim = p.neq2 * HSV; + const uint p_base = (batch_idx * p.neq1 * p.neq2 + head_v_idx * gqa_ratio + out_row_idx * Wr * p.neq2) * p_dim / 4; + const uint max_base = (batch_idx * p.neq1 * p.neq2 + head_v_idx * gqa_ratio + out_row_idx * Wr * p.neq2) * max_dim; + const uint v_base = batch_v_idx * p.nbv3 + head_v_idx * p.nbv2 + v_idx * Bc + sg * TK * v_dim; + const uint o_base = (batch_idx * p.nb3 + v_idx * Bc + head_v_idx * gqa_ratio * HSV + out_row_idx * o_dim * Wr) / 4; + const uint psh_offset = psh_base + wg_lane * 4; + const float NEG_FLT_MAX_OVER_2 = uintBitsToFloat(0xFEFFFFFF); + const float POS_FLT_MAX_OVER_2 = uintBitsToFloat(0x7EFFFFFF); + uint v_offset = v_base; + + coopmat OMat[omat_row][omat_col]; + f16vec4 pv4_temp[p_loops]; + + float eMf[p_loops]; + + uint p_row_base[p_loops]; + uint p_col_base[p_loops]; + float Lf[p_loops]; + float global_max[p_loops]; + uint max_row_base[max_loops]; + uint max_col_base[max_loops]; + uint o_offset[o_scalar_row]; + bool o_mask[o_scalar_row]; + float sink[o_scalar_row]; + + [[unroll]] for (uint r = 0; r < o_scalar_row; ++r) { + const uint flat_offset = r * SubGroupSize + lane; + const uint row_idx = flat_offset / gqa_ratio; + const uint col_idx = flat_offset % gqa_ratio; + o_mask[r] = (Wr * out_row_idx + row_idx < p.neq1) && (col_idx < gqa_ratio) && (row_idx < Wr); + o_offset[r] = (row_idx * o_dim + col_idx * HSV) / 4; + if ((p.flag & SINK_ENABLE_BIT) != 0) { + sink[r] = data_s[col_idx + head_v_idx * gqa_ratio]; } - } - fp32MaxReduce = subgroupMax(fp32MaxReduce); - if (lane == 0) { - slm_pool_f32.slm_pool_o[SLM_OFFSET_GLOBAL_MAX + maxCount * SUBGROUP_COUNT + localLinearId] = fp32MaxReduce; - } - } else { - if (lane == 0) { - slm_pool_f32.slm_pool_o[SLM_OFFSET_GLOBAL_MAX + maxCount * SUBGROUP_COUNT + localLinearId] = fp32Max; - } - } - } - - barrier(); - - [[unroll]] for (uint pLoad = 0; pLoad < P_LOAD_LOOPS; pLoad++) { - const uint flatOffset = (pLoad * GROUPSIZE + wgLane) / P_REDUCE_VEC4; - const uint rowIdxFlat = flatOffset % GQA_RATIO; - const uint tokenIdxFlat = min(flatOffset / GQA_RATIO, N_TOKS_PER_GROUP - 1); - loadColBase[pLoad] = (pLoad * GROUPSIZE + wgLane) % P_REDUCE_VEC4; - loadRowBase[pLoad] = (tokenIdxFlat * p.qHead + rowIdxFlat); - fp32SoftMaxSum[pLoad] = 0.0f; - fp32GlobalMaxP[pLoad] = slm_pool_f32.slm_pool_o[SLM_OFFSET_GLOBAL_MAX + flatOffset]; - } - - [[unroll]] for (uint maxLoad = 0; maxLoad < MAX_LOAD_LOOPS; maxLoad++) { - const uint flatOffset = (maxLoad * GROUPSIZE + wgLane) / MAX_PER_LOOP; - const uint rowIdxFlat = flatOffset % GQA_RATIO; - const uint tokenIdxFlat = min(flatOffset / GQA_RATIO, N_TOKS_PER_GROUP - 1); - maxColBase[maxLoad] = (maxLoad * GROUPSIZE + wgLane) % MAX_PER_LOOP; - maxRowBase[maxLoad] = (tokenIdxFlat * p.qHead + rowIdxFlat); - } - - [[unroll]] for (uint maxLoad = 0; maxLoad < MAX_LOAD_LOOPS; maxLoad++) { - const uint flatMaxOffset = maxRowBase[maxLoad] * maxDim + maxColBase[maxLoad]; - slm_pool_f32.slm_pool_o[SLM_OFFSET_LOCAL_MAX + maxLoad * GROUPSIZE + wgLane] = max_f32[offsetBaseMax + flatMaxOffset]; - maxColBase[maxLoad] = maxColBase[maxLoad] + MATP_PER_LOOP / MATP_REDUCE; - } - - [[unroll]] for (uint pLoad = 0; pLoad < P_LOAD_LOOPS; pLoad++) { - const uint flatOffset = loadRowBase[pLoad] * pDim / 4 + loadColBase[pLoad]; - pStateTemp[pLoad] = pStateVec4[offsetBaseP + flatOffset]; - } - - [[unroll]] for (uint n = 0; n < ALIGNED_P_M; n++) { - [[unroll]] for (uint i = 0; i < MAT_O_N; i++) { - sums[n][i] = coopmat(0.0f); - } - } - - barrier(); - - [[unroll]] for (uint pLoad = 0; pLoad < P_LOAD_LOOPS; pLoad++) { - const uint maxOffset = (pLoad * GROUPSIZE + wgLane) / SLM_MAX_STRIDE; - if (loadColBase[pLoad] < pDim / 4) { - fp32CompensationP[pLoad] = slm_pool_f32.slm_pool_o[SLM_OFFSET_LOCAL_MAX + maxOffset]; - float pTemp[4] = float[4](pStateTemp[pLoad].x, pStateTemp[pLoad].y, pStateTemp[pLoad].z, pStateTemp[pLoad].w); - float compTemp = exp(fp32CompensationP[pLoad] - fp32GlobalMaxP[pLoad]); - [[unroll]] for (uint kk = 0; kk < 4; kk++) { - pTemp[kk] = pTemp[kk] * compTemp; - fp32SoftMaxSum[pLoad] = fp32SoftMaxSum[pLoad] + pTemp[kk]; - slm_pool_alias_f16.slm_pool_pv[offsetSlmP + pLoad * GROUPSIZE * 4 + kk] = float16_t(pTemp[kk]); - } - } else { - [[unroll]] for (uint kk = 0; kk < 4; kk++) { - slm_pool_alias_f16.slm_pool_pv[offsetSlmP + pLoad * GROUPSIZE * 4 + kk] = float16_t(0.0f); - } } - loadColBase[pLoad] = loadColBase[pLoad] + P_REDUCE_VEC4; - } + if (p.nek1 == 0) { + if (sg == 0) { + [[unroll]] for (uint c = 0; c < Bc / 4; ++c) { + [[unroll]] for (uint r = 0; r < o_scalar_row; ++r) { + if (o_mask[r]) { + data_ov4[o_base + o_offset[r] + c] = vec4(0.0f, 0.0f, 0.0f, 0.0f); + } + } + } + } - const uint loopCount = (p.kvSeqLen + MATP_PER_LOOP - 1) / MATP_PER_LOOP; + return; + } - for (uint loop = 0; loop < loopCount; loop++) { - const uint slmPingPongLoad = (loop & 0x1); - const uint slmPingPongStore = ((loop + 1) & 0x1); + [[unroll]] for (uint idx = 0; idx < global_max_loops; ++idx) { + const uint flat_offset = idx * sg_count + sg; + const uint row_idx = flat_offset % gqa_ratio; + const uint tok_idx = flat_offset / gqa_ratio; + + if (tok_idx < Wr) { + float Maxf = NEG_FLT_MAX_OVER_2; + const uint max_offset = max_base + (tok_idx * p.neq2 + row_idx) * max_dim; + [[unroll]] for (uint max_reduce = 0; max_reduce < (max_dim + SubGroupSize - 1) / SubGroupSize; ++max_reduce) { + if (max_reduce * SubGroupSize + lane < max_dim) { + Maxf = max(Maxf, data_max[max_offset + max_reduce * SubGroupSize + lane]); + } + } + Maxf = subgroupMax(Maxf); + if (lane == 0) { + F32Poolsh.F32sh[global_maxsh_offset + idx * sg_count + sg] = Maxf; + } + } else { + if (lane == 0) { + F32Poolsh.F32sh[global_maxsh_offset + idx * sg_count + sg] = POS_FLT_MAX_OVER_2; + } + } + } - if (loop + 1 < loopCount) { - [[unroll]] for (uint pLoad = 0; pLoad < P_LOAD_LOOPS; pLoad++) { - const uint flatOffset = loadRowBase[pLoad] * pDim / 4 + loadColBase[pLoad]; - pStateTemp[pLoad] = pStateVec4[offsetBaseP + flatOffset]; - } + barrier(); - [[unroll]] for (uint maxLoad = 0; maxLoad < MAX_LOAD_LOOPS; maxLoad++) { - const uint flatMaxOffset = maxRowBase[maxLoad] * maxDim + maxColBase[maxLoad]; - slm_pool_f32.slm_pool_o[SLM_OFFSET_LOCAL_MAX + slmPingPongStore * SLM_POOL_SIZE_LOCAL_MAX + maxLoad * GROUPSIZE + wgLane] = max_f32[offsetBaseMax + flatMaxOffset]; - maxColBase[maxLoad] = maxColBase[maxLoad] + MATP_PER_LOOP / MATP_REDUCE; - } + [[unroll]] for (uint idx = 0; idx < p_loops; ++idx) { + const uint flat_offset = (idx * WorkGroupSize + wg_lane) / p_v4; + const uint row_idx = flat_offset % gqa_ratio; + const uint tok_idx = min(flat_offset / gqa_ratio, Wr - 1); + p_col_base[idx] = (idx * WorkGroupSize + wg_lane) % p_v4; + p_row_base[idx] = (tok_idx * p.neq2 + row_idx); + Lf[idx] = 0.0f; + global_max[idx] = F32Poolsh.F32sh[global_maxsh_offset + flat_offset]; } - barrier(); + [[unroll]] for (uint idx = 0; idx < max_loops; ++idx) { + const uint flat_offset = (idx * WorkGroupSize + wg_lane) / max_per_loop; + const uint row_idx = flat_offset % gqa_ratio; + const uint tok_idx = min(flat_offset / gqa_ratio, Wr - 1); + max_col_base[idx] = (idx * WorkGroupSize + wg_lane) % max_per_loop; + max_row_base[idx] = (tok_idx * p.neq2 + row_idx); + } - { - const uint coopMatOffsetP = SLM_OFFSET_P + slmPingPongLoad * SLM_POOL_SIZE_P + splitIdx * TK; - coopmat matV[MAT_O_N]; - [[unroll]] for (uint cc = 0; cc < MAT_O_N; cc++) { - coopMatLoad( - matV[cc], - vState, - offsetV + TM * cc, - kvDim, - gl_CooperativeMatrixLayoutColumnMajor); - } - [[unroll]] for (uint mo = 0; mo < ALIGNED_P_M; mo++) { - coopmat matP; - coopMatLoad( - matP, - slm_pool_alias_f16.slm_pool_pv, - coopMatOffsetP + mo * TN * MATP_PER_LOOP, - MATP_PER_LOOP, - gl_CooperativeMatrixLayoutColumnMajor); - - [[unroll]] for (uint no = 0; no < MAT_O_N; no++) { - sums[mo][no] = coopMatMulAdd(matV[no], matP, sums[mo][no]); + [[unroll]] for (uint idx = 0; idx < max_loops; ++idx) { + const uint flat_offset = max_row_base[idx] * max_dim + max_col_base[idx]; + if (seq_boundary_check) { + F32Poolsh.F32sh[maxsh_offset + idx * WorkGroupSize + wg_lane] = + (max_col_base[idx] < max_dim) ? data_max[max_base + flat_offset] : NEG_FLT_MAX_OVER_2; + } else { + F32Poolsh.F32sh[maxsh_offset + idx * WorkGroupSize + wg_lane] = data_max[max_base + flat_offset]; } - } + max_col_base[idx] = max_col_base[idx] + p_per_loop / p_reduce; } - offsetV += MATP_PER_LOOP * kvDim; - if (loop * MATP_PER_LOOP + splitIdx * TK >= p.kvSeqLen) { - offsetV = 0; - } - if (loop + 1 < loopCount) { - [[unroll]] for (uint pLoad = 0; pLoad < P_LOAD_LOOPS; pLoad++) { - const uint maxOffset = (pLoad * GROUPSIZE + wgLane) / SLM_MAX_STRIDE; - if (loadColBase[pLoad] < pDim / 4) { - fp32CompensationP[pLoad] = slm_pool_f32.slm_pool_o[SLM_OFFSET_LOCAL_MAX + slmPingPongStore * SLM_POOL_SIZE_LOCAL_MAX + maxOffset]; - float pTemp[4] = float[4](pStateTemp[pLoad].x, pStateTemp[pLoad].y, pStateTemp[pLoad].z, pStateTemp[pLoad].w); - float compTemp = exp(fp32CompensationP[pLoad] - fp32GlobalMaxP[pLoad]); - [[unroll]] for (uint kk = 0; kk < 4; kk++) { - pTemp[kk] = pTemp[kk] * compTemp; - fp32SoftMaxSum[pLoad] = fp32SoftMaxSum[pLoad] + pTemp[kk]; - slm_pool_alias_f16.slm_pool_pv[offsetSlmP + slmPingPongStore * SLM_POOL_SIZE_P + pLoad * GROUPSIZE * 4 + kk] = float16_t(pTemp[kk]); - } + [[unroll]] for (uint idx = 0; idx < p_loops; ++idx) { + const uint flat_offset = p_row_base[idx] * p_dim / 4 + p_col_base[idx]; + if (seq_boundary_check) { + pv4_temp[idx] = (p_col_base[idx] < p_dim / 4) ? data_pv4[p_base + flat_offset] : f16vec4(0.0f, 0.0f, 0.0f, 0.0f); } else { - [[unroll]] for (uint kk = 0; kk < 4; kk++) { - slm_pool_alias_f16.slm_pool_pv[offsetSlmP + slmPingPongStore * SLM_POOL_SIZE_P + pLoad * GROUPSIZE * 4 + kk] = float16_t(0.0f); - } + pv4_temp[idx] = data_pv4[p_base + flat_offset]; } - loadColBase[pLoad] = loadColBase[pLoad] + P_REDUCE_VEC4; - } } - } - - barrier(); - - [[unroll]] for (uint pLoad = 0; pLoad < P_LOAD_LOOPS; pLoad++) { - fp32SoftMaxSum[pLoad] = subgroupAdd(fp32SoftMaxSum[pLoad]); - } - - [[unroll]] for (uint mo = 0; mo < ALIGNED_P_M; mo++) { - [[unroll]] for (uint no = 0; no < MAT_O_N; no++) { - coopMatStore( - sums[mo][no], - slm_pool_f32.slm_pool_o, - SLM_OFFSET_O + mo * TN * WARP_V_DIM + TM * no + localLinearId * ALIGNED_P_M * TN * WARP_V_DIM, - WARP_V_DIM, - gl_CooperativeMatrixLayoutColumnMajor); + + [[unroll]] for (uint r = 0; r < omat_row; ++r) { + [[unroll]] for (uint c = 0; c < omat_col; ++c) { + OMat[r][c] = coopmat(0.0f); + } } - } - [[unroll]] for (uint pLoad = 0; pLoad < P_LOAD_LOOPS; pLoad++) { - slm_pool_f32.slm_pool_o[SLM_OFFSET_SOFTMAX_SUM + pLoad * SUBGROUP_COUNT + localLinearId] = fp32SoftMaxSum[pLoad]; - } + barrier(); - barrier(); + [[unroll]] for (uint idx = 0; idx < p_loops; ++idx) { + const uint max_offset = (idx * WorkGroupSize + wg_lane) / maxsh_stride; + eMf[idx] = F32Poolsh.F32sh[maxsh_offset + max_offset]; + float p_temp[4] = float[4](pv4_temp[idx].x, pv4_temp[idx].y, pv4_temp[idx].z, pv4_temp[idx].w); + float eMf_temp = exp(eMf[idx] - global_max[idx]); + [[unroll]] for (uint i = 0; i < 4; ++i) { + p_temp[i] = p_temp[i] * eMf_temp; + Lf[idx] = Lf[idx] + p_temp[i]; + F16Poolsh.F16sh[psh_offset + idx * WorkGroupSize * 4 + i] = float16_t(p_temp[i]); + } + p_col_base[idx] = p_col_base[idx] + p_v4; + } - if (localLinearId == 1) { - const uint sumBase = SLM_OFFSET_SOFTMAX_SUM + lane * SUB_GROUPS_PER_LINE; - float sumTemp[ALIGNED_P_M][SUB_GROUPS_PER_LINE]; - [[unroll]] for (uint pm = 0; pm < ALIGNED_P_M; pm++) { - [[unroll]] for (uint reduce = 0; reduce < SUB_GROUPS_PER_LINE; reduce++) { - sumTemp[pm][reduce] = slm_pool_f32.slm_pool_o[sumBase + pm * WARPSIZE * SUB_GROUPS_PER_LINE + reduce]; - } + const uint loop_count = seq_boundary_check ? ((p.nek1 + p_per_loop - 1) / p_per_loop - 1) : (p.nek1 / p_per_loop); + uint sh_pingpong_load = 0; + uint sh_pingpong_store = 0; + + for (uint loop = 0; loop < loop_count; ++loop) { + sh_pingpong_load = loop & 0x1; + sh_pingpong_store = (loop + 1) & 0x1; + + if (seq_boundary_check || loop + 1 < loop_count) { + [[unroll]] for (uint idx = 0; idx < p_loops; ++idx) { + const uint flat_offset = p_row_base[idx] * p_dim / 4 + p_col_base[idx]; + if (seq_boundary_check) { + pv4_temp[idx] = (p_col_base[idx] < p_dim / 4) ? data_pv4[p_base + flat_offset] : f16vec4(0.0f, 0.0f, 0.0f, 0.0f); + } else { + pv4_temp[idx] = data_pv4[p_base + flat_offset]; + } + } + + [[unroll]] for (uint idx = 0; idx < max_loops; ++idx) { + const uint flat_offset = max_row_base[idx] * max_dim + max_col_base[idx]; + if (seq_boundary_check) { + F32Poolsh.F32sh[maxsh_offset + sh_pingpong_store * SLM_POOL_SIZE_MAX + idx * WorkGroupSize + wg_lane] = + (max_col_base[idx] < max_dim) ? data_max[max_base + flat_offset] : NEG_FLT_MAX_OVER_2; + } else { + F32Poolsh.F32sh[maxsh_offset + sh_pingpong_store * SLM_POOL_SIZE_MAX + idx * WorkGroupSize + wg_lane] = data_max[max_base + flat_offset]; + } + max_col_base[idx] = max_col_base[idx] + p_per_loop / p_reduce; + } + } + + barrier(); + + { + const uint coord_p = psh_base + sh_pingpong_load * psh_size + sg * TK; + coopmat VMat[omat_col]; + [[unroll]] for (uint c = 0; c < omat_col; ++c) { + coopMatLoad(VMat[c], data_v, v_offset + TM * c, v_dim, gl_CooperativeMatrixLayoutColumnMajor); + } + [[unroll]] for (uint r = 0; r < omat_row; ++r) { + coopmat PMat; + coopMatLoad(PMat, F16Poolsh.F16sh, coord_p + r * TN * p_per_loop, p_per_loop, gl_CooperativeMatrixLayoutColumnMajor); + + [[unroll]] for (uint c = 0; c < omat_col; ++c) { + OMat[r][c] = coopMatMulAdd(VMat[c], PMat, OMat[r][c]); + } + } + } + + v_offset += p_per_loop * v_dim; + + if (seq_boundary_check || loop + 1 < loop_count) { + [[unroll]] for (uint idx = 0; idx < p_loops; ++idx) { + const uint max_offset = (idx * WorkGroupSize + wg_lane) / maxsh_stride; + eMf[idx] = F32Poolsh.F32sh[maxsh_offset + sh_pingpong_store * SLM_POOL_SIZE_MAX + max_offset]; + float p_temp[4] = float[4](pv4_temp[idx].x, pv4_temp[idx].y, pv4_temp[idx].z, pv4_temp[idx].w); + float eMf_temp = exp(eMf[idx] - global_max[idx]); + [[unroll]] for (uint i = 0; i < 4; ++i) { + p_temp[i] = p_temp[i] * eMf_temp; + Lf[idx] = Lf[idx] + p_temp[i]; + F16Poolsh.F16sh[psh_offset + sh_pingpong_store * psh_size + idx * WorkGroupSize * 4 + i] = float16_t(p_temp[i]); + } + p_col_base[idx] = p_col_base[idx] + p_v4; + } + } } - [[unroll]] for (uint pm = 0; pm < ALIGNED_P_M; pm++) { - [[unroll]] for (uint reduce = 1; reduce < SUB_GROUPS_PER_LINE; reduce++) { - sumTemp[pm][0] = sumTemp[pm][0] + sumTemp[pm][reduce]; - } + barrier(); + + // last loop for oob handling + if (seq_boundary_check) { + const uint coord_p = psh_base + sh_pingpong_store * psh_size + sg * TK; + coopmat VMat[omat_col]; + coopmat PMat[omat_row]; + + [[unroll]] for (uint r = 0; r < omat_row; ++r) { + coopMatLoad(PMat[r], F16Poolsh.F16sh, coord_p + r * TN * p_per_loop, p_per_loop, gl_CooperativeMatrixLayoutColumnMajor); + } + + barrier(); + const uint v_global_coord_base = sg * TK + loop_count * p_per_loop; + const uint v_global_offset = (v_base + loop_count * p_per_loop * p.nbv1) / 4; + const uint vv4sh_base = psh_base / 4 + sg * TM * TK / 4; + [[unroll]] for (uint c = 0; c < omat_col; ++c) { + [[unroll]] for (uint idx = 0; idx < TM * TK / SubGroupSize / 4; ++idx) { + const uint vv4_idx = idx * SubGroupSize + lane; + const uint row_idx = vv4_idx / (TM / 4); + const uint col_idx = vv4_idx % (TM / 4); + const uint vv4_offset = v_global_offset + row_idx * p.nbv1 / 4 + c * TM / 4 + col_idx; + + f16vec4 vv4_temp = (v_global_coord_base + row_idx < p.nek1) ? data_vv4[vv4_offset] : f16vec4(0.0f, 0.0f, 0.0f, 0.0f); + F16V4Poolsh.F16V4sh[vv4sh_base + vv4_idx] = vv4_temp; + } + + subgroupBarrier(); + coopMatLoad(VMat[c], F16V4Poolsh.F16V4sh, vv4sh_base, TM / 4, gl_CooperativeMatrixLayoutColumnMajor); + subgroupBarrier(); + } + + [[unroll]] for (uint r = 0; r < omat_row; ++r) { + [[unroll]] for (uint c = 0; c < omat_col; ++c) { + OMat[r][c] = coopMatMulAdd(VMat[c], PMat[r], OMat[r][c]); + } + } } - if ((0x1 & p.flag) != 0) { - [[unroll]] for (uint pm = 0; pm < ALIGNED_P_M; pm++) { - float fp32GlobalMax = slm_pool_f32.slm_pool_o[SLM_OFFSET_GLOBAL_MAX + pm * WARPSIZE + lane]; - float sinkCompensation = fp32GlobalMax - fp32SinkCoeff[pm]; - sinkCompensation = exp(sinkCompensation); - float softmaxSumTemp = sumTemp[pm][0] * sinkCompensation; - sumTemp[pm][0] = sumTemp[pm][0] + 1.0f / sinkCompensation; - sumTemp[pm][0] = 1.0f / sumTemp[pm][0]; - sinkCompensation = sinkCompensation / (1.0f + softmaxSumTemp); - sumTemp[pm][0] = fp32GlobalMax < fp32SinkCoeff[pm] ? sinkCompensation : sumTemp[pm][0]; - slm_pool_f32.slm_pool_o[SLM_OFFSET_SOFTMAX_SUM + pm * WARPSIZE + lane] = sumTemp[pm][0]; - } - } else { - [[unroll]] for (uint pm = 0; pm < ALIGNED_P_M; pm++) { - slm_pool_f32.slm_pool_o[SLM_OFFSET_SOFTMAX_SUM + pm * WARPSIZE + lane] = 1.0f / sumTemp[pm][0]; - } + barrier(); + + [[unroll]] for (uint idx = 0; idx < p_loops; ++idx) { + Lf[idx] = subgroupAdd(Lf[idx]); } - } - - [[unroll]] for (uint reduce = 2; reduce < SPLIT_P_GROUPS; reduce = reduce << 1 ) { - const uint stride = (reduce >> 1) * ALIGNED_P_M * TN * MAT_O_N * TM; - if ((localLinearId % reduce) == 0) { - const uint reduceBase = localLinearId * ALIGNED_P_M * TN * MAT_O_N * TM + SLM_OFFSET_O; - float sumTemp0[4]; - float sumTemp1[4]; - const uint reduceVec4Count = ALIGNED_P_M * TN * MAT_O_N * TM / 4 / WARPSIZE; - [[unroll]] for (uint totalLoads = 0; totalLoads < reduceVec4Count; totalLoads++) { - [[unroll]] for (uint kk = 0; kk < 4; kk++) { - sumTemp0[kk] = slm_pool_f32.slm_pool_o[reduceBase + totalLoads * 4 * WARPSIZE + 4 * lane + kk]; - sumTemp1[kk] = slm_pool_f32.slm_pool_o[reduceBase + stride + totalLoads * 4 * WARPSIZE + 4 * lane + kk]; - } - [[unroll]] for (uint kk = 0; kk < 4; kk++) { - sumTemp0[kk] = sumTemp0[kk] + sumTemp1[kk]; + [[unroll]] for (uint r = 0; r < omat_row; ++r) { + [[unroll]] for (uint c = 0; c < omat_col; ++c) { + const uint coord = osh_offset + r * TN * Bc + TM * c + sg * omat_row * TN * Bc; + coopMatStore(OMat[r][c], F32Poolsh.F32sh, coord, Bc, gl_CooperativeMatrixLayoutColumnMajor); } + } - [[unroll]] for (uint kk = 0; kk < 4; kk++) { - slm_pool_f32.slm_pool_o[reduceBase + totalLoads * 4 * WARPSIZE + 4 * lane + kk] = sumTemp0[kk]; + [[unroll]] for (uint idx = 0; idx < p_loops; ++idx) { + if (lane == 0) { + F32Poolsh.F32sh[lfsh_offset + idx * sg_count + sg] = Lf[idx]; } - } } + barrier(); - } - if (localLinearId == 0) { - const uint slmBase0 = SLM_OFFSET_O + lane * WARP_V_DIM; - const uint slmBase1 = slmBase0 + SPLIT_P_GROUPS / 2 * ALIGNED_P_M * TN * MAT_O_N * TM; + if (sg == 1) { + const uint lf_base = lfsh_offset + lane * sg_per_line; + float Lfrcp_temp[o_scalar_row][sg_per_line]; + [[unroll]] for (uint r = 0; r < o_scalar_row; ++r) { + [[unroll]] for (uint reduce = 0; reduce < sg_per_line; ++reduce) { + Lfrcp_temp[r][reduce] = F32Poolsh.F32sh[lf_base + r * SubGroupSize * sg_per_line + reduce]; + } + } + + [[unroll]] for (uint pm = 0; pm < o_scalar_row; ++pm) { + [[unroll]] for (uint reduce = 1; reduce < sg_per_line; ++reduce) { + Lfrcp_temp[pm][0] = Lfrcp_temp[pm][0] + Lfrcp_temp[pm][reduce]; + } + } - const uint offsetOutBase = (d * p.batchStrideO + vWarpIdx * WARP_V_DIM + v * GQA_RATIO * HEAD_DIM + outTokIdx * oDim * N_TOKS_PER_GROUP) / 4; - float fp32SoftMaxMul[ALIGNED_P_M]; - float fp32Output[ALIGNED_P_M][4]; - [[unroll]] for (uint pm = 0; pm < ALIGNED_P_M; pm++) { - fp32SoftMaxMul[pm] = slm_pool_f32.slm_pool_o[SLM_OFFSET_SOFTMAX_SUM + pm * WARPSIZE + lane]; + subgroupBarrier(); + + if ((p.flag & SINK_ENABLE_BIT) != 0) { + [[unroll]] for (uint r = 0; r < o_scalar_row; ++r) { + float Mf = F32Poolsh.F32sh[global_maxsh_offset + r * SubGroupSize + lane]; + float sink_emf = Mf - sink[r]; + sink_emf = exp(sink_emf); + float Lf_sink = Lfrcp_temp[r][0] * sink_emf; + Lfrcp_temp[r][0] = Lfrcp_temp[r][0] + 1.0f / sink_emf; + Lfrcp_temp[r][0] = 1.0f / Lfrcp_temp[r][0]; + sink_emf = sink_emf / (1.0f + Lf_sink); + Lfrcp_temp[r][0] = (Mf < sink[r]) ? sink_emf : Lfrcp_temp[r][0]; + F32Poolsh.F32sh[lfsh_offset + r * SubGroupSize + lane] = Lfrcp_temp[r][0]; + } + } else { + [[unroll]] for (uint r = 0; r < o_scalar_row; ++r) { + F32Poolsh.F32sh[lfsh_offset + r * SubGroupSize + lane] = (Lfrcp_temp[r][0] == 0.0f) ? 0.0f : (1.0f / Lfrcp_temp[r][0]); + } + } } - [[unroll]] for (uint vg = 0; vg < WARP_V_DIM / 4; vg++) { - [[unroll]] for (uint pm = 0; pm < ALIGNED_P_M; pm++) { - [[unroll]] for (uint vc = 0; vc < 4; vc++) { - fp32Output[pm][vc] = slm_pool_f32.slm_pool_o[slmBase0 + pm * WARPSIZE * WARP_V_DIM + vg * 4 + vc] * fp32SoftMaxMul[pm]; - fp32Output[pm][vc] = fp32Output[pm][vc] + slm_pool_f32.slm_pool_o[slmBase1 + pm * WARPSIZE * WARP_V_DIM + vg * 4 + vc] * fp32SoftMaxMul[pm]; + [[unroll]] for (uint reduce = 2; reduce < split_p_sg; reduce = reduce << 1) { + const uint stride = (reduce >> 1) * omat_row * TN * omat_col * TM; + if ((sg % reduce) == 0) { + const uint reduce_base = sg * omat_row * TN * omat_col * TM + osh_offset; + float sum_temp0[4]; + float sum_temp1[4]; + const uint reduce_count = omat_row * TN * omat_col * TM / 4 / SubGroupSize; + [[unroll]] for (uint idx = 0; idx < reduce_count; ++idx) { + [[unroll]] for (uint i = 0; i < 4; ++i) { + sum_temp0[i] = F32Poolsh.F32sh[reduce_base + idx * 4 * SubGroupSize + 4 * lane + i]; + sum_temp1[i] = F32Poolsh.F32sh[reduce_base + stride + idx * 4 * SubGroupSize + 4 * lane + i]; + } + + [[unroll]] for (uint i = 0; i < 4; ++i) { + sum_temp0[i] = sum_temp0[i] + sum_temp1[i]; + } + + [[unroll]] for (uint i = 0; i < 4; ++i) { + F32Poolsh.F32sh[reduce_base + idx * 4 * SubGroupSize + 4 * lane + i] = sum_temp0[i]; + } + } + } + barrier(); + } + + if (sg == 0) { + const uint sh_base0 = osh_offset + lane * Bc; + const uint sh_base1 = sh_base0 + split_p_sg / 2 * omat_row * TN * omat_col * TM; + + float Lfrcp[o_scalar_row]; + float Of[o_scalar_row][4]; + [[unroll]] for (uint r = 0; r < o_scalar_row; ++r) { + Lfrcp[r] = F32Poolsh.F32sh[lfsh_offset + r * SubGroupSize + lane]; } - } - [[unroll]] for (uint pm = 0; pm < ALIGNED_P_M; pm++) { - if (outputMask[pm] == true) { - out_f32_vec4[offsetOutBase + outOffsets[pm] + vg] = vec4(fp32Output[pm][0], fp32Output[pm][1], fp32Output[pm][2], fp32Output[pm][3]); + [[unroll]] for (uint c = 0; c < Bc / 4; ++c) { + [[unroll]] for (uint r = 0; r < o_scalar_row; ++r) { + [[unroll]] for (uint vc = 0; vc < 4; ++vc) { + Of[r][vc] = F32Poolsh.F32sh[sh_base0 + r * SubGroupSize * Bc + c * 4 + vc] * Lfrcp[r]; + Of[r][vc] = Of[r][vc] + F32Poolsh.F32sh[sh_base1 + r * SubGroupSize * Bc + c * 4 + vc] * Lfrcp[r]; + } + } + + [[unroll]] for (uint r = 0; r < o_scalar_row; ++r) { + if (o_mask[r]) { + data_ov4[o_base + o_offset[r] + c] = vec4(Of[r][0], Of[r][1], Of[r][2], Of[r][3]); + } + } } - } } - } } diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_prefill.comp b/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_prefill.comp new file mode 100644 index 000000000000..407654ca5c40 --- /dev/null +++ b/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_prefill.comp @@ -0,0 +1,817 @@ +#version 450 + +#extension GL_EXT_control_flow_attributes : enable +#extension GL_EXT_shader_16bit_storage : require +#extension GL_EXT_shader_explicit_arithmetic_types_float16 : require +#extension GL_KHR_cooperative_matrix : enable +#extension GL_KHR_memory_scope_semantics : enable +#extension GL_KHR_shader_subgroup_basic : enable +#extension GL_EXT_shared_memory_block : enable +#extension GL_KHR_shader_subgroup_shuffle : enable +#extension GL_KHR_shader_subgroup_shuffle_relative : enable + +layout (local_size_x_id = 0, local_size_y = 1, local_size_z = 1) in; + +layout (binding = 0) readonly buffer Q {float data_q[];}; +layout (binding = 0) readonly buffer QM2x4 {mat2x4 data_qm2x4[];}; +layout (binding = 1) readonly buffer K {float16_t data_k[];}; +layout (binding = 1) readonly buffer KV4 {f16vec4 data_kv4[];}; +layout (binding = 1) readonly buffer KM2x4 {f16mat2x4 data_km2x4[];}; +layout (binding = 2) readonly buffer V {float16_t data_v[];}; +layout (binding = 2) readonly buffer VV4 {f16vec4 data_vv4[];}; +layout (binding = 3) readonly buffer M {float16_t data_m[];}; +layout (binding = 3) readonly buffer MV2 {f16vec2 data_mv2[];}; +layout (binding = 3) readonly buffer MV4 {f16vec4 data_mv4[];}; +layout (binding = 4) readonly buffer S {float data_s[];}; +layout (binding = 5) buffer OV4 {vec4 data_ov4[];}; + +layout (push_constant) uniform parameter +{ + uint neq1; + uint neq2; + uint neq3; + uint nek1; + uint nek2; + uint nek3; + uint nev2; + uint nev3; + uint nem1; + uint nem2; + uint nem3; + uint nbq1; + uint nbq2; + uint nbk1; + uint nbk2; + uint nbv1; + uint nbv2; + uint nbq3; + uint nbk3; + uint nbv3; + uint nb3; + uint flag; + float scale; +} p; + +layout (constant_id = 0) const uint WorkGroupSize = 256; +layout (constant_id = 1) const uint Wr = 128; +layout (constant_id = 2) const uint Bc = 64; +layout (constant_id = 3) const uint HSK = 128; +layout (constant_id = 4) const uint HSV = 128; +layout (constant_id = 5) const uint qk_ratio = 8; +layout (constant_id = 6) const uint step_p = 32; +layout (constant_id = 7) const uint TM = 8; +layout (constant_id = 8) const uint TN = 16; +layout (constant_id = 9) const uint q_reg = 96; +layout (constant_id = 10) const uint shmem_size = 64; +layout (constant_id = 11) const uint acc_count = 1; +layout (constant_id = 12) const uint Flags = 0; +layout (constant_id = 13) const uint SubGroupSize = 16; +layout (constant_id = 14) const uint gqa_fold = 1; + +#define UNALIGNED_SEQ_LEN_BIT (1 << 0) +#define UNALIGNED_Q_STRIDE_BIT (1 << 1) +#define PARTIAL_ROW_MASK_BIT (1 << 2) +#define INTEL_GPU_BIT (1 << 3) +#define SINK_ENABLE_BIT (1 << 16) + +const bool aligned_q_stride = (Flags & UNALIGNED_Q_STRIDE_BIT) == 0; +const bool seq_boundary_check = (Flags & UNALIGNED_SEQ_LEN_BIT) != 0; +const bool partial_row_mask = (Flags & PARTIAL_ROW_MASK_BIT) != 0; +const uint TK = 16; +const uint sg_count = WorkGroupSize / SubGroupSize; +const uint matp_count = step_p / TM; +const uint mato_count = Bc / TM; +const uint qk_sg = Wr / TN; +const uint folded_row_wg = Wr / gqa_fold; +const uint loadv_sg = (qk_sg == sg_count) ? sg_count : (sg_count - qk_sg); +const uint k_col = (HSK % 64 == 0 && HSK <= 512 && WorkGroupSize <= 256) ? 8 : 4; +const uint k_loop_col = WorkGroupSize * k_col; +const uint k_loops = (step_p * HSK + k_loop_col - 1) / k_loop_col; +const bool k_boundary_check = (step_p * HSK) % k_loop_col != 0; +const uint v_vert_groups = step_p / TK; +const uint v_hori_groups = loadv_sg / v_vert_groups; +const uint v_loops = (HSV / TM + v_hori_groups - 1) / v_hori_groups; +const bool v_boundary_check = HSV % (v_hori_groups * TM) != 0; +const uint m_vert_groups = folded_row_wg / SubGroupSize; +const uint m_hori_groups = sg_count / m_vert_groups; +const uint m_loops = (step_p + m_hori_groups - 1) / m_hori_groups; + +const uint matq_count = q_reg / TK; +const uint q_shmem = HSK - q_reg; +const uint matq_shmem_count = q_shmem / TK; +const uint max_q_reg_staging = shmem_size / 2 / Wr; + +const bool intel_gpu = (Flags & INTEL_GPU_BIT) != 0; +const bool k_in_shmem = HSK <= 576; +const bool v_in_shmem = HSK <= 128 && HSV <= 128; +const bool mask_in_shmem = folded_row_wg <= 64; + +const uint oobsh_staging_size = sg_count * TM * TK; +const uint qsh_size = Wr * q_shmem; +const uint ksh_size = k_in_shmem ? (k_loops * k_loop_col) : oobsh_staging_size; +const uint msh_size = mask_in_shmem ? (folded_row_wg * step_p) : 0; +const uint psh_size = Wr * step_p; +const uint vsh_size = v_in_shmem ? (step_p * HSV) : 0; +const uint emfsh_size = (qk_sg == sg_count) ? 0 : Wr; +const uint pvsh_size = intel_gpu ? 0 : (acc_count * TM * TN * sg_count); + +const uint qsh_base = 0; +const uint emfsh_base = (qsh_base + qsh_size) / 2; +const uint vsh_base = (emfsh_base + emfsh_size) * 2; +const uint psh_base = (vsh_base + vsh_size) / 2; +const uint ksh_base = (psh_base + psh_size) * 2; +const uint pvsh_base = ksh_base / 2; +const uint msh_base = ksh_base + ksh_size; +const uint vsh_oob_staging_base = ksh_base / 4; + +const uint sg_mat_reduce = SubGroupSize / TN; +const uint softmax_loops = step_p / sg_mat_reduce; +const uint acc_loops = acc_count * TM / sg_mat_reduce; +const uint CM1_ELEMS = (TM * TN) / SubGroupSize; +const uint tk_sg_reduce = (TK + SubGroupSize - 1) / SubGroupSize; +const uint o_sg_cols = (sg_count + qk_sg - 1) / qk_sg; +const bool v_row_overshoot_check = o_sg_cols * Bc > HSV; + +shared sh_block { + float F32sh[shmem_size / 4]; +} F32Poolsh; + +shared shf16_alias { + float16_t F16sh[shmem_size / 2]; +} F16Poolsh; + +shared shf16v2_alias { + f16vec2 F16V2sh[shmem_size / 4]; +} F16V2Poolsh; + +shared shf16v4_alias { + f16vec4 F16V4sh[shmem_size / 8]; +} F16V4Poolsh; + +shared shf16m2x4_alias { + f16mat2x4 M2x4sh[shmem_size / 16]; +} F16M2x4Poolsh; + +void main() { + const uint lane = gl_SubgroupInvocationID; + const uint lane_c = lane % TN; + const uint lane_r = lane / TN; + const uint pv_ratio = p.neq2 / p.nev2; + const uint head_q_idx = gl_WorkGroupID.x; + const uint head_k_idx = head_q_idx * gqa_fold / qk_ratio; + const uint head_v_idx = head_q_idx * gqa_fold / pv_ratio; + const uint batch_k_ratio = p.neq3 / p.nek3; + const uint batch_v_ratio = p.neq3 / p.nev3; + const uint head_mask_idx = head_q_idx % p.nem2; + const uint out_r = gl_WorkGroupID.y; + const uint batch_idx = gl_WorkGroupID.z; + const uint batch_k_idx = batch_idx / batch_k_ratio; + const uint batch_v_idx = batch_idx / batch_v_ratio; + const uint batch_mask_idx = batch_idx % p.nem3; + const uint sg = gl_SubgroupID; + const uint batch_mask_stride = p.nek1 * p.nem1 * p.nem2; + uint sg_v_r = sg / v_vert_groups; + const uint sg_v_c = sg % v_vert_groups; + if (qk_sg < sg_count && sg >= qk_sg) { + sg_v_r -= qk_sg / v_vert_groups; + } + const uint sg_m_c = sg % m_hori_groups; + const uint sg_m_r = sg / m_hori_groups; + const uint sg_o_c = sg / qk_sg; + const uint sg_o_r = sg % qk_sg; + const uint sg_o_r_h = sg_o_r % gqa_fold; + const uint sg_o_r_row = sg_o_r / gqa_fold; + const uint o_dim = p.neq2 * HSV; + uint q_offset = out_r * folded_row_wg + sg_o_r_row * TN + lane_c; + q_offset = min(q_offset, p.neq1 - 1); + if (aligned_q_stride) { + q_offset = (q_offset * p.nbq1 + batch_idx * p.nbq3 + (head_q_idx * gqa_fold + sg_o_r_h) * p.nbq2) / 8 + lane_r; + } else { + q_offset = (q_offset * p.nbq1 + batch_idx * p.nbq3 + (head_q_idx * gqa_fold + sg_o_r_h) * p.nbq2) + lane_r; + } + uint k_base; + if (k_in_shmem) { + k_base = (batch_k_idx * p.nbk3 + head_k_idx * p.nbk2) / k_col; + } else { + k_base = batch_k_idx * p.nbk3 + head_k_idx * p.nbk2; + } + uint k_offset[k_loops]; + const uint v_base = batch_v_idx * p.nbv3 + head_v_idx * p.nbv2; + uint v_offset; + if (v_in_shmem) { + v_offset = v_base + sg_v_r * TM + sg_v_c * p.nbv1 * TK; + } else { + v_offset = v_base + sg_o_c * Bc; + } + uint m_offset; + bool mask_in_boundary; + if (mask_in_shmem) { + m_offset = out_r * folded_row_wg + sg_m_r * SubGroupSize + lane; + mask_in_boundary = m_offset < p.nem1; + m_offset = min(m_offset, p.nem1 - 1); + m_offset = m_offset * p.nek1 + batch_mask_idx * batch_mask_stride + head_mask_idx * p.nek1 * p.nem1; + if (seq_boundary_check) { + m_offset += sg_m_c; + } else if (m_loops % 4 == 0) { + m_offset = m_offset / 4 + sg_m_c; + } else if (m_loops % 2 == 0) { + m_offset = m_offset / 2 + sg_m_c; + } else { + m_offset += sg_m_c; + } + } else { + m_offset = out_r * folded_row_wg + sg_o_r_row * TN + lane_c; + mask_in_boundary = m_offset < p.nem1; + m_offset = min(m_offset, p.nem1 - 1); + m_offset = m_offset * p.nek1 + batch_mask_idx * batch_mask_stride + lane_r * softmax_loops + head_mask_idx * p.nek1 * p.nem1; + if (!seq_boundary_check) { + m_offset /= 4; + } + } + const bool mask_row_valid = !partial_row_mask || mask_in_boundary; + const uint o_offset = (batch_idx * p.nb3 + out_r * folded_row_wg * o_dim + (head_q_idx * gqa_fold + sg_o_r_h) * HSV + + sg_o_r_row * TN * o_dim + sg_o_c * Bc + lane_c * o_dim) / 4 + lane_r; + const uint vsh_offset = sg_v_c * TK * HSV + sg_v_r * TM * TK + vsh_base; + const uint msh_store_offset = (sg_m_r * SubGroupSize + lane) * step_p + msh_base; + const uint msh_load_offset = sg_o_r_row * TN * step_p + lane_c * step_p + lane_r * softmax_loops + msh_base; + const uint psh_load_offset = psh_base + sg * TN * step_p + lane_c * step_p + lane_r * softmax_loops; + const uint psh_store_offset = (psh_base + sg * TN * step_p + lane_c * step_p) * 2 + lane_r * softmax_loops; + const float NEG_FLT_MAX_OVER_2 = uintBitsToFloat(0xFEFFFFFF); + const float16_t NEG_F16_MAX_OVER_2 = float16_t(NEG_FLT_MAX_OVER_2); + const float sink = ((p.flag & SINK_ENABLE_BIT) != 0) ? data_s[head_q_idx * gqa_fold + sg_o_r_h] : 0.0f; + coopmat QMat[matq_count]; + uint ksh_offset[k_loops]; + bool mask_load_k[k_loops]; + + if (q_reg <= max_q_reg_staging) { + if (sg < qk_sg) { + if (aligned_q_stride) { + [[dont_unroll]] for (uint i = 0; i < q_reg / sg_mat_reduce / 8; ++i) { + const uint qsh_offset = (qsh_base + sg * q_reg * TN + lane_c * q_reg) / 8 + i * sg_mat_reduce + lane_r; + mat2x4 q_temp = data_qm2x4[q_offset]; + q_temp *= p.scale; + F16M2x4Poolsh.M2x4sh[qsh_offset] = f16mat2x4(q_temp); + q_offset += sg_mat_reduce; + } + } else { + [[dont_unroll]] for (uint i = 0; i < q_reg / sg_mat_reduce; ++i) { + const uint qsh_offset = (qsh_base + sg * q_reg * TN + lane_c * q_reg) + i * sg_mat_reduce + lane_r; + float q_temp = data_q[q_offset]; + q_temp *= p.scale; + F16Poolsh.F16sh[qsh_offset] = float16_t(q_temp); + q_offset += sg_mat_reduce; + } + } + } + + subgroupBarrier(); + + if (sg < qk_sg) { + const uint coord_q = sg * TN * q_reg + qsh_base; + [[unroll]] for (uint i = 0; i < matq_count; ++i) { + coopMatLoad(QMat[i], F16Poolsh.F16sh, coord_q + TK * i, q_reg, gl_CooperativeMatrixLayoutColumnMajor); + } + } + } else { + if (sg < qk_sg) { + const uint coord_q = sg * TN * TK + qsh_base; + [[unroll]] for (uint iq = 0; iq < q_reg / TK; ++iq) { + if (aligned_q_stride) { + [[dont_unroll]] for (uint idx = 0; idx < TK / sg_mat_reduce / 8; ++idx) { + const uint qsh_offset = (qsh_base + sg * TK * TN + lane_c * TK) / 8 + idx * sg_mat_reduce + lane_r; + mat2x4 q_temp = data_qm2x4[q_offset]; + q_temp *= p.scale; + F16M2x4Poolsh.M2x4sh[qsh_offset] = f16mat2x4(q_temp); + q_offset += sg_mat_reduce; + } + } else { + [[dont_unroll]] for (uint idx = 0; idx < TK / sg_mat_reduce; ++idx) { + const uint qsh_offset = (qsh_base + sg * TK * TN + lane_c * TK) + idx * sg_mat_reduce + lane_r; + float q_temp = data_q[q_offset]; + q_temp *= p.scale; + F16Poolsh.F16sh[qsh_offset] = float16_t(q_temp); + q_offset += sg_mat_reduce; + } + } + + subgroupBarrier(); + coopMatLoad(QMat[iq], F16Poolsh.F16sh, coord_q, TK, gl_CooperativeMatrixLayoutColumnMajor); + subgroupBarrier(); + } + } + } + + float Of[Bc / sg_mat_reduce]; + [[unroll]] for (uint i = 0; i < Bc / sg_mat_reduce; ++i) { + Of[i] = 0; + } + + barrier(); + + if (sg < qk_sg) { + if (aligned_q_stride) { + [[dont_unroll]] for (uint i = 0; i < q_shmem / sg_mat_reduce / 8; ++i) { + const uint qsh_offset = (qsh_base + sg * q_shmem * TN + lane_c * q_shmem) / 8 + i * sg_mat_reduce + lane_r; + mat2x4 q_temp = data_qm2x4[q_offset + i * sg_mat_reduce]; + q_temp *= p.scale; + F16M2x4Poolsh.M2x4sh[qsh_offset] = f16mat2x4(q_temp); + } + } else { + [[dont_unroll]] for (uint i = 0; i < q_shmem / sg_mat_reduce; ++i) { + const uint qsh_offset = (qsh_base + sg * q_shmem * TN + lane_c * q_shmem) + i * sg_mat_reduce + lane_r; + float q_temp = data_q[q_offset + i * sg_mat_reduce]; + q_temp *= p.scale; + F16Poolsh.F16sh[qsh_offset] = float16_t(q_temp); + } + } + } + + if (k_in_shmem) { + [[unroll]] for (uint idx = 0; idx < k_loops; ++idx) { + const uint flat_offset = idx * WorkGroupSize + sg * SubGroupSize + lane; + const uint row_idx = flat_offset / (HSK / k_col); + const uint col_idx = flat_offset % (HSK / k_col); + const uint sh_chunk_c = col_idx % (TK / k_col); + const uint sh_chunk_r = col_idx / (TK / k_col); + k_offset[idx] = k_base + row_idx * p.nbk1 / k_col + col_idx; + ksh_offset[idx] = (ksh_base + row_idx * TK + sh_chunk_r * TK * step_p) / k_col + sh_chunk_c; + if (k_boundary_check) { + mask_load_k[idx] = row_idx < step_p; + } + } + } + + const uint loop_count = (p.nek1 + step_p - 1) / step_p; + const uint k_seq_boundary = ((p.nek1 - 1) * p.nbk1 + batch_k_idx * p.nbk3 + head_k_idx * p.nbk2 + HSK) / k_col - 1; + float Mf = NEG_FLT_MAX_OVER_2; + float eMf = 0.0f; + float Lf = 0; + for (uint loop = 0; loop < loop_count; ++loop) { + if (k_in_shmem) { + if (k_boundary_check) { + [[unroll]] for (uint idx = 0; idx < k_loops; ++idx) { + if (seq_boundary_check && loop + 1 == loop_count) { + k_offset[idx] = min(k_offset[idx], k_seq_boundary); + } + + if (mask_load_k[idx]) { + if (k_col == 4) { + F16V4Poolsh.F16V4sh[ksh_offset[idx]] = data_kv4[k_offset[idx]]; + } else { + F16M2x4Poolsh.M2x4sh[ksh_offset[idx]] = data_km2x4[k_offset[idx]]; + } + } + k_offset[idx] += step_p * p.nbk1 / k_col; + } + } else { + [[unroll]] for (uint idx = 0; idx < k_loops; ++idx) { + if (seq_boundary_check && loop + 1 == loop_count) { + k_offset[idx] = min(k_offset[idx], k_seq_boundary); + } + + if (k_col == 4) { + F16V4Poolsh.F16V4sh[ksh_offset[idx]] = data_kv4[k_offset[idx]]; + } else { + F16M2x4Poolsh.M2x4sh[ksh_offset[idx]] = data_km2x4[k_offset[idx]]; + } + k_offset[idx] += step_p * p.nbk1 / k_col; + } + } + } + + if (mask_in_shmem) { + if (seq_boundary_check) { + [[unroll]] for (uint idx = 0; idx < m_loops; ++idx) { + if (idx * m_hori_groups + loop * step_p + sg_m_c < p.nek1) { + F16Poolsh.F16sh[msh_store_offset + sg_m_c + idx * m_hori_groups] = mask_row_valid ? data_m[m_offset] : float16_t(0.0f); + m_offset += m_hori_groups; + } else { + F16Poolsh.F16sh[msh_store_offset + sg_m_c + idx * m_hori_groups] = NEG_F16_MAX_OVER_2; + } + } + } else if (m_loops % 4 == 0) { + [[unroll]] for (uint idx = 0; idx < m_loops / 4; ++idx) { + F16V4Poolsh.F16V4sh[msh_store_offset / 4 + sg_m_c + idx * m_hori_groups] = + mask_row_valid ? data_mv4[m_offset] : f16vec4(0.0f, 0.0f, 0.0f, 0.0f); + m_offset += m_hori_groups; + } + } else if (m_loops % 2 == 0) { + [[unroll]] for (uint idx = 0; idx < m_loops / 2; ++idx) { + F16V2Poolsh.F16V2sh[msh_store_offset / 2 + sg_m_c + idx * m_hori_groups] = mask_row_valid ? data_mv2[m_offset] : f16vec2(0.0f, 0.0f); + m_offset += m_hori_groups; + } + } else { + [[unroll]] for (uint idx = 0; idx < m_loops; ++idx) { + F16Poolsh.F16sh[msh_store_offset + sg_m_c + idx * m_hori_groups] = mask_row_valid ? data_m[m_offset] : float16_t(0.0f); + m_offset += m_hori_groups; + } + } + } + + barrier(); + if (qk_sg == sg_count || sg < qk_sg) { + { + coopmat PMat[matp_count]; + coopmat KMat[matp_count]; + [[unroll]] for (uint pc = 0; pc < matp_count; ++pc) { + PMat[pc] = coopmat(0.0f); + } + + [[unroll]] for (uint d0 = 0; d0 < matq_count; ++d0) { + if (k_in_shmem) { + [[unroll]] for (uint pc = 0; pc < matp_count; ++pc) { + const uint coord = ksh_base + step_p * TK * d0 + TM * TK * pc; + coopMatLoad(KMat[pc], F16Poolsh.F16sh, coord, TK, gl_CooperativeMatrixLayoutRowMajor); + } + } else { + [[unroll]] for (uint pc = 0; pc < matp_count; ++pc) { + const uint k_base_row = loop * step_p + pc * TM; + if (seq_boundary_check && loop + 1 == loop_count) { + const uint k_base_offset = (k_base + k_base_row * p.nbk1 + TK * d0) / 4; + [[unroll]] for (uint idx = 0; idx < TM * TK / 4 / SubGroupSize; ++idx) { + const uint flat_offset = idx * SubGroupSize + lane; + const uint row_idx = flat_offset / (TK / 4); + const uint col_idx = flat_offset % (TK / 4); + const uint k_temp_offset = k_base_offset + row_idx * p.nbk1 / 4 + col_idx; + const uint ksh_temp_offset = ksh_base / 4 + sg * TM * TK / 4 + idx * SubGroupSize + lane; + f16vec4 kv4_temp = (k_base_row + row_idx) < p.nek1 ? data_kv4[k_temp_offset] : f16vec4(0.0f, 0.0f, 0.0f, 0.0f); + F16V4Poolsh.F16V4sh[ksh_temp_offset] = kv4_temp; + } + + subgroupBarrier(); + coopMatLoad(KMat[pc], F16Poolsh.F16sh, ksh_base + sg * TM * TK, TK, gl_CooperativeMatrixLayoutRowMajor); + subgroupBarrier(); + } else { + const uint coord = k_base + k_base_row * p.nbk1 + TK * d0; + coopMatLoad(KMat[pc], data_k, coord, p.nbk1, gl_CooperativeMatrixLayoutRowMajor); + } + } + } + + [[unroll]] for (uint pc = 0; pc < matp_count; ++pc) { + PMat[pc] = coopMatMulAdd(KMat[pc], QMat[d0], PMat[pc]); + } + } + + [[unroll]] for (uint d0 = 0; d0 < matq_shmem_count; ++d0) { + coopmat QMatsh; + if (k_in_shmem) { + [[unroll]] for (uint pc = 0; pc < matp_count; ++pc) { + const uint coord = ksh_base + matq_count * TK * step_p + step_p * TK * d0 + TM * TK * pc; + coopMatLoad(KMat[pc], F16Poolsh.F16sh, coord, TK, gl_CooperativeMatrixLayoutRowMajor); + } + } else { + [[unroll]] for (uint pc = 0; pc < matp_count; ++pc) { + const uint k_base_row = loop * step_p + pc * TM; + if (seq_boundary_check && loop + 1 == loop_count) { + const uint k_base_offset = (k_base + k_base_row * p.nbk1 + TK * d0 + TK * matq_count) / 4; + [[unroll]] for (uint idx = 0; idx < TM * TK / 4 / SubGroupSize; ++idx) { + const uint flat_offset = idx * SubGroupSize + lane; + const uint row_idx = flat_offset / (TK / 4); + const uint col_idx = flat_offset % (TK / 4); + const uint k_temp_offset = k_base_offset + row_idx * p.nbk1 / 4 + col_idx; + const uint ksh_temp_offset = ksh_base / 4 + sg * TM * TK / 4 + idx * SubGroupSize + lane; + f16vec4 kv4_temp = (k_base_row + row_idx) < p.nek1 ? data_kv4[k_temp_offset] : f16vec4(0.0f, 0.0f, 0.0f, 0.0f); + F16V4Poolsh.F16V4sh[ksh_temp_offset] = kv4_temp; + } + + subgroupBarrier(); + coopMatLoad(KMat[pc], F16Poolsh.F16sh, ksh_base + sg * TM * TK, TK, gl_CooperativeMatrixLayoutRowMajor); + subgroupBarrier(); + } else { + const uint coord = k_base + k_base_row * p.nbk1 + TK * d0 + TK * matq_count; + coopMatLoad(KMat[pc], data_k, coord, p.nbk1, gl_CooperativeMatrixLayoutRowMajor); + } + } + } + + const uint coord = qsh_base + sg * TN * q_shmem + TK * d0; + coopMatLoad(QMatsh, F16Poolsh.F16sh, coord, q_shmem, gl_CooperativeMatrixLayoutColumnMajor); + [[unroll]] for (uint pc = 0; pc < matp_count; ++pc) { + PMat[pc] = coopMatMulAdd(KMat[pc], QMatsh, PMat[pc]); + } + } + + [[unroll]] for (uint pc = 0; pc < matp_count; ++pc) { + coopMatStore(PMat[pc], F32Poolsh.F32sh, psh_base + TM * pc + sg * TN * step_p, step_p, gl_CooperativeMatrixLayoutColumnMajor); + } + } + + subgroupBarrier(); + + float Pf[softmax_loops]; + float Mnewf = Mf; + + [[unroll]] for (uint idx = 0; idx < softmax_loops; ++idx) { + Pf[idx] = F32Poolsh.F32sh[psh_load_offset + idx]; + } + + if (mask_in_shmem) { + [[unroll]] for (uint idx = 0; idx < softmax_loops; ++idx) { + Pf[idx] += float(F16Poolsh.F16sh[msh_load_offset + idx]); + } + } else { + if (seq_boundary_check) { + [[unroll]] for (uint idx = 0; idx < softmax_loops; ++idx) { + if (loop * step_p + idx + lane_r * softmax_loops < p.nek1) { + Pf[idx] += mask_row_valid ? float(data_m[m_offset + idx]) : 0.0f; + } else { + Pf[idx] += NEG_FLT_MAX_OVER_2; + } + } + m_offset += step_p; + } else { + [[unroll]] for (uint idx = 0; idx < softmax_loops / 4; ++idx) { + f16vec4 mv4_temp = mask_row_valid ? data_mv4[m_offset + idx] : f16vec4(0.0f, 0.0f, 0.0f, 0.0f); + float mask_f_temp[4] = float[4](float(mv4_temp.x), float(mv4_temp.y), float(mv4_temp.z), float(mv4_temp.w)); + [[unroll]] for (uint vec_idx = 0; vec_idx < 4; ++vec_idx) { + Pf[4 * idx + vec_idx] += mask_f_temp[vec_idx]; + } + } + m_offset += step_p / 4; + } + } + + if (seq_boundary_check && loop + 1 == loop_count) { + [[unroll]] for (uint idx = 0; idx < softmax_loops; ++idx) { + if (loop * step_p + idx + lane_r * softmax_loops < p.nek1) { + Mnewf = max(Mnewf, Pf[idx]); + } + } + } else { + [[unroll]] for (uint idx = 0; idx < softmax_loops; ++idx) { + Mnewf = max(Mnewf, Pf[idx]); + } + } + + [[unroll]] for (uint idx = 1; idx < sg_mat_reduce; ++idx) { + Mnewf = max(Mnewf, subgroupShuffleDown(Mnewf, idx * TN)); + } + + Mnewf = subgroupShuffle(Mnewf, lane & (TN - 1)); + eMf = exp(Mf - Mnewf); + Lf *= eMf; + + if (seq_boundary_check && loop + 1 == loop_count) { + [[unroll]] for (uint idx = 0; idx < softmax_loops; ++idx) { + if (loop * step_p + idx + lane_r * softmax_loops < p.nek1) { + Pf[idx] = exp(Pf[idx] - Mnewf); + } else { + Pf[idx] = 0.0f; + } + Lf += Pf[idx]; + } + } else { + [[unroll]] for (uint idx = 0; idx < softmax_loops; ++idx) { + Pf[idx] = exp(Pf[idx] - Mnewf); + Lf += Pf[idx]; + } + } + + if (qk_sg != sg_count && lane_r == 0) { + F32Poolsh.F32sh[emfsh_base + sg * TN + lane_c] = eMf; + } + + if (SubGroupSize > TN) { + subgroupBarrier(); + } + + [[unroll]] for (uint idx = 0; idx < softmax_loops; ++idx) { + F16Poolsh.F16sh[psh_store_offset + idx] = float16_t(Pf[idx]); + } + Mf = Mnewf; + } + + if (v_in_shmem && (qk_sg == sg_count || sg >= sg_count - loadv_sg)) { + if (seq_boundary_check && loop + 1 == loop_count) { + const uint v_global_coord_base = loop * step_p + sg_v_c * TK + lane_c; + if (v_boundary_check) { + [[dont_unroll]] for (uint c = 0; c < tk_sg_reduce; ++c) { + const uint v_coord_temp = v_global_coord_base + c * SubGroupSize; + uint v_oob_offset = (v_base + v_coord_temp * p.nbv1 + sg_v_r * TM) / 4 + lane_r; + uint vsh_temp_offset = vsh_offset + c * SubGroupSize + lane_c + lane_r * 4 * TK; + [[dont_unroll]] for (uint r = 0; r < v_loops - 1; ++r) { + [[dont_unroll]] for (uint idx = 0; idx < TM / sg_mat_reduce / 4; ++idx) { + f16vec4 vv4_temp = (v_coord_temp < p.nek1) ? data_vv4[v_oob_offset + idx * sg_mat_reduce] : f16vec4(0.0f, 0.0f, 0.0f, 0.0f); + F16Poolsh.F16sh[vsh_temp_offset + idx * sg_mat_reduce * TK * 4 + 0 * TK] = vv4_temp.x; + F16Poolsh.F16sh[vsh_temp_offset + idx * sg_mat_reduce * TK * 4 + 1 * TK] = vv4_temp.y; + F16Poolsh.F16sh[vsh_temp_offset + idx * sg_mat_reduce * TK * 4 + 2 * TK] = vv4_temp.z; + F16Poolsh.F16sh[vsh_temp_offset + idx * sg_mat_reduce * TK * 4 + 3 * TK] = vv4_temp.w; + } + v_oob_offset += TM * v_hori_groups / 4; + vsh_temp_offset += TM * TK * v_hori_groups; + } + } + + if ((sg_v_r + (v_loops - 1) * v_hori_groups) * TM < HSV) { + [[dont_unroll]] for (uint c = 0; c < tk_sg_reduce; ++c) { + const uint v_coord_temp = v_global_coord_base + c * SubGroupSize; + const uint v_oob_offset = (v_base + v_coord_temp * p.nbv1 + sg_v_r * TM + TM * (v_loops - 1) * v_hori_groups) / 4 + lane_r; + const uint vsh_temp_offset = vsh_offset + TM * (v_loops - 1) * TK * v_hori_groups + c * SubGroupSize + lane_c + lane_r * 4 * TK; + [[dont_unroll]] for (uint idx = 0; idx < TM / sg_mat_reduce / 4; ++idx) { + f16vec4 vv4_temp = (v_coord_temp < p.nek1) ? data_vv4[v_oob_offset + idx * sg_mat_reduce] : f16vec4(0.0f, 0.0f, 0.0f, 0.0f); + F16Poolsh.F16sh[vsh_temp_offset + idx * sg_mat_reduce * TK * 4 + 0 * TK] = vv4_temp.x; + F16Poolsh.F16sh[vsh_temp_offset + idx * sg_mat_reduce * TK * 4 + 1 * TK] = vv4_temp.y; + F16Poolsh.F16sh[vsh_temp_offset + idx * sg_mat_reduce * TK * 4 + 2 * TK] = vv4_temp.z; + F16Poolsh.F16sh[vsh_temp_offset + idx * sg_mat_reduce * TK * 4 + 3 * TK] = vv4_temp.w; + } + } + } + } else { + [[dont_unroll]] for (uint c = 0; c < tk_sg_reduce; ++c) { + const uint v_coord_temp = v_global_coord_base + c * SubGroupSize; + uint v_oob_offset = (v_base + v_coord_temp * p.nbv1 + sg_v_r * TM) / 4 + lane_r; + uint vsh_temp_offset = vsh_offset + c * SubGroupSize + lane_c + lane_r * 4 * TK; + [[dont_unroll]] for (uint r = 0; r < v_loops; ++r) { + [[dont_unroll]] for (uint idx = 0; idx < TM / sg_mat_reduce / 4; ++idx) { + f16vec4 vv4_temp = (v_coord_temp < p.nek1) ? data_vv4[v_oob_offset + idx * sg_mat_reduce] : f16vec4(0.0f, 0.0f, 0.0f, 0.0f); + F16Poolsh.F16sh[vsh_temp_offset + idx * sg_mat_reduce * TK * 4 + 0 * TK] = vv4_temp.x; + F16Poolsh.F16sh[vsh_temp_offset + idx * sg_mat_reduce * TK * 4 + 1 * TK] = vv4_temp.y; + F16Poolsh.F16sh[vsh_temp_offset + idx * sg_mat_reduce * TK * 4 + 2 * TK] = vv4_temp.z; + F16Poolsh.F16sh[vsh_temp_offset + idx * sg_mat_reduce * TK * 4 + 3 * TK] = vv4_temp.w; + } + v_oob_offset += TM * v_hori_groups / 4; + vsh_temp_offset += TM * TK * v_hori_groups; + } + } + } + } else { + if (v_boundary_check) { + [[unroll]] for (uint r = 0; r < v_loops - 1; ++r) { + coopmat VMat; + coopMatLoad(VMat, data_v, v_offset + TM * r * v_hori_groups, p.nbv1, gl_CooperativeMatrixLayoutColumnMajor); + coopMatStore(VMat, F16Poolsh.F16sh, vsh_offset + TM * r * TK * v_hori_groups, TK, gl_CooperativeMatrixLayoutRowMajor); + } + + if ((sg_v_r + (v_loops - 1) * v_hori_groups) * TM < HSV) { + coopmat VMat; + coopMatLoad(VMat, data_v, v_offset + TM * (v_loops - 1) * v_hori_groups, p.nbv1, gl_CooperativeMatrixLayoutColumnMajor); + coopMatStore(VMat, F16Poolsh.F16sh, vsh_offset + TM * (v_loops - 1) * TK * v_hori_groups, TK, gl_CooperativeMatrixLayoutRowMajor); + } + } else { + [[unroll]] for (uint r = 0; r < v_loops; ++r) { + coopmat VMat; + coopMatLoad(VMat, data_v, v_offset + TM * r * v_hori_groups, p.nbv1, gl_CooperativeMatrixLayoutColumnMajor); + coopMatStore(VMat, F16Poolsh.F16sh, vsh_offset + TM * r * TK * v_hori_groups, TK, gl_CooperativeMatrixLayoutRowMajor); + } + } + } + } + + barrier(); + { + const uint coord_p = psh_base * 2 + sg_o_r * TN * step_p * 2; + const uint coord_v_base = vsh_base + sg_o_c * mato_count * TM * TK; + const uint coord_pv = pvsh_base + sg * acc_count * TN * TM; + const uint pvsh_offset = pvsh_base + sg * acc_count * TN * TM + lane_c * acc_count * TM + lane_r * 4; + if (qk_sg != sg_count) { + eMf = F32Poolsh.F32sh[emfsh_base + sg_o_r * TN + lane_c]; + } + coopmat PMat[step_p / TK]; + [[unroll]] for (uint d0 = 0; d0 < step_p / TK; ++d0) { + coopMatLoad(PMat[d0], F16Poolsh.F16sh, coord_p + TK * d0, 2 * step_p, gl_CooperativeMatrixLayoutColumnMajor); + } + + [[unroll]] for (uint pv_col = 0; pv_col < mato_count / acc_count; ++pv_col) { + const uint coord_v = coord_v_base + pv_col * acc_count * TM * TK; + coopmat PVMat[acc_count]; + coopmat VMat[acc_count]; + [[unroll]] for (uint idx = 0; idx < acc_count; ++idx) { + PVMat[idx] = coopmat(0.0f); + } + + [[unroll]] for (uint d0 = 0; d0 < step_p / TK; ++d0) { + if (v_in_shmem) { + [[unroll]] for (uint idx = 0; idx < acc_count; ++idx) { + coopMatLoad(VMat[idx], F16Poolsh.F16sh, coord_v + HSV * TK * d0 + TM * TK * idx, TK, gl_CooperativeMatrixLayoutRowMajor); + } + } else { + if (seq_boundary_check && loop + 1 == loop_count) { + const uint v_global_coord_base = d0 * TK + loop * step_p + lane_c; + const uint vsh_staging_store = vsh_oob_staging_base + (sg * TM * TK + lane_c * TM) / 4 + lane_r; + const uint vsh_staging_load = vsh_oob_staging_base * 4 + sg * TM * TK; + [[unroll]] for (uint idx = 0; idx < acc_count; ++idx) { + subgroupBarrier(); + [[dont_unroll]] for (uint c = 0; c < tk_sg_reduce; ++c) { + const uint v_coord_temp = v_global_coord_base + c * SubGroupSize; + const uint v_oob_offset = (v_base + v_coord_temp * p.nbv1 + sg_o_c * Bc + pv_col * acc_count * TM + idx * TM) / 4 + lane_r; + + if (v_row_overshoot_check) { + const uint v_row_temp = sg_o_c * Bc + pv_col * acc_count * TM + idx * TM; + if (v_row_temp < HSV) { + [[dont_unroll]] for (uint r = 0; r < TM / sg_mat_reduce / 4; ++r) { + f16vec4 vv4_temp = (v_coord_temp < p.nek1) ? data_vv4[v_oob_offset + sg_mat_reduce * r] : f16vec4(0.0f, 0.0f, 0.0f, 0.0f); + F16V4Poolsh.F16V4sh[vsh_staging_store + c * SubGroupSize * TM / 4 + sg_mat_reduce * r] = vv4_temp; + } + } else { + [[dont_unroll]] for (uint r = 0; r < TM / sg_mat_reduce / 4; ++r) { + F16V4Poolsh.F16V4sh[vsh_staging_store + c * SubGroupSize * TM / 4 + sg_mat_reduce * r] = f16vec4(0.0f, 0.0f, 0.0f, 0.0f); + } + } + } else { + [[dont_unroll]] for (uint r = 0; r < TM / sg_mat_reduce / 4; ++r) { + f16vec4 vv4_temp = (v_coord_temp < p.nek1) ? data_vv4[v_oob_offset + sg_mat_reduce * r] : f16vec4(0.0f, 0.0f, 0.0f, 0.0f); + F16V4Poolsh.F16V4sh[vsh_staging_store + c * SubGroupSize * TM / 4 + sg_mat_reduce * r] = vv4_temp; + } + } + } + subgroupBarrier(); + coopMatLoad(VMat[idx], F16Poolsh.F16sh, vsh_staging_load, TM, gl_CooperativeMatrixLayoutColumnMajor); + } + } else { + const uint v_coord_global = v_offset + pv_col * acc_count * TM + d0 * TK * p.nbv1; + if (v_row_overshoot_check) { + [[unroll]] for (uint idx = 0; idx < acc_count; ++idx) { + const uint v_row_temp = sg_o_c * Bc + pv_col * acc_count * TM + idx * TM; + if (v_row_temp < HSV) { + coopMatLoad(VMat[idx], data_v, v_coord_global + TM * idx, p.nbv1, gl_CooperativeMatrixLayoutColumnMajor); + } else { + VMat[idx] = coopmat(0.0f); + } + } + } else { + [[unroll]] for (uint idx = 0; idx < acc_count; ++idx) { + coopMatLoad(VMat[idx], data_v, v_coord_global + TM * idx, p.nbv1, gl_CooperativeMatrixLayoutColumnMajor); + } + } + } + } + + [[unroll]] for (uint idx = 0; idx < acc_count; ++idx) { + PVMat[idx] = coopMatMulAdd(VMat[idx], PMat[d0], PVMat[idx]); + } + } + + if (intel_gpu) { + [[unroll]] for (uint idx = 0; idx < acc_count; ++idx) { + [[unroll]] for (uint e = 0; e < CM1_ELEMS; ++e) { + Of[pv_col * acc_loops + idx * CM1_ELEMS + e] = fma(Of[pv_col * acc_loops + idx * CM1_ELEMS + e], + eMf, + PVMat[idx][e]); + } + } + } else { + barrier(); + + [[unroll]] for (uint idx = 0; idx < acc_count; ++idx) { + coopMatStore(PVMat[idx], F32Poolsh.F32sh, coord_pv + TM * idx, acc_count * TM, gl_CooperativeMatrixLayoutColumnMajor); + } + + subgroupBarrier(); + + [[unroll]] for (uint idx = 0; idx < acc_loops / 4; ++idx) { + [[unroll]] for (uint vec_idx = 0; vec_idx < 4; ++vec_idx) { + Of[pv_col * acc_loops + idx * 4 + vec_idx] = fma(Of[pv_col * acc_loops + idx * 4 + vec_idx], + eMf, + F32Poolsh.F32sh[pvsh_offset + idx * sg_mat_reduce * 4 + vec_idx]); + } + } + barrier(); + } + } + } + + v_offset += p.nbv1 * step_p; + barrier(); + } + + if (sg < qk_sg) { + [[unroll]] for (uint idx = 1; idx < sg_mat_reduce; ++idx) { + Lf += subgroupShuffleDown(Lf, idx * TN); + } + if (lane_r == 0) { + F32Poolsh.F32sh[sg * TN + lane_c] = Mf; + F32Poolsh.F32sh[qk_sg * TN + sg * TN + lane_c] = Lf; + } + } + + barrier(); + + Mf = F32Poolsh.F32sh[sg_o_r * TN + lane_c]; + Lf = F32Poolsh.F32sh[qk_sg * TN + sg_o_r * TN + lane_c]; + + float Lfrcp; + if ((p.flag & SINK_ENABLE_BIT) != 0) { + float sink_emf = Mf - sink; + sink_emf = exp(sink_emf); + float Lf_sink = Lf * sink_emf; + Lf += 1.0f / sink_emf; + Lf = 1.0f / Lf; + sink_emf /= (1.0f + Lf_sink); + Lfrcp = (Mf < sink) ? sink_emf : Lf; + } else { + Lfrcp = (Lf == 0.0f) ? 0.0f : (1.0f / Lf); + } + + [[unroll]] for (uint out_c = 0; out_c < Bc / sg_mat_reduce; ++out_c) { + Of[out_c] *= Lfrcp; + } + + const uint sg_out_r = out_r * folded_row_wg + sg_o_r_row * TN + lane_c; + [[unroll]] for (uint o_chunk = 0; o_chunk < Bc / sg_mat_reduce / 4; ++o_chunk) { + const uint out_c = sg_mat_reduce * 4 * o_chunk + sg_o_c * Bc + lane_r; + if (sg_out_r < p.neq1 && out_c < HSV) { + data_ov4[o_offset + o_chunk * sg_mat_reduce] = vec4(Of[4 * o_chunk], Of[4 * o_chunk + 1], Of[4 * o_chunk + 2], Of[4 * o_chunk + 3]); + } + } +} diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/fwht.comp b/ggml/src/ggml-vulkan/vulkan-shaders/fwht.comp index a2069964adbe..d279d4306596 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/fwht.comp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/fwht.comp @@ -5,11 +5,13 @@ #extension GL_KHR_shader_subgroup_basic : enable #extension GL_KHR_shader_subgroup_shuffle : enable #endif +#extension GL_EXT_shader_16bit_storage : require layout(constant_id = 0) const uint BLOCK_SIZE = 32; layout(constant_id = 1) const uint N = 128; +layout(constant_id = 2) const uint ROWS = 4; -layout(local_size_x_id = 0, local_size_y = 4, local_size_z = 1) in; +layout(local_size_x_id = 0, local_size_y_id = 2, local_size_z = 1) in; layout(push_constant) uniform parameter { @@ -19,13 +21,13 @@ layout(push_constant) uniform parameter float scale; }; -layout(binding = 0, std430) readonly buffer A { float data_a[]; }; +layout(binding = 0, std430) readonly buffer A { A_TYPE data_a[]; }; layout(binding = 1, std430) writeonly buffer D { float data_d[]; }; const uint EL_W = N / BLOCK_SIZE; #ifdef FWHT_SHMEM -shared float shmem[4 * N]; +shared float shmem[ROWS * N]; #endif void main() { @@ -54,7 +56,7 @@ void main() { [[unroll]] for (uint i = 0; i < EL_W; ++i) { - reg[i] = row < n_rows ? data_a[src_offset + row_offset + i * BLOCK_SIZE + tid] * scale : 0.0; + reg[i] = row < n_rows ? float(data_a[src_offset + row_offset + i * BLOCK_SIZE + tid]) * scale : 0.0; } #ifdef FWHT_SHMEM diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/group_norm.comp b/ggml/src/ggml-vulkan/vulkan-shaders/group_norm.comp index bdf97dbb5dc9..334321e7bc4a 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/group_norm.comp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/group_norm.comp @@ -18,8 +18,12 @@ void main() { const float eps = p.param1; const uint tid = gl_LocalInvocationID.x; - const uint start = gl_WorkGroupID.x * group_size + tid; - const uint end = (gl_WorkGroupID.x + 1) * group_size; + const uint group = gl_WorkGroupID.x + gl_WorkGroupID.y * gl_NumWorkGroups.x + gl_WorkGroupID.z * gl_NumWorkGroups.x * gl_NumWorkGroups.y; + if (group >= p.KY) { + return; + } + const uint start = group * group_size + tid; + const uint end = (group + 1) * group_size; tmp[tid] = 0.0f; diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/l2_norm.comp b/ggml/src/ggml-vulkan/vulkan-shaders/l2_norm.comp index 9039ed1ded3b..4ac040de92ef 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/l2_norm.comp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/l2_norm.comp @@ -14,6 +14,8 @@ void main() { const uint row = gl_WorkGroupID.z * 262144 + gl_WorkGroupID.y * 512 + gl_WorkGroupID.x; const uint tid = gl_LocalInvocationID.x; + if (row >= p.ne01 * p.ne02 * p.ne03) { return; } + const uint a_base = get_aoffset() + src0_idx(row * p.ne00); const uint d_base = get_doffset() + dst_idx(row * p.ne10); diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mmq_cm1.comp b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mmq_cm1.comp index 17280d17b417..eaf7eb65d4e2 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mmq_cm1.comp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mmq_cm1.comp @@ -173,7 +173,7 @@ void main() { cm1_kvalues[gl_LocalInvocationIndex] = kvalues_mxfp4_const[gl_LocalInvocationIndex]; } #if !defined(USE_OCP_FP4) - for (uint i = gl_LocalInvocationIndex; i < 128u; i += BLOCK_SIZE) { + for (uint i = gl_LocalInvocationIndex; i < 256u; i += BLOCK_SIZE) { ue4m3_fp32_lut[i] = ue4m3_to_fp32_build(i); } #endif diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/norm.comp b/ggml/src/ggml-vulkan/vulkan-shaders/norm.comp index 792012d57e85..ebf9c44995ab 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/norm.comp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/norm.comp @@ -14,6 +14,8 @@ void main() { const uint row = gl_WorkGroupID.z * 262144 + gl_WorkGroupID.y * 512 + gl_WorkGroupID.x; const uint tid = gl_LocalInvocationID.x; + if (row >= p.ne01 * p.ne02 * p.ne03) { return; } + const uint a_base = get_aoffset() + src0_idx(row * p.ne00); const uint d_base = get_doffset() + dst_idx(row * p.ne10); diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/rms_norm_back.comp b/ggml/src/ggml-vulkan/vulkan-shaders/rms_norm_back.comp index 87707fc1494d..b8cada5aea6c 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/rms_norm_back.comp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/rms_norm_back.comp @@ -19,6 +19,8 @@ void main() { const uint row = gl_WorkGroupID.z * 262144 + gl_WorkGroupID.y * 512 + gl_WorkGroupID.x; const uint tid = gl_LocalInvocationID.x; + if (row >= p.KY) { return; } + // Compute derivative of x[i]/norm(x) = g[i]/norm(x) - x[i] dot(x,g)/KX / norm(x)^1.5 // partial sums for thread in warp diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/sum_rows.comp b/ggml/src/ggml-vulkan/vulkan-shaders/sum_rows.comp index 13ba2e99dccc..94bab1dfd6a6 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/sum_rows.comp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/sum_rows.comp @@ -19,6 +19,8 @@ void main() { const uint col = gl_LocalInvocationID.x; const float weight = p.weight; + if (row >= p.ne01 * p.ne02 * p.ne03) { return; } + const uint i03 = fastdiv(row, p.ne0_12mp, p.ne0_12L); const uint i03_offset = i03 * p.ne01*p.ne02; const uint i02 = fastdiv(row - i03_offset, p.ne0_1mp, p.ne0_1L); diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/sum_rows.glsl b/ggml/src/ggml-vulkan/vulkan-shaders/sum_rows.glsl index 1cb0f7827a38..b50db0a97889 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/sum_rows.glsl +++ b/ggml/src/ggml-vulkan/vulkan-shaders/sum_rows.glsl @@ -5,7 +5,7 @@ layout (push_constant) uniform parameter { uint n_cols; - uint ne01, ne02; + uint ne01, ne02, ne03; uint nb01, nb02, nb03; uint nb11, nb12, nb13; float weight; diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/types.glsl b/ggml/src/ggml-vulkan/vulkan-shaders/types.glsl index ec0a80c7e7d7..99ce1458a38b 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/types.glsl +++ b/ggml/src/ggml-vulkan/vulkan-shaders/types.glsl @@ -1921,8 +1921,9 @@ shared int8_t kvalues_mxfp4[16]; #endif #if defined(DATA_A_NVFP4) && !defined(USE_OCP_FP4) -// UE4M3 scale in NVFP4 blocks use only 7 bits; sign (bit 7) is always zero. -shared float ue4m3_fp32_lut[128]; +// Cover all scale bytes accepted by ggml_validate_row_data. +// ue4m3_to_fp32_build() masks the exponent and mantissa, but checks the full byte for 0 and 0x7F. +shared float ue4m3_fp32_lut[256]; float ue4m3_to_fp32_build(uint u) { if (u == 0u || u == 127u) { @@ -1947,7 +1948,7 @@ void init_iq_shmem(uvec3 wgsize) kvalues_mxfp4[i] = kvalues_mxfp4_const[i]; } #if defined(DATA_A_NVFP4) - for (uint i = gl_LocalInvocationIndex.x; i < 128u; i += wgsize.x) { + for (uint i = gl_LocalInvocationIndex.x; i < 256u; i += wgsize.x) { ue4m3_fp32_lut[i] = ue4m3_to_fp32_build(i); } #endif diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp b/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp index 3af09fb48a68..7ccccfa1d3cb 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp @@ -939,6 +939,7 @@ void process_shaders() { #if defined(GGML_VULKAN_COOPMAT_GLSLC_SUPPORT) string_to_spv("fa_decode_ph1", "flash_attn_decode_phase_1.comp", {}, true, true, false, false); string_to_spv("fa_decode_ph2", "flash_attn_decode_phase_2.comp", {}, true, true, false, false); + string_to_spv("fa_prefill", "flash_attn_prefill.comp", {}, true, true, false, false); #endif string_to_spv("fa_sparse_compact", "flash_attn_sparse_compact.comp", {}); @@ -969,6 +970,9 @@ void process_shaders() { string_to_spv("concat_i16", "concat.comp", {{"A_TYPE", "uint16_t"}, {"B_TYPE", "uint16_t"}, {"D_TYPE", "uint16_t"}}); string_to_spv("concat_i32", "concat.comp", {{"A_TYPE", "uint"}, {"B_TYPE", "uint"}, {"D_TYPE", "uint"}}); string_to_spv("concat_i64", "concat.comp", {{"A_TYPE", "uvec2"}, {"B_TYPE", "uvec2"}, {"D_TYPE", "uvec2"}}); + string_to_spv("concat_transpose_i8", "concat_transpose.comp", {{"A_TYPE", "uint8_t"}, {"B_TYPE", "uint8_t"}, {"D_TYPE", "uint8_t"}}); + string_to_spv("concat_transpose_i16", "concat_transpose.comp", {{"A_TYPE", "uint16_t"}, {"B_TYPE", "uint16_t"}, {"D_TYPE", "uint16_t"}}); + string_to_spv("concat_transpose_i32", "concat_transpose.comp", {{"A_TYPE", "uint"}, {"B_TYPE", "uint"}, {"D_TYPE", "uint"}}); string_to_spv("upscale_f32", "upscale.comp", {{"A_TYPE", "float"}, {"B_TYPE", "float"}, {"D_TYPE", "float"}}); @@ -1071,8 +1075,6 @@ void process_shaders() { string_to_spv("silu_back_f32", "silu_back.comp", {{"A_TYPE", "float"}, {"B_TYPE", "float"}, {"D_TYPE", "float"}}); - string_to_spv("diag_mask_inf_f32", "diag_mask_inf.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}}); - string_to_spv("soft_max_f32", "soft_max.comp", merge_maps(base_dict, {{"A_TYPE", "float"}, {"B_TYPE", "float"}, {"D_TYPE", "float"}})); string_to_spv("soft_max_f32_f16", "soft_max.comp", merge_maps(base_dict, {{"A_TYPE", "float"}, {"B_TYPE", "float16_t"}, {"D_TYPE", "float"}})); string_to_spv("soft_max_back_f32", "soft_max_back.comp", merge_maps(base_dict, {{"A_TYPE", "float"}, {"B_TYPE", "float"}, {"D_TYPE", "float"}})); @@ -1110,8 +1112,10 @@ void process_shaders() { string_to_spv("sum_rows_f32", "sum_rows.comp", merge_maps(base_dict, {{"A_TYPE", "float"}, {"D_TYPE", "float"}})); string_to_spv("cross_entropy_loss_f32", "cross_entropy_loss.comp", merge_maps(base_dict, {{"A_TYPE", "float"}, {"B_TYPE", "float"}, {"D_TYPE", "float"}})); string_to_spv("cross_entropy_loss_back_f32", "cross_entropy_loss_back.comp", merge_maps(base_dict, {{"A_TYPE", "float"}, {"B_TYPE", "float"}, {"D_TYPE", "float"}})); - string_to_spv("fwht_f32", "fwht.comp", {}); - string_to_spv("fwht_shmem_f32", "fwht.comp", {{"FWHT_SHMEM", "1"}}); + string_to_spv("fwht_f32", "fwht.comp", {{"A_TYPE", "float"}}); + string_to_spv("fwht_shmem_f32", "fwht.comp", {{"A_TYPE", "float"}, {"FWHT_SHMEM", "1"}}); + string_to_spv("fwht_f16", "fwht.comp", {{"A_TYPE", "float16_t"}}); + string_to_spv("fwht_shmem_f16", "fwht.comp", {{"A_TYPE", "float16_t"}, {"FWHT_SHMEM", "1"}}); string_to_spv("count_equal_i32", "count_equal.comp", merge_maps(base_dict, {{"A_TYPE", "int"}, {"B_TYPE", "int"}, {"D_TYPE", "int"}})); string_to_spv("dsv4_hc_comb_f32", "dsv4_hc_comb.comp", {}); string_to_spv("dsv4_hc_pre_f32", "dsv4_hc_pre.comp", {}); diff --git a/ggml/src/ggml-webgpu/ggml-webgpu-shader-lib.hpp b/ggml/src/ggml-webgpu/ggml-webgpu-shader-lib.hpp index 1e63a9fac052..3cc6a70c8b22 100644 --- a/ggml/src/ggml-webgpu/ggml-webgpu-shader-lib.hpp +++ b/ggml/src/ggml-webgpu/ggml-webgpu-shader-lib.hpp @@ -27,6 +27,7 @@ #define GGML_WEBGPU_KV_SEQ_PAD 256u #define GGML_WEBGPU_ARGSORT_MERGE_MAX_WG_SIZE 512u +#define GGML_WEBGPU_COUNT_EQUAL_WG_SIZE 256u // Matrix multiplication parameters @@ -251,6 +252,31 @@ struct ggml_webgpu_get_rows_pipeline_key_hash { } }; +/** Group Norm **/ + +struct ggml_webgpu_group_norm_pipeline_key { + ggml_type src_type; + ggml_type dst_type; + bool inplace; + uint32_t wg_size; + + bool operator==(const ggml_webgpu_group_norm_pipeline_key & other) const { + return src_type == other.src_type && dst_type == other.dst_type && inplace == other.inplace && + wg_size == other.wg_size; + } +}; + +struct ggml_webgpu_group_norm_pipeline_key_hash { + size_t operator()(const ggml_webgpu_group_norm_pipeline_key & key) const { + size_t seed = 0; + ggml_webgpu_hash_combine(seed, key.src_type); + ggml_webgpu_hash_combine(seed, key.dst_type); + ggml_webgpu_hash_combine(seed, key.inplace); + ggml_webgpu_hash_combine(seed, key.wg_size); + return seed; + } +}; + /** Row Norm **/ struct ggml_webgpu_row_norm_pipeline_key { @@ -307,14 +333,18 @@ struct ggml_webgpu_rms_norm_mul_shader_decisions { /** Pad **/ struct ggml_webgpu_pad_pipeline_key { bool circular; + bool reflect; - bool operator==(const ggml_webgpu_pad_pipeline_key & other) const { return circular == other.circular; } + bool operator==(const ggml_webgpu_pad_pipeline_key & other) const { + return circular == other.circular && reflect == other.reflect; + } }; struct ggml_webgpu_pad_pipeline_key_hash { size_t operator()(const ggml_webgpu_pad_pipeline_key & key) const { size_t seed = 0; ggml_webgpu_hash_combine(seed, key.circular); + ggml_webgpu_hash_combine(seed, key.reflect); return seed; } }; @@ -1225,14 +1255,19 @@ inline bool ggml_webgpu_can_use_mmvq(const ggml_tensor * src0, } class ggml_webgpu_shader_lib { + std::unordered_map + group_norm_pipelines; + wgpu::Device device; pre_wgsl::Preprocessor preprocessor; - std::unordered_map sum_rows_pipelines; // key is fixed, no variants yet + std::unordered_map sum_rows_pipelines; // key is whether to compute the mean std::unordered_map argmax_pipelines; // key is vec4 std::unordered_map argsort_pipelines; // key is order std::unordered_map argsort_merge_pipelines; // key is order std::unordered_map cumsum_pipelines; // key is fixed, no variants yet + std::unordered_map arange_pipelines; // key is fixed, no variants yet + std::unordered_map count_equal_pipelines; // key is src_overlap std::unordered_map row_norm_pipelines; // op/inplace @@ -1317,17 +1352,70 @@ class ggml_webgpu_shader_lib { public: ggml_webgpu_shader_lib(wgpu::Device device) { this->device = device; } + webgpu_pipeline get_count_equal_pipeline(bool src_overlap) { + const int key = src_overlap; + auto it = count_equal_pipelines.find(key); + if (it != count_equal_pipelines.end()) { + return it->second; + } + std::vector defines = { "WG_SIZE=" + std::to_string(GGML_WEBGPU_COUNT_EQUAL_WG_SIZE) }; + if (src_overlap) { + defines.push_back("SRC_OVERLAP"); + } + auto processed = preprocessor.preprocess(wgsl_count_equal, defines); + count_equal_pipelines[key] = ggml_webgpu_create_pipeline(device, processed, "count_equal"); + return count_equal_pipelines[key]; + } + webgpu_pipeline get_sum_rows_pipeline(const ggml_webgpu_shader_lib_context & context) { - auto it = sum_rows_pipelines.find(1); + const bool mean = context.dst->op == GGML_OP_MEAN; + auto it = sum_rows_pipelines.find(mean); if (it != sum_rows_pipelines.end()) { return it->second; } std::vector defines; defines.push_back(std::string("WG_SIZE=") + std::to_string(context.max_wg_size)); + if (mean) { + defines.push_back("MEAN"); + } + + auto processed = preprocessor.preprocess(wgsl_sum_rows, defines); + sum_rows_pipelines[mean] = ggml_webgpu_create_pipeline(device, processed, mean ? "mean" : "sum_rows"); + return sum_rows_pipelines[mean]; + } + + webgpu_pipeline get_group_norm_pipeline(const ggml_webgpu_shader_lib_context & context) { + const ggml_webgpu_group_norm_pipeline_key key = { + context.src0->type, + context.dst->type, + ggml_webgpu_tensor_equal(context.src0, context.dst), + std::min(context.max_wg_size, 128u), + }; + auto it = group_norm_pipelines.find(key); + if (it != group_norm_pipelines.end()) { + return it->second; + } - auto processed = preprocessor.preprocess(wgsl_sum_rows, defines); - sum_rows_pipelines[1] = ggml_webgpu_create_pipeline(device, processed, "sum_rows"); - return sum_rows_pipelines[1]; + std::vector defines = { + "WG_SIZE=" + std::to_string(key.wg_size), + key.src_type == GGML_TYPE_F16 ? "SRC_F16" : "SRC_F32", + key.dst_type == GGML_TYPE_F16 ? "DST_F16" : "DST_F32", + }; + std::string variant = + std::string("group_norm_src_") + ggml_type_name(key.src_type) + "_dst_" + ggml_type_name(key.dst_type); + if (key.inplace) { + GGML_ASSERT(key.src_type == key.dst_type); + defines.push_back("INPLACE"); + variant += "_inplace"; + } + auto processed = preprocessor.preprocess(wgsl_group_norm, defines); + auto decisions = std::make_shared(); + decisions->wg_size = key.wg_size; + decisions->inplace = key.inplace; + auto pipeline = ggml_webgpu_create_pipeline(device, processed, variant); + pipeline.context = decisions; + group_norm_pipelines[key] = pipeline; + return pipeline; } webgpu_pipeline get_row_norm_pipeline(const ggml_webgpu_shader_lib_context & context) { @@ -1943,9 +2031,28 @@ class ggml_webgpu_shader_lib { return gated_delta_net_pipelines[key]; } + webgpu_pipeline get_arange_pipeline(const ggml_webgpu_shader_lib_context & context) { + auto it = arange_pipelines.find(1); + if (it != arange_pipelines.end()) { + return it->second; + } + + std::vector defines; + defines.push_back(std::string("WG_SIZE=") + std::to_string(context.max_wg_size)); + + auto processed = preprocessor.preprocess(wgsl_arange, defines); + auto decisions = std::make_shared(); + decisions->wg_size = context.max_wg_size; + webgpu_pipeline pipeline = ggml_webgpu_create_pipeline(device, processed, "arange"); + pipeline.context = decisions; + arange_pipelines[1] = pipeline; + return arange_pipelines[1]; + } + webgpu_pipeline get_pad_pipeline(const ggml_webgpu_shader_lib_context & context) { ggml_webgpu_pad_pipeline_key key = {}; - key.circular = ggml_get_op_params_i32(context.dst, 8) != 0; + key.reflect = context.dst->op == GGML_OP_PAD_REFLECT_1D; + key.circular = !key.reflect && ggml_get_op_params_i32(context.dst, 8) != 0; auto it = pad_pipelines.find(key); if (it != pad_pipelines.end()) { @@ -1955,6 +2062,10 @@ class ggml_webgpu_shader_lib { std::vector defines; std::string variant = "pad"; + if (key.reflect) { + defines.push_back("REFLECT"); + variant += "_reflect"; + } if (key.circular) { defines.push_back("CIRCULAR"); variant += "_circular"; diff --git a/ggml/src/ggml-webgpu/ggml-webgpu.cpp b/ggml/src/ggml-webgpu/ggml-webgpu.cpp index 44ff0b10cd99..c08aa415fbe7 100644 --- a/ggml/src/ggml-webgpu/ggml-webgpu.cpp +++ b/ggml/src/ggml-webgpu/ggml-webgpu.cpp @@ -182,7 +182,6 @@ struct webgpu_capabilities { uint32_t subgroup_size = 0; uint32_t min_subgroup_size = 0; uint32_t max_subgroup_size = 0; - size_t memset_bytes_per_thread; }; // Stores global webgpu members @@ -203,6 +202,11 @@ struct webgpu_global_context_struct { wgpu::Buffer memset_params_buf; webgpu_pipeline memset_pipeline; + // Packed source data and write descriptors for set_tensor_n + wgpu::Buffer set_tensor_n_src_buf; + wgpu::Buffer set_tensor_n_descs_buf; + webgpu_pipeline set_tensor_n_pipeline; + std::string vendor; // TODO: We should rework the CPU profiling time handling to make it more useful. ref: https://github.com/ggml-org/llama.cpp/pull/22050 @@ -227,6 +231,14 @@ struct webgpu_global_context_struct { this->memset_params_buf.Destroy(); this->memset_params_buf = nullptr; } + if (this->set_tensor_n_src_buf) { + this->set_tensor_n_src_buf.Destroy(); + this->set_tensor_n_src_buf = nullptr; + } + if (this->set_tensor_n_descs_buf) { + this->set_tensor_n_descs_buf.Destroy(); + this->set_tensor_n_descs_buf = nullptr; + } #ifdef GGML_WEBGPU_DEBUG if (this->debug_host_buf) { this->debug_host_buf.Destroy(); @@ -256,8 +268,6 @@ struct webgpu_context_struct { wgpu::ComputePassEncoder active_compute_pass; bool batch_compute_passes = true; - size_t memset_bytes_per_thread; - #ifdef GGML_WEBGPU_GPU_PROFILE // Profiling: per-shader GPU time in ms std::unordered_map shader_gpu_time_ms; @@ -672,38 +682,54 @@ static void ggml_backend_webgpu_buffer_memset(webgpu_global_context & ctx, size_t size) { std::lock_guard lock(ctx->mutex); - std::vector params = { (uint32_t) offset, (uint32_t) size, value }; - std::vector entries = { ggml_webgpu_make_bind_group_entry(0, buf, 0, buf.GetSize()) }; - size_t bytes_per_wg = - ctx->capabilities.limits.maxComputeInvocationsPerWorkgroup * ctx->capabilities.memset_bytes_per_thread; - uint32_t wg_x = CEIL_DIV(size + 3, bytes_per_wg); - - ctx->queue.WriteBuffer(ctx->memset_params_buf, 0, params.data(), params.size() * sizeof(uint32_t)); - - wgpu::BindGroupEntry params_entry = {}; - params_entry.binding = 1; - params_entry.buffer = ctx->memset_params_buf; - params_entry.offset = 0; - params_entry.size = WEBGPU_PARAMS_BUF_SIZE_BYTES; - entries.push_back(params_entry); - - wgpu::BindGroupDescriptor bind_group_desc; - bind_group_desc.layout = ctx->memset_pipeline.pipeline.GetBindGroupLayout(0); - bind_group_desc.entryCount = entries.size(); - bind_group_desc.entries = entries.data(); - bind_group_desc.label = ctx->memset_pipeline.name.c_str(); - wgpu::BindGroup bind_group = ctx->device.CreateBindGroup(&bind_group_desc); - - wgpu::CommandEncoder encoder = ctx->device.CreateCommandEncoder(); - wgpu::ComputePassEncoder pass = encoder.BeginComputePass(); - pass.SetPipeline(ctx->memset_pipeline.pipeline); - pass.SetBindGroup(0, bind_group); - pass.DispatchWorkgroups(wg_x, 1, 1); - pass.End(); - - wgpu::CommandBuffer command = encoder.Finish(); - std::vector commands = { command }; - ctx->queue.Submit(commands.size(), commands.data()); + const size_t max_bind_size = ctx->capabilities.limits.maxStorageBufferBindingSize; + const size_t max_wg_size = ctx->capabilities.limits.maxComputeInvocationsPerWorkgroup; + + while (size > 0) { + const size_t align_offset = offset & ~(ctx->capabilities.limits.minStorageBufferOffsetAlignment - 1); + const size_t lead = offset - align_offset; + const size_t chunk_size = std::min(size, max_bind_size - lead); + const size_t bind_size = ROUNDUP_POW2(chunk_size + lead, WEBGPU_STORAGE_BUF_BINDING_MULT); + + std::vector params = { (uint32_t) lead, (uint32_t) chunk_size, value }; + std::vector entries = { ggml_webgpu_make_bind_group_entry(0, buf, align_offset, + bind_size) }; + uint32_t wg_x; + uint32_t wg_y; + uint32_t n_values = CEIL_DIV(chunk_size + lead, sizeof(uint32_t)); + uint32_t total_wg = CEIL_DIV(n_values, max_wg_size); + compute_2d_workgroups(total_wg, ctx->capabilities.limits.maxComputeWorkgroupsPerDimension, wg_x, wg_y); + + ctx->queue.WriteBuffer(ctx->memset_params_buf, 0, params.data(), params.size() * sizeof(uint32_t)); + + wgpu::BindGroupEntry params_entry = {}; + params_entry.binding = 1; + params_entry.buffer = ctx->memset_params_buf; + params_entry.offset = 0; + params_entry.size = WEBGPU_PARAMS_BUF_SIZE_BYTES; + entries.push_back(params_entry); + + wgpu::BindGroupDescriptor bind_group_desc; + bind_group_desc.layout = ctx->memset_pipeline.pipeline.GetBindGroupLayout(0); + bind_group_desc.entryCount = entries.size(); + bind_group_desc.entries = entries.data(); + bind_group_desc.label = ctx->memset_pipeline.name.c_str(); + wgpu::BindGroup bind_group = ctx->device.CreateBindGroup(&bind_group_desc); + + wgpu::CommandEncoder encoder = ctx->device.CreateCommandEncoder(); + wgpu::ComputePassEncoder pass = encoder.BeginComputePass(); + pass.SetPipeline(ctx->memset_pipeline.pipeline); + pass.SetBindGroup(0, bind_group); + pass.DispatchWorkgroups(wg_x, wg_y, 1); + pass.End(); + + wgpu::CommandBuffer command = encoder.Finish(); + std::vector commands = { command }; + ctx->queue.Submit(commands.size(), commands.data()); + + offset += chunk_size; + size -= chunk_size; + } } /** End WebGPU Actions */ @@ -856,6 +882,32 @@ static webgpu_encoded_op ggml_webgpu_set(webgpu_context & ctx, return ggml_backend_webgpu_build(ctx, pipeline, params, entries, wg_x, wg_y); } +static webgpu_encoded_op ggml_webgpu_arange(webgpu_context & ctx, ggml_tensor * dst) { + ggml_webgpu_shader_lib_context shader_lib_ctx = {}; + shader_lib_ctx.max_wg_size = ctx->global_ctx->capabilities.limits.maxComputeInvocationsPerWorkgroup; + + webgpu_pipeline pipeline = ctx->shader_lib->get_arange_pipeline(shader_lib_ctx); + auto * decisions = static_cast(pipeline.context.get()); + + const uint32_t ne = (uint32_t) ggml_nelements(dst); + std::vector params = { + ne, + (uint32_t) (ggml_webgpu_tensor_misalignment(ctx, dst) / ggml_type_size(dst->type)), + ggml_webgpu_u32_from_f32(ggml_get_op_params_f32(dst, 0)), + ggml_webgpu_u32_from_f32(ggml_get_op_params_f32(dst, 2)), + }; + + std::vector entries = { + ggml_webgpu_make_tensor_bind_group_entry(ctx, 0, dst), + }; + + uint32_t wg_x; + uint32_t wg_y; + uint32_t total_wg = CEIL_DIV(ne, decisions->wg_size); + compute_2d_workgroups(total_wg, ctx->global_ctx->capabilities.limits.maxComputeWorkgroupsPerDimension, wg_x, wg_y); + return ggml_backend_webgpu_build(ctx, pipeline, params, entries, wg_x, wg_y); +} + static webgpu_encoded_op ggml_webgpu_pad(webgpu_context & ctx, ggml_tensor * src, ggml_tensor * dst) { ggml_webgpu_shader_lib_context shader_lib_ctx = {}; shader_lib_ctx.src0 = src; @@ -2707,6 +2759,47 @@ static std::optional ggml_webgpu_rms_norm_mul(webgpu_context return ggml_backend_webgpu_build(ctx, pipeline, params, entries, wg_x, wg_y); } +static webgpu_encoded_op ggml_webgpu_group_norm(webgpu_context & ctx, ggml_tensor * src, ggml_tensor * dst) { + const size_t src_type_size = ggml_type_size(src->type); + const size_t dst_type_size = ggml_type_size(dst->type); + const uint32_t num_groups = (uint32_t) ggml_get_op_params_i32(dst, 0); + std::vector params = { + (uint32_t) src->ne[0], + (uint32_t) src->ne[1], + (uint32_t) src->ne[2], + (uint32_t) src->ne[3], + num_groups, + ggml_webgpu_u32_from_f32(ggml_get_op_params_f32(dst, 1)), + (uint32_t) (src->nb[1] / src_type_size), + (uint32_t) (src->nb[0] / src_type_size), + (uint32_t) (src->nb[2] / src_type_size), + (uint32_t) (src->nb[3] / src_type_size), + (uint32_t) (dst->nb[1] / dst_type_size), + (uint32_t) (dst->nb[0] / dst_type_size), + (uint32_t) (dst->nb[2] / dst_type_size), + (uint32_t) (dst->nb[3] / dst_type_size), + (uint32_t) (ggml_webgpu_tensor_misalignment(ctx, src) / src_type_size), + (uint32_t) (ggml_webgpu_tensor_misalignment(ctx, dst) / dst_type_size), + }; + + ggml_webgpu_shader_lib_context shader_lib_ctx = {}; + shader_lib_ctx.src0 = src; + shader_lib_ctx.dst = dst; + shader_lib_ctx.max_wg_size = ctx->global_ctx->capabilities.limits.maxComputeInvocationsPerWorkgroup; + webgpu_pipeline pipeline = ctx->shader_lib->get_group_norm_pipeline(shader_lib_ctx); + auto * decisions = static_cast(pipeline.context.get()); + + std::vector entries = { ggml_webgpu_make_tensor_bind_group_entry(ctx, 0, src) }; + if (!decisions->inplace) { + entries.push_back(ggml_webgpu_make_tensor_bind_group_entry(ctx, 1, dst)); + } + uint32_t wg_x; + uint32_t wg_y; + compute_2d_workgroups(num_groups * (uint32_t) src->ne[3], + ctx->global_ctx->capabilities.limits.maxComputeWorkgroupsPerDimension, wg_x, wg_y); + return ggml_backend_webgpu_build(ctx, pipeline, params, entries, wg_x, wg_y); +} + static webgpu_encoded_op ggml_webgpu_row_norm(webgpu_context & ctx, ggml_tensor * src, ggml_tensor * dst) { std::vector params = { (uint32_t) (ggml_webgpu_tensor_misalignment(ctx, src) / ggml_type_size(src->type)), @@ -2787,7 +2880,8 @@ static webgpu_encoded_op ggml_webgpu_rope(webgpu_context & ctx, float theta_scale = powf(freq_base, -2.0f / n_dims); float corr_dims[2]; - ggml_rope_yarn_corr_dims(n_dims, n_ctx_orig, freq_base, beta_fast, beta_slow, corr_dims); + ggml_rope_yarn_corr_dims(n_dims, n_ctx_orig, freq_base, beta_fast, beta_slow, ggml_rope_get_truncate(dst), + corr_dims); std::vector params = { (uint32_t) (ggml_webgpu_tensor_misalignment(ctx, src0) / ggml_type_size(src0->type)), @@ -3048,6 +3142,52 @@ static webgpu_encoded_op ggml_webgpu_argmax(webgpu_context & ctx, ggml_tensor * return ggml_backend_webgpu_build(ctx, pipeline, params, entries, wg_x); } +static webgpu_encoded_op ggml_webgpu_count_equal(webgpu_context & ctx, + ggml_tensor * src0, + ggml_tensor * src1, + ggml_tensor * dst) { + const uint32_t ne = ggml_nelements(src0); + const uint32_t offset_dst = ggml_webgpu_tensor_misalignment(ctx, dst); + std::vector dispatches; + // Reset both words of the I64 result before accumulating partial counts. + dispatches.push_back({ + ctx->global_ctx->memset_pipeline, + { offset_dst, sizeof(int64_t), 0 }, + { ggml_webgpu_make_tensor_bind_group_entry(ctx, 0, dst) }, + { 1, 1 } + }); + + if (ne == 0) { + return ggml_backend_webgpu_build_multi(ctx, dispatches); + } + + const bool src_overlap = ggml_webgpu_tensor_binding_overlap(ctx->global_ctx, src0, src1); + webgpu_pipeline pipeline = ctx->shader_lib->get_count_equal_pipeline(src_overlap); + + uint32_t offset_src0 = ggml_webgpu_tensor_misalignment(ctx, src0) / sizeof(int32_t); + uint32_t offset_src1 = ggml_webgpu_tensor_misalignment(ctx, src1) / sizeof(int32_t); + std::vector entries; + if (src_overlap) { + const ggml_webgpu_merged_binding_range range = ggml_webgpu_tensor_merged_binding_range(ctx, { src0, src1 }); + offset_src0 = ggml_webgpu_tensor_merged_element_offset(src0, range); + offset_src1 = ggml_webgpu_tensor_merged_element_offset(src1, range); + entries.push_back(ggml_webgpu_make_bind_group_entry(0, ggml_webgpu_tensor_buf(src0), range.offset, range.size)); + } else { + entries.push_back(ggml_webgpu_make_tensor_bind_group_entry(ctx, 0, src0)); + entries.push_back(ggml_webgpu_make_tensor_bind_group_entry(ctx, 1, src1)); + } + entries.push_back(ggml_webgpu_make_tensor_bind_group_entry(ctx, entries.size(), dst)); + std::vector params = { offset_src0, offset_src1, offset_dst / (uint32_t) sizeof(uint32_t), ne }; + uint32_t wg_x; + uint32_t wg_y; + const uint32_t total_wg = CEIL_DIV(ne, GGML_WEBGPU_COUNT_EQUAL_WG_SIZE); + compute_2d_workgroups(total_wg, ctx->global_ctx->capabilities.limits.maxComputeWorkgroupsPerDimension, wg_x, wg_y); + dispatches.push_back({ + pipeline, std::move(params), std::move(entries), { wg_x, wg_y } + }); + return ggml_backend_webgpu_build_multi(ctx, dispatches); +} + static webgpu_encoded_op ggml_webgpu_argsort(webgpu_context & ctx, ggml_tensor * src, ggml_tensor * dst) { bool is_top_k = dst->op == GGML_OP_TOP_K; @@ -3226,7 +3366,8 @@ static webgpu_encoded_op ggml_webgpu_sum_rows(webgpu_context & ctx, ggml_tensor total_sum ? 0 : (uint32_t) (src->nb[3] / ggml_type_size(src->type)), total_sum ? static_cast(ggml_nelements(src)) : (uint32_t) src->ne[0], total_sum ? 1 : (uint32_t) src->ne[1], - total_sum ? 1 : (uint32_t) src->ne[2] }; + total_sum ? 1 : (uint32_t) src->ne[2], + total_sum ? 1 : (uint32_t) ggml_nrows(dst) }; std::vector entries = { ggml_webgpu_make_tensor_bind_group_entry(ctx, 0, src), ggml_webgpu_make_tensor_bind_group_entry(ctx, 1, dst) }; @@ -3434,6 +3575,8 @@ static std::optional ggml_webgpu_encode(webgpu_context ctx, return ggml_webgpu_concat(ctx, src0, src1, node); case GGML_OP_REPEAT: return ggml_webgpu_repeat(ctx, src0, node); + case GGML_OP_GROUP_NORM: + return ggml_webgpu_group_norm(ctx, src0, node); case GGML_OP_RMS_NORM: if (ggml_webgpu_can_fuse_rms_norm_mul(cgraph, node_idx)) { num_encoded_ops = 2; @@ -3480,8 +3623,13 @@ static std::optional ggml_webgpu_encode(webgpu_context ctx, return ggml_webgpu_gated_delta_net(ctx, src0, src1, src2, node->src[3], node->src[4], node->src[5], node, nullptr); } + case GGML_OP_ARANGE: + return ggml_webgpu_arange(ctx, node); + case GGML_OP_PAD_REFLECT_1D: case GGML_OP_PAD: return ggml_webgpu_pad(ctx, src0, node); + case GGML_OP_COUNT_EQUAL: + return ggml_webgpu_count_equal(ctx, src0, src1, node); case GGML_OP_ARGMAX: return ggml_webgpu_argmax(ctx, src0, node); case GGML_OP_ARGSORT: @@ -3492,6 +3640,7 @@ static std::optional ggml_webgpu_encode(webgpu_context ctx, return ggml_webgpu_cumsum(ctx, src0, node); case GGML_OP_SUM: case GGML_OP_SUM_ROWS: + case GGML_OP_MEAN: return ggml_webgpu_sum_rows(ctx, src0, node); case GGML_OP_CONV_2D: return ggml_webgpu_conv_2d(ctx, src0, src1, node); @@ -3859,6 +4008,112 @@ static void ggml_backend_webgpu_buffer_set_tensor(ggml_backend_buffer_t buffer, WEBGPU_CPU_PROFILE_TOTAL_END(set_tensor, buf_ctx->global_ctx); } +static void ggml_backend_webgpu_buffer_set_tensor_n(ggml_backend_buffer_t buffer, + struct ggml_tensor * const * tensors, + const void * const * data, + const size_t * offsets, + const size_t * sizes, + size_t n_tensors) { + WEBGPU_CPU_PROFILE_TOTAL_START(set_tensor_n); + ggml_backend_webgpu_buffer_context * buf_ctx = (ggml_backend_webgpu_buffer_context *) buffer->context; + webgpu_global_context & ctx = buf_ctx->global_ctx; + + WEBGPU_LOG_DEBUG("ggml_backend_webgpu_buffer_set_tensor_n(" << buf_ctx->label << ", " << (const void *) tensors + << ", " << (const void *) data << ", " << offsets + << ", " << sizes << ", " << n_tensors << ")"); + + const size_t max_binding_size = ctx->capabilities.limits.maxStorageBufferBindingSize; + const size_t max_wg_per_dim = ctx->capabilities.limits.maxComputeWorkgroupsPerDimension; + + std::lock_guard lock(ctx->mutex); + + std::vector src; // we use src to copy the data of many tensors + std::vector descs; // [dst offset in bytes, src offset in words, size in bytes, unused] + wgpu::Buffer dst_buf; + + auto grow_buffer = [&ctx](wgpu::Buffer & buf, size_t size, const char * label) { + if (buf == nullptr || buf.GetSize() < size) { + if (buf) { + buf.Destroy(); + } + ggml_webgpu_create_buffer(ctx->device, buf, size, wgpu::BufferUsage::CopyDst | wgpu::BufferUsage::Storage, + label); + } + }; + + auto flush_tensors = [&]() { + if (descs.empty()) { + return; + } + + src.resize(ROUNDUP_POW2(src.size(), 4)); + const size_t descs_size = descs.size() * sizeof(uint32_t); + const size_t n_dsts = descs.size() / 4; + + grow_buffer(ctx->set_tensor_n_src_buf, src.size(), "set_tensor_n_src_buf"); + grow_buffer(ctx->set_tensor_n_descs_buf, descs_size, "set_tensor_n_descs_buf"); + ctx->queue.WriteBuffer(ctx->set_tensor_n_src_buf, 0, src.data(), src.size()); + ctx->queue.WriteBuffer(ctx->set_tensor_n_descs_buf, 0, descs.data(), descs_size); + + std::vector entries = { + ggml_webgpu_make_bind_group_entry(0, dst_buf, 0, dst_buf.GetSize()), + ggml_webgpu_make_bind_group_entry(1, ctx->set_tensor_n_src_buf, 0, src.size()), + ggml_webgpu_make_bind_group_entry(2, ctx->set_tensor_n_descs_buf, 0, descs_size), + }; + + wgpu::BindGroupDescriptor bind_group_desc; + bind_group_desc.layout = ctx->set_tensor_n_pipeline.pipeline.GetBindGroupLayout(0); + bind_group_desc.entryCount = entries.size(); + bind_group_desc.entries = entries.data(); + bind_group_desc.label = ctx->set_tensor_n_pipeline.name.c_str(); + wgpu::BindGroup bind_group = ctx->device.CreateBindGroup(&bind_group_desc); + + uint32_t wg_x; + uint32_t wg_y; + uint32_t total_wg = n_dsts; + compute_2d_workgroups(total_wg, max_wg_per_dim, wg_x, wg_y); + + wgpu::CommandEncoder encoder = ctx->device.CreateCommandEncoder(); + wgpu::ComputePassEncoder pass = encoder.BeginComputePass(); + pass.SetPipeline(ctx->set_tensor_n_pipeline.pipeline); + pass.SetBindGroup(0, bind_group); + pass.DispatchWorkgroups(wg_x, wg_y, 1); + pass.End(); + + wgpu::CommandBuffer commands = encoder.Finish(); + ctx->queue.Submit(1, &commands); + + src.clear(); + descs.clear(); + }; + + if (n_tensors > 0) { + dst_buf = ggml_webgpu_tensor_buf(tensors[0]); + } + + for (size_t i = 0; i < n_tensors; i++) { + if (sizes[i] == 0) { + continue; + } + wgpu::Buffer buf = ggml_webgpu_tensor_buf(tensors[i]); + size_t offset = ggml_webgpu_tensor_offset(tensors[i]) + offsets[i]; + if (!descs.empty() && (dst_buf.Get() != buf.Get() || src.size() + 4 + sizes[i] > max_binding_size || + (descs.size() + 4) * sizeof(uint32_t) > max_binding_size)) { + flush_tensors(); + } + dst_buf = buf; + + // Place the data at the same in-word byte lane as its destination + size_t src_offset = ROUNDUP_POW2(src.size(), 4) + offset % 4; + src.resize(src_offset + sizes[i]); + memcpy(src.data() + src_offset, data[i], sizes[i]); + descs.insert(descs.end(), { (uint32_t) offset, (uint32_t) (src_offset / 4), (uint32_t) sizes[i], 0 }); + } + flush_tensors(); + + WEBGPU_CPU_PROFILE_TOTAL_END(set_tensor_n, ctx); +} + static void ggml_backend_webgpu_buffer_get_tensor(ggml_backend_buffer_t buffer, const ggml_tensor * tensor, void * data, @@ -3919,6 +4174,117 @@ static void ggml_backend_webgpu_buffer_get_tensor(ggml_backend_buffer_t buffer, WEBGPU_CPU_PROFILE_TOTAL_END(get_tensor, buf_ctx->global_ctx); } +static void ggml_backend_webgpu_buffer_get_tensor_n(ggml_backend_buffer_t buffer, + const struct ggml_tensor * const * tensors, + void * const * data, + const size_t * offsets, + const size_t * sizes, + size_t n_tensors) { + WEBGPU_CPU_PROFILE_TOTAL_START(get_tensor_n); + ggml_backend_webgpu_buffer_context * buf_ctx = (ggml_backend_webgpu_buffer_context *) buffer->context; + WEBGPU_LOG_DEBUG("ggml_backend_webgpu_buffer_get_tensor_n(" << buf_ctx->label << ", " << (const void *) tensors + << ", " << (const void *) data << ", " << offsets + << ", " << sizes << ", " << n_tensors << ")"); + webgpu_global_context global_ctx = buf_ctx->global_ctx; + wgpu::Device device = global_ctx->device; + + typedef struct { + size_t staging_size; + size_t align_offset; + size_t local_offset; + } tensor_desc; + + std::vector t_descs; + std::vector> + staging_chunk_descs; // pair (boundary id, sum of the tensor size to the bounary id) + t_descs.reserve(n_tensors); + + size_t sum_tensor_size = 0; + for (size_t i = 0; i < n_tensors; i++) { + const ggml_tensor * tensor = tensors[i]; + tensor_desc td; + + td.align_offset = ggml_webgpu_tensor_offset(tensor) + offsets[i]; + + td.local_offset = td.align_offset % 4; + if (td.local_offset != 0) { + // If offset is not a multiple of 4, we need to round it down to the previous + // multiple of 4 + td.align_offset -= td.local_offset; + } + + td.staging_size = sizes[i] + td.local_offset; + if (td.staging_size % 4 != 0) { + // If size is not a multiple of 4, we need to round it up to the next + // multiple of 4 + td.staging_size += 4 - (td.staging_size % 4); + } + + t_descs.push_back(td); + + if (sum_tensor_size + td.staging_size > global_ctx->capabilities.limits.maxBufferSize) { + if (i == 0) { + GGML_ABORT("it is not expected that one tensor size exceed the one wgpu::Buffer limit size."); + } + staging_chunk_descs.push_back({ i - 1, sum_tensor_size }); + sum_tensor_size = td.staging_size; + } else { + sum_tensor_size += td.staging_size; + } + } + staging_chunk_descs.push_back({ n_tensors - 1, sum_tensor_size }); + + std::lock_guard lock(global_ctx->mutex); + + size_t tsi = 0; + for (auto & [tei, sum_chunk_size] : staging_chunk_descs) { + if (global_ctx->get_tensor_staging_buf == nullptr || + global_ctx->get_tensor_staging_buf.GetSize() < sum_chunk_size) { + // Create a new staging buffer if it doesn't exist or is too small + if (global_ctx->get_tensor_staging_buf) { + global_ctx->get_tensor_staging_buf.Destroy(); + } + ggml_webgpu_create_buffer(device, global_ctx->get_tensor_staging_buf, sum_chunk_size, + wgpu::BufferUsage::CopyDst | wgpu::BufferUsage::MapRead, + "get_tensor_staging_buf"); + } + + wgpu::CommandEncoder encoder = device.CreateCommandEncoder(); + + // Copy the data of the each tensor from the buffer to the staging buffer + size_t i_offset = 0; + for (size_t i = tsi; i <= tei; i++) { + encoder.CopyBufferToBuffer(ggml_webgpu_tensor_buf(tensors[i]), t_descs[i].align_offset, + global_ctx->get_tensor_staging_buf, i_offset, t_descs[i].staging_size); + i_offset += t_descs[i].staging_size; + } + + wgpu::CommandBuffer commands = encoder.Finish(); + + // Submit the command buffer to the queue + global_ctx->queue.Submit(1, &commands); + + // Map the staging buffer to read the data + ggml_backend_webgpu_map_buffer(global_ctx, global_ctx->get_tensor_staging_buf, wgpu::MapMode::Read, 0, + sum_chunk_size); + // Must specify size here since the staging buffer might be larger than the tensor size + const void * mapped_range = global_ctx->get_tensor_staging_buf.GetConstMappedRange(0, sum_chunk_size); + + i_offset = 0; + for (size_t i = tsi; i <= tei; i++) { + // Copy the data from the mapped range to the output buffer + std::memcpy(data[i], (const void *) ((const char *) mapped_range + i_offset + t_descs[i].local_offset), + sizes[i]); + i_offset += t_descs[i].staging_size; + } + + tsi = tei + 1; + global_ctx->get_tensor_staging_buf.Unmap(); + } + + WEBGPU_CPU_PROFILE_TOTAL_END(get_tensor_n, buf_ctx->global_ctx); +} + static void ggml_backend_webgpu_buffer_clear(ggml_backend_buffer_t buffer, uint8_t value) { WEBGPU_LOG_DEBUG("ggml_backend_webgpu_buffer_clear(" << buffer << ", " << (uint32_t) value << ")"); WEBGPU_CPU_PROFILE_TOTAL_START(clear); @@ -3939,8 +4305,8 @@ static ggml_backend_buffer_i ggml_backend_webgpu_buffer_interface = { /* .cpy_tensor = */ NULL, // TODO: optional, implement this /* .clear = */ ggml_backend_webgpu_buffer_clear, /* .reset = */ NULL, // TODO: optional, think it coordinates with - /* .set_tensor_n = */ NULL, - /* .get_tensor_n = */ NULL, + /* .set_tensor_n = */ ggml_backend_webgpu_buffer_set_tensor_n, + /* .get_tensor_n = */ ggml_backend_webgpu_buffer_get_tensor_n, /* .cpy_tensor_n = */ NULL, }; @@ -3978,12 +4344,10 @@ static size_t ggml_backend_webgpu_buffer_type_get_alignment(ggml_backend_buffer_ return dev_ctx->webgpu_global_ctx->capabilities.limits.minStorageBufferOffsetAlignment; } -// maxBufferSize might be larger, but you can't bind more than -// maxStorageBufferBindingSize to a single binding. static size_t ggml_backend_webgpu_buffer_type_get_max_size(ggml_backend_buffer_type_t buft) { ggml_backend_webgpu_device_context * dev_ctx = static_cast(buft->device->context); - return dev_ctx->webgpu_global_ctx->capabilities.limits.maxStorageBufferBindingSize; + return dev_ctx->webgpu_global_ctx->capabilities.limits.maxBufferSize; } static size_t ggml_backend_webgpu_buffer_type_get_alloc_size(ggml_backend_buffer_type_t buft, @@ -4148,20 +4512,19 @@ static ggml_guid_t ggml_backend_webgpu_guid(void) { } static void ggml_webgpu_init_memset_pipeline(webgpu_global_context & ctx) { - // we use the maximum workgroup size for the memset pipeline - size_t max_threads = ctx->capabilities.limits.maxComputeInvocationsPerWorkgroup * - ctx->capabilities.limits.maxComputeWorkgroupsPerDimension; - // Size the bytes_per_thread so that the largest buffer size can be handled - ctx->capabilities.memset_bytes_per_thread = - CEIL_DIV(ctx->capabilities.limits.maxStorageBufferBindingSize, max_threads); - std::vector constants(2); + std::vector constants(1); constants[0].key = "wg_size"; constants[0].value = ctx->capabilities.limits.maxComputeInvocationsPerWorkgroup; - constants[1].key = "bytes_per_thread"; - constants[1].value = ctx->capabilities.memset_bytes_per_thread; ctx->memset_pipeline = ggml_webgpu_create_pipeline(ctx->device, wgsl_memset, "memset", constants); } +static void ggml_webgpu_init_scatter_write_pipeline(webgpu_global_context & ctx) { + std::vector constants(1); + constants[0].key = "wg_size"; + constants[0].value = ctx->capabilities.limits.maxComputeInvocationsPerWorkgroup; + ctx->set_tensor_n_pipeline = ggml_webgpu_create_pipeline(ctx->device, wgsl_set_tensor_n, "set_tensor_n", constants); +} + static void ggml_backend_webgpu_request_adapter(wgpu::Instance & instance, wgpu::Adapter & adapter) { wgpu::RequestAdapterOptions options = {}; @@ -4305,6 +4668,7 @@ static void create_webgpu_device(ggml_backend_webgpu_reg_context * ctx) { GGML_ASSERT(ctx->webgpu_global_ctx->device != nullptr); ggml_webgpu_init_memset_pipeline(ctx->webgpu_global_ctx); + ggml_webgpu_init_scatter_write_pipeline(ctx->webgpu_global_ctx); ggml_webgpu_create_buffer(ctx->webgpu_global_ctx->device, ctx->webgpu_global_ctx->memset_params_buf, WEBGPU_PARAMS_BUF_SIZE_BYTES, wgpu::BufferUsage::CopyDst | wgpu::BufferUsage::Uniform, "memset_params_buf"); @@ -4547,6 +4911,9 @@ static bool ggml_backend_webgpu_device_supports_op(ggml_backend_dev_t dev, const default: break; } + if (ggml_prec_get_src(op, 1) == GGML_PREC_BF16 || ggml_prec_get_src(op, 1) == GGML_PREC_F32) { + supports_op = false; + } break; } case GGML_OP_MUL_MAT_ID: @@ -4590,7 +4957,7 @@ static bool ggml_backend_webgpu_device_supports_op(ggml_backend_dev_t dev, const default: break; } - if (ggml_get_op_params_i32(op, 3) == GGML_PREC_F32) { + if (ggml_prec_get_src(op, 1) == GGML_PREC_BF16 || ggml_prec_get_src(op, 1) == GGML_PREC_F32) { supports_op = false; } break; @@ -4657,6 +5024,18 @@ static bool ggml_backend_webgpu_device_supports_op(ggml_backend_dev_t dev, const supports_op = max_kv_tile > 0; break; } + case GGML_OP_GROUP_NORM: + { + bool valid_type = (src0->type == GGML_TYPE_F32 && op->type == GGML_TYPE_F32) || + (src0->type == GGML_TYPE_F16 && op->type == GGML_TYPE_F16); + // One workgroup per normalization group within each batch item. + const uint64_t num_workgroups = uint64_t(ggml_get_op_params_i32(op, 0)) * uint64_t(src0->ne[3]); + const uint64_t max_per_dim = + ctx->webgpu_global_ctx->capabilities.limits.maxComputeWorkgroupsPerDimension; + const bool dispatch_fits = num_workgroups <= max_per_dim * max_per_dim; + supports_op = valid_type && dispatch_fits; + break; + } case GGML_OP_RMS_NORM: case GGML_OP_NORM: case GGML_OP_L2_NORM: @@ -4784,6 +5163,12 @@ static bool ggml_backend_webgpu_device_supports_op(ggml_backend_dev_t dev, const case GGML_OP_COS: supports_op = (op->type == GGML_TYPE_F32 || op->type == GGML_TYPE_F16) && (src0->type == op->type); break; + case GGML_OP_ARANGE: + supports_op = op->type == GGML_TYPE_F32 && ggml_is_contiguous(op); + break; + case GGML_OP_PAD_REFLECT_1D: + supports_op = op->type == GGML_TYPE_F32 && src0->type == GGML_TYPE_F32; + break; case GGML_OP_PAD: supports_op = op->type == GGML_TYPE_F32 && src0->type == GGML_TYPE_F32; break; @@ -4802,7 +5187,12 @@ static bool ggml_backend_webgpu_device_supports_op(ggml_backend_dev_t dev, const case GGML_OP_SUM: supports_op = op->type == GGML_TYPE_F32 && src0->type == op->type && ggml_is_contiguously_allocated(src0); break; + case GGML_OP_COUNT_EQUAL: + supports_op = src0->type == GGML_TYPE_I32 && src1->type == GGML_TYPE_I32 && ggml_is_contiguous(src0) && + ggml_is_contiguous(src1); + break; case GGML_OP_SUM_ROWS: + case GGML_OP_MEAN: supports_op = op->type == GGML_TYPE_F32 && src0->type == op->type && ggml_is_contiguous_rows(src0); break; case GGML_OP_UPSCALE: diff --git a/ggml/src/ggml-webgpu/wgsl-shaders/arange.wgsl b/ggml/src/ggml-webgpu/wgsl-shaders/arange.wgsl new file mode 100644 index 000000000000..707a66536222 --- /dev/null +++ b/ggml/src/ggml-webgpu/wgsl-shaders/arange.wgsl @@ -0,0 +1,22 @@ +@group(0) @binding(0) +var dst: array; + +struct Params { + ne: u32, + offset_dst: u32, + start: f32, + step: f32, +}; + +@group(0) @binding(1) +var params: Params; + +@compute @workgroup_size(WG_SIZE) +fn main(@builtin(num_workgroups) num_wg: vec3, @builtin(global_invocation_id) gid: vec3) { + let i = gid.x + (num_wg.x * u32(WG_SIZE)) * gid.y; + if (i >= params.ne) { + return; + } + + dst[params.offset_dst + i] = params.start + params.step * f32(i); +} diff --git a/ggml/src/ggml-webgpu/wgsl-shaders/count_equal.wgsl b/ggml/src/ggml-webgpu/wgsl-shaders/count_equal.wgsl new file mode 100644 index 000000000000..1ab6a635bc02 --- /dev/null +++ b/ggml/src/ggml-webgpu/wgsl-shaders/count_equal.wgsl @@ -0,0 +1,57 @@ +struct Params { + offset_src0: u32, + offset_src1: u32, + offset_dst: u32, + ne: u32, +}; + +#ifdef SRC_OVERLAP +@group(0) @binding(0) +var merged_src: array; + +@group(0) @binding(1) +var dst: array>; + +@group(0) @binding(2) +var params: Params; +#else +@group(0) @binding(0) +var src0: array; + +@group(0) @binding(1) +var src1: array; + +@group(0) @binding(2) +var dst: array>; + +@group(0) @binding(3) +var params: Params; +#endif + +var shared_count: array; + +@compute @workgroup_size(WG_SIZE) +fn main(@builtin(workgroup_id) wid: vec3, + @builtin(num_workgroups) num_wg: vec3, + @builtin(local_invocation_id) lid: vec3) { + var count = 0u; + let i = (wid.x + wid.y * num_wg.x) * WG_SIZE + lid.x; + if (i < params.ne) { +#ifdef SRC_OVERLAP + count += select(0u, 1u, merged_src[params.offset_src0 + i] == merged_src[params.offset_src1 + i]); +#else + count += select(0u, 1u, src0[params.offset_src0 + i] == src1[params.offset_src1 + i]); +#endif + } + shared_count[lid.x] = count; + workgroupBarrier(); + for (var offset: u32 = WG_SIZE >> 1; offset > 0; offset >>= 1) { + if (lid.x < offset) { + shared_count[lid.x] += shared_count[lid.x + offset]; + } + workgroupBarrier(); + } + if (lid.x == 0) { + atomicAdd(&dst[params.offset_dst], shared_count[0]); + } +} diff --git a/ggml/src/ggml-webgpu/wgsl-shaders/group_norm.wgsl b/ggml/src/ggml-webgpu/wgsl-shaders/group_norm.wgsl new file mode 100644 index 000000000000..a2dcd42c0231 --- /dev/null +++ b/ggml/src/ggml-webgpu/wgsl-shaders/group_norm.wgsl @@ -0,0 +1,154 @@ +#if defined(SRC_F16) || defined(DST_F16) +enable f16; +#endif + +#ifdef SRC_F16 +#define SRC_TYPE f16 +#else +#define SRC_TYPE f32 +#endif + +#ifdef DST_F16 +#define DST_TYPE f16 +#else +#define DST_TYPE f32 +#endif + +struct Params { + width: u32, + height: u32, + num_channels: u32, + batch_size: u32, + num_groups: u32, + epsilon: f32, + // input strides + row_stride: u32, + col_stride: u32, + channel_stride: u32, + batch_stride: u32, + // output strides + output_row_stride: u32, + output_col_stride: u32, + output_channel_stride: u32, + output_batch_stride: u32, + // offsets + input_offset: u32, + output_offset: u32, +}; + +@group(0) @binding(0) var input: array; +#ifdef INPLACE +@group(0) @binding(1) var params: Params; +#else +@group(0) @binding(1) var output: array; +@group(0) @binding(2) var params: Params; +#endif + +var partials: array; +var mean: f32; +var variance: f32; + +fn input_index(logical_position: u32, group_base: u32) -> u32 { + let col: u32 = logical_position % params.width; + let row: u32 = (logical_position / params.width) % params.height; + let channel: u32 = (logical_position / params.width) / params.height; + return group_base + + (row * params.row_stride) + + (col * params.col_stride) + + (channel * params.channel_stride); +} + +fn output_index(logical_position: u32, group_base: u32) -> u32 { + let col: u32 = logical_position % params.width; + let row: u32 = (logical_position / params.width) % params.height; + let channel: u32 = (logical_position / params.width) / params.height; + return group_base + + (row * params.output_row_stride) + + (col * params.output_col_stride) + + (channel * params.output_channel_stride); +} + +@compute @workgroup_size(WG_SIZE) +fn main( + @builtin(workgroup_id) workgroup_id: vec3, + @builtin(num_workgroups) num_workgroups: vec3, + @builtin(local_invocation_index) lane: u32 +) { + if (params.num_groups == 0u) { + return; + } + let linear_group_id = workgroup_id.x + workgroup_id.y * num_workgroups.x; + let batch_index = linear_group_id / params.num_groups; + if (batch_index >= params.batch_size) { + return; + } + let group_index = linear_group_id % params.num_groups; + let channels_per_group = params.num_channels / params.num_groups + + select(0u, 1u, params.num_channels % params.num_groups != 0u); + if (group_index > params.num_channels / max(channels_per_group, 1u)) { + return; + } + let first_channel = group_index * channels_per_group; + if (first_channel >= params.num_channels) { + return; + } + let group_channels = min(channels_per_group, params.num_channels - first_channel); + let num_elems_per_group = group_channels * params.width * params.height; + let batch_base = params.input_offset + batch_index * params.batch_stride; + let group_base = batch_base + first_channel * params.channel_stride; + + var partial_sum : f32 = 0.0f; + for (var i = lane; i < num_elems_per_group; i += WG_SIZE) { + partial_sum += f32(input[input_index(i, group_base)]); + } + partials[lane] = partial_sum; + + workgroupBarrier(); + + if (lane == 0) { + var total_sum : f32 = 0.0f; + for (var i: u32 = 0u; i < WG_SIZE; i++) { + total_sum += partials[i]; + } + mean = total_sum / f32(num_elems_per_group); + } + + workgroupBarrier(); + + var partial_sq_dev : f32 = 0.0f; + for (var i = lane; i < num_elems_per_group; i += WG_SIZE) { + let delta = f32(input[input_index(i, group_base)]) - mean; + partial_sq_dev += delta * delta; + } + partials[lane] = partial_sq_dev; + + workgroupBarrier(); + + if (lane == 0) { + var total_sum : f32 = 0.0f; + for (var i = 0u; i < WG_SIZE; i++) { + total_sum += partials[i]; + } + variance = total_sum / f32(num_elems_per_group); + } + + workgroupBarrier(); + + let scale = inverseSqrt(variance + params.epsilon); +#ifndef INPLACE + let output_group_base = params.output_offset + + batch_index * params.output_batch_stride + + first_channel * params.output_channel_stride; +#endif + for (var i = lane; i < num_elems_per_group; i += WG_SIZE) { + let index = input_index(i, group_base); + let normalized = (f32(input[index]) - mean) * scale; +#ifdef INPLACE + input[index] = SRC_TYPE(normalized); +#else + output[output_index(i, output_group_base)] = DST_TYPE(normalized); +#endif + } + + +} diff --git a/ggml/src/ggml-webgpu/wgsl-shaders/memset.wgsl b/ggml/src/ggml-webgpu/wgsl-shaders/memset.wgsl index 194d2d6f58c7..af5a23bc0f69 100644 --- a/ggml/src/ggml-webgpu/wgsl-shaders/memset.wgsl +++ b/ggml/src/ggml-webgpu/wgsl-shaders/memset.wgsl @@ -11,30 +11,33 @@ struct Params { var params: Params; override wg_size: u32; -override bytes_per_thread: u32; @compute @workgroup_size(wg_size) -fn main(@builtin(global_invocation_id) gid: vec3) { - let i = gid.x * bytes_per_thread; +fn main(@builtin(workgroup_id) wid: vec3, + @builtin(num_workgroups) num_wg: vec3, + @builtin(local_invocation_id) lid: vec3) { + + let i = (wid.x + wid.y * num_wg.x) * wg_size + lid.x; let start = params.offset; let end = params.offset + params.size; + let id = (start >> 2u) + i; + let id_u32 = id * 4; + + if (id_u32 >= end) { + return; + } + + let lo = max(start, id_u32) - id_u32; + let hi = min(end, id_u32 + 4u) - id_u32; + var mask = 0xFFFFFFFFu << (8u * lo); + if (hi < 4u) { + mask &= (1u << (8u * hi)) - 1u; + } - for (var j: u32 = 0u; j < bytes_per_thread; j += 4) { - let byte_index = start + i + j; - if (byte_index + 4 <= end) { - output_buffer[byte_index >> 2] = params.value; - } else { - // Handle tail (unaligned) - for (var k: u32 = 0; k < 4; k++) { - let idx = byte_index + k; - if (idx < end) { - let word_idx = idx >> 2; - let bit_offset = (idx & 3) * 8u; - let mask = ~(0xffu << bit_offset); - let existing = output_buffer[word_idx]; - output_buffer[word_idx] = (existing & mask) | (params.value & (0xffu << bit_offset)); - } - } - } + if (mask == 0xFFFFFFFFu) { + output_buffer[id] = params.value; + } else { + let existing = output_buffer[id]; + output_buffer[id] = (existing & ~(mask)) | (params.value & mask); } } diff --git a/ggml/src/ggml-webgpu/wgsl-shaders/pad.wgsl b/ggml/src/ggml-webgpu/wgsl-shaders/pad.wgsl index 2cb999bef288..6c7b2e0ac184 100644 --- a/ggml/src/ggml-webgpu/wgsl-shaders/pad.wgsl +++ b/ggml/src/ggml-webgpu/wgsl-shaders/pad.wgsl @@ -64,6 +64,18 @@ fn main(@builtin(num_workgroups) num_wg: vec3, var value: f32 = 0.0; +#ifdef REFLECT + var x: u32; + if (i0 < params.lp0) { + x = params.lp0 - i0; + } else if (i0 < params.lp0 + params.src_ne0) { + x = i0 - params.lp0; + } else { + x = params.src_ne0 - 2 - (i0 - params.lp0 - params.src_ne0); + } + value = src[params.offset_src + x * params.stride_src0 + i1 * params.stride_src1 + + i2 * params.stride_src2 + i3 * params.stride_src3]; +#else #ifdef CIRCULAR let ci0 = wrap_around(i32(i0) - i32(params.lp0), params.src_ne0); let ci1 = wrap_around(i32(i1) - i32(params.lp1), params.src_ne1); @@ -85,5 +97,6 @@ fn main(@builtin(num_workgroups) num_wg: vec3, } #endif +#endif dst[params.offset_dst + gid_i] = value; } diff --git a/ggml/src/ggml-webgpu/wgsl-shaders/set_tensor_n.wgsl b/ggml/src/ggml-webgpu/wgsl-shaders/set_tensor_n.wgsl new file mode 100644 index 000000000000..f9ba2641604d --- /dev/null +++ b/ggml/src/ggml-webgpu/wgsl-shaders/set_tensor_n.wgsl @@ -0,0 +1,42 @@ +@group(0) @binding(0) +var output_buffer: array>; + +@group(0) @binding(1) +var src_buffer: array; + +@group(0) @binding(2) +var descs: array>; + +override wg_size: u32; + +@compute @workgroup_size(wg_size) +fn main(@builtin(workgroup_id) wid: vec3, + @builtin(num_workgroups) num_wg: vec3, + @builtin(local_invocation_id) lid: vec3) { + + let w = wid.y * num_wg.x + wid.x; + + if (w >= arrayLength(&descs)) { + return; + } + + let dst_i = descs[w].x / 4u; + let src_i = descs[w].y; + let lane = descs[w].x & 3u; + let end = lane + descs[w].z; + let n_words = (end + 3u) / 4u; + + for (var i = lid.x; i < n_words; i += wg_size) { + // [lo, hi) is the byte range of this word that belongs to the descs + let lo = select(0u, lane, i == 0u); + let hi = min(end - i * 4u, 4u); + let mask = select((1u << (hi * 8u)) - 1u, 0xffffffffu, hi == 4u) & (0xffffffffu << (lo * 8u)); + let value = src_buffer[src_i + i]; + if (mask == 0xffffffffu) { + atomicStore(&output_buffer[dst_i + i], value); + } else { + atomicAnd(&output_buffer[dst_i + i], ~mask); + atomicOr(&output_buffer[dst_i + i], value & mask); + } + } +} diff --git a/ggml/src/ggml-webgpu/wgsl-shaders/sum_rows.wgsl b/ggml/src/ggml-webgpu/wgsl-shaders/sum_rows.wgsl index 1ead3ac3f7d8..e4f1752624ec 100644 --- a/ggml/src/ggml-webgpu/wgsl-shaders/sum_rows.wgsl +++ b/ggml/src/ggml-webgpu/wgsl-shaders/sum_rows.wgsl @@ -15,7 +15,8 @@ struct Params { ne0: u32, ne1: u32, - ne2: u32 + ne2: u32, + nrows: u32 }; @group(0) @binding(2) @@ -28,7 +29,11 @@ fn main(@builtin(workgroup_id) wid: vec3, @builtin(num_workgroups) num_wg: vec3, @builtin(local_invocation_id) lid: vec3) { - var i = wid.x + wid.y * num_wg.x; + let row = wid.x + wid.y * num_wg.x; + if (row >= params.nrows) { + return; + } + var i = row; let i3 = i / (params.ne2 * params.ne1); i = i % (params.ne2 * params.ne1); let i2 = i / params.ne1; @@ -51,6 +56,10 @@ fn main(@builtin(workgroup_id) wid: vec3, } if (lid.x == 0) { - dst[params.offset_dst + wid.x] = shared_sum[0]; +#ifdef MEAN + dst[params.offset_dst + row] = shared_sum[0] / f32(params.ne0); +#else + dst[params.offset_dst + row] = shared_sum[0]; +#endif } } diff --git a/ggml/src/ggml-zdnn/common.hpp b/ggml/src/ggml-zdnn/common.hpp index 7ed044f91332..e28e7608b6a2 100644 --- a/ggml/src/ggml-zdnn/common.hpp +++ b/ggml/src/ggml-zdnn/common.hpp @@ -16,8 +16,7 @@ #include #include -#define GGML_ZDNN_NAME "zDNN" -#define GGML_ZDNN_VERSION ZDNN_VERNUM +#define GGML_ZDNN_NAME "zDNN" #define ZDNN_CHECK(stmt) \ do { \ diff --git a/ggml/src/ggml-zdnn/fattn.cpp b/ggml/src/ggml-zdnn/fattn.cpp index 7c7801cbb1dc..e6a0c24cadaf 100644 --- a/ggml/src/ggml-zdnn/fattn.cpp +++ b/ggml/src/ggml-zdnn/fattn.cpp @@ -20,8 +20,8 @@ bool ggml_zdnn_supports_flash_attn_ext(const ggml_backend_zdnn_device_context * memcpy(&max_bias, (const float *) op->op_params + 1, sizeof(float)); memcpy(&logit_softcap, (const float *) op->op_params + 2, sizeof(float)); - // sinks, alibi, softcap and sparse k/v sets are not implemented - if (op->src[4] != nullptr || max_bias != 0.0f || logit_softcap != 0.0f || ggml_get_op_params_i32(op, 4) != 0) { + // sinks, alibi, softcap, sparse k/v sets and k/v row lists are not implemented + if (op->src[4] != nullptr || op->src[5] != nullptr || max_bias != 0.0f || logit_softcap != 0.0f || ggml_get_op_params_i32(op, 4) != 0) { return false; } @@ -246,10 +246,4 @@ void ggml_zdnn_flash_attn_ext(ggml_backend_zdnn_context * ctx, ggml_tensor * dst memcpy((char *) dst->data + h*dst->nb[1] + iq*dst->nb[2], o_host + (h*n_q + iq)*d_v, d_v*sizeof(float)); } } - - // the result is written to dst->data directly, so the dst ztensor is stale - ggml_backend_zdnn_buffer * dst_extra = (ggml_backend_zdnn_buffer *) dst->extra; - if (dst_extra != nullptr) { - dst_extra->ztensor.is_transformed = false; - } } diff --git a/ggml/src/ggml-zdnn/ggml-zdnn.cpp b/ggml/src/ggml-zdnn/ggml-zdnn.cpp index 71ae85b3ec5e..ccdd4f38280d 100644 --- a/ggml/src/ggml-zdnn/ggml-zdnn.cpp +++ b/ggml/src/ggml-zdnn/ggml-zdnn.cpp @@ -95,6 +95,10 @@ static int64_t ggml_zdnn_mul_mat_min_batch(ggml_type type, int32_t n_threads) { } } +static bool ggml_zdnn_is_tensor_own(const ggml_tensor * tensor) { + return tensor->buffer != nullptr && tensor->buffer->buft == ggml_backend_zdnn_buffer_type(); +} + static bool ggml_zdnn_supports_op(const ggml_backend_zdnn_device_context * ctx_dev, const ggml_tensor * op) { switch (op->op) { case GGML_OP_NONE: @@ -115,6 +119,10 @@ static bool ggml_zdnn_supports_op(const ggml_backend_zdnn_device_context * ctx_d const int64_t max_batch = ctx_dev->max_size; + if (ggml_prec_get_src(op, 1) == GGML_PREC_BF16 || ggml_prec_get_src(op, 1) == GGML_PREC_F32) { + return false; + } + if (!ggml_is_matrix(weights) || !ggml_is_matrix(inputs) || !ggml_is_contiguous(weights) || !ggml_is_contiguous(inputs) || weights->view_src != nullptr || @@ -127,14 +135,24 @@ static bool ggml_zdnn_supports_op(const ggml_backend_zdnn_device_context * ctx_d return false; } + // tensors with no buffer yet get a zDNN buffer, computed or graph input weights and tensors in other buffers have no ztensor + if (weights->op != GGML_OP_NONE || (weights->flags & GGML_TENSOR_FLAG_INPUT) || + (weights->buffer != nullptr && !ggml_zdnn_is_tensor_own(weights)) || + (op->buffer != nullptr && !ggml_zdnn_is_tensor_own(op))) { + return false; + } + + // training writes allocated weights through ->data, test-backend-ops sets PARAM before it allocates and uses set_tensor + if ((weights->flags & GGML_TENSOR_FLAG_PARAM) && weights->buffer != nullptr) { + return false; + } + switch (weights->type) { case GGML_TYPE_F32: case GGML_TYPE_F16: case GGML_TYPE_BF16: - // the inputs ztensor is used, and views have none - return inputs->view_src == nullptr; + return true; case GGML_TYPE_Q8_0: - // the inputs are quantized from their data, so views work as well return ctx_dev->has_parmblkformat_1 && inputs->type == GGML_TYPE_F32; default: return false; @@ -274,6 +292,11 @@ static enum ggml_status ggml_backend_zdnn_buffer_init_tensor(ggml_backend_buffer return GGML_STATUS_SUCCESS; } + // ops read other compute tensors through ->data + if (tensor->op != GGML_OP_NONE && tensor->op != GGML_OP_MUL_MAT) { + return GGML_STATUS_SUCCESS; + } + ggml_backend_zdnn_buffer_context * ctx = (ggml_backend_zdnn_buffer_context *)buffer->context; const int64_t tsize = ggml_nbytes(tensor); @@ -324,23 +347,27 @@ static enum ggml_status ggml_backend_zdnn_buffer_init_tensor(ggml_backend_buffer GGML_UNUSED(buffer_idx); } +static void ggml_zdnn_reset_tensor(const ggml_tensor * tensor) { + const ggml_tensor * src = tensor->view_src != nullptr ? tensor->view_src : tensor; + + ggml_backend_zdnn_buffer * extra = (ggml_backend_zdnn_buffer *)src->extra; + if (extra != nullptr) { + zdnn_reset_ztensor(&extra->ztensor); + } +} + static void ggml_backend_zdnn_buffer_memset_tensor(ggml_backend_buffer_t buffer, ggml_tensor * tensor, uint8_t value, size_t offset, size_t size) { memset((char *)tensor->data + offset, value, size); + ggml_zdnn_reset_tensor(tensor); + GGML_UNUSED(buffer); } static void ggml_backend_zdnn_buffer_set_tensor(ggml_backend_buffer_t buffer, ggml_tensor * tensor, const void * data, size_t offset, size_t size) { memcpy((char *)tensor->data + offset, data, size); - // views share the ztensor of their source, which is now stale, and empty tensors have none - if (tensor->view_src != nullptr) { - ggml_backend_zdnn_buffer * src_extra = (ggml_backend_zdnn_buffer *)tensor->view_src->extra; - if (src_extra != nullptr && src_extra->ztensor.is_transformed) { - zdnn_reset_ztensor(&src_extra->ztensor); - } - return; - } + ggml_zdnn_reset_tensor(tensor); ggml_backend_zdnn_buffer * extra = (ggml_backend_zdnn_buffer *)tensor->extra; if (extra == nullptr) { @@ -355,12 +382,10 @@ static void ggml_backend_zdnn_buffer_set_tensor(ggml_backend_buffer_t buffer, gg // Fixes the LLAMA_SET_ROWS bug // see: https://github.com/ggml-org/llama.cpp/issues/15414 - // only weights are transformed here, others on first use, as not every op reads the ztensor and some data (masks) is not finite - if (tensor->buffer->usage != GGML_BACKEND_BUFFER_USAGE_WEIGHTS) { - zdnn_reset_ztensor(&extra->ztensor); - return; + // only complete writes to weights are transformed here, others on first use, as not every op reads the ztensor and some data (masks) is not finite + if (tensor->buffer->usage == GGML_BACKEND_BUFFER_USAGE_WEIGHTS && size == ggml_nbytes(tensor)) { + ggml_zdnn_load_tensor(extra->ztensor, tensor->data); } - if (extra->ztensor.is_transformed == false) ggml_zdnn_load_tensor(extra->ztensor, tensor->data); GGML_UNUSED(buffer); } @@ -375,6 +400,17 @@ static void ggml_backend_zdnn_buffer_clear(ggml_backend_buffer_t buffer, uint8_t ggml_backend_zdnn_buffer_context * ctx = (ggml_backend_zdnn_buffer_context *)buffer->context; memset(ctx->all_data, value, ctx->all_size); + + for (const auto & buf_ptr : ctx->buffers) { + zdnn_reset_ztensor(&buf_ptr->ztensor); + } + + // the bias has its own data, and no op transforms it on first use + for (const auto & buf_ptr : ctx->buffers) { + if (buf_ptr->extra != nullptr) { + ggml_zdnn_load_tensor(buf_ptr->extra->ztensor, buf_ptr->extra->data); + } + } } static void ggml_backend_zdnn_buffer_reset(ggml_backend_buffer_t buffer) { @@ -620,8 +656,7 @@ static bool ggml_backend_zdnn_device_supports_op(ggml_backend_dev_t dev, const g } static bool ggml_backend_zdnn_device_supports_buft(ggml_backend_dev_t dev, ggml_backend_buffer_type_t buft) { - return - buft->iface.get_name == ggml_backend_zdnn_buffer_type_get_name; + return ggml_backend_buft_is_host(buft); GGML_UNUSED(dev); } @@ -728,7 +763,7 @@ ggml_backend_reg_t ggml_backend_zdnn_reg(void) { { g_ggml_backend_zdnn_reg = { - /* .api_version = */ GGML_ZDNN_VERSION, + /* .api_version = */ GGML_BACKEND_API_VERSION, /* .iface = */ ggml_backend_zdnn_reg_i, /* .context = */ NULL }; diff --git a/ggml/src/ggml-zdnn/mmf.cpp b/ggml/src/ggml-zdnn/mmf.cpp index 0551fb88b3cf..acd443e5be59 100644 --- a/ggml/src/ggml-zdnn/mmf.cpp +++ b/ggml/src/ggml-zdnn/mmf.cpp @@ -3,7 +3,7 @@ #include "utils.hpp" void ggml_zdnn_mul_mat_f( - const ggml_backend_zdnn_context * ctx, + ggml_backend_zdnn_context * ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { @@ -31,10 +31,14 @@ void ggml_zdnn_mul_mat_f( ggml_tensor * output = dst; ggml_backend_zdnn_buffer * weights_extra = (ggml_backend_zdnn_buffer *)weights->extra; - ggml_backend_zdnn_buffer * inputs_extra = (ggml_backend_zdnn_buffer *)inputs->extra; ggml_backend_zdnn_buffer * output_extra = (ggml_backend_zdnn_buffer *)output->extra; ggml_backend_zdnn_buffer * bias_extra = (ggml_backend_zdnn_buffer *)output_extra->extra; + zdnn_tensor_desc inputs_pre_tfm_desc; + zdnn_tensor_desc inputs_tfm_desc; + zdnn_ztensor inputs_ztensor; + ggml_zdnn_pool_alloc inputs_ztensor_alloc; + const int64_t weights_rows = ne01; const int64_t weights_cols = ne00; const int64_t inputs_rows = ne11; @@ -53,37 +57,29 @@ void ggml_zdnn_mul_mat_f( // weights_extra->pre_tfm_desc.dim3, // weights_extra->pre_tfm_desc.dim4); - // GGML_LOG_INFO("%s: tensor '%s' tensor dimensions: [%ld, %ld, %ld, %ld] pre_tfm_desc dimensions: [%ld, %ld, %ld, %ld]\n", - // __func__, inputs_extra->name, - // inputs->ne[3], inputs->ne[2], inputs->ne[1], inputs->ne[0], - // inputs_extra->pre_tfm_desc.dim1, - // inputs_extra->pre_tfm_desc.dim2, - // inputs_extra->pre_tfm_desc.dim3, - // inputs_extra->pre_tfm_desc.dim4); - GGML_ASSERT(weights_extra->pre_tfm_desc.dim1 == weights->ne[0] && "weights_extra->pre_tfm_desc.dim1 must match weights->ne[0]"); GGML_ASSERT(weights_extra->pre_tfm_desc.dim2 == weights->ne[1] && "weights_extra->pre_tfm_desc.dim2 must match weights->ne[1]"); - GGML_ASSERT(inputs_extra->pre_tfm_desc.dim1 == inputs->ne[0] && "inputs_extra->pre_tfm_desc.dim1 must match inputs->ne[0]"); - GGML_ASSERT(inputs_extra->pre_tfm_desc.dim2 == inputs->ne[1] && "inputs_extra->pre_tfm_desc.dim2 must match inputs->ne[1]"); - // compute tensors are transformed on first use + // weights read straight into a host buffer skip set_tensor, so transform them on first use if (!weights_extra->ztensor.is_transformed) { ggml_zdnn_load_tensor(weights_extra->ztensor, weights->data); } - if (!inputs_extra->ztensor.is_transformed) { - ggml_zdnn_load_tensor(inputs_extra->ztensor, inputs->data); + + // the inputs can be in any host buffer or be a view, so transform them from their data on every call + { + zdnn_init_pre_transformed_desc(ZDNN_2D, ggml_zdnn_type_mapping(inputs->type), &inputs_pre_tfm_desc, inputs_rows, inputs_cols); + ZDNN_CHECK(zdnn_generate_transformed_desc(&inputs_pre_tfm_desc, &inputs_tfm_desc)); + ggml_zdnn_init_scratch_ztensor_dlf16(ctx, &inputs_pre_tfm_desc, &inputs_tfm_desc, &inputs_ztensor, inputs_ztensor_alloc); + ggml_zdnn_load_tensor(inputs_ztensor, inputs->data); } - ZDNN_CHECK(zdnn_matmul_transpose_op(&inputs_extra->ztensor, &weights_extra->ztensor, &bias_extra->ztensor, + ZDNN_CHECK(zdnn_matmul_transpose_op(&inputs_ztensor, &weights_extra->ztensor, &bias_extra->ztensor, false, true, MATMUL_OP_ADDITION, &output_extra->ztensor)); // TODO: Remove in the future as we are currently DLF16 -> FP32 then in the next op, FP32 -> DLF16 again. Inefficient. ZDNN_CHECK(zdnn_transform_origtensor(&output_extra->ztensor, output->data)); - GGML_UNUSED(ctx); GGML_UNUSED(weights_rows); GGML_UNUSED(weights_cols); - GGML_UNUSED(inputs_rows); - GGML_UNUSED(inputs_cols); GGML_UNUSED(output_rows); GGML_UNUSED(output_cols); } diff --git a/ggml/src/ggml-zdnn/mmf.hpp b/ggml/src/ggml-zdnn/mmf.hpp index a12f1b8f8a0e..afc02bc0ebeb 100644 --- a/ggml/src/ggml-zdnn/mmf.hpp +++ b/ggml/src/ggml-zdnn/mmf.hpp @@ -4,7 +4,7 @@ #include "common.hpp" void ggml_zdnn_mul_mat_f( - const ggml_backend_zdnn_context * ctx, + ggml_backend_zdnn_context * ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst); diff --git a/ggml/src/ggml-zdnn/mmq.cpp b/ggml/src/ggml-zdnn/mmq.cpp index 0c4472845f54..285479b4184e 100644 --- a/ggml/src/ggml-zdnn/mmq.cpp +++ b/ggml/src/ggml-zdnn/mmq.cpp @@ -188,7 +188,6 @@ void ggml_zdnn_mul_mat_q( GGML_ASSERT(inputs->type == GGML_TYPE_F32); ggml_backend_zdnn_buffer * weights_extra = (ggml_backend_zdnn_buffer *)weights->extra; - ggml_backend_zdnn_buffer * output_extra = (ggml_backend_zdnn_buffer *)output->extra; // weights read straight into a host buffer skip set_tensor, so transform them on first use if (!weights_extra->ztensor.is_transformed) { @@ -202,13 +201,11 @@ void ggml_zdnn_mul_mat_q( GGML_ASSERT(weights_extra->pre_tfm_desc.dim1 == weights->ne[1] && "weights_extra->pre_tfm_desc.dim1 must match weights->ne[1]"); ggml_zdnn_mul_mat_q_grouped(ctx, n_threads, inputs, weights_extra, output); - - // the result is written to output->data directly, so the output ztensor is stale - output_extra->ztensor.is_transformed = false; return; } - ggml_backend_zdnn_buffer * bias_extra = (ggml_backend_zdnn_buffer *)output_extra->extra; + ggml_backend_zdnn_buffer * output_extra = (ggml_backend_zdnn_buffer *)output->extra; + ggml_backend_zdnn_buffer * bias_extra = (ggml_backend_zdnn_buffer *)output_extra->extra; GGML_ASSERT(weights_extra->pre_tfm_desc.dim2 == weights->ne[0] && "weights_extra->pre_tfm_desc.dim2 must match weights->ne[0]"); GGML_ASSERT(weights_extra->pre_tfm_desc.dim1 == weights->ne[1] && "weights_extra->pre_tfm_desc.dim1 must match weights->ne[1]"); @@ -271,9 +268,6 @@ void ggml_zdnn_mul_mat_q( } }); - // the result is written to output->data directly, so the output ztensor is stale - output_extra->ztensor.is_transformed = false; - const float * weights_scales = weights_extra->scales.data(); ggml_zdnn_parallel_for(n_threads, ne1, [&](int64_t i1_start, int64_t i1_end) { for (int64_t i1 = i1_start; i1 < i1_end; i1++) { diff --git a/ggml/src/ggml-zendnn/ggml-zendnn.cpp b/ggml/src/ggml-zendnn/ggml-zendnn.cpp index 357cfcd99a2c..85f1263fb48f 100644 --- a/ggml/src/ggml-zendnn/ggml-zendnn.cpp +++ b/ggml/src/ggml-zendnn/ggml-zendnn.cpp @@ -720,6 +720,10 @@ static bool ggml_backend_zendnn_device_supports_op(ggml_backend_dev_t dev, const return false; } + if (ggml_prec_get_src(op, 1) == GGML_PREC_BF16 || ggml_prec_get_src(op, 1) == GGML_PREC_F32) { + return false; + } + if (ggml_zendnn_adaptive_fallback_enabled()) { const int64_t K = inputs->ne[0]; const int64_t N = (inputs->ne[1]*inputs->ne[2]*inputs->ne[3]); diff --git a/ggml/src/ggml.c b/ggml/src/ggml.c index 3d60c44e594f..1fca258a531d 100644 --- a/ggml/src/ggml.c +++ b/ggml/src/ggml.c @@ -485,27 +485,25 @@ void ggml_fp16_to_fp32_row(const ggml_fp16_t * x, float * y, int64_t n) { } void ggml_fp32_to_fp16_row(const float * x, ggml_fp16_t * y, int64_t n) { - int i = 0; - for (; i < n; ++i) { + for (int64_t i = 0; i < n; i++) { y[i] = GGML_FP32_TO_FP16(x[i]); } } void ggml_bf16_to_fp32_row(const ggml_bf16_t * x, float * y, int64_t n) { - int i = 0; - for (; i < n; ++i) { + for (int64_t i = 0; i < n; i++) { y[i] = GGML_BF16_TO_FP32(x[i]); } } void ggml_fp32_to_bf16_row_ref(const float * x, ggml_bf16_t * y, int64_t n) { - for (int i = 0; i < n; i++) { + for (int64_t i = 0; i < n; i++) { y[i] = ggml_compute_fp32_to_bf16(x[i]); } } void ggml_fp32_to_bf16_row(const float * x, ggml_bf16_t * y, int64_t n) { - int i = 0; + int64_t i = 0; #if defined(__AVX512BF16__) // subnormals are flushed to zero on this platform for (; i + 32 <= n; i += 32) { @@ -716,13 +714,13 @@ static const struct ggml_type_traits type_traits[GGML_TYPE_COUNT] = { .from_float_ref = (ggml_from_float_t) quantize_row_q4_1_ref, }, [4] = { // GGML_TYPE_Q4_2 - .type_name = "DEPRECATED", + .type_name = "REMOVED", .blck_size = 0, .type_size = 0, .is_quantized = false, }, [5] = { // GGML_TYPE_Q4_3 - .type_name = "DEPRECATED", + .type_name = "REMOVED", .blck_size = 0, .type_size = 0, .is_quantized = false, @@ -1011,7 +1009,6 @@ static const char * GGML_OP_NAME[GGML_OP_COUNT] = { "DUP", "ADD", "ADD_ID", - "ADD1", "ACC", "SUB", "MUL", @@ -1053,8 +1050,6 @@ static const char * GGML_OP_NAME[GGML_OP_COUNT] = { "GET_ROWS_BACK", "SET_ROWS", "DIAG", - "DIAG_MASK_INF", - "DIAG_MASK_ZERO", "SOFT_MAX", "SOFT_MAX_BACK", "ROPE", @@ -1119,7 +1114,7 @@ static const char * GGML_OP_NAME[GGML_OP_COUNT] = { "GLU", }; -static_assert(GGML_OP_COUNT == 102, "GGML_OP_COUNT != 102"); +static_assert(GGML_OP_COUNT == 99, "GGML_OP_COUNT != 99"); static const char * GGML_OP_SYMBOL[GGML_OP_COUNT] = { "none", @@ -1127,7 +1122,6 @@ static const char * GGML_OP_SYMBOL[GGML_OP_COUNT] = { "x", "x+y", "x[i]+y", - "x+y", "view(x,nb,offset)+=y->x", "x-y", "x*y", @@ -1169,8 +1163,6 @@ static const char * GGML_OP_SYMBOL[GGML_OP_COUNT] = { "get_rows_back(x)", "set_rows(x)", "diag(x)", - "diag_mask_inf(x)", - "diag_mask_zero(x)", "soft_max(x)", "soft_max_back(x)", "rope(x)", @@ -1235,7 +1227,7 @@ static const char * GGML_OP_SYMBOL[GGML_OP_COUNT] = { "glu(x)", }; -static_assert(GGML_OP_COUNT == 102, "GGML_OP_COUNT != 102"); +static_assert(GGML_OP_COUNT == 99, "GGML_OP_COUNT != 99"); static_assert(GGML_OP_POOL_COUNT == 2, "GGML_OP_POOL_COUNT != 2"); @@ -1362,12 +1354,6 @@ size_t ggml_row_size(enum ggml_type type, int64_t ne) { return ggml_type_size(type)*ne/ggml_blck_size(type); } -double ggml_type_sizef(enum ggml_type type) { - assert(type >= 0); - assert(type < GGML_TYPE_COUNT); - return ((double)(type_traits[type].type_size))/type_traits[type].blck_size; -} - const char * ggml_type_name(enum ggml_type type) { assert(type >= 0); assert(type < GGML_TYPE_COUNT); @@ -1788,6 +1774,7 @@ static struct ggml_tensor * ggml_new_tensor_impl( enum ggml_type type, int n_dims, const int64_t * ne, + const size_t * nb, struct ggml_tensor * view_src, size_t view_offs) { @@ -1814,8 +1801,6 @@ static struct ggml_tensor * ggml_new_tensor_impl( data_size *= ne[i]; } - GGML_ASSERT(view_src == NULL || data_size == 0 || data_size + view_offs <= ggml_nbytes(view_src)); - void * data = view_src != NULL ? view_src->data : NULL; if (data != NULL) { data = (char *) data + view_offs; @@ -1859,12 +1844,23 @@ static struct ggml_tensor * ggml_new_tensor_impl( result->ne[i] = ne[i]; } - result->nb[0] = ggml_type_size(type); - result->nb[1] = result->nb[0]*(result->ne[0]/ggml_blck_size(type)); - for (int i = 2; i < GGML_MAX_DIMS; i++) { - result->nb[i] = result->nb[i - 1]*result->ne[i - 1]; + // a view built with strides takes them from the caller, any other tensor is contiguous + if (nb != NULL) { + for (int i = 0; i < GGML_MAX_DIMS; i++) { + result->nb[i] = nb[i]; + } + } else { + result->nb[0] = ggml_type_size(type); + result->nb[1] = result->nb[0]*(result->ne[0]/ggml_blck_size(type)); + for (int i = 2; i < GGML_MAX_DIMS; i++) { + result->nb[i] = result->nb[i - 1]*result->ne[i - 1]; + } } + // a view stays inside its source, over the bytes its strides actually reach + const size_t view_size = ggml_nbytes(result); + GGML_ASSERT(view_src == NULL || view_size == 0 || view_size + view_offs <= ggml_nbytes(view_src)); + ctx->n_objects++; return result; @@ -1875,7 +1871,7 @@ struct ggml_tensor * ggml_new_tensor( enum ggml_type type, int n_dims, const int64_t * ne) { - return ggml_new_tensor_impl(ctx, type, n_dims, ne, NULL, 0); + return ggml_new_tensor_impl(ctx, type, n_dims, ne, NULL, NULL, 0); } struct ggml_tensor * ggml_new_tensor_1d( @@ -1992,13 +1988,9 @@ struct ggml_tensor * ggml_format_name(struct ggml_tensor * tensor, const char * struct ggml_tensor * ggml_view_tensor( struct ggml_context * ctx, struct ggml_tensor * src) { - struct ggml_tensor * result = ggml_new_tensor_impl(ctx, src->type, GGML_MAX_DIMS, src->ne, src, 0); + struct ggml_tensor * result = ggml_new_tensor_impl(ctx, src->type, GGML_MAX_DIMS, src->ne, src->nb, src, 0); ggml_format_name(result, "%s (view)", src->name); - for (int i = 0; i < GGML_MAX_DIMS; i++) { - result->nb[i] = src->nb[i]; - } - return result; } @@ -2168,39 +2160,6 @@ struct ggml_tensor * ggml_add_id( return result; } -// ggml_add1 - -static struct ggml_tensor * ggml_add1_impl( - struct ggml_context * ctx, - struct ggml_tensor * a, - struct ggml_tensor * b, - bool inplace) { - GGML_ASSERT(ggml_is_scalar(b)); - GGML_ASSERT(ggml_is_padded_1d(a)); - - struct ggml_tensor * result = inplace ? ggml_view_tensor(ctx, a) : ggml_dup_tensor(ctx, a); - - result->op = GGML_OP_ADD1; - result->src[0] = a; - result->src[1] = b; - - return result; -} - -struct ggml_tensor * ggml_add1( - struct ggml_context * ctx, - struct ggml_tensor * a, - struct ggml_tensor * b) { - return ggml_add1_impl(ctx, a, b, false); -} - -struct ggml_tensor * ggml_add1_inplace( - struct ggml_context * ctx, - struct ggml_tensor * a, - struct ggml_tensor * b) { - return ggml_add1_impl(ctx, a, b, true); -} - // ggml_acc static struct ggml_tensor * ggml_acc_impl( @@ -2948,7 +2907,7 @@ static struct ggml_tensor * ggml_glu_impl( } int64_t ne[GGML_MAX_DIMS] = { a->ne[0] / 2 }; for (int i = 1; i < GGML_MAX_DIMS; i++) ne[i] = a->ne[i]; - struct ggml_tensor * result = ggml_new_tensor_impl(ctx, a->type, GGML_MAX_DIMS, b ? a->ne : ne, NULL, 0); + struct ggml_tensor * result = ggml_new_tensor_impl(ctx, a->type, GGML_MAX_DIMS, b ? a->ne : ne, NULL, NULL, 0); ggml_set_op_params_i32(result, 0, (int32_t) op); ggml_set_op_params_i32(result, 1, (int32_t) swapped); @@ -3415,16 +3374,6 @@ struct ggml_tensor * ggml_mul_mat( return result; } -void ggml_mul_mat_set_prec( - struct ggml_tensor * a, - enum ggml_prec prec) { - GGML_ASSERT(a->op == GGML_OP_MUL_MAT); - - const int32_t prec_i32 = (int32_t) prec; - - ggml_set_op_params_i32(a, 0, prec_i32); -} - void ggml_mul_mat_set_hint( struct ggml_tensor * a, enum ggml_op_hint hint) { @@ -3757,7 +3706,7 @@ struct ggml_tensor * ggml_reshape( // as only the shape of b is relevant, and not its memory layout, b is allowed to be non contiguous. GGML_ASSERT(ggml_nelements(a) == ggml_nelements(b)); - struct ggml_tensor * result = ggml_new_tensor_impl(ctx, a->type, GGML_MAX_DIMS, b->ne, a, 0); + struct ggml_tensor * result = ggml_new_tensor_impl(ctx, a->type, GGML_MAX_DIMS, b->ne, NULL, a, 0); ggml_format_name(result, "%s (reshaped)", a->name); result->op = GGML_OP_RESHAPE; @@ -3774,7 +3723,7 @@ struct ggml_tensor * ggml_reshape_1d( GGML_ASSERT(ggml_nelements(a) == ne0); const int64_t ne[1] = { ne0 }; - struct ggml_tensor * result = ggml_new_tensor_impl(ctx, a->type, 1, ne, a, 0); + struct ggml_tensor * result = ggml_new_tensor_impl(ctx, a->type, 1, ne, NULL, a, 0); ggml_format_name(result, "%s (reshaped)", a->name); result->op = GGML_OP_RESHAPE; @@ -3792,7 +3741,7 @@ struct ggml_tensor * ggml_reshape_2d( GGML_ASSERT(ggml_nelements(a) == ne0*ne1); const int64_t ne[2] = { ne0, ne1 }; - struct ggml_tensor * result = ggml_new_tensor_impl(ctx, a->type, 2, ne, a, 0); + struct ggml_tensor * result = ggml_new_tensor_impl(ctx, a->type, 2, ne, NULL, a, 0); ggml_format_name(result, "%s (reshaped)", a->name); result->op = GGML_OP_RESHAPE; @@ -3811,7 +3760,7 @@ struct ggml_tensor * ggml_reshape_3d( GGML_ASSERT(ggml_nelements(a) == ne0*ne1*ne2); const int64_t ne[3] = { ne0, ne1, ne2 }; - struct ggml_tensor * result = ggml_new_tensor_impl(ctx, a->type, 3, ne, a, 0); + struct ggml_tensor * result = ggml_new_tensor_impl(ctx, a->type, 3, ne, NULL, a, 0); ggml_format_name(result, "%s (reshaped)", a->name); result->op = GGML_OP_RESHAPE; @@ -3831,7 +3780,7 @@ struct ggml_tensor * ggml_reshape_4d( GGML_ASSERT(ggml_nelements(a) == ne0*ne1*ne2*ne3); const int64_t ne[4] = { ne0, ne1, ne2, ne3 }; - struct ggml_tensor * result = ggml_new_tensor_impl(ctx, a->type, 4, ne, a, 0); + struct ggml_tensor * result = ggml_new_tensor_impl(ctx, a->type, 4, ne, NULL, a, 0); ggml_format_name(result, "%s (reshaped)", a->name); result->op = GGML_OP_RESHAPE; @@ -3845,8 +3794,12 @@ static struct ggml_tensor * ggml_view_impl( struct ggml_tensor * a, int n_dims, const int64_t * ne, + const size_t * nb, size_t offset) { - struct ggml_tensor * result = ggml_new_tensor_impl(ctx, a->type, n_dims, ne, a, offset); + // a strided view counts its strides from elements that are contiguous along dim 0 of its source + GGML_ASSERT(nb == NULL || a->nb[0] == ggml_type_size(a->type)); + + struct ggml_tensor * result = ggml_new_tensor_impl(ctx, a->type, n_dims, ne, nb, a, offset); ggml_format_name(result, "%s (view)", a->name); ggml_set_op_params(result, &offset, sizeof(offset)); @@ -3864,7 +3817,7 @@ struct ggml_tensor * ggml_view_1d( struct ggml_tensor * a, int64_t ne0, size_t offset) { - struct ggml_tensor * result = ggml_view_impl(ctx, a, 1, &ne0, offset); + struct ggml_tensor * result = ggml_view_impl(ctx, a, 1, &ne0, NULL, offset); return result; } @@ -3879,12 +3832,9 @@ struct ggml_tensor * ggml_view_2d( size_t nb1, size_t offset) { const int64_t ne[2] = { ne0, ne1 }; + const size_t nb[4] = { ggml_type_size(a->type), nb1, nb1*ne1, nb1*ne1 }; - struct ggml_tensor * result = ggml_view_impl(ctx, a, 2, ne, offset); - - result->nb[1] = nb1; - result->nb[2] = result->nb[1]*ne1; - result->nb[3] = result->nb[2]; + struct ggml_tensor * result = ggml_view_impl(ctx, a, 2, ne, nb, offset); return result; } @@ -3901,12 +3851,9 @@ struct ggml_tensor * ggml_view_3d( size_t nb2, size_t offset) { const int64_t ne[3] = { ne0, ne1, ne2 }; + const size_t nb[4] = { ggml_type_size(a->type), nb1, nb2, nb2*ne2 }; - struct ggml_tensor * result = ggml_view_impl(ctx, a, 3, ne, offset); - - result->nb[1] = nb1; - result->nb[2] = nb2; - result->nb[3] = result->nb[2]*ne2; + struct ggml_tensor * result = ggml_view_impl(ctx, a, 3, ne, nb, offset); return result; } @@ -3925,12 +3872,9 @@ struct ggml_tensor * ggml_view_4d( size_t nb3, size_t offset) { const int64_t ne[4] = { ne0, ne1, ne2, ne3 }; + const size_t nb[4] = { ggml_type_size(a->type), nb1, nb2, nb3 }; - struct ggml_tensor * result = ggml_view_impl(ctx, a, 4, ne, offset); - - result->nb[1] = nb1; - result->nb[2] = nb2; - result->nb[3] = nb3; + struct ggml_tensor * result = ggml_view_impl(ctx, a, 4, ne, nb, offset); return result; } @@ -4103,70 +4047,6 @@ struct ggml_tensor * ggml_diag( return result; } -// ggml_diag_mask_inf - -static struct ggml_tensor * ggml_diag_mask_inf_impl( - struct ggml_context * ctx, - struct ggml_tensor * a, - int n_past, - bool inplace) { - struct ggml_tensor * result = inplace ? ggml_view_tensor(ctx, a) : ggml_dup_tensor(ctx, a); - - int32_t params[] = { n_past }; - ggml_set_op_params(result, params, sizeof(params)); - - result->op = GGML_OP_DIAG_MASK_INF; - result->src[0] = a; - - return result; -} - -struct ggml_tensor * ggml_diag_mask_inf( - struct ggml_context * ctx, - struct ggml_tensor * a, - int n_past) { - return ggml_diag_mask_inf_impl(ctx, a, n_past, false); -} - -struct ggml_tensor * ggml_diag_mask_inf_inplace( - struct ggml_context * ctx, - struct ggml_tensor * a, - int n_past) { - return ggml_diag_mask_inf_impl(ctx, a, n_past, true); -} - -// ggml_diag_mask_zero - -static struct ggml_tensor * ggml_diag_mask_zero_impl( - struct ggml_context * ctx, - struct ggml_tensor * a, - int n_past, - bool inplace) { - struct ggml_tensor * result = inplace ? ggml_view_tensor(ctx, a) : ggml_dup_tensor(ctx, a); - - int32_t params[] = { n_past }; - ggml_set_op_params(result, params, sizeof(params)); - - result->op = GGML_OP_DIAG_MASK_ZERO; - result->src[0] = a; - - return result; -} - -struct ggml_tensor * ggml_diag_mask_zero( - struct ggml_context * ctx, - struct ggml_tensor * a, - int n_past) { - return ggml_diag_mask_zero_impl(ctx, a, n_past, false); -} - -struct ggml_tensor * ggml_diag_mask_zero_inplace( - struct ggml_context * ctx, - struct ggml_tensor * a, - int n_past) { - return ggml_diag_mask_zero_impl(ctx, a, n_past, true); -} - // ggml_clamp static struct ggml_tensor * ggml_clamp_impl( @@ -4360,7 +4240,7 @@ static struct ggml_tensor * ggml_rope_impl( struct ggml_tensor * result = inplace ? ggml_view_tensor(ctx, a) : ggml_dup_tensor(ctx, a); - int32_t params[16] = { /*n_past*/ 0, n_dims, mode, /*n_ctx*/ 0, n_ctx_orig }; + int32_t params[16] = { /*n_past*/ 0, n_dims, mode, /*no_trunc*/ 0, n_ctx_orig }; memcpy(params + 5, &freq_base, sizeof(float)); memcpy(params + 6, &freq_scale, sizeof(float)); memcpy(params + 7, &ext_factor, sizeof(float)); @@ -4488,44 +4368,6 @@ struct ggml_tensor * ggml_rope_ext_inplace( ); } -struct ggml_tensor * ggml_rope_custom( - struct ggml_context * ctx, - struct ggml_tensor * a, - struct ggml_tensor * b, - int n_dims, - int mode, - int n_ctx_orig, - float freq_base, - float freq_scale, - float ext_factor, - float attn_factor, - float beta_fast, - float beta_slow) { - return ggml_rope_impl( - ctx, a, b, NULL, n_dims, NULL, mode, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow, false - ); -} - -struct ggml_tensor * ggml_rope_custom_inplace( - struct ggml_context * ctx, - struct ggml_tensor * a, - struct ggml_tensor * b, - int n_dims, - int mode, - int n_ctx_orig, - float freq_base, - float freq_scale, - float ext_factor, - float attn_factor, - float beta_fast, - float beta_slow) { - return ggml_rope_impl( - ctx, a, b, NULL, n_dims, NULL, mode, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow, true - ); -} - // Apparently solving `n_rot = 2pi * x * base^((2 * max_pos_emb) / n_dims)` for x, we get // `corr_dim(n_rot) = n_dims * log(max_pos_emb / (n_rot * 2pi)) / (2 * log(base))` static float ggml_rope_yarn_corr_dim(int n_dims, int n_ctx_orig, float n_rot, float base) { @@ -4533,11 +4375,15 @@ static float ggml_rope_yarn_corr_dim(int n_dims, int n_ctx_orig, float n_rot, fl } void ggml_rope_yarn_corr_dims( - int n_dims, int n_ctx_orig, float freq_base, float beta_fast, float beta_slow, float dims[2] + int n_dims, int n_ctx_orig, float freq_base, float beta_fast, float beta_slow, bool truncate, float dims[2] ) { // start and end correction dims - float start = floorf(ggml_rope_yarn_corr_dim(n_dims, n_ctx_orig, beta_fast, freq_base)); - float end = ceilf(ggml_rope_yarn_corr_dim(n_dims, n_ctx_orig, beta_slow, freq_base)); + float start = ggml_rope_yarn_corr_dim(n_dims, n_ctx_orig, beta_fast, freq_base); + float end = ggml_rope_yarn_corr_dim(n_dims, n_ctx_orig, beta_slow, freq_base); + if (truncate) { + start = floorf(start); + end = ceilf(end); + } dims[0] = MAX(0, start); dims[1] = MIN(n_dims - 1, end); } @@ -4598,6 +4444,19 @@ struct ggml_tensor * ggml_rope_set_offset( return a; } +struct ggml_tensor * ggml_rope_set_truncate( + struct ggml_tensor * a, + bool truncate) { + GGML_ASSERT(a->op == GGML_OP_ROPE || a->op == GGML_OP_ROPE_BACK); + ggml_set_op_params_i32(a, 3, !truncate); + return a; +} + +bool ggml_rope_get_truncate(const struct ggml_tensor * a) { + GGML_ASSERT(a->op == GGML_OP_ROPE || a->op == GGML_OP_ROPE_BACK); + return ggml_get_op_params_i32(a, 3) == 0; +} + static int64_t ggml_calc_conv_output_size(int64_t ins, int64_t ks, int s, int p, int d) { return (ins + 2 * p - d * (ks - 1) - 1) / s + 1; } @@ -5248,17 +5107,6 @@ struct ggml_tensor * ggml_upscale( return ggml_interpolate_impl(ctx, a, a->ne[0] * scale_factor, a->ne[1] * scale_factor, a->ne[2], a->ne[3], mode); } -struct ggml_tensor * ggml_upscale_ext( - struct ggml_context * ctx, - struct ggml_tensor * a, - int ne0, - int ne1, - int ne2, - int ne3, - enum ggml_scale_mode mode) { - return ggml_interpolate_impl(ctx, a, ne0, ne1, ne2, ne3, mode); -} - struct ggml_tensor * ggml_interpolate( struct ggml_context * ctx, struct ggml_tensor * a, @@ -5389,10 +5237,10 @@ struct ggml_tensor * ggml_roll( int shift2, int shift3) { GGML_ASSERT(a->nb[0] == ggml_type_size(a->type)); - GGML_ASSERT(abs(shift0) < a->ne[0]); - GGML_ASSERT(abs(shift1) < a->ne[1]); - GGML_ASSERT(abs(shift2) < a->ne[2]); - GGML_ASSERT(abs(shift3) < a->ne[3]); + GGML_ASSERT(shift0 > -a->ne[0] && shift0 < a->ne[0]); + GGML_ASSERT(shift1 > -a->ne[1] && shift1 < a->ne[1]); + GGML_ASSERT(shift2 > -a->ne[2] && shift2 < a->ne[2]); + GGML_ASSERT(shift3 > -a->ne[3] && shift3 < a->ne[3]); struct ggml_tensor * result = ggml_dup_tensor(ctx, a); @@ -5638,16 +5486,6 @@ struct ggml_tensor * ggml_flash_attn_ext_rows( } -void ggml_flash_attn_ext_set_prec( - struct ggml_tensor * a, - enum ggml_prec prec) { - GGML_ASSERT(a->op == GGML_OP_FLASH_ATTN_EXT); - - const int32_t prec_i32 = (int32_t) prec; - - ggml_set_op_params_i32(a, 3, prec_i32); // scale is on first pos, max_bias on second -} - enum ggml_prec ggml_flash_attn_ext_get_prec( const struct ggml_tensor * a) { GGML_ASSERT(a->op == GGML_OP_FLASH_ATTN_EXT); @@ -6857,22 +6695,6 @@ static void ggml_acc_or_set( ggml_build_forward_expand(cgraph, cgraph->grads[isrc]); } -static void ggml_add1_or_set( - struct ggml_context * ctx, - struct ggml_cgraph * cgraph, - size_t isrc, - struct ggml_tensor * tensor) { - struct ggml_tensor * src = cgraph->visited_hash_set.keys[isrc]; - GGML_ASSERT(src); - if (cgraph->grads[isrc]) { - cgraph->grads[isrc] = ggml_add1_impl(ctx, cgraph->grads[isrc], tensor, cgraph->grad_accs[isrc]); - } else { - cgraph->grads[isrc] = ggml_repeat(ctx, tensor, src); - } - ggml_format_name(cgraph->grads[isrc], "grad for %s", src->name); - ggml_build_forward_expand(cgraph, cgraph->grads[isrc]); -} - static void ggml_sub_or_set( struct ggml_context * ctx, struct ggml_cgraph * cgraph, @@ -6927,14 +6749,6 @@ static void ggml_compute_backward( ggml_add_or_set(ctx, cgraph, isrc1, tmp); } } break; - case GGML_OP_ADD1: { - if (src0_needs_grads) { - ggml_add_or_set(ctx, cgraph, isrc0, grad); - } - if (src1_needs_grads) { - ggml_add_or_set(ctx, cgraph, isrc1, ggml_mean(ctx, grad)); // TODO: should probably be sum instead of mean - } - } break; case GGML_OP_ACC: { if (src0_needs_grads) { ggml_add_or_set(ctx, cgraph, isrc0, grad); @@ -7007,7 +6821,7 @@ static void ggml_compute_backward( } break; case GGML_OP_SUM: { if (src0_needs_grads) { - ggml_add1_or_set(ctx, cgraph, isrc0, grad); + ggml_add_or_set(ctx, cgraph, isrc0, ggml_repeat(ctx, grad, src0)); } } break; case GGML_OP_SUM_ROWS: { @@ -7017,7 +6831,7 @@ static void ggml_compute_backward( } break; case GGML_OP_MEAN: { if (src0_needs_grads) { - ggml_add1_or_set(ctx, cgraph, isrc0, ggml_scale_impl(ctx, grad, 1.0f/src0->ne[0], 0.0, false)); + ggml_add_or_set(ctx, cgraph, isrc0, ggml_repeat(ctx, ggml_scale_impl(ctx, grad, 1.0f/src0->ne[0], 0.0, false), src0)); } } break; case GGML_OP_REPEAT: { @@ -7207,18 +7021,9 @@ static void ggml_compute_backward( // noop } } break; - case GGML_OP_DIAG_MASK_INF: { - if (src0_needs_grads) { - /* ggml_diag_mask_inf_impl() shouldn't be here */ - /* ref: https://github.com/ggml-org/llama.cpp/pull/4203#discussion_r1412377992 */ - const int n_past = ((const int32_t *) tensor->op_params)[0]; - ggml_add_or_set(ctx, cgraph, isrc0, ggml_diag_mask_zero_impl(ctx, grad, n_past, false)); - } - } break; - case GGML_OP_DIAG_MASK_ZERO: { + case GGML_OP_TRI: { if (src0_needs_grads) { - const int n_past = ((const int32_t *) tensor->op_params)[0]; - ggml_add_or_set(ctx, cgraph, isrc0, ggml_diag_mask_zero_impl(ctx, grad, n_past, false)); + ggml_add_or_set(ctx, cgraph, isrc0, ggml_tri(ctx, grad, (enum ggml_tri_type) ggml_get_op_params_i32(tensor, 0))); } } break; case GGML_OP_SOFT_MAX: { @@ -7238,7 +7043,6 @@ static void ggml_compute_backward( //const int n_past = ((int32_t *) tensor->op_params)[0]; const int n_dims = ((const int32_t *) tensor->op_params)[1]; const int mode = ((const int32_t *) tensor->op_params)[2]; - //const int n_ctx = ((int32_t *) tensor->op_params)[3]; const int n_ctx_orig = ((const int32_t *) tensor->op_params)[4]; float freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow; int sections[4] = {0, 0, 0, 0}; @@ -7256,6 +7060,7 @@ static void ggml_compute_backward( mode, n_ctx_orig, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow) : ggml_rope_multi_back(ctx, grad, src1, src2, n_dims, sections, mode, n_ctx_orig, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow); + ggml_rope_set_truncate(rope_back, ggml_rope_get_truncate(tensor)); ggml_add_or_set(ctx, cgraph, isrc0, rope_back); } GGML_ASSERT((!src2 || !src2_needs_grads) && "gradients for freq factors not implemented"); diff --git a/ggml/src/gguf.cpp b/ggml/src/gguf.cpp index 75ceade7d5fb..c80b682185ce 100644 --- a/ggml/src/gguf.cpp +++ b/ggml/src/gguf.cpp @@ -1463,6 +1463,7 @@ struct gguf_writer_base { // we bet on devirtualization virtual void write(int8_t val) = 0; virtual void write(const std::vector & val) = 0; + virtual void write(const void * data, size_t size) = 0; virtual void write_tensor_data(const struct gguf_tensor_info & info, size_t offset_data, size_t alignment) = 0; template @@ -1579,21 +1580,31 @@ struct gguf_writer_buf final : public gguf_writer_base { written_bytes += val.size(); } + void write(const void * data, const size_t size) override { + buf.insert(buf.end(), (const int8_t *) data, (const int8_t *) data + size); + written_bytes += size; + } + void write_tensor_data(const struct gguf_tensor_info & info, const size_t offset_data, const size_t alignment) override { GGML_ASSERT(buf.size() - offset_data == info.offset); GGML_ASSERT(ggml_is_contiguous(&info.t)); - const size_t offset = buf.size(); const size_t nbytes = ggml_nbytes(&info.t); - buf.resize(offset + nbytes); - if (info.t.buffer) { - ggml_backend_tensor_get(&info.t, buf.data() + offset, 0, nbytes); + if (info.t.buffer && ggml_backend_buffer_is_host(info.t.buffer)) { + GGML_ASSERT(info.t.data != NULL && "tensor not allocated"); + write(info.t.data, nbytes); } else { - GGML_ASSERT(info.t.data); - memcpy(buf.data() + offset, info.t.data, nbytes); + const size_t offset = buf.size(); + buf.resize(offset + nbytes); + if (info.t.buffer) { + ggml_backend_tensor_get(&info.t, buf.data() + offset, 0, nbytes); + } else { + GGML_ASSERT(info.t.data); + memcpy(buf.data() + offset, info.t.data, nbytes); + } + written_bytes += nbytes; } - written_bytes += nbytes; pad(alignment); } @@ -1624,20 +1635,33 @@ struct gguf_writer_file final : public gguf_writer_base { } } + void write(const void * data, const size_t size) override { + const auto ret = fwrite(data, 1, size, file); + written_bytes += size; + if (ret != size) { + throw std::runtime_error("unexpected fwrite number of bytes written, '" + std::to_string(ret) + "' instead of '" + std::to_string(size) + "'"); + } + } + void write_tensor_data(const struct gguf_tensor_info & info, const size_t offset_data, const size_t alignment) override { GGML_ASSERT(written_bytes - offset_data == info.offset); GGML_ASSERT(ggml_is_contiguous(&info.t)); const size_t nbytes = ggml_nbytes(&info.t); - std::vector buf(nbytes); - if (info.t.buffer) { - ggml_backend_tensor_get(&info.t, buf.data(), 0, nbytes); + if (info.t.buffer && ggml_backend_buffer_is_host(info.t.buffer)) { + GGML_ASSERT(info.t.data != NULL && "tensor not allocated"); + write(info.t.data, nbytes); } else { - GGML_ASSERT(info.t.data); - memcpy(buf.data(), info.t.data, nbytes); + std::vector buf(nbytes); + if (info.t.buffer) { + ggml_backend_tensor_get(&info.t, buf.data(), 0, nbytes); + } else { + GGML_ASSERT(info.t.data); + memcpy(buf.data(), info.t.data, nbytes); + } + write(buf); } - write(buf); pad(alignment); } diff --git a/gguf-py/gguf/constants.py b/gguf-py/gguf/constants.py index 4d057295c128..ad0bd8e67004 100644 --- a/gguf-py/gguf/constants.py +++ b/gguf-py/gguf/constants.py @@ -162,6 +162,7 @@ class LLM: EMBD_LENGTH_PER_LAYER_INP = "{arch}.embedding_length_per_layer_input" SWIGLU_CLAMP_EXP = "{arch}.swiglu_clamp_exp" SWIGLU_CLAMP_SHEXP = "{arch}.swiglu_clamp_shexp" + SWIGLU_CLAMP = "{arch}.swiglu_clamp" HIDDEN_ACT = "{arch}.hidden_activation" DENSE_FEAT_IN_SIZE = "{arch}.{dense}_feat_in" DENSE_FEAT_OUT_SIZE = "{arch}.{dense}_feat_out" @@ -177,6 +178,7 @@ class LLM: NORM_BEFORE_RESIDUAL = "{arch}.norm_before_residual" NORM_BEFORE_FC = "{arch}.norm_before_fc" REALTIME_DELAY_TOKENS = "{arch}.realtime_delay_tokens" + CLASSIFIER_FREE_GUIDANCE = "{arch}.classifier_free_guidance" class Adapters: COUNT = "{arch}.adapters.count" @@ -203,6 +205,7 @@ class Attention: ICLR_LORA_RANK = "{arch}.attention.iclr_lora_rank" VALUE_RESIDUAL_MIX_LORA_RANK = "{arch}.attention.value_residual_mix_lora_rank" GATE_LORA_RANK = "{arch}.attention.gate_lora_rank" + PASS_LORA_RANK = "{arch}.attention.pass_lora_rank" REL_BUCKETS_COUNT = "{arch}.attention.relative_buckets_count" SLIDING_WINDOW = "{arch}.attention.sliding_window" SCALE = "{arch}.attention.scale" @@ -315,6 +318,12 @@ class HRM: L_CYCLES = "{arch}.hrm.l_cycles" PREFIX_LM = "{arch}.hrm.prefix_lm" + class Metis: + PASS_COUNT = "{arch}.metis.pass_count" + MEMORY_LENGTH = "{arch}.metis.memory_length" + ROUTE_LENGTH = "{arch}.metis.route_length" + CONTINUATION_LOGIT_LIMIT = "{arch}.metis.continuation_logit_limit" + class SSM: CONV_KERNEL = "{arch}.ssm.conv_kernel" INNER_SIZE = "{arch}.ssm.inner_size" @@ -322,6 +331,7 @@ class SSM: TIME_STEP_RANK = "{arch}.ssm.time_step_rank" GROUP_COUNT = "{arch}.ssm.group_count" DT_B_C_RMS = "{arch}.ssm.dt_b_c_rms" + NORM_EPSILON = "{arch}.ssm.norm_epsilon" class KDA: HEAD_DIM = "{arch}.kda.head_dim" @@ -506,6 +516,11 @@ class ClipAudio: WINDOW_SIZE = "clip.audio.window_size" LOCAL_BLOCK_COUNT = "clip.audio.local_block_count" # mimo-v2.5: input_local_transformer layer count LOCAL_GROUP_SIZE = "clip.audio.local_group_size" # mimo-v2.5: input_local_transformer grouping size + ROPE_DIMS = "clip.audio.rope_dims" # nemotron-transcribe + + class Speaker: # nemotron-transcribe + BLOCK_COUNT = "clip.audio.spk.block_count" + HEAD_COUNT = "clip.audio.spk.head_count" class Attention: HEAD_COUNT = "clip.audio.attention.head_count" @@ -574,7 +589,7 @@ class GGUFType: class MODEL_ARCH(IntEnum): - MMPROJ = auto() # dummy arch for clip.cpp + MMPROJ = auto() # dummy arch for mmproj LLAMA = auto() LLAMA4 = auto() DECI = auto() @@ -663,6 +678,7 @@ class MODEL_ARCH(IntEnum): DEEPSEEK32 = auto() DEEPSEEK4 = auto() DEEPSEEK41 = auto() + XING4_0 = auto() CHATGLM = auto() GLM4 = auto() GLM4_MOE = auto() @@ -676,6 +692,7 @@ class MODEL_ARCH(IntEnum): NEMOTRON = auto() NEMOTRON_H = auto() NEMOTRON_H_MOE = auto() + METIS16 = auto() EXAONE = auto() EXAONE4 = auto() EXAONE_MOE = auto() @@ -695,6 +712,7 @@ class MODEL_ARCH(IntEnum): ARCEE = auto() AFMOE = auto() LAGUNA = auto() + RICURSIVE = auto() ERNIE4_5 = auto() ERNIE4_5_MOE = auto() HUNYUAN_MOE = auto() @@ -734,6 +752,7 @@ class MODEL_ARCH(IntEnum): MAION_CODER = auto() KIMI_LINEAR = auto() KIMI_K3 = auto() + KIMI_K31 = auto() TALKIE = auto() MELLUM = auto() NANBEIGE = auto() @@ -825,6 +844,36 @@ class MODEL_TENSOR(IntEnum): FFN_GATE_TID2EID = auto() MOE_LATENT_DOWN = auto() # nemotron 3 super MOE_LATENT_UP = auto() # nemotron 3 super + STREAM_EMBD = auto() + DEPTH_MEM_STATE_WRITE = auto() + DEPTH_MEM_META_WRITE = auto() + DEPTH_MEM_ENTRY_BIAS = auto() + DEPTH_MEM_QUERY = auto() + DEPTH_MEM_KEY = auto() + DEPTH_MEM_VALUE = auto() + DEPTH_MEM_OUT = auto() + DEPTH_MEM_ROUTE_STATE = auto() + DEPTH_MEM_ROUTE_RETR = auto() + DEPTH_MEM_ROUTE_DIFF = auto() + DEPTH_MEM_ROUTE_HIST = auto() + DEPTH_MEM_ROUTE = auto() + DEPTH_MEM_STREAM_GATE = auto() + CONT_HIDDEN_STATE = auto() + CONT_HIDDEN_RETR = auto() + CONT_HIDDEN_DIFF = auto() + CONT_HIDDEN_ROUTE = auto() + CONT_HIDDEN = auto() + CONT_OUTPUT = auto() + HC_ATTN_MIX = auto() + HC_ATTN_READ = auto() + HC_ATTN_WRITE = auto() + HC_FFN_MIX = auto() + HC_FFN_READ = auto() + HC_FFN_WRITE = auto() + ATTN_PASS_LORA_DOWN = auto() + ATTN_PASS_LORA_UP = auto() + ATTN_PASS_GATE = auto() + FFN_EXP_EMBD = auto() FFN_RESID_ALPHA = auto() FFN_RESID_BETA = auto() FFN_PRE_EXP_NORM = auto() @@ -833,6 +882,7 @@ class MODEL_TENSOR(IntEnum): ATTN_K_NORM = auto() LAYER_OUT_NORM = auto() LAYER_OUT_SCALE = auto() + ATTN_VRES_LAMBDA = auto() PER_LAYER_TOKEN_EMBD = auto() # gemma3n PER_LAYER_MODEL_PROJ = auto() # gemma3n PER_LAYER_INP_GATE = auto() # gemma3n @@ -948,8 +998,8 @@ class MODEL_TENSOR(IntEnum): HC_FFN_INJECT = auto() # qwen4exp HC_ATTN_DOWN_INJECT = auto() # qwen4exp, [hc_attn_down | hc_attn_inject] HC_FFN_DOWN_INJECT = auto() # qwen4exp, [hc_ffn_down | hc_ffn_inject] - ATTN_QKV_GATE = auto() # qwen4exp, [attn_qkv | attn_gate] - SSM_BETA_ALPHA_CAT = auto() # qwen4exp, [ssm_beta | ssm_alpha], unlike the grouped qwen3next ssm_ba + ATTN_QKV_GATE = auto() # qwen35, qwen4exp, [attn_qkv | attn_gate] + SSM_BETA_ALPHA_CAT = auto() # qwen35, qwen4exp, [ssm_beta | ssm_alpha], unlike the grouped qwen3next ssm_ba PLE_KEY = auto() # qwen4exp PLE_VALUE = auto() # qwen4exp PLE_NORM_KEY = auto() # qwen4exp @@ -1062,6 +1112,7 @@ class MODEL_TENSOR(IntEnum): V_ENC_EMBD_NORM = auto() V_ENC_EMBD_PATCH_NORM = auto() # allow multiple norms in the same embd, e.g. for gemma4u V_ENC_EMBD_POS = auto() + V_ENC_ROPE_FREQS = auto() V_ENC_INPUT_NORM = auto() V_ENC_ATTN_QKV = auto() V_ENC_ATTN_Q = auto() @@ -1251,6 +1302,11 @@ class MODEL_TENSOR(IntEnum): A_GEN_CODE_ATTN_GATE = auto() # lfm3a: per-head sigmoid gate on the attention output A_GEN_CODE_HEAD_NORM = auto() # lfm3a: per-codebook norm before the output head, merged 2D [n_codebooks, dim] A_GEN_CODE_MASK_EMBD = auto() # lfm3a: MaskGIT mask embedding + A_GEN_CODE_ENORM = auto() # drama3: per-layer norm of the code embedding + A_GEN_CODE_HNORM = auto() # drama3: per-layer norm of the input hidden state + A_GEN_CODE_EH_PROJ = auto() # drama3: per-layer [code embd; hidden] -> hidden projection + A_GEN_CODE_BLK_EMBD = auto() # drama3: per-layer code embedding table + A_GEN_CODE_BLK_OUT_NORM = auto() # drama3: per-layer output norm # qwen3tts code2wav: RVQ codes -> raw PCM A_GEN_WAV_QUANT_FIRST_IN = auto() # semantic RVQ, in_proj (1x1 conv, loaded as 2D) A_GEN_WAV_QUANT_FIRST_OUT = auto() # semantic RVQ, out_proj @@ -1411,6 +1467,24 @@ class MODEL_TENSOR(IntEnum): A_ENC_CONV_NORM_VAR = auto() # parakeet A_ENC_MEL_FILTERS = auto() # parakeet A_ENC_WINDOW = auto() # parakeet + A_ENC_ATTN_QKV = auto() # nemotron-transcribe + A_SPK_INP_PROJ = auto() # nemotron-transcribe + A_SPK_PRE_NORM = auto() # nemotron-transcribe + A_SPK_POST_NORM = auto() # nemotron-transcribe + A_SPK_ATTN_NORM = auto() # nemotron-transcribe + A_SPK_ATTN_QKV = auto() # nemotron-transcribe + A_SPK_ATTN_OUT = auto() # nemotron-transcribe + A_SPK_FFN_NORM = auto() # nemotron-transcribe + A_SPK_FFN_UP = auto() # nemotron-transcribe + A_SPK_FFN_DOWN = auto() # nemotron-transcribe + A_SPK_HEAD_PROJ = auto() # nemotron-transcribe + A_SPK_HEAD_UPSAMPLE = auto() # nemotron-transcribe + A_SPK_HEAD_FC1 = auto() # nemotron-transcribe + A_SPK_HEAD_FC2 = auto() # nemotron-transcribe + A_SPK_SIL_EMB = auto() # nemotron-transcribe + A_FUSION_ASR_NORM = auto() # nemotron-transcribe + A_FUSION_SPK_NORM = auto() # nemotron-transcribe + A_FUSION_SPK_KERNEL = auto() # nemotron-transcribe A_CTC_OUT = auto() A_CTC_OUT_MID = auto() A_ENC_ATTN_REL_POS_EMB = auto() @@ -1434,7 +1508,7 @@ class MODEL_TENSOR(IntEnum): MODEL_ARCH_NAMES: dict[MODEL_ARCH, str] = { - MODEL_ARCH.MMPROJ: "clip", # dummy arch for clip.cpp + MODEL_ARCH.MMPROJ: "clip", # dummy arch for mmproj files (tools/mtmd) MODEL_ARCH.LLAMA: "llama", MODEL_ARCH.LLAMA4: "llama4", MODEL_ARCH.DECI: "deci", @@ -1521,6 +1595,7 @@ class MODEL_TENSOR(IntEnum): MODEL_ARCH.DEEPSEEK2OCR: "deepseek2-ocr", MODEL_ARCH.DEEPSEEK32: "deepseek32", MODEL_ARCH.DEEPSEEK4: "deepseek4", + MODEL_ARCH.XING4_0: "xing4_0", MODEL_ARCH.DEEPSEEK41: "deepseek41", MODEL_ARCH.CHATGLM: "chatglm", MODEL_ARCH.GLM4: "glm4", @@ -1535,6 +1610,7 @@ class MODEL_TENSOR(IntEnum): MODEL_ARCH.NEMOTRON: "nemotron", MODEL_ARCH.NEMOTRON_H: "nemotron_h", MODEL_ARCH.NEMOTRON_H_MOE: "nemotron_h_moe", + MODEL_ARCH.METIS16: "metis16", MODEL_ARCH.EXAONE: "exaone", MODEL_ARCH.EXAONE4: "exaone4", MODEL_ARCH.EXAONE_MOE: "exaone-moe", @@ -1554,6 +1630,7 @@ class MODEL_TENSOR(IntEnum): MODEL_ARCH.ARCEE: "arcee", MODEL_ARCH.AFMOE: "afmoe", MODEL_ARCH.LAGUNA: "laguna", + MODEL_ARCH.RICURSIVE: "ricursive", MODEL_ARCH.ERNIE4_5: "ernie4_5", MODEL_ARCH.ERNIE4_5_MOE: "ernie4_5-moe", MODEL_ARCH.FALCON_H1: "falcon-h1", @@ -1594,6 +1671,7 @@ class MODEL_TENSOR(IntEnum): MODEL_ARCH.MAION_CODER: "maion-coder", MODEL_ARCH.KIMI_LINEAR: "kimi-linear", MODEL_ARCH.KIMI_K3: "kimi-k3", + MODEL_ARCH.KIMI_K31: "kimi-k31", MODEL_ARCH.TALKIE: "talkie", MODEL_ARCH.MELLUM: "mellum", MODEL_ARCH.NANBEIGE: "nanbeige", @@ -1685,12 +1763,43 @@ class MODEL_TENSOR(IntEnum): MODEL_TENSOR.FFN_GATE_TID2EID: "blk.{bid}.ffn_gate_tid2eid", MODEL_TENSOR.MOE_LATENT_DOWN: "blk.{bid}.ffn_latent_down", # nemotron 3 super MODEL_TENSOR.MOE_LATENT_UP: "blk.{bid}.ffn_latent_up", # nemotron 3 super + MODEL_TENSOR.STREAM_EMBD: "stream_embd", + MODEL_TENSOR.DEPTH_MEM_STATE_WRITE: "depth_mem_state_write", + MODEL_TENSOR.DEPTH_MEM_META_WRITE: "depth_mem_meta_write", + MODEL_TENSOR.DEPTH_MEM_ENTRY_BIAS: "depth_mem_entry_bias", + MODEL_TENSOR.DEPTH_MEM_QUERY: "depth_mem_query", + MODEL_TENSOR.DEPTH_MEM_KEY: "depth_mem_key", + MODEL_TENSOR.DEPTH_MEM_VALUE: "depth_mem_value", + MODEL_TENSOR.DEPTH_MEM_OUT: "depth_mem_out", + MODEL_TENSOR.DEPTH_MEM_ROUTE_STATE: "depth_mem_route_state", + MODEL_TENSOR.DEPTH_MEM_ROUTE_RETR: "depth_mem_route_retr", + MODEL_TENSOR.DEPTH_MEM_ROUTE_DIFF: "depth_mem_route_diff", + MODEL_TENSOR.DEPTH_MEM_ROUTE_HIST: "depth_mem_route_hist", + MODEL_TENSOR.DEPTH_MEM_ROUTE: "depth_mem_route", + MODEL_TENSOR.DEPTH_MEM_STREAM_GATE: "depth_mem_stream_gate", + MODEL_TENSOR.CONT_HIDDEN_STATE: "cont_hidden_state", + MODEL_TENSOR.CONT_HIDDEN_RETR: "cont_hidden_retr", + MODEL_TENSOR.CONT_HIDDEN_DIFF: "cont_hidden_diff", + MODEL_TENSOR.CONT_HIDDEN_ROUTE: "cont_hidden_route", + MODEL_TENSOR.CONT_HIDDEN: "cont_hidden", + MODEL_TENSOR.CONT_OUTPUT: "cont_output", + MODEL_TENSOR.HC_ATTN_MIX: "blk.{bid}.hc_attn_mix", + MODEL_TENSOR.HC_ATTN_READ: "blk.{bid}.hc_attn_read", + MODEL_TENSOR.HC_ATTN_WRITE: "blk.{bid}.hc_attn_write", + MODEL_TENSOR.HC_FFN_MIX: "blk.{bid}.hc_ffn_mix", + MODEL_TENSOR.HC_FFN_READ: "blk.{bid}.hc_ffn_read", + MODEL_TENSOR.HC_FFN_WRITE: "blk.{bid}.hc_ffn_write", + MODEL_TENSOR.ATTN_PASS_LORA_DOWN: "blk.{bid}.attn_pass_lora_down", + MODEL_TENSOR.ATTN_PASS_LORA_UP: "blk.{bid}.attn_pass_lora_up", + MODEL_TENSOR.ATTN_PASS_GATE: "blk.{bid}.attn_pass_gate", + MODEL_TENSOR.FFN_EXP_EMBD: "blk.{bid}.ffn_exp_embd", MODEL_TENSOR.FFN_RESID_ALPHA: "blk.{bid}.ffn_resid_alpha", MODEL_TENSOR.FFN_RESID_BETA: "blk.{bid}.ffn_resid_beta", MODEL_TENSOR.FFN_PRE_EXP_NORM: "blk.{bid}.ffn_pre_exp_norm", MODEL_TENSOR.FFN_POST_EXP_NORM: "blk.{bid}.ffn_post_exp_norm", MODEL_TENSOR.LAYER_OUT_NORM: "blk.{bid}.layer_output_norm", MODEL_TENSOR.LAYER_OUT_SCALE: "blk.{bid}.layer_output_scale", + MODEL_TENSOR.ATTN_VRES_LAMBDA: "blk.{bid}.attn_vres_lambda", MODEL_TENSOR.PER_LAYER_TOKEN_EMBD: "per_layer_token_embd", # gemma3n MODEL_TENSOR.PER_LAYER_MODEL_PROJ: "per_layer_model_proj", # gemma3n MODEL_TENSOR.PER_LAYER_PROJ_NORM: "per_layer_proj_norm", # gemma3n @@ -1918,6 +2027,7 @@ class MODEL_TENSOR(IntEnum): MODEL_TENSOR.V_ENC_EMBD_NORM: "v.norm_embd", MODEL_TENSOR.V_ENC_EMBD_PATCH_NORM: "v.patch_norm.{bid}", MODEL_TENSOR.V_ENC_EMBD_POS: "v.position_embd", + MODEL_TENSOR.V_ENC_ROPE_FREQS: "v.rope_freqs", MODEL_TENSOR.V_ENC_ATTN_QKV: "v.blk.{bid}.attn_qkv", MODEL_TENSOR.V_ENC_ATTN_Q: "v.blk.{bid}.attn_q", MODEL_TENSOR.V_ENC_ATTN_Q_NORM: "v.blk.{bid}.attn_q_norm", @@ -2090,14 +2200,14 @@ class MODEL_TENSOR(IntEnum): MODEL_TENSOR.A_GEN_CODE_EMBD: "a.gen.code.embd", MODEL_TENSOR.A_GEN_CODE_HEAD: "a.gen.code.head", MODEL_TENSOR.A_GEN_CODE_OUT_EMBD: "a.gen.code.out_embd", - MODEL_TENSOR.A_GEN_CODE_ATTN_NORM: "a.gen.code.blk.{bid}.ln1", # reuses the generic clip.cpp block loader (TN_LN_1) + MODEL_TENSOR.A_GEN_CODE_ATTN_NORM: "a.gen.code.blk.{bid}.ln1", # reuses the generic mtmd block loader (TN_LN_1) MODEL_TENSOR.A_GEN_CODE_ATTN_Q: "a.gen.code.blk.{bid}.attn_q", MODEL_TENSOR.A_GEN_CODE_ATTN_Q_NORM: "a.gen.code.blk.{bid}.attn_q_norm", MODEL_TENSOR.A_GEN_CODE_ATTN_K: "a.gen.code.blk.{bid}.attn_k", MODEL_TENSOR.A_GEN_CODE_ATTN_K_NORM: "a.gen.code.blk.{bid}.attn_k_norm", MODEL_TENSOR.A_GEN_CODE_ATTN_V: "a.gen.code.blk.{bid}.attn_v", MODEL_TENSOR.A_GEN_CODE_ATTN_OUT: "a.gen.code.blk.{bid}.attn_out", - MODEL_TENSOR.A_GEN_CODE_FFN_NORM: "a.gen.code.blk.{bid}.ln2", # reuses the generic clip.cpp block loader (TN_LN_2) + MODEL_TENSOR.A_GEN_CODE_FFN_NORM: "a.gen.code.blk.{bid}.ln2", # reuses the generic mtmd block loader (TN_LN_2) MODEL_TENSOR.A_GEN_CODE_FFN_GATE: "a.gen.code.blk.{bid}.ffn_gate", MODEL_TENSOR.A_GEN_CODE_FFN_UP: "a.gen.code.blk.{bid}.ffn_up", MODEL_TENSOR.A_GEN_CODE_FFN_DOWN: "a.gen.code.blk.{bid}.ffn_down", @@ -2105,6 +2215,11 @@ class MODEL_TENSOR(IntEnum): MODEL_TENSOR.A_GEN_CODE_ATTN_GATE: "a.gen.code.blk.{bid}.attn_gate", MODEL_TENSOR.A_GEN_CODE_HEAD_NORM: "a.gen.code.head_norm", MODEL_TENSOR.A_GEN_CODE_MASK_EMBD: "a.gen.code.mask_embd", + MODEL_TENSOR.A_GEN_CODE_ENORM: "a.gen.code.blk.{bid}.enorm", + MODEL_TENSOR.A_GEN_CODE_HNORM: "a.gen.code.blk.{bid}.hnorm", + MODEL_TENSOR.A_GEN_CODE_EH_PROJ: "a.gen.code.blk.{bid}.eh_proj", + MODEL_TENSOR.A_GEN_CODE_BLK_EMBD: "a.gen.code.blk.{bid}.embd", + MODEL_TENSOR.A_GEN_CODE_BLK_OUT_NORM: "a.gen.code.blk.{bid}.out_norm", MODEL_TENSOR.A_GEN_WAV_QUANT_FIRST_IN: "a.gen.wav.quant.first.in_proj", MODEL_TENSOR.A_GEN_WAV_QUANT_FIRST_OUT: "a.gen.wav.quant.first.out_proj", MODEL_TENSOR.A_GEN_WAV_QUANT_FIRST_CB: "a.gen.wav.quant.first.codebook", @@ -2219,6 +2334,24 @@ class MODEL_TENSOR(IntEnum): MODEL_TENSOR.A_ENC_CONV_NORM_VAR: "a.blk.{bid}.conv_norm_var", MODEL_TENSOR.A_ENC_MEL_FILTERS: "a.mel_filters", MODEL_TENSOR.A_ENC_WINDOW: "a.window", + MODEL_TENSOR.A_ENC_ATTN_QKV: "a.blk.{bid}.attn_qkv", + MODEL_TENSOR.A_SPK_INP_PROJ: "a.spk.input_projection", + MODEL_TENSOR.A_SPK_PRE_NORM: "a.spk.pre_ln", + MODEL_TENSOR.A_SPK_POST_NORM: "a.spk.post_ln", + MODEL_TENSOR.A_SPK_ATTN_NORM: "a.spk.blk.{bid}.ln1", + MODEL_TENSOR.A_SPK_ATTN_QKV: "a.spk.blk.{bid}.attn_qkv", + MODEL_TENSOR.A_SPK_ATTN_OUT: "a.spk.blk.{bid}.attn_out", + MODEL_TENSOR.A_SPK_FFN_NORM: "a.spk.blk.{bid}.ln2", + MODEL_TENSOR.A_SPK_FFN_UP: "a.spk.blk.{bid}.ffn_up", + MODEL_TENSOR.A_SPK_FFN_DOWN: "a.spk.blk.{bid}.ffn_down", + MODEL_TENSOR.A_SPK_HEAD_PROJ: "a.spk.head.proj", + MODEL_TENSOR.A_SPK_HEAD_UPSAMPLE: "a.spk.head.upsample", + MODEL_TENSOR.A_SPK_HEAD_FC1: "a.spk.head.fc1", + MODEL_TENSOR.A_SPK_HEAD_FC2: "a.spk.head.fc2", + MODEL_TENSOR.A_SPK_SIL_EMB: "a.spk.head.sil_emb", + MODEL_TENSOR.A_FUSION_ASR_NORM: "a.fusion.asr_norm", + MODEL_TENSOR.A_FUSION_SPK_NORM: "a.fusion.spk_norm", + MODEL_TENSOR.A_FUSION_SPK_KERNEL: "a.fusion.spk_kernel", MODEL_TENSOR.A_CTC_OUT: "a.enc_ctc_out", MODEL_TENSOR.A_CTC_OUT_MID: "a.enc_ctc_out_mid", MODEL_TENSOR.A_ENC_ATTN_REL_POS_EMB: "a.blk.{bid}.attn_rel_pos_emb", @@ -2295,6 +2428,7 @@ class MODEL_TENSOR(IntEnum): MODEL_TENSOR.V_ENC_EMBD_NORM, MODEL_TENSOR.V_ENC_EMBD_PATCH_NORM, MODEL_TENSOR.V_ENC_EMBD_POS, + MODEL_TENSOR.V_ENC_ROPE_FREQS, MODEL_TENSOR.V_ENC_EMBD_IMGNL, MODEL_TENSOR.V_ENC_EMBD_VSEP, MODEL_TENSOR.V_TOK_EMBD_SEP, @@ -2504,6 +2638,11 @@ class MODEL_TENSOR(IntEnum): MODEL_TENSOR.A_GEN_CODE_ATTN_GATE, MODEL_TENSOR.A_GEN_CODE_HEAD_NORM, MODEL_TENSOR.A_GEN_CODE_MASK_EMBD, + MODEL_TENSOR.A_GEN_CODE_ENORM, + MODEL_TENSOR.A_GEN_CODE_HNORM, + MODEL_TENSOR.A_GEN_CODE_EH_PROJ, + MODEL_TENSOR.A_GEN_CODE_BLK_EMBD, + MODEL_TENSOR.A_GEN_CODE_BLK_OUT_NORM, MODEL_TENSOR.A_GEN_WAV_QUANT_FIRST_IN, MODEL_TENSOR.A_GEN_WAV_QUANT_FIRST_OUT, MODEL_TENSOR.A_GEN_WAV_QUANT_FIRST_CB, @@ -2587,6 +2726,24 @@ class MODEL_TENSOR(IntEnum): MODEL_TENSOR.A_ENC_CONV_NORM_VAR, MODEL_TENSOR.A_ENC_MEL_FILTERS, MODEL_TENSOR.A_ENC_WINDOW, + MODEL_TENSOR.A_ENC_ATTN_QKV, + MODEL_TENSOR.A_SPK_INP_PROJ, + MODEL_TENSOR.A_SPK_PRE_NORM, + MODEL_TENSOR.A_SPK_POST_NORM, + MODEL_TENSOR.A_SPK_ATTN_NORM, + MODEL_TENSOR.A_SPK_ATTN_QKV, + MODEL_TENSOR.A_SPK_ATTN_OUT, + MODEL_TENSOR.A_SPK_FFN_NORM, + MODEL_TENSOR.A_SPK_FFN_UP, + MODEL_TENSOR.A_SPK_FFN_DOWN, + MODEL_TENSOR.A_SPK_HEAD_PROJ, + MODEL_TENSOR.A_SPK_HEAD_UPSAMPLE, + MODEL_TENSOR.A_SPK_HEAD_FC1, + MODEL_TENSOR.A_SPK_HEAD_FC2, + MODEL_TENSOR.A_SPK_SIL_EMB, + MODEL_TENSOR.A_FUSION_ASR_NORM, + MODEL_TENSOR.A_FUSION_SPK_NORM, + MODEL_TENSOR.A_FUSION_SPK_KERNEL, MODEL_TENSOR.A_MM_INP_PROJ, MODEL_TENSOR.A_MM_SOFT_EMB_NORM, MODEL_TENSOR.A_MM_EMBEDDING, @@ -3252,6 +3409,7 @@ class MODEL_TENSOR(IntEnum): MODEL_TENSOR.ATTN_POST_NORM, MODEL_TENSOR.ATTN_GATE, MODEL_TENSOR.ATTN_QKV, + MODEL_TENSOR.ATTN_QKV_GATE, MODEL_TENSOR.FFN_GATE, MODEL_TENSOR.FFN_DOWN, MODEL_TENSOR.FFN_UP, @@ -3261,6 +3419,7 @@ class MODEL_TENSOR(IntEnum): MODEL_TENSOR.SSM_NORM, MODEL_TENSOR.SSM_BETA, MODEL_TENSOR.SSM_ALPHA, + MODEL_TENSOR.SSM_BETA_ALPHA_CAT, MODEL_TENSOR.SSM_OUT, # NextN/MTP tensors - preserved but unused MODEL_TENSOR.NEXTN_EH_PROJ, @@ -4377,6 +4536,51 @@ class MODEL_TENSOR(IntEnum): MODEL_TENSOR.NEXTN_SHARED_HEAD_HEAD, MODEL_TENSOR.NEXTN_SHARED_HEAD_NORM, ], + MODEL_ARCH.XING4_0: [ + MODEL_TENSOR.TOKEN_EMBD, + MODEL_TENSOR.OUTPUT_NORM, + MODEL_TENSOR.OUTPUT, + MODEL_TENSOR.ROPE_FREQS, + MODEL_TENSOR.ATTN_NORM, + MODEL_TENSOR.ATTN_Q, + MODEL_TENSOR.ATTN_Q_A, + MODEL_TENSOR.ATTN_Q_B, + MODEL_TENSOR.ATTN_KV_A_MQA, + MODEL_TENSOR.ATTN_KV_B, + MODEL_TENSOR.ATTN_K, + MODEL_TENSOR.ATTN_K_B, + MODEL_TENSOR.ATTN_V, + MODEL_TENSOR.ATTN_V_B, + MODEL_TENSOR.ATTN_Q_A_NORM, + MODEL_TENSOR.ATTN_KV_A_NORM, + MODEL_TENSOR.ATTN_OUT, + MODEL_TENSOR.HC_ATTN_FN, + MODEL_TENSOR.HC_ATTN_BASE, + MODEL_TENSOR.HC_ATTN_SCALE, + MODEL_TENSOR.ATTN_ROT_EMBD, + MODEL_TENSOR.HC_FFN_FN, + MODEL_TENSOR.HC_FFN_BASE, + MODEL_TENSOR.HC_FFN_SCALE, + MODEL_TENSOR.FFN_GATE_INP, + MODEL_TENSOR.FFN_NORM, + MODEL_TENSOR.FFN_GATE, + MODEL_TENSOR.FFN_DOWN, + MODEL_TENSOR.FFN_UP, + MODEL_TENSOR.FFN_GATE_EXP, + MODEL_TENSOR.FFN_DOWN_EXP, + MODEL_TENSOR.FFN_UP_EXP, + MODEL_TENSOR.FFN_GATE_SHEXP, + MODEL_TENSOR.FFN_DOWN_SHEXP, + MODEL_TENSOR.FFN_UP_SHEXP, + MODEL_TENSOR.FFN_EXP_PROBS_B, + # NextN/MTP tensors + MODEL_TENSOR.NEXTN_EH_PROJ, + MODEL_TENSOR.NEXTN_EMBED_TOKENS, + MODEL_TENSOR.NEXTN_ENORM, + MODEL_TENSOR.NEXTN_HNORM, + MODEL_TENSOR.NEXTN_SHARED_HEAD_HEAD, + MODEL_TENSOR.NEXTN_SHARED_HEAD_NORM, + ], MODEL_ARCH.DEEPSEEK2OCR: [ MODEL_TENSOR.TOKEN_EMBD, MODEL_TENSOR.OUTPUT_NORM, @@ -4858,6 +5062,61 @@ class MODEL_TENSOR(IntEnum): MODEL_TENSOR.FFN_DOWN, MODEL_TENSOR.FFN_UP, ], + MODEL_ARCH.METIS16: [ + MODEL_TENSOR.TOKEN_EMBD, + MODEL_TENSOR.OUTPUT_NORM, + MODEL_TENSOR.OUTPUT, + MODEL_TENSOR.ATTN_NORM, + MODEL_TENSOR.FFN_NORM, + MODEL_TENSOR.STREAM_EMBD, + MODEL_TENSOR.DEPTH_MEM_STATE_WRITE, + MODEL_TENSOR.DEPTH_MEM_META_WRITE, + MODEL_TENSOR.DEPTH_MEM_ENTRY_BIAS, + MODEL_TENSOR.DEPTH_MEM_QUERY, + MODEL_TENSOR.DEPTH_MEM_KEY, + MODEL_TENSOR.DEPTH_MEM_VALUE, + MODEL_TENSOR.DEPTH_MEM_OUT, + MODEL_TENSOR.DEPTH_MEM_ROUTE_STATE, + MODEL_TENSOR.DEPTH_MEM_ROUTE_RETR, + MODEL_TENSOR.DEPTH_MEM_ROUTE_DIFF, + MODEL_TENSOR.DEPTH_MEM_ROUTE_HIST, + MODEL_TENSOR.DEPTH_MEM_ROUTE, + MODEL_TENSOR.DEPTH_MEM_STREAM_GATE, + MODEL_TENSOR.CONT_HIDDEN_STATE, + MODEL_TENSOR.CONT_HIDDEN_RETR, + MODEL_TENSOR.CONT_HIDDEN_DIFF, + MODEL_TENSOR.CONT_HIDDEN_ROUTE, + MODEL_TENSOR.CONT_HIDDEN, + MODEL_TENSOR.CONT_OUTPUT, + MODEL_TENSOR.HC_ATTN_MIX, + MODEL_TENSOR.HC_ATTN_READ, + MODEL_TENSOR.HC_ATTN_WRITE, + MODEL_TENSOR.HC_FFN_MIX, + MODEL_TENSOR.HC_FFN_READ, + MODEL_TENSOR.HC_FFN_WRITE, + MODEL_TENSOR.ATTN_PASS_LORA_DOWN, + MODEL_TENSOR.ATTN_PASS_LORA_UP, + MODEL_TENSOR.ATTN_PASS_GATE, + MODEL_TENSOR.FFN_EXP_EMBD, + MODEL_TENSOR.SSM_IN, + MODEL_TENSOR.SSM_CONV1D, + MODEL_TENSOR.SSM_DT, + MODEL_TENSOR.SSM_A, + MODEL_TENSOR.SSM_D, + MODEL_TENSOR.SSM_NORM, + MODEL_TENSOR.SSM_OUT, + MODEL_TENSOR.ATTN_QKV, + MODEL_TENSOR.ATTN_OUT, + MODEL_TENSOR.MOE_LATENT_DOWN, + MODEL_TENSOR.MOE_LATENT_UP, + MODEL_TENSOR.FFN_GATE_INP, + MODEL_TENSOR.FFN_EXP_PROBS_B, + MODEL_TENSOR.FFN_GATE_UP_EXP, + MODEL_TENSOR.FFN_DOWN_EXP, + MODEL_TENSOR.FFN_GATE_SHEXP, + MODEL_TENSOR.FFN_UP_SHEXP, + MODEL_TENSOR.FFN_DOWN_SHEXP, + ], MODEL_ARCH.NEMOTRON_H: [ MODEL_TENSOR.TOKEN_EMBD, MODEL_TENSOR.OUTPUT_NORM, @@ -5334,6 +5593,26 @@ class MODEL_TENSOR(IntEnum): MODEL_TENSOR.FFN_POST_NORM, MODEL_TENSOR.FFN_EXP_PROBS_B, ], + MODEL_ARCH.RICURSIVE: [ + MODEL_TENSOR.TOKEN_EMBD, + MODEL_TENSOR.OUTPUT_NORM, + MODEL_TENSOR.OUTPUT, + MODEL_TENSOR.ATTN_NORM, + MODEL_TENSOR.ATTN_Q, + MODEL_TENSOR.ATTN_K, + MODEL_TENSOR.ATTN_V, + MODEL_TENSOR.ATTN_OUT, + MODEL_TENSOR.ATTN_Q_NORM, + MODEL_TENSOR.ATTN_K_NORM, + MODEL_TENSOR.ATTN_GATE, + MODEL_TENSOR.ATTN_VRES_LAMBDA, + MODEL_TENSOR.ATTN_POST_NORM, + MODEL_TENSOR.FFN_PRE_NORM, + MODEL_TENSOR.FFN_GATE, + MODEL_TENSOR.FFN_DOWN, + MODEL_TENSOR.FFN_UP, + MODEL_TENSOR.FFN_POST_NORM, + ], MODEL_ARCH.LAGUNA: [ MODEL_TENSOR.TOKEN_EMBD, MODEL_TENSOR.OUTPUT_NORM, @@ -6264,6 +6543,60 @@ class MODEL_TENSOR(IntEnum): MODEL_TENSOR.FFN_ROUTED_UP, MODEL_TENSOR.FFN_ROUTED_NORM, ], + MODEL_ARCH.KIMI_K31: [ + MODEL_TENSOR.TOKEN_EMBD, + MODEL_TENSOR.OUTPUT_NORM, + MODEL_TENSOR.OUTPUT, + MODEL_TENSOR.OUTPUT_RES_SCORE, + MODEL_TENSOR.ATTN_NORM, + MODEL_TENSOR.ATTN_RES_SCORE, + MODEL_TENSOR.FFN_RES_SCORE, + # MLA + DSA indexer + MODEL_TENSOR.ATTN_Q, + MODEL_TENSOR.ATTN_K, + MODEL_TENSOR.ATTN_V, + MODEL_TENSOR.ATTN_OUT, + MODEL_TENSOR.ATTN_GATE, + MODEL_TENSOR.ATTN_Q_A, + MODEL_TENSOR.ATTN_Q_B, + MODEL_TENSOR.ATTN_KV_A_MQA, + MODEL_TENSOR.ATTN_KV_B, + MODEL_TENSOR.ATTN_K_B, + MODEL_TENSOR.ATTN_V_B, + MODEL_TENSOR.ATTN_Q_A_NORM, + MODEL_TENSOR.ATTN_KV_A_NORM, + MODEL_TENSOR.INDEXER_ATTN_Q_B, + MODEL_TENSOR.INDEXER_ATTN_K, + MODEL_TENSOR.INDEXER_K_NORM, + MODEL_TENSOR.INDEXER_PROJ, + # KDA (linear-attention layers) + MODEL_TENSOR.SSM_CONV1D_Q, + MODEL_TENSOR.SSM_CONV1D_K, + MODEL_TENSOR.SSM_CONV1D_V, + MODEL_TENSOR.SSM_F_A, + MODEL_TENSOR.SSM_F_B, + MODEL_TENSOR.SSM_BETA, + MODEL_TENSOR.SSM_A, + MODEL_TENSOR.SSM_G, + MODEL_TENSOR.SSM_DT, + MODEL_TENSOR.SSM_NORM, + # FFN + MODEL_TENSOR.FFN_NORM, + MODEL_TENSOR.FFN_GATE, + MODEL_TENSOR.FFN_DOWN, + MODEL_TENSOR.FFN_UP, + MODEL_TENSOR.FFN_GATE_INP, + MODEL_TENSOR.FFN_EXP_PROBS_B, + MODEL_TENSOR.FFN_GATE_EXP, + MODEL_TENSOR.FFN_DOWN_EXP, + MODEL_TENSOR.FFN_UP_EXP, + MODEL_TENSOR.FFN_GATE_SHEXP, + MODEL_TENSOR.FFN_DOWN_SHEXP, + MODEL_TENSOR.FFN_UP_SHEXP, + MODEL_TENSOR.FFN_ROUTED_DOWN, + MODEL_TENSOR.FFN_ROUTED_UP, + MODEL_TENSOR.FFN_ROUTED_NORM, + ], MODEL_ARCH.TALKIE: [ MODEL_TENSOR.TOKEN_EMBD, MODEL_TENSOR.OUTPUT, @@ -6468,6 +6801,10 @@ class MODEL_TENSOR(IntEnum): MODEL_TENSOR.ROPE_FREQS, MODEL_TENSOR.ATTN_ROT_EMBD, ], + MODEL_ARCH.XING4_0: [ + MODEL_TENSOR.ROPE_FREQS, + MODEL_TENSOR.ATTN_ROT_EMBD, + ], MODEL_ARCH.CHATGLM: [ MODEL_TENSOR.ROPE_FREQS, ], @@ -6709,6 +7046,7 @@ class VisionProjectorType: KIMIVL = "kimivl" PADDLEOCR = "paddleocr" KIMIK25 = "kimik25" + KIMIK31 = "kimik31" LIGHTONOCR = "lightonocr" COGVLM = "cogvlm" JANUS_PRO = "janus_pro" @@ -6732,7 +7070,10 @@ class VisionProjectorType: QWEN3TTS_GEN = "qwen3tts_gen" # audio generation: code_predictor POCKETTTS_SPKENC = "pockettts_spkenc" # audio: mimi encoder as voice-prompt encoder POCKETTTS_GEN = "pockettts_gen" # audio generation: flow-matching decoder + mimi decoder + CHATTERBOX_SPKENC = "chatterbox_spkenc" # audio: CAMPPlus speaker encoder + CHATTERBOX = "chatterbox" # audio generation: s3gen flow matching + HiFT vocoder SOPRANO = "soprano" # audio generation: Vocos decoder + DRAMA3_GEN = "drama3_gen" # audio generation: codebook tower + Fish DAC decoder HUNYUANVL = "hunyuanvl" PARAKEET = "parakeet" # audio MINIMAXM3 = "minimax_m3" @@ -6740,7 +7081,9 @@ class VisionProjectorType: MINICPMV4_7 = "minicpmv4_7" MINICPM_OCR_1_0 = "minicpm_ocr_1_0" GRANITE_SPEECH = "granite_speech" # audio + NEMOTRON_TRANSCRIBE = "nemotron_transcribe" # audio MIMOVL = "mimovl" + MAGEVL = "magevl" MIMO_AUDIO = "mimo_audio" GRANITE4_VISION = "granite4_vision" MUSE_GLIMMER = "muse-glimmer" diff --git a/gguf-py/gguf/gguf_writer.py b/gguf-py/gguf/gguf_writer.py index a0800e24c0a6..364eeaa75b24 100644 --- a/gguf-py/gguf/gguf_writer.py +++ b/gguf-py/gguf/gguf_writer.py @@ -11,7 +11,7 @@ from math import prod from pathlib import Path from io import BufferedWriter -from typing import IO, Any, Sequence, Mapping +from typing import IO, Any, Callable, Sequence, Mapping from string import ascii_letters, digits import numpy as np @@ -86,7 +86,8 @@ class GGUFWriter: def __init__( self, path: os.PathLike[str] | str | None, arch: str, use_temp_file: bool = False, endianess: GGUFEndian = GGUFEndian.LITTLE, - split_max_tensors: int = 0, split_max_size: int = 0, dry_run: bool = False, small_first_shard: bool = False + split_max_tensors: int = 0, split_max_size: int = 0, dry_run: bool = False, small_first_shard: bool = False, + tensor_filter: Callable[[str], bool] | None = None, ): self.fout = None self.path = Path(path) if path else None @@ -101,6 +102,7 @@ def __init__( self.split_max_size = split_max_size self.dry_run = dry_run self.small_first_shard = small_first_shard + self.tensor_filter = tensor_filter logger.info("gguf: This GGUF file is for {0} Endian only".format( "Big" if self.endianess == GGUFEndian.BIG else "Little", )) @@ -381,6 +383,9 @@ def add_tensor( self, name: str, tensor: np.ndarray[Any, Any], raw_shape: Sequence[int] | None = None, raw_dtype: GGMLQuantizationType | None = None, tensor_endianess: GGUFEndian | None = None ) -> None: + if self.tensor_filter is not None and not self.tensor_filter(name): + return + # if tensor endianness is not passed, assume it's native to system if tensor_endianess is None: tensor_endianess = GGUFEndian.BIG if sys.byteorder == 'big' else GGUFEndian.LITTLE @@ -912,6 +917,9 @@ def add_expert_gating_func(self, value: ExpertGatingFuncType) -> None: def add_swiglu_clamp_exp(self, values: Sequence[float]) -> None: self.add_array(Keys.LLM.SWIGLU_CLAMP_EXP.format(arch=self.arch), values) + def add_swiglu_clamp(self, values: Sequence[float]) -> None: + self.add_array(Keys.LLM.SWIGLU_CLAMP.format(arch=self.arch), values) + def add_swiglu_clamp_shexp(self, values: Sequence[float]) -> None: self.add_array(Keys.LLM.SWIGLU_CLAMP_SHEXP.format(arch=self.arch), values) @@ -975,6 +983,18 @@ def add_hrm_l_cycles(self, value: int) -> None: def add_hrm_prefix_lm(self, value: bool) -> None: self.add_bool(Keys.HRM.PREFIX_LM.format(arch=self.arch), value) + def add_metis_pass_count(self, value: int) -> None: + self.add_uint32(Keys.Metis.PASS_COUNT.format(arch=self.arch), value) + + def add_metis_memory_length(self, value: int) -> None: + self.add_uint32(Keys.Metis.MEMORY_LENGTH.format(arch=self.arch), value) + + def add_metis_route_length(self, value: int) -> None: + self.add_uint32(Keys.Metis.ROUTE_LENGTH.format(arch=self.arch), value) + + def add_metis_continuation_logit_limit(self, value: float) -> None: + self.add_float32(Keys.Metis.CONTINUATION_LOGIT_LIMIT.format(arch=self.arch), value) + def add_adapter_count(self, count: int) -> None: self.add_uint32(Keys.Adapters.COUNT.format(arch=self.arch), count) @@ -1086,6 +1106,9 @@ def add_attention_output_group_count(self, count: int) -> None: def add_attention_output_lora_rank(self, length: int) -> None: self.add_uint32(Keys.Attention.OUTPUT_LORA_RANK.format(arch=self.arch), length) + def add_attention_pass_lora_rank(self, length: int) -> None: + self.add_uint32(Keys.Attention.PASS_LORA_RANK.format(arch=self.arch), length) + def add_attention_compress_ratios(self, values: Sequence[int]) -> None: self.add_array(Keys.Attention.COMPRESS_RATIOS.format(arch=self.arch), values) @@ -1184,6 +1207,9 @@ def add_attn_temperature_scale(self, value: float) -> None: def add_realtime_delay_tokens(self, value: int) -> None: self.add_uint32(Keys.LLM.REALTIME_DELAY_TOKENS.format(arch=self.arch), value) + def add_classifier_free_guidance(self, value: bool) -> None: + self.add_bool(Keys.LLM.CLASSIFIER_FREE_GUIDANCE.format(arch=self.arch), value) + def add_pooling_type(self, value: PoolingType) -> None: self.add_uint32(Keys.LLM.POOLING_TYPE.format(arch=self.arch), value.value) @@ -1267,6 +1293,9 @@ def add_ssm_group_count(self, value: int) -> None: def add_ssm_dt_b_c_rms(self, value: bool) -> None: self.add_bool(Keys.SSM.DT_B_C_RMS.format(arch=self.arch), value) + def add_ssm_norm_epsilon(self, value: float) -> None: + self.add_float32(Keys.SSM.NORM_EPSILON.format(arch=self.arch), value) + def add_expert_latent_length(self, value: int) -> None: self.add_uint32(Keys.LLM.EXPERT_LATENT_LENGTH.format(arch=self.arch), value) @@ -1712,6 +1741,15 @@ def add_audio_projector_downsample_rate(self, value: int) -> None: def add_audio_projector_head_count(self, value: int) -> None: self.add_uint32(Keys.ClipAudio.Projector.HEAD_COUNT, value) + def add_audio_rope_dims(self, value: int) -> None: + self.add_uint32(Keys.ClipAudio.ROPE_DIMS, value) + + def add_audio_spk_block_count(self, value: int) -> None: + self.add_uint32(Keys.ClipAudio.Speaker.BLOCK_COUNT, value) + + def add_audio_spk_head_count(self, value: int) -> None: + self.add_uint32(Keys.ClipAudio.Speaker.HEAD_COUNT, value) + # audio generation (mmproj) def add_clip_gen_audio_projector_type(self, value: str) -> None: diff --git a/gguf-py/gguf/lazy.py b/gguf-py/gguf/lazy.py index a39f22321597..eae6ab5c49ec 100644 --- a/gguf-py/gguf/lazy.py +++ b/gguf-py/gguf/lazy.py @@ -138,7 +138,7 @@ def wrapped_fn(*args, **kwargs): if isinstance(meta_noop, tuple): dtype, shape = meta_noop assert callable(shape) - res = cls.meta_with_dtype_and_shape(dtype, shape(res.shape)) # ty: ignore[call-top-callable] + res = cls.meta_with_dtype_and_shape(dtype, shape(res.shape)) else: res = cls.meta_with_dtype_and_shape(meta_noop, res.shape) diff --git a/gguf-py/gguf/tensor_mapping.py b/gguf-py/gguf/tensor_mapping.py index d06dc4ec5fdc..bb7d6dcaeec4 100644 --- a/gguf-py/gguf/tensor_mapping.py +++ b/gguf-py/gguf/tensor_mapping.py @@ -35,6 +35,7 @@ class TensorNameMap: "model.transformer.wte", # llada "embed_tokens", # qwen3-embedding "model.embed", # talkie + "embedding", # metis16 ), # Masked embeddings @@ -224,6 +225,7 @@ class TensorNameMap: block_mappings_cfg: dict[MODEL_TENSOR, tuple[str, ...]] = { # Attention norm MODEL_TENSOR.ATTN_NORM: ( + "layers.{bid}.mixer_connection.norm", # metis16 "model.lilicorr.layers.{bid}.attn_norm", # lilicorr "gpt_neox.layers.{bid}.input_layernorm", # gptneox "transformer.h.{bid}.ln_1", # gpt2 gpt-j refact qwen jais exaone @@ -294,6 +296,7 @@ class TensorNameMap: "model.layers.{bid}.linear_attn.in_proj_qkv", # qwen3.5 "head.layers.{bid}.self_attn.in_proj", # laya "layers.{bid}.linear_attn.in_proj_qkv", # qwen3.5 text + "layers.{bid}.mixer.qkv", # metis16 ), # Attention query @@ -401,6 +404,7 @@ class TensorNameMap: "model.layers.{bid}.self_attn.language_expert_dense", # cogvlm "model.blocks.{bid}.attn.attn_resid", # talkie "head.layers.{bid}.self_attn.out_proj", # laya + "layers.{bid}.mixer.out", # metis16 ), # Attention output norm @@ -447,8 +451,13 @@ class TensorNameMap: "model.layers.{bid}.self_attn.attn_gate_proj", # k2-horizon ), + MODEL_TENSOR.ATTN_VRES_LAMBDA: ( + "model.layers.{bid}.self_attn.value_res_lambda", # ricursive + ), + # Feed-forward norm MODEL_TENSOR.FFN_NORM: ( + "layers.{bid}.moe_connection.norm", # metis16 "model.lilicorr.layers.{bid}.mlp_norm", # lilicorr "model.lilicorr.feature_mlp.{bid}.norm", # lilicorr "gpt_neox.layers.{bid}.post_attention_layernorm", # gptneox @@ -536,6 +545,7 @@ class TensorNameMap: "backbone.layers.{bid}.mixer.gate", # nemotron-h-moe "model.layers.{bid}.moe.gate", # step3.5 "model.layers.{bid}.router.proj", # gemma4 + "layers.{bid}.moe.expert_router", # metis16 ), MODEL_TENSOR.FFN_GATE_INP_SHEXP: ( @@ -543,6 +553,7 @@ class TensorNameMap: ), MODEL_TENSOR.FFN_EXP_PROBS_B: ( + "layers.{bid}.moe.selection_bias", # metis16 "model.layers.{bid}.mlp.gate.e_score_correction", # deepseek-v3 dots1 "model.layers.{bid}.mlp.gate.e_score_correction_bias", # muse-spark "model.layers.{bid}.mlp.moe_statics.e_score_correction", # ernie4.5-moe @@ -617,6 +628,7 @@ class TensorNameMap: ), MODEL_TENSOR.FFN_UP_SHEXP: ( + "layers.{bid}.moe.shared_expert.up", # metis16 "model.layers.{bid}.mlp.shared_expert.up_proj", # qwen2moe "model.layers.{bid}.mlp.shared_experts.up_proj", # deepseek deepseek2 "model.layers.{bid}.feed_forward.shared_expert.up_proj", # llama4 @@ -670,6 +682,7 @@ class TensorNameMap: ), MODEL_TENSOR.FFN_GATE_SHEXP: ( + "layers.{bid}.moe.shared_expert.gate", # metis16 "model.layers.{bid}.mlp.shared_expert.gate_proj", # qwen2moe "model.layers.{bid}.mlp.shared_experts.gate_proj", # deepseek deepseek2 "model.layers.{bid}.feed_forward.shared_expert.gate_proj", # llama4 @@ -686,18 +699,94 @@ class TensorNameMap: MODEL_TENSOR.FFN_GATE_UP_EXP: ( "model.layers.{bid}.mlp.experts.gate_up_proj", "model.layers.{bid}.experts.gate_up_proj", # gemma4 + "layers.{bid}.moe.local_experts.gate_up", # metis16 ), MODEL_TENSOR.MOE_LATENT_DOWN: ( "backbone.layers.{bid}.mixer.fc1_latent_proj", # nemotron 3 super "model.layers.{bid}.mlp.pre_expert_proj", # muse-spark + "layers.{bid}.moe.latent_down", # metis16 ), MODEL_TENSOR.MOE_LATENT_UP: ( "backbone.layers.{bid}.mixer.fc2_latent_proj", # nemotron 3 super "model.layers.{bid}.mlp.post_expert_proj", # muse-spark + "layers.{bid}.moe.latent_up", # metis16 + ), + + MODEL_TENSOR.STREAM_EMBD: ( + "stream_embeddings", # metis16 + ), + MODEL_TENSOR.DEPTH_MEM_STATE_WRITE: ( + "depth_memory.state_write", # metis16 + ), + MODEL_TENSOR.DEPTH_MEM_META_WRITE: ( + "depth_memory.metadata_write", # metis16 + ), + MODEL_TENSOR.DEPTH_MEM_QUERY: ( + "depth_memory.query", # metis16 + ), + MODEL_TENSOR.DEPTH_MEM_KEY: ( + "depth_memory.key", # metis16 + ), + MODEL_TENSOR.DEPTH_MEM_VALUE: ( + "depth_memory.value", # metis16 + ), + MODEL_TENSOR.DEPTH_MEM_OUT: ( + "depth_memory.output", # metis16 + ), + MODEL_TENSOR.DEPTH_MEM_ROUTE_STATE: ( + "depth_memory.route_state", # metis16 + ), + MODEL_TENSOR.DEPTH_MEM_ROUTE_RETR: ( + "depth_memory.route_retr", # metis16 + ), + MODEL_TENSOR.DEPTH_MEM_ROUTE_DIFF: ( + "depth_memory.route_diff", # metis16 + ), + MODEL_TENSOR.DEPTH_MEM_ROUTE_HIST: ( + "depth_memory.route_hist", # metis16 + ), + MODEL_TENSOR.DEPTH_MEM_ROUTE: ( + "depth_memory.route_projection", # metis16 + ), + MODEL_TENSOR.DEPTH_MEM_STREAM_GATE: ( + "depth_memory.stream_gate", # metis16 + ), + MODEL_TENSOR.CONT_HIDDEN_STATE: ( + "continuation.hidden_state", # metis16 + ), + MODEL_TENSOR.CONT_HIDDEN_RETR: ( + "continuation.hidden_retr", # metis16 + ), + MODEL_TENSOR.CONT_HIDDEN_DIFF: ( + "continuation.hidden_diff", # metis16 + ), + MODEL_TENSOR.CONT_HIDDEN_ROUTE: ( + "continuation.hidden_route", # metis16 + ), + MODEL_TENSOR.CONT_HIDDEN: ( + "continuation.hidden", # metis16 + ), + MODEL_TENSOR.CONT_OUTPUT: ( + "continuation.output", # metis16 + ), + MODEL_TENSOR.ATTN_PASS_LORA_DOWN: ( + "layers.{bid}.mixer.pass_lora_down", # metis16 + ), + MODEL_TENSOR.ATTN_PASS_LORA_UP: ( + "layers.{bid}.mixer.pass_lora_up", # metis16 + ), + MODEL_TENSOR.ATTN_PASS_GATE: ( + "layers.{bid}.mixer.pass_gates", # metis16 + ), + MODEL_TENSOR.FFN_EXP_EMBD: ( + "layers.{bid}.moe.expert_embeddings", # metis16 ), + + + MODEL_TENSOR.FFN_PRE_EXP_NORM: ( "model.layers.{bid}.mlp.pre_expert_norm", # muse-spark ), @@ -759,6 +848,7 @@ class TensorNameMap: "model.layers.{bid}.block_sparse_moe.experts.down", # smallthinker "model.layers.{bid}.moe.down_proj", # step3.5 "model.layers.{bid}.experts.down_proj", # gemma4 + "layers.{bid}.moe.local_experts.down", # metis16 ), MODEL_TENSOR.FFN_DOWN_SHEXP: ( @@ -771,6 +861,7 @@ class TensorNameMap: "backbone.layers.{bid}.mixer.shared_experts.down_proj", # nemotron-h-moe "model.layers.{bid}.block_sparse_moe.shared_experts.down_proj", # kimi "model.layers.{bid}.share_expert.down_proj", # step3.5 + "layers.{bid}.moe.shared_expert.down", # metis16 ), MODEL_TENSOR.FFN_DOWN_CHEXP: ( @@ -906,6 +997,7 @@ class TensorNameMap: "model.layers.{bid}.mamba.in_proj", # jamba falcon-h1 granite-hybrid "model.layers.layers.{bid}.mixer.in_proj", # plamo2 "model.layers.{bid}.linear_attn.in_proj_qkvz", # qwen3next + "layers.{bid}.mixer.impl.mixer.in_proj", # metis16 ), MODEL_TENSOR.SSM_CONV1D: ( @@ -915,6 +1007,7 @@ class TensorNameMap: "model.layers.layers.{bid}.mixer.conv1d", # plamo2 "model.layers.{bid}.linear_attn.conv1d", # qwen3next "layers.{bid}.linear_attn.conv1d", # qwen3.5 text + "layers.{bid}.mixer.impl.mixer.conv1d", # metis16 ), MODEL_TENSOR.SSM_X: ( @@ -925,6 +1018,7 @@ class TensorNameMap: ), MODEL_TENSOR.SSM_DT: ( + "layers.{bid}.mixer.impl.mixer.dt_bias", # metis16 "model.layers.{bid}.dt_proj", # mamba-hf "backbone.layers.{bid}.mixer.dt_proj", # mamba "model.layers.{bid}.mamba.dt_proj", # jamba falcon-h1 granite-hybrid @@ -942,6 +1036,7 @@ class TensorNameMap: ), MODEL_TENSOR.SSM_A: ( + "layers.{bid}.mixer.impl.mixer.A_log", # metis16 "model.layers.{bid}.A_log", # mamba-hf "backbone.layers.{bid}.mixer.A_log", # mamba "model.layers.{bid}.mamba.A_log", # jamba falcon-h1 granite-hybrid @@ -965,6 +1060,7 @@ class TensorNameMap: ), MODEL_TENSOR.SSM_D: ( + "layers.{bid}.mixer.impl.mixer.D", # metis16 "model.layers.{bid}.D", # mamba-hf "backbone.layers.{bid}.mixer.D", # mamba "model.layers.{bid}.mamba.D", # jamba falcon-h1 granite-hybrid @@ -972,6 +1068,7 @@ class TensorNameMap: ), MODEL_TENSOR.SSM_NORM: ( + "layers.{bid}.mixer.impl.mixer.norm", # metis16 "model.layers.{bid}.mamba.norm", # falcon-h1 granite-hybrid "model.layers.{bid}.linear_attn.norm", # qwen3next "layers.{bid}.linear_attn.norm", # qwen3.5 text @@ -987,6 +1084,7 @@ class TensorNameMap: "model.layers.{bid}.linear_attn.out_proj", # qwen3next "layers.{bid}.linear_attn.out_proj", # qwen3.5 text "model.layers.layers.{bid}.mixer.out_proj", # plamo2 + "layers.{bid}.mixer.impl.mixer.out_proj", # metis16 ), MODEL_TENSOR.SSM_ALPHA: ( @@ -1765,7 +1863,12 @@ class TensorNameMap: "model.vision_model.embeddings.class_embedding", # Deepseek-OCR ), + MODEL_TENSOR.V_ENC_ROPE_FREQS: ( + "visual.rope_freqs", + ), + MODEL_TENSOR.V_ENC_EMBD_PATCH: ( + "visual.embeddings.patch_embedding", # mage-vl "vision.patch_embed.proj", # deepseek4v "model.vision_tower.vision_model.embeddings.patch_embedding", # Granite4Vision "vision_tower.vision_model.embeddings.patch_embedding", @@ -1835,6 +1938,7 @@ class TensorNameMap: ), MODEL_TENSOR.V_ENC_ATTN_QKV: ( + "visual.encoder.layers.{bid}.self_attn.qkv", # mage-vl "vision.blocks.{bid}.attn.wqkv", # deepseek4v "visual.blocks.{bid}.attn.qkv", # qwen3vl "vision_tower.blocks.{bid}.attn.qkv", # dots.ocr @@ -1931,6 +2035,7 @@ class TensorNameMap: ), MODEL_TENSOR.V_ENC_INPUT_NORM: ( + "visual.encoder.layers.{bid}.layer_norm1", # mage-vl "vision_model.encoder.layer.{bid}.norm1", # Nemotron Omni "vision.blocks.{bid}.norm1", # deepseek4v "model.vision_tower.vision_model.encoder.layers.{bid}.layer_norm1", # Granite4Vision @@ -1959,6 +2064,7 @@ class TensorNameMap: ), MODEL_TENSOR.V_ENC_ATTN_O: ( + "visual.encoder.layers.{bid}.self_attn.proj", # mage-vl "vision_model.encoder.layer.{bid}.attention.output.dense", # Nemotron Omni "vision.blocks.{bid}.attn.wo", # deepseek4v "model.vision_tower.vision_model.encoder.layers.{bid}.self_attn.out_proj", # Granite4Vision @@ -1993,6 +2099,7 @@ class TensorNameMap: ), MODEL_TENSOR.V_ENC_POST_ATTN_NORM: ( + "visual.encoder.layers.{bid}.layer_norm2", # mage-vl "vision_model.encoder.layer.{bid}.norm2", # Nemotron Omni "vision.blocks.{bid}.norm2", # deepseek4v "model.vision_tower.vision_model.encoder.layers.{bid}.layer_norm2", # Granite4Vision @@ -2022,6 +2129,7 @@ class TensorNameMap: ), MODEL_TENSOR.V_ENC_FFN_UP: ( + "visual.encoder.layers.{bid}.mlp.fc1", # mage-vl "vision_model.encoder.layer.{bid}.mlp.fc1", # Nemotron Omni "vision.blocks.{bid}.mlp.w1_up", # deepseek4v (split from fused w1) "vision_encoder.blocks.{bid}.mlp.fc3", # dots3note @@ -2062,6 +2170,7 @@ class TensorNameMap: ), MODEL_TENSOR.V_ENC_FFN_DOWN: ( + "visual.encoder.layers.{bid}.mlp.fc2", # mage-vl "vision_model.encoder.layer.{bid}.mlp.fc2", # Nemotron Omni "vision.blocks.{bid}.mlp.w2", # deepseek4v "vision_encoder.blocks.{bid}.mlp.fc2", # dots3note @@ -2140,6 +2249,7 @@ class TensorNameMap: ), MODEL_TENSOR.V_PRE_NORM: ( + "visual.layernorm_pre", # mage-vl "vision_tower.vision_model.pre_layrnorm", "vision_tower.ln_pre", # pixtral-hf "vision_encoder.ln_pre", # pixtral @@ -2172,6 +2282,7 @@ class TensorNameMap: "vision_tower.post_trunk_norm", # dots.ocr "vision_encoder.post_trunk_norm", # dots3note "vit.perceive.after_rms", # HunyuanVL + "mm_projector.post_norm", # Kimi-K3.1 ), MODEL_TENSOR.V_MM_INP_PROJ: ( @@ -2532,6 +2643,7 @@ class TensorNameMap: MODEL_TENSOR.A_ENC_INP_PROJ: ( "conformer.subsample_conv_projection.input_proj_linear", # gemma4 + "perception.encoder.asr_encoder.pre_encode.proj", # nemotron-transcribe (feature stacking) "encoder.input_linear", ), @@ -2546,13 +2658,16 @@ class TensorNameMap: "speaker_encoder.mfa.conv", # qwen3tts speaker encoder: multi-layer feature aggregation ), - MODEL_TENSOR.A_PRE_NORM: (), + MODEL_TENSOR.A_PRE_NORM: ( + "perception.encoder.asr_encoder.embed_norm", # nemotron-transcribe + ), MODEL_TENSOR.A_POST_NORM: ( "audio_tower.layer_norm", # ultravox "audio_tower.norm", # voxtral realtime "audio_encoder.dots_encoder.speech_encoder.layer_norm", # dots3note "audio_tower.ln_post", # qwen2omni + "perception.encoder.asr_encoder.final_norm", # nemotron-transcribe "encoder.layer_norm", # mimo-audio-tokenizer "encoder.lfm.lfm.embedding_norm", # lfm3a ), @@ -2595,10 +2710,12 @@ class TensorNameMap: MODEL_TENSOR.A_ENC_ATTN_Q_NORM: ( "encoder.lfm.lfm.layers.{bid}.self_attn.q_layernorm", # lfm3a + "perception.encoder.asr_encoder.layers.{bid}.attn.q_norm", # nemotron-transcribe ), MODEL_TENSOR.A_ENC_ATTN_K_NORM: ( "encoder.lfm.lfm.layers.{bid}.self_attn.k_layernorm", # lfm3a + "perception.encoder.asr_encoder.layers.{bid}.attn.k_norm", # nemotron-transcribe ), MODEL_TENSOR.A_ENC_ATTN_K_REL: ( @@ -2629,6 +2746,7 @@ class TensorNameMap: "encoder.lfm.lfm.layers.{bid}.operator_norm", # lfm3a "conformer.layers.{bid}.attention.pre_attn_norm", # gemma3n "sound_encoder.encoder.layers.{bid}.norm_self_att", # parakeet + "perception.encoder.asr_encoder.layers.{bid}.norm1", # nemotron-transcribe "encoder.layers.{bid}.attn.pre_norm", # granite_speech "encoder.layers.{bid}.self_attn_layer_norm", # mimo-audio-tokenizer ), @@ -2642,6 +2760,7 @@ class TensorNameMap: "conformer.layers.{bid}.attention.post", # gemma3n "conformer.layers.{bid}.self_attn.post", # gemma4 "sound_encoder.encoder.layers.{bid}.self_attn.o_proj", # parakeet + "perception.encoder.asr_encoder.layers.{bid}.attn.out_proj", # nemotron-transcribe "encoder.layers.{bid}.attn.to_out", # granite_speech "encoder.layers.{bid}.self_attn.out_proj", # mimo-audio-tokenizer ), @@ -2653,6 +2772,7 @@ class TensorNameMap: "encoder.lfm.lfm.layers.{bid}.ffn_norm", # lfm3a "conformer.layers.{bid}.attention.post_norm", # gemma3n "sound_encoder.encoder.layers.{bid}.norm_out", # parakeet + "perception.encoder.asr_encoder.layers.{bid}.norm2", # nemotron-transcribe "encoder.layers.{bid}.post_norm", # granite_speech "encoder.layers.{bid}.final_layer_norm", # mimo-audio-tokenizer ), @@ -2683,6 +2803,7 @@ class TensorNameMap: "conformer.layers.{bid}.ffw_layer_start.ffw_layer_1", # gemma3n "conformer.layers.{bid}.feed_forward1.ffw_layer_1", # gemma4 "sound_encoder.encoder.layers.{bid}.feed_forward1.linear1", # parakeet + "perception.encoder.asr_encoder.layers.{bid}.ffn.net.0", # nemotron-transcribe "encoder.layers.{bid}.ff1.up_proj", # granite_speech "encoder.layers.{bid}.fc1", # mimo-audio-tokenizer ), @@ -2702,6 +2823,7 @@ class TensorNameMap: "conformer.layers.{bid}.ffw_layer_start.ffw_layer_2", # gemma3n "conformer.layers.{bid}.feed_forward1.ffw_layer_2", # gemma4 "sound_encoder.encoder.layers.{bid}.feed_forward1.linear2", # parakeet + "perception.encoder.asr_encoder.layers.{bid}.ffn.net.3", # nemotron-transcribe "encoder.layers.{bid}.ff1.down_proj", # granite_speech "encoder.layers.{bid}.fc2", # mimo-audio-tokenizer ), @@ -2792,6 +2914,7 @@ class TensorNameMap: "audio_tower.proj", # qwen2omni "model.audio_tower.output_proj", # gemma4 "speaker_encoder.fc", # qwen3tts speaker encoder: final speaker embedding projection + "perception.proj", # nemotron-transcribe ), MODEL_TENSOR.A_MM_NORM_PRE: ( @@ -2914,12 +3037,85 @@ class TensorNameMap: MODEL_TENSOR.A_ENC_MEL_FILTERS: ( "sound_encoder.encoder.feature_extractor.featurizer.fb", # parakeet + "perception.preprocessor.featurizer.fb", # nemotron-transcribe ), MODEL_TENSOR.A_ENC_WINDOW: ( "sound_encoder.encoder.feature_extractor.featurizer.window", # parakeet + "perception.preprocessor.featurizer.window", # nemotron-transcribe + ), + + MODEL_TENSOR.A_ENC_ATTN_QKV: ( + "perception.encoder.asr_encoder.layers.{bid}.attn.w_qkv", # nemotron-transcribe ), + MODEL_TENSOR.A_SPK_INP_PROJ: ( + "perception.encoder.diarization_model.encoder.pre_encode.proj", # nemotron-transcribe + ), + + MODEL_TENSOR.A_SPK_PRE_NORM: ( + "perception.encoder.diarization_model.encoder.embed_norm", # nemotron-transcribe + ), + + MODEL_TENSOR.A_SPK_POST_NORM: ( + "perception.encoder.diarization_model.encoder.final_norm", # nemotron-transcribe + ), + + MODEL_TENSOR.A_SPK_ATTN_NORM: ( + "perception.encoder.diarization_model.encoder.layers.{bid}.norm1", # nemotron-transcribe + ), + + MODEL_TENSOR.A_SPK_ATTN_QKV: ( + "perception.encoder.diarization_model.encoder.layers.{bid}.attn.w_qkv", # nemotron-transcribe + ), + + MODEL_TENSOR.A_SPK_ATTN_OUT: ( + "perception.encoder.diarization_model.encoder.layers.{bid}.attn.out_proj", # nemotron-transcribe + ), + + MODEL_TENSOR.A_SPK_FFN_NORM: ( + "perception.encoder.diarization_model.encoder.layers.{bid}.norm2", # nemotron-transcribe + ), + + MODEL_TENSOR.A_SPK_FFN_UP: ( + "perception.encoder.diarization_model.encoder.layers.{bid}.ffn.net.0", # nemotron-transcribe + ), + + MODEL_TENSOR.A_SPK_FFN_DOWN: ( + "perception.encoder.diarization_model.encoder.layers.{bid}.ffn.net.3", # nemotron-transcribe + ), + + MODEL_TENSOR.A_SPK_HEAD_PROJ: ( + "perception.encoder.diarization_model.sortformer_modules.encoder_proj", # nemotron-transcribe + ), + + MODEL_TENSOR.A_SPK_HEAD_UPSAMPLE: ( + "perception.encoder.diarization_model.sortformer_modules.subpixel_upsample", # nemotron-transcribe + ), + + MODEL_TENSOR.A_SPK_HEAD_FC1: ( + "perception.encoder.diarization_model.sortformer_modules.first_hidden_to_hidden", # nemotron-transcribe + ), + + MODEL_TENSOR.A_SPK_HEAD_FC2: ( + "perception.encoder.diarization_model.sortformer_modules.single_hidden_to_spks", # nemotron-transcribe + ), + + MODEL_TENSOR.A_SPK_SIL_EMB: ( + "perception.encoder.diarization_model.sortformer_modules.learnable_sil_emb", # nemotron-transcribe + ), + + MODEL_TENSOR.A_FUSION_ASR_NORM: ( + "perception.encoder.asr_norm", # nemotron-transcribe + ), + + MODEL_TENSOR.A_FUSION_SPK_NORM: ( + "perception.encoder.diar_norm", # nemotron-transcribe + ), + + # generated in conversion code (nemotron-transcribe) + MODEL_TENSOR.A_FUSION_SPK_KERNEL: (), + MODEL_TENSOR.A_MM_EMBEDDING: ( "model.embed_audio.embedding", # gemma3n ), diff --git a/gguf-py/gguf/vocab.py b/gguf-py/gguf/vocab.py index 3b82c3961496..9b8b87cf6be7 100644 --- a/gguf-py/gguf/vocab.py +++ b/gguf-py/gguf/vocab.py @@ -14,12 +14,12 @@ SentencePieceProcessor: Any = None try: - from mistral_common.tokens.tokenizers.mistral import MistralTokenizer # type: ignore[import-not-found, ty:unresolved-import] - from mistral_common.tokens.tokenizers.tekken import Tekkenizer # type: ignore[import-not-found, ty:unresolved-import] - from mistral_common.tokens.tokenizers.utils import ( # type: ignore[import-not-found, ty:unresolved-import] + from mistral_common.tokens.tokenizers.mistral import MistralTokenizer # ty: ignore[import-not-found] + from mistral_common.tokens.tokenizers.tekken import Tekkenizer # ty: ignore[import-not-found] + from mistral_common.tokens.tokenizers.utils import ( # ty: ignore[import-not-found] _filter_valid_tokenizer_files, ) - from mistral_common.tokens.tokenizers.sentencepiece import ( # type: ignore[import-not-found, ty:unresolved-import] + from mistral_common.tokens.tokenizers.sentencepiece import ( # ty: ignore[import-not-found] SentencePieceTokenizer, ) except ImportError: @@ -32,7 +32,7 @@ _mistral_common_installed = True try: - from mistral_common.tokens.tokenizers.utils import ( # type: ignore[import-not-found, ty:unresolved-import] + from mistral_common.tokens.tokenizers.utils import ( # ty: ignore[import-not-found] get_one_valid_tokenizer_file, ) except ImportError: @@ -221,10 +221,12 @@ def _try_load_from_tokenizer_json(self, path: Path) -> bool: if isinstance(merges, list) and merges: if isinstance(merges[0], str): self.merges = merges - elif isinstance(merges[0], list) and len(merges[0]) == 2 and isinstance(merges[0][0], str): + elif isinstance(merges[0], list): # New format since transformers 4.45 to support spaces in merges # ref: https://github.com/ggml-org/llama.cpp/issues/9692 # TODO: internally store as the new format instead of converting to old + if not all(isinstance(pair, list) and len(pair) == 2 and all(isinstance(s, str) for s in pair) for pair in merges): + raise ValueError("Unknown tokenizer merges format") if any(' ' in s for pair in merges for s in pair): logger.warning(f'Spaces in merges detected, encoding as {chr(ord(" ") + 256)!r}') self.merges = [ @@ -350,21 +352,20 @@ def _try_load_from_tokenizer_json(self, path: Path) -> bool: continue if not tokenizer_config: return True - chat_template_alt = None + chat_template = tokenizer_config.get('chat_template') chat_template_json = path / 'chat_template.json' chat_template_jinja = path / 'chat_template.jinja' if chat_template_jinja.is_file(): with open(chat_template_jinja, encoding = 'utf-8') as f: - chat_template_alt = f.read() + chat_template = f.read() if additional_templates := list((path / 'additional_chat_templates').glob('*.jinja')): - chat_template_alt = [{'name': 'default', 'template': chat_template_alt}] + chat_template = [{'name': 'default', 'template': chat_template}] for template_path in additional_templates: with open(template_path, encoding = 'utf-8') as fp: - chat_template_alt.append({'name': template_path.stem, 'template': fp.read()}) + chat_template.append({'name': template_path.stem, 'template': fp.read()}) elif chat_template_json.is_file(): with open(chat_template_json, encoding = 'utf-8') as f: - chat_template_alt = json.load(f).get('chat_template') - chat_template = tokenizer_config.get('chat_template', chat_template_alt) + chat_template = json.load(f).get('chat_template') if chat_template is None or isinstance(chat_template, (str, list)): self.chat_template = chat_template else: @@ -628,7 +629,8 @@ def __init__(self, base_path: Path): cache_dir=base_path, local_files_only=True, ) - assert self.tokenizer.is_fast # assume tokenizer.json is used # ty: ignore[unresolved-attribute] + assert self.tokenizer is not None + assert self.tokenizer.is_fast # assume tokenizer.json is used # Initialize lists and dictionaries for added tokens self.added_tokens_list = [] @@ -637,30 +639,31 @@ def __init__(self, base_path: Path): # Process added tokens for tok, tokidx in sorted( - self.tokenizer.get_added_vocab().items(), key=lambda x: x[1] # ty: ignore[unresolved-attribute] + self.tokenizer.get_added_vocab().items(), key=lambda x: x[1] ): # Only consider added tokens that are not in the base vocabulary - if tokidx >= self.tokenizer.vocab_size: # ty: ignore[unresolved-attribute] + if tokidx >= self.tokenizer.vocab_size: self.added_tokens_list.append(tok) self.added_tokens_dict[tok] = tokidx self.added_tokens_ids.add(tokidx) # Store special tokens and their IDs self.specials = { - tok: self.tokenizer.get_vocab()[tok] # ty: ignore[unresolved-attribute] - for tok in self.tokenizer.all_special_tokens # ty: ignore[unresolved-attribute] + tok: self.tokenizer.get_vocab()[tok] + for tok in self.tokenizer.all_special_tokens } - self.special_ids = set(self.tokenizer.all_special_ids) # ty: ignore[unresolved-attribute] + self.special_ids = set(self.tokenizer.all_special_ids) # Set vocabulary sizes - self.vocab_size_base = self.tokenizer.vocab_size # ty: ignore[unresolved-attribute] + self.vocab_size_base = self.tokenizer.vocab_size self.vocab_size = self.vocab_size_base + len(self.added_tokens_list) self.fname_tokenizer = fname_tokenizer def hf_tokens(self) -> Iterable[tuple[bytes, float, gguf.TokenType]]: + assert self.tokenizer is not None reverse_vocab = { - id: encoded_tok for encoded_tok, id in self.tokenizer.get_vocab().items() # ty: ignore[unresolved-attribute] + id: encoded_tok for encoded_tok, id in self.tokenizer.get_vocab().items() } for token_id in range(self.vocab_size_base): @@ -701,7 +704,8 @@ def added_tokens(self) -> Iterable[tuple[bytes, float, gguf.TokenType]]: yield text.encode("utf-8"), score, toktype def has_newline_token(self): - return "<0x0A>" in self.tokenizer.vocab or "\n" in self.tokenizer.vocab # ty: ignore[unresolved-attribute] + assert self.tokenizer is not None + return "<0x0A>" in self.tokenizer.vocab or "\n" in self.tokenizer.vocab def all_tokens(self) -> Iterable[tuple[bytes, float, gguf.TokenType]]: yield from self.hf_tokens() diff --git a/gguf-py/tests/test_gguf_writer.py b/gguf-py/tests/test_gguf_writer.py new file mode 100644 index 000000000000..606b2c5f9bca --- /dev/null +++ b/gguf-py/tests/test_gguf_writer.py @@ -0,0 +1,23 @@ +import numpy as np +import pytest + +from gguf import GGUFReader, GGUFWriter + + +@pytest.mark.parametrize("use_temp_file", [False, True]) +def test_tensor_filter_preserves_data_offsets(tmp_path, use_temp_file): + path = tmp_path / "filtered.gguf" + data = np.arange(32, dtype=np.float32).reshape(1, 32) + writer = GGUFWriter(path, "qwen4exp", use_temp_file=use_temp_file, + tensor_filter=lambda name: name != "excluded") + writer.add_tensor("first", data) + writer.add_tensor("excluded", data + 100) + writer.add_tensor("last", data + 200) + writer.write_header_to_file() + writer.write_kv_data_to_file() + writer.write_tensors_to_file() + writer.close() + tensors = GGUFReader(path).tensors + assert [t.name for t in tensors] == ["first", "last"] + np.testing.assert_array_equal(tensors[0].data, data) + np.testing.assert_array_equal(tensors[1].data, data + 200) diff --git a/include/llama.h b/include/llama.h index 3877b69e2d1e..181d840a6c88 100644 --- a/include/llama.h +++ b/include/llama.h @@ -270,7 +270,7 @@ extern "C" { llama_pos * pos; int32_t * n_seq_id; llama_seq_id ** seq_id; - int8_t * logits; // TODO: rename this to "output" + int8_t * logits; } llama_batch; enum llama_model_kv_override_type { @@ -389,7 +389,6 @@ extern "C" { float yarn_beta_fast; // YaRN low correction dim float yarn_beta_slow; // YaRN high correction dim uint32_t yarn_orig_ctx; // YaRN original context size - float defrag_thold; // [DEPRECATED] defragment the KV cache if holes/size > thold, <= 0 disabled (default) ggml_backend_sched_eval_callback cb_eval; void * cb_eval_user_data; @@ -397,7 +396,8 @@ extern "C" { enum ggml_type type_k; // data type for K cache [EXPERIMENTAL] enum ggml_type type_v; // data type for V cache [EXPERIMENTAL] - size_t moe_cache_size; // device cache in bytes for the experts kept in host memory, split among the devices like the layers, 0 = disabled [EXPERIMENTAL] + size_t moe_cache_size; // device cache in bytes for the experts kept in host memory, split among the devices like the layers, 0 = disabled [EXPERIMENTAL] + float moe_cache_cpu_share; // share of the MoE cache misses computed on the CPU instead of uploaded, 0 = all uploaded [EXPERIMENTAL] // Abort callback // if it returns true, execution of llama_decode() will be aborted @@ -515,11 +515,6 @@ extern "C" { void * set_tensor_data_ud, // userdata for function struct llama_model_params params); - DEPRECATED(LLAMA_API struct llama_model * llama_load_model_from_file( - const char * path_model, - struct llama_model_params params), - "use llama_model_load_from_file instead"); - // Load a model from a file // If the file is split into multiple parts, the file name must follow this pattern: -%05d-of-%05d.gguf // If the split file name does not follow this pattern, use llama_model_load_from_splits @@ -545,20 +540,12 @@ extern "C" { const struct llama_model * model, const char * path_model); - DEPRECATED(LLAMA_API void llama_free_model(struct llama_model * model), - "use llama_model_free instead"); - LLAMA_API void llama_model_free(struct llama_model * model); LLAMA_API struct llama_context * llama_init_from_model( struct llama_model * model, struct llama_context_params params); - DEPRECATED(LLAMA_API struct llama_context * llama_new_context_with_model( - struct llama_model * model, - struct llama_context_params params), - "use llama_init_from_model instead"); - // Frees all allocated memory LLAMA_API void llama_free(struct llama_context * ctx); @@ -583,13 +570,6 @@ extern "C" { LLAMA_API uint32_t llama_n_seq_max (const struct llama_context * ctx); LLAMA_API uint32_t llama_n_rs_seq (const struct llama_context * ctx); - DEPRECATED(LLAMA_API int32_t llama_n_ctx_train(const struct llama_model * model), "use llama_model_n_ctx_train instead"); - DEPRECATED(LLAMA_API int32_t llama_n_embd (const struct llama_model * model), "use llama_model_n_embd instead"); - DEPRECATED(LLAMA_API int32_t llama_n_layer (const struct llama_model * model), "use llama_model_n_layer instead"); - DEPRECATED(LLAMA_API int32_t llama_n_head (const struct llama_model * model), "use llama_model_n_head instead"); - - DEPRECATED(LLAMA_API int32_t llama_n_vocab (const struct llama_vocab * vocab), "use llama_vocab_n_tokens instead"); - LLAMA_API const struct llama_model * llama_get_model (const struct llama_context * ctx); LLAMA_API llama_memory_t llama_get_memory (const struct llama_context * ctx); LLAMA_API enum llama_pooling_type llama_pooling_type(const struct llama_context * ctx); // TODO: rename to llama_get_pooling_type @@ -683,6 +663,19 @@ extern "C" { const char * fname_out, const llama_model_quantize_params * params); + // + // NextN / speculative draft model queries + // + + // DFlash2 selector: number of candidate tokens the draft keeps per block position from its logits + // (the packed lattice also holds the top_k x top_k pairwise transition scores); 0 if the model has no selector head + LLAMA_API int32_t llama_model_dflash_selector_top_k(const struct llama_model * model); + + // returns pointer to the target-model layer indices + LLAMA_API const int32_t * llama_model_target_layer_ids (const struct llama_model * model); + // returns the number of extracted layers from target model + LLAMA_API uint32_t llama_model_target_layer_ids_n(const struct llama_model * model); + // // Adapters // @@ -829,8 +822,6 @@ extern "C" { // (logits, embedding and memory) // Only use when saving the state, not when restoring it, otherwise the size may be too small. LLAMA_API size_t llama_state_get_size(struct llama_context * ctx); - LLAMA_API DEPRECATED(size_t llama_get_state_size(struct llama_context * ctx), - "use llama_state_get_size instead"); // Copies the state to the specified destination address. // Destination needs to have allocated enough memory. @@ -839,10 +830,6 @@ extern "C" { struct llama_context * ctx, uint8_t * dst, size_t size); - LLAMA_API DEPRECATED(size_t llama_copy_state_data( - struct llama_context * ctx, - uint8_t * dst), - "use llama_state_get_data instead"); // Set the state reading from the specified address // Returns the number of bytes read @@ -850,10 +837,6 @@ extern "C" { struct llama_context * ctx, const uint8_t * src, size_t size); - LLAMA_API DEPRECATED(size_t llama_set_state_data( - struct llama_context * ctx, - const uint8_t * src), - "use llama_state_set_data instead"); // Save/load session file LLAMA_API bool llama_state_load_file( @@ -862,25 +845,12 @@ extern "C" { llama_token * tokens_out, size_t n_token_capacity, size_t * n_token_count_out); - LLAMA_API DEPRECATED(bool llama_load_session_file( - struct llama_context * ctx, - const char * path_session, - llama_token * tokens_out, - size_t n_token_capacity, - size_t * n_token_count_out), - "use llama_state_load_file instead"); LLAMA_API bool llama_state_save_file( struct llama_context * ctx, const char * path_session, const llama_token * tokens, size_t n_token_count); - LLAMA_API DEPRECATED(bool llama_save_session_file( - struct llama_context * ctx, - const char * path_session, - const llama_token * tokens, - size_t n_token_count), - "use llama_state_save_file instead"); // Get the exact size needed to copy the state of a single sequence LLAMA_API size_t llama_state_seq_get_size( @@ -1064,6 +1034,15 @@ extern "C" { int32_t idx, struct llama_embd embd); + // Add an entry whose embedding is row `row` of the layer input device buffer of ctx_other. + // All entries of the batch must be of this kind. Positions are set with llama_batch_ext_set_pos(), n_pos_per_embd of them. + // Returns the batch index (>= 0), or the llama_batch_ext_add() error codes. + LLAMA_API int32_t llama_batch_ext_add_embd_row(struct llama_batch_ext * batch, llama_seq_id seq_id, int32_t row); + + // The state of entry idx is row `row` of the h_nextn device buffer of the context set with llama_set_ctx_state_src(). + // All entries of a batch have a state row, or none. + LLAMA_API bool llama_batch_ext_set_embd_state_row(struct llama_batch_ext * batch, int32_t idx, int32_t row); + // Set if output embeddings should be available for the token at index idx in the batch // Note: for now, this is equivalent to setting the output logits LLAMA_API bool llama_batch_ext_set_output_embd( @@ -1117,14 +1096,6 @@ extern "C" { // Returns whether the context is currently using causal attention LLAMA_API bool llama_get_causal_attn(const struct llama_context * ctx); - // Set whether the model is in warmup mode or not - // If true, all model tensors are activated during llama_decode() to load and cache their weights. - // - // note: using this can cause extra graph reallocations because it changes the graph topology with MoE models, - // so it is generally not recommended to use in practice. will be removed in the future - DEPRECATED(LLAMA_API void llama_set_warmup(struct llama_context * ctx, bool warmup), - "user code should do warmup runs manually [TAG_LLAMA_GRAPH_NO_WARMUP]"); - // Set abort callback LLAMA_API void llama_set_abort_callback(struct llama_context * ctx, ggml_abort_callback abort_callback, void * abort_callback_data); @@ -1169,6 +1140,57 @@ extern "C" { // otherwise: float[n_embd] (1-dimensional) LLAMA_API float * llama_get_embeddings_seq(struct llama_context * ctx, llama_seq_id seq_id); + // + // NextN / speculative decoding API + // + + // Set whether the context outputs nextn embeddings or not + // If masked == true, output the embeddings only for the tokens with batch.logits != 0 + // If masked == false, output the embeddings for all tokens in the batch regardless of batch.logits + LLAMA_API void llama_set_embeddings_nextn(struct llama_context * ctx, bool value, bool masked); + + // Select which appended NextN block the DECODER_MTP graph runs (offset past + // the trunk: il = n_layer() + offset). Used by the speculative NextN driver to + // chain multiple trained NextN heads. Default 0 (first head). + LLAMA_API void llama_set_nextn_layer_offset(struct llama_context * ctx, int32_t offset); + + // Get all nextn embeddings, mirroring llama_get_embeddings() + LLAMA_API float * llama_get_embeddings_nextn(struct llama_context * ctx); + + // Get the nextn embeddings for the ith token, mirroring llama_get_embeddings_ith() + LLAMA_API float * llama_get_embeddings_nextn_ith(struct llama_context * ctx, int32_t i); + + // Set whether the context outputs the input embeddings of a specific layer + LLAMA_API void llama_set_embeddings_layer_inp(struct llama_context * ctx, uint32_t lid, bool value); + + // Get the input embeddings of a specific layer, mirroring llama_get_embeddings() + LLAMA_API float * llama_get_embeddings_layer_inp(struct llama_context * ctx, uint32_t lid); + + // Write the enabled layer inputs to a device buffer [n_layer_inp*n_embd, n_batch] instead of host memory. + // Row r holds the token at batch index r, layers in ascending layer id order. + LLAMA_API void llama_set_embeddings_layer_inp_dev(struct llama_context * ctx, bool value); + + // Whether decode accepts entries added with llama_batch_ext_add_embd_row() + // (needs a DFlash/DSpark decoder context whose ctx_other has a layer input device buffer). + LLAMA_API bool llama_supports_embd_row(const struct llama_context * ctx); + + // Write the unmasked h_nextn rows (see llama_set_embeddings_nextn) of each decode to a device buffer + // [n_embd_out, n_batch + n_seq_max] instead of host memory: row r < n_batch is the token at batch index r of the last decode, + // row n_batch + s is free for the caller, e.g. for seq s (see llama_set_embeddings_nextn_rows()). + // Returns false if the buffer cannot be used, the rows are then extracted to host memory. + LLAMA_API bool llama_set_embeddings_nextn_dev(struct llama_context * ctx, bool value); + + // Copy rows of the h_nextn device buffer to or from host memory, ordered after the last decode and its readers + LLAMA_API bool llama_get_embeddings_nextn_rows(struct llama_context * ctx, int64_t row0, int64_t n_rows, float * out); + LLAMA_API bool llama_set_embeddings_nextn_rows(struct llama_context * ctx, int64_t row0, int64_t n_rows, const float * data); + + // Read the state rows of batches (see llama_batch_ext_set_embd_state_row()) from the h_nextn device buffer of ctx_src. + // Returns whether decode accepts batches with state rows. + LLAMA_API bool llama_set_ctx_state_src(struct llama_context * ctx, struct llama_context * ctx_src); + + // The paired context (llama_context_params::ctx_other), e.g. the target context of a draft context; may be nullptr + LLAMA_API struct llama_context * llama_get_ctx_other(struct llama_context * ctx); + // // backend sampling API [EXPERIMENTAL] // note: use only if the llama_context was created with at least one llama_sampler_seq_config @@ -1237,31 +1259,6 @@ extern "C" { LLAMA_API llama_token llama_vocab_fim_rep(const struct llama_vocab * vocab); LLAMA_API llama_token llama_vocab_fim_sep(const struct llama_vocab * vocab); - DEPRECATED(LLAMA_API const char * llama_token_get_text(const struct llama_vocab * vocab, llama_token token), "use llama_vocab_get_text instead"); - DEPRECATED(LLAMA_API float llama_token_get_score(const struct llama_vocab * vocab, llama_token token), "use llama_vocab_get_score instead"); - DEPRECATED(LLAMA_API enum llama_token_attr llama_token_get_attr(const struct llama_vocab * vocab, llama_token token), "use llama_vocab_get_attr instead"); - DEPRECATED(LLAMA_API bool llama_token_is_eog(const struct llama_vocab * vocab, llama_token token), "use llama_vocab_is_eog instead"); - DEPRECATED(LLAMA_API bool llama_token_is_control(const struct llama_vocab * vocab, llama_token token), "use llama_vocab_is_control instead"); - DEPRECATED(LLAMA_API llama_token llama_token_bos(const struct llama_vocab * vocab), "use llama_vocab_bos instead"); - DEPRECATED(LLAMA_API llama_token llama_token_eos(const struct llama_vocab * vocab), "use llama_vocab_eos instead"); - DEPRECATED(LLAMA_API llama_token llama_token_eot(const struct llama_vocab * vocab), "use llama_vocab_eot instead"); - DEPRECATED(LLAMA_API llama_token llama_token_cls(const struct llama_vocab * vocab), "use llama_vocab_cls instead"); - DEPRECATED(LLAMA_API llama_token llama_token_sep(const struct llama_vocab * vocab), "use llama_vocab_sep instead"); - DEPRECATED(LLAMA_API llama_token llama_token_nl (const struct llama_vocab * vocab), "use llama_vocab_nl instead"); - DEPRECATED(LLAMA_API llama_token llama_token_pad(const struct llama_vocab * vocab), "use llama_vocab_pad instead"); - DEPRECATED(LLAMA_API bool llama_add_bos_token(const struct llama_vocab * vocab), "use llama_vocab_get_add_bos instead"); - DEPRECATED(LLAMA_API bool llama_add_eos_token(const struct llama_vocab * vocab), "use llama_vocab_get_add_eos instead"); - DEPRECATED(LLAMA_API llama_token llama_token_fim_pre(const struct llama_vocab * vocab), "use llama_vocab_fim_pre instead"); - DEPRECATED(LLAMA_API llama_token llama_token_fim_suf(const struct llama_vocab * vocab), "use llama_vocab_fim_suf instead"); - DEPRECATED(LLAMA_API llama_token llama_token_fim_mid(const struct llama_vocab * vocab), "use llama_vocab_fim_mid instead"); - DEPRECATED(LLAMA_API llama_token llama_token_fim_pad(const struct llama_vocab * vocab), "use llama_vocab_fim_pad instead"); - DEPRECATED(LLAMA_API llama_token llama_token_fim_rep(const struct llama_vocab * vocab), "use llama_vocab_fim_rep instead"); - DEPRECATED(LLAMA_API llama_token llama_token_fim_sep(const struct llama_vocab * vocab), "use llama_vocab_fim_sep instead"); - - // CLS is equivalent to BOS - DEPRECATED(LLAMA_API llama_token llama_vocab_cls(const struct llama_vocab * vocab), // classification - "use llama_vocab_bos instead"); - // // Tokenization // @@ -1533,17 +1530,6 @@ extern "C" { const char * grammar_str, const char * grammar_root); - DEPRECATED(LLAMA_API struct llama_sampler * llama_sampler_init_grammar_lazy( - const struct llama_vocab * vocab, - const char * grammar_str, - const char * grammar_root, - const char ** trigger_words, - size_t num_trigger_words, - const llama_token * trigger_tokens, - size_t num_trigger_tokens), - "use llama_sampler_init_grammar_lazy_patterns instead"); - - /// @details Lazy grammar sampler, introduced in https://github.com/ggml-org/llama.cpp/pull/9639 /// @param trigger_patterns A list of patterns that will trigger the grammar sampler. Pattern will be matched from the start of the generation output, and grammar sampler will be fed content starting from its first match group. /// @param trigger_tokens A list of tokens that will trigger the grammar sampler. Grammar sampler will be fed content starting from the trigger token included. diff --git a/media/llama0-banner.png b/media/llama0-banner.png deleted file mode 100644 index cee3a87f1a7b..000000000000 Binary files a/media/llama0-banner.png and /dev/null differ diff --git a/media/llama0-logo.png b/media/llama0-logo.png deleted file mode 100644 index e55b38bd9c0b..000000000000 Binary files a/media/llama0-logo.png and /dev/null differ diff --git a/media/llama1-banner.png b/media/llama1-banner.png deleted file mode 100644 index 1e469584e0ce..000000000000 Binary files a/media/llama1-banner.png and /dev/null differ diff --git a/media/llama1-icon-transparent.png b/media/llama1-icon-transparent.png deleted file mode 100644 index 432d6c2223bb..000000000000 Binary files a/media/llama1-icon-transparent.png and /dev/null differ diff --git a/media/llama1-icon-transparent.svg b/media/llama1-icon-transparent.svg deleted file mode 100644 index e28203f4e82d..000000000000 --- a/media/llama1-icon-transparent.svg +++ /dev/null @@ -1,77 +0,0 @@ - - - - - - - - - - - - - - - - - diff --git a/media/llama1-icon.png b/media/llama1-icon.png deleted file mode 100644 index 0e44672e54bf..000000000000 Binary files a/media/llama1-icon.png and /dev/null differ diff --git a/media/llama1-icon.svg b/media/llama1-icon.svg deleted file mode 100644 index dcbe9cce9bad..000000000000 --- a/media/llama1-icon.svg +++ /dev/null @@ -1,87 +0,0 @@ - - - - - - - - - - - - - - - - - - diff --git a/media/llama1-logo.png b/media/llama1-logo.png deleted file mode 100644 index 365c5b865f3f..000000000000 Binary files a/media/llama1-logo.png and /dev/null differ diff --git a/media/llama1-logo.svg b/media/llama1-logo.svg deleted file mode 100644 index e080481fa67c..000000000000 --- a/media/llama1-logo.svg +++ /dev/null @@ -1,34 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/models/templates/google-gemma-4-31B-it-interleaved.jinja b/models/templates/google-gemma-4-31B-it-interleaved.jinja deleted file mode 100644 index 85791c4fe5dc..000000000000 --- a/models/templates/google-gemma-4-31B-it-interleaved.jinja +++ /dev/null @@ -1,282 +0,0 @@ -{%- macro format_parameters(properties, required) -%} - {%- set standard_keys = ['description', 'type', 'properties', 'required', 'nullable'] -%} - {%- set ns = namespace(found_first=false) -%} - {%- for key, value in properties | dictsort -%} - {%- set add_comma = false -%} - {%- if key not in standard_keys -%} - {%- if ns.found_first %},{% endif -%} - {%- set ns.found_first = true -%} - {{ key }}:{ - {%- if value['description'] -%} - description:<|"|>{{ value['description'] }}<|"|> - {%- set add_comma = true -%} - {%- endif -%} - {%- if value['nullable'] %} - {%- if add_comma %},{%- else -%} {%- set add_comma = true -%} {% endif -%} - nullable:true - {%- endif -%} - {%- if value['type'] | upper == 'STRING' -%} - {%- if value['enum'] -%} - {%- if add_comma %},{%- else -%} {%- set add_comma = true -%} {% endif -%} - enum:{{ format_argument(value['enum']) }} - {%- endif -%} - {%- elif value['type'] | upper == 'OBJECT' -%} - ,properties:{ - {%- if value['properties'] is defined and value['properties'] is mapping -%} - {{- format_parameters(value['properties'], value['required'] | default([])) -}} - {%- elif value is mapping -%} - {{- format_parameters(value, value['required'] | default([])) -}} - {%- endif -%} - } - {%- if value['required'] -%} - ,required:[ - {%- for item in value['required'] | default([]) -%} - <|"|>{{- item -}}<|"|> - {%- if not loop.last %},{% endif -%} - {%- endfor -%} - ] - {%- endif -%} - {%- elif value['type'] | upper == 'ARRAY' -%} - {%- if value['items'] is mapping and value['items'] -%} - ,items:{ - {%- set ns_items = namespace(found_first=false) -%} - {%- for item_key, item_value in value['items'] | dictsort -%} - {%- if item_value is not none -%} - {%- if ns_items.found_first %},{% endif -%} - {%- set ns_items.found_first = true -%} - {%- if item_key == 'properties' -%} - properties:{ - {%- if item_value is mapping -%} - {{- format_parameters(item_value, value['items']['required'] | default([])) -}} - {%- endif -%} - } - {%- elif item_key == 'required' -%} - required:[ - {%- for req_item in item_value -%} - <|"|>{{- req_item -}}<|"|> - {%- if not loop.last %},{% endif -%} - {%- endfor -%} - ] - {%- elif item_key == 'type' -%} - {%- if item_value is string -%} - type:{{ format_argument(item_value | upper) }} - {%- else -%} - type:{{ format_argument(item_value | map('upper') | list) }} - {%- endif -%} - {%- else -%} - {{ item_key }}:{{ format_argument(item_value) }} - {%- endif -%} - {%- endif -%} - {%- endfor -%} - } - {%- endif -%} - {%- endif -%} - {%- if add_comma %},{%- else -%} {%- set add_comma = true -%} {% endif -%} - type:<|"|>{{ value['type'] | upper }}<|"|>} - {%- endif -%} - {%- endfor -%} -{%- endmacro -%} -{%- macro format_function_declaration(tool_data) -%} - declaration:{{- tool_data['function']['name'] -}}{description:<|"|>{{- tool_data['function']['description'] -}}<|"|> - {%- set params = tool_data['function']['parameters'] -%} - {%- if params -%} - ,parameters:{ - {%- if params['properties'] -%} - properties:{ {{- format_parameters(params['properties'], params['required']) -}} }, - {%- endif -%} - {%- if params['required'] -%} - required:[ - {%- for item in params['required'] -%} - <|"|>{{- item -}}<|"|> - {{- ',' if not loop.last -}} - {%- endfor -%} - ], - {%- endif -%} - {%- if params['type'] -%} - type:<|"|>{{- params['type'] | upper -}}<|"|>} - {%- endif -%} - {%- endif -%} - {%- if 'response' in tool_data['function'] -%} - {%- set response_declaration = tool_data['function']['response'] -%} - ,response:{ - {%- if response_declaration['description'] -%} - description:<|"|>{{- response_declaration['description'] -}}<|"|>, - {%- endif -%} - {%- if response_declaration['type'] | upper == 'OBJECT' -%} - type:<|"|>{{- response_declaration['type'] | upper -}}<|"|>} - {%- endif -%} - {%- endif -%} - } -{%- endmacro -%} -{%- macro format_argument(argument, escape_keys=True) -%} - {%- if argument is string -%} - {{- '<|"|>' + argument + '<|"|>' -}} - {%- elif argument is boolean -%} - {{- 'true' if argument else 'false' -}} - {%- elif argument is mapping -%} - {{- '{' -}} - {%- set ns = namespace(found_first=false) -%} - {%- for key, value in argument | dictsort -%} - {%- if ns.found_first %},{% endif -%} - {%- set ns.found_first = true -%} - {%- if escape_keys -%} - {{- '<|"|>' + key + '<|"|>' -}} - {%- else -%} - {{- key -}} - {%- endif -%} - :{{- format_argument(value, escape_keys=escape_keys) -}} - {%- endfor -%} - {{- '}' -}} - {%- elif argument is sequence -%} - {{- '[' -}} - {%- for item in argument -%} - {{- format_argument(item, escape_keys=escape_keys) -}} - {%- if not loop.last %},{% endif -%} - {%- endfor -%} - {{- ']' -}} - {%- else -%} - {{- argument -}} - {%- endif -%} -{%- endmacro -%} -{%- macro strip_thinking(text) -%} - {%- set ns = namespace(result='') -%} - {%- for part in text.split('') -%} - {%- if '<|channel>' in part -%} - {%- set ns.result = ns.result + part.split('<|channel>')[0] -%} - {%- else -%} - {%- set ns.result = ns.result + part -%} - {%- endif -%} - {%- endfor -%} - {{- ns.result | trim -}} -{%- endmacro -%} - -{%- set ns = namespace(prev_message_type=None, last_user_message=-1) -%} -{%- set loop_messages = messages -%} -{{- bos_token -}} -{#- Handle System/Tool Definitions Block -#} -{%- if (enable_thinking is defined and enable_thinking) or tools or messages[0]['role'] in ['system', 'developer'] -%} - {{- '<|turn>system\n' -}} - - {#- Inject Thinking token at the very top of the FIRST system turn -#} - {%- if enable_thinking is defined and enable_thinking -%} - {{- '<|think|>\n' -}} - {%- set ns.prev_message_type = 'think' -%} - {%- endif -%} - - {%- if messages[0]['role'] in ['system', 'developer'] -%} - {{- messages[0]['content'] | trim -}} - {%- set loop_messages = messages[1:] -%} - {%- endif -%} - - {%- if tools -%} - {%- for tool in tools %} - {{- '<|tool>' -}} - {{- format_function_declaration(tool) | trim -}} - {{- '' -}} - {%- endfor %} - {%- set ns.prev_message_type = 'tool' -%} - {%- endif -%} - - {{- '\n' -}} -{%- endif %} - -{#- Find last user message -#} -{%- for message in loop_messages -%} - {%- if message['role'] == 'user' -%} - {%- set ns.last_user_message = loop.index0 -%} - {%- endif -%} -{%- endfor -%} - -{#- Loop through messages -#} -{%- for message in loop_messages -%} - {%- set role = 'model' if message['role'] == 'assistant' else message['role'] -%} - {%- if not (ns.prev_message_type == 'tool_response' and message['tool_calls']) -%} - {{- '<|turn>' + role + '\n' }} - {%- endif -%} - - {%- set ns.prev_message_type = None -%} - - {%- if message['tool_calls'] -%} - {#- Preserve reasoning between tool calls for model turns that come after the last user turn -#} - {%- if message['reasoning_content'] and loop.index0 > ns.last_user_message -%} - {{- '<|channel>thought\n' -}} - {{- message['reasoning_content'] -}} - {{- '' -}} - {%- endif -%} - {%- for tool_call in message['tool_calls'] -%} - {%- set function = tool_call['function'] -%} - {{- '<|tool_call>call:' + function['name'] + '{' -}} - {%- if function['arguments'] is mapping -%} - {%- set ns_args = namespace(found_first=false) -%} - {%- for key, value in function['arguments'] | dictsort -%} - {%- if ns_args.found_first %},{% endif -%} - {%- set ns_args.found_first = true -%} - {{- key -}}:{{- format_argument(value, escape_keys=False) -}} - {%- endfor -%} - {%- elif function['arguments'] is string -%} - {{- function['arguments'] -}} - {%- endif -%} - {{- '}' -}} - {%- endfor -%} - {%- set ns.prev_message_type = 'tool_call' -%} - {%- endif -%} - - {%- if message['tool_responses'] -%} - {#- Tool Response handling -#} - {%- for tool_response in message['tool_responses'] -%} - {{- '<|tool_response>' -}} - {%- if tool_response['response'] is mapping -%} - {{- 'response:' + tool_response['name'] | default('unknown') + '{' -}} - {%- for key, value in tool_response['response'] | dictsort -%} - {{- key -}}:{{- format_argument(value, escape_keys=False) -}} - {%- if not loop.last %},{% endif -%} - {%- endfor -%} - {{- '}' -}} - {%- else -%} - {{- 'response:' + tool_response['name'] | default('unknown') + '{value:' + format_argument(tool_response['response'], escape_keys=False) + '}' -}} - {%- endif -%} - {{- '' -}} - {%- endfor -%} - {%- set ns.prev_message_type = 'tool_response' -%} - {%- endif -%} - - {%- if message['content'] is string -%} - {%- if role == 'model' -%} - {{- strip_thinking(message['content']) -}} - {%- else -%} - {{- message['content'] | trim -}} - {%- endif -%} - {%- elif message['content'] is sequence -%} - {%- for item in message['content'] -%} - {%- if item['type'] == 'text' -%} - {%- if role == 'model' -%} - {{- strip_thinking(item['text']) -}} - {%- else -%} - {{- item['text'] | trim -}} - {%- endif -%} - {%- elif item['type'] == 'image' -%} - {{- '<|image|>' -}} - {%- set ns.prev_message_type = 'image' -%} - {%- elif item['type'] == 'audio' -%} - {{- '<|audio|>' -}} - {%- set ns.prev_message_type = 'audio' -%} - {%- elif item['type'] == 'video' -%} - {{- '<|video|>' -}} - {%- set ns.prev_message_type = 'video' -%} - {%- endif -%} - {%- endfor -%} - {%- endif -%} - - {%- if not (message['tool_responses'] and not message['content']) -%} - {{- '\n' -}} - {%- endif -%} -{%- endfor -%} - -{%- if add_generation_prompt -%} - {%- if ns.prev_message_type != 'tool_response' -%} - {{- '<|turn>model\n' -}} - {%- endif -%} - {%- if not enable_thinking | default(false) -%} - {{- '<|channel>thought\n' -}} - {%- endif -%} -{%- endif -%} diff --git a/scripts/chat_tts_norm.py b/scripts/chat_tts_norm.py new file mode 100644 index 000000000000..a5fbd55b68ed --- /dev/null +++ b/scripts/chat_tts_norm.py @@ -0,0 +1,556 @@ +#!/usr/bin/env python3 +"""Reference implementation of TTS normalization patterns, the tts_norm_scan() and tts_norm_sub() template functions. + +The specification is in docs/tts-norm-patterns.md. This file must give the same result as the C++ implementation +(common/chat-tts-norm.cpp) for every input; tests/test-jinja.cpp checks this in python mode (test-jinja -py). + +Usage with jinja2: + + env.globals["tts_norm_scan"] = chat_tts_norm.tts_norm_scan + env.globals["tts_norm_sub"] = chat_tts_norm.tts_norm_sub +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Any + +# character classes, explicit codepoint ranges (keep in sync with common/chat-tts-norm.cpp) + +TTS_NORM_SPACE = [ + (0x0009, 0x000D), (0x0020, 0x0020), (0x0085, 0x0085), (0x00A0, 0x00A0), (0x1680, 0x1680), + (0x2000, 0x200A), (0x2028, 0x2029), (0x202F, 0x202F), (0x205F, 0x205F), (0x3000, 0x3000), +] + +TTS_NORM_PUNCT = [ + (0x0021, 0x002F), (0x003A, 0x0040), (0x005B, 0x0060), (0x007B, 0x007E), + (0x00A1, 0x00A9), (0x00AB, 0x00B4), (0x00B6, 0x00B9), (0x00BB, 0x00BF), (0x00D7, 0x00D7), (0x00F7, 0x00F7), + (0x2010, 0x2027), (0x2030, 0x205E), (0x20A0, 0x20CF), (0x2100, 0x218F), (0x2190, 0x2BFF), + (0x3001, 0x3003), (0x3008, 0x3020), (0x3030, 0x3030), (0x303D, 0x303D), (0x30FB, 0x30FB), + (0xFE10, 0xFE19), (0xFE30, 0xFE4F), (0xFE50, 0xFE6B), + (0xFF01, 0xFF0F), (0xFF1A, 0xFF20), (0xFF3B, 0xFF40), (0xFF5B, 0xFF65), + (0x1F000, 0x1FAFF), +] + +TTS_NORM_CONTROL = [ + (0x0000, 0x001F), (0x007F, 0x009F), (0x200B, 0x200F), (0x2060, 0x206F), (0xFEFF, 0xFEFF), +] + +TTS_NORM_HAN = [ + (0x2E80, 0x2FDF), (0x3005, 0x3005), (0x3007, 0x3007), (0x3021, 0x3029), (0x3038, 0x303B), + (0x3400, 0x4DBF), (0x4E00, 0x9FFF), (0xF900, 0xFAFF), (0x20000, 0x323AF), +] + +CLASS_LETTERS = "dspawulhc" +INF = 1 << 31 +MAX_REPEAT = 1000 +MAX_PATTERN_LEN = 4096 + + +def _in(ranges: list[tuple[int, int]], c: int) -> bool: + return any(lo <= c <= hi for lo, hi in ranges) + + +def _is_alpha(c: int) -> bool: + if ord("a") <= c <= ord("z") or ord("A") <= c <= ord("Z"): + return True + if c in (0xAA, 0xB5, 0xBA): + return True + return c >= 0xC0 and not _in(TTS_NORM_SPACE, c) and not _in(TTS_NORM_PUNCT, c) and not _in(TTS_NORM_CONTROL, c) + + +def _class_match(kind: str, c: int) -> bool: + if kind == "d": + return ord("0") <= c <= ord("9") + if kind == "s": + return _in(TTS_NORM_SPACE, c) + if kind == "p": + return _in(TTS_NORM_PUNCT, c) + if kind == "a": + return _is_alpha(c) + if kind == "w": + return _is_alpha(c) or ord("0") <= c <= ord("9") + if kind == "u": + return ord("A") <= c <= ord("Z") + if kind == "l": + return ord("a") <= c <= ord("z") + if kind == "h": + return _in(TTS_NORM_HAN, c) + if kind == "c": + return _in(TTS_NORM_CONTROL, c) + return False + + +def _ascii_lower(c: int) -> int: + return c + 32 if ord("A") <= c <= ord("Z") else c + + +def _ascii_swapcase(c: int) -> int: + if ord("A") <= c <= ord("Z"): + return c + 32 + if ord("a") <= c <= ord("z"): + return c - 32 + return c + + +def _is_ascii_alnum(c: int) -> bool: + return ord("0") <= c <= ord("9") or ord("a") <= c <= ord("z") or ord("A") <= c <= ord("Z") + + +# AST + +LITERAL, SET, ANY, START, END, BOUNDARY, GROUP, LOOKAHEAD, LOOKBEHIND = range(9) +ZERO_WIDTH = (START, END, BOUNDARY, LOOKAHEAD, LOOKBEHIND) + + +@dataclass +class _Set: + neg: bool = False + ranges: list[tuple[int, int]] = field(default_factory=list) + classes: list[tuple[str, bool]] = field(default_factory=list) + + +@dataclass +class _Node: + op: int = GROUP + cp: int = 0 + set: _Set = field(default_factory=_Set) + alts: list[list[_Item]] = field(default_factory=list) + capture: int = -1 + neg: bool = False + + +@dataclass +class _Item: + atom: _Node + min: int = 1 + max: int = 1 + + +class PatternError(ValueError): + pass + + +class _Parser: + def __init__(self, s: list[int]): + self.s = s + self.p = 0 + self.n_caps = 0 + + def fail(self, msg: str): + raise PatternError(f"invalid TTS norm pattern: {msg} at position {self.p}") + + def eof(self) -> bool: + return self.p >= len(self.s) + + def peek_is(self, c: str) -> bool: + return not self.eof() and self.s[self.p] == ord(c) + + def expect(self, c: str, what: str): + if not self.peek_is(c): + self.fail(f"expected {what}") + self.p += 1 + + def parse_alts(self) -> list[list[_Item]]: + alts = [self.parse_seq()] + while self.peek_is("|"): + self.p += 1 + alts.append(self.parse_seq()) + return alts + + def parse_seq(self) -> list[_Item]: + seq: list[_Item] = [] + while not self.eof() and self.s[self.p] not in (ord("|"), ord(")")): + item = _Item(self.parse_atom()) + self.parse_quantifier(item) + seq.append(item) + return seq + + def parse_int(self) -> int: + if self.eof() or not (ord("0") <= self.s[self.p] <= ord("9")): + self.fail("expected number in quantifier") + v = 0 + while not self.eof() and ord("0") <= self.s[self.p] <= ord("9"): + v = v * 10 + self.s[self.p] - ord("0") + if v > MAX_REPEAT: + self.fail(f"repetition count exceeds {MAX_REPEAT}") + self.p += 1 + return v + + def parse_quantifier(self, item: _Item): + if self.eof(): + return + c = chr(self.s[self.p]) + if c not in "?*+{": + return + if item.atom.op in ZERO_WIDTH: + self.fail("zero-width assertion cannot be repeated") + self.p += 1 + if c == "?": + item.min, item.max = 0, 1 + elif c == "*": + item.min, item.max = 0, INF + elif c == "+": + item.min, item.max = 1, INF + else: + item.min = self.parse_int() + item.max = item.min + if self.peek_is(","): + self.p += 1 + item.max = INF if self.peek_is("}") else self.parse_int() + self.expect("}", "'}' to close quantifier") + if item.max < item.min: + self.fail("quantifier max is smaller than min") + if not self.eof() and chr(self.s[self.p]) in "?*+{": + self.fail("multiple quantifiers") + + def parse_group(self) -> _Node: + node = _Node(op=GROUP) + if self.peek_is("?"): + self.p += 1 + if self.eof(): + self.fail("unterminated group") + c = chr(self.s[self.p]) + if c == ":": + self.p += 1 + elif c in "=!": + self.p += 1 + node.op = LOOKAHEAD + node.neg = c == "!" + elif c == "<" and self.p + 1 < len(self.s) and self.s[self.p + 1] in (ord("="), ord("!")): + node.op = LOOKBEHIND + node.neg = self.s[self.p + 1] == ord("!") + self.p += 2 + if self.eof() or self.s[self.p] == ord(")"): + self.fail("lookbehind must contain exactly one codepoint matcher") + atom = self.parse_atom() + if atom.op not in (LITERAL, SET, ANY): + self.fail("lookbehind must contain exactly one codepoint matcher") + if not self.peek_is(")"): + self.fail("lookbehind must contain exactly one codepoint matcher") + self.p += 1 + node.alts = [[_Item(atom)]] + return node + elif c == "i": + self.fail("(?i) is only allowed at the start of the pattern") + else: + self.fail("unknown group type") + else: + node.capture = self.n_caps + self.n_caps += 1 + node.alts = self.parse_alts() + self.expect(")", "')' to close group") + return node + + # returns ("class", (kind, neg)) | ("boundary", None) | ("literal", cp) + def parse_escape(self, in_set: bool) -> tuple[str, Any]: + if self.eof(): + self.fail("dangling '%'") + e = self.s[self.p] + self.p += 1 + if e < 0x80 and chr(e).lower() in CLASS_LETTERS: + return "class", (chr(e).lower(), ord("A") <= e <= ord("Z")) + if e == ord("b") and not in_set: + return "boundary", None + if _is_ascii_alnum(e): + self.p -= 1 + self.fail(f"unknown escape '%{chr(e)}'") + return "literal", e + + def parse_set(self) -> _Node: + node = _Node(op=SET) + if self.peek_is("^"): + node.set.neg = True + self.p += 1 + empty = True + while True: + if self.eof(): + self.fail("unterminated set") + if self.s[self.p] == ord("]"): + self.p += 1 + break + empty = False + if self.s[self.p] == ord("%"): + self.p += 1 + kind, val = self.parse_escape(True) + if kind == "class": + node.set.classes.append(val) + continue + lo = val + else: + lo = self.s[self.p] + self.p += 1 + hi = lo + if self.peek_is("-") and self.p + 1 < len(self.s) and self.s[self.p + 1] != ord("]"): + self.p += 1 + if self.s[self.p] == ord("%"): + self.p += 1 + kind, val = self.parse_escape(True) + if kind == "class": + self.fail("character class cannot be used in a range") + hi = val + else: + hi = self.s[self.p] + self.p += 1 + if hi < lo: + self.fail("invalid range in set") + node.set.ranges.append((lo, hi)) + if empty: + self.fail("empty set") + return node + + def parse_atom(self) -> _Node: + c = chr(self.s[self.p]) + self.p += 1 + if c == "(": + return self.parse_group() + if c == "[": + return self.parse_set() + if c == ".": + return _Node(op=ANY) + if c == "^": + return _Node(op=START) + if c == "$": + return _Node(op=END) + if c == "%": + kind, val = self.parse_escape(False) + if kind == "class": + node = _Node(op=SET) + node.set.classes.append(val) + return node + if kind == "boundary": + return _Node(op=BOUNDARY) + return _Node(op=LITERAL, cp=val) + if c in "?*+{": + self.p -= 1 + self.fail("nothing to repeat") + if c in "]}": + self.p -= 1 + self.fail(f"unescaped '{c}', use '%{c}'") + return _Node(op=LITERAL, cp=ord(c)) + + +class _Matcher: + def __init__(self, t: list[int], icase: bool, n_groups: int): + self.t = t + self.icase = icase + self.caps: list[tuple[int, int]] = [(-1, -1)] * n_groups + + def is_w(self, i: int) -> bool: + return i < len(self.t) and _class_match("w", self.t[i]) + + def set_match(self, st: _Set, c: int) -> bool: + def test(x: int) -> bool: + if any(lo <= x <= hi for lo, hi in st.ranges): + return True + return any(_class_match(kind, x) != neg for kind, neg in st.classes) + + m = test(c) + if not m and self.icase: + o = _ascii_swapcase(c) + if o != c: + m = test(o) + return m != st.neg + + def match_alts(self, alts: list[list[_Item]], i: int) -> int: + """returns the end position, or -1 if no alternative matches""" + for seq in alts: + saved = list(self.caps) + end = self.match_seq(seq, i) + if end >= 0: + return end + self.caps = saved + return -1 + + def match_seq(self, seq: list[_Item], i: int) -> int: + pos = i + for it in seq: + count = 0 + while count < it.max: + is_group = it.atom.op == GROUP + saved = list(self.caps) if is_group else None + e = self.match_once(it.atom, pos) + if e < 0: + if saved is not None: + self.caps = saved + break + if e == pos: + # an empty iteration would repeat forever, it satisfies any remaining minimum + count = max(count + 1, it.min) + break + pos = e + count += 1 + if count < it.min: + return -1 + return pos + + def match_once(self, node: _Node, i: int) -> int: + """returns the end position, or -1 on failure""" + n = len(self.t) + op = node.op + if op == LITERAL: + if i < n and (self.t[i] == node.cp or (self.icase and _ascii_lower(self.t[i]) == _ascii_lower(node.cp))): + return i + 1 + return -1 + if op == SET: + return i + 1 if i < n and self.set_match(node.set, self.t[i]) else -1 + if op == ANY: + return i + 1 if i < n else -1 + if op == START: + return i if i == 0 else -1 + if op == END: + return i if i == n else -1 + if op == BOUNDARY: + return i if (i > 0 and self.is_w(i - 1)) != self.is_w(i) else -1 + if op == GROUP: + e = self.match_alts(node.alts, i) + if e < 0: + return -1 + if node.capture >= 0: + self.caps[node.capture] = (i, e) + return e + if op == LOOKAHEAD: + # captures made inside a lookahead are discarded + saved = list(self.caps) + ok = self.match_alts(node.alts, i) >= 0 + self.caps = saved + return i if ok != node.neg else -1 + if op == LOOKBEHIND: + ok = i > 0 and self.match_once(node.alts[0][0].atom, i - 1) == i + return i if ok != node.neg else -1 + return -1 + + +class Pattern: + def __init__(self, src: str): + if len(src.encode("utf-8")) > MAX_PATTERN_LEN: + raise PatternError(f"invalid TTS norm pattern: pattern is longer than {MAX_PATTERN_LEN} bytes") + self.icase = False + parser = _Parser([ord(c) for c in src]) + if src.startswith("(?i)"): + self.icase = True + parser.p = 4 + self.root = _Node(op=GROUP, alts=parser.parse_alts()) + if not parser.eof(): + parser.fail("unbalanced ')'") + self.n_caps = parser.n_caps + + def match_at(self, t: list[int], pos: int) -> tuple[int, list[tuple[int, int]]] | None: + m = _Matcher(t, self.icase, self.n_caps) + end = m.match_alts(self.root.alts, pos) + if end < 0: + return None + return end, m.caps + + +# (rule index or -1, start, end, groups) with codepoint offsets +Segment = tuple[int, int, int, list[tuple[int, int]]] + + +def scan_segments(text: str, rules: list[Pattern]) -> list[Segment]: + t = [ord(c) for c in text] + n = len(t) + out: list[Segment] = [] + i = 0 + pending = 0 + while i < n: + matched = False + for r, rule in enumerate(rules): + res = rule.match_at(t, i) + if res is not None and res[0] > i: + end, groups = res + if pending < i: + out.append((-1, pending, i, [])) + out.append((r, i, end, groups)) + i = end + pending = i + matched = True + break + if not matched: + i += 1 + if pending < n: + out.append((-1, pending, n, [])) + return out + + +def _rules_from_value(rules: Any) -> list[tuple[str, str]]: + if isinstance(rules, str): + return [("match", rules)] + if isinstance(rules, dict): + out = [] + for k, v in rules.items(): + if not isinstance(v, str): + raise TypeError("tts_norm_scan: rule pattern must be a string") + out.append((str(k), v)) + return out + if isinstance(rules, (list, tuple)): + out = [] + for item in rules: + if not isinstance(item, (list, tuple)) or len(item) != 2 or not isinstance(item[0], str) or not isinstance(item[1], str): + raise TypeError("tts_norm_scan: each rule must be a [name, pattern] pair of strings") + out.append((item[0], item[1])) + return out + raise TypeError("tts_norm_scan: rules must be a string, a dict or a list of [name, pattern] pairs") + + +def tts_norm_scan(text: str, rules: Any) -> list[dict[str, Any]]: + rule_srcs = _rules_from_value(rules) + patterns = [Pattern(src) for _, src in rule_srcs] + out: list[dict[str, Any]] = [] + for r, start, end, groups in scan_segments(text, patterns): + # groups that did not participate are empty strings, so they render as nothing and are falsy + vals = ["" if gs < 0 else text[gs:ge] for gs, ge in groups] + out.append({"rule": rule_srcs[r][0] if r >= 0 else None, "text": text[start:end], "groups": vals}) + return out + + +def _parse_repl(repl: str, pattern: Pattern) -> list[str | int]: + pieces: list[str | int] = [] + lit = "" + i = 0 + while i < len(repl): + c = repl[i] + if c != "%": + lit += c + i += 1 + continue + if i + 1 >= len(repl): + raise PatternError("invalid TTS norm replacement: dangling '%'") + i += 1 + c = repl[i] + if c == "%": + lit += "%" + elif "0" <= c <= "9": + g = ord(c) - ord("0") + if g > pattern.n_caps: + raise PatternError(f"invalid TTS norm replacement: group %{c} does not exist") + if lit: + pieces.append(lit) + lit = "" + pieces.append(g) + else: + raise PatternError(f"invalid TTS norm replacement: unknown escape '%{c}'") + i += 1 + if lit: + pieces.append(lit) + return pieces + + +def tts_norm_sub(text: str, pattern: str, repl: str) -> str: + pat = Pattern(pattern) + pieces = _parse_repl(repl, pat) + out = [] + for r, start, end, groups in scan_segments(text, [pat]): + if r < 0: + out.append(text[start:end]) + continue + for piece in pieces: + if isinstance(piece, str): + out.append(piece) + elif piece == 0: + out.append(text[start:end]) + else: + gs, ge = groups[piece - 1] + if gs >= 0: + out.append(text[gs:ge]) + return "".join(out) diff --git a/scripts/hip/gcn-cdna-vgpr-check.py b/scripts/hip/gcn-cdna-vgpr-check.py index 146bc8ff58e4..a09891fdc3a5 100644 --- a/scripts/hip/gcn-cdna-vgpr-check.py +++ b/scripts/hip/gcn-cdna-vgpr-check.py @@ -64,27 +64,27 @@ def main(): '_ZL12rwkv_wkv_f32ILi128EEviiiiPKfS1_S1_S1_S1_S1_Pf', '_ZL9mul_mat_qIL9ggml_type10ELi64ELb1EEvPKcPKiS4_S4_PfS5_PKf15HIP_vector_typeIjLj3EEiiiiiS9_S9_iiiS9_S9_iiiS9_', '_ZL9mul_mat_qIL9ggml_type42ELi128ELb1EEvPKcPKiS4_S4_PfS5_PKf15HIP_vector_typeIjLj3EEiiiiiS9_S9_iiiS9_S9_iiiS9_', - '_ZL18flash_attn_ext_f16ILi576ELi512ELi2ELi32ELb0ELb1ELb0EEvPKcS1_S1_S1_S1_PKiPfP15HIP_vector_typeIfLj2EEffffjfiS5_IjLj3EEiiiiiiiiiiiliiliiiiil', - '_ZL18flash_attn_ext_f16ILi512ELi512ELi16ELi4ELb0ELb0ELb0EEvPKcS1_S1_S1_S1_PKiPfP15HIP_vector_typeIfLj2EEffffjfiS5_IjLj3EEiiiiiiiiiiiliiliiiiil', - '_ZL18flash_attn_ext_f16ILi512ELi512ELi16ELi4ELb1ELb0ELb0EEvPKcS1_S1_S1_S1_PKiPfP15HIP_vector_typeIfLj2EEffffjfiS5_IjLj3EEiiiiiiiiiiiliiliiiiil', - '_ZL18flash_attn_ext_f16ILi512ELi512ELi32ELi2ELb0ELb0ELb0EEvPKcS1_S1_S1_S1_PKiPfP15HIP_vector_typeIfLj2EEffffjfiS5_IjLj3EEiiiiiiiiiiiliiliiiiil', - '_ZL18flash_attn_ext_f16ILi512ELi512ELi32ELi2ELb1ELb0ELb0EEvPKcS1_S1_S1_S1_PKiPfP15HIP_vector_typeIfLj2EEffffjfiS5_IjLj3EEiiiiiiiiiiiliiliiiiil', - '_ZL18flash_attn_ext_f16ILi512ELi512ELi8ELi8ELb0ELb0ELb0EEvPKcS1_S1_S1_S1_PKiPfP15HIP_vector_typeIfLj2EEffffjfiS5_IjLj3EEiiiiiiiiiiiliiliiiiil', - '_ZL18flash_attn_ext_f16ILi512ELi512ELi8ELi8ELb1ELb0ELb0EEvPKcS1_S1_S1_S1_PKiPfP15HIP_vector_typeIfLj2EEffffjfiS5_IjLj3EEiiiiiiiiiiiliiliiiiil', - '_ZL18flash_attn_ext_f16ILi576ELi512ELi16ELi4ELb0ELb1ELb0EEvPKcS1_S1_S1_S1_PKiPfP15HIP_vector_typeIfLj2EEffffjfiS5_IjLj3EEiiiiiiiiiiiliiliiiiil', - '_ZL18flash_attn_ext_f16ILi576ELi512ELi4ELi16ELb0ELb1ELb0EEvPKcS1_S1_S1_S1_PKiPfP15HIP_vector_typeIfLj2EEffffjfiS5_IjLj3EEiiiiiiiiiiiliiliiiiil', - '_ZL18flash_attn_ext_f16ILi512ELi512ELi16ELi2ELb0ELb0ELb0EEvPKcS1_S1_S1_S1_PKiPfP15HIP_vector_typeIfLj2EEffffjfiS5_IjLj3EEiiiiiiiiiiiliiliiiiil', - '_ZL18flash_attn_ext_f16ILi512ELi512ELi16ELi2ELb1ELb0ELb0EEvPKcS1_S1_S1_S1_PKiPfP15HIP_vector_typeIfLj2EEffffjfiS5_IjLj3EEiiiiiiiiiiiliiliiiiil', - '_ZL18flash_attn_ext_f16ILi512ELi512ELi4ELi8ELb0ELb0ELb0EEvPKcS1_S1_S1_S1_PKiPfP15HIP_vector_typeIfLj2EEffffjfiS5_IjLj3EEiiiiiiiiiiiliiliiiiil', - '_ZL18flash_attn_ext_f16ILi512ELi512ELi4ELi8ELb1ELb0ELb0EEvPKcS1_S1_S1_S1_PKiPfP15HIP_vector_typeIfLj2EEffffjfiS5_IjLj3EEiiiiiiiiiiiliiliiiiil', - '_ZL18flash_attn_ext_f16ILi512ELi512ELi8ELi4ELb0ELb0ELb0EEvPKcS1_S1_S1_S1_PKiPfP15HIP_vector_typeIfLj2EEffffjfiS5_IjLj3EEiiiiiiiiiiiliiliiiiil', - '_ZL18flash_attn_ext_f16ILi512ELi512ELi8ELi4ELb1ELb0ELb0EEvPKcS1_S1_S1_S1_PKiPfP15HIP_vector_typeIfLj2EEffffjfiS5_IjLj3EEiiiiiiiiiiiliiliiiiil', - '_ZL18flash_attn_ext_f16ILi576ELi512ELi1ELi32ELb0ELb1ELb0EEvPKcS1_S1_S1_S1_PKiPfP15HIP_vector_typeIfLj2EEffffjfiS5_IjLj3EEiiiiiiiiiiiliiliiiiil', - '_ZL18flash_attn_ext_f16ILi576ELi512ELi2ELi16ELb0ELb1ELb0EEvPKcS1_S1_S1_S1_PKiPfP15HIP_vector_typeIfLj2EEffffjfiS5_IjLj3EEiiiiiiiiiiiliiliiiiil', - '_ZL18flash_attn_ext_f16ILi576ELi512ELi8ELi4ELb0ELb1ELb0EEvPKcS1_S1_S1_S1_PKiPfP15HIP_vector_typeIfLj2EEffffjfiS5_IjLj3EEiiiiiiiiiiiliiliiiiil', - '_ZL18flash_attn_ext_vecILi256ELi1EL9ggml_type2ELS0_2ELb0EEvPKcS2_S2_S2_S2_PKiPfP15HIP_vector_typeIfLj2EEffffjfiS6_IjLj3EEiiiiiiiiiiiliiliiiiil', - '_ZL18flash_attn_ext_f16ILi320ELi256ELi2ELi32ELb0ELb0ELb0EEvPKcS1_S1_S1_S1_PKiPfP15HIP_vector_typeIfLj2EEffffjfiS5_IjLj3EEiiiiiiiiiiiliiliiiiil', - '_ZL18flash_attn_ext_f16ILi576ELi512ELi1ELi16ELb0ELb1ELb1EEvPKcS1_S1_S1_S1_PKiPfP15HIP_vector_typeIfLj2EEffffjfiS5_IjLj3EEiiiiiiiiiiiliiliiiiil', + '_ZL18flash_attn_ext_f16ILi576ELi512ELi2ELi32ELb0ELb1ELb0EEvPKcS1_S1_S1_S1_PKiS3_PfP15HIP_vector_typeIfLj2EEffffjfiS5_IjLj3EEiiiiiiiiiiiliiliiiiil', + '_ZL18flash_attn_ext_f16ILi512ELi512ELi16ELi4ELb0ELb0ELb0EEvPKcS1_S1_S1_S1_PKiS3_PfP15HIP_vector_typeIfLj2EEffffjfiS5_IjLj3EEiiiiiiiiiiiliiliiiiil', + '_ZL18flash_attn_ext_f16ILi512ELi512ELi16ELi4ELb1ELb0ELb0EEvPKcS1_S1_S1_S1_PKiS3_PfP15HIP_vector_typeIfLj2EEffffjfiS5_IjLj3EEiiiiiiiiiiiliiliiiiil', + '_ZL18flash_attn_ext_f16ILi512ELi512ELi32ELi2ELb0ELb0ELb0EEvPKcS1_S1_S1_S1_PKiS3_PfP15HIP_vector_typeIfLj2EEffffjfiS5_IjLj3EEiiiiiiiiiiiliiliiiiil', + '_ZL18flash_attn_ext_f16ILi512ELi512ELi32ELi2ELb1ELb0ELb0EEvPKcS1_S1_S1_S1_PKiS3_PfP15HIP_vector_typeIfLj2EEffffjfiS5_IjLj3EEiiiiiiiiiiiliiliiiiil', + '_ZL18flash_attn_ext_f16ILi512ELi512ELi8ELi8ELb0ELb0ELb0EEvPKcS1_S1_S1_S1_PKiS3_PfP15HIP_vector_typeIfLj2EEffffjfiS5_IjLj3EEiiiiiiiiiiiliiliiiiil', + '_ZL18flash_attn_ext_f16ILi512ELi512ELi8ELi8ELb1ELb0ELb0EEvPKcS1_S1_S1_S1_PKiS3_PfP15HIP_vector_typeIfLj2EEffffjfiS5_IjLj3EEiiiiiiiiiiiliiliiiiil', + '_ZL18flash_attn_ext_f16ILi576ELi512ELi16ELi4ELb0ELb1ELb0EEvPKcS1_S1_S1_S1_PKiS3_PfP15HIP_vector_typeIfLj2EEffffjfiS5_IjLj3EEiiiiiiiiiiiliiliiiiil', + '_ZL18flash_attn_ext_f16ILi576ELi512ELi4ELi16ELb0ELb1ELb0EEvPKcS1_S1_S1_S1_PKiS3_PfP15HIP_vector_typeIfLj2EEffffjfiS5_IjLj3EEiiiiiiiiiiiliiliiiiil', + '_ZL18flash_attn_ext_f16ILi512ELi512ELi16ELi2ELb0ELb0ELb0EEvPKcS1_S1_S1_S1_PKiS3_PfP15HIP_vector_typeIfLj2EEffffjfiS5_IjLj3EEiiiiiiiiiiiliiliiiiil', + '_ZL18flash_attn_ext_f16ILi512ELi512ELi16ELi2ELb1ELb0ELb0EEvPKcS1_S1_S1_S1_PKiS3_PfP15HIP_vector_typeIfLj2EEffffjfiS5_IjLj3EEiiiiiiiiiiiliiliiiiil', + '_ZL18flash_attn_ext_f16ILi512ELi512ELi4ELi8ELb0ELb0ELb0EEvPKcS1_S1_S1_S1_PKiS3_PfP15HIP_vector_typeIfLj2EEffffjfiS5_IjLj3EEiiiiiiiiiiiliiliiiiil', + '_ZL18flash_attn_ext_f16ILi512ELi512ELi4ELi8ELb1ELb0ELb0EEvPKcS1_S1_S1_S1_PKiS3_PfP15HIP_vector_typeIfLj2EEffffjfiS5_IjLj3EEiiiiiiiiiiiliiliiiiil', + '_ZL18flash_attn_ext_f16ILi512ELi512ELi8ELi4ELb0ELb0ELb0EEvPKcS1_S1_S1_S1_PKiS3_PfP15HIP_vector_typeIfLj2EEffffjfiS5_IjLj3EEiiiiiiiiiiiliiliiiiil', + '_ZL18flash_attn_ext_f16ILi512ELi512ELi8ELi4ELb1ELb0ELb0EEvPKcS1_S1_S1_S1_PKiS3_PfP15HIP_vector_typeIfLj2EEffffjfiS5_IjLj3EEiiiiiiiiiiiliiliiiiil', + '_ZL18flash_attn_ext_f16ILi576ELi512ELi1ELi32ELb0ELb1ELb0EEvPKcS1_S1_S1_S1_PKiS3_PfP15HIP_vector_typeIfLj2EEffffjfiS5_IjLj3EEiiiiiiiiiiiliiliiiiil', + '_ZL18flash_attn_ext_f16ILi576ELi512ELi2ELi16ELb0ELb1ELb0EEvPKcS1_S1_S1_S1_PKiS3_PfP15HIP_vector_typeIfLj2EEffffjfiS5_IjLj3EEiiiiiiiiiiiliiliiiiil', + '_ZL18flash_attn_ext_f16ILi576ELi512ELi8ELi4ELb0ELb1ELb0EEvPKcS1_S1_S1_S1_PKiS3_PfP15HIP_vector_typeIfLj2EEffffjfiS5_IjLj3EEiiiiiiiiiiiliiliiiiil', + '_ZL18flash_attn_ext_vecILi256ELi1EL9ggml_type2ELS0_2ELb0EEvPKcS2_S2_S2_S2_PKiS4_PfP15HIP_vector_typeIfLj2EEffffjfiS6_IjLj3EEiiiiiiiiiiiliiliiiiil', + '_ZL18flash_attn_ext_f16ILi320ELi256ELi2ELi32ELb0ELb0ELb0EEvPKcS1_S1_S1_S1_PKiS3_PfP15HIP_vector_typeIfLj2EEffffjfiS5_IjLj3EEiiiiiiiiiiiliiliiiiil', + '_ZL18flash_attn_ext_f16ILi576ELi512ELi1ELi16ELb0ELb1ELb1EEvPKcS1_S1_S1_S1_PKiS3_PfP15HIP_vector_typeIfLj2EEffffjfiS5_IjLj3EEiiiiiiiiiiiliiliiiiil', } functions = parse_log_file(log_file) diff --git a/scripts/jinja/jinja-tester.py b/scripts/jinja/jinja-tester.py index 6d36ecfa575d..f37350ec1170 100755 --- a/scripts/jinja/jinja-tester.py +++ b/scripts/jinja/jinja-tester.py @@ -241,10 +241,10 @@ def highlight_current_line(self): if not self.isReadOnly(): selection = QTextEdit.ExtraSelection() line_color = QColorConstants.Yellow.lighter(160) - selection.format.setBackground(line_color) # pyright: ignore[reportAttributeAccessIssue] # ty: ignore[unresolved-attribute] - selection.format.setProperty(QTextFormat.Property.FullWidthSelection, True) # pyright: ignore[reportAttributeAccessIssue] # ty: ignore[unresolved-attribute] - selection.cursor = self.textCursor() # pyright: ignore[reportAttributeAccessIssue] # ty: ignore[unresolved-attribute] - selection.cursor.clearSelection() # pyright: ignore[reportAttributeAccessIssue] # ty: ignore[unresolved-attribute] + selection.format.setBackground(line_color) # pyright: ignore[reportAttributeAccessIssue] # ty: ignore[missing-attribute] + selection.format.setProperty(QTextFormat.Property.FullWidthSelection, True) # pyright: ignore[reportAttributeAccessIssue] # ty: ignore[missing-attribute] + selection.cursor = self.textCursor() # pyright: ignore[reportAttributeAccessIssue] # ty: ignore[missing-attribute] + selection.cursor.clearSelection() # pyright: ignore[reportAttributeAccessIssue] # ty: ignore[missing-attribute] extra_selections.append(selection) self.setExtraSelections(extra_selections) @@ -262,8 +262,8 @@ def highlight_position(self, lineno: int, col: int, color: QColor): ) extra = QTextEdit.ExtraSelection() - extra.format.setBackground(color.lighter(160)) # pyright: ignore[reportAttributeAccessIssue] # ty: ignore[unresolved-attribute] - extra.cursor = cursor # pyright: ignore[reportAttributeAccessIssue] # ty: ignore[unresolved-attribute] + extra.format.setBackground(color.lighter(160)) # pyright: ignore[reportAttributeAccessIssue] # ty: ignore[missing-attribute] + extra.cursor = cursor # pyright: ignore[reportAttributeAccessIssue] # ty: ignore[missing-attribute] self.setExtraSelections(self.extraSelections() + [extra]) @@ -274,8 +274,8 @@ def highlight_line(self, lineno: int, color: QColor): cursor.select(QTextCursor.SelectionType.LineUnderCursor) extra = QTextEdit.ExtraSelection() - extra.format.setBackground(color.lighter(160)) # pyright: ignore[reportAttributeAccessIssue] # ty: ignore[unresolved-attribute] - extra.cursor = cursor # pyright: ignore[reportAttributeAccessIssue] # ty: ignore[unresolved-attribute] + extra.format.setBackground(color.lighter(160)) # pyright: ignore[reportAttributeAccessIssue] # ty: ignore[missing-attribute] + extra.cursor = cursor # pyright: ignore[reportAttributeAccessIssue] # ty: ignore[missing-attribute] self.setExtraSelections(self.extraSelections() + [extra]) diff --git a/scripts/moe-cache-tune.py b/scripts/moe-cache-tune.py new file mode 100755 index 000000000000..d0680fa74300 --- /dev/null +++ b/scripts/moe-cache-tune.py @@ -0,0 +1,630 @@ +#!/usr/bin/env python3 +# Picks --moe-cache-mib and --moe-cache-cpu-share for a llama-server command (see docs/moe-cache.md): +# python3 scripts/moe-cache-tune.py [--max-mib N] -- llama-server -m model.gguf -cmoe [args] +# Runs the server at 2 cache sizes x 3 CPU shares, fits the time per token, and replays the recorded +# routing through a copy of the cache logic (src/llama-moe-cache.cpp) to predict other sizes and shares. + +import argparse +import json +import logging +import math +import os +import re +import shlex +import signal +import socket +import struct +import subprocess +import sys +import time +import urllib.request + +logger = logging.getLogger("moe-cache-tune") + +MAX_BATCH = 32 # llama_moe_cache::impl::max_batch +SHARES = [0.0, 0.5, 0.8, 0.9, 0.95, 0.98] +CALIB_SHARES = [0.0, 0.5, 0.95] +N_FIRST = 2048 # tokens per request, first run +N_CALIB = 512 # tokens per request, other runs +PROMPT = ("The lighthouse keeper had kept a logbook for thirty-one years, and in all that time he had never once written down a lie. " + "So when the ship appeared on the horizon on the first calm morning of October, with no flag, no lights and no one on deck, " + "he sat at his small wooden table for a long while before he picked up the pen. Below him the sea was flat and grey, and the " + "gulls had gone quiet. He wrote the date, the hour and the direction of the wind, and then he stopped, because the next line " + "would have to say what he had seen, and he was no longer sure that he had seen it at all. This is what he finally wrote:") + + +def f32(x): + return struct.unpack(" budget: + return 0 + lo, hi = 0, len(g.layers) * g.n_expert + while lo < hi: + mid = (lo + hi + 1) // 2 + if alloc(mid) <= budget: + lo = mid + else: + hi = mid - 1 + return lo if lo >= n_expert_used else 0 + + +def full_mib(trace, share): + """smallest cache that holds all experts""" + _, n_expert_used, groups, _ = trace + n_zero = min(n_expert_used, 64) if f32(share) > 0.0 else 0 + res = 0.0 + for g in groups: + alloc = sum(pad(nb2 * (len(g.layers) * g.n_expert + n_zero + 1), g.alignment) for nb2 in g.nb2) + res = max(res, alloc / (g.split / g.split_sum * g.host_bytes / g.dev_host_bytes)) + return math.ceil(res / 2**20) + 1 + + +class ClassLRU: + """moe_cache_lru, with its float32 rounding of the CRF""" + N_CLASS = 8 + CRF_HALF_LIFE = 128.0 # tokens + + def __init__(self, layers, n_expert, n_slots): + self.n_expert = n_expert + self.n_slots = n_slots + self.slot_map = {il: [n_slots] * n_expert for il in layers} + self.key_of = [-1] * n_slots + self.prev = [s - 1 for s in range(n_slots)] + self.next = [s + 1 if s + 1 < n_slots else -1 for s in range(n_slots)] + self.head = [0] + [-1] * (self.N_CLASS - 1) + self.tail = [n_slots - 1] + [-1] * (self.N_CLASS - 1) + self.list_of = [0] * n_slots + self.n_filled = 0 + self.crf = {} # key -> CRF, 0 before the first use + self.last_use = {} # key -> layer clock at the last use + self.clock = {il: 0 for il in layers} + self.seen = [0] * n_expert + self.seen_gen = 0 + + def move(self, s, c): + frm = self.list_of[s] + if frm == c and s == self.tail[frm]: + return + prev, nxt, head, tail = self.prev, self.next, self.head, self.tail + if prev[s] >= 0: + nxt[prev[s]] = nxt[s] + else: + head[frm] = nxt[s] + if nxt[s] >= 0: + prev[nxt[s]] = prev[s] + else: + tail[frm] = prev[s] + prev[s] = tail[c] + nxt[s] = -1 + if tail[c] >= 0: + nxt[tail[c]] = s + else: + head[c] = s + tail[c] = s + self.list_of[s] = c + + def class_of(self, key): + return min(self.N_CLASS - 1, int(f32(math.log2(self.crf[key])))) # (int) log2f + + def decayed_crf(self, il, key): + crf = self.crf.get(key, 0.0) + if crf == 0.0: + return 0.0 + x = f32(f32(-(self.clock[il] - self.last_use[key])) / self.CRF_HALF_LIFE) + return f32(crf * f32(2.0 ** x)) + + def use(self, il, key): + self.crf[key] = f32(self.decayed_crf(il, key) + 1.0) + self.last_use[key] = self.clock[il] + + def is_selected(self, il, key): + return key >= 0 and key // self.n_expert == il and self.seen[key % self.n_expert] == self.seen_gen + + def victim(self, il): + res, res_score = -1, -1.0 + for c in range(self.N_CLASS): + s = self.head[c] + while s >= 0 and self.is_selected(il, self.key_of[s]): + s = self.next[s] + if s < 0: + continue + key = self.key_of[s] + if key < 0: + return s # unused slot + score = math.ldexp(float(self.clock[key // self.n_expert] - self.last_use[key]), -c) # age/2^class + if score > res_score: + res, res_score = s, score + if res < 0: + raise RuntimeError("no MoE cache victim") + return res + + def plan(self, il, ids, n_used, n_zero, n_cpu): + """returns (uploaded, computed on the CPU)""" + self.seen_gen += 1 + seen, gen = self.seen, self.seen_gen + uniq = [] + for e in ids: + if seen[e] != gen: + seen[e] = gen + uniq.append(e) + if len(uniq) > self.n_slots: + raise RuntimeError(f"layer {il}: {len(uniq)} experts for {self.n_slots} slots") + + slots = self.slot_map[il] + n_slots = self.n_slots + n_tokens = len(ids) // n_used + misses = [] + for e in uniq: + key = il * self.n_expert + e + self.use(il, key) + if slots[e] < n_slots: + self.move(slots[e], self.class_of(key)) + else: + misses.append(e) + + # the misses with the lowest CRF go to the CPU + n_c = min(n_cpu(len(misses)), len(misses)) + n_cpu_done = 0 + if n_c > 0: + misses.sort(key=lambda e: (self.crf[il * self.n_expert + e], e)) + # distinct zero slots per token, a miss without one is uploaded + token_zero = [0] * n_tokens + n_keep = n_c + i = 0 + while i < n_keep: + e = misses[i] + toks = [it for it in range(n_tokens) if e in ids[it * n_used:(it + 1) * n_used]] + used = 0 + for it in toks: + used |= token_zero[it] + z = ((~used) & (used + 1)).bit_length() - 1 + if z >= n_zero: + n_keep -= 1 + misses[i], misses[n_keep] = misses[n_keep], misses[i] + continue + for it in toks: + token_zero[it] |= 1 << z + slots[e] = n_slots + z + n_cpu_done += 1 + i += 1 + del misses[:n_keep] + + misses.sort() + for e in misses: + s = self.victim(il) + key = self.key_of[s] + if key >= 0: + self.slot_map[key // self.n_expert][key % self.n_expert] = n_slots + else: + self.n_filled += 1 + key = il * self.n_expert + e + self.key_of[s] = key + slots[e] = s + self.move(s, self.class_of(key)) + self.clock[il] += n_tokens + return len(misses), n_cpu_done + + +def replay(trace, size_mib, share, skip_tokens): + """GiB of experts uploaded and computed on the CPU per token, after skip_tokens""" + n_layer, n_expert_used, groups, records = trace + share = f32(share) + share_q = int(math.floor(f32(share * 65536.0) + 0.5)) # std::lround(cpu_share*65536.0f) + n_zero = min(n_expert_used, 64) if share > 0.0 else 0 + caches, group_of = [], {} + for ig, g in enumerate(groups): + n = n_slots_of(g, size_mib, n_zero, n_expert_used) + caches.append(ClassLRU(g.layers, g.n_expert, n) if n > 0 else None) + group_of.update((il, ig) for il in g.layers) + + first = min(il for il, _, _ in records) + acc = [0] * n_layer + tokens = seen = up = cpu = 0 + for il, n_used, ids in records: + n_tokens = len(ids) // n_used + if il == first: + seen += n_tokens + if seen > skip_tokens: + tokens += n_tokens + count = seen > skip_tokens + g, c = groups[group_of[il]], caches[group_of[il]] + if c is None or n_tokens > MAX_BATCH or min(n_tokens * n_used, g.n_expert) > c.n_slots: + if count: + cpu += len(set(ids)) * g.expert_bytes # not cached + continue + + def n_cpu(n_miss): + v = n_miss * (share_q if c.n_filled == c.n_slots else 0) + acc[il] # the share applies once the cache is full + acc[il] = v & 0xFFFF + return v >> 16 + + n_fill, n_c = c.plan(il, ids, n_used, n_zero, n_cpu) + if count: + up += n_fill * g.expert_bytes + cpu += n_c * g.expert_bytes + tokens = max(tokens, 1) + return up / 2**30 / tokens, cpu / 2**30 / tokens + + +# time model: ms/token = base + up*up_ms + cpu*cpu_ms + split*(share > 0) + overlap*min(up, cpu), up and cpu in GiB/token + +def model_ms(m, up, cpu, share): + return m["base"] + m["up"] * up + m["cpu"] * cpu + (m["split"] if share > 0 else 0.0) + m["overlap"] * min(up, cpu) + + +def solve(A, b): + """least squares by the normal equations""" + n = len(A[0]) + M: list = [[sum(A[k][i] * A[k][j] for k in range(len(A))) for j in range(n)] + [sum(A[k][i] * b[k] for k in range(len(A)))] for i in range(n)] + for i in range(n): + p = max(range(i, n), key=lambda r: abs(M[r][i])) + M[i], M[p] = M[p], M[i] + if abs(M[i][i]) < 1e-12: + sys.exit("the runs do not determine the time model") + for r in range(n): + if r != i: + f = M[r][i] / M[i][i] + M[r] = [x - f * y for x, y in zip(M[r], M[i])] + return [M[i][n] / M[i][i] for i in range(n)] + + +def fit(runs): + """returns the model, the relative error per run and whether the model is physical""" + y = [r["ms"] / r["tokens"] for r in runs] + rows = [[1.0, r["up_gib"], r["cpu_gib"], float(r["share"] > 0), min(r["up_gib"], r["cpu_gib"])] for r in runs] + + def solve_model(cols): + coef = dict(zip(cols, solve([[row[i] for i in cols.values()] for row in rows], y))) + if coef.get("split", 0) < 0: # noise + del cols["split"] + return solve_model(cols) + return {k: coef.get(k, 0.0) for k in ("base", "up", "cpu", "split", "overlap")} + + def physical(m): + # overlap hides at most the cheaper path, the rest of the token is a good part of the fastest run + return m["up"] > 0 and m["cpu"] > 0 and m["base"] >= 0.2 * min(y) and -m["overlap"] <= min(m["up"], m["cpu"]) + + m = solve_model({"base": 0, "up": 1, "cpu": 2, "split": 3, "overlap": 4}) + if not physical(m): + m = solve_model({"base": 0, "up": 1, "cpu": 2, "split": 3}) + return m, [(model_ms(m, r["up_gib"], r["cpu_gib"], r["share"]) - yy) / yy for r, yy in zip(runs, y)], physical(m) + + +# output + +FMT = {"markdown": False, "tty": False} + + +def say(text=""): + print(text) # noqa: NP100 + + +def best(text): + if FMT["markdown"]: + return f"**{text}**" + return f"\033[1m{text}\033[0m" if FMT["tty"] else f"{text}*" + + +def code(text): + return f"`{text}`" if FMT["markdown"] else text + + +def table(header, rows): + rows = [[str(c) for c in row] for row in rows] + if FMT["markdown"]: + return "\n".join(["| " + " | ".join(header) + " |", "|" + "---:|" * len(header)] + ["| " + " | ".join(row) + " |" for row in rows]) + + def width(text): + return len(re.sub(r"\033\[[0-9;]*m", "", text)) + + w = [max(width(c) for c in col) for col in zip(header, *rows)] + return "\n".join(" " + " ".join(" " * (wi - width(c)) + c for c, wi in zip(row, w)) for row in [header] + rows) + + +# model file + +GGUF_SCALAR = {0: "B", 1: "b", 2: "H", 3: "h", 4: "I", 5: "i", 6: "f", 7: "?", 10: "Q", 11: "q", 12: "d"} +EXPS_RE = re.compile(r"\.ffn_(up|down|gate|gate_up)_(ch|)exps") # LLM_FFN_EXPS_REGEX + + +def model_files(path): + m = re.match(r"(.*)-(\d{5})-of-(\d{5})\.gguf$", path) + return [f"{m.group(1)}-{i:05d}-of-{m.group(3)}.gguf" for i in range(1, int(m.group(3)) + 1)] if m else [path] + + +def gguf_tensor_sizes(path): + """tensor sizes from the data offsets""" + with open(path, "rb") as f: + def take(fmt): + return struct.unpack("<" + fmt, f.read(struct.calcsize("<" + fmt))) + + def string(): + (n,) = take("Q") + return f.read(n).decode("utf-8", "replace") + + def value(t): + if t == 8: + return string() + if t == 9: + et, n = take("IQ") + if et in (8, 9): + for _ in range(n): + value(et) + else: + f.seek(n * struct.calcsize("<" + GGUF_SCALAR[et]), 1) + return None + return take(GGUF_SCALAR[t])[0] + + if f.read(4) != b"GGUF": + sys.exit(f"{path}: not a GGUF file") + take("I") + n_tensors, n_kv = take("QQ") + kv = {} + for _ in range(n_kv): + key = string() + kv[key] = value(take("I")[0]) + infos = [] + for _ in range(n_tensors): + name = string() + take(f"{take('I')[0]}Q") + take("I") + infos.append((take("Q")[0], name)) + data = pad(f.tell(), kv.get("general.alignment", 32)) + end = os.path.getsize(path) - data + infos.sort() + return {name: (infos[i + 1][0] if i + 1 < len(infos) else end) - off for i, (off, name) in enumerate(infos)} + + +def expert_mib(path): + sizes = {} + for p in model_files(path): + sizes.update(gguf_tensor_sizes(p)) + return sum(v for k, v in sizes.items() if EXPS_RE.search(k)) / 2**20 + + +# server runs + +def http(port, endpoint, body=None, timeout=3600): + req = urllib.request.Request(f"http://127.0.0.1:{port}{endpoint}", data=json.dumps(body).encode() if body else None, + headers={"Content-Type": "application/json"}) + with urllib.request.urlopen(req, timeout=timeout) as resp: + return json.loads(resp.read()) + + +def cache_stats(log): + """the statistics the cache logs at exit (-lv 4)""" + r = {"hits": 0, "misses": 0, "cpu": 0, "up_mib": 0.0} + for m in re.finditer(r"llama_moe_cache: ubatch .. 8: hits = (\d+), misses = (\d+)(?: \((\d+) on the CPU\))?, hit rate = [0-9.]+%, uploaded = ([0-9.]+) MiB", log): + r["hits"] += int(m.group(1)) + r["misses"] += int(m.group(2)) + r["cpu"] += int(m.group(3) or 0) + r["up_mib"] += float(m.group(4)) + return r + + +def device_room(log): + """host experts, cache device, its free memory at load and the memory the model uses on it""" + host = sum(float(x) for x in re.findall(r"MoE cache size =\s+[0-9.]+ MiB for ([0-9.]+) MiB of host experts", log)) + m = re.search(r"(\S+) MoE cache size =", log) + if host == 0 or not m: + sys.exit("no MoE cache: the experts must be in host memory (-cmoe, -ncmoe or -ot)") + free = {d: int(x) for d, x in re.findall(r"using device (\S+) \(.*- (\d+) MiB free", log)} + used = sum(float(x) for d, x in re.findall(r"(\S+) +(?:model|KV|RS|compute|output) +buffer size =\s+([0-9.]+) MiB", log) if d == m.group(1)) + if m.group(1) not in free: + sys.exit(f"the free memory of {m.group(1)} is not in the log, pass --max-mib") + return host, m.group(1), free[m.group(1)], used + + +def run_server(cmd, out, mib, share, n_predict, seed, label, warm): + """two generations with the cache, reused if already in out""" + base = os.path.join(out, f"run-{mib}-{share:g}") + if os.path.isfile(base + ".json") and os.path.isfile(base + ".trace"): + r = json.load(open(base + ".json")) + else: + warm() + with socket.socket() as sock: + sock.bind(("127.0.0.1", 0)) + port = sock.getsockname()[1] + args = cmd + ["--host", "127.0.0.1", "--port", str(port), "-lv", "4", "--moe-cache-mib", str(mib), "--moe-cache-cpu-share", str(share)] + timings: list = [] + for attempt in range(2): + with open(base + ".log", "w") as log: + proc = subprocess.Popen(args, stdout=log, stderr=subprocess.STDOUT, env=dict(os.environ, LLAMA_MOE_CACHE_TRACE=base + ".trace")) + try: + while True: + if proc.poll() is not None: + sys.exit(f"llama-server exited ({proc.returncode}), see {base}.log") + try: + if http(port, "/health", timeout=5).get("status") == "ok": + break + except OSError: + pass + time.sleep(1) + n = min(n_predict, http(port, "/props")["default_generation_settings"]["n_ctx"] - 256) + timings = [http(port, "/completion", {"prompt": PROMPT, "n_predict": n, "ignore_eos": True, "cache_prompt": False, + "temperature": 0.8, "top_p": 0.95, "seed": seed + k})["timings"] for k in (0, 1)] + finally: + proc.send_signal(signal.SIGINT) + try: + proc.wait(timeout=120) + except subprocess.TimeoutExpired: + proc.kill() + # the second generation has a warmer cache, it is much slower only if the machine is busy + if timings[1]["predicted_per_second"] > timings[0]["predicted_per_second"] / 1.5 or attempt == 1: + break + logger.info(" slower second generation, machine busy? running again") + r = cache_stats(open(base + ".log", errors="replace").read()) + r.update(mib=mib, share=share, ms=sum(t["predicted_ms"] for t in timings), tokens=sum(t["predicted_n"] for t in timings)) + json.dump(r, open(base + ".json", "w")) + logger.info(f"{label} {mib} MiB, share {share:g}: {1000 * r['tokens'] / r['ms']:.1f} t/s") + r["hit"] = 100 * r["hits"] / max(r["hits"] + r["misses"], 1) + return r + + +def tune(cmd, out, max_mib): + model = next((cmd[i + 1] for i, a in enumerate(cmd[:-1]) if a in ("-m", "--model")), None) + if model is None: + sys.exit("the server command needs -m") + os.makedirs(out, exist_ok=True) + command = os.path.join(out, "command.txt") + if os.path.isfile(command) and open(command).read() != shlex.join(cmd): + for f in os.listdir(out): + os.remove(os.path.join(out, f)) + open(command, "w").write(shlex.join(cmd)) + + load_mode = next((cmd[i + 1] for i, a in reversed(list(enumerate(cmd[:-1]))) if a in ("-lm", "--load-mode")), os.environ.get("LLAMA_ARG_LOAD_MODE", "auto")) + warmed = [] + + def warm(): + # read an mmap'd model once, so that no run reads it from disk + if not warmed and load_mode in ("auto", "mmap", "mmap+mlock"): + for p in model_files(model): + with open(p, "rb") as f: + while f.read(64 << 20): + pass + warmed.append(True) + + def rnd(x): + return max(64, int(x) // 64 * 64) + + n = 2 * len(CALIB_SHARES) + small = rnd(0.10 * expert_mib(model)) + runs = [run_server(cmd, out, small, 0.0, N_FIRST, 1, f"1/{n}", warm)] + host, dev, free, used = device_room(open(os.path.join(out, f"run-{small}-0.log"), errors="replace").read()) + room = int(free - used - 512) + cap = min(max_mib, room) if max_mib else room + large = rnd(min(0.30 * host, room)) + if large <= small: + sys.exit(f"no room on {dev} for a cache larger than {small} MiB") + for i, (mib, share) in enumerate([(small, s) for s in CALIB_SHARES[1:]] + [(large, s) for s in CALIB_SHARES]): + runs.append(run_server(cmd, out, mib, share, N_CALIB, 2 * i + 3, f"{i + 2}/{n}", warm)) + + trace = concat_traces([os.path.join(out, f"run-{r['mib']}-{r['share']:g}.trace") for r in runs]) + groups = trace[2] + gib = sum(g.expert_bytes * len(g.layers) for g in groups) / sum(len(g.layers) for g in groups) / 2**30 + for r in runs: + r["up_gib"], r["cpu_gib"] = r["up_mib"] / 1024 / r["tokens"], r["cpu"] * gib / r["tokens"] + m, errs, ok = fit(runs) + + full = full_mib(trace, max(SHARES)) + top = min(cap, full) + sizes = sorted({rnd(min(f * host, top)) for f in (0.05, 0.10, 0.20, 0.30, 0.40, 0.50)} | {small, large, full if cap >= full else rnd(top)}) + skip = trace_tokens(trace) // 2 # warm-up + rows, pick = [], {} + for mib in [s for s in sizes if s <= top]: + ms = [model_ms(m, *replay(trace, mib, share, skip), share) for share in SHARES] + k = min(i for i in range(len(ms)) if ms[i] <= 1.05 * min(ms)) # lowest share within 5% of the fastest + rows.append([mib] + [best(f"{1000 / x:.1f}") if i == k else f"{1000 / x:.1f}" for i, x in enumerate(ms)]) + pick[mib] = (SHARES[k], 1000 / ms[k], 1000 / ms[0]) + + say(f"{code(os.path.basename(model))}: {host:.0f} MiB of experts, {dev} {free} MiB free, cache up to {cap} MiB") + say() + say(table(["cache MiB", "share", "t/s", "hits"], [[r["mib"], f"{r['share']:g}", f"{1000 * r['tokens'] / r['ms']:.1f}", f"{r['hit']:.0f}%"] for r in runs])) + say() + say(f"machine: uploads {1000 / m['up']:.1f} GiB/s, CPU {1000 / m['cpu']:.1f} GiB/s, CPU part {m['split']:.2f} ms, overlap {m['overlap']:.1f} ms/GiB," + f" rest {m['base']:.2f} ms per token, fit within {100 * max(abs(e) for e in errs):.1f}%") + say() + say("predicted t/s, warm cache:") + say(table(["cache MiB"] + [f"{s:g}" for s in SHARES], rows)) + say() + if not ok or max(abs(e) for e in errs) > 0.10: + sys.exit(f"the runs do not fit the time model, was the machine busy? delete {out} and run again") + if cap >= full: + say(f"all experts fit on {dev} ({full} MiB): drop {code('-cmoe')}, or tune a smaller cache with {code('--max-mib')}") + return + share, tps, tps0 = pick[max(pick)] + say(f"recommended: {code(f'--moe-cache-mib {max(pick)} --moe-cache-cpu-share {share:g}')} ({tps:.1f} t/s" + (f", {tps0:.1f} with share 0)" if share > 0 else ")")) + + +def main(): + ap = argparse.ArgumentParser(description="picks --moe-cache-mib and --moe-cache-cpu-share for a llama-server command", + usage="%(prog)s [--max-mib N] [--out DIR] [--markdown] -- llama-server -m model.gguf -cmoe [args]") + ap.add_argument("--max-mib", type=int, help="largest cache in MiB (default: what fits)") + ap.add_argument("--out", default="moe-cache-tune", help="run directory, reused for the same command (default: moe-cache-tune)") + ap.add_argument("--markdown", action="store_true", help="Markdown tables") + ap.add_argument("cmd", nargs=argparse.REMAINDER, help="llama-server command") + args = ap.parse_args() + cmd = args.cmd[1:] if args.cmd[:1] == ["--"] else args.cmd + if not cmd: + ap.error("pass the llama-server command after --") + logging.basicConfig(level=logging.INFO, format="%(message)s") + FMT["markdown"] = args.markdown + FMT["tty"] = sys.stdout.isatty() and not args.markdown + tune(cmd, args.out, args.max_mib) + + +if __name__ == "__main__": + main() diff --git a/scripts/snapdragon/build.py b/scripts/snapdragon/build.py index 05e51516742d..1bf35b08bf14 100755 --- a/scripts/snapdragon/build.py +++ b/scripts/snapdragon/build.py @@ -106,6 +106,7 @@ def main(): parser.add_argument("--no-docker", action="store_true", help="Build natively on the host instead of in a docker container") parser.add_argument("--preset", help="Override the CMake preset to use") parser.add_argument("--debug", action="store_true", help="Build in debug mode (uses -debug presets instead of -release)") + parser.add_argument("--hex-debug", "--hexagon-debug", action="store_true", help="Enable Hexagon HTP debug output (sets GGML_HEXAGON_HTP_DEBUG=ON)") parser.add_argument("--hex-only", "--hexagon-only", action="store_true", help="Only build and push ggml-hexagon and libggml-htp libraries") parser.add_argument("--hex-arch", choices=["v73", "v75", "v79", "v81", "v85", "73", "75", "79", "81", "85"], help="Specific Hexagon DSP architecture to build with --hex-only (default: all)") @@ -197,13 +198,15 @@ def main(): else: hex_targets = ["ggml-hexagon", "htp-v73", "htp-v75", "htp-v79", "htp-v81", "htp-v85"] + hex_debug_arg = f"-DGGML_HEXAGON_HTP_DEBUG={'ON' if args.hex_debug else 'OFF'}" + if args.no_docker: # Native/local host build logger.info("Running native/local CMake build...") install_prefix = os.path.join(repo_root, install_dir, "llama.cpp") # Configure - configure_cmd = ["cmake", f"--preset={preset}", "-B", build_dir] + configure_cmd = ["cmake", f"--preset={preset}", hex_debug_arg, "-B", build_dir] logger.info(f"+ {' '.join(configure_cmd)}") res = subprocess.run(configure_cmd, cwd=repo_root) if res.returncode != 0: @@ -238,7 +241,7 @@ def main(): target_args = f"--target {' '.join(hex_targets)} " if args.hex_only else "" build_sh_cmd = ( - f"cmake --preset {preset} -B /workspace/{build_dir} && " + f"cmake --preset {preset} {hex_debug_arg} -B /workspace/{build_dir} && " f"cmake --build /workspace/{build_dir} {target_args}-j {jobs} && " f"cmake --install /workspace/{build_dir} --prefix {install_prefix_container}" ) @@ -261,6 +264,17 @@ def main(): logger.error("Docker-based build failed.") sys.exit(res.returncode) + install_lib_dir = os.path.join(repo_root, install_dir, "llama.cpp", "lib") + if os.path.isdir(install_lib_dir): + if args.hex_debug: + farf_src = os.path.join(repo_root, "scripts", "snapdragon", "adb", "llama-cli.farf") + farf_apps = ["llama-server", "llama-cli", "llama-completion", "test-backend-ops", "test-llama-archs"] + for app in farf_apps: + shutil.copyfile(farf_src, os.path.join(install_lib_dir, f"{app}.farf")) + else: + for f in glob.glob(os.path.join(install_lib_dir, "*.farf")): + os.remove(f) + logger.info("\nBuild and installation completed successfully!") # Push/deploy if requested @@ -286,6 +300,9 @@ def main(): else: hex_libs += glob.glob(os.path.join(lib_dir, "libggml-htp*")) + if args.hex_debug: + hex_libs += glob.glob(os.path.join(lib_dir, "*.farf")) + if not hex_libs: logger.error(f"Error: no hexagon libraries found in {lib_dir} to push.") sys.exit(1) @@ -307,6 +324,10 @@ def main(): if res.returncode != 0: logger.error("ADB mkdir failed.") sys.exit(res.returncode) + if not args.hex_debug: + rm_farf_cmd = adb_cmd + ["shell", f"rm -f {shlex.quote(target_lib_dir)}/*.farf"] + logger.info(f"+ {' '.join(rm_farf_cmd)}") + subprocess.run(rm_farf_cmd) for lib in hex_libs: push_cmd = adb_cmd + ["push", lib, f"{target_lib_dir}/"] logger.info(f"+ {' '.join(push_cmd)}") @@ -383,6 +404,10 @@ def main(): if args.hex_only: logger.info(f"\nDeploying Hexagon libraries to Linux device {ssh_host} via SCP...") target_lib_dir = f"{target_dir}/lib" + if not args.hex_debug: + rm_farf_cmd = ["ssh", ssh_host, f"rm -f {target_lib_dir}/*.farf"] + logger.info(f"+ {' '.join(rm_farf_cmd)}") + subprocess.run(rm_farf_cmd) deploy_cmd = ["scp"] + hex_libs + [f"{ssh_host}:{target_lib_dir}/"] logger.info(f"+ {' '.join(deploy_cmd)}") res = subprocess.run(deploy_cmd) diff --git a/scripts/snapdragon/ggml-hexagon-align-macros.py b/scripts/snapdragon/ggml-hexagon-align-macros.py index b64db3e654a8..dc1475c52f85 100755 --- a/scripts/snapdragon/ggml-hexagon-align-macros.py +++ b/scripts/snapdragon/ggml-hexagon-align-macros.py @@ -6,6 +6,7 @@ align-macros.py [paths...] # Check and report misaligned macros align-macros.py --diff [paths...] # Show unified diff of fixes align-macros.py --fix [paths...] # Fix misaligned macros in-place + align-macros.py --fix --reflow false ... # Disable line reflowing in mostly-aligned macros align-macros.py --fix --mode majority ... # Align to the dominant column align-macros.py --fix --pad 2 ... # Align to (max_content_len + pad) @@ -127,9 +128,227 @@ def is_macro_aligned(macro: MacroDef) -> bool: return len(set(bs_cols)) == 1 and not has_trailing_ws +def str_to_bool(val: str) -> bool: + if isinstance(val, bool): + return val + v = val.strip().lower() + if v in ("true", "1", "yes", "y", "on"): + return True + if v in ("false", "0", "no", "n", "off"): + return False + raise argparse.ArgumentTypeError(f"Boolean value expected, got '{val}'") + + +def find_paren_comma_groups(s: str) -> List[Tuple[int, int, List[int]]]: + """Find parenthesized groups (open_idx, close_idx, comma_indices) ordered from outermost to innermost.""" + stack: List[Tuple[int, List[int]]] = [] + groups: List[Tuple[int, int, List[int]]] = [] + in_str: Optional[str] = None + escape = False + + for idx, ch in enumerate(s): + if escape: + escape = False + continue + if ch == "\\": + escape = True + continue + if in_str is not None: + if ch == in_str: + in_str = None + continue + if ch in ('"', "'"): + in_str = ch + continue + if ch == "(": + stack.append((idx, [])) + elif ch == ")" and stack: + open_idx, commas = stack.pop() + if commas: + groups.append((open_idx, idx, commas)) + elif ch == "," and stack: + stack[-1][1].append(idx) + + groups.sort(key=lambda g: g[0]) + return groups + + +def find_top_level_op_splits(s: str) -> Tuple[Optional[int], List[int]]: + """Find split positions after binary operators (+, -, &&, ||) at paren depth 0.""" + depth = 0 + in_str: Optional[str] = None + escape = False + assign_idx: Optional[int] = None + splits: List[int] = [] + n = len(s) + i = 0 + + while i < n: + ch = s[i] + if escape: + escape = False + i += 1 + continue + if ch == "\\": + escape = True + i += 1 + continue + if in_str is not None: + if ch == in_str: + in_str = None + i += 1 + continue + if ch in ('"', "'"): + in_str = ch + i += 1 + continue + if ch in "([{": + depth += 1 + elif ch in ")]}": + depth = max(0, depth - 1) + elif depth == 0: + if s.startswith("&&", i) or s.startswith("||", i): + splits.append(i + 2) + i += 2 + continue + if ch == "=" and (i == 0 or s[i - 1] not in "=!<>+-*/%&|^") and (i + 1 >= n or s[i + 1] != "="): + if assign_idx is None: + assign_idx = i + elif ch in "+-" and i > 0 and s[i - 1] == " " and i + 1 < n and s[i + 1] == " ": + splits.append(i + 1) + i += 1 + + return assign_idx, splits + + +def reflow_c_line(content: str, max_len: int) -> Optional[List[str]]: + """Try to cleanly reflow a C statement/signature line so each piece has len <= max_len.""" + if len(content) <= max_len: + return [content] + + stripped = content.lstrip() + if not stripped or stripped.startswith("#") or "//" in content or "/*" in content: + return None + + base_indent = content[:len(content) - len(stripped)] + + # 1. Try splitting comma-separated arguments/parameters inside (...) + for open_idx, _, commas in find_paren_comma_groups(content): + pts = [0] + [c + 1 for c in commas] + [len(content)] + segs: List[str] = [] + for idx in range(len(pts) - 1): + seg = content[pts[idx]:pts[idx + 1]].rstrip() + if idx > 0: + seg = seg.lstrip() + segs.append(seg) + + for cont_indent in [" " * (open_idx + 1), base_indent + " ", base_indent + " "]: + lines = [segs[0]] + for seg in segs[1:]: + cand = f"{lines[-1]} {seg}" + if len(cand) <= max_len: + lines[-1] = cand + else: + lines.append(f"{cont_indent}{seg}") + if len(lines) > 1 and all(len(line) <= max_len for line in lines): + return lines + + # 2. Try splitting after top-level binary operators (+, -, &&, ||) or assignment (=) + assign_idx, op_splits = find_top_level_op_splits(content) + if not op_splits and assign_idx is not None: + op_splits = [assign_idx + 1] + if op_splits: + pts = [0] + op_splits + [len(content)] + segs = [] + for idx in range(len(pts) - 1): + seg = content[pts[idx]:pts[idx + 1]].rstrip() + if idx > 0: + seg = seg.lstrip() + segs.append(seg) + + indents = [] + if assign_idx is not None: + rhs_start = assign_idx + 1 + while rhs_start < len(content) and content[rhs_start] == " ": + rhs_start += 1 + indents.append(" " * rhs_start) + indents.extend([base_indent + " ", base_indent + " "]) + + for cont_indent in indents: + lines = [segs[0]] + for seg in segs[1:]: + cand = f"{lines[-1]} {seg}" + if len(cand) <= max_len: + lines[-1] = cand + else: + lines.append(f"{cont_indent}{seg}") + if len(lines) > 1 and all(len(line) <= max_len for line in lines): + return lines + + return None + + +def reflow_macro_if_needed( + macro: MacroDef, + pad: int, + dom_ratio_threshold: float = 0.80, + fit_ratio_threshold: float = 0.90, +) -> Tuple[MacroDef, Optional[int]]: + """ + If a macro is already mostly aligned at a dominant column (>= dom_ratio_threshold) + and >= fit_ratio_threshold of continuation lines fit within that column, reflow any + exceeding lines so the macro can keep its dominant column instead of shifting all lines. + """ + cont_lines = [ml for ml in macro.lines if ml.bs_col is not None] + if not cont_lines: + return macro, None + + counts = Counter(ml.bs_col for ml in cont_lines if ml.bs_col is not None) + dom_col, dom_count = sorted(counts.items(), key=lambda x: (-x[1], -x[0]))[0] + max_allowed_len = dom_col - pad + if max_allowed_len <= 0: + return macro, None + + dom_ratio = dom_count / len(cont_lines) + fit_count = sum(1 for ml in cont_lines if len(ml.content) <= max_allowed_len) + fit_ratio = fit_count / len(cont_lines) + + effective_dom_threshold = 0.70 if fit_count == len(cont_lines) else dom_ratio_threshold + if dom_ratio < effective_dom_threshold or fit_ratio < fit_ratio_threshold: + return macro, None + + new_macro_lines: List[MacroLine] = [] + for ml in macro.lines: + if ml.bs_col is None or len(ml.content) <= max_allowed_len: + new_macro_lines.append(ml) + else: + reflowed = reflow_c_line(ml.content, max_allowed_len) + if not reflowed: + return macro, None + nl = "\r\n" if ml.raw.endswith("\r\n") else "\n" + for piece in reflowed: + new_macro_lines.append(MacroLine( + line_num=ml.line_num, + raw=f"{piece} \\{nl}", + content=piece, + bs_col=dom_col, + trailing_ws=False, + )) + + return MacroDef( + name=macro.name, + filepath=macro.filepath, + start_line=macro.start_line, + end_line=macro.end_line, + lines=new_macro_lines, + ), dom_col + + def compute_target_column(macro: MacroDef, mode: str, pad: int, target_col: Optional[int]) -> int: """Determine the column where backslashes should be aligned.""" - max_content_len = max(len(ml.content) for ml in macro.lines) + cont_lines = [ml for ml in macro.lines if ml.bs_col is not None] + lines_for_len = cont_lines if cont_lines else macro.lines + max_content_len = max(len(ml.content) for ml in lines_for_len) min_needed = max_content_len + pad if target_col is not None: @@ -181,6 +400,7 @@ def process_file(filepath: str, args: argparse.Namespace) -> Tuple[int, int, Opt misaligned_count = 0 modified = False new_file_lines = list(file_lines) + line_offset = 0 for macro in macros: aligned = is_macro_aligned(macro) @@ -188,8 +408,9 @@ def process_file(filepath: str, args: argparse.Namespace) -> Tuple[int, int, Opt if not aligned: misaligned_count += 1 - bs_cols = [ml.bs_col for ml in macro.lines if ml.bs_col is not None] - max_content = max(len(ml.content) for ml in macro.lines) + cont_lines = [ml for ml in macro.lines if ml.bs_col is not None] + bs_cols = [ml.bs_col for ml in cont_lines if ml.bs_col is not None] + max_content = max(len(ml.content) for ml in (cont_lines or macro.lines)) col_counts = Counter(bs_cols) if not args.quiet: @@ -200,16 +421,28 @@ def process_file(filepath: str, args: argparse.Namespace) -> Tuple[int, int, Opt if trailing_ws_lines: logger.warning(f" Warning: Trailing whitespace after backslash on line(s): {trailing_ws_lines}") - target_col = compute_target_column(macro, args.mode, args.pad, args.target_col) + reflow_col: Optional[int] = None + target_macro = macro + if args.reflow and args.target_col is None: + target_macro, reflow_col = reflow_macro_if_needed(macro, args.pad) + + if reflow_col is not None: + target_col = reflow_col + if not args.quiet and len(target_macro.lines) != len(macro.lines): + logger.info(f" -> Reflowed {len(target_macro.lines) - len(macro.lines)} line(s) to preserve column {target_col}") + else: + target_col = compute_target_column(target_macro, args.mode, args.pad, args.target_col) + if not args.quiet: logger.info(f" -> Target alignment column: {target_col}") - realigned = realign_macro_lines(macro, target_col) + realigned = realign_macro_lines(target_macro, target_col) - start_idx = macro.start_line - 1 + start_idx = macro.start_line - 1 + line_offset end_idx = start_idx + len(macro.lines) if new_file_lines[start_idx:end_idx] != realigned: new_file_lines[start_idx:end_idx] = realigned + line_offset += len(realigned) - len(macro.lines) modified = True diff_text = None @@ -258,6 +491,8 @@ def main(): parser.add_argument("--check", action="store_true", help="Exit with code 1 if misaligned macros exist") parser.add_argument("--mode", choices=["min", "max", "majority"], default="min", help="Alignment mode: 'min' (max_len + pad), 'max' (max existing col), 'majority' (dominant col)") + parser.add_argument("--reflow", type=str_to_bool, nargs="?", const=True, default=True, + help="Reflow long lines in mostly-aligned macros instead of shifting the whole macro (default: true)") parser.add_argument("--pad", type=int, default=2, help="Spaces between longest line and backslash (default: 2)") parser.add_argument("--target-col", type=int, default=None, help="Force alignment to an exact column") parser.add_argument("--all", action="store_true", help="Realign all macros even if already aligned (default: only misaligned)") diff --git a/scripts/snapdragon/ggml-hexagon-profile.py b/scripts/snapdragon/ggml-hexagon-profile.py index 7c95ec8cba9b..f6a949e983d4 100755 --- a/scripts/snapdragon/ggml-hexagon-profile.py +++ b/scripts/snapdragon/ggml-hexagon-profile.py @@ -466,10 +466,11 @@ def print_bubbles_timeline(op): logger.info(f" Avg Thread DMA IDLE : {avg_dma_idle:.1f}%") logger.info("-" * 100) + hmx_threads = {e['thread'] for e in events if e['event'].startswith('HMX')} logger.info(" Per-Thread Idle Analysis:") for t in active_threads: stats = thread_stats[t] - thread_name = f"Thread {t:<2} (HVX)" if t != 10 else "Thread 10 (HMX)" + thread_name = f"Thread {t:<2} (HMX)" if t in hmx_threads else f"Thread {t:<2} (HVX)" logger.info(f" {thread_name} -> Compute Idle: {stats['compute_idle_pct']:.1f}% | DMA Idle: {stats['dma_idle_pct']:.1f}%") all_bubbles = [] @@ -531,8 +532,9 @@ def print_ascii_summary(op_name, dims, types, usec, cycles, events): norm_evt = normalize_event_name(evt) thread_totals[t][norm_evt] += dur + hmx_threads = {e['thread'] for e in events if e['event'].startswith('HMX')} for t in sorted(thread_totals.keys()): - thread_name = f"Thread {t} (HVX)" if t != 10 else "Thread 10 (HMX)" + thread_name = f"Thread {t} (HMX)" if t in hmx_threads else f"Thread {t} (HVX)" sorted_evts = sorted(thread_totals[t].items(), key=lambda item: item[0]) evt_strs = [] diff --git a/scripts/snapdragon/ggml-hexagon-trace.py b/scripts/snapdragon/ggml-hexagon-trace.py index 5241dca15311..308591b83f80 100755 --- a/scripts/snapdragon/ggml-hexagon-trace.py +++ b/scripts/snapdragon/ggml-hexagon-trace.py @@ -433,7 +433,7 @@ def generate_perfetto_trace(filtered_ops, trace_events, output_path): time_mappers = {dev: DeviceTimeMapper(dev, filtered_ops) for dev in unique_devices} # Process events - completed_events = [] + completed_events: List[Dict[str, Any]] = [] if trace_events: trace_events = sorted(trace_events, key=lambda e: e['unwrapped_cycles']) @@ -516,6 +516,9 @@ def generate_perfetto_trace(filtered_ops, trace_events, output_path): dur_ns = tm.dur_cycles_to_ns(e['start_cyc'], e['end_cyc'] - e['start_cyc']) e['dur_ns'] = max(dur_ns, 1) + # Identify HMX threads dynamically from HMX* events + hmx_threads = {(e['device'], e['thread']) for e in completed_events if e['event'].startswith('HMX')} + # Allocate slots (sub-tracks) to prevent overlaps on same virtual track active_slots = defaultdict(list) for e in completed_events: @@ -528,7 +531,7 @@ def generate_perfetto_trace(filtered_ops, trace_events, output_path): norm_evt = normalize_event_name(evt, e['info']) if norm_evt == "DMA": track_key = (dev, t, "DMA") - elif t == 10: + elif (dev, t) in hmx_threads: track_key = (dev, t, "HMX") else: track_key = (dev, t, "HVX") @@ -552,23 +555,24 @@ def generate_perfetto_trace(filtered_ops, trace_events, output_path): evt = e['event'] slot = e['slot'] dev = e['device'] + is_hmx = (dev, t) in hmx_threads norm_evt = normalize_event_name(evt, e['info']) if norm_evt == "DMA": track_evt = "DMA" evt_id = 1 - elif t == 10: + elif is_hmx: track_evt = "HMX" evt_id = 3 else: track_evt = "HVX" evt_id = 2 - t_sort = 1 if t == 10 else t + 2 + t_sort = 1 if is_hmx else t + 2 dev_idx = device_to_idx[dev] # Unique UUID for each sub-track - if t == 10: + if is_hmx: uuid = dev_idx * 10000000 + 20 # HMX thread track UUID else: uuid = int(dev_idx * 10000000 + t_sort * 1000000 + evt_id * 1000 + slot) @@ -601,9 +605,9 @@ def generate_perfetto_trace(filtered_ops, trace_events, output_path): hmx_packet = make_trace_packet(0, track_descriptor=make_track_descriptor(hmx_track_uuid, parent_uuid=proc_uuid, thread=hmx_thread_desc)) write_trace_packet_to_file(f, hmx_packet) - # Define Thread Tracks (T0, T1, ..., T9) for this device + # Define Thread Tracks for this device dev_used_tracks = {uuid: val for uuid, val in used_tracks.items() if val[0] == dev} - unique_threads = sorted(list(set(t for (_, t, _, _) in dev_used_tracks.values() if t != 10))) + unique_threads = sorted(list(set(t for (_, t, _, _) in dev_used_tracks.values() if (dev, t) not in hmx_threads))) for t in unique_threads: thread_uuid = dev_idx * 10000000 + 10 + t thread_name = f"T{t}" @@ -623,7 +627,7 @@ def generate_perfetto_trace(filtered_ops, trace_events, output_path): for uuid in sorted(used_tracks.keys()): dev, t, evt, slot = used_tracks[uuid] dev_idx = device_to_idx[dev] - if t == 10: + if (dev, t) in hmx_threads: continue name = f"T{t} {evt}" rank = 0 if evt == "HVX" else 1 diff --git a/scripts/snapdragon/run.py b/scripts/snapdragon/run.py index 99b43bb2b347..4336409205d2 100755 --- a/scripts/snapdragon/run.py +++ b/scripts/snapdragon/run.py @@ -33,12 +33,14 @@ "GGML_HEXAGON_MM_SELECT", "GGML_HEXAGON_FA_SELECT", "GGML_HEXAGON_FA_HEAD_SPLIT", + "GGML_HEXAGON_LI_SELECT", "GGML_HEXAGON_GDN_SELECT", "GGML_HEXAGON_AR_SELECT", "GGML_HEXAGON_AR_SCATTER", "GGML_HEXAGON_ETM", "GGML_HEXAGON_ARCH", "GGML_HEXAGON_OPTRACE", + "GGML_HEXAGON_LIBHTP_DIR", "GGML_OPENCL_PLATFORM", "GGML_OPENCL_DEVICE", "GGML_OPENCL_OPFILTER", @@ -49,26 +51,12 @@ "GGML_OPENCL_ADRENO_XMEM_GEMM", "GGML_OPENCL_ADRENO_USE_LARGE_BUFFER", "GGML_SCHED_DEBUG", - "ADSP_LIBRARY_PATH", - "CDSP_LIBRARY_PATH", - "CDSP1_LIBRARY_PATH", - "CDSP2_LIBRARY_PATH", - "CDSP3_LIBRARY_PATH", "MTMD_BACKEND_DEVICE", "D", "DEVICE", ) -DSP_LIBRARY_ENV_NAMES = ( - "ADSP_LIBRARY_PATH", - "CDSP_LIBRARY_PATH", - "CDSP1_LIBRARY_PATH", - "CDSP2_LIBRARY_PATH", - "CDSP3_LIBRARY_PATH", -) - - def parse_target(target_str): if not target_str: return None, None, None @@ -111,10 +99,7 @@ def default_target_dir(target_type, target_transport, home_dir): def library_env_parts(lib_dir): - env_parts = [f"LD_LIBRARY_PATH={lib_dir}"] - for name in DSP_LIBRARY_ENV_NAMES: - env_parts.append(f"{name}={lib_dir}") - return env_parts + return [f"LD_LIBRARY_PATH={lib_dir}"] def adb_root(adb_base): @@ -177,11 +162,17 @@ def device_arg_from_devices(devices): def normalize_cmd_device_args(cmd_args): + exec_name = os.path.basename(cmd_args[0]) if cmd_args else "" + if exec_name.endswith(".exe"): + exec_name = exec_name[:-4] + for i, arg in enumerate(cmd_args): if arg == "--device" and i + 1 < len(cmd_args): cmd_args[i + 1] = device_arg_from_devices(cmd_args[i + 1]) elif arg.startswith("--device="): cmd_args[i] = "--device=" + device_arg_from_devices(arg.split("=", 1)[1]) + elif exec_name in ("test-backend-ops", "test-llama-archs") and arg in ("-b", "--backend") and i + 1 < len(cmd_args): + cmd_args[i + 1] = device_arg_from_devices(cmd_args[i + 1]) def main(): @@ -229,11 +220,13 @@ def main(): parser.add_argument("--hex-mm-select", help="Select MUL_MAT and MUL_MAT_ID kernel (GGML_HEXAGON_MM_SELECT) 2:HMX,1:HVX,0:disable") parser.add_argument("--hex-fa-select", help="Select Flash Attention kernel (GGML_HEXAGON_FA_SELECT) 2:HMX,1:HVX,0:disable") parser.add_argument("--hex-fa-head-split", help="Enable (1) or disable (0) head-parallel flash_attn partitioning (GGML_HEXAGON_FA_HEAD_SPLIT)") + parser.add_argument("--hex-li-select", help="Select Lightning Indexer kernel (GGML_HEXAGON_LI_SELECT) 2:HMX,1:HVX,0:disable") parser.add_argument("--hex-gdn-select", help="Select Gated Delta Net kernel (GGML_HEXAGON_GDN_SELECT) 2:HMX,1:HVX,0:disable") parser.add_argument("--hex-ar-select", help="Select All-Reduce kernel (GGML_HEXAGON_AR_SELECT) 1:enable,0:disable") parser.add_argument("--hex-ar-scatter", help="Enable (1) or disable (0) reduce-scatter for fused ALLREDUCE+ADD (GGML_HEXAGON_AR_SCATTER)") parser.add_argument("--hex-etm", help="Enable Embedded Trace Macrocell hardware tracing / trace logging (GGML_HEXAGON_ETM)") parser.add_argument("--hex-arch", help="Target Hexagon NPU architecture version override (v73, v75, v79, v81, etc.) (GGML_HEXAGON_ARCH)") + parser.add_argument("--hex-libhtp-dir", default=os.environ.get("GGML_HEXAGON_LIBHTP_DIR"), help="Directory containing libggml-htp-v.so (GGML_HEXAGON_LIBHTP_DIR)") parser.add_argument("--hex-optrace", help="Trace buffer size in number of records (GGML_HEXAGON_OPTRACE)") # OpenCL specific parameters @@ -299,6 +292,10 @@ def set_env(env_name, opt_val): env_vars[env_name] = str(opt_val) # Resolve and filter devices (HTP vs OpenCL) + exec_name = os.path.basename(cmd_args[0]) if cmd_args else "" + if exec_name.endswith(".exe"): + exec_name = exec_name[:-4] + device_in_cmd = None for i, arg in enumerate(cmd_args): if arg == "--device" and i + 1 < len(cmd_args): @@ -307,6 +304,9 @@ def set_env(env_name, opt_val): elif arg.startswith("--device="): device_in_cmd = arg.split("=", 1)[1] break + elif exec_name in ("test-backend-ops", "test-llama-archs") and arg in ("-b", "--backend") and i + 1 < len(cmd_args): + device_in_cmd = cmd_args[i + 1] + break if args.devices is not None: devices_val = args.devices @@ -375,11 +375,13 @@ def set_env(env_name, opt_val): set_env("GGML_HEXAGON_MM_SELECT", args.hex_mm_select) set_env("GGML_HEXAGON_FA_SELECT", args.hex_fa_select) set_env("GGML_HEXAGON_FA_HEAD_SPLIT", args.hex_fa_head_split) + set_env("GGML_HEXAGON_LI_SELECT", args.hex_li_select) set_env("GGML_HEXAGON_GDN_SELECT", args.hex_gdn_select) set_env("GGML_HEXAGON_AR_SELECT", args.hex_ar_select) set_env("GGML_HEXAGON_AR_SCATTER", args.hex_ar_scatter) set_env("GGML_HEXAGON_ETM", args.hex_etm) set_env("GGML_HEXAGON_ARCH", args.hex_arch) + set_env("GGML_HEXAGON_LIBHTP_DIR", args.hex_libhtp_dir) set_env("GGML_HEXAGON_OPTRACE", args.hex_optrace) set_env("MTMD_BACKEND_DEVICE", args.mtmd_device) @@ -402,7 +404,7 @@ def set_env(env_name, opt_val): # Resolve executable path executable = cmd_args[0] - known_binaries = ["llama-cli", "llama-bench", "llama-completion", "llama-mtmd-cli", "test-backend-ops"] + known_binaries = ["llama-cli", "llama-bench", "llama-completion", "llama-mtmd-cli", "test-backend-ops", "test-llama-archs"] if executable in known_binaries: if target_type in ("android", "linux"): resolved_exec = f"./bin/{executable}" @@ -419,22 +421,23 @@ def set_env(env_name, opt_val): basename = basename[:-4] device_val = None - if basename == "test-backend-ops": - for i in range(len(cmd_args)): - if cmd_args[i] in ("-p", "--params") and i + 1 < len(cmd_args): - val = cmd_args[i + 1] - new_val = "" - for j, char in enumerate(val): - if char in ('[', ']'): - if j > 0 and val[j - 1] == '\\': - new_val += char + if basename in ("test-backend-ops", "test-llama-archs"): + if basename == "test-backend-ops": + for i in range(len(cmd_args)): + if cmd_args[i] in ("-p", "--params") and i + 1 < len(cmd_args): + val = cmd_args[i + 1] + new_val = "" + for j, char in enumerate(val): + if char in ('[', ']'): + if j > 0 and val[j - 1] == '\\': + new_val += char + else: + new_val += '\\' + char else: - new_val += '\\' + char - else: - new_val += char - cmd_args[i + 1] = new_val + new_val += char + cmd_args[i + 1] = new_val - has_b = any(arg == "-b" for arg in cmd_args) + has_b = any(arg in ("-b", "--backend") for arg in cmd_args) if not has_b: if args.devices: device_val = device_arg_from_devices(args.devices) @@ -462,9 +465,13 @@ def set_env(env_name, opt_val): or args.hex_profile is not None or args.hex_optrace is not None ) - if verbose_trigger and basename in ("llama-cli", "llama-completion", "llama-bench", "llama-server", "llama-mtmd-cli"): - if "-v" not in cmd_args and "--verbose" not in cmd_args: - cmd_args.append("-v") + if verbose_trigger: + if basename == "test-llama-archs": + if "-v" not in cmd_args: + cmd_args += ["-v", "5"] + elif basename in ("llama-cli", "llama-completion", "llama-bench", "llama-server", "llama-mtmd-cli"): + if "-v" not in cmd_args and "--verbose" not in cmd_args: + cmd_args.append("-v") # Inject defaults for llama-cli, llama-completion, and llama-server if not overridden by the user if basename in ("llama-cli", "llama-completion", "llama-server"): @@ -551,8 +558,6 @@ def set_env(env_name, opt_val): for k in MANAGED_ENV_NAMES: local_env.pop(k, None) - for name in DSP_LIBRARY_ENV_NAMES: - local_env[name] = lib_dir for k, v in env_vars.items(): local_env[k] = v diff --git a/scripts/sync-ggml.last b/scripts/sync-ggml.last index 7ecc9a407565..43167cdceab2 100644 --- a/scripts/sync-ggml.last +++ b/scripts/sync-ggml.last @@ -1 +1 @@ -facdc44a67da17df4433adcc468b424f48e9ded2 +57e2c2ca2a7cae756b1c4dfafa0cdfef9051d234 diff --git a/scripts/sync_vendor.py b/scripts/sync_vendor.py index 7cb535ac56a1..fed4914fff9a 100755 --- a/scripts/sync_vendor.py +++ b/scripts/sync_vendor.py @@ -5,7 +5,7 @@ import sys import subprocess -HTTPLIB_VERSION = "refs/tags/v0.64.0" +HTTPLIB_VERSION = "refs/tags/v0.66.0" # used by examples/gguf-hash, these repos have no release tag, so we pin a commit XXHASH_COMMIT = "9f465f1ea932d6ad9a26cd77496311ffa544cd68" diff --git a/scripts/tool_bench.py b/scripts/tool_bench.py index fb7df10f3bc5..2015f6682fba 100755 --- a/scripts/tool_bench.py +++ b/scripts/tool_bench.py @@ -52,8 +52,8 @@ sys.path.insert(0, Path(__file__).parent.parent.as_posix()) if True: - from tools.server.tests.utils import ServerProcess # ty: ignore[unresolved-import] - from tools.server.tests.unit.test_tool_call import do_test_calc_result, do_test_hello_world, do_test_weather # ty: ignore[unresolved-import] + from tools.server.tests.utils import ServerProcess # ty: ignore[import-not-found] + from tools.server.tests.unit.test_tool_call import do_test_calc_result, do_test_hello_world, do_test_weather # ty: ignore[import-not-found] @contextmanager diff --git a/skills/add-new-model/SKILL.md b/skills/add-new-model/SKILL.md index 710a1ebb4488..3e4d178ebeb7 100644 --- a/skills/add-new-model/SKILL.md +++ b/skills/add-new-model/SKILL.md @@ -50,7 +50,7 @@ Skill-specific addition: before writing `src/models/.cpp`, read at least 1 ## Step 4 - Optional: multimodal encoder -Only do this if the contributor flagged a vision/audio encoder in Step 0. Follow HOWTO-add-model.md section 4 and `docs/multimodal.md` for the actual touch points (`MmprojModel` subclass, `clip.cpp`, `mtmd.cpp`, encoder graph in `tools/mtmd/models`, etc.). +Only do this if the contributor flagged a vision/audio encoder in Step 0. Follow HOWTO-add-model.md section 4 and `docs/multimodal.md` for the actual touch points (`MmprojModel` subclass, `mtmd-model.cpp`, `mtmd-subcontext.cpp`, `mtmd.cpp`, encoder graph in `tools/mtmd/models`, etc.). Skill-specific addition, and read this carefully: **whether the multimodal encoder can be bundled into the same PR as the base text-model support depends on how conventional the change is.** It's OK to bundle it if the encoder support is conventional - i.e. no new infra or logic is needed, it's just a new cgraph reusing existing preprocessing/projector machinery (e.g. siglip/pixtral/qwen with just a new projector). If it requires anything beyond that - a new preprocessor, non-standard projector logic, or changes to shared `libmtmd` infra/logic - STOP, tell the contributor this is non-conventional, and have them land the text model first with the encoder as a dedicated follow-up PR. Do not let this decision pass silently - call it out explicitly to the contributor before writing any `clip.cpp`/`mtmd.cpp` code. diff --git a/skills/code-review/SKILL.md b/skills/code-review/SKILL.md index 4c8e4bbcca03..d72b46712cb9 100644 --- a/skills/code-review/SKILL.md +++ b/skills/code-review/SKILL.md @@ -85,6 +85,7 @@ See the `add-new-model` skill and `docs/development/HOWTO-add-model.md` for the - Hparams that the model can't run correctly without must be mandatory (hard-error if missing), not read with a silent default fallback. Only genuinely-optional-across-configs values get a fallback accessor. - New/optional weight tensors (scales, etc.) must route through `build_lora_mm` and the existing helpers, matching convention - don't leave raw matmuls copied from another arch. - Don't hack RoPE with a custom sin/cos implementation. If `ggml_rope_ext` genuinely can't express it, that's an issue for discussion, not a PR. +- In `src/models/`, apply RoPE with `build_rope(cur, pos, il)`, not a direct `ggml_rope_ext`/`ggml_rope_multi` call (see #31683). Per-layer differences go in a `get_rope_params` override: the KV-cache K-shift reads the same params, so a direct call lets the graph and the shift drift apart. The remaining direct calls (deepseek4/41, dflash, sensenova-u1) are not a pattern to copy. - Test the quantized-KV path (`-ctk`/`-ctv q8_0`), not just default f16 - new speculative/attention features silently break there. - Preserve existing explanatory comments about model-specific quirks when copying code; note the provenance ("copied from X, with Y added"). - Remove dead code/branches left over from adapting a reference implementation. diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 97250c2e8856..426c487411ee 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -69,7 +69,8 @@ target_include_directories(llama PRIVATE . ${CMAKE_CURRENT_BINARY_DIR}) target_include_directories(llama PUBLIC ../include) target_compile_features (llama PRIVATE cxx_std_17) # don't bump -target_link_libraries(llama PUBLIC ggml) +find_package(Threads REQUIRED) +target_link_libraries(llama PUBLIC ggml PRIVATE Threads::Threads) if (BUILD_SHARED_LIBS) set_target_properties(llama PROPERTIES POSITION_INDEPENDENT_CODE ON) diff --git a/src/llama-arch.cpp b/src/llama-arch.cpp index 40f608371c54..6dcf34e77501 100644 --- a/src/llama-arch.cpp +++ b/src/llama-arch.cpp @@ -90,6 +90,7 @@ static const std::map LLM_ARCH_NAMES = { { LLM_ARCH_DEEPSEEK32, "deepseek32" }, { LLM_ARCH_DEEPSEEK4, "deepseek4" }, { LLM_ARCH_DEEPSEEK41, "deepseek41" }, + { LLM_ARCH_XING4_0, "xing4_0" }, { LLM_ARCH_CHATGLM, "chatglm" }, { LLM_ARCH_GLM4, "glm4" }, { LLM_ARCH_GLM4_MOE, "glm4moe" }, @@ -102,6 +103,7 @@ static const std::map LLM_ARCH_NAMES = { { LLM_ARCH_NEMOTRON, "nemotron" }, { LLM_ARCH_NEMOTRON_H, "nemotron_h" }, { LLM_ARCH_NEMOTRON_H_MOE, "nemotron_h_moe" }, + { LLM_ARCH_METIS16, "metis16" }, { LLM_ARCH_EXAONE, "exaone" }, { LLM_ARCH_EXAONE4, "exaone4" }, { LLM_ARCH_EXAONE_MOE, "exaone-moe" }, @@ -166,6 +168,7 @@ static const std::map LLM_ARCH_NAMES = { { LLM_ARCH_MAION_CODER, "maion-coder" }, { LLM_ARCH_KIMI_LINEAR, "kimi-linear" }, { LLM_ARCH_KIMI_K3, "kimi-k3" }, + { LLM_ARCH_KIMI_K31, "kimi-k31" }, { LLM_ARCH_GLM5_NEXT, "glm5-next" }, { LLM_ARCH_TALKIE, "talkie" }, { LLM_ARCH_MELLUM, "mellum" }, @@ -175,6 +178,7 @@ static const std::map LLM_ARCH_NAMES = { { LLM_ARCH_K2_HORIZON, "k2-horizon" }, { LLM_ARCH_BEAM, "beam" }, { LLM_ARCH_KOLIBRI1, "kolibri1" }, + { LLM_ARCH_RICURSIVE, "ricursive" }, { LLM_ARCH_UNKNOWN, "(unknown)" }, }; @@ -224,6 +228,7 @@ static const std::map LLM_KV_NAMES = { { LLM_KV_EXPERT_CHUNK_FEED_FORWARD_LENGTH, "%s.expert_chunk_feed_forward_length" }, { LLM_KV_SWIGLU_CLAMP_EXP, "%s.swiglu_clamp_exp" }, { LLM_KV_SWIGLU_CLAMP_SHEXP, "%s.swiglu_clamp_shexp" }, + { LLM_KV_SWIGLU_CLAMP, "%s.swiglu_clamp" }, { LLM_KV_USE_PARALLEL_RESIDUAL, "%s.use_parallel_residual" }, { LLM_KV_TENSOR_DATA_LAYOUT, "%s.tensor_data_layout" }, { LLM_KV_EXPERT_COUNT, "%s.expert_count" }, @@ -272,6 +277,10 @@ static const std::map LLM_KV_NAMES = { { LLM_KV_HRM_H_CYCLES, "%s.hrm.h_cycles" }, { LLM_KV_HRM_L_CYCLES, "%s.hrm.l_cycles" }, { LLM_KV_HRM_PREFIX_LM, "%s.hrm.prefix_lm" }, + { LLM_KV_METIS_PASS_COUNT, "%s.metis.pass_count" }, + { LLM_KV_METIS_MEMORY_LENGTH, "%s.metis.memory_length" }, + { LLM_KV_METIS_ROUTE_LENGTH, "%s.metis.route_length" }, + { LLM_KV_METIS_CONTINUATION_LOGIT_LIMIT, "%s.metis.continuation_logit_limit" }, { LLM_KV_MUDD_FEED_FORWARD_LENGTH, "%s.mudd.feed_forward_length" }, { LLM_KV_MUDD_TAP_COUNT, "%s.mudd.tap_count" }, { LLM_KV_MUDD_TAP_INDICES, "%s.mudd.tap_indices" }, @@ -307,6 +316,7 @@ static const std::map LLM_KV_NAMES = { { LLM_KV_ATTENTION_TEMPERATURE_LENGTH, "%s.attention.temperature_length" }, { LLM_KV_ATTENTION_TEMPERATURE_SCALE, "%s.attention.temperature_scale" }, { LLM_KV_REALTIME_DELAY_TOKENS, "%s.realtime_delay_tokens" }, + { LLM_KV_CLASSIFIER_FREE_GUIDANCE, "%s.classifier_free_guidance" }, { LLM_KV_ATTENTION_KEY_LENGTH_MLA, "%s.attention.key_length_mla" }, { LLM_KV_ATTENTION_VALUE_LENGTH_MLA, "%s.attention.value_length_mla" }, { LLM_KV_ATTENTION_KEY_LENGTH_SWA, "%s.attention.key_length_swa" }, @@ -329,6 +339,7 @@ static const std::map LLM_KV_NAMES = { { LLM_KV_ATTENTION_KV_SOURCE_LAYERS, "%s.attention.kv_source_layers" }, { LLM_KV_ATTENTION_OUTPUT_GROUP_COUNT, "%s.attention.output_group_count" }, { LLM_KV_ATTENTION_OUTPUT_LORA_RANK, "%s.attention.output_lora_rank" }, + { LLM_KV_ATTENTION_PASS_LORA_RANK, "%s.attention.pass_lora_rank" }, { LLM_KV_ATTENTION_COMPRESS_ROPE_FREQ_BASE, "%s.attention.compress_rope_freq_base" }, { LLM_KV_ATTENTION_COMPRESS_RATIOS, "%s.attention.compress_ratios" }, { LLM_KV_ATTENTION_SHARED_KV_LAYERS, "%s.attention.shared_kv_layers" }, @@ -387,6 +398,7 @@ static const std::map LLM_KV_NAMES = { { LLM_KV_SSM_TIME_STEP_RANK, "%s.ssm.time_step_rank" }, { LLM_KV_SSM_GROUP_COUNT, "%s.ssm.group_count" }, { LLM_KV_SSM_DT_B_C_RMS, "%s.ssm.dt_b_c_rms" }, + { LLM_KV_SSM_NORM_EPS, "%s.ssm.norm_epsilon" }, { LLM_KV_KDA_HEAD_DIM, "%s.kda.head_dim" }, { LLM_KV_KDA_SAFE_GATE, "%s.kda.safe_gate" }, @@ -548,6 +560,36 @@ static const std::map LLM_TENSOR_NAMES = { { LLM_TENSOR_FFN_EXP_PROBS_B_VL, "blk.%d.exp_probs_b_vl" }, { LLM_TENSOR_FFN_LATENT_DOWN, "blk.%d.ffn_latent_down" }, { LLM_TENSOR_FFN_LATENT_UP, "blk.%d.ffn_latent_up" }, + { LLM_TENSOR_STREAM_EMBD, "stream_embd" }, + { LLM_TENSOR_DEPTH_MEM_STATE_WRITE, "depth_mem_state_write" }, + { LLM_TENSOR_DEPTH_MEM_META_WRITE, "depth_mem_meta_write" }, + { LLM_TENSOR_DEPTH_MEM_ENTRY_BIAS, "depth_mem_entry_bias" }, + { LLM_TENSOR_DEPTH_MEM_QUERY, "depth_mem_query" }, + { LLM_TENSOR_DEPTH_MEM_KEY, "depth_mem_key" }, + { LLM_TENSOR_DEPTH_MEM_VALUE, "depth_mem_value" }, + { LLM_TENSOR_DEPTH_MEM_OUT, "depth_mem_out" }, + { LLM_TENSOR_DEPTH_MEM_ROUTE_STATE, "depth_mem_route_state" }, + { LLM_TENSOR_DEPTH_MEM_ROUTE_RETR, "depth_mem_route_retr" }, + { LLM_TENSOR_DEPTH_MEM_ROUTE_DIFF, "depth_mem_route_diff" }, + { LLM_TENSOR_DEPTH_MEM_ROUTE_HIST, "depth_mem_route_hist" }, + { LLM_TENSOR_DEPTH_MEM_ROUTE, "depth_mem_route" }, + { LLM_TENSOR_DEPTH_MEM_STREAM_GATE, "depth_mem_stream_gate" }, + { LLM_TENSOR_CONT_HIDDEN_STATE, "cont_hidden_state" }, + { LLM_TENSOR_CONT_HIDDEN_RETR, "cont_hidden_retr" }, + { LLM_TENSOR_CONT_HIDDEN_DIFF, "cont_hidden_diff" }, + { LLM_TENSOR_CONT_HIDDEN_ROUTE, "cont_hidden_route" }, + { LLM_TENSOR_CONT_HIDDEN, "cont_hidden" }, + { LLM_TENSOR_CONT_OUTPUT, "cont_output" }, + { LLM_TENSOR_HC_ATTN_MIX, "blk.%d.hc_attn_mix" }, + { LLM_TENSOR_HC_ATTN_READ, "blk.%d.hc_attn_read" }, + { LLM_TENSOR_HC_ATTN_WRITE, "blk.%d.hc_attn_write" }, + { LLM_TENSOR_HC_FFN_MIX, "blk.%d.hc_ffn_mix" }, + { LLM_TENSOR_HC_FFN_READ, "blk.%d.hc_ffn_read" }, + { LLM_TENSOR_HC_FFN_WRITE, "blk.%d.hc_ffn_write" }, + { LLM_TENSOR_ATTN_PASS_LORA_DOWN, "blk.%d.attn_pass_lora_down" }, + { LLM_TENSOR_ATTN_PASS_LORA_UP, "blk.%d.attn_pass_lora_up" }, + { LLM_TENSOR_ATTN_PASS_GATE, "blk.%d.attn_pass_gate" }, + { LLM_TENSOR_FFN_EXP_EMBD, "blk.%d.ffn_exp_embd" }, { LLM_TENSOR_FFN_PRE_EXP_NORM, "blk.%d.ffn_pre_exp_norm" }, { LLM_TENSOR_FFN_POST_EXP_NORM, "blk.%d.ffn_post_exp_norm" }, { LLM_TENSOR_FFN_RESID_ALPHA, "blk.%d.ffn_resid_alpha" }, @@ -556,6 +598,7 @@ static const std::map LLM_TENSOR_NAMES = { { LLM_TENSOR_ATTN_QKV, "blk.%d.attn_qkv" }, { LLM_TENSOR_LAYER_OUT_NORM, "blk.%d.layer_output_norm" }, { LLM_TENSOR_LAYER_OUT_SCALE, "blk.%d.layer_output_scale" }, + { LLM_TENSOR_ATTN_VRES_LAMBDA, "blk.%d.attn_vres_lambda" }, { LLM_TENSOR_ATTN_OUT_NORM, "blk.%d.attn_output_norm" }, { LLM_TENSOR_POS_EMBD, "position_embd" }, { LLM_TENSOR_FFN_ACT, "blk.%d.ffn.act" }, @@ -1061,6 +1104,7 @@ static const std::map LLM_TENSOR_INFOS = { {LLM_TENSOR_ATTN_K_NORM, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}}, {LLM_TENSOR_LAYER_OUT_NORM, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}}, {LLM_TENSOR_LAYER_OUT_SCALE, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}}, + {LLM_TENSOR_ATTN_VRES_LAMBDA, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}}, {LLM_TENSOR_ATTN_Q_A_NORM, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}}, {LLM_TENSOR_ATTN_KV_A_NORM, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}}, {LLM_TENSOR_ATTN_SUB_NORM, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}}, @@ -1158,6 +1202,36 @@ static const std::map LLM_TENSOR_INFOS = { // latent projections feed ggml_mul_mat, the buft probe must use MUL_MAT to keep them on GPU {LLM_TENSOR_FFN_LATENT_DOWN, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}}, {LLM_TENSOR_FFN_LATENT_UP, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}}, + {LLM_TENSOR_STREAM_EMBD, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_ADD}}, + {LLM_TENSOR_DEPTH_MEM_STATE_WRITE, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}}, + {LLM_TENSOR_DEPTH_MEM_META_WRITE, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}}, + {LLM_TENSOR_DEPTH_MEM_ENTRY_BIAS, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}}, + {LLM_TENSOR_DEPTH_MEM_QUERY, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}}, + {LLM_TENSOR_DEPTH_MEM_KEY, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}}, + {LLM_TENSOR_DEPTH_MEM_VALUE, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}}, + {LLM_TENSOR_DEPTH_MEM_OUT, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}}, + {LLM_TENSOR_DEPTH_MEM_ROUTE_STATE, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}}, + {LLM_TENSOR_DEPTH_MEM_ROUTE_RETR, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}}, + {LLM_TENSOR_DEPTH_MEM_ROUTE_DIFF, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}}, + {LLM_TENSOR_DEPTH_MEM_ROUTE_HIST, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}}, + {LLM_TENSOR_DEPTH_MEM_ROUTE, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}}, + {LLM_TENSOR_DEPTH_MEM_STREAM_GATE, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}}, + {LLM_TENSOR_CONT_HIDDEN_STATE, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}}, + {LLM_TENSOR_CONT_HIDDEN_RETR, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}}, + {LLM_TENSOR_CONT_HIDDEN_DIFF, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}}, + {LLM_TENSOR_CONT_HIDDEN_ROUTE, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}}, + {LLM_TENSOR_CONT_HIDDEN, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}}, + {LLM_TENSOR_CONT_OUTPUT, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}}, + {LLM_TENSOR_HC_ATTN_MIX, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}}, + {LLM_TENSOR_HC_ATTN_READ, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}}, + {LLM_TENSOR_HC_ATTN_WRITE, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}}, + {LLM_TENSOR_HC_FFN_MIX, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}}, + {LLM_TENSOR_HC_FFN_READ, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}}, + {LLM_TENSOR_HC_FFN_WRITE, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}}, + {LLM_TENSOR_ATTN_PASS_LORA_DOWN, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}}, + {LLM_TENSOR_ATTN_PASS_LORA_UP, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}}, + {LLM_TENSOR_ATTN_PASS_GATE, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}}, + {LLM_TENSOR_FFN_EXP_EMBD, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_GET_ROWS}}, {LLM_TENSOR_FFN_PRE_EXP_NORM, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}}, {LLM_TENSOR_FFN_POST_EXP_NORM, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}}, {LLM_TENSOR_FFN_RESID_ALPHA, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}}, @@ -1286,10 +1360,12 @@ bool llm_arch_is_hybrid(const llm_arch & arch) { case LLM_ARCH_LFM3: case LLM_ARCH_NEMOTRON_H: case LLM_ARCH_NEMOTRON_H_MOE: + case LLM_ARCH_METIS16: case LLM_ARCH_QWEN3NEXT: case LLM_ARCH_KIMI_LINEAR: case LLM_ARCH_BAILINGMOE3: case LLM_ARCH_KIMI_K3: + case LLM_ARCH_KIMI_K31: case LLM_ARCH_GLM5_NEXT: case LLM_ARCH_QWEN35: case LLM_ARCH_QWEN35MOE: @@ -1319,6 +1395,7 @@ bool llm_arch_is_diffusion(const llm_arch & arch) { bool llm_arch_supports_rs_rollback(const llm_arch & arch) { switch (arch) { case LLM_ARCH_KIMI_K3: + case LLM_ARCH_KIMI_K31: case LLM_ARCH_QWEN35: case LLM_ARCH_QWEN35MOE: case LLM_ARCH_CLOVER_MOE: @@ -1373,6 +1450,7 @@ bool llm_arch_supports_sm_tensor(const llm_arch & arch) { case LLM_ARCH_DEEPSEEK32: case LLM_ARCH_HY_V4: case LLM_ARCH_DOTS3NOTE: + case LLM_ARCH_XING4_0: case LLM_ARCH_GLM_DSA: case LLM_ARCH_BITNET: case LLM_ARCH_T5: @@ -1390,7 +1468,7 @@ bool llm_arch_supports_sm_tensor(const llm_arch & arch) { case LLM_ARCH_KIMI_LINEAR: case LLM_ARCH_BAILINGMOE3: case LLM_ARCH_KIMI_K3: - case LLM_ARCH_GLM5_NEXT: + case LLM_ARCH_KIMI_K31: case LLM_ARCH_QWEN3TTS: case LLM_ARCH_LIMITE: return false; diff --git a/src/llama-arch.h b/src/llama-arch.h index b98be40cf29b..3b8f45d26f9b 100644 --- a/src/llama-arch.h +++ b/src/llama-arch.h @@ -94,6 +94,7 @@ enum llm_arch { LLM_ARCH_DEEPSEEK32, LLM_ARCH_DEEPSEEK4, LLM_ARCH_DEEPSEEK41, + LLM_ARCH_XING4_0, LLM_ARCH_CHATGLM, LLM_ARCH_GLM4, LLM_ARCH_GLM4_MOE, @@ -106,6 +107,7 @@ enum llm_arch { LLM_ARCH_NEMOTRON, LLM_ARCH_NEMOTRON_H, LLM_ARCH_NEMOTRON_H_MOE, + LLM_ARCH_METIS16, LLM_ARCH_EXAONE, LLM_ARCH_EXAONE4, LLM_ARCH_EXAONE_MOE, @@ -164,6 +166,7 @@ enum llm_arch { LLM_ARCH_MAION_CODER, LLM_ARCH_KIMI_LINEAR, LLM_ARCH_KIMI_K3, + LLM_ARCH_KIMI_K31, LLM_ARCH_GLM5_NEXT, LLM_ARCH_TALKIE, LLM_ARCH_MELLUM, @@ -179,6 +182,7 @@ enum llm_arch { LLM_ARCH_BEAM, LLM_ARCH_KOLIBRI1, LLM_ARCH_LIMITE, + LLM_ARCH_RICURSIVE, LLM_ARCH_UNKNOWN, }; @@ -228,6 +232,7 @@ enum llm_kv { LLM_KV_EXPERT_CHUNK_FEED_FORWARD_LENGTH, LLM_KV_SWIGLU_CLAMP_EXP, LLM_KV_SWIGLU_CLAMP_SHEXP, + LLM_KV_SWIGLU_CLAMP, LLM_KV_USE_PARALLEL_RESIDUAL, LLM_KV_TENSOR_DATA_LAYOUT, LLM_KV_EXPERT_COUNT, @@ -276,7 +281,12 @@ enum llm_kv { LLM_KV_HRM_H_CYCLES, LLM_KV_HRM_L_CYCLES, LLM_KV_HRM_PREFIX_LM, + LLM_KV_METIS_PASS_COUNT, + LLM_KV_METIS_MEMORY_LENGTH, + LLM_KV_METIS_ROUTE_LENGTH, + LLM_KV_METIS_CONTINUATION_LOGIT_LIMIT, LLM_KV_REALTIME_DELAY_TOKENS, + LLM_KV_CLASSIFIER_FREE_GUIDANCE, LLM_KV_MUDD_FEED_FORWARD_LENGTH, LLM_KV_MUDD_TAP_COUNT, LLM_KV_MUDD_TAP_INDICES, @@ -333,6 +343,7 @@ enum llm_kv { LLM_KV_ATTENTION_KV_SOURCE_LAYERS, LLM_KV_ATTENTION_OUTPUT_GROUP_COUNT, LLM_KV_ATTENTION_OUTPUT_LORA_RANK, + LLM_KV_ATTENTION_PASS_LORA_RANK, LLM_KV_ATTENTION_COMPRESS_ROPE_FREQ_BASE, LLM_KV_ATTENTION_COMPRESS_RATIOS, LLM_KV_ATTENTION_SHARED_KV_LAYERS, @@ -391,6 +402,7 @@ enum llm_kv { LLM_KV_SSM_TIME_STEP_RANK, LLM_KV_SSM_GROUP_COUNT, LLM_KV_SSM_DT_B_C_RMS, + LLM_KV_SSM_NORM_EPS, LLM_KV_KDA_HEAD_DIM, LLM_KV_KDA_SAFE_GATE, @@ -566,6 +578,36 @@ enum llm_tensor { LLM_TENSOR_FFN_EXP_PROBS_B_VL, LLM_TENSOR_FFN_LATENT_DOWN, LLM_TENSOR_FFN_LATENT_UP, + LLM_TENSOR_STREAM_EMBD, + LLM_TENSOR_DEPTH_MEM_STATE_WRITE, + LLM_TENSOR_DEPTH_MEM_META_WRITE, + LLM_TENSOR_DEPTH_MEM_ENTRY_BIAS, + LLM_TENSOR_DEPTH_MEM_QUERY, + LLM_TENSOR_DEPTH_MEM_KEY, + LLM_TENSOR_DEPTH_MEM_VALUE, + LLM_TENSOR_DEPTH_MEM_OUT, + LLM_TENSOR_DEPTH_MEM_ROUTE_STATE, + LLM_TENSOR_DEPTH_MEM_ROUTE_RETR, + LLM_TENSOR_DEPTH_MEM_ROUTE_DIFF, + LLM_TENSOR_DEPTH_MEM_ROUTE_HIST, + LLM_TENSOR_DEPTH_MEM_ROUTE, + LLM_TENSOR_DEPTH_MEM_STREAM_GATE, + LLM_TENSOR_CONT_HIDDEN_STATE, + LLM_TENSOR_CONT_HIDDEN_RETR, + LLM_TENSOR_CONT_HIDDEN_DIFF, + LLM_TENSOR_CONT_HIDDEN_ROUTE, + LLM_TENSOR_CONT_HIDDEN, + LLM_TENSOR_CONT_OUTPUT, + LLM_TENSOR_HC_ATTN_MIX, + LLM_TENSOR_HC_ATTN_READ, + LLM_TENSOR_HC_ATTN_WRITE, + LLM_TENSOR_HC_FFN_MIX, + LLM_TENSOR_HC_FFN_READ, + LLM_TENSOR_HC_FFN_WRITE, + LLM_TENSOR_ATTN_PASS_LORA_DOWN, + LLM_TENSOR_ATTN_PASS_LORA_UP, + LLM_TENSOR_ATTN_PASS_GATE, + LLM_TENSOR_FFN_EXP_EMBD, LLM_TENSOR_FFN_PRE_EXP_NORM, LLM_TENSOR_FFN_POST_EXP_NORM, LLM_TENSOR_FFN_RESID_ALPHA, @@ -574,6 +616,7 @@ enum llm_tensor { LLM_TENSOR_ATTN_K_NORM, LLM_TENSOR_LAYER_OUT_NORM, LLM_TENSOR_LAYER_OUT_SCALE, + LLM_TENSOR_ATTN_VRES_LAMBDA, LLM_TENSOR_POST_ATTN_NORM, LLM_TENSOR_POST_MLP_NORM, LLM_TENSOR_PER_LAYER_TOKEN_EMBD, // gemma3n diff --git a/src/llama-batch.cpp b/src/llama-batch.cpp index 321aa9d23647..7fce6624f670 100644 --- a/src/llama-batch.cpp +++ b/src/llama-batch.cpp @@ -52,19 +52,38 @@ bool llama_batch_allocr::init( // an entry can carry a token id, a token embedding, or both // all entries must carry the same combination, or be a mix of token and embd entries // a state embedding (e.g. MTP hook batches) is set on all entries or on none + // an embd row (embd read from a device buffer) is set on all entries or on none, with no token id or embedding // int32_t n_tok_only = 0; int32_t n_embd_only = 0; int32_t n_both = 0; - const bool has_state = batch_inp.tokens[0].has_state; + const bool has_state = batch_inp.tokens[0].has_state; + const bool has_row = batch_inp.tokens[0].embd_row >= 0; + const bool has_state_row = batch_inp.tokens[0].state_row >= 0; + + if (has_state && has_state_row) { + LLAMA_LOG_ERROR("%s: the entries have a state embedding and a state row\n", __func__); + return false; + } for (int32_t i = 0; i < n_tok; ++i) { const bool is_tok = batch_inp.tokens[i].id != LLAMA_TOKEN_NULL; const bool is_emb = batch_inp.tokens[i].has_embd; + const bool is_row = batch_inp.tokens[i].embd_row >= 0; + + if (is_row != has_row) { + LLAMA_LOG_ERROR("%s: all entries in the batch must have the same embd row presence\n", __func__); + return false; + } - if (!is_tok && !is_emb) { + if (is_row) { + if (is_tok || is_emb) { + LLAMA_LOG_ERROR("%s: entry %d has an embd row and a token id or embedding\n", __func__, i); + return false; + } + } else if (!is_tok && !is_emb) { LLAMA_LOG_ERROR("%s: entry %d has neither a token id nor an embedding\n", __func__, i); return false; } @@ -74,6 +93,11 @@ bool llama_batch_allocr::init( return false; } + if ((batch_inp.tokens[i].state_row >= 0) != has_state_row) { + LLAMA_LOG_ERROR("%s: all entries in the batch must have the same state row presence\n", __func__); + return false; + } + n_tok_only += is_tok && !is_emb; n_embd_only += is_emb && !is_tok; n_both += is_tok && is_emb; @@ -137,6 +161,20 @@ bool llama_batch_allocr::init( state_vec = batch_inp.state; } + if (has_row) { + embd_row_vec.resize(n_tok); + for (int32_t i = 0; i < n_tok; ++i) { + embd_row_vec[i] = batch_inp.tokens[i].embd_row; + } + } + + if (has_state_row) { + state_row_vec.resize(n_tok); + for (int32_t i = 0; i < n_tok; ++i) { + state_row_vec[i] = batch_inp.tokens[i].state_row; + } + } + // // build flat pos array, section-major: pos[j*n_tok + i] = section j of entry i // token entry: [p, p, p, 0] (M-RoPE text position) @@ -306,6 +344,7 @@ bool llama_batch_allocr::init( /*.token =*/ batch.token, /*.embd =*/ batch.embd, /*.embd_state =*/ state_vec.empty() ? nullptr : state_vec.data(), + /*.embd_row =*/ embd_row_vec.empty() ? nullptr : embd_row_vec.data(), /*.pos =*/ batch.pos, /*.n_seq_id =*/ batch.n_seq_id, /*.seq_id =*/ batch.seq_id, @@ -531,6 +570,7 @@ llama_ubatch llama_batch_allocr::ubatch_reserve(uint32_t n_seq_tokens, uint32_t /*.token =*/ udata->token.data(), /*.embd =*/ nullptr, /*.embd_state =*/ nullptr, + /*.embd_row =*/ nullptr, /*.pos =*/ udata->pos.data(), /*.n_seq_id =*/ udata->n_seq_id.data(), /*.seq_id =*/ udata->seq_id.data(), @@ -838,6 +878,8 @@ void llama_batch_allocr::clear() { embd_vec .clear(); is_embd_vec .clear(); state_vec .clear(); + embd_row_vec.clear(); + state_row_vec.clear(); seq_id_data .clear(); pos .clear(); n_seq_id .clear(); @@ -881,6 +923,8 @@ llama_ubatch llama_batch_allocr::ubatch_add(const std::vector & idxs, u const bool use_token = batch.token && !(mixed_batch && n_embd_rows == n_tokens); const bool use_embd = batch.embd && !(mixed_batch && n_embd_rows == 0); const bool has_state = !state_vec.empty(); + const bool has_row = !embd_row_vec.empty(); + const bool has_srow = !state_row_vec.empty(); const int64_t n_embd_all = use_embd ? (int64_t) n_tokens*n_embd : 0; const int64_t n_state_all = has_state ? (int64_t) n_tokens*n_embd_state : 0; @@ -889,6 +933,8 @@ llama_ubatch llama_batch_allocr::ubatch_add(const std::vector & idxs, u udata->token .resize(n_tokens); udata->embd .resize(n_embd_all); udata->embd_state.resize(n_state_all); + udata->embd_row .resize(has_row ? n_tokens : 0); + udata->embd_state_row.resize(has_srow ? n_tokens : 0); udata->pos .resize(n_pos_all); udata->n_seq_id .resize(n_tokens); udata->seq_id .resize(n_tokens); @@ -920,6 +966,14 @@ llama_ubatch llama_batch_allocr::ubatch_add(const std::vector & idxs, u memcpy(udata->embd_state.data() + i*n_embd_state, state_vec.data() + (int64_t) idxs[i]*n_embd_state, n_embd_state*sizeof(float)); } + if (has_row) { + udata->embd_row[i] = embd_row_vec[idxs[i]]; + } + + if (has_srow) { + udata->embd_state_row[i] = state_row_vec[idxs[i]]; + } + for (size_t j = 0; j < (size_t)n_pos_per_embd; ++j) { udata->pos[j*n_tokens + i] = batch.pos[j*batch.n_tokens + idxs[i]]; } @@ -967,6 +1021,7 @@ llama_ubatch llama_batch_allocr::ubatch_add(const std::vector & idxs, u /*.token =*/ use_token ? udata->token.data() : nullptr, /*.embd =*/ use_embd ? udata->embd.data() : nullptr, /*.embd_state =*/ has_state ? udata->embd_state.data() : nullptr, + /*.embd_row =*/ has_row ? udata->embd_row.data() : nullptr, /*.pos =*/ udata->pos.data(), /*.n_seq_id =*/ udata->n_seq_id.data(), /*.seq_id =*/ udata->seq_id.data(), @@ -1019,6 +1074,7 @@ void llama_batch_allocr::ubatch_print(const llama_ubatch & ubatch, int debug) { LLAMA_LOG_DEBUG("%s: token = %p\n", __func__, (void *) ubatch.token); LLAMA_LOG_DEBUG("%s: embd = %p\n", __func__, (void *) ubatch.embd); LLAMA_LOG_DEBUG("%s: embd_state = %p\n", __func__, (void *) ubatch.embd_state); + LLAMA_LOG_DEBUG("%s: embd_row = %p\n", __func__, (void *) ubatch.embd_row); LLAMA_LOG_DEBUG("%s: pos = %p\n", __func__, (void *) ubatch.pos); LLAMA_LOG_DEBUG("%s: n_seq_id = %p\n", __func__, (void *) ubatch.n_seq_id); LLAMA_LOG_DEBUG("%s: seq_id = %p\n", __func__, (void *) ubatch.seq_id); @@ -1303,6 +1359,43 @@ bool llama_batch_ext::set_token_state(int32_t idx, llama_embd state_in) { return true; } +bool llama_batch_ext::set_token_embd_row(int32_t idx, int32_t row) { + if (idx < 0 || idx >= (int32_t) tokens.size()) { + return false; + } + if (row < 0) { + return false; + } + + token & t = tokens[idx]; + + if (t.id != LLAMA_TOKEN_NULL || t.has_embd || t.embd_row >= 0) { + LLAMA_LOG_ERROR("%s: entry %d already has a token id, an embedding or an embd row\n", __func__, idx); + return false; + } + + t.embd_row = row; + + return true; +} + +bool llama_batch_ext::set_token_state_row(int32_t idx, int32_t row) { + if (idx < 0 || idx >= (int32_t) tokens.size() || row < 0) { + return false; + } + + token & t = tokens[idx]; + + if (t.has_state || t.state_row >= 0) { + LLAMA_LOG_ERROR("%s: entry %d already has a state embedding or a state row\n", __func__, idx); + return false; + } + + t.state_row = row; + + return true; +} + bool llama_batch_ext::set_token_pos(int32_t idx, const llama_pos * pos_in) { if (idx < 0 || idx >= (int32_t) tokens.size()) { return false; @@ -1377,6 +1470,17 @@ int32_t llama_batch_ext_add_embd(llama_batch_ext * batch, llama_seq_id seq_id, l return idx; } +int32_t llama_batch_ext_add_embd_row(llama_batch_ext * batch, llama_seq_id seq_id, int32_t row) { + int32_t idx = batch->add_token(seq_id); + if (idx < 0) { + return idx; + } + if (!batch->set_token_embd_row(idx, row)) { + return -2; + } + return idx; +} + bool llama_batch_ext_add_seq(llama_batch_ext * batch, int32_t idx, llama_seq_id seq_id) { return batch->add_seq(idx, seq_id); } @@ -1393,6 +1497,10 @@ bool llama_batch_ext_set_embd_state(llama_batch_ext * batch, int32_t idx, llama_ return batch->set_token_state(idx, embd); } +bool llama_batch_ext_set_embd_state_row(llama_batch_ext * batch, int32_t idx, int32_t row) { + return batch->set_token_state_row(idx, row); +} + bool llama_batch_ext_set_output_embd(llama_batch_ext * batch, int32_t idx, bool value) { return batch->set_output(idx, value); } diff --git a/src/llama-batch.h b/src/llama-batch.h index e1e43979b540..d16662898f7e 100644 --- a/src/llama-batch.h +++ b/src/llama-batch.h @@ -52,6 +52,7 @@ struct llama_ubatch { llama_token * token; // [n_tokens] | i | id, token float * embd; // [n_embd, n_tokens] | i | embd float * embd_state; // [n_embd_state, n_tokens] | i | hidden state carried over from a previous stage (e.g. MTP) + int32_t * embd_row; // [n_tokens] | i | embd is this row of ctx_other's layer input device buffer llama_pos * pos; // [n_tokens*n_pos] | i | pos int32_t * n_seq_id; // [n_tokens] | i | - llama_seq_id ** seq_id; // [n_tokens] | s | s0, s1, seq_id @@ -65,6 +66,8 @@ struct llama_ubatch { std::vector token; std::vector embd; std::vector embd_state; + std::vector embd_row; + std::vector embd_state_row; // the state is this row of ctx_state_src's h_nextn device buffer, empty if none std::vector pos; std::vector n_seq_id; std::vector seq_id; // these point into the seq_id_data below @@ -114,6 +117,8 @@ struct llama_batch_ext { size_t embd_off = 0; // index offset in the embd array bool has_state = false; // whether state_off is set size_t state_off = 0; // index offset in the state array + int32_t embd_row = -1; // row of ctx_other's layer input device buffer used as embd, -1 if none + int32_t state_row = -1; // row of ctx_state_src's h_nextn device buffer used as state, -1 if none bool output = false; // TODO: have dedicated output flags int32_t decision_order = 0; // see llama_batch_ext_set_decision_order() std::unordered_set seq_ids; @@ -145,6 +150,8 @@ struct llama_batch_ext { bool set_token_id(int32_t idx, llama_token id); bool set_token_embd(int32_t idx, llama_embd embd_in); bool set_token_state(int32_t idx, llama_embd state_in); + bool set_token_embd_row(int32_t idx, int32_t row); + bool set_token_state_row(int32_t idx, int32_t row); bool set_token_pos(int32_t idx, const llama_pos * pos_in); bool set_output(int32_t idx, bool output_last); bool set_decision_order(int32_t idx, int32_t order); @@ -223,6 +230,8 @@ class llama_batch_allocr { std::vector embd_vec; // owned embeddings built from llama_batch_ext std::vector is_embd_vec; // mixed batch only (= 1 if embd, 0 if text token) std::vector state_vec; // owned state embeddings built from llama_batch_ext, llama_batch has no slot for them + std::vector embd_row_vec; // owned embd rows built from llama_batch_ext, llama_batch has no slot for them + std::vector state_row_vec; // owned state rows built from llama_batch_ext, llama_batch has no slot for them std::vector seq_id_data; // flat storage for seq_id pointers below std::vector pos; diff --git a/src/llama-context.cpp b/src/llama-context.cpp index 66b724c18b62..9066e061ef0e 100644 --- a/src/llama-context.cpp +++ b/src/llama-context.cpp @@ -14,6 +14,7 @@ #include "llama-sampler.h" #include "llama.h" +#include #include #include #include @@ -135,18 +136,6 @@ static const llm_fused_op_probe llm_fused_op_flash_attn_kv_rows_probe = { /*.n_tokens_per_seq =*/ 1, }; -static const llm_fused_op_probe llm_fused_op_gdn_ar_probe = { - /*.op =*/ LLM_FUSED_OP_GDN_AR, - /*.name =*/ "fused Gated Delta Net (autoregressive)", - /*.n_tokens_per_seq =*/ 1, -}; - -static const llm_fused_op_probe llm_fused_op_gdn_ch_probe = { - /*.op =*/ LLM_FUSED_OP_GDN_CH, - /*.name =*/ "fused Gated Delta Net (chunked)", - /*.n_tokens_per_seq =*/ 16, -}; - static const llm_fused_op_probe llm_fused_op_lid_probe = { /*.op =*/ LLM_FUSED_OP_LIGHTNING_INDEXER, /*.name =*/ "Lightning Indexer", @@ -201,7 +190,7 @@ llama_context::llama_context( } cparams.n_rs_seq = params.n_rs_seq; - if (cparams.n_rs_seq > 0 && !llm_arch_supports_rs_rollback(model.arch)) { + if (cparams.n_rs_seq > 0 && (!llm_arch_supports_rs_rollback(model.arch) || !hparams.causal_attn)) { LLAMA_LOG_DEBUG("%s: n_rs_seq=%u requested but model does not support recurrent partial rollback; clamping to 0\n", __func__, cparams.n_rs_seq); cparams.n_rs_seq = 0; @@ -218,10 +207,12 @@ llama_context::llama_context( cparams.embeddings_nextn_masked = false; cparams.offload_kqv = params.offload_kqv; cparams.no_perf = params.no_perf; - cparams.warmup = false; // +1: id n_layer() taps the output of the last layer ("input" of the head) cparams.embeddings_layer_inp.resize(hparams.n_layer() + 1, false); + cparams.embeddings_layer_inp_dev = false; + cparams.embeddings_nextn_dev = false; + cparams.ctx_state_src = nullptr; embd_layer_inp.resize(hparams.n_layer() + 1); cparams.ctx_type = params.ctx_type; @@ -257,6 +248,9 @@ llama_context::llama_context( throw std::runtime_error(model.arch_name() + " requires ctx_other to be set (this warning is normal during memory fitting)"); } cparams.ctx_other = params.ctx_other; + } else if (model.arch == LLM_ARCH_DFLASH) { + // used to read the target's layer input device buffer + cparams.ctx_other = params.ctx_other; } } @@ -328,9 +322,17 @@ llama_context::llama_context( cparams.flash_attn = params.flash_attn_type != LLAMA_FLASH_ATTN_TYPE_DISABLED; cparams.auto_fa = params.flash_attn_type == LLAMA_FLASH_ATTN_TYPE_AUTO; - cparams.fused_gdn_ar = true; - cparams.fused_gdn_ch = true; - cparams.auto_fgdn = false; + // the KV cache is shared with ctx_other, so follow its flash attention setting instead of resolving it here: + // the probe disables it when the layers and the shared KV are on different devices, which breaks a quantized V cache + if (model.arch == LLM_ARCH_GEMMA4_ASSISTANT) { + const bool flash_attn_other = cparams.ctx_other->get_cparams().flash_attn; + if (!cparams.auto_fa && cparams.flash_attn != flash_attn_other) { + LLAMA_LOG_WARN("%s: flash_attn = %d does not match the context the KV cache is shared with, using %d\n", + __func__, cparams.flash_attn, flash_attn_other); + } + cparams.flash_attn = flash_attn_other; + cparams.auto_fa = false; + } cparams.fused_lid = true; cparams.auto_flid = false; @@ -374,7 +376,8 @@ llama_context::llama_context( cparams.op_offload = params.op_offload; cparams.kv_unified = params.kv_unified; - cparams.moe_cache_size = params.moe_cache_size; + cparams.moe_cache_size = params.moe_cache_size; + cparams.moe_cache_cpu_share = params.moe_cache_cpu_share; // resolved after flash attention, see resolve_fused_ops() { @@ -445,6 +448,15 @@ llama_context::llama_context( __func__, cparams.n_ctx_seq, hparams.n_ctx_train); } + { + const int64_t max_nodes = graph_max_nodes(); + + gf_res_reserve = std::make_unique(max_nodes); + for (auto & res : gf_res_prev) { + res = std::make_unique(max_nodes); + } + } + if (!hparams.vocab_only) { // GPU backends for (const auto & dev : model.devices) { @@ -455,6 +467,22 @@ llama_context::llama_context( backends.emplace_back(backend); } + // tensors of ctx_other (e.g. a shared KV cache) can be on devices that this model does not use + if (cparams.ctx_other) { + for (const auto & dev_other : llama_get_model(cparams.ctx_other)->devices) { + const bool found = std::any_of(model.devices.begin(), model.devices.end(), + [&](const llama_device & dev) { return dev.dev == dev_other.dev; }); + if (found) { + continue; + } + ggml_backend_t backend = ggml_backend_dev_init(dev_other.dev, nullptr); + if (backend == nullptr) { + throw std::runtime_error(format("failed to initialize %s backend", ggml_backend_dev_name(dev_other.dev))); + } + backends.emplace_back(backend); + } + } + // add ACCEL backends (such as BLAS) for (size_t i = 0; i < ggml_backend_dev_count(); ++i) { ggml_backend_dev_t dev = ggml_backend_dev_get(i); @@ -579,7 +607,7 @@ llama_context::llama_context( } if (cparams.moe_cache_size > 0) { - moe_cache = std::make_unique(model, model.no_alloc(), backend_ptrs, backend_buft, cparams.moe_cache_size); + moe_cache = std::make_unique(model, model.no_alloc(), backend_ptrs, backend_buft, cparams.moe_cache_size, cparams.moe_cache_cpu_share); } sched_reserve(); @@ -688,13 +716,6 @@ void llama_context::resolve_fused_ops(const llama_memory_context_i * mctx, uint3 cparams.auto_fkvr = false; } - if (cparams.auto_fgdn) { - LLAMA_LOG_INFO("%s: resolving fused Gated Delta Net support:\n", func); - resolve(llm_fused_op_gdn_ar_probe, cparams.fused_gdn_ar); - resolve(llm_fused_op_gdn_ch_probe, cparams.fused_gdn_ch); - cparams.auto_fgdn = false; - } - if (cparams.auto_flid) { LLAMA_LOG_INFO("%s: resolving fused Lightning Indexer support:\n", func); resolve(llm_fused_op_lid_probe, cparams.fused_lid); @@ -747,12 +768,21 @@ static int llama_graph_n_input_tensors(ggml_cgraph * gf) { return (int) users.size(); } -void llama_context::sched_new(size_t max_nodes, bool parallel) { +void llama_context::sched_new(const sched_params & params) { + // reusing the scheduler avoids reallocating its buffers (incl. the compute buffers) + if (sched && params.can_reuse(sched_params_cur)) { + LLAMA_LOG_DEBUG("%s: reusing scheduler: max_nodes = %zu (current max_nodes = %zu)\n", + __func__, params.max_nodes, sched_params_cur.max_nodes); + return; + } + sched.reset(ggml_backend_sched_new( - backend_ptrs.data(), backend_buft.data(), backend_ptrs.size(), max_nodes, parallel, cparams.op_offload)); + backend_ptrs.data(), backend_buft.data(), backend_ptrs.size(), params.max_nodes, params.parallel, params.op_offload)); ggml_backend_sched_set_copy_callback(sched.get(), sched_copy_experts, this); ggml_backend_sched_set_realloc_callback(sched.get(), sched_graph_realloc, this); + + sched_params_cur = params; } void llama_context::sched_reserve() { @@ -771,17 +801,24 @@ void llama_context::sched_reserve() { const uint32_t n_seqs = cparams.n_seq_max; const uint32_t n_tokens = std::min(cparams.n_ctx, cparams.n_ubatch); - const size_t max_nodes = this->graph_max_nodes(n_tokens); + const size_t max_nodes = this->graph_max_nodes(); LLAMA_LOG_DEBUG("%s: max_nodes = %zu\n", __func__, max_nodes); + gf_res_prev_active = nullptr; + + gf_res_reserve->set_max_nodes(max_nodes); + gf_res_reserve->reset(); for (auto & res : gf_res_prev) { - res.reset(); + res->set_max_nodes(max_nodes); + res->reset(); } - gf_res_reserve.reset(new llm_graph_result(max_nodes)); - gf_res_prev_active = nullptr; - sched_new(max_nodes, cparams.pipeline_parallel); + sched_new({ + /*.max_nodes =*/ max_nodes, + /*.parallel =*/ cparams.pipeline_parallel, + /*.op_offload =*/ cparams.op_offload, + }); llama_memory_context_ptr mctx; if (memory) { @@ -820,7 +857,11 @@ void llama_context::sched_reserve() { if (cparams.pipeline_parallel) { LLAMA_LOG_WARN("%s: compute buffer allocation failed, retrying without pipeline parallelism\n", __func__); cparams.pipeline_parallel = false; - sched_new(max_nodes, false); + sched_new({ + /*.max_nodes =*/ max_nodes, + /*.parallel =*/ false, + /*.op_offload =*/ cparams.op_offload, + }); gf = graph_reserve(n_tokens, n_seqs, n_outputs_pp, mctx.get()); } if (!gf) { @@ -1017,9 +1058,7 @@ bool llama_context::memory_update(bool optimize) { // reset the previous graph results to make sure that they won't be reused // TODO: make mctx->apply() report if a graph reserve is needed, then reset graph results only if the memory module reset the scheduler for (auto & res : gf_res_prev) { - if (res) { - res->reset(); - } + res->reset(); } gf_res_prev_active = nullptr; @@ -1385,6 +1424,12 @@ void llama_context::set_embeddings_nextn(bool value, bool masked) { cparams.embeddings_nextn = value; cparams.embeddings_nextn_masked = masked; + + // the device rows are batch rows of the unmasked extraction + if (cparams.embeddings_nextn_dev && (!value || masked)) { + cparams.embeddings_nextn_dev = false; + h_nextn_dev_alloc(); + } } void llama_context::set_embeddings_layer_inp(uint32_t lid, bool enable) { @@ -1396,6 +1441,245 @@ void llama_context::set_embeddings_layer_inp(uint32_t lid, bool enable) { // note: without this reserve, the draft acceptance drops to zero. not sure why - this is unexpected sched_need_reserve = true; + + if (cparams.embeddings_layer_inp_dev) { + layer_inp_dev_alloc(); + } +} + +void llama_context::set_embeddings_layer_inp_dev(bool value) { + LLAMA_LOG_DEBUG("%s: value = %d\n", __func__, value); + + if (cparams.embeddings_layer_inp_dev != value) { + sched_need_reserve = true; + } + + cparams.embeddings_layer_inp_dev = value; + + layer_inp_dev_alloc(); +} + +bool llama_context::supports_embd_row() const { + // the DFlash/DSpark decoder graphs read the device buffer + if (model.arch != LLM_ARCH_DFLASH) { + return false; + } + + const ggml_tensor * src = cparams.ctx_other ? cparams.ctx_other->get_layer_inp_dev() : nullptr; + if (src == nullptr) { + return false; + } + + // one of our backends must be able to read the buffer, e.g. not a draft on another GPU than the target output + ggml_backend_buffer_type_t buft = ggml_backend_buffer_get_type(src->buffer); + for (const auto & backend : backends) { + if (ggml_backend_supports_buft(backend.get(), buft)) { + return true; + } + } + + return false; +} + +ggml_backend_t llama_context::backend_from_dev(ggml_backend_dev_t dev) const { + for (const auto & backend : backends) { + if (ggml_backend_get_device(backend.get()) == dev) { + return backend.get(); + } + } + return nullptr; +} + +void llama_context::layer_inp_dev_alloc() { + synchronize(); + + layer_inp_dev = nullptr; + layer_inp_dev_dev = nullptr; + layer_inp_backend = nullptr; + buf_layer_inp_dev.reset(); + ctx_layer_inp_dev.reset(); + + if (!cparams.embeddings_layer_inp_dev) { + return; + } + + int64_t n_layer_inp = 0; + for (bool enabled : cparams.embeddings_layer_inp) { + n_layer_inp += enabled ? 1 : 0; + } + if (n_layer_inp == 0) { + return; + } + + ggml_init_params params = { + /*.mem_size =*/ ggml_tensor_overhead(), + /*.mem_buffer =*/ nullptr, + /*.no_alloc =*/ true, + }; + ctx_layer_inp_dev.reset(ggml_init(params)); + + layer_inp_dev = ggml_new_tensor_2d(ctx_layer_inp_dev.get(), GGML_TYPE_F32, n_layer_inp*model.hparams.n_embd, cparams.n_batch); + ggml_set_name(layer_inp_dev, "layer_inp_dev"); + + layer_inp_dev_dev = model.dev_output(); + auto * buft = layer_inp_dev_dev ? ggml_backend_dev_buffer_type(layer_inp_dev_dev) : ggml_backend_cpu_buffer_type(); + + buf_layer_inp_dev.reset(ggml_backend_alloc_ctx_tensors_from_buft(ctx_layer_inp_dev.get(), buft)); + if (!buf_layer_inp_dev) { + // e.g. a large n_batch that does not fit next to the model, keep the host copies + LLAMA_LOG_WARN("%s: failed to allocate the layer input device buffer, using host copies\n", __func__); + cparams.embeddings_layer_inp_dev = false; + layer_inp_dev = nullptr; + layer_inp_dev_dev = nullptr; + ctx_layer_inp_dev.reset(); + return; + } + + LLAMA_LOG_INFO("%s: %10s layer input buffer size = %8.2f MiB (%d layers)\n", __func__, + ggml_backend_buffer_name(buf_layer_inp_dev.get()), ggml_backend_buffer_get_size(buf_layer_inp_dev.get())/1024.0/1024.0, (int) n_layer_inp); + + layer_inp_backend = layer_inp_dev_dev ? backend_from_dev(layer_inp_dev_dev) : nullptr; + if (layer_inp_backend && !ev_layer_inp_written) { + ev_layer_inp_written.reset(ggml_backend_event_new(layer_inp_dev_dev)); + ev_layer_inp_read.reset(ggml_backend_event_new(layer_inp_dev_dev)); + ev_layer_inp_read_recorded = false; + } +} + +bool llama_context::h_nextn_dev_alloc() { + synchronize(); + + auto reset = [&]() { + h_nextn_dev = nullptr; + h_nextn_dev_dev = nullptr; + h_nextn_backend = nullptr; + buf_h_nextn_dev.reset(); + ctx_h_nextn_dev.reset(); + ev_h_nextn_written_recorded = false; + ev_h_nextn_read_recorded = false; + }; + + // __func__ inside the lambda would be "operator()" + const char * func = __func__; + auto fail = [&](const char * why) { + LLAMA_LOG_WARN("%s: %s, extracting the h_nextn rows to host memory\n", func, why); + cparams.embeddings_nextn_dev = false; + reset(); + return false; + }; + + reset(); + + if (!cparams.embeddings_nextn_dev) { + return true; + } + + h_nextn_dev_dev = model.dev_output(); + h_nextn_backend = h_nextn_dev_dev ? backend_from_dev(h_nextn_dev_dev) : nullptr; + if (h_nextn_backend == nullptr) { + return fail("no device backend for the model output"); + } + + ggml_init_params params = { + /*.mem_size =*/ ggml_tensor_overhead(), + /*.mem_buffer =*/ nullptr, + /*.no_alloc =*/ true, + }; + ctx_h_nextn_dev.reset(ggml_init(params)); + + h_nextn_dev = ggml_new_tensor_2d(ctx_h_nextn_dev.get(), GGML_TYPE_F32, model.hparams.n_embd_out(), cparams.n_batch + cparams.n_seq_max); + ggml_set_name(h_nextn_dev, "h_nextn_dev"); + + buf_h_nextn_dev.reset(ggml_backend_alloc_ctx_tensors_from_buft(ctx_h_nextn_dev.get(), ggml_backend_dev_buffer_type(h_nextn_dev_dev))); + if (!buf_h_nextn_dev) { + return fail("failed to allocate the h_nextn device buffer"); + } + + if (!ev_h_nextn_written) { + ev_h_nextn_written.reset(ggml_backend_event_new(h_nextn_dev_dev)); + ev_h_nextn_read.reset(ggml_backend_event_new(h_nextn_dev_dev)); + } + if (!ev_h_nextn_written || !ev_h_nextn_read) { + return fail("the device has no events"); + } + + LLAMA_LOG_INFO("%s: %10s h_nextn buffer size = %8.2f MiB\n", __func__, + ggml_backend_buffer_name(buf_h_nextn_dev.get()), ggml_backend_buffer_get_size(buf_h_nextn_dev.get())/1024.0/1024.0); + + return true; +} + +bool llama_context::set_embeddings_nextn_dev(bool value) { + LLAMA_LOG_DEBUG("%s: value = %d\n", __func__, value); + + // the device rows are batch rows, so the extraction must not be masked + if (value && (!cparams.embeddings_nextn || cparams.embeddings_nextn_masked)) { + return false; + } + + if (cparams.embeddings_nextn_dev != value) { + sched_need_reserve = true; + } + + cparams.embeddings_nextn_dev = value; + + return h_nextn_dev_alloc() && cparams.embeddings_nextn_dev == value; +} + +bool llama_context::get_h_nextn_rows(int64_t row0, int64_t n_rows, float * out) { + if (h_nextn_dev == nullptr || row0 < 0 || n_rows < 0 || row0 + n_rows > h_nextn_dev->ne[1]) { + return false; + } + if (ev_h_nextn_written_recorded) { + ggml_backend_event_synchronize(ev_h_nextn_written.get()); + } else { + synchronize(); + } + ggml_backend_tensor_get(h_nextn_dev, out, row0*h_nextn_dev->nb[1], n_rows*h_nextn_dev->nb[1]); + return true; +} + +bool llama_context::set_h_nextn_rows(int64_t row0, int64_t n_rows, const float * data) { + if (h_nextn_dev == nullptr || row0 < 0 || n_rows < 0 || row0 + n_rows > h_nextn_dev->ne[1]) { + return false; + } + // a reader may still read these rows + if (ev_h_nextn_read_recorded) { + ggml_backend_event_synchronize(ev_h_nextn_read.get()); + } + ggml_backend_tensor_set(h_nextn_dev, data, row0*h_nextn_dev->nb[1], n_rows*h_nextn_dev->nb[1]); + return true; +} + +bool llama_context::set_ctx_state_src(llama_context * ctx_src) { + if (cparams.ctx_state_src != ctx_src) { + sched_need_reserve = true; + } + cparams.ctx_state_src = ctx_src; + return supports_state_row(); +} + +bool llama_context::supports_state_row() const { + // the MTP graphs read the rows, see llm_graph_context::build_inp_embd_state() + if (cparams.ctx_type != LLAMA_CONTEXT_TYPE_MTP) { + return false; + } + + // the rows replace the host state rows, so they must have the width the graph takes for those + const ggml_tensor * src = cparams.ctx_state_src ? cparams.ctx_state_src->get_h_nextn_dev() : nullptr; + if (src == nullptr || src->ne[0] != (int64_t) llama_batch_ext_select_n_embd_state(cparams.ctx_type, model.hparams)) { + return false; + } + + // one of our backends must be able to read the buffer + ggml_backend_buffer_type_t buft = ggml_backend_buffer_get_type(src->buffer); + for (const auto & backend : backends) { + if (ggml_backend_supports_buft(backend.get(), buft)) { + return true; + } + } + + return false; } void llama_context::set_nextn_layer_offset(int32_t offset) { @@ -1414,14 +1698,6 @@ bool llama_context::get_causal_attn() const { return cparams.causal_attn; } -void llama_context::set_warmup(bool value) { - LLAMA_LOG_DEBUG("%s: value = %d\n", __func__, value); - - // no scheduler reserve needed because warmups are usually with small batches - - cparams.warmup = value; -} - bool llama_context::set_sampler(llama_seq_id seq_id, llama_sampler * sampler) { if (!sampler && sampling.samplers.count(seq_id) == 0) { return true; @@ -1627,6 +1903,11 @@ int llama_context::encode(const llama_batch_ext & batch_inp) { return -1; } + if (!batch_inp.tokens.empty() && batch_inp.tokens[0].embd_row >= 0) { + LLAMA_LOG_ERROR("%s: embd row batches are not supported by encode\n", __func__); + return -1; + } + // eagle3/DFlash: features as encoder input, and non-draft paths fall back to model's input dim const int64_t n_vocab = model.vocab.n_tokens(); @@ -1873,6 +2154,18 @@ int llama_context::decode(const llama_batch_ext & batch_inp) { return -1; } + const bool has_embd_row = !batch_inp.tokens.empty() && batch_inp.tokens[0].embd_row >= 0; + if (has_embd_row && !supports_embd_row()) { + LLAMA_LOG_ERROR("%s: this context cannot read embd rows (needs a DFlash/DSpark decoder and a layer input device buffer in ctx_other)\n", __func__); + return -1; + } + + const bool has_state_row = !batch_inp.tokens.empty() && batch_inp.tokens[0].state_row >= 0; + if (has_state_row && !supports_state_row()) { + LLAMA_LOG_ERROR("%s: this context cannot read state rows (needs an MTP context and an h_nextn device buffer in ctx_state_src)\n", __func__); + return -1; + } + const auto & vocab = model.vocab; const auto & hparams = model.hparams; @@ -2016,6 +2309,40 @@ int llama_context::decode(const llama_batch_ext & batch_inp) { int64_t n_outputs_prev = 0; int64_t n_tokens_prev = 0; + // ordering for the layer input device buffer + llama_context * inject_src = has_embd_row ? cparams.ctx_other : nullptr; + ggml_backend_t inject_backend = nullptr; + if (inject_src) { + inject_backend = inject_src->ev_layer_inp_written ? backend_from_dev(inject_src->layer_inp_dev_dev) : nullptr; + if (inject_backend) { + // wait on the host: a device-side wait would stall our queue for the whole target batch, + // which trips the Metal command buffer timeout on long prompts + ggml_backend_event_synchronize(inject_src->ev_layer_inp_written.get()); + } else { + inject_src->synchronize(); + } + } + if (layer_inp_dev && ev_layer_inp_read && ev_layer_inp_read_recorded) { + // do not overwrite rows the reader has not consumed yet + ggml_backend_event_wait(layer_inp_backend, ev_layer_inp_read.get()); + } + + // ordering for the h_nextn device buffer, as for the layer input buffer + llama_context * state_src = has_state_row ? cparams.ctx_state_src : nullptr; + ggml_backend_t state_backend = nullptr; + if (state_src) { + state_backend = backend_from_dev(state_src->h_nextn_dev_dev); + if (state_src->ev_h_nextn_written_recorded) { + ggml_backend_event_synchronize(state_src->ev_h_nextn_written.get()); + } else { + state_src->synchronize(); + } + } + const bool write_h_nextn = cparams.embeddings_nextn_dev && h_nextn_dev != nullptr; + if (write_h_nextn && ev_h_nextn_read_recorded) { + ggml_backend_event_wait(h_nextn_backend, ev_h_nextn_read.get()); + } + do { const auto & ubatch = mctx->get_ubatch(); @@ -2168,7 +2495,8 @@ int llama_context::decode(const llama_batch_ext & batch_inp) { const int64_t n_rows = masked ? n_outputs : (int64_t) ubatch.n_tokens; const int64_t offset = masked ? n_outputs_prev : n_tokens_prev; - if (embd_nextn.data && t_h_nextn && n_rows > 0 && cparams.pooling_type == LLAMA_POOLING_TYPE_NONE) { + // the device buffer replaces the host extraction, see build_h_nextn_dev() + if (embd_nextn.data && t_h_nextn && n_rows > 0 && cparams.pooling_type == LLAMA_POOLING_TYPE_NONE && !write_h_nextn) { ggml_backend_t backend_h = ggml_backend_sched_get_tensor_backend(sched.get(), t_h_nextn); GGML_ASSERT(backend_h != nullptr); @@ -2202,6 +2530,30 @@ int llama_context::decode(const llama_batch_ext & batch_inp) { n_tokens_prev += ubatch.n_tokens; } while (mctx->next()); + if (layer_inp_dev && ev_layer_inp_written) { + ggml_backend_event_record(ev_layer_inp_written.get(), layer_inp_backend); + } + if (write_h_nextn) { + ggml_backend_event_record(ev_h_nextn_written.get(), h_nextn_backend); + ev_h_nextn_written_recorded = true; + } + if (state_src) { + if (state_backend) { + ggml_backend_event_record(state_src->ev_h_nextn_read.get(), state_backend); + state_src->ev_h_nextn_read_recorded = true; + } else { + synchronize(); + } + } + if (inject_src) { + if (inject_backend) { + ggml_backend_event_record(inject_src->ev_layer_inp_read.get(), inject_backend); + inject_src->ev_layer_inp_read_recorded = true; + } else { + synchronize(); + } + } + // set to total number of outputs in the batch, for use in llama_get_logits_ith n_outputs = n_outputs_all; @@ -2298,7 +2650,7 @@ uint32_t llama_context::output_reserve(int32_t n_outputs) { } for (bool enabled : cparams.embeddings_layer_inp) { - if (enabled) { + if (enabled && !cparams.embeddings_layer_inp_dev) { embd_layer_inp_float_count += (size_t) n_embd * n_batch; } } @@ -2370,7 +2722,7 @@ uint32_t llama_context::output_reserve(int32_t n_outputs) { offset += embd_nextn.size * sizeof(float); for (uint32_t il = 0; il < embd_layer_inp.size(); ++il) { - if (cparams.embeddings_layer_inp[il]) { + if (cparams.embeddings_layer_inp[il] && !cparams.embeddings_layer_inp_dev) { embd_layer_inp[il] = buffer_view{(float *) (base + offset), (size_t) n_embd * n_batch}; offset += embd_layer_inp[il].size * sizeof(float); } else { @@ -2426,6 +2778,9 @@ uint32_t llama_context::output_reserve(int32_t n_outputs) { bool llama_context::extract_layer_inputs(const llm_graph_result * res, size_t token_offset, size_t n_tokens) { bool extracted = false; + if (cparams.embeddings_layer_inp_dev) { + return extracted; + } for (uint32_t il = 0; il < cparams.embeddings_layer_inp.size(); ++il) { if (!cparams.embeddings_layer_inp[il]) { continue; @@ -2542,16 +2897,14 @@ void llama_context::output_reorder() { // graph // -uint32_t llama_context::graph_max_nodes(uint32_t n_tokens) const { +uint32_t llama_context::graph_max_nodes() const { uint32_t res; - if (model.arch == LLM_ARCH_KIMI_K3 || model.arch == LLM_ARCH_GLM5_NEXT) { - // the n_tokens*40 budget below is exhausted at ubatch 3840 - res = std::max(n_tokens * 160, 64u * model.n_tensors()); - } else if (model.arch == LLM_ARCH_HRM_TEXT) { - // the 128-slot looped graph needs roughly one stack per token budget - res = std::max(n_tokens * 80, 64u * model.n_tensors()); + if (model.arch == LLM_ARCH_KIMI_K3 || model.arch == LLM_ARCH_KIMI_K31 || model.arch == LLM_ARCH_GLM5_NEXT || + model.arch == LLM_ARCH_HRM_TEXT) { + res = std::max(1024u, 64u * model.n_tensors()); } else if (model.arch == LLM_ARCH_QWEN3NEXT || model.arch == LLM_ARCH_KIMI_LINEAR || + model.arch == LLM_ARCH_METIS16 || model.arch == LLM_ARCH_BAILINGMOE3 || model.arch == LLM_ARCH_QWEN35 || model.arch == LLM_ARCH_QWEN35MOE || @@ -2560,12 +2913,13 @@ uint32_t llama_context::graph_max_nodes(uint32_t n_tokens) const { model.arch == LLM_ARCH_QWEN4EXP || model.arch == LLM_ARCH_DEEPSEEK4 || model.arch == LLM_ARCH_DEEPSEEK41 || + model.arch == LLM_ARCH_XING4_0 || (model.arch == LLM_ARCH_DFLASH && model.hparams.dsv4_hc_mult > 0) || model.arch == LLM_ARCH_NANBEIGE || model.arch == LLM_ARCH_MINIMAX_01 || model.arch == LLM_ARCH_MINIMAX_M3 || model.arch == LLM_ARCH_HY_V4) { - res = std::max(n_tokens * 40, 32u * model.n_tensors()); + res = std::max(1024u, 32u * model.n_tensors()); } else if (model.arch == LLM_ARCH_DFLASH && model.hparams.dflash_selector_rank > 0) { // DFlash2's convolutions and selector are shape work rather than matmuls, // so they cost ~8.6 nodes per tensor against ~5.9 for a plain DFlash draft @@ -2594,11 +2948,7 @@ llm_graph_result * llama_context::get_gf_res_reserve() const { } llm_graph_result * llama_context::get_gf_res_prev() { - auto & res = gf_res_prev[n_outputs > 0]; - if (!res) { - res.reset(new llm_graph_result(gf_res_reserve->get_max_nodes())); - } - return res.get(); + return gf_res_prev[n_outputs > 0].get(); } // pack sampler outputs into as few sequences as possible before using sequences without samplers @@ -2672,9 +3022,7 @@ ggml_cgraph * llama_context::graph_reserve( // when the scheduler is reset, we cannot reuse old graphs, so we reset the previous graph results for (auto & res : gf_res_prev) { - if (res) { - res->reset(); - } + res->reset(); } gf_res_prev_active = nullptr; @@ -2728,7 +3076,7 @@ llm_graph_params llama_context::graph_params( const llama_memory_context_i * mctx, llm_graph_type gtype) const { return { - /*.arch =*/ model.arch, + /*.model =*/ &model, /*.hparams =*/ model.hparams, /*.cparams =*/ cparams, /*.ubatch =*/ ubatch, @@ -2742,6 +3090,10 @@ llm_graph_params llama_context::graph_params( /*.moe_cache =*/ moe_cache.get(), /*.prec_policy =*/ &model.prec_policy, /*.hdmd =*/ model.hdmd.rot.empty() ? nullptr : &model.hdmd, + /*.layer_inp_dev =*/ layer_inp_dev, + /*.layer_inp_src =*/ supports_embd_row() ? cparams.ctx_other->get_layer_inp_dev() : nullptr, + /*.h_nextn_dev =*/ cparams.embeddings_nextn_dev ? h_nextn_dev : nullptr, + /*.h_nextn_src =*/ supports_state_row() ? cparams.ctx_state_src->get_h_nextn_dev() : nullptr, /*.samplers =*/ sampling.samplers, /*.n_outputs =*/ n_outputs, /*.cb =*/ graph_get_cb(), @@ -3167,7 +3519,8 @@ class llama_io_write_device : public llama_io_write_i { for (const auto & winfo : winfos) { auto * buft = ggml_backend_buffer_get_type(winfo.tensor->buffer); - const int64_t n = winfo.size/ggml_element_size(winfo.tensor); + GGML_ASSERT(winfo.size % ggml_type_size(winfo.tensor->type) == 0); + const int64_t n = winfo.size/ggml_type_size(winfo.tensor->type)*ggml_blck_size(winfo.tensor->type); auto & mbuf = mbufs_new[buft]; @@ -3298,7 +3651,8 @@ class llama_io_read_device : public llama_io_read_i { for (const auto & rinfo : rinfos) { auto * buft = ggml_backend_buffer_get_type(rinfo.tensor->buffer); - const int64_t n = rinfo.size/ggml_element_size(rinfo.tensor); + GGML_ASSERT(rinfo.size % ggml_type_size(rinfo.tensor->type) == 0); + const int64_t n = rinfo.size/ggml_type_size(rinfo.tensor->type)*ggml_blck_size(rinfo.tensor->type); auto & mbuf = mbufs_new[buft]; @@ -3370,8 +3724,8 @@ class llama_io_read_device : public llama_io_read_i { const size_t n_copy = std::min(src_size - src_off, dst_size - dst_off); - const size_t el = ggml_element_size(src_t); - const int64_t n_el = (int64_t) (n_copy / el); + GGML_ASSERT(n_copy % ggml_type_size(src_t->type) == 0); + const int64_t n_el = n_copy / ggml_type_size(src_t->type) * ggml_blck_size(src_t->type); auto * src_v = ggml_view_1d(ctx_scratch, src_t, n_el, src_off); ggml_backend_view_init(src_v); @@ -4081,12 +4435,12 @@ llama_context_params llama_context_default_params() { /*.yarn_beta_fast =*/ -1.0f, /*.yarn_beta_slow =*/ -1.0f, /*.yarn_orig_ctx =*/ 0, - /*.defrag_thold =*/ -1.0f, /*.cb_eval =*/ nullptr, /*.cb_eval_user_data =*/ nullptr, /*.type_k =*/ GGML_TYPE_F16, /*.type_v =*/ GGML_TYPE_F16, /*.moe_cache_size =*/ 0, + /*.moe_cache_cpu_share =*/ 0.0f, /*.abort_callback =*/ nullptr, /*.abort_callback_data =*/ nullptr, /*.embeddings =*/ false, @@ -4145,6 +4499,45 @@ llama_context * llama_init_from_model( return nullptr; } + const bool fp8_cache = params.type_k == GGML_TYPE_F8_E4M3 || params.type_v == GGML_TYPE_F8_E4M3; + // Direct cache consumers do not use the shared attention scaling helpers. + if (fp8_cache && (model->hparams.is_mla() || + model->arch == LLM_ARCH_DEEPSEEK32 || + model->arch == LLM_ARCH_DEEPSEEK4 || + model->arch == LLM_ARCH_DEEPSEEK41 || + model->arch == LLM_ARCH_GLM_DSA || + model->arch == LLM_ARCH_GLM5_NEXT || + model->arch == LLM_ARCH_DOTS3NOTE || + model->arch == LLM_ARCH_MISTRAL_LARGE4 || + model->arch == LLM_ARCH_HY_V4 || + model->arch == LLM_ARCH_MINIMAX_M3 || + model->arch == LLM_ARCH_QWEN4EXP || + model->arch == LLM_ARCH_DFLASH)) { + LLAMA_LOG_ERROR("%s: FP8 cache is not supported for architecture '%s'\n", __func__, llm_arch_name(model->arch)); + return nullptr; + } + + if (fp8_cache) { + for (uint32_t il = 0; il < model->hparams.n_layer_all; ++il) { + if (!model->hparams.has_kv(il)) { + continue; + } + + for (ggml_tensor * scale : { model->layers[il].k_cache_s, model->layers[il].v_cache_s }) { + if (!scale || !scale->data) { + continue; + } + + float value; + ggml_backend_tensor_get(scale, &value, 0, sizeof(value)); + if (!std::isfinite(value) || value == 0.0f) { + LLAMA_LOG_ERROR("%s: FP8 cache scale '%s' must be finite and nonzero\n", __func__, ggml_get_name(scale)); + return nullptr; + } + } + } + } + if (ggml_is_quantized(params.type_v) && params.flash_attn_type != LLAMA_FLASH_ATTN_TYPE_ENABLED) { if (params.flash_attn_type == LLAMA_FLASH_ATTN_TYPE_AUTO) { LLAMA_LOG_INFO("%s: enabling flash_attn since it is required for quantized V cache\n", __func__); @@ -4210,13 +4603,6 @@ llama_context * llama_init_from_model( return nullptr; } -// deprecated -llama_context * llama_new_context_with_model( - llama_model * model, - llama_context_params params) { - return llama_init_from_model(model, params); -} - void llama_free(llama_context * ctx) { delete ctx; } @@ -4292,10 +4678,6 @@ bool llama_get_causal_attn(const llama_context * ctx) { return ctx->get_causal_attn(); } -void llama_set_warmup(llama_context * ctx, bool warmup) { - ctx->set_warmup(warmup); -} - void llama_synchronize(llama_context * ctx) { ctx->synchronize(); } @@ -4342,6 +4724,30 @@ void llama_set_embeddings_nextn(llama_context * ctx, bool value, bool masked) { ctx->set_embeddings_nextn(value, masked); } +bool llama_set_embeddings_nextn_dev(llama_context * ctx, bool value) { + return ctx->set_embeddings_nextn_dev(value); +} + +bool llama_get_embeddings_nextn_rows(llama_context * ctx, int64_t row0, int64_t n_rows, float * out) { + return ctx->get_h_nextn_rows(row0, n_rows, out); +} + +bool llama_set_embeddings_nextn_rows(llama_context * ctx, int64_t row0, int64_t n_rows, const float * data) { + return ctx->set_h_nextn_rows(row0, n_rows, data); +} + +bool llama_set_ctx_state_src(llama_context * ctx, llama_context * ctx_src) { + return ctx->set_ctx_state_src(ctx_src); +} + +void llama_set_embeddings_layer_inp_dev(llama_context * ctx, bool value) { + ctx->set_embeddings_layer_inp_dev(value); +} + +bool llama_supports_embd_row(const llama_context * ctx) { + return ctx->supports_embd_row(); +} + void llama_set_embeddings_layer_inp(llama_context * ctx, uint32_t lid, bool value) { ctx->set_embeddings_layer_inp(lid, value); } @@ -4566,31 +4972,6 @@ bool llama_memory_can_shift(llama_memory_t mem) { // llama state API -// deprecated -size_t llama_get_state_size(llama_context * ctx) { - return llama_state_get_size(ctx); -} - -// deprecated -size_t llama_copy_state_data(llama_context * ctx, uint8_t * dst) { - return llama_state_get_data(ctx, dst, -1); -} - -// deprecated -size_t llama_set_state_data(llama_context * ctx, const uint8_t * src) { - return llama_state_set_data(ctx, src, -1); -} - -// deprecated -bool llama_load_session_file(llama_context * ctx, const char * path_session, llama_token * tokens_out, size_t n_token_capacity, size_t * n_token_count_out) { - return llama_state_load_file(ctx, path_session, tokens_out, n_token_capacity, n_token_count_out); -} - -// deprecated -bool llama_save_session_file(llama_context * ctx, const char * path_session, const llama_token * tokens, size_t n_token_count) { - return llama_state_save_file(ctx, path_session, tokens, n_token_count); -} - // Returns the *actual* size of the state. // Intended to be used when saving to state to a buffer. size_t llama_state_get_size(llama_context * ctx) { diff --git a/src/llama-context.h b/src/llama-context.h index 6938fb103cde..45eca4566116 100644 --- a/src/llama-context.h +++ b/src/llama-context.h @@ -93,6 +93,9 @@ struct llama_context { float * get_embeddings_layer_inp(uint32_t lid); + ggml_tensor * get_layer_inp_dev() const { return layer_inp_dev; } + ggml_tensor * get_h_nextn_dev() const { return h_nextn_dev; } + llama_token * get_sampled_tokens() const; llama_token get_sampled_token_ith(int32_t idx); @@ -120,9 +123,21 @@ struct llama_context { void set_embeddings (bool value); void set_embeddings_nextn(bool value, bool masked); void set_embeddings_layer_inp(uint32_t lid, bool enable); + void set_embeddings_layer_inp_dev(bool value); + + // whether decode accepts entries whose embd is a row of ctx_other's layer input device buffer + bool supports_embd_row() const; + + // h_nextn device buffer, see llama_set_embeddings_nextn_dev() + bool set_embeddings_nextn_dev(bool value); + bool get_h_nextn_rows(int64_t row0, int64_t n_rows, float * out); + bool set_h_nextn_rows(int64_t row0, int64_t n_rows, const float * data); + + // state rows read from the h_nextn device buffer of ctx_src, see llama_set_ctx_state_src() + bool set_ctx_state_src(llama_context * ctx_src); + bool supports_state_row() const; void set_nextn_layer_offset(int32_t offset); void set_causal_attn(bool value); - void set_warmup(bool value); void set_adapters_lora(llama_adapter_lora ** adapters, size_t n_adapters, float * scales); @@ -243,12 +258,20 @@ struct llama_context { // from backend into host-side embd_layer_inp buffers bool extract_layer_inputs(const llm_graph_result * res, size_t token_offset, size_t n_tokens); + // (re)allocate the device buffer for cparams.embeddings_layer_inp_dev + void layer_inp_dev_alloc(); + + // (re)allocate the device buffer for cparams.embeddings_nextn_dev, false if it does not fit + bool h_nextn_dev_alloc(); + + ggml_backend_t backend_from_dev(ggml_backend_dev_t dev) const; + // // graph // public: - uint32_t graph_max_nodes(uint32_t n_tokens) const; + uint32_t graph_max_nodes() const; // can reuse the llm_graph_result instance of the context (for example to update a memory module) llm_graph_result * get_gf_res_reserve() const; @@ -273,8 +296,22 @@ struct llama_context { llm_graph_cb graph_get_cb() const; - // create a new backend scheduler for a graph of the specified size and register the callbacks below - void sched_new(size_t max_nodes, bool parallel); + // parameters defining the backend scheduler instance + struct sched_params { + size_t max_nodes = 0; + bool parallel = false; + bool op_offload = false; + + // a scheduler can serve any graph of up to max_nodes nodes + bool can_reuse(const sched_params & other) const { + return parallel == other.parallel + && op_offload == other.op_offload + && max_nodes <= other.max_nodes; + } + }; + + // create a new backend scheduler with the given parameters, unless the current one can be reused + void sched_new(const sched_params & params); // ggml_backend_sched copy callback, copies only the experts used by MUL_MAT_ID and updates the MoE cache static bool sched_copy_experts(ggml_backend_t backend, const ggml_tensor * src, ggml_tensor * dst, ggml_cgraph * graph, void * user_data); @@ -366,6 +403,8 @@ struct llama_context { ggml_backend_sched_ptr sched; + sched_params sched_params_cur; + bool sched_need_reserve = true; // whether the reserve graph has a logits tensor, set in sched_reserve() @@ -420,6 +459,33 @@ struct llama_context { // host buffer for the model output (logits and embeddings) ggml_backend_buffer_ptr buf_output; + // device buffer with enabled layer inputs, [n_layer_inp*n_embd, n_batch], row = batch index + ggml_context_ptr ctx_layer_inp_dev; + ggml_backend_buffer_ptr buf_layer_inp_dev; + ggml_tensor * layer_inp_dev = nullptr; + ggml_backend_dev_t layer_inp_dev_dev = nullptr; + ggml_backend_t layer_inp_backend = nullptr; + + // ordering with the reader context (no host sync): + // written - recorded by this context after the batch, waited on by the reader + // read - recorded by the reader after it consumed the rows, waited on by this context + ggml_backend_event_ptr ev_layer_inp_written; + ggml_backend_event_ptr ev_layer_inp_read; + bool ev_layer_inp_read_recorded = false; + + // device buffer with the h_nextn rows, F32 [n_embd_out, n_batch + n_seq_max]: + // row r < n_batch is the token at batch index r of the last decode, row n_batch + s is set by the host for seq s + // ordered with the reader context like the layer input buffer + ggml_context_ptr ctx_h_nextn_dev; + ggml_backend_buffer_ptr buf_h_nextn_dev; + ggml_tensor * h_nextn_dev = nullptr; + ggml_backend_dev_t h_nextn_dev_dev = nullptr; + ggml_backend_t h_nextn_backend = nullptr; + ggml_backend_event_ptr ev_h_nextn_written; + ggml_backend_event_ptr ev_h_nextn_read; + bool ev_h_nextn_written_recorded = false; + bool ev_h_nextn_read_recorded = false; + // keep copies of the per-sequence memory on the device std::map mem_storage; diff --git a/src/llama-cparams.h b/src/llama-cparams.h index c02e8bcac8ab..52c85bdd2bf7 100644 --- a/src/llama-cparams.h +++ b/src/llama-cparams.h @@ -39,9 +39,6 @@ struct llama_cparams { bool offload_kqv; bool flash_attn; bool auto_fa; - bool fused_gdn_ar; // use fused gated delta net (autoregressive) - bool fused_gdn_ch; // use fused gated delta net (chunked) - bool auto_fgdn; bool fused_lid; // use fused lightning indexer bool auto_flid; bool fused_dsv4_hc_pre; @@ -53,15 +50,17 @@ struct llama_cparams { bool fused_kv_rows; // unified cache: attend to the cells of each sequence only (ggml_flash_attn_ext_rows) bool auto_fkvr; bool no_perf; - bool warmup; // TODO: remove [TAG_LLAMA_GRAPH_NO_WARMUP] bool op_offload; bool kv_unified; bool pipeline_parallel; bool training; // set by llama_opt_init() size_t moe_cache_size; + float moe_cache_cpu_share; std::vector embeddings_layer_inp; // [n_layer()] extract input embeddings for layer + bool embeddings_layer_inp_dev; // write enabled layer inputs to a device buffer instead of host + bool embeddings_nextn_dev; // write the unmasked h_nextn rows to a device buffer instead of host, see llama_set_embeddings_nextn_dev() enum llama_context_type ctx_type; enum llama_rope_scaling_type rope_scaling_type; @@ -71,4 +70,7 @@ struct llama_cparams { void * cb_eval_user_data; llama_context * ctx_other; + + // context whose h_nextn device buffer holds the state rows of a batch, see llama_set_ctx_state_src() + llama_context * ctx_state_src; }; diff --git a/src/llama-ext.h b/src/llama-ext.h index fcd91e912b1e..3ac617b0fa03 100644 --- a/src/llama-ext.h +++ b/src/llama-ext.h @@ -123,16 +123,6 @@ LLAMA_API std::vector llama_memory_get_buffers(const llam // Useful for testing purposes. LLAMA_API void llama_model_set_attn_rot(struct llama_model * model, bool value); -// Set whether the context outputs nextn embeddings or not -// If masked == true, output the embeddings only for the tokens with batch.logits != 0 -// If masked == false, output the embeddings for all tokens in the batch regardless of batch.logits -LLAMA_API void llama_set_embeddings_nextn(struct llama_context * ctx, bool value, bool masked); - -// Select which appended NextN block the DECODER_MTP graph runs (offset past -// the trunk: il = n_layer() + offset). Used by the speculative NextN driver to -// chain multiple trained NextN heads. Default 0 (first head). -LLAMA_API void llama_set_nextn_layer_offset(struct llama_context * ctx, int32_t offset); - // Marks the entries that a decision head (clef, vllm-decision2) reads, the default is 0 // See https://github.com/ggml-org/llama.cpp/pull/29831 for details // A run of entries with the same value is one span, spans must be separated by entries with value 0 @@ -150,33 +140,10 @@ LLAMA_API bool llama_batch_ext_set_decision_order(struct llama_batch_ext * batch // Label of option i of a decision model (".decision.labels"), nullptr past the last one LLAMA_API const char * llama_model_decision_label(const struct llama_model * model, uint32_t i); -// mirrors: -// LLAMA_API float * llama_get_embeddings(struct llama_context * ctx); -LLAMA_API float * llama_get_embeddings_nextn(struct llama_context * ctx); - -// LLAMA_API float * llama_get_embeddings_ith(struct llama_context * ctx, int32_t i); -LLAMA_API float * llama_get_embeddings_nextn_ith(struct llama_context * ctx, int32_t i); - -// Set whether the context outputs the input embeddings of a specific layer -LLAMA_API void llama_set_embeddings_layer_inp(struct llama_context * ctx, uint32_t lid, bool value); - -// mirrors: -// LLAMA_API float * llama_get_embeddings(struct llama_context * ctx); -LLAMA_API float * llama_get_embeddings_layer_inp(struct llama_context * ctx, uint32_t lid); - -LLAMA_API llama_context * llama_get_ctx_other(struct llama_context * ctx); - // // model/context data extraction // -LLAMA_API int32_t llama_model_dflash_selector_top_k(const struct llama_model * model); - -// returns pointer to the target-model layer indices -LLAMA_API const int32_t * llama_model_target_layer_ids (const struct llama_model * model); -// returns the number of extracted layers from target model -LLAMA_API uint32_t llama_model_target_layer_ids_n(const struct llama_model * model); - // retrieves the whole token embedding matrix in F32 format (n_embd * n_vocab) // returns total number of elements or 0 on error // if out is nullptr, returns the number of tokens without writing to out @@ -189,3 +156,6 @@ LLAMA_API uint32_t llama_model_get_tok_embd_rows(const struct llama_model * mode // realtime models: number of tokens the output lags behind the audio input, used by mtmd; 0 if not applicable LLAMA_API uint32_t llama_model_n_tokens_delay(const struct llama_model * model); + +// each sequence is paired with an unconditional one at n_seq_max / 2 + seq_id, in a unified KV cache +LLAMA_API bool llama_model_classifier_free_guidance(const struct llama_model * model); diff --git a/src/llama-graph.cpp b/src/llama-graph.cpp index 725bdb048e51..3cee72df0a0d 100644 --- a/src/llama-graph.cpp +++ b/src/llama-graph.cpp @@ -19,6 +19,8 @@ #include #include +#include +#include #include #include #include @@ -124,8 +126,6 @@ static llm_attn_mask build_attn_inp_attn_mask( ggml_set_name(attn_mask.raw, "attn_inp_kq_mask"); } - attn_mask.cnv = attn_mask.raw; - return attn_mask; } @@ -138,6 +138,12 @@ void llm_graph_input_embd::set_input(const llama_ubatch * ubatch) { ggml_backend_tensor_set(tokens, ubatch->token, 0, n_tokens*ggml_element_size(tokens)); } + if (ubatch->embd_row && embd_rows) { + const int64_t n_tokens = ubatch->n_tokens; + + ggml_backend_tensor_set(embd_rows, ubatch->embd_row, 0, n_tokens*ggml_element_size(embd_rows)); + } + if (ubatch->embd && embd && !ubatch->is_mixed()) { GGML_ASSERT(n_embd == embd->ne[0]); @@ -197,6 +203,7 @@ bool llm_graph_input_embd::can_reuse(const llm_graph_params & params) { res &= (!mixed_tokens) || mixed_tokens->ne[0] == llm_graph_n_tok_rows(params.ubatch); res &= (!mixed_embd) || mixed_embd->ne[1] == params.ubatch.n_tokens; res &= (!scale_rows) || scale_rows->ne[1] == params.ubatch.n_tokens; + res &= (!params.ubatch.embd_row) || (embd_rows && embd_rows->ne[0] == params.ubatch.n_tokens); return res; } @@ -216,6 +223,12 @@ void llm_graph_input_embd_h::set_input(const llama_ubatch * ubatch) { ggml_backend_tensor_set(embd, ubatch->embd, 0, n_tokens*n_embd*ggml_element_size(embd)); } + if (h_rows) { + GGML_ASSERT(ubatch->data && ubatch->data->embd_state_row.size() == (size_t) n_tokens); + ggml_backend_tensor_set(h_rows, ubatch->data->embd_state_row.data(), 0, n_tokens*ggml_element_size(h_rows)); + return; + } + GGML_ASSERT(ubatch->embd_state && "this graph requires a state embedding, see llama_batch_ext_set_embd_state()"); GGML_ASSERT(n_embd_state == h->ne[0]); @@ -229,9 +242,29 @@ bool llm_graph_input_embd_h::can_reuse(const llm_graph_params & params) { res &= (!params.ubatch.embd) || (embd && embd->ne[1] == params.ubatch.n_tokens); res &= (!params.ubatch.embd_state) || (h && h->ne[1] == params.ubatch.n_tokens); + const bool has_rows = params.ubatch.data && !params.ubatch.data->embd_state_row.empty(); + res &= has_rows == (h_rows != nullptr); + res &= !has_rows || h_rows->ne[0] == params.ubatch.n_tokens; + return res; } +void llm_graph_input_batch_idxs::set_input(const llama_ubatch * ubatch) { + const int64_t n_tokens = ubatch->n_tokens; + + GGML_ASSERT(ubatch->data && ubatch->data->batch_idxs.size() == (size_t) n_tokens); + GGML_ASSERT(ggml_backend_buffer_is_host(idxs->buffer)); + + int64_t * data = (int64_t *) idxs->data; + for (int64_t i = 0; i < n_tokens; ++i) { + data[i] = ubatch->data->batch_idxs[i]; + } +} + +bool llm_graph_input_batch_idxs::can_reuse(const llm_graph_params & params) { + return idxs->ne[0] == params.ubatch.n_tokens; +} + void llm_graph_input_pos::set_input(const llama_ubatch * ubatch) { if (ubatch->pos && pos) { const int64_t n_tokens = ubatch->n_tokens; @@ -1380,6 +1413,10 @@ int64_t llm_graph_result::get_max_nodes() const { return max_nodes; } +void llm_graph_result::set_max_nodes(int64_t max_nodes) { + this->max_nodes = max_nodes; +} + void llm_graph_result::reset() { t_inp_tokens = nullptr; t_inp_embd = nullptr; @@ -1395,6 +1432,7 @@ void llm_graph_result::reset() { t_sampled_probs.clear(); t_sampled_logits.clear(); t_candidates.clear(); + t_sampling_all.clear(); params = {}; @@ -1402,11 +1440,16 @@ void llm_graph_result::reset() { fused_nodes.clear(); hdmd_inputs.clear(); - buf_compute_meta.resize(ggml_tensor_overhead()*max_nodes + ggml_graph_overhead_custom(max_nodes, false)); + const size_t meta_size = ggml_tensor_overhead()*max_nodes + ggml_graph_overhead_custom(max_nodes, false); + if (meta_size != buf_compute_meta_size) { + // not make_unique: it would zero the whole reserve and make its pages resident + buf_compute_meta.reset(new uint8_t[meta_size]); + buf_compute_meta_size = meta_size; + } ggml_init_params params = { - /*.mem_size =*/ buf_compute_meta.size(), - /*.mem_buffer =*/ buf_compute_meta.data(), + /*.mem_size =*/ buf_compute_meta_size, + /*.mem_buffer =*/ buf_compute_meta.get(), /*.no_alloc =*/ true, }; @@ -1436,29 +1479,15 @@ void llm_graph_result::set_outputs(const llm_graph_params & params) { } { const auto & embeddings_layer_inp = params.cparams.embeddings_layer_inp; - for (size_t il = 0; il < embeddings_layer_inp.size(); ++il) { + for (size_t il = 0; il < embeddings_layer_inp.size() && !params.cparams.embeddings_layer_inp_dev; ++il) { if (embeddings_layer_inp[il]) { GGML_ASSERT(t_layer_inp[il] != nullptr && "layer input tensor is null"); ggml_set_output(t_layer_inp[il]); } } } - for (auto * tensor : t_sampled) { - if (tensor != nullptr) { - ggml_set_output(tensor); - } - } - for (auto * tensor : t_sampled_probs) { - if (tensor != nullptr) { - ggml_set_output(tensor); - } - } - for (auto * tensor : t_sampled_logits) { - if (tensor != nullptr) { - ggml_set_output(tensor); - } - } - for (auto * tensor : t_candidates) { + // the chains are read on the host after compute, so keep every one alive + for (auto * tensor : t_sampling_all) { if (tensor != nullptr) { ggml_set_output(tensor); } @@ -1524,7 +1553,7 @@ void llm_graph_result::set_params(const llm_graph_params & params) { // llm_graph_context::llm_graph_context(const llm_graph_params & params) : - arch (params.arch), + model (*params.model), hparams (params.hparams), cparams (params.cparams), ubatch (params.ubatch), @@ -1540,7 +1569,7 @@ llm_graph_context::llm_graph_context(const llm_graph_params & params) : n_embd_head_v (hparams.n_embd_head_v()), n_embd_v_gqa (hparams.n_embd_v_gqa()), n_expert (hparams.n_expert), - n_expert_used (cparams.warmup ? hparams.n_expert : hparams.n_expert_used()), + n_expert_used (hparams.n_expert_used()), freq_base (cparams.rope_freq_base), freq_scale (cparams.rope_freq_scale), ext_factor (cparams.yarn_ext_factor), @@ -1563,6 +1592,10 @@ llm_graph_context::llm_graph_context(const llm_graph_params & params) : moe_cache (params.moe_cache), prec_policy (params.prec_policy), hdmd (params.hdmd), + layer_inp_dev (params.layer_inp_dev), + layer_inp_src (params.layer_inp_src), + h_nextn_dev (params.h_nextn_dev), + h_nextn_src (params.h_nextn_src), samplers (params.samplers), cb_func (params.cb), res (params.res), @@ -1616,6 +1649,37 @@ ggml_tensor * llm_graph_context::build_hadamard_input( return x; } +static ggml_tensor * build_rope_impl(ggml_context * ctx, ggml_tensor * cur, ggml_tensor * pos, const llama_hparams & hparams, const llama_rope_params & rp) { + if (rp.n_rot == 0) { + return cur; + } + + if (rp.type & GGML_ROPE_TYPE_MROPE) { + int sections[GGML_MROPE_SECTIONS]; + std::copy(hparams.rope_sections.begin(), hparams.rope_sections.end(), sections); + + return ggml_rope_multi(ctx, cur, pos, rp.factors, + rp.n_rot, sections, rp.type, rp.n_ctx_orig, rp.freq_base, rp.freq_scale, + rp.ext_factor, rp.attn_factor, rp.beta_fast, rp.beta_slow); + } + + return ggml_rope_ext(ctx, cur, pos, rp.factors, + rp.n_rot, rp.type, rp.n_ctx_orig, rp.freq_base, rp.freq_scale, + rp.ext_factor, rp.attn_factor, rp.beta_fast, rp.beta_slow); +} + +ggml_tensor * llm_graph_context::build_rope(ggml_tensor * cur, ggml_tensor * pos, int il) const { + return build_rope_impl(ctx0, cur, pos, hparams, model.get_rope_params(cparams, il)); +} + +ggml_tensor * llm_graph_context::build_rope_indexer(ggml_tensor * cur, ggml_tensor * pos, int il) const { + const llama_rope_params rp = model.get_rope_params_indexer(cparams, il); + + ggml_tensor * res = build_rope_impl(ctx0, cur, pos, hparams, rp); + + return rp.n_rot > 0 && rp.n_offs > 0 ? ggml_rope_set_offset(res, rp.n_offs) : res; +} + ggml_tensor * llm_graph_context::build_lora_mm( ggml_tensor * w, ggml_tensor * cur, @@ -1699,13 +1763,14 @@ ggml_tensor * llm_graph_context::build_lora_mm_id( ggml_tensor * cur, // ggml_tensor * b ggml_tensor * ids, ggml_tensor * w_s, - ggml_tensor * slots) const { + ggml_tensor * slots, + bool cached) const { // as in build_lora_mm: the folded weight reads the transformed input, the LoRA delta reads `cur` ggml_tensor * cur_mm = build_hadamard_input(w, cur); // the experts in the MoE cache are selected by their slots ggml_tensor * res = slots == nullptr ? ggml_mul_mat_id(ctx0, w, cur_mm, ids) : - ggml_mul_mat_id(ctx0, moe_cache->get_experts(w), cur_mm, slots); + ggml_mul_mat_id(ctx0, cached ? moe_cache->get_experts(w) : w, cur_mm, slots); if (prec_policy) { prec_policy->apply(res); @@ -1926,6 +1991,8 @@ ggml_tensor * llm_graph_context::build_ffn( llm_ffn_op_type type_op, llm_ffn_gate_type type_gate, int il) const { + const llm_arch arch = model.arch; + // NVFP4 support is currently restricted to // 1) LORA absence (*_s would be applied after LORA residual, which is incorrect) // 2) bias absense (*_s would be applied after bias addition, which is incorrect) @@ -2113,8 +2180,8 @@ ggml_tensor * llm_graph_context::build_ffn( if (down) { cur = build_lora_mm(down, cur); if (arch == LLM_ARCH_GLM4 || arch == LLM_ARCH_GLM4_MOE || arch == LLM_ARCH_JAIS2) { - // GLM4, GLM4_MOE, and JAIS2 seem to have numerical issues with half-precision accumulators - ggml_prec_set_acc(cur, GGML_PREC_F32); + // GLM4, GLM4_MOE, and JAIS2 have numerical issues with F16 accumulators + ggml_prec_set_acc(cur, GGML_PREC_BF16); } } @@ -2209,6 +2276,7 @@ ggml_tensor * llm_graph_context::build_moe_ffn( ggml_tensor * selected_experts_in, ggml_tensor * post_exp_norm, float post_exp_norm_eps) const { + const llm_arch arch = model.arch; const int64_t n_embd = cur->ne[0]; const int64_t n_tokens = cur->ne[1]; const bool weight_before_ffn = arch == LLM_ARCH_LLAMA4; // for llama4, we apply the sigmoid-ed weights before the FFN @@ -2218,6 +2286,7 @@ ggml_tensor * llm_graph_context::build_moe_ffn( if (probs_in == nullptr) { logits = build_lora_mm(gate_inp, cur); // [n_expert, n_tokens] if (gating_op == LLAMA_EXPERT_GATING_FUNC_TYPE_SQRT_SOFTPLUS) { + // the reference computes the router in F32: https://huggingface.co/deepseek-ai/DeepSeek-V4-Flash/blob/60d8d70770c6776ff598c94bb586a859a38244f1/inference/model.py#L565 ggml_prec_set_acc(logits, GGML_PREC_F32); } cb(logits, "ffn_moe_logits", il); @@ -2353,8 +2422,15 @@ ggml_tensor * llm_graph_context::build_moe_ffn( //call early so that topk-moe can be used ggml_build_forward_expand(gf, weights); - // the experts of host-resident layers may be read from the MoE cache - ggml_tensor * slots = build_moe_cache_slots(selected_experts, up_exps, gate_exps, down_exps, gate_up_exps, il); + // compute the expert input before the slot lookup, so that the scheduler prefetches it for the CPU part (the router can read a different tensor) + ggml_build_forward_expand(gf, cur); + + // the experts of host-resident layers may be read from the MoE cache, and part of the misses computed from the host experts + // the CPU part does not support weights before the FFN, LoRAs or rotated experts + const std::array exps = { up_exps, gate_exps, down_exps, gate_up_exps }; + const bool can_cpu_share = !weight_before_ffn && loras->empty() && + std::none_of(exps.begin(), exps.end(), [&](ggml_tensor * w) { return w != nullptr && hdmd != nullptr && hdmd->rot.count(w) > 0; }); + const llm_graph_moe_cache_slots cache = build_moe_cache_slots(selected_experts, up_exps, gate_exps, down_exps, gate_up_exps, can_cpu_share, il); cur = ggml_reshape_3d(ctx0, cur, n_embd, 1, n_tokens); @@ -2365,145 +2441,146 @@ ggml_tensor * llm_graph_context::build_moe_ffn( cb(cur, "ffn_moe_weighted", il); } - ggml_tensor * up = nullptr; - ggml_tensor * experts = nullptr; + // the experts selected by mm_ids, up to the down projection, from the MoE cache if cached + // ids are the selected experts, for the biases and scales + auto build_experts = [&](ggml_tensor * cur, ggml_tensor * ids, ggml_tensor * mm_ids, bool cached) { + ggml_tensor * up = nullptr; - if (gate_up_exps) { - // merged gate_up path: one mul_mat_id, then split into gate and up views - ggml_tensor * gate_up = build_lora_mm_id(gate_up_exps, cur, selected_experts, up_exps_s, slots); // [n_ff*2, n_expert_used, n_tokens] - cb(gate_up, "ffn_moe_gate_up", il); + if (gate_up_exps) { + // merged gate_up path: one mul_mat_id, then split into gate and up views + ggml_tensor * gate_up = build_lora_mm_id(gate_up_exps, cur, ids, up_exps_s, mm_ids, cached); // [n_ff*2, n_expert_used, n_tokens] + cb(gate_up, "ffn_moe_gate_up", il); - if (up_exps_s) { - cb(gate_up, "ffn_moe_gate_up_scaled", il); - } + if (up_exps_s) { + cb(gate_up, "ffn_moe_gate_up_scaled", il); + } - if (gate_up_exps_b) { - gate_up = ggml_add_id(ctx0, gate_up, gate_up_exps_b, selected_experts); - cb(gate_up, "ffn_moe_gate_up_biased", il); - } + if (gate_up_exps_b) { + gate_up = ggml_add_id(ctx0, gate_up, gate_up_exps_b, ids); + cb(gate_up, "ffn_moe_gate_up_biased", il); + } - const int64_t n_ff = gate_up->ne[0] / 2; - cur = ggml_view_3d(ctx0, gate_up, n_ff, gate_up->ne[1], gate_up->ne[2], gate_up->nb[1], gate_up->nb[2], 0); - cb(cur, "ffn_moe_gate", il); - up = ggml_view_3d(ctx0, gate_up, n_ff, gate_up->ne[1], gate_up->ne[2], gate_up->nb[1], gate_up->nb[2], n_ff * gate_up->nb[0]); - cb(up, "ffn_moe_up", il); - } else { - // separate gate and up path - up = build_lora_mm_id(up_exps, cur, selected_experts, up_exps_s, slots); // [n_ff, n_expert_used, n_tokens] - cb(up, "ffn_moe_up", il); + const int64_t n_ff = gate_up->ne[0] / 2; + cur = ggml_view_3d(ctx0, gate_up, n_ff, gate_up->ne[1], gate_up->ne[2], gate_up->nb[1], gate_up->nb[2], 0); + cb(cur, "ffn_moe_gate", il); + up = ggml_view_3d(ctx0, gate_up, n_ff, gate_up->ne[1], gate_up->ne[2], gate_up->nb[1], gate_up->nb[2], n_ff * gate_up->nb[0]); + cb(up, "ffn_moe_up", il); + } else { + // separate gate and up path + up = build_lora_mm_id(up_exps, cur, ids, up_exps_s, mm_ids, cached); // [n_ff, n_expert_used, n_tokens] + cb(up, "ffn_moe_up", il); - if (up_exps_s) { - cb(up, "ffn_moe_up_scaled", il); - } + if (up_exps_s) { + cb(up, "ffn_moe_up_scaled", il); + } - if (up_exps_b) { - up = ggml_add_id(ctx0, up, up_exps_b, selected_experts); - cb(up, "ffn_moe_up_biased", il); - } + if (up_exps_b) { + up = ggml_add_id(ctx0, up, up_exps_b, ids); + cb(up, "ffn_moe_up_biased", il); + } - if (gate_exps) { - cur = build_lora_mm_id(gate_exps, cur, selected_experts, gate_exps_s, slots); // [n_ff, n_expert_used, n_tokens] - cb(cur, "ffn_moe_gate", il); - } else { - cur = up; - } + if (gate_exps) { + cur = build_lora_mm_id(gate_exps, cur, ids, gate_exps_s, mm_ids, cached); // [n_ff, n_expert_used, n_tokens] + cb(cur, "ffn_moe_gate", il); + } else { + cur = up; + } - if (gate_exps_s) { - cb(cur, "ffn_moe_gate_scaled", il); - } + if (gate_exps_s) { + cb(cur, "ffn_moe_gate_scaled", il); + } - if (gate_exps_b) { - cur = ggml_add_id(ctx0, cur, gate_exps_b, selected_experts); - cb(cur, "ffn_moe_gate_biased", il); + if (gate_exps_b) { + cur = ggml_add_id(ctx0, cur, gate_exps_b, ids); + cb(cur, "ffn_moe_gate_biased", il); + } } - } - - const bool has_gate = gate_exps || gate_up_exps; - switch (type_op) { - case LLM_FFN_SILU: - if (gate_exps) { - if (il >= 0) { - const float limit = hparams.swiglu_clamp_exp[il]; - constexpr float eps = 1e-6f; - if (limit > eps) { - if (arch == LLM_ARCH_MAPLE || arch == LLM_ARCH_DEEPSEEK4 || arch == LLM_ARCH_DEEPSEEK41 || arch == LLM_ARCH_GLM5_NEXT || (arch == LLM_ARCH_DFLASH && hparams.dsv4_hc_mult > 0) || arch == LLM_ARCH_HY_V4) { - cur = ggml_swiglu_clamp(ctx0, cur, up, limit); - } else { - up = ggml_clamp(ctx0, up, -limit, limit); - cb(up, "ffn_moe_up_clamped", il); - ggml_tensor * gate_act = ggml_silu(ctx0, cur); - cb(gate_act, "ffn_moe_silu", il); - gate_act = ggml_clamp(ctx0, gate_act, -INFINITY, limit); - cb(gate_act, "ffn_moe_silu_clamped", il); - cur = ggml_mul(ctx0, gate_act, up); + const bool has_gate = gate_exps || gate_up_exps; + + switch (type_op) { + case LLM_FFN_SILU: + if (gate_exps) { + if (il >= 0) { + const float limit = hparams.swiglu_clamp_exp[il]; + constexpr float eps = 1e-6f; + if (limit > eps) { + if (arch == LLM_ARCH_MAPLE || arch == LLM_ARCH_DEEPSEEK4 || arch == LLM_ARCH_DEEPSEEK41 || arch == LLM_ARCH_GLM5_NEXT || (arch == LLM_ARCH_DFLASH && hparams.dsv4_hc_mult > 0) || arch == LLM_ARCH_HY_V4) { + cur = ggml_swiglu_clamp(ctx0, cur, up, limit); + } else { + up = ggml_clamp(ctx0, up, -limit, limit); + cb(up, "ffn_moe_up_clamped", il); + ggml_tensor * gate_act = ggml_silu(ctx0, cur); + cb(gate_act, "ffn_moe_silu", il); + gate_act = ggml_clamp(ctx0, gate_act, -INFINITY, limit); + cb(gate_act, "ffn_moe_silu_clamped", il); + cur = ggml_mul(ctx0, gate_act, up); + } + cb(cur, "ffn_moe_swiglu_limited", il); + break; } - cb(cur, "ffn_moe_swiglu_limited", il); - break; } } - } - if (has_gate) { - cur = ggml_swiglu_split(ctx0, cur, up); - cb(cur, "ffn_moe_swiglu", il); - } else { - cur = ggml_silu(ctx0, cur); - cb(cur, "ffn_moe_silu", il); - } break; - case LLM_FFN_SITU: - { - // situ(gate, up) = beta*tanh(gate/beta)*sigmoid(gate) * lb*tanh(up/lb) - GGML_ASSERT(has_gate); - const float beta = hparams.situ_beta; - const float lb = hparams.situ_linear_beta; - - ggml_tensor * act = ggml_scale(ctx0, ggml_tanh(ctx0, ggml_scale(ctx0, cur, 1.0f/beta)), beta); - act = ggml_mul(ctx0, act, ggml_sigmoid(ctx0, cur)); - if (lb > 0.0f) { - up = ggml_scale(ctx0, ggml_tanh(ctx0, ggml_scale(ctx0, up, 1.0f/lb)), lb); - } - cur = ggml_mul(ctx0, act, up); - cb(cur, "ffn_moe_situ", il); - } break; - case LLM_FFN_GELU: - if (has_gate) { - cur = ggml_geglu_split(ctx0, cur, up); - cb(cur, "ffn_moe_geglu", il); - } else { - cur = ggml_gelu(ctx0, cur); - cb(cur, "ffn_moe_gelu", il); - } break; - case LLM_FFN_SWIGLU_OAI_MOE: - { - // TODO: move to hparams? - constexpr float alpha = 1.702f; - constexpr float limit = 7.0f; - cur = ggml_swiglu_oai(ctx0, cur, up, alpha, limit); - cb(cur, "ffn_moe_swiglu_oai", il); - } break; - case LLM_FFN_RELU: - if (has_gate) { - cur = ggml_reglu_split(ctx0, cur, up); - cb(cur, "ffn_moe_reglu", il); - } else { - cur = ggml_relu(ctx0, cur); - cb(cur, "ffn_moe_relu", il); - } break; - case LLM_FFN_RELU_SQR: - if (has_gate) { - // TODO: add support for gated squared relu - GGML_ABORT("fatal error: gated squared relu not implemented"); - } else { - cur = ggml_relu(ctx0, cur); - cur = ggml_sqr(ctx0, cur); - cb(cur, "ffn_moe_relu_sqr", il); - } break; - default: - GGML_ABORT("fatal error"); - } + if (has_gate) { + cur = ggml_swiglu_split(ctx0, cur, up); + cb(cur, "ffn_moe_swiglu", il); + } else { + cur = ggml_silu(ctx0, cur); + cb(cur, "ffn_moe_silu", il); + } break; + case LLM_FFN_SITU: + { + // situ(gate, up) = beta*tanh(gate/beta)*sigmoid(gate) * lb*tanh(up/lb) + GGML_ASSERT(has_gate); + const float beta = hparams.situ_beta; + const float lb = hparams.situ_linear_beta; + + ggml_tensor * act = ggml_scale(ctx0, ggml_tanh(ctx0, ggml_scale(ctx0, cur, 1.0f/beta)), beta); + act = ggml_mul(ctx0, act, ggml_sigmoid(ctx0, cur)); + if (lb > 0.0f) { + up = ggml_scale(ctx0, ggml_tanh(ctx0, ggml_scale(ctx0, up, 1.0f/lb)), lb); + } + cur = ggml_mul(ctx0, act, up); + cb(cur, "ffn_moe_situ", il); + } break; + case LLM_FFN_GELU: + if (has_gate) { + cur = ggml_geglu_split(ctx0, cur, up); + cb(cur, "ffn_moe_geglu", il); + } else { + cur = ggml_gelu(ctx0, cur); + cb(cur, "ffn_moe_gelu", il); + } break; + case LLM_FFN_SWIGLU_OAI_MOE: + { + // TODO: move to hparams? + constexpr float alpha = 1.702f; + constexpr float limit = 7.0f; + cur = ggml_swiglu_oai(ctx0, cur, up, alpha, limit); + cb(cur, "ffn_moe_swiglu_oai", il); + } break; + case LLM_FFN_RELU: + if (has_gate) { + cur = ggml_reglu_split(ctx0, cur, up); + cb(cur, "ffn_moe_reglu", il); + } else { + cur = ggml_relu(ctx0, cur); + cb(cur, "ffn_moe_relu", il); + } break; + case LLM_FFN_RELU_SQR: + if (has_gate) { + // TODO: add support for gated squared relu + GGML_ABORT("fatal error: gated squared relu not implemented"); + } else { + cur = ggml_relu(ctx0, cur); + cur = ggml_sqr(ctx0, cur); + cb(cur, "ffn_moe_relu_sqr", il); + } break; + default: + GGML_ABORT("fatal error"); + } - { // TODO: this graph-level rescaling should only be applied when the corresponding backend does not support // ggml_prec_set_src(experts, GGML_PREC_BF16, 1); // the amax op cannot be split across devices, so these architectures are denied -sm tensor [TAG_MOE_AMAX_SCALE] @@ -2515,13 +2592,24 @@ ggml_tensor * llm_graph_context::build_moe_ffn( cur = amax_src1.res; } - experts = build_lora_mm_id(down_exps, cur, selected_experts, down_exps_s, slots); // [n_embd, n_expert_used, n_tokens] + ggml_tensor * experts = build_lora_mm_id(down_exps, cur, ids, down_exps_s, mm_ids, cached); // [n_embd, n_expert_used, n_tokens] //ggml_prec_set_src(experts, GGML_PREC_BF16, 1); // (see TODO above) cb(experts, "ffn_moe_down", il); if (amax_src1.scale_s) { experts = ggml_mul(ctx0, experts, amax_src1.scale_s); } + + return experts; + }; + + ggml_tensor * experts = build_experts(cur, selected_experts, cache.slots, true); + if (cache.cpu_ids != nullptr) { + // the cached experts and the misses computed from the host experts, each reads the others as zero + // the cached experts come first in the graph, so that they are computed while the misses are + ggml_build_forward_expand(gf, experts); + experts = ggml_add(ctx0, experts, build_experts(cur, selected_experts, cache.cpu_ids, false)); + cb(experts, "ffn_moe_down_merged", il); } if (down_exps_s) { @@ -2581,24 +2669,35 @@ ggml_tensor * llm_graph_context::build_moe_ffn( return moe_out; } -ggml_tensor * llm_graph_context::build_moe_cache_slots( +llm_graph_moe_cache_slots llm_graph_context::build_moe_cache_slots( ggml_tensor * selected_experts, ggml_tensor * up_exps, ggml_tensor * gate_exps, ggml_tensor * down_exps, ggml_tensor * gate_up_exps, + bool cpu_part, int il) const { if (moe_cache == nullptr) { - return nullptr; + return { nullptr, nullptr }; } - ggml_tensor * slot_map = moe_cache->get_slot_map(il, selected_experts->ne[1], selected_experts->ne[0]); + ggml_tensor * cpu_map = cpu_part ? moe_cache->get_cpu_map(il, selected_experts->ne[1]) : nullptr; + ggml_tensor * slot_map = moe_cache->get_slot_map(il, selected_experts->ne[1], selected_experts->ne[0], cpu_map != nullptr); if (slot_map == nullptr) { - return nullptr; + return { nullptr, nullptr }; } for (ggml_tensor * w : { up_exps, gate_exps, down_exps, gate_up_exps }) { - if (w != nullptr && moe_cache->get_experts(w) == nullptr) { - return nullptr; + if (w == nullptr) { + continue; + } + ggml_tensor * experts = moe_cache->get_experts(w); + if (experts == nullptr) { + return { nullptr, nullptr }; + } + // the device of the cache must compute the matmul, else the scheduler copies the cache to another backend + ggml_tensor * b = ggml_new_tensor_3d(ctx0, GGML_TYPE_F32, experts->ne[0], w == down_exps ? selected_experts->ne[0] : 1, selected_experts->ne[1]); + if (!ggml_backend_supports_op(moe_cache->backend(il), ggml_mul_mat_id(ctx0, experts, b, selected_experts))) { + return { nullptr, nullptr }; } } @@ -2612,7 +2711,7 @@ ggml_tensor * llm_graph_context::build_moe_cache_slots( // the callback reads the selected experts, uploads the missing ones and updates the slot map ggml_tensor * slots = ggml_get_rows(ctx0, slot_map, ids); // [1, n_expert_used*n_tokens] if (!ggml_backend_supports_op(moe_cache->backend(il), slots)) { - return nullptr; + return { nullptr, nullptr }; } ggml_backend_sched_set_tensor_backend(sched, slots, moe_cache->backend(il)); cb(slots, "ffn_moe_slots", il); @@ -2620,7 +2719,14 @@ ggml_tensor * llm_graph_context::build_moe_cache_slots( // add the lookup now: the transform of a Hadamard-folded expert reads host weights and would start its split first ggml_build_forward_expand(gf, slots); - return ggml_reshape_2d(ctx0, slots, selected_experts->ne[0], selected_experts->ne[1]); // [n_expert_used, n_tokens] + // the misses computed from the host experts, read after the lookup has picked them + ggml_tensor * cpu_ids = nullptr; + if (cpu_map != nullptr) { + cpu_ids = ggml_get_rows(ctx0, cpu_map, ids); // [1, n_expert_used*n_tokens] + cpu_ids = ggml_reshape_2d(ctx0, cpu_ids, selected_experts->ne[0], selected_experts->ne[1]); + } + + return { ggml_reshape_2d(ctx0, slots, selected_experts->ne[0], selected_experts->ne[1]), cpu_ids }; } // input embeddings with optional lora @@ -2634,7 +2740,7 @@ ggml_tensor * llm_graph_context::build_inp_embd(ggml_tensor * tok_embd, float to // mixed path (ubatch.is_mixed()): set_rows the token rows into a copy of the embd rows, with its own inputs as select branches must not share tensors // TODO: use inp->tokens and inp->embd once ggml_build_forward_select allows it - const bool has_mixed = llm_arch_supports_mixed_batch(arch) && cparams.ctx_type == LLAMA_CONTEXT_TYPE_DEFAULT; + const bool has_mixed = llm_arch_supports_mixed_batch(model.arch) && cparams.ctx_type == LLAMA_CONTEXT_TYPE_DEFAULT; const int64_t n_tok_rows = has_mixed ? llm_graph_n_tok_rows(ubatch) : 0; @@ -2794,6 +2900,24 @@ ggml_tensor * llm_graph_context::build_inp_embd(ggml_tensor * tok_embd, float to return cur; } +ggml_tensor * llm_graph_context::build_inp_embd_state(llm_graph_input_embd_h * inp, int64_t n_embd_state) const { + if (ubatch.data && !ubatch.data->embd_state_row.empty()) { + GGML_ASSERT(h_nextn_src != nullptr && h_nextn_src->ne[0] == n_embd_state); + + inp->h_rows = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_tokens); + ggml_set_input(inp->h_rows); + ggml_set_name(inp->h_rows, "mtp_h_rows"); + + return ggml_get_rows(ctx0, h_nextn_src, inp->h_rows); + } + + ggml_tensor * h = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_embd_state, n_tokens); + ggml_set_input(h); + ggml_set_name(h, "mtp_h_input"); + + return h; +} + ggml_tensor * llm_graph_context::build_inp_pos() const { auto inp = std::make_unique(hparams.n_pos_per_embd(), hparams.rope_section_order); @@ -2860,7 +2984,7 @@ ggml_tensor * llm_graph_context::build_inp_mean() const { } ggml_tensor * llm_graph_context::build_inp_cls() const { - auto inp = std::make_unique(cparams, arch); + auto inp = std::make_unique(cparams, model.arch); auto & cur = inp->cls; @@ -3027,11 +3151,9 @@ ggml_tensor * llm_graph_context::build_attn_mha( ggml_tensor * kq = ggml_mul_mat(ctx0, k, q); cb(kq, "kq", il); - // note: this op tends to require high floating point range - // while for some models F16 is enough, for others it is not, so we default to F32 here - ggml_prec_set_acc(kq, GGML_PREC_F32); + ggml_prec_set_acc(kq, GGML_PREC_BF16); - if (arch == LLM_ARCH_GROK) { + if (model.arch == LLM_ARCH_GROK) { // need to do the following: // multiply by attn_output_multiplier // and then : @@ -3106,16 +3228,11 @@ llm_graph_input_attn_no_cache * llm_graph_context::build_attn_inp_no_cache() con ggml_set_input(inp->self_attn_mask.raw); cb(inp->self_attn_mask.raw, "self_kq_mask", -1); - inp->self_attn_mask.cnv = inp->self_attn_mask.raw; - if (hparams.swa_type != LLAMA_SWA_TYPE_NONE) { inp->self_attn_mask_swa.raw = ggml_new_tensor_4d(ctx0, type_mask, n_kv, n_tokens, 1, 1); ggml_set_input(inp->self_attn_mask_swa.raw); - - inp->self_attn_mask_swa.cnv = inp->self_attn_mask_swa.raw; } else { inp->self_attn_mask_swa.raw = nullptr; - inp->self_attn_mask_swa.cnv = nullptr; } return (llm_graph_input_attn_no_cache *) res->add_input(std::move(inp)); @@ -3144,7 +3261,7 @@ ggml_tensor * llm_graph_context::build_attn( const bool is_swa = hparams.is_swa(il); - const auto & kq_mask = is_swa ? inp->self_attn_mask_swa.cnv : inp->self_attn_mask.cnv; + const auto & kq_mask = is_swa ? inp->self_attn_mask_swa.raw : inp->self_attn_mask.raw; const auto & kv_rows = is_swa ? inp->self_attn_mask_swa.rows : inp->self_attn_mask.rows; // [TAG_NO_CACHE_PAD] @@ -3216,6 +3333,43 @@ llm_graph_input_attn_kv * llm_graph_context::build_attn_inp_kv() const { return (llm_graph_input_attn_kv *) res->add_input(std::move(inp)); } +static void build_attn_scale_kv_inputs( + ggml_context * ctx, + const llama_kv_cache_context * mctx, + ggml_tensor * & q, + ggml_tensor * & k, + ggml_tensor * & v, + int32_t il) { + if (ggml_tensor * scale = mctx->get_k_scale(il)) { + // Store K/s and move its dequant scale to Q: (Q*s) dot (K/s) = Q dot K. + q = ggml_mul(ctx, q, scale); + if (k) { + k = ggml_div(ctx, k, scale); + } + } + + if (v) { + // Store V/s here and restore its dequant scale after the weighted sum. + ggml_tensor * scale = mctx->get_v_scale(il); + if (scale) { + v = ggml_div(ctx, v, scale); + } + } +} + +static ggml_tensor * build_attn_scale_kv_output( + ggml_context * ctx, + const llama_kv_cache_context * mctx, + ggml_tensor * cur, + int32_t il) { + if (ggml_tensor * scale = mctx->get_v_scale(il)) { + // P*(V/s)*s = P*V. + cur = ggml_mul(ctx, cur, scale); + } + + return cur; +} + ggml_tensor * llm_graph_context::build_attn( llm_graph_input_attn_kv * inp, ggml_tensor * wo, @@ -3229,6 +3383,8 @@ ggml_tensor * llm_graph_context::build_attn( ggml_tensor * v_mla, // TODO: remove float kq_scale, int il) const { + const llm_arch arch = model.arch; + GGML_ASSERT(v_mla == nullptr); if (inp->self_k_rot[il]) { @@ -3240,6 +3396,10 @@ ggml_tensor * llm_graph_context::build_attn( v_cur = llama_mul_mat_hadamard(ctx0, v_cur, inp->self_v_rot[il]); } + const auto * mctx_cur = inp->mctx; + + build_attn_scale_kv_inputs(ctx0, mctx_cur, q_cur, k_cur, v_cur, il); + // these nodes are added to the graph together so that they are not reordered // by doing so, the number of splits in the graph is reduced // expand k later to enable rope fusion which directly writes into k-v cache @@ -3247,8 +3407,6 @@ ggml_tensor * llm_graph_context::build_attn( ggml_build_forward_expand(gf, v_cur); ggml_build_forward_expand(gf, k_cur); - const auto * mctx_cur = inp->mctx; - ggml_tensor * q = q_cur; ggml_tensor * k; ggml_tensor * v; @@ -3271,10 +3429,11 @@ ggml_tensor * llm_graph_context::build_attn( v = mctx_cur->get_v(ctx0, il); } - const auto & kq_mask = inp->self_attn_mask.cnv; + const auto & kq_mask = inp->self_attn_mask.raw; const auto & kv_rows = inp->self_attn_mask.rows; ggml_tensor * cur = build_attn_mha(q, k, v, kq_b, kq_mask, kv_rows, sinks, v_mla, 0, kq_scale, il); + cur = build_attn_scale_kv_output(ctx0, mctx_cur, cur, il); cb(cur, "kqv_out", il); if (inp->self_v_rot[il]) { @@ -3283,9 +3442,9 @@ ggml_tensor * llm_graph_context::build_attn( if (wo) { if (arch == LLM_ARCH_GLM4 || arch == LLM_ARCH_GLM4_MOE || arch == LLM_ARCH_JAIS2) { - // GLM4, GLM4_MOE, and JAIS2 seem to have numerical issues with half-precision accumulators + // GLM4, GLM4_MOE, and JAIS2 have numerical issues with F16 accumulators cur = build_lora_mm(wo, cur); - ggml_prec_set_acc(cur, GGML_PREC_F32); + ggml_prec_set_acc(cur, GGML_PREC_BF16); if (wo_s) { cur = ggml_mul(ctx0, cur, wo_s); } @@ -3316,7 +3475,6 @@ static std::unique_ptr build_attn_inp_k_impl( inp->self_k_idxs = mctx_cur->build_input_k_idxs(ctx0, ubatch); inp->self_attn_mask.raw = build_attn_inp_kq_mask(ctx0, mctx_cur, ubatch, cparams); - inp->self_attn_mask.cnv = inp->self_attn_mask.raw; } return inp; @@ -3343,6 +3501,8 @@ ggml_tensor * llm_graph_context::build_attn( ggml_tensor * v_mla, float kq_scale, int il) const { + const llm_arch arch = model.arch; + // these nodes are added to the graph together so that they are not reordered // by doing so, the number of splits in the graph is reduced // expand k later to enable rope fusion which directly writes into k-v cache @@ -3359,7 +3519,7 @@ ggml_tensor * llm_graph_context::build_attn( ggml_build_forward_expand(gf, mctx_cur->cpy_k(ctx0, k_cur, k_idxs, il)); } - const auto & kq_mask = inp->self_attn_mask.cnv; + const auto & kq_mask = inp->self_attn_mask.raw; const auto & kv_rows = inp->self_attn_mask.rows; ggml_tensor * q = q_cur; @@ -3371,9 +3531,9 @@ ggml_tensor * llm_graph_context::build_attn( if (wo) { if (arch == LLM_ARCH_GLM4 || arch == LLM_ARCH_GLM4_MOE) { - // GLM4 and GLM4_MOE seem to have numerical issues with half-precision accumulators + // GLM4 and GLM4_MOE have numerical issues with F16 accumulators cur = build_lora_mm(wo, cur); - ggml_prec_set_acc(cur, GGML_PREC_F32); + ggml_prec_set_acc(cur, GGML_PREC_BF16); if (wo_s) { cur = ggml_mul(ctx0, cur, wo_s); } @@ -3401,6 +3561,41 @@ ggml_tensor * llm_graph_context::build_fused_top_k_mask( return cur; } +ggml_tensor * llm_graph_context::build_top_k_mask( + ggml_tensor * kq_mask, + ggml_tensor * top_k, + int il) const { + if (cparams.fused_top_k_mask) { + return build_fused_top_k_mask(kq_mask, top_k, il); + } + + // prepare new kq mask - starts filled with -INFINITY + ggml_tensor * kq_mask_all = ggml_fill(ctx0, kq_mask, -INFINITY); + + // reshape KQ mask into tensor with rows of size 1: + // [n_kv, n_batch, 1, n_stream] -> [1, n_kv, n_batch, n_stream] + kq_mask_all = ggml_view_4d(ctx0, kq_mask_all, 1, kq_mask_all->ne[0], kq_mask_all->ne[1], kq_mask_all->ne[3], kq_mask_all->nb[0], kq_mask_all->nb[1], kq_mask_all->nb[2], 0); + + // reshape top_k indices: [n_top_k, n_batch, 1, n_stream] -> [n_top_k, n_batch, n_stream, 1] + ggml_tensor * top_k_3d = ggml_view_4d(ctx0, top_k, top_k->ne[0], top_k->ne[1], top_k->ne[3], 1, top_k->nb[1], top_k->nb[2], top_k->ne[3]*top_k->nb[3], 0); + + // prepare zero-filled tensor with rows of size 1: [1, n_top_k, n_batch, n_stream] + // this will be our source of zero values for unmasking top k mask elements + ggml_tensor * zeros = ggml_new_tensor_4d(ctx0, GGML_TYPE_F32, 1, top_k_3d->ne[0], top_k_3d->ne[1], top_k_3d->ne[2]); + zeros = ggml_fill(ctx0, zeros, 0.0f); + + // modify KQ mask by unmasking elements that are in top_k indices + // ggml_set_rows([1, n_kv, n_batch, n_stream], [1, n_top_k, n_batch, n_stream], [n_top_k, n_batch, n_stream, 1]) + ggml_tensor * kq_mask_top_k = ggml_set_rows(ctx0, kq_mask_all, zeros, top_k_3d); + + // reshape to restore the original shape of KQ mask: + // [1, n_kv, n_batch, n_stream] -> [n_kv, n_batch, 1, n_stream] + kq_mask_top_k = ggml_view_4d(ctx0, kq_mask_top_k, kq_mask_top_k->ne[1], kq_mask_top_k->ne[2], 1, kq_mask_top_k->ne[3], kq_mask_top_k->nb[2], kq_mask_top_k->nb[3], kq_mask_top_k->nb[3], 0); + + // combine with the original kq mask + return ggml_add(ctx0, kq_mask_top_k, kq_mask); +} + ggml_tensor * llm_graph_context::build_attn( llm_graph_input_attn_k_dsa * inp, ggml_tensor * wo, @@ -3440,41 +3635,12 @@ ggml_tensor * llm_graph_context::build_attn( // top_k indexes the cells, and the indexer mask of the same type is built the same way // let's use it, so when no other layer reads the MLA mask is not allocated - ggml_tensor * kq_mask = inp->self_attn_mask_mla.cnv; - if (inp->self_attn_mask_lid.cnv->type == kq_mask->type) { - kq_mask = inp->self_attn_mask_lid.cnv; + ggml_tensor * kq_mask = inp->self_attn_mask_mla.raw; + if (inp->self_attn_mask_lid.raw->type == kq_mask->type) { + kq_mask = inp->self_attn_mask_lid.raw; } - ggml_tensor * kq_mask_top_k = nullptr; - if (cparams.fused_top_k_mask) { - kq_mask_top_k = build_fused_top_k_mask(kq_mask, top_k, il); - } else { - // prepare new kq mask - starts filled with -INFINITY - ggml_tensor * kq_mask_all = ggml_fill(ctx0, kq_mask, -INFINITY); - - // reshape KQ mask into tensor with rows of size 1: - // [n_kv, n_batch, 1, n_stream] -> [1, n_kv, n_batch, n_stream] - kq_mask_all = ggml_view_4d(ctx0, kq_mask_all, 1, kq_mask_all->ne[0], kq_mask_all->ne[1], kq_mask_all->ne[3], kq_mask_all->nb[0], kq_mask_all->nb[1], kq_mask_all->nb[2], 0); - - // reshape top_k indices: [n_top_k, n_batch, 1, n_stream] -> [n_top_k, n_batch, n_stream, 1] - ggml_tensor * top_k_3d = ggml_view_4d(ctx0, top_k, top_k->ne[0], top_k->ne[1], top_k->ne[3], 1, top_k->nb[1], top_k->nb[2], top_k->ne[3]*top_k->nb[3], 0); - - // prepare zero-filled tensor with rows of size 1: [1, n_top_k, n_batch, n_stream] - // this will be our source of zero values for unmasking top k mask elements - ggml_tensor * zeros = ggml_new_tensor_4d(ctx0, GGML_TYPE_F32, 1, top_k_3d->ne[0], top_k_3d->ne[1], top_k_3d->ne[2]); - zeros = ggml_fill(ctx0, zeros, 0.0f); - - // modify KQ mask by unmasking elements that are in top_k indices - // ggml_set_rows([1, n_kv, n_batch, n_stream], [1, n_top_k, n_batch, n_stream], [n_top_k, n_batch, n_stream, 1]) - kq_mask_top_k = ggml_set_rows(ctx0, kq_mask_all, zeros, top_k_3d); - - // reshape to restore the original shape of KQ mask: - // [1, n_kv, n_batch, n_stream] -> [n_kv, n_batch, 1, n_stream] - kq_mask_top_k = ggml_view_4d(ctx0, kq_mask_top_k, kq_mask_top_k->ne[1], kq_mask_top_k->ne[2], 1, kq_mask_top_k->ne[3], kq_mask_top_k->nb[2], kq_mask_top_k->nb[3], kq_mask_top_k->nb[3], 0); - - // combine with the original kq mask - kq_mask_top_k = ggml_add(ctx0, kq_mask_top_k, kq_mask); - } + ggml_tensor * kq_mask_top_k = build_top_k_mask(kq_mask, top_k, il); ggml_tensor * q = q_cur; ggml_tensor * k = mctx_cur->get_k(ctx0, il); @@ -3527,6 +3693,11 @@ ggml_tensor * llm_graph_context::build_attn( } } + const auto * mctx_iswa = inp->mctx; + const auto * mctx_cur = is_swa ? mctx_iswa->get_swa() : mctx_iswa->get_base(); + + build_attn_scale_kv_inputs(ctx0, mctx_cur, q_cur, k_cur, v_cur, il); + // these nodes are added to the graph together so that they are not reordered // by doing so, the number of splits in the graph is reduced ggml_build_forward_expand(gf, q_cur); @@ -3539,10 +3710,6 @@ ggml_tensor * llm_graph_context::build_attn( ggml_build_forward_expand(gf, v_cur); } - const auto * mctx_iswa = inp->mctx; - - const auto * mctx_cur = is_swa ? mctx_iswa->get_swa() : mctx_iswa->get_base(); - // whole seq fits into batch in training mode const bool use_kv_cur = cparams.training && k_cur && v_cur; if (use_kv_cur) { @@ -3564,7 +3731,7 @@ ggml_tensor * llm_graph_context::build_attn( ggml_build_forward_expand(gf, mctx_cur->cpy_v(ctx0, v_cur, v_idxs, il)); } - const auto & kq_mask = is_swa ? inp->self_attn_mask_swa.cnv : inp->self_attn_mask.cnv; + const auto & kq_mask = is_swa ? inp->self_attn_mask_swa.raw : inp->self_attn_mask.raw; const auto & kv_rows = is_swa ? inp->self_attn_mask_swa.rows : inp->self_attn_mask.rows; ggml_tensor * q = q_cur; @@ -3572,6 +3739,7 @@ ggml_tensor * llm_graph_context::build_attn( ggml_tensor * v = use_kv_cur ? v_cur : mctx_cur->get_v(ctx0, il); ggml_tensor * cur = build_attn_mha(q, k, v, kq_b, kq_mask, kv_rows, sinks, v_mla, 0, kq_scale, il); + cur = build_attn_scale_kv_output(ctx0, mctx_cur, cur, il); cb(cur, "kqv_out", il); if (v_rot) { @@ -3635,7 +3803,7 @@ ggml_tensor * llm_graph_context::build_attn( ggml_build_forward_expand(gf, mctx_cur->cpy_k(ctx0, k_cur, k_idxs, il)); } - const auto & kq_mask = is_swa ? inp->self_attn_mask_swa.cnv : inp->self_attn_mask.cnv; + const auto & kq_mask = is_swa ? inp->self_attn_mask_swa.raw : inp->self_attn_mask.raw; const auto & kv_rows = is_swa ? inp->self_attn_mask_swa.rows : inp->self_attn_mask.rows; // MLA-style attention: the cached K is used as V @@ -3672,8 +3840,6 @@ llm_graph_input_attn_cross * llm_graph_context::build_attn_inp_cross() const { inp->cross_attn_mask.raw = ggml_new_tensor_4d(ctx0, type_mask, n_enc, n_tokens, 1, 1); ggml_set_input(inp->cross_attn_mask.raw); - inp->cross_attn_mask.cnv = inp->cross_attn_mask.raw; - return (llm_graph_input_attn_cross *) res->add_input(std::move(inp)); } @@ -3696,7 +3862,7 @@ ggml_tensor * llm_graph_context::build_attn( ggml_build_forward_expand(gf, k_cur); ggml_build_forward_expand(gf, v_cur); - const auto & kq_mask = inp->cross_attn_mask.cnv; + const auto & kq_mask = inp->cross_attn_mask.raw; const auto & kv_rows = inp->cross_attn_mask.rows; ggml_tensor * q = q_cur; @@ -3734,7 +3900,6 @@ static std::unique_ptr build_attn_inp_k_dsa_impl( inp->self_k_idxs_mla = mctx_cur->get_mla()->build_input_k_idxs(ctx0, ubatch); inp->self_attn_mask_mla.raw = build_attn_inp_kq_mask(ctx0, mctx_cur->get_mla(), ubatch, cparams); - inp->self_attn_mask_mla.cnv = inp->self_attn_mask_mla.raw; } { @@ -3745,7 +3910,6 @@ static std::unique_ptr build_attn_inp_k_dsa_impl( cparams_copy.flash_attn = cparams.fused_lid; inp->self_attn_mask_lid.raw = build_attn_inp_kq_mask(ctx0, mctx_cur->get_lid(), ubatch, cparams_copy); - inp->self_attn_mask_lid.cnv = inp->self_attn_mask_lid.raw; inp->self_k_rot_lid = mctx_cur->get_lid()->get_k_rot(); } @@ -3772,7 +3936,6 @@ llm_graph_input_attn_k_dsa_iswa * llm_graph_context::build_attn_inp_k_dsa_iswa() inp_swa->self_k_idxs = mctx_cur->get_swa()->build_input_k_idxs(ctx0, ubatch); inp_swa->self_attn_mask.raw = build_attn_inp_kq_mask(ctx0, mctx_cur->get_swa(), ubatch, cparams); - inp_swa->self_attn_mask.cnv = inp_swa->self_attn_mask.raw; auto inp = std::make_unique(std::move(inp_dsa), std::move(inp_swa), mctx_cur); @@ -3794,7 +3957,6 @@ llm_graph_input_attn_kv_msa * llm_graph_context::build_attn_inp_kv_msa(bool msa_ inp->self_v_idxs = mctx_base->build_input_v_idxs(ctx0, ubatch); inp->self_attn_mask.raw = build_attn_inp_kq_mask(ctx0, mctx_base, ubatch, cparams); - inp->self_attn_mask.cnv = inp->self_attn_mask.raw; } inp->self_k_rot = mctx_base->get_k_rot(); @@ -3849,7 +4011,6 @@ llm_graph_input_attn_k_iswa * llm_graph_context::build_attn_inp_k_iswa() const { inp->self_k_idxs = mctx_cur->get_base()->build_input_k_idxs(ctx0, ubatch); inp->self_attn_mask.raw = build_attn_inp_kq_mask(ctx0, mctx_cur->get_base(), ubatch, cparams); - inp->self_attn_mask.cnv = inp->self_attn_mask.raw; } { @@ -3858,7 +4019,6 @@ llm_graph_input_attn_k_iswa * llm_graph_context::build_attn_inp_k_iswa() const { inp->self_k_idxs_swa = mctx_cur->get_swa()->build_input_k_idxs(ctx0, ubatch); inp->self_attn_mask_swa.raw = build_attn_inp_kq_mask(ctx0, mctx_cur->get_swa(), ubatch, cparams); - inp->self_attn_mask_swa.cnv = inp->self_attn_mask_swa.raw; } inp->self_k_rot = mctx_cur->get_base()->get_k_rot(); @@ -3880,7 +4040,6 @@ llm_graph_input_dsv4 * llm_graph_context::build_inp_dsv4() const { inp_raw->self_k_idxs = raw_ctx->build_input_k_idxs(ctx0, ubatch); inp_raw->self_attn_mask.raw = dsv4_build_raw_kq_mask(ctx0, raw_ctx, ubatch, cparams, n_stream); - inp_raw->self_attn_mask.cnv = inp_raw->self_attn_mask.raw; inp_raw->self_k_rot = raw_ctx->get_k_rot(); auto inp = std::make_unique(cparams, std::move(inp_raw), mctx_cur); @@ -3986,6 +4145,67 @@ ggml_tensor * llm_graph_context::build_rs( get_state_rows); } +void llm_graph_context::build_conv_state_store( + llm_graph_input_rs * inp, + ggml_tensor * s, + ggml_tensor * conv_x, + int64_t n_cols, + size_t offs) const { + const auto * mctx_cur = inp->mctx; + + const int64_t n = conv_x->ne[0] - n_cols; // n_seq_tokens + const int64_t channels = conv_x->ne[1]; + const int64_t n_seqs = conv_x->ne[2]; + + const size_t row_size = s->nb[1]; + const size_t cell_offs = mctx_cur->get_head()*row_size + offs; + + if (cparams.n_rs_seq == 0) { + ggml_tensor * last = ggml_view_3d(ctx0, conv_x, + n_cols, channels, n_seqs, + conv_x->nb[1], conv_x->nb[2], + ggml_row_size(conv_x->type, n)); + + ggml_tensor * cell = ggml_view_2d(ctx0, s, + n_cols*channels, n_seqs, + row_size, cell_offs); + + ggml_build_forward_expand(gf, ggml_cpy(ctx0, last, cell)); + return; + } + + // [TAG_RECURRENT_ROLLBACK_SPLITS] + // this logic assumes that the last (n_rs_seq + 1) tokens of a sequence in a batch are inside + // the same ubatch, which `split_equal()` guarantees via its n_keep_tail argument + // a rollback of k tokens reads slot k, and the memory allows k < n only (see rs_max), so slots from n on are left unwritten + const int64_t K = (int64_t) cparams.n_rs_seq + 1; + const int64_t Kw = std::min(n, K); + + // each conv layer builds its own slot order {K-1, ..., 0}, on the device of the layer, and takes its tail + // {Kw-1, ..., 0}; the size K never depends on the ubatch, so the reserved graph covers them all + ggml_tensor * s_slot_rev = ggml_arange(ctx0, 0.0f, (float) K, 1.0f); + s_slot_rev = ggml_scale_bias(ctx0, s_slot_rev, -1.0f, (float) (K - 1)); + s_slot_rev = ggml_cast(ctx0, s_slot_rev, GGML_TYPE_I32); + cb(s_slot_rev, "s_slot_rev", -1); + + ggml_tensor * slot_order = ggml_view_1d(ctx0, s_slot_rev, Kw, ggml_row_size(GGML_TYPE_I32, K - Kw)); + + // window w starts at column n - (Kw - 1) + w (ascending time) and is rollback slot Kw - 1 - w, + // consecutive windows start one element apart, so they overlap + ggml_tensor * windows = ggml_view_4d(ctx0, conv_x, + n_cols, Kw, channels, n_seqs, + ggml_element_size(conv_x), conv_x->nb[1], conv_x->nb[2], + ggml_row_size(conv_x->type, n - (Kw - 1))); + + // [n_cols, slot, channel, seq] view of the cells of this ubatch + ggml_tensor * slots = ggml_view_4d(ctx0, s, + n_cols, K, channels, n_seqs, + (size_t) mctx_cur->get_size()*row_size, ggml_row_size(s->type, n_cols), row_size, + cell_offs); + + ggml_build_forward_expand(gf, ggml_set_rows(ctx0, slots, windows, slot_order)); +} + ggml_tensor * llm_graph_context::build_rwkv_token_shift_load( llm_graph_input_rs * inp, const llama_ubatch & ubatch, @@ -4064,7 +4284,6 @@ llm_graph_input_mem_hybrid_iswa * llm_graph_context::build_inp_mem_hybrid_iswa() inp_attn->self_v_idxs = attn_ctx->get_base()->build_input_v_idxs(ctx0, ubatch); inp_attn->self_attn_mask.raw = build_attn_inp_kq_mask(ctx0, attn_ctx->get_base(), ubatch, cparams); - inp_attn->self_attn_mask.cnv = inp_attn->self_attn_mask.raw; } { @@ -4072,7 +4291,6 @@ llm_graph_input_mem_hybrid_iswa * llm_graph_context::build_inp_mem_hybrid_iswa() inp_attn->self_v_idxs_swa = attn_ctx->get_swa()->build_input_v_idxs(ctx0, ubatch); inp_attn->self_attn_mask_swa.raw = build_attn_inp_kq_mask(ctx0, attn_ctx->get_swa(), ubatch, cparams); - inp_attn->self_attn_mask_swa.cnv = inp_attn->self_attn_mask_swa.raw; } inp_attn->self_k_rot = attn_ctx->get_base()->get_k_rot(); @@ -4117,6 +4335,8 @@ void llm_graph_context::build_pooling( ggml_tensor * cls_out, ggml_tensor * cls_out_b, ggml_tensor * cls_norm) const { + const llm_arch arch = model.arch; + if (!cparams.embeddings) { // a graph without logits still computes t_embd, so that it is never empty if (res->t_logits == nullptr && res->t_embd != nullptr) { @@ -4223,6 +4443,55 @@ void llm_graph_context::build_logits() const { } } +void llm_graph_context::build_layer_inp_dev() const { + if (!cparams.embeddings_layer_inp_dev || layer_inp_dev == nullptr) { + return; + } + + auto inp = std::make_unique(); + + inp->idxs = ggml_new_tensor_1d(ctx0, GGML_TYPE_I64, n_tokens); + ggml_set_input(inp->idxs); + + // row layout: [layer_0 | layer_1 | ...] in ascending layer id order + int64_t col = 0; + for (size_t il = 0; il < cparams.embeddings_layer_inp.size(); ++il) { + if (!cparams.embeddings_layer_inp[il]) { + continue; + } + + ggml_tensor * src = res->t_layer_inp[il]; + GGML_ASSERT(src != nullptr && "layer input tensor is null"); + GGML_ASSERT(src->ne[0] == n_embd); + + ggml_tensor * dst = ggml_view_2d(ctx0, layer_inp_dev, n_embd, layer_inp_dev->ne[1], layer_inp_dev->nb[1], col*n_embd*ggml_element_size(layer_inp_dev)); + ggml_build_forward_expand(gf, ggml_set_rows(ctx0, dst, ggml_reshape_2d(ctx0, src, n_embd, n_tokens), inp->idxs)); + col++; + } + GGML_ASSERT(col*n_embd == layer_inp_dev->ne[0]); + + res->add_input(std::move(inp)); +} + +void llm_graph_context::build_h_nextn_dev() const { + ggml_tensor * src = res->t_h_nextn; + if (h_nextn_dev == nullptr || src == nullptr) { + return; + } + + // the device buffer is only enabled for the unmasked extraction, which has a row per batch token + GGML_ASSERT(src->ne[0] == h_nextn_dev->ne[0] && ggml_nrows(src) == n_tokens); + + auto inp = std::make_unique(); + + inp->idxs = ggml_new_tensor_1d(ctx0, GGML_TYPE_I64, n_tokens); + ggml_set_input(inp->idxs); + + ggml_build_forward_expand(gf, ggml_set_rows(ctx0, h_nextn_dev, ggml_reshape_2d(ctx0, src, src->ne[0], n_tokens), inp->idxs)); + + res->add_input(std::move(inp)); +} + void llm_graph_context::build_sampling() const { if (samplers.empty() || !res->t_logits) { return; @@ -4285,6 +4554,11 @@ void llm_graph_context::build_sampling() const { assert(sampler->iface->backend_apply); sampler->iface->backend_apply(sampler, ctx0, gf, &data); + if (data.sampled != nullptr) { res->t_sampling_all.push_back(data.sampled); } + if (data.probs != nullptr) { res->t_sampling_all.push_back(data.probs); } + if (data.logits != nullptr) { res->t_sampling_all.push_back(data.logits); } + if (data.candidates != nullptr) { res->t_sampling_all.push_back(data.candidates); } + if (data.sampled != nullptr) { if (active) { res->t_sampled[row] = data.sampled; diff --git a/src/llama-graph.h b/src/llama-graph.h index b528ee11df46..d3b527dce9b4 100644 --- a/src/llama-graph.h +++ b/src/llama-graph.h @@ -19,6 +19,7 @@ struct ggml_tensor; struct llama_cparams; struct llama_layer; +struct llama_model; struct llama_prec_policy; struct llama_hadamard; @@ -48,8 +49,6 @@ enum llm_graph_type { enum llm_fused_op { LLM_FUSED_OP_FLASH_ATTN, LLM_FUSED_OP_FLASH_ATTN_KV_ROWS, - LLM_FUSED_OP_GDN_AR, - LLM_FUSED_OP_GDN_CH, LLM_FUSED_OP_LIGHTNING_INDEXER, LLM_FUSED_OP_DSV4_HC_PRE, LLM_FUSED_OP_DSV4_HC_COMB, @@ -147,6 +146,7 @@ class llm_graph_input_embd : public llm_graph_input_i { ggml_tensor * mixed_slots = nullptr; // I64 [n_tok_rows], mixed path: batch index of the token rows ggml_tensor * mixed_embd = nullptr; // F32 [n_embd, n_batch], mixed path: embd rows, token rows are overwritten ggml_tensor * scale_rows = nullptr; // F32 [1, n_batch], per-row scale: scale_tok for token rows, 1 for embd rows + ggml_tensor * embd_rows = nullptr; // I32 [n_batch], embd row path: rows of ctx_other's layer input device buffer float scale_tok = 1.0f; @@ -165,12 +165,26 @@ class llm_graph_input_embd_h : public llm_graph_input_i { ggml_tensor * tokens = nullptr; // I32 [n_batch] ggml_tensor * embd = nullptr; // F32 [n_embd, n_batch] - ggml_tensor * h = nullptr; // F32 [n_embd_state, n_batch] + ggml_tensor * h = nullptr; // F32 [n_embd_state, n_batch], the host input or the rows gathered with h_rows + ggml_tensor * h_rows = nullptr; // I32 [n_batch], rows of the h_nextn device buffer of ctx_state_src, nullptr on the host path const int64_t n_embd = 0; const int64_t n_embd_state = 0; }; +// original batch index of each ubatch token, used as set_rows destination rows +class llm_graph_input_batch_idxs : public llm_graph_input_i { +public: + llm_graph_input_batch_idxs() = default; + virtual ~llm_graph_input_batch_idxs() = default; + + void set_input(const llama_ubatch * ubatch) override; + + bool can_reuse(const llm_graph_params & params) override; + + ggml_tensor * idxs = nullptr; // I64 [n_batch] +}; + class llm_graph_input_pos : public llm_graph_input_i { public: llm_graph_input_pos(uint32_t n_pos_per_embd, llama_rope_section_order rope_section_order = LLAMA_ROPE_SECTION_ORDER_TYXZ) @@ -319,12 +333,13 @@ class llm_graph_input_cross_embd : public llm_graph_input_i { // the mask has two shapes depending on whether `rows` is used: // rows != nullptr: [n_kv_rows, n_batch/n_slices, 1, n_slices] // rows == nullptr: [n_kv, n_batch/n_stream, 1, n_stream] -// TODO: cnv is redundant (always aliases raw) except dflash overwrites it with a derived block mask [TAG_ATTN_MASK_CNV] struct llm_attn_mask { ggml_tensor * raw = nullptr; - ggml_tensor * cnv = nullptr; // same shape ggml_tensor * rows = nullptr; // I32 [n_kv_rows, n_slices] uint32_t n_kv = 0; + + // fill every token of a sequence with the mask row of its first token (dspark draft blocks) + bool block_uniform = false; }; class llm_graph_input_attn_no_cache : public llm_graph_input_i { @@ -772,7 +787,7 @@ using llm_graph_cb = std::function samplers; static bool samplers_equal( @@ -833,7 +853,8 @@ struct llm_graph_params { (!ubatch.embd && !other.ubatch.embd) || (ubatch.token && other.ubatch.token && ubatch.embd && other.ubatch.embd) ) && - (!ubatch.embd_state == !other.ubatch.embd_state); + (!ubatch.embd_state == !other.ubatch.embd_state) && + (!ubatch.embd_row == !other.ubatch.embd_row); // when we split the batch using "equal_seqs" we have to verify that the participating sequences are the same // the reason is because the set of attention streams would be different for different sequences @@ -885,7 +906,11 @@ struct llm_graph_params { cparams.embeddings_nextn == other.cparams.embeddings_nextn && cparams.embeddings_nextn_masked == other.cparams.embeddings_nextn_masked && cparams.causal_attn == other.cparams.causal_attn && - arch == other.arch && + layer_inp_dev == other.layer_inp_dev && + layer_inp_src == other.layer_inp_src && + h_nextn_dev == other.h_nextn_dev && + h_nextn_src == other.h_nextn_src && + model == other.model && gtype == other.gtype && cvec == other.cvec && loras == other.loras && @@ -918,6 +943,8 @@ class llm_graph_result { int64_t get_max_nodes() const; + void set_max_nodes(int64_t max_nodes); + void reset(); void set_inputs(const llama_ubatch * ubatch); @@ -957,6 +984,8 @@ class llm_graph_result { std::vector t_sampled_logits; std::vector t_candidates; + std::vector t_sampling_all; + std::vector inputs; std::vector fused_nodes; @@ -965,13 +994,14 @@ class llm_graph_result { ggml_context_ptr ctx_compute; // memory buffers used to evaluate the model - std::vector buf_compute_meta; + std::unique_ptr buf_compute_meta; + size_t buf_compute_meta_size = 0; ggml_cgraph * gf; +private: int64_t max_nodes; -private: // keep a copy of the previous graph parameters // we will use this to determine whether the graph can be reused by comparing them with the new parameters // note: these are updated after constructing the new graph @@ -1001,8 +1031,13 @@ struct llm_graph_amax_src1 { ggml_tensor * scale_s; // scale to undo on the matmul result, or nullptr }; +struct llm_graph_moe_cache_slots { + ggml_tensor * slots; // [n_expert_used, n_tokens] slots of the selected experts in the MoE cache, or nullptr + ggml_tensor * cpu_ids; // [n_expert_used, n_tokens] misses computed from the host experts, -1 for the other experts, or nullptr +}; + struct llm_graph_context { - const llm_arch arch; + const llama_model & model; const llama_hparams & hparams; const llama_cparams & cparams; @@ -1052,6 +1087,11 @@ struct llm_graph_context { const llama_hadamard * hdmd; + ggml_tensor * layer_inp_dev; + ggml_tensor * layer_inp_src; + ggml_tensor * h_nextn_dev; + ggml_tensor * h_nextn_src; + std::map samplers; const llm_graph_cb & cb_func; @@ -1096,13 +1136,14 @@ struct llm_graph_context { ggml_tensor * w_s = nullptr) const; // do mat_mul_id, while optionally apply lora and per-expert scale - // if slots is set, the experts are read from the MoE cache at these slots (see build_moe_cache_slots) + // if slots is set, the experts are selected by slots instead of ids, from the MoE cache if cached (see build_moe_cache_slots) ggml_tensor * build_lora_mm_id( ggml_tensor * w, // ggml_tensor * as ggml_tensor * cur, // ggml_tensor * b ggml_tensor * ids, - ggml_tensor * w_s = nullptr, - ggml_tensor * slots = nullptr) const; + ggml_tensor * w_s = nullptr, + ggml_tensor * slots = nullptr, + bool cached = true) const; // rescale `cur` (the src1 of a matmul) by a power of two, computed from its maximum absolute // keep both individual values and Q8 activation block sums within F16 range @@ -1209,12 +1250,14 @@ struct llm_graph_context { float post_exp_norm_eps = 1e-5f) const; // the slots of the selected experts in the MoE cache, nullptr if the experts of the layer are not read from the cache - ggml_tensor * build_moe_cache_slots( + // with cpu_part, also the misses computed from the host experts, if the cache leaves any to the CPU + llm_graph_moe_cache_slots build_moe_cache_slots( ggml_tensor * selected_experts, ggml_tensor * up_exps, ggml_tensor * gate_exps, ggml_tensor * down_exps, ggml_tensor * gate_up_exps, + bool cpu_part, int il) const; // @@ -1223,7 +1266,18 @@ struct llm_graph_context { // tok_scale: applied to token rows only ggml_tensor * build_inp_embd(ggml_tensor * tok_embd, float tok_scale = 1.0f) const; + + // the state input of an MTP graph: rows of the h_nextn device buffer of ctx_state_src (see llama_set_ctx_state_src()), + // or uploaded from ubatch.embd_state + ggml_tensor * build_inp_embd_state(llm_graph_input_embd_h * inp, int64_t n_embd_state) const; + ggml_tensor * build_inp_pos() const; + + // applies the RoPE of layer il, cur is returned as is if the layer has none + ggml_tensor * build_rope(ggml_tensor * cur, ggml_tensor * pos, int il) const; + + // same, for the lightning indexer + ggml_tensor * build_rope_indexer(ggml_tensor * cur, ggml_tensor * pos, int il) const; ggml_tensor * build_inp_attn_scale() const; ggml_tensor * build_inp_out_ids() const; ggml_tensor * build_inp_mean() const; @@ -1326,6 +1380,12 @@ struct llm_graph_context { ggml_tensor * top_k, // [n_top, n_tokens] int il) const; + // kq_mask with only the cells named by top_k left visible, for sparse attention + ggml_tensor * build_top_k_mask( + ggml_tensor * kq_mask, // [n_kv, n_tokens/n_stream, 1, n_stream] + ggml_tensor * top_k, // [n_top, n_tokens] + int il) const; + llm_graph_input_attn_kv_iswa * build_attn_inp_kv_iswa() const; llm_graph_input_dsv4 * build_inp_dsv4() const; @@ -1409,6 +1469,15 @@ struct llm_graph_context { int32_t n_seqs, const llm_graph_get_rows_fn & get_state_rows = nullptr) const; + // store the conv history of the ubatch cells, the last n_cols columns of conv_x {n_cols + n_seq_tokens, channels, n_seqs}, + // at byte offset offs of each row of s, and with n_rs_seq > 0 once per rollback slot of this ubatch + void build_conv_state_store( + llm_graph_input_rs * inp, + ggml_tensor * s, + ggml_tensor * conv_x, + int64_t n_cols, + size_t offs) const; + ggml_tensor * build_rwkv_token_shift_load( llm_graph_input_rs * inp, const llama_ubatch & ubatch, @@ -1450,6 +1519,12 @@ struct llm_graph_context { void build_sampling() const; + // write enabled layer inputs into layer_inp_dev at their batch rows + void build_layer_inp_dev() const; + + // write the unmasked h_nextn rows into h_nextn_dev at their batch rows + void build_h_nextn_dev() const; + // // dense (out) // diff --git a/src/llama-hparams.h b/src/llama-hparams.h index 4d2cb89314d2..7b738407ddb1 100644 --- a/src/llama-hparams.h +++ b/src/llama-hparams.h @@ -209,6 +209,7 @@ struct llama_hparams { uint32_t ssm_d_state = 0; uint32_t ssm_dt_rank = 0; uint32_t ssm_n_group = 0; + float ssm_norm_eps = 0.0f; // 0 = use f_norm_rms_eps // for MiniMax-Text-01 linear attention uint32_t n_embd_head_la = 0; @@ -230,6 +231,13 @@ struct llama_hparams { uint32_t n_hrm_l_cycles = 0; bool hrm_prefix_lm = false; + // metis16 (passes over shared layers, depth memory) + uint32_t metis_n_pass = 0; + uint32_t metis_n_mem = 0; + uint32_t metis_n_route = 0; + uint32_t metis_pass_lora_rank = 0; + float metis_cont_limit = 0.0f; + // limite float f_xsa_eps = 0.0f; std::array final_logit_sigmoid_capping = {}; // scale, shift, temperature @@ -296,6 +304,9 @@ struct llama_hparams { // realtime models: number of tokens the output lags behind the audio input uint32_t n_tokens_delay = 0; + // each sequence is paired with an unconditional one + bool classifier_free_guidance = false; + // gemma3n altup uint32_t n_altup = 4; // altup_num_inputs uint32_t i_altup_act = 0; // altup_active_idx @@ -423,6 +434,7 @@ struct llama_hparams { // Step35: optional per-layer clamps for (Swi)GLU std::array swiglu_clamp_exp; // clamping for expert FFN std::array swiglu_clamp_shexp; // shared expert + std::array swiglu_clamp; // dense FFN // this value n_pattern means that every nth layer is dense (i.e. non-SWA) // dense_first means whether the pattern is start with a dense layer diff --git a/src/llama-kv-cache-dsa.cpp b/src/llama-kv-cache-dsa.cpp index 405eee36b7ae..1221f7f2bfae 100644 --- a/src/llama-kv-cache-dsa.cpp +++ b/src/llama-kv-cache-dsa.cpp @@ -44,7 +44,6 @@ llama_kv_cache_dsa::llama_kv_cache_dsa( // DSA lightning indexer uses MQA with single key head std::fill(hparams_lid.n_head_kv_arr.begin(), hparams_lid.n_head_kv_arr.end(), 1); hparams_lid.n_embd_head_k_full = model.hparams.indexer_head_size; - hparams_lid.rope_type = LLAMA_ROPE_TYPE_NEOX; // KV sharing applies to the MLA cache only, each indexer owns its keys std::iota(hparams_lid.kv_source_impl.begin(), hparams_lid.kv_source_impl.end(), 0); @@ -54,7 +53,7 @@ llama_kv_cache_dsa::llama_kv_cache_dsa( kv_lid = std::make_unique( model, no_alloc, hparams_lid, type_k, type_v, v_trans, offload, unified, kv_size, n_seq_max, - n_swa, swa_type, nullptr, filter_lid, nullptr, nullptr); + n_swa, swa_type, nullptr, filter_lid, nullptr, nullptr, "", true); } void llama_kv_cache_dsa::clear(bool data) { diff --git a/src/llama-kv-cache-dsv4.cpp b/src/llama-kv-cache-dsv4.cpp index 10da95ae85cc..ef1be94a27a9 100644 --- a/src/llama-kv-cache-dsv4.cpp +++ b/src/llama-kv-cache-dsv4.cpp @@ -160,6 +160,7 @@ static llama_ubatch dsv4_build_raw_write_ubatch(const llama_ubatch & ubatch) { /*.token =*/ data->token.empty() ? nullptr : data->token.data(), /*.embd =*/ nullptr, /*.embd_state =*/ nullptr, + /*.embd_row =*/ nullptr, /*.pos =*/ data->pos.data(), /*.n_seq_id =*/ data->n_seq_id.data(), /*.seq_id =*/ data->seq_id.data(), @@ -427,25 +428,6 @@ static void dsv4_state_read_k_cache( dsv4_state_read_tensor_streams(io, kv->get_k_storage(il), kv_size, n_rows_ref, s0, ns); } - - if (n_rows_ref == 0) { - return; - } - - // refill the mirrors from their sources; a mirror gets the same rows as its source, so a whole copy is exact - for (uint32_t il : kv->get_layer_ids()) { - const int32_t src = kv_src[il]; - if (src == (int32_t) il) { - continue; - } - ggml_tensor * t_src = kv->get_k_storage(src); - ggml_tensor * t_dst = kv->get_k_storage(il); - if (t_src->type != t_dst->type || !ggml_are_same_shape(t_src, t_dst)) { - throw std::runtime_error("DSV4 K-cache mirror layout mismatch"); - } - - ggml_backend_tensor_copy(t_src, t_dst); - } } static std::string dsv4_plan_positions(const std::vector & values) { @@ -1796,6 +1778,9 @@ void llama_kv_cache_dsv4::state_read(llama_io_read_i & io, llama_seq_id seq_id, dsv4_state_read_k_cache(io, kv_csa.get(), seq_id, flags, comp_il_src); dsv4_state_read_k_cache(io, kv_hca.get(), seq_id, flags, comp_il_src); dsv4_state_read_k_cache(io, kv_lid.get(), seq_id, flags, comp_il_src); + + // the io can defer the tensor reads until it is destroyed, so the mirrors are refilled before the next graph + comp_mirrors_stale = true; } csa_state->state_read(io, seq_id, flags); @@ -1881,6 +1866,30 @@ void llama_kv_cache_dsv4::update_rs_for_ubatches(const std::vector } } +void llama_kv_cache_dsv4::refill_comp_mirrors() { + if (!comp_mirrors_stale) { + return; + } + + comp_mirrors_stale = false; + + // a mirror gets the same rows as its source, so a whole copy is exact + for (const llama_kv_cache * kv : { kv_csa.get(), kv_hca.get(), kv_lid.get() }) { + for (uint32_t il : kv->get_layer_ids()) { + const int32_t src = comp_il_src[il]; + if (src == (int32_t) il) { + continue; + } + + ggml_tensor * t_src = kv->get_k_storage(src); + ggml_tensor * t_dst = kv->get_k_storage(il); + GGML_ASSERT(t_src->type == t_dst->type && ggml_are_same_shape(t_src, t_dst)); + + ggml_backend_tensor_copy(t_src, t_dst); + } + } +} + void llama_kv_cache_dsv4::clear_compressed(llama_seq_id seq_id, bool data) { if (seq_id < 0) { kv_csa->clear(data); @@ -2280,6 +2289,8 @@ bool llama_kv_cache_dsv4_context::apply() { lid_state->apply_copies(sc_info_lid); } + kv->refill_comp_mirrors(); + return res; } diff --git a/src/llama-kv-cache-dsv4.h b/src/llama-kv-cache-dsv4.h index 1c8d23ed2801..50cb11b626ad 100644 --- a/src/llama-kv-cache-dsv4.h +++ b/src/llama-kv-cache-dsv4.h @@ -157,6 +157,9 @@ class llama_kv_cache_dsv4 : public llama_memory_i { const std::vector & get_rs_idx() const; void update_rs_for_ubatches(const std::vector & ubatches); + // copy the sources into their mirrors after a state read + void refill_comp_mirrors(); + private: llama_hparams hparams_raw; llama_hparams hparams_csa; @@ -178,6 +181,8 @@ class llama_kv_cache_dsv4 : public llama_memory_i { // [n_layer] layer -> layer whose compressed-KV storage it reads (itself, its V4.1 kv source or a mirror), -1 = none std::vector comp_read; + bool comp_mirrors_stale = false; + std::unique_ptr kv_raw; std::unique_ptr kv_csa; std::unique_ptr kv_hca; @@ -422,5 +427,5 @@ class llama_kv_cache_dsv4_context : public llama_memory_context_i { const llama_memory_status status; - const llama_kv_cache_dsv4 * kv = nullptr; + llama_kv_cache_dsv4 * kv = nullptr; }; diff --git a/src/llama-kv-cache.cpp b/src/llama-kv-cache.cpp index 84f547ae18d6..4ec7fb89c14c 100644 --- a/src/llama-kv-cache.cpp +++ b/src/llama-kv-cache.cpp @@ -79,9 +79,10 @@ llama_kv_cache::llama_kv_cache( const layer_filter_cb & filter, const layer_reuse_cb & reuse, const layer_share_cb & share, - const char * name_tag) : + const char * name_tag, + bool is_indexer) : model(model), hparams(hparams), v_trans(v_trans), - n_seq_max(n_seq_max), n_stream(unified ? 1 : n_seq_max), n_swa(n_swa), + n_seq_max(n_seq_max), n_stream(unified ? 1 : n_seq_max), n_swa(n_swa), is_indexer(is_indexer), attn_rot_bufs(no_alloc), swa_type(swa_type), ctxs_bufs(no_alloc), other(static_cast(mem_other)), v_cells_impl(other ? other->v_cells_impl : std::make_shared()), @@ -245,7 +246,7 @@ llama_kv_cache::llama_kv_cache( map_layer_ids[il] = layers.size(); - layers.push_back({ il, k, v, k_stream, v_stream, }); + layers.push_back({ il, k, v, model.layers[il].k_cache_s, model.layers[il].v_cache_s, k_stream, v_stream, }); } if (reuse) { @@ -1413,7 +1414,8 @@ uint32_t llama_kv_cache::get_n_kv_rows(const llama_ubatch & ubatch) const { } // pad like n_kv so that the graph can be reused - return std::min(GGML_PAD(n_max, n_pad_cur), get_size()); + // never 0, it means that kv_rows are not used (f.ex. the sequences have no cells yet in a cache that shares the cells of another one) + return std::min(std::max(n_pad_cur, GGML_PAD(n_max, n_pad_cur)), get_size()); } ggml_tensor * llama_kv_cache::get_k(ggml_context * ctx, int32_t il, uint32_t n_kv, const slot_info & sinfo) const { @@ -1468,6 +1470,30 @@ ggml_tensor * llama_kv_cache::get_v(ggml_context * ctx, int32_t il, uint32_t n_k ggml_row_size(v->type, kv_size*n_embd_v_gqa)*sinfo.s0); } +static bool is_type_scaled(ggml_type type) { + return type == GGML_TYPE_F8_E4M3; +} + +ggml_tensor * llama_kv_cache::get_k_scale(int32_t il) const { + if (!is_type_scaled(type_k())) { + return nullptr; + } + + const int32_t ikv = map_layer_ids.at(il); + + return layers[ikv].k_scale; +} + +ggml_tensor * llama_kv_cache::get_v_scale(int32_t il) const { + if (!is_type_scaled(type_v())) { + return nullptr; + } + + const int32_t ikv = map_layer_ids.at(il); + + return layers[ikv].v_scale; +} + ggml_tensor * llama_kv_cache::cpy_k(ggml_context * ctx, ggml_tensor * k_cur, ggml_tensor * k_idxs, int32_t il, const slot_info & sinfo) const { GGML_UNUSED(sinfo); @@ -1682,6 +1708,9 @@ struct args_set_input_attn_mask { // cell of each mask column per stream, nullptr if the columns are the cells const int32_t * rows; + + // every token of a sequence reuses the mask row of the first one (dspark draft blocks) + bool block_uniform; }; template @@ -1708,10 +1737,15 @@ static void set_input_attn_mask_impl(const args_set_input_attn_mask & args, T * llama_pos seq_pos_min[LLAMA_MAX_SEQ]; std::fill(seq_pos_min, seq_pos_min + LLAMA_MAX_SEQ, INT32_MAX); + // first token of each sequence, used when the mask rows are shared within a sequence + uint32_t seq_first[LLAMA_MAX_SEQ]; + std::fill(seq_first, seq_first + LLAMA_MAX_SEQ, ubatch->n_tokens); + for (uint32_t i = 0; i < ubatch->n_tokens; ++i) { const llama_seq_id seq_id = ubatch->seq_id[i][0]; seq_pos_min[seq_id] = std::min(seq_pos_min[seq_id], ubatch->pos[i]); + seq_first[seq_id] = std::min(seq_first[seq_id], i); } for (uint32_t s = 0; s < n_stream; ++s) { @@ -1728,12 +1762,15 @@ static void set_input_attn_mask_impl(const args_set_input_attn_mask & args, T * const auto & cells = v_cells.at(seq_to_stream[seq_id]); + // shared mask rows: every query of the sequence uses the row of its first token + const uint32_t iq = args.block_uniform ? seq_first[seq_id] : i; + llama_pos p0 = -1; - const llama_pos p1 = ubatch->pos[i]; + const llama_pos p1 = ubatch->pos[iq]; // for M-RoPE - const llama_pos p1_x = is_2d ? ubatch->pos[i + ubatch->n_tokens*2] : 0; - const llama_pos p1_y = is_2d ? ubatch->pos[i + ubatch->n_tokens] : 0; + const llama_pos p1_x = is_2d ? ubatch->pos[iq + ubatch->n_tokens*2] : 0; + const llama_pos p1_y = is_2d ? ubatch->pos[iq + ubatch->n_tokens] : 0; const uint64_t idst = n_kv*i; @@ -1940,6 +1977,7 @@ void llama_kv_cache::set_input_attn_mask(const llm_attn_mask & attn_mask, const /*.n_stream =*/ n_stream, /*.n_tps =*/ n_tps, /*.rows =*/ rows, + /*.block_uniform =*/ attn_mask.block_uniform, }; if (dst->type == GGML_TYPE_F16) { @@ -2076,53 +2114,69 @@ size_t llama_kv_cache::size_v_bytes() const { ggml_tensor * llama_kv_cache::build_rope_shift( const llama_cparams & cparams, ggml_context * ctx, - ggml_tensor * cur, + ggml_tensor * k, ggml_tensor * shift, ggml_tensor * rot, - ggml_tensor * factors, - float freq_base, - float freq_scale, uint32_t il) const { - const auto & n_ctx_orig = cparams.n_ctx_orig_yarn; - - const auto & yarn_ext_factor = cparams.yarn_ext_factor; - const auto & yarn_beta_fast = cparams.yarn_beta_fast; - const auto & yarn_beta_slow = cparams.yarn_beta_slow; - const auto & yarn_attn_factor = cparams.yarn_attn_factor; + const llama_rope_params rp = is_indexer ? model.get_rope_params_indexer(cparams, il) : model.get_rope_params(cparams, il); - const auto & n_rot = hparams.n_rot(il); - const auto & rope_type = hparams.rope_type == LLAMA_ROPE_TYPE_MROPE || hparams.rope_type == LLAMA_ROPE_TYPE_IMROPE + const auto rope_type = rp.type == LLAMA_ROPE_TYPE_MROPE || rp.type == LLAMA_ROPE_TYPE_IMROPE // @ngxson : this is a workaround // for M-RoPE, we want to rotate the whole vector when doing KV shift // a normal RoPE should work, we just need to use the correct ordering // ref: https://github.com/ggml-org/llama.cpp/pull/13870 ? LLAMA_ROPE_TYPE_NEOX - : hparams.rope_type; - ggml_tensor * tmp; + : rp.type; - if (ggml_is_quantized(cur->type)) { - // dequantize to f32 -> RoPE -> quantize back - tmp = ggml_cast(ctx, cur, GGML_TYPE_F32); + const int64_t n_head_kv = hparams.n_head_kv(il); + const int64_t n_embd_k_gqa = hparams.n_embd_k_gqa(il); + const int64_t n_embd_head_k = hparams.n_embd_head_k(il); + const int64_t n_cells = ggml_nelements(k)/n_embd_k_gqa; - // rotate back - tmp = llama_mul_mat_hadamard(ctx, tmp, rot); + if (!ggml_is_quantized(k->type) && k->type != GGML_TYPE_F8_E4M3 && !rot && !rp.k_norm) { + // we rotate only the n_rot dimensions + ggml_tensor * k_rope = + ggml_view_3d(ctx, k, + rp.n_rot, n_head_kv, n_cells, + ggml_row_size(k->type, n_embd_head_k), + ggml_row_size(k->type, n_embd_k_gqa), + ggml_row_size(k->type, rp.n_offs)); - tmp = ggml_rope_ext(ctx, tmp, - shift, factors, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - yarn_ext_factor, yarn_attn_factor, yarn_beta_fast, yarn_beta_slow); + return ggml_rope_ext_inplace(ctx, k_rope, + shift, rp.factors, rp.n_rot, rope_type, rp.n_ctx_orig, rp.freq_base, rp.freq_scale, + rp.ext_factor, rp.attn_factor, rp.beta_fast, rp.beta_slow); + } - // rotate fwd + // convert to f32 -> RoPE -> convert back, on whole rows as a head is not always a multiple of the block size + ggml_tensor * tmp = ggml_reshape_3d(ctx, ggml_cast(ctx, k, GGML_TYPE_F32), n_embd_head_k, n_head_kv, n_cells); + + // rotate back + if (rot) { tmp = llama_mul_mat_hadamard(ctx, tmp, rot); + } - tmp = ggml_cpy(ctx, tmp, cur); - } else { - // we rotate only the first n_rot dimensions - tmp = ggml_rope_ext_inplace(ctx, cur, - shift, factors, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - yarn_ext_factor, yarn_attn_factor, yarn_beta_fast, yarn_beta_slow); + // the RMS norm commutes with RoPE, but the weight does not + if (rp.k_norm) { + tmp = ggml_div(ctx, tmp, rp.k_norm); + } + + tmp = ggml_rope_ext(ctx, tmp, + shift, rp.factors, rp.n_rot, rope_type, rp.n_ctx_orig, rp.freq_base, rp.freq_scale, + rp.ext_factor, rp.attn_factor, rp.beta_fast, rp.beta_slow); + if (rp.n_offs > 0) { + tmp = ggml_rope_set_offset(tmp, rp.n_offs); + } + + if (rp.k_norm) { + tmp = ggml_mul(ctx, tmp, rp.k_norm); + } + + // rotate fwd + if (rot) { + tmp = llama_mul_mat_hadamard(ctx, tmp, rot); } - return tmp; + return ggml_cpy(ctx, tmp, k); } class llm_graph_input_k_shift : public llm_graph_input_i { @@ -2166,26 +2220,7 @@ ggml_cgraph * llama_kv_cache::build_graph_shift(llm_graph_result * res, llama_co continue; } - const int64_t n_head_kv = hparams.n_head_kv(il); - const int64_t n_embd_k_gqa = hparams.n_embd_k_gqa(il); - - const auto n_rot = hparams.n_rot(il); - const auto n_embd_head_k = hparams.n_embd_head_k(il); - const auto n_embd_nope = hparams.n_lora_kv > 0 ? n_embd_head_k - n_rot : 0; - - const float freq_base_l = model.get_rope_freq_base (cparams, il); - const float freq_scale_l = model.get_rope_freq_scale(cparams, il); - - ggml_tensor * rope_factors = model.get_rope_factors(cparams, il); - - ggml_tensor * k = - ggml_view_3d(ctx, layer.k, - n_rot, n_head_kv, get_size()*n_stream, - ggml_row_size(layer.k->type, n_embd_head_k), - ggml_row_size(layer.k->type, n_embd_k_gqa), - ggml_row_size(layer.k->type, n_embd_nope)); - - ggml_tensor * cur = build_rope_shift(cparams, ctx, k, inp->k_shift, layer.k_rot, rope_factors, freq_base_l, freq_scale_l, il); + ggml_tensor * cur = build_rope_shift(cparams, ctx, layer.k, inp->k_shift, layer.k_rot, il); ggml_build_forward_expand(gf, cur); } @@ -3038,6 +3073,14 @@ ggml_tensor * llama_kv_cache_context::get_v(ggml_context * ctx, int32_t il) cons return kv->get_v(ctx, il, n_kv, sinfos[i_cur]); } +ggml_tensor * llama_kv_cache_context::get_k_scale(int32_t il) const { + return kv->get_k_scale(il); +} + +ggml_tensor * llama_kv_cache_context::get_v_scale(int32_t il) const { + return kv->get_v_scale(il); +} + ggml_tensor * llama_kv_cache_context::cpy_k(ggml_context * ctx, ggml_tensor * k_cur, ggml_tensor * k_idxs, int32_t il) const { return kv->cpy_k(ctx, k_cur, k_idxs, il, sinfos[i_cur]); } diff --git a/src/llama-kv-cache.h b/src/llama-kv-cache.h index 9ed36c562a45..a81cd8498b3c 100644 --- a/src/llama-kv-cache.h +++ b/src/llama-kv-cache.h @@ -114,7 +114,9 @@ class llama_kv_cache : public llama_memory_i { const layer_reuse_cb & reuse, const layer_share_cb & share, // a model can hold more than one cache, so the tensor names have to stay unique - const char * name_tag = ""); + const char * name_tag = "", + // lightning indexer keys are rotated with the indexer RoPE parameters + bool is_indexer = false); ~llama_kv_cache() = default; @@ -205,6 +207,9 @@ class llama_kv_cache : public llama_memory_i { ggml_tensor * get_k(ggml_context * ctx, int32_t il, uint32_t n_kv, const slot_info & sinfo) const; ggml_tensor * get_v(ggml_context * ctx, int32_t il, uint32_t n_kv, const slot_info & sinfo) const; + ggml_tensor * get_k_scale(int32_t il) const; + ggml_tensor * get_v_scale(int32_t il) const; + // store k_cur and v_cur in the cache based on the provided head location ggml_tensor * cpy_k(ggml_context * ctx, ggml_tensor * k_cur, ggml_tensor * k_idxs, int32_t il, const slot_info & sinfo) const; ggml_tensor * cpy_v(ggml_context * ctx, ggml_tensor * v_cur, ggml_tensor * v_idxs, int32_t il, const slot_info & sinfo) const; @@ -268,6 +273,9 @@ class llama_kv_cache : public llama_memory_i { ggml_tensor * k; ggml_tensor * v; + ggml_tensor * k_scale; + ggml_tensor * v_scale; + std::vector k_stream; std::vector v_stream; @@ -283,6 +291,8 @@ class llama_kv_cache : public llama_memory_i { // SWA const uint32_t n_swa = 0; + const bool is_indexer = false; + // env: LLAMA_ATTN_ROT_DISABLE uint32_t n_rot_k = 0; uint32_t n_rot_v = 0; @@ -339,12 +349,9 @@ class llama_kv_cache : public llama_memory_i { ggml_tensor * build_rope_shift( const llama_cparams & cparams, ggml_context * ctx, - ggml_tensor * cur, + ggml_tensor * k, ggml_tensor * shift, ggml_tensor * rot, - ggml_tensor * factors, - float freq_base, - float freq_scale, uint32_t il) const; ggml_cgraph * build_graph_shift( @@ -419,6 +426,9 @@ class llama_kv_cache_context : public llama_memory_context_i { ggml_tensor * get_k(ggml_context * ctx, int32_t il) const; ggml_tensor * get_v(ggml_context * ctx, int32_t il) const; + ggml_tensor * get_k_scale(int32_t il) const; + ggml_tensor * get_v_scale(int32_t il) const; + // store k_cur and v_cur in the cache based on the provided head location // note: the heads in k_cur and v_cur should be laid out contiguously in memory // - k_cur [n_embd_head_k, n_head_k, n_tokens] diff --git a/src/llama-mmap.cpp b/src/llama-mmap.cpp index 4d1cd31e9863..9688a7ed4e92 100644 --- a/src/llama-mmap.cpp +++ b/src/llama-mmap.cpp @@ -30,6 +30,7 @@ #ifndef NOMINMAX #define NOMINMAX #endif + #include #include #ifndef PATH_MAX #define PATH_MAX MAX_PATH @@ -68,7 +69,10 @@ static std::string llama_format_win_err(DWORD err) { struct llama_file::impl { #if defined(_WIN32) - HANDLE fp_win32; + // Keep the buffered handle as the logical cursor; the direct handle only issues aligned I/O. + HANDLE fp_win32 = INVALID_HANDLE_VALUE; + HANDLE fp_win32_direct = INVALID_HANDLE_VALUE; + std::string GetErrorMessageWin32(DWORD error_code) const { std::string ret; LPSTR lpMsgBuf = NULL; @@ -81,13 +85,93 @@ struct llama_file::impl { LocalFree(lpMsgBuf); } + while (!ret.empty() && (ret.back() == '\r' || ret.back() == '\n')) { + ret.pop_back(); + } + return ret; } - impl(const char * fname, const char * mode, [[maybe_unused]] const bool use_direct_io = false) { - fp = ggml_fopen(fname, mode); + impl(const char * fname, const char * mode, const bool use_direct_io = false) : fname(fname) { + init_buffered_io(mode); + if (use_direct_io && std::strcmp(mode, "rb") == 0) { + DWORD error = ERROR_SUCCESS; + if (!init_direct_io(error)) { + LLAMA_LOG_WARN("Failed to open file '%s' with unbuffered I/O: %s. Falling back to buffered I/O\n", + fname, GetErrorMessageWin32(error).c_str()); + } + } + } + + bool init_direct_io(DWORD& error) { + error = ERROR_SUCCESS; + alignment = 1; + + fp_win32_direct = ReOpenFile( + fp_win32, + GENERIC_READ, + FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, + FILE_FLAG_NO_BUFFERING); + + if (fp_win32_direct == INVALID_HANDLE_VALUE) { + error = GetLastError(); + return false; + } + + const bool ok = [&]() -> bool { +#if defined(NTDDI_VERSION) && defined(NTDDI_WIN8) && (NTDDI_VERSION >= NTDDI_WIN8) + FILE_STORAGE_INFO storage_info{}; + if (!GetFileInformationByHandleEx( + fp_win32_direct, + FileStorageInfo, + &storage_info, + static_cast(sizeof(storage_info)))) { + error = GetLastError(); + return false; + } + + alignment = std::max({ + static_cast(storage_info.LogicalBytesPerSector), + static_cast(storage_info.PhysicalBytesPerSectorForAtomicity), + static_cast(storage_info.PhysicalBytesPerSectorForPerformance), + static_cast(storage_info.FileSystemEffectivePhysicalBytesPerSectorForAtomicity) + }); +#else + error = ERROR_NOT_SUPPORTED; + return false; +#endif + // Required for bit-mask rounding and aligned allocation schemes. + if (alignment <= 1 || (alignment & (alignment - 1)) != 0) { + error = ERROR_NOT_SUPPORTED; + return false; + } + + // ReOpenFile creates a distinct handle; explicitly start direct I/O at offset 0. + LARGE_INTEGER zero{}; + if (!SetFilePointerEx(fp_win32_direct, zero, nullptr, FILE_BEGIN)) { + error = GetLastError(); + return false; + } + + return true; + }(); + + if (!ok) { + CloseHandle(fp_win32_direct); + fp_win32_direct = INVALID_HANDLE_VALUE; + alignment = 1; + return false; + } + + LLAMA_LOG_INFO("%s: file '%s' opened with unbuffered I/O (alignment: %zu bytes)\n", + __func__, fname.c_str(), alignment); + return true; + } + + void init_buffered_io(const char * mode) { + fp = ggml_fopen(fname.c_str(), mode); if (fp == NULL) { - throw std::runtime_error(format("failed to open %s: %s", fname, strerror(errno))); + throw std::runtime_error(format("failed to open %s: %s", fname.c_str(), strerror(errno))); } fp_win32 = (HANDLE) _get_osfhandle(_fileno(fp)); seek(0, SEEK_END); @@ -95,7 +179,7 @@ struct llama_file::impl { seek(0, SEEK_SET); } - impl(FILE * file) : owns_fp(false) { + impl(FILE * file) : fname("(file*)"), owns_fp(false) { fp = file; fp_win32 = (HANDLE) _get_osfhandle(_fileno(fp)); seek(0, SEEK_END); @@ -108,7 +192,7 @@ struct llama_file::impl { li.QuadPart = 0; BOOL ret = SetFilePointerEx(fp_win32, li, &li, FILE_CURRENT); if (!ret) { - throw std::runtime_error(format("read error: %s", GetErrorMessageWin32(GetLastError()).c_str())); + throw std::runtime_error(format("tell error: %s", GetErrorMessageWin32(GetLastError()).c_str())); } return li.QuadPart; @@ -123,24 +207,159 @@ struct llama_file::impl { li.QuadPart = offset; BOOL ret = SetFilePointerEx(fp_win32, li, NULL, whence); if (!ret) { - throw std::runtime_error(format("read error: %s", GetErrorMessageWin32(GetLastError()).c_str())); + throw std::runtime_error(format("seek error: %s", GetErrorMessageWin32(GetLastError()).c_str())); } } - void read_raw(void * ptr, size_t len) { + void read_raw_unsafe(void * ptr, size_t len) { + if (len == 0) { + return; + } + + const size_t read_offset = tell(); + if (read_offset > size) { + throw std::runtime_error("unexpectedly reached end of file"); + } + + if (!has_direct_io()) { + const size_t read_size = std::min(len, size - read_offset); + size_t bytes_read = 0; + while (bytes_read < read_size) { + const DWORD to_read = (DWORD) std::min(read_size - bytes_read, MAXDWORD); + DWORD chunk_read = 0; + BOOL result = ReadFile(fp_win32, reinterpret_cast(ptr) + bytes_read, + to_read, &chunk_read, NULL); + if (!result) { + throw std::runtime_error(format("read error: %s", GetErrorMessageWin32(GetLastError()).c_str())); + } + if (chunk_read < to_read) { + throw std::runtime_error("unexpectedly reached end of file"); + } + + bytes_read += chunk_read; + } + return; + } + + DWORD error = ERROR_SUCCESS; + if ((read_offset & (alignment - 1)) != 0 || + (len & (alignment - 1)) != 0 || + (reinterpret_cast(ptr) & (alignment - 1)) != 0) { + error = ERROR_INVALID_PARAMETER; + } else { + LARGE_INTEGER li; + li.QuadPart = read_offset; + if (!SetFilePointerEx(fp_win32_direct, li, NULL, FILE_BEGIN)) { + error = GetLastError(); + } + } + size_t bytes_read = 0; - while (bytes_read < len) { - size_t chunk_size = std::min(len - bytes_read, 64*1024*1024); + const size_t max_read_size = (size_t) MAXDWORD & ~(alignment - 1); + while (error == ERROR_SUCCESS && bytes_read < len) { + const DWORD to_read = (DWORD) std::min(len - bytes_read, max_read_size); DWORD chunk_read = 0; - BOOL result = ReadFile(fp_win32, reinterpret_cast(ptr) + bytes_read, chunk_size, &chunk_read, NULL); + BOOL result = ReadFile(fp_win32_direct, reinterpret_cast(ptr) + bytes_read, + to_read, &chunk_read, NULL); if (!result) { - throw std::runtime_error(format("read error: %s", GetErrorMessageWin32(GetLastError()).c_str())); - } - if (chunk_read < chunk_size || chunk_read == 0) { - throw std::runtime_error("unexpectedly reached end of file"); + error = GetLastError(); + if (error == ERROR_HANDLE_EOF && bytes_read == size - read_offset) { + error = ERROR_SUCCESS; + break; + } + break; } bytes_read += chunk_read; + if (chunk_read < to_read) { + if (bytes_read == size - read_offset) { + break; + } + error = ERROR_HANDLE_EOF; + } + } + + if (error == ERROR_INVALID_PARAMETER || error == ERROR_INVALID_USER_BUFFER || error == ERROR_NOACCESS) { + LLAMA_LOG_WARN("%s: file '%s' falling back to buffered I/O due to %s\n", + __func__, fname.c_str(), GetErrorMessageWin32(error).c_str()); + CloseHandle(fp_win32_direct); + fp_win32_direct = INVALID_HANDLE_VALUE; + alignment = 1; + seek(read_offset, SEEK_SET); + read_raw_unsafe(ptr, len); + return; + } + if (error != ERROR_SUCCESS) { + throw std::runtime_error(format("read error: %s", GetErrorMessageWin32(error).c_str())); + } + + seek(len > size - read_offset ? size : read_offset + len, SEEK_SET); + } + + void read_aligned_chunk(void * dest, size_t len) { + if (len == 0) { + return; + } + + const size_t offset = tell(); + if (offset > size || len > size - offset) { + throw std::runtime_error("unexpectedly reached end of file"); + } + + const size_t aligned_offset = offset & ~(alignment - 1); + const size_t offset_from_alignment = offset - aligned_offset; + const size_t bytes_to_read = (offset_from_alignment + len + alignment - 1) & ~(alignment - 1); + const size_t buffer_size = std::min(bytes_to_read, LLAMA_DIRECT_IO_BUFFER_SIZE); + + struct aligned_buffer_deleter { + void operator()(void * p) const { _aligned_free(p); } + }; + std::unique_ptr buffer(_aligned_malloc(buffer_size, alignment)); + if (!buffer) { + LLAMA_LOG_WARN("%s: Falling back to buffered I/O due to %s\n", + __func__, GetErrorMessageWin32(ERROR_NOT_ENOUGH_MEMORY).c_str()); + CloseHandle(fp_win32_direct); + fp_win32_direct = INVALID_HANDLE_VALUE; + alignment = 1; + seek(offset, SEEK_SET); + read_raw_unsafe(dest, len); + return; + } + + seek(aligned_offset, SEEK_SET); + + size_t skip = offset_from_alignment; + size_t copied = 0; + for (size_t done = 0; done < bytes_to_read; ) { + const size_t n = std::min(buffer_size, bytes_to_read - done); + read_raw_unsafe(buffer.get(), n); + + const size_t chunk_skip = std::min(skip, n); + const size_t count = std::min(n - chunk_skip, len - copied); + memcpy(reinterpret_cast(dest) + copied, reinterpret_cast(buffer.get()) + chunk_skip, count); + + copied += count; + skip -= chunk_skip; + done += n; + } + + seek(offset + len, SEEK_SET); + } + + void read_raw(void * ptr, size_t len) { + if (len == 0) { + return; + } + + const size_t offset = tell(); + if (offset > size || len > size - offset) { + throw std::runtime_error("unexpectedly reached end of file"); + } + + if (has_direct_io()) { + read_aligned_chunk(ptr, len); + } else { + read_raw_unsafe(ptr, len); } } @@ -153,7 +372,7 @@ struct llama_file::impl { void write_raw(const void * ptr, size_t len) const { size_t bytes_written = 0; while (bytes_written < len) { - size_t chunk_size = std::min(len - bytes_written, 64*1024*1024); + size_t chunk_size = std::min(len - bytes_written, LLAMA_DIRECT_IO_BUFFER_SIZE); DWORD chunk_written = 0; BOOL result = WriteFile(fp_win32, reinterpret_cast(ptr) + bytes_written, chunk_size, &chunk_written, NULL); if (!result) { @@ -172,23 +391,29 @@ struct llama_file::impl { } bool has_direct_io() const { - return true; + return fp_win32_direct != INVALID_HANDLE_VALUE && alignment > 1; } ~impl() { + if (fp_win32_direct != INVALID_HANDLE_VALUE) { + CloseHandle(fp_win32_direct); + } if (fp && owns_fp) { std::fclose(fp); } } + std::string fname; #else impl(const char * fname, const char * mode, [[maybe_unused]] const bool use_direct_io = false) : fname(fname) { #ifdef __linux__ // Try unbuffered I/O for read only if (use_direct_io && std::strcmp(mode, "rb") == 0) { if (init_fd()) { + LLAMA_LOG_INFO("%s: file '%s' opened with direct I/O (alignment: %zu bytes)\n", + __func__, fname, alignment); return; } - LLAMA_LOG_WARN("Failed to open file '%s' with error: %s. Falling back to buffered I/O", + LLAMA_LOG_WARN("Failed to open file '%s' with unbuffered I/O: %s. Falling back to buffered I/O\n", fname, strerror(errno)); } #endif @@ -268,8 +493,8 @@ struct llama_file::impl { } errno = 0; if (fd == -1) { - const size_t curr_off = tell(); - const size_t to_read = std::min(len, size - curr_off); + const size_t read_offset = tell(); + const size_t to_read = std::min(len, size - read_offset); std::size_t ret = std::fread(ptr, to_read, 1, fp); if (ferror(fp)) { @@ -286,24 +511,23 @@ struct llama_file::impl { if (ret == -1) { if (errno == EINTR) { - continue; // Interrupted by signal, retry + continue; } - // Fallback to std::fread in case the DMA controller cannot access the buffer if (errno == EFAULT || errno == EINVAL) { - LLAMA_LOG_WARN("%s: Falling back to buffered IO due to %s\n", __func__, strerror(errno)); - auto curr_off = tell(); + LLAMA_LOG_WARN("%s: file '%s' falling back to buffered I/O due to %s\n", + __func__, fname.c_str(), strerror(errno)); + const size_t read_offset = tell(); close(fd); fd = -1; alignment = 1; init_fp("rb"); - seek(curr_off, SEEK_SET); + seek(read_offset, SEEK_SET); read_raw_unsafe(ptr, len); return; } throw std::runtime_error(format("read error: %s", strerror(errno))); } if (ret == 0) { - // EOF: allow if this read was only pulling alignment padding past file end off_t pos = lseek(fd, 0, SEEK_CUR); if (pos != -1 && (size_t) pos == size) { std::memset(reinterpret_cast(ptr) + bytes_read, 0, len - bytes_read); @@ -317,11 +541,11 @@ struct llama_file::impl { } } - void read_aligned_chunk(void * dest, size_t size) { + void read_aligned_chunk(void * dest, size_t len) { size_t offset = tell(); off_t aligned_offset = offset & ~(alignment - 1); off_t offset_from_alignment = offset - aligned_offset; - size_t bytes_to_read = (offset_from_alignment + size + alignment - 1) & ~(alignment - 1); + size_t bytes_to_read = (offset_from_alignment + len + alignment - 1) & ~(alignment - 1); // stage through a bounded buffer, so that a large tensor is not held in memory twice while it loads const size_t buffer_size = std::min(bytes_to_read, LLAMA_DIRECT_IO_BUFFER_SIZE); @@ -329,7 +553,15 @@ struct llama_file::impl { void * raw_buffer = nullptr; int ret = posix_memalign(&raw_buffer, alignment, buffer_size); if (ret != 0) { - throw std::runtime_error(format("posix_memalign failed with error %d", ret)); + LLAMA_LOG_WARN("%s: file '%s' falling back to buffered I/O due to %s\n", + __func__, fname.c_str(), strerror(ret)); + close(fd); + fd = -1; + alignment = 1; + init_fp("rb"); + seek(offset, SEEK_SET); + read_raw_unsafe(dest, len); + return; } struct aligned_buffer_deleter { @@ -345,13 +577,15 @@ struct llama_file::impl { const size_t n = std::min(buffer_size, bytes_to_read - done); read_raw_unsafe(buffer.get(), n); - const size_t count = std::min(n - skip, size - copied); + const size_t count = std::min(n - skip, len - copied); memcpy(reinterpret_cast(dest) + copied, reinterpret_cast(buffer.get()) + skip, count); copied += count; skip = 0; done += n; } + + seek(offset + len, SEEK_SET); } void read_raw(void * ptr, size_t len) { @@ -390,7 +624,7 @@ struct llama_file::impl { ~impl() { if (fd != -1) { close(fd); - } else if (owns_fp) { + } else if (fp && owns_fp) { std::fclose(fp); } } @@ -439,11 +673,7 @@ int llama_file::file_id() const { void llama_file::seek(size_t offset, int whence) const { pimpl->seek(offset, whence); } void llama_file::read_raw(void * ptr, size_t len) { pimpl->read_raw(ptr, len); } -#ifdef _WIN32 -void llama_file::read_raw_unsafe(void * ptr, size_t len) { pimpl->read_raw(ptr, len); } -#else void llama_file::read_raw_unsafe(void * ptr, size_t len) { pimpl->read_raw_unsafe(ptr, len); } -#endif uint32_t llama_file::read_u32() { return pimpl->read_u32(); } diff --git a/src/llama-model-loader.cpp b/src/llama-model-loader.cpp index 485483ba1f27..2cbee61d4a69 100644 --- a/src/llama-model-loader.cpp +++ b/src/llama-model-loader.cpp @@ -597,85 +597,167 @@ llama_model_loader::llama_model_loader( n_bytes += ggml_nbytes(cur); weights_map.emplace(tensor_name, llama_tensor_weight(files.back().get(), 0, metadata, cur)); } + auto paths = splits; uint16_t n_split = 0; + size_t n_backbone = 1; + const std::string kv_split_no = llm_kv(LLM_KV_SPLIT_NO); get_key(llm_kv(LLM_KV_SPLIT_COUNT), n_split, false); - // Load additional GGML contexts if (n_split > 1) { - // make sure the main file is loaded first uint16_t idx = 0; - const std::string kv_split_no = llm_kv(LLM_KV_SPLIT_NO); get_key(kv_split_no, idx); if (idx != 0) { throw std::runtime_error(format("illegal split file idx: %d (file: %s), model must be loaded with the first split", idx, fname.c_str())); } - // generate list of splits if needed - if (splits.empty()) { - splits = llama_get_list_splits(fname, idx, n_split); + std::vector prefix(llama_path_max(), 0); + if (paths.empty() || llama_split_prefix(prefix.data(), prefix.size(), fname.c_str(), idx, n_split)) { + auto backbone = llama_get_list_splits(fname, idx, n_split); + auto file_name = [](const std::string & path) { + return path.substr(path.find_last_of("/\\") + 1); + }; + std::vector extras; + for (const auto & path : paths) { + auto it = std::find_if(backbone.begin(), backbone.end(), [&](const std::string & b) { + return file_name(b) == file_name(path); + }); + if (it != backbone.end()) { + *it = path; // given path wins, splits can be in different dirs + } else { + extras.push_back(path); + } + } + paths = std::move(backbone); + paths.insert(paths.end(), extras.begin(), extras.end()); + } else if (paths.size() != n_split) { + throw std::runtime_error(format("invalid split count, given: %zu splits, but expected %d", paths.size(), n_split)); } + n_backbone = n_split; + } else if (paths.empty()) { + paths.push_back(fname); + } - // in case user give a custom list of splits, check if it matches the expected number - if (n_split != (uint16_t)splits.size()) { - throw std::runtime_error(format("invalid split count, given: %zu splits, but expected %d", splits.size(), n_split)); - } + if (paths.size() > UINT16_MAX) { + throw std::runtime_error("too many model files"); + } + const bool has_extras = paths.size() > n_backbone; - if (trace > 0) { - LLAMA_LOG_INFO("%s: loading additional %d GGUFs\n", __func__, n_split); + // special case for PLE-only GGUFs: no need to follow the exact split numbering (allow flexibility) + // ref: https://github.com/ggml-org/llama.cpp/pull/31185 + auto is_ple_only = [](const gguf_context * ctx) { + return gguf_get_n_tensors(ctx) == 1 && std::strcmp(gguf_get_tensor_name(ctx, 0), "per_layer_token_embd.weight") == 0; + }; + bool skip_shard_order_validation = false; + for (size_t idx = n_backbone; idx < paths.size(); idx++) { + struct gguf_init_params probe_params = { + /*.no_alloc = */ true, + /*.ctx = */ nullptr, + }; + gguf_context_ptr probe { gguf_init_from_file(paths[idx].c_str(), probe_params) }; + skip_shard_order_validation |= probe && is_ple_only(probe.get()); + } + + size_t n_skipped = 0; + size_t n_backbone_files = 1; + int n_backbone_tensors = gguf_get_n_tensors(metadata); + for (size_t idx = 1; idx < paths.size(); idx++) { + const char * fname_split = paths[idx].c_str(); + if (skip_shard_order_validation && idx < n_backbone) { + FILE * f = ggml_fopen(fname_split, "rb"); + if (f == nullptr) { + LLAMA_LOG_INFO("%s: GGUF split %s not found, assuming it only holds the per-layer embedding table\n", __func__, fname_split); + n_skipped++; + continue; + } + fclose(f); + } + ctx = nullptr; // still owned by contexts, not set by gguf on failure + struct gguf_init_params split_params = { + /*.no_alloc = */ true, + /*.ctx = */ &ctx, + }; + gguf_context_ptr ctx_gguf { gguf_init_from_file(fname_split, split_params) }; + ggml_context_ptr ctx_ptr(ctx); + if (!ctx_gguf) { + throw std::runtime_error(format("%s: failed to load GGUF split from %s", __func__, fname_split)); } - // load other splits - for (idx = 1; idx < n_split; idx++) { - const char * fname_split = splits[idx].c_str(); - - struct gguf_init_params split_params = { - /*.no_alloc = */ true, - /*.ctx = */ &ctx, - }; - gguf_context_ptr ctx_gguf { gguf_init_from_file(fname_split, split_params) }; - if (!ctx_gguf) { - throw std::runtime_error(format("%s: failed to load GGUF split from %s", __func__, fname_split)); + bool allow_override_tensor = false; + if (idx < n_backbone) { + const int kid = gguf_find_key(ctx_gguf.get(), kv_split_no.c_str()); + if (kid < 0) { + throw std::runtime_error(format("missing key %s in GGUF split %s", kv_split_no.c_str(), fname_split)); } - - // check idx - { - const int kid = gguf_find_key(ctx_gguf.get(), kv_split_no.c_str()); - if (kid < 0) { - throw std::runtime_error(format("missing key %s in GGUF split %s", kv_split_no.c_str(), fname_split)); - } - int idx_gguf = gguf_get_val_u16(ctx_gguf.get(), kid); - if (idx_gguf != idx) { - throw std::runtime_error(format("invalid split file idx: %d (file: %s), expected %d", idx_gguf, fname_split, idx)); - } + if (gguf_get_kv_type(ctx_gguf.get(), kid) != GGUF_TYPE_UINT16) { + throw std::runtime_error(format("invalid type for %s in GGUF split %s", kv_split_no.c_str(), fname_split)); + } + const uint16_t idx_gguf = gguf_get_val_u16(ctx_gguf.get(), kid); + if (idx_gguf != idx) { + throw std::runtime_error(format("invalid split file idx: %d (file: %s), expected %zu", idx_gguf, fname_split, idx)); } + if (skip_shard_order_validation && is_ple_only(ctx_gguf.get())) { + LLAMA_LOG_INFO("%s: skipping GGUF split %s, per-layer embedding table given separately\n", __func__, fname_split); + n_skipped++; + continue; + } + n_backbone_tensors += gguf_get_n_tensors(ctx_gguf.get()); + } else { + // an external PLE table replaces the one in the model, if any + allow_override_tensor = is_ple_only(ctx_gguf.get()); + const bool numbered = gguf_find_key(ctx_gguf.get(), kv_split_no.c_str()) >= 0 || + gguf_find_key(ctx_gguf.get(), llm_kv(LLM_KV_SPLIT_COUNT).c_str()) >= 0 || + gguf_find_key(ctx_gguf.get(), llm_kv(LLM_KV_SPLIT_TENSORS_COUNT).c_str()) >= 0; + LLAMA_LOG_INFO("%s: appended extra GGUF shard %s%s\n", __func__, fname_split, + numbered ? " (declared split numbering ignored)" : ""); + } - files.emplace_back(new llama_file(fname_split, "rb", use_direct_io)); - contexts.emplace_back(ctx); + files.emplace_back(new llama_file(fname_split, "rb", use_direct_io)); + contexts.emplace_back(std::move(ctx_ptr)); + if (idx < n_backbone) { + n_backbone_files = files.size(); + } - // Save tensors data offset info of the shard. - for (ggml_tensor * cur = ggml_get_first_tensor(ctx); cur; cur = ggml_get_next_tensor(ctx, cur)) { - std::string tensor_name = std::string(cur->name); - // make sure there is no duplicated tensor names - if (weights_map.find(tensor_name) != weights_map.end()) { + // skipped splits are not in files, so idx != file index + const uint16_t file_idx = files.size() - 1; + for (ggml_tensor * cur = ggml_get_first_tensor(ctx); cur; cur = ggml_get_next_tensor(ctx, cur)) { + const std::string tensor_name = cur->name; + auto it = weights_map.find(tensor_name); + if (it != weights_map.end()) { + // only override tensors of the model, not of another extra shard + if (!allow_override_tensor || it->second.idx >= n_backbone_files) { throw std::runtime_error(format("invalid model: tensor '%s' is duplicated", ggml_get_name(cur))); } - n_elements += ggml_nelements(cur); - n_bytes += ggml_nbytes(cur); - weights_map.emplace(tensor_name, llama_tensor_weight(files.back().get(), idx, ctx_gguf.get(), cur)); + LLAMA_LOG_INFO("%s: tensor '%s' overridden by %s\n", __func__, tensor_name.c_str(), fname_split); + n_elements -= ggml_nelements(it->second.tensor); + n_bytes -= ggml_nbytes(it->second.tensor); + weights_map.erase(it); } + n_elements += ggml_nelements(cur); + n_bytes += ggml_nbytes(cur); + weights_map.emplace(tensor_name, llama_tensor_weight(files.back().get(), file_idx, ctx_gguf.get(), cur)); } + } - get_key(llm_kv(LLM_KV_SPLIT_TENSORS_COUNT), n_tensors); - - // sanity check - { - const int n_tensors_loaded = (int) weights_map.size(); - if (n_tensors != n_tensors_loaded) { - throw std::runtime_error(format("corrupted model: %d tensors expected but %d found", n_tensors, n_tensors_loaded)); + if (n_skipped > 0) { + // the external table can only replace one tensor of the numbered set + int declared_tensors = 0; + get_key(llm_kv(LLM_KV_SPLIT_TENSORS_COUNT), declared_tensors); + if (n_skipped != 1 || n_backbone_tensors + 1 != declared_tensors) { + throw std::runtime_error(format("incomplete model: %zu split(s) skipped, %d of %d tensors found in the split files", n_skipped, n_backbone_tensors, declared_tensors)); + } + } + if (n_split > 1 || has_extras) { + int declared_tensors = gguf_get_n_tensors(metadata); + get_key(llm_kv(LLM_KV_SPLIT_TENSORS_COUNT), declared_tensors, n_split > 1 && !has_extras); + if (declared_tensors != (int) weights_map.size()) { + if (n_split > 1 && !has_extras) { + throw std::runtime_error(format("corrupted model: %d tensors expected but %zu found", declared_tensors, weights_map.size())); } + LLAMA_LOG_INFO("%s: using %zu registered tensors instead of %d declared tensors\n", __func__, weights_map.size(), declared_tensors); } - - LLAMA_LOG_INFO("%s: additional %d GGUFs metadata loaded.\n", __func__, n_split - 1); + } + if (files.size() > 1) { + LLAMA_LOG_INFO("%s: additional %zu GGUFs metadata loaded.\n", __func__, files.size() - 1); } } else if (file != nullptr) { struct ggml_context * ctx = NULL; diff --git a/src/llama-model-saver.cpp b/src/llama-model-saver.cpp index 36f3850d5180..b667bc49ff37 100644 --- a/src/llama-model-saver.cpp +++ b/src/llama-model-saver.cpp @@ -210,6 +210,8 @@ void llama_model_saver::add_kv_from_model() { add_kv(LLM_KV_MOE_LATENT_SIZE, hparams.moe_latent_size); add_kv(LLM_KV_EXPERT_SHARED_FEED_FORWARD_LENGTH, hparams.n_ff_shexp); add_kv(LLM_KV_EXPERT_CHUNK_FEED_FORWARD_LENGTH, hparams.n_ff_chexp); + add_kv(LLM_KV_SWIGLU_CLAMP, std::vector( + hparams.swiglu_clamp.begin(), hparams.swiglu_clamp.begin() + hparams.n_layer_all)); add_kv(LLM_KV_SWIGLU_CLAMP_EXP, std::vector( hparams.swiglu_clamp_exp.begin(), hparams.swiglu_clamp_exp.begin() + hparams.n_layer_all)); add_kv(LLM_KV_SWIGLU_CLAMP_SHEXP, std::vector( @@ -247,6 +249,10 @@ void llama_model_saver::add_kv_from_model() { add_kv(LLM_KV_HRM_H_CYCLES, hparams.n_hrm_h_cycles); add_kv(LLM_KV_HRM_L_CYCLES, hparams.n_hrm_l_cycles); add_kv(LLM_KV_HRM_PREFIX_LM, hparams.hrm_prefix_lm); + add_kv(LLM_KV_METIS_PASS_COUNT, hparams.metis_n_pass); + add_kv(LLM_KV_METIS_MEMORY_LENGTH, hparams.metis_n_mem); + add_kv(LLM_KV_METIS_ROUTE_LENGTH, hparams.metis_n_route); + add_kv(LLM_KV_METIS_CONTINUATION_LOGIT_LIMIT, hparams.metis_cont_limit); if (hparams.final_logit_sigmoid_capping[0] != 0.0f) { add_kv(LLM_KV_FINAL_LOGIT_SIGMOID_CAPPING, std::vector( hparams.final_logit_sigmoid_capping.begin(), hparams.final_logit_sigmoid_capping.end())); @@ -323,8 +329,9 @@ void llama_model_saver::add_kv_from_model() { add_kv(LLM_KV_ATTENTION_RECURRENT_LAYERS, hparams.is_recr_impl, true); add_kv(LLM_KV_ATTENTION_OUTPUT_GROUP_COUNT, hparams.dsv4_o_group_count); add_kv(LLM_KV_ATTENTION_OUTPUT_LORA_RANK, hparams.dsv4_o_lora_rank); + add_kv(LLM_KV_ATTENTION_PASS_LORA_RANK, hparams.metis_pass_lora_rank); add_kv(LLM_KV_ATTENTION_COMPRESS_ROPE_FREQ_BASE, hparams.dsv4_compress_rope_base); - if (model->arch == LLM_ARCH_DEEPSEEK4 || hparams.dsv4_hc_mult > 0) { + if (model->arch == LLM_ARCH_DEEPSEEK4 || (hparams.dsv4_hc_mult > 0 && model->arch != LLM_ARCH_XING4_0)) { // the loader requires one compress ratio per layer, including nextn layers const std::vector compress_ratios( hparams.dsv4_compress_ratios.begin(), hparams.dsv4_compress_ratios.begin() + hparams.n_layer_all); @@ -419,6 +426,7 @@ void llama_model_saver::add_kv_from_model() { add_kv(LLM_KV_SSM_TIME_STEP_RANK, hparams.ssm_dt_rank); add_kv(LLM_KV_SSM_GROUP_COUNT, hparams.ssm_n_group); add_kv(LLM_KV_SSM_DT_B_C_RMS, hparams.ssm_dt_b_c_rms); + add_kv(LLM_KV_SSM_NORM_EPS, hparams.ssm_norm_eps); add_kv(LLM_KV_KDA_HEAD_DIM, hparams.n_embd_head_kda); add_kv(LLM_KV_KDA_SAFE_GATE, hparams.kda_safe_gate); @@ -521,6 +529,32 @@ void llama_model_saver::add_tensors_from_model() { add_tensor(model->cls_out_b); add_tensor(model->cls_norm); add_tensor(model->hrm_z_l_init); + add_tensor(model->stream_embd); + add_tensor(model->depth_mem_state_write); + add_tensor(model->depth_mem_meta_write); + add_tensor(model->depth_mem_entry_bias); + add_tensor(model->depth_mem_query); + add_tensor(model->depth_mem_query_b); + add_tensor(model->depth_mem_key); + add_tensor(model->depth_mem_key_b); + add_tensor(model->depth_mem_value); + add_tensor(model->depth_mem_value_b); + add_tensor(model->depth_mem_out); + add_tensor(model->depth_mem_out_b); + add_tensor(model->depth_mem_route_state); + add_tensor(model->depth_mem_route_retr); + add_tensor(model->depth_mem_route_diff); + add_tensor(model->depth_mem_route_hist); + add_tensor(model->depth_mem_route_b); + add_tensor(model->depth_mem_stream_gate); + add_tensor(model->depth_mem_stream_gate_b); + add_tensor(model->cont_hidden_state); + add_tensor(model->cont_hidden_retr); + add_tensor(model->cont_hidden_diff); + add_tensor(model->cont_hidden_route); + add_tensor(model->cont_hidden_b); + add_tensor(model->cont_output); + add_tensor(model->cont_output_b); add_tensor(model->hc_head_fn); add_tensor(model->hc_head_base); add_tensor(model->hc_head_scale); diff --git a/src/llama-model.cpp b/src/llama-model.cpp index 52893876dfaf..7c75ab55ef30 100644 --- a/src/llama-model.cpp +++ b/src/llama-model.cpp @@ -214,6 +214,8 @@ static llama_model * llama_model_mapping(llm_arch arch, const llama_model_params return new llama_model_deepseek4(params); case LLM_ARCH_DEEPSEEK41: return new llama_model_deepseek41(params); + case LLM_ARCH_XING4_0: + return new llama_model_xing4_0(params); case LLM_ARCH_GLM_DSA: return new llama_model_glm_dsa(params); case LLM_ARCH_MISTRAL4: @@ -242,6 +244,8 @@ static llama_model * llama_model_mapping(llm_arch arch, const llama_model_params return new llama_model_nemotron_h(params); case LLM_ARCH_NEMOTRON_H_MOE: return new llama_model_nemotron_h_moe(params); + case LLM_ARCH_METIS16: + return new llama_model_metis16(params); case LLM_ARCH_EXAONE: return new llama_model_exaone(params); case LLM_ARCH_EXAONE4: @@ -290,6 +294,8 @@ static llama_model * llama_model_mapping(llm_arch arch, const llama_model_params return new llama_model_afmoe(params); case LLM_ARCH_LAGUNA: return new llama_model_laguna(params); + case LLM_ARCH_RICURSIVE: + return new llama_model_ricursive(params); case LLM_ARCH_ERNIE4_5: return new llama_model_ernie4_5(params); case LLM_ARCH_ERNIE4_5_MOE: @@ -366,6 +372,8 @@ static llama_model * llama_model_mapping(llm_arch arch, const llama_model_params return new llama_model_kimi_linear(params); case LLM_ARCH_KIMI_K3: return new llama_model_kimi_k3(params); + case LLM_ARCH_KIMI_K31: + return new llama_model_kimi_k31(params); case LLM_ARCH_GLM5_NEXT: return new llama_model_glm5_next(params); case LLM_ARCH_STEP35: @@ -410,11 +418,13 @@ struct ggml_backend_meta_split_state llama_meta_device_get_split_state(const str const std::string tensor_name = tensor->name; const bool is_dsv4 = ud->model->arch == LLM_ARCH_DEEPSEEK4 || ud->model->arch == LLM_ARCH_DEEPSEEK41 || (ud->model->arch == LLM_ARCH_DFLASH && hparams.dsv4_hc_mult > 0); + const bool is_glm5_next = ud->model->arch == LLM_ARCH_GLM5_NEXT; static const std::regex pattern_q_weight ("blk\\.\\d*\\.attn_q.weight"); static const std::regex pattern_kv_weight ("blk\\.\\d*\\.attn_(k|v).weight"); static const std::regex pattern_v_exps_weight ("blk\\.\\d*\\.attn_v_exps.weight"); // K2 Horizon MoVA static const std::regex pattern_qkv_weight ("blk\\.\\d*\\.attn_qkv.weight"); + static const std::regex pattern_qkv_gate_weight ("blk\\.\\d*\\.attn_qkv_gate.weight"); static const std::regex pattern_q_bias ("blk\\.\\d*\\.attn_q\\.bias"); static const std::regex pattern_kv_bias ("blk\\.\\d*\\.attn_(k|v)\\.bias"); static const std::regex pattern_qkv_bias ("blk\\.\\d*\\.attn_qkv.bias"); @@ -429,6 +439,7 @@ struct ggml_backend_meta_split_state llama_meta_device_get_split_state(const str static const std::regex pattern_attn_out_a_weight("blk\\.\\d*\\.attn_output_a\\.weight"); static const std::regex pattern_attn_out_b_weight("blk\\.\\d*\\.attn_output_b\\.weight"); static const std::regex pattern_attn_q_b_weight ("blk\\.\\d*\\.attn_q_b\\.weight"); + static const std::regex pattern_attn_kv_b_weight("blk\\.\\d*\\.attn_(k|v)_b\\.weight"); static const std::regex pattern_attn_gate_weight("blk\\.\\d*\\.attn_gate.weight"); static const std::regex pattern_ssm_dt ("blk\\.\\d*\\.ssm_dt.bias"); @@ -436,11 +447,14 @@ struct ggml_backend_meta_split_state llama_meta_device_get_split_state(const str static const std::regex pattern_ssm_alpha ("blk\\.\\d*\\.ssm_alpha.weight"); static const std::regex pattern_ssm_beta ("blk\\.\\d*\\.ssm_beta.weight"); static const std::regex pattern_ssm_beta_alpha ("blk\\.\\d*\\.ssm_ba.weight"); + static const std::regex pattern_ssm_beta_alpha_cat("blk\\.\\d*\\.ssm_beta_alpha.weight"); static const std::regex pattern_r_cache ("cache_r_l\\d*"); static const std::regex pattern_ple_r_cache ("cache_ple_r_l\\d*"); static const std::regex pattern_s_cache ("cache_s_l\\d*"); static const std::regex pattern_ssm_conv1d ("blk\\.\\d*\\.ssm_conv1d.weight"); static const std::regex pattern_ssm_out_weight ("blk\\.\\d*\\.ssm_out.weight"); + static const std::regex pattern_kda_conv1d ("blk\\.\\d*\\.ssm_conv1d_(q|k|v)\\.weight"); + static const std::regex pattern_kda_gate_b ("blk\\.\\d*\\.ssm_(f|g)_b\\.weight"); static const std::regex pattern_ffn_up_weight ("blk\\.\\d*\\.ffn_up(_exps)?.weight"); static const std::regex pattern_ffn_up_bias ("blk\\.\\d*\\.ffn_up(_exps)?.bias"); @@ -511,7 +525,7 @@ struct ggml_backend_meta_split_state llama_meta_device_get_split_state(const str }; auto get_tensor_config = [&]() -> tensor_config { - if (ud->model->arch == LLM_ARCH_HRM_TEXT) { + if (ud->model->arch == LLM_ARCH_HRM_TEXT || ud->model->arch == LLM_ARCH_METIS16) { // aliased cache slots cannot satisfy the meta-split invariants, so replicate all tensors return {GGML_BACKEND_SPLIT_AXIS_MIRRORED, tensor, 0, 0}; } @@ -532,6 +546,25 @@ struct ggml_backend_meta_split_state llama_meta_device_get_split_state(const str if (std::regex_match(tensor_name, pattern_attn_out_b_weight)) { return get_tensor_config_impl(GGML_BACKEND_SPLIT_AXIS_0); } + } + if (is_glm5_next) { + // split the KDA and MLA heads, the MLA latent cache and the indexer stay mirrored + if (std::regex_match(tensor_name, pattern_kv_cache)) { + return get_tensor_config_impl(GGML_BACKEND_SPLIT_AXIS_MIRRORED); + } + if (std::regex_match(tensor_name, pattern_attn_q_b_weight) || std::regex_match(tensor_name, pattern_kda_gate_b) || + std::regex_match(tensor_name, pattern_ssm_beta)) { + return get_tensor_config_impl(GGML_BACKEND_SPLIT_AXIS_1, "attn_output.weight"); + } + if (std::regex_match(tensor_name, pattern_attn_kv_b_weight) || std::regex_match(tensor_name, pattern_kda_conv1d)) { + return get_tensor_config_impl(GGML_BACKEND_SPLIT_AXIS_2, "attn_output.weight"); + } + if (std::regex_match(tensor_name, pattern_ssm_dt) || std::regex_match(tensor_name, pattern_ssm_a) || + std::regex_match(tensor_name, pattern_r_cache) || std::regex_match(tensor_name, pattern_s_cache)) { + return get_tensor_config_impl(GGML_BACKEND_SPLIT_AXIS_0, "attn_output.weight"); + } + } + if (is_dsv4 || is_glm5_next) { if (std::regex_match(tensor_name, pattern_ffn_up_shexp_weight) || std::regex_match(tensor_name, pattern_ffn_gate_shexp_weight)) { return get_tensor_config_impl(GGML_BACKEND_SPLIT_AXIS_1, "ffn_down_shexp.weight"); @@ -562,7 +595,7 @@ struct ggml_backend_meta_split_state llama_meta_device_get_split_state(const str if (std::regex_match(tensor_name, pattern_q_bias) || std::regex_match(tensor_name, pattern_kv_bias)) { return get_tensor_config_impl(GGML_BACKEND_SPLIT_AXIS_0, "attn_output.weight", "ssm_out.weight"); } - if (std::regex_match(tensor_name, pattern_qkv_weight)) { + if (std::regex_match(tensor_name, pattern_qkv_weight) || std::regex_match(tensor_name, pattern_qkv_gate_weight)) { return get_tensor_config_impl(GGML_BACKEND_SPLIT_AXIS_1, "attn_output.weight", "ssm_out.weight"); } if ( std::regex_match(tensor_name, pattern_qkv_bias)) { @@ -589,7 +622,7 @@ struct ggml_backend_meta_split_state llama_meta_device_get_split_state(const str return get_tensor_config_impl(GGML_BACKEND_SPLIT_AXIS_0, "ssm_out.weight"); } if (std::regex_match(tensor_name, pattern_ssm_alpha) || std::regex_match(tensor_name, pattern_ssm_beta) || - std::regex_match(tensor_name, pattern_ssm_beta_alpha)) { + std::regex_match(tensor_name, pattern_ssm_beta_alpha) || std::regex_match(tensor_name, pattern_ssm_beta_alpha_cat)) { return get_tensor_config_impl(GGML_BACKEND_SPLIT_AXIS_1, "ssm_out.weight"); } if (std::regex_match(tensor_name, pattern_r_cache) || std::regex_match(tensor_name, pattern_s_cache)) { @@ -645,6 +678,12 @@ struct ggml_backend_meta_split_state llama_meta_device_get_split_state(const str // if a model has fused tensors they need to be separated into "segments", see the comments on ggml_backend_meta_split_state struct auto get_split_segments = [&](int axis, uint32_t il) -> std::vector> { + if (is_glm5_next && std::regex_match(tensor_name, pattern_r_cache)) { + // the conv state holds the q, k and v windows one after the other + const int64_t conv_state_size = (hparams.ssm_d_conv - 1) * hparams.n_head(il) * hparams.n_embd_head_kda; + GGML_ASSERT(tensor->ne[axis] == 3*conv_state_size); + return {{conv_state_size, 3}}; + } // TODO: deduplicate condition [TAG_SPLIT_QGATE_QWEN] if (ud->model->arch == LLM_ARCH_QWEN3NEXT || ud->model->arch == LLM_ARCH_QWEN35 || ud->model->arch == LLM_ARCH_QWEN35MOE || ud->model->arch == LLM_ARCH_CLEF || ud->model->arch == LLM_ARCH_CLOVER_MOE || ud->model->arch == LLM_ARCH_QWEN4EXP) { @@ -678,6 +717,14 @@ struct ggml_backend_meta_split_state llama_meta_device_get_split_state(const str } } else { const int64_t head_ratio = n_v_heads / n_k_heads; + if (std::regex_match(tensor_name, pattern_qkv_gate_weight)) { + GGML_ASSERT(tensor->ne[axis] == 2*key_dim + 2*value_dim); + return {{key_dim, 2 + 2*head_ratio}}; + } + if (std::regex_match(tensor_name, pattern_ssm_beta_alpha_cat)) { + GGML_ASSERT(tensor->ne[axis] == 2*n_v_heads); + return {{n_k_heads, 2*head_ratio}}; + } if (std::regex_match(tensor_name, pattern_qkv_weight) || std::regex_match(tensor_name, pattern_ssm_conv1d)) { GGML_ASSERT(tensor->ne[axis] == 2*key_dim + value_dim); return {{key_dim, 2 + head_ratio}}; @@ -736,17 +783,51 @@ struct ggml_backend_meta_split_state llama_meta_device_get_split_state(const str auto get_split_granularity = [&](int64_t blck_size, uint32_t il, const std::vector> & segments) -> std::vector { // for better performance it may make sense to round up blck_size to a higher power of 2 so that more efficient kernels can be used - if (hparams.is_recr(il)) { + if (is_glm5_next && hparams.is_recr(il)) { + // KDA, each device holds whole heads + const int64_t head_dim = hparams.n_embd_head_kda; + const int64_t blck_size_perf = std::lcm(blck_size, 128); + const int64_t granularity_qkv = std::lcm(blck_size_perf, head_dim); + if (std::regex_match(tensor_name, pattern_q_weight) || std::regex_match(tensor_name, pattern_kv_weight) || + std::regex_match(tensor_name, pattern_kda_conv1d) || std::regex_match(tensor_name, pattern_kda_gate_b) || + std::regex_match(tensor_name, pattern_ssm_dt) || std::regex_match(tensor_name, pattern_attn_out_weight)) { + return {granularity_qkv}; + } + if (std::regex_match(tensor_name, pattern_ssm_a) || std::regex_match(tensor_name, pattern_ssm_beta)) { + return {granularity_qkv / head_dim}; + } + if (std::regex_match(tensor_name, pattern_r_cache)) { + return std::vector(segments.size(), granularity_qkv * (hparams.ssm_d_conv - 1)); + } + if (std::regex_match(tensor_name, pattern_s_cache)) { + return {granularity_qkv * head_dim}; + } + } else if (is_glm5_next) { + // MLA, each device holds whole heads + // an even number of heads per device, the HIP tile FA kernel for D=512 needs an even GQA ratio + const int64_t granularity_head = std::lcm(std::lcm(hparams.n_embd_head_v_mla(), blck_size) / hparams.n_embd_head_v_mla(), int64_t(2)); + if (std::regex_match(tensor_name, pattern_attn_q_b_weight)) { + return {granularity_head * hparams.n_embd_head_k_mla()}; + } + if (std::regex_match(tensor_name, pattern_attn_kv_b_weight)) { + return {granularity_head}; + } + if (std::regex_match(tensor_name, pattern_attn_out_weight)) { + return {granularity_head * hparams.n_embd_head_v_mla()}; + } + } else if (hparams.is_recr(il)) { // linear attention const int64_t head_dim = hparams.ssm_d_state; const int64_t blck_size_perf = std::lcm(blck_size, 128); const int64_t granularity_qkv = std::lcm(blck_size_perf, head_dim); if (std::regex_match(tensor_name, pattern_qkv_weight) || std::regex_match(tensor_name, pattern_attn_gate_weight) || - std::regex_match(tensor_name, pattern_ssm_conv1d) || std::regex_match(tensor_name, pattern_ssm_out_weight)) { + std::regex_match(tensor_name, pattern_ssm_conv1d) || std::regex_match(tensor_name, pattern_ssm_out_weight) || + std::regex_match(tensor_name, pattern_qkv_gate_weight)) { return std::vector(segments.size(), granularity_qkv); } if (std::regex_match(tensor_name, pattern_ssm_dt) || std::regex_match(tensor_name, pattern_ssm_a) || - std::regex_match(tensor_name, pattern_ssm_alpha) || std::regex_match(tensor_name, pattern_ssm_beta)) { + std::regex_match(tensor_name, pattern_ssm_alpha) || std::regex_match(tensor_name, pattern_ssm_beta) || + std::regex_match(tensor_name, pattern_ssm_beta_alpha_cat)) { return std::vector(segments.size(), granularity_qkv / head_dim); } if (std::regex_match(tensor_name, pattern_ssm_beta_alpha)) { @@ -988,6 +1069,7 @@ const char * llm_type_name(llm_type type) { case LLM_TYPE_7B: return "7B"; case LLM_TYPE_8B: return "8B"; case LLM_TYPE_9B: return "9B"; + case LLM_TYPE_10B: return "10B"; case LLM_TYPE_11B: return "11B"; case LLM_TYPE_12B: return "12B"; case LLM_TYPE_13B: return "13B"; @@ -1408,6 +1490,7 @@ void llama_model_base::load_hparams(llama_model_loader & ml) { ml.get_key(LLM_KV_ATTENTION_CAUSAL, hparams.causal_attn, false); ml.get_key(LLM_KV_POOLING_TYPE, hparams.pooling_type, false); ml.get_key(LLM_KV_CLASSIFIER_POOLING_TYPE, hparams.pooling_type_cls, false); + ml.get_key(LLM_KV_CLASSIFIER_FREE_GUIDANCE, hparams.classifier_free_guidance, false); std::string act_cls; if (ml.get_key(LLM_KV_CLASSIFIER_ACTIVATION, act_cls, false)) { const auto it = LLM_CLS_ACT_TYPES_FROM_STRING.find(act_cls); @@ -1482,6 +1565,7 @@ void llama_model_base::load_hparams(llama_model_loader & ml) { std::fill(hparams.swiglu_clamp_exp.begin(), hparams.swiglu_clamp_exp.end(), 0.0f); std::fill(hparams.swiglu_clamp_shexp.begin(), hparams.swiglu_clamp_shexp.end(), 0.0f); + std::fill(hparams.swiglu_clamp.begin(), hparams.swiglu_clamp.end(), 0.0f); ml.get_key_or_arr(LLM_KV_FEED_FORWARD_LENGTH, hparams.n_ff_arr, hparams.n_layer_all, false); ml.get_key_or_arr(LLM_KV_ATTENTION_HEAD_COUNT, hparams.n_head_arr, hparams.n_layer_all, false); @@ -1625,8 +1709,8 @@ bool llama_model_base::load_tensors(llama_model_loader & ml) { } } - // resolve AUTO on systems without mmap support (e.g. iGPUs): fall back to OFF; see #28160 - if (ml.lazy.mode == LLAMA_LAZY_MODE_AUTO) { + // resolve AUTO on systems without mmap support (e.g. iGPUs): fall back to OFF unless mmap is in use; see #28160, #31943 + if (ml.lazy.mode == LLAMA_LAZY_MODE_AUTO && !ml.use_mmap) { for (const auto & dev : devices) { ggml_backend_dev_props props; ggml_backend_dev_get_props(dev.dev, &props); @@ -1741,6 +1825,16 @@ bool llama_model_base::load_tensors(llama_model_loader & ml) { for (int i = 0; i < n_layer_all; ++i) { auto & layer = layers[i]; + if (hparams.has_kv(i)) { + layer.k_cache_s = create_tensor(tn(LLM_TENSOR_ATTN_K, "k_scale", i), {1}, TENSOR_NOT_REQUIRED); + layer.v_cache_s = create_tensor(tn(LLM_TENSOR_ATTN_V, "v_scale", i), {1}, TENSOR_NOT_REQUIRED); + + if ((layer.k_cache_s && layer.k_cache_s->type != GGML_TYPE_F32) || + (layer.v_cache_s && layer.v_cache_s->type != GGML_TYPE_F32)) { + throw std::runtime_error(format("KV cache scales for layer %d must be F32", i)); + } + } + // attention weight scales (per-tensor, shape {1}) if (!layer.wq_s && layer.wq) { layer.wq_s = create_tensor(tn(LLM_TENSOR_ATTN_Q, "scale", i), {1}, TENSOR_NOT_REQUIRED); @@ -2786,6 +2880,36 @@ const ggml_tensor * llama_model::get_tensor(const char * name) const { return it->second; } +llama_rope_params llama_model::get_rope_params(const llama_cparams & cparams, int il) const { + llama_rope_params res; + + res.n_rot = hparams.has_rope(il) ? hparams.n_rot(il) : 0; + res.n_offs = hparams.n_lora_kv > 0 ? hparams.n_embd_head_k(il) - hparams.n_rot(il) : 0; + res.type = hparams.rope_type; + res.n_ctx_orig = cparams.n_ctx_orig_yarn; + res.freq_base = get_rope_freq_base (cparams, il); + res.freq_scale = get_rope_freq_scale(cparams, il); + res.ext_factor = cparams.yarn_ext_factor; + res.attn_factor = cparams.yarn_attn_factor; + res.beta_fast = cparams.yarn_beta_fast; + res.beta_slow = cparams.yarn_beta_slow; + res.factors = get_rope_factors(cparams, il); + + return res; +} + +llama_rope_params llama_model::get_rope_params_indexer(const llama_cparams & cparams, int il) const { + llama_rope_params res = get_rope_params(cparams, il); + + res.n_offs = 0; + res.type = LLAMA_ROPE_TYPE_NEOX; + res.freq_base = cparams.rope_freq_base; + res.freq_scale = cparams.rope_freq_scale; + res.factors = nullptr; + + return res; +} + float llama_model::get_rope_freq_base (const llama_cparams & cparams, int il) const { return hparams.is_swa(il) ? hparams.rope_freq_base_train_swa : cparams.rope_freq_base; } @@ -3182,8 +3306,20 @@ llama_memory_i * llama_model::create_memory(const llama_memory_params & params, // only the sparse-attention architectures use llama_memory_hybrid_idx // a null filter_idx means the GGUF has no indexer tensors llama_memory_hybrid::layer_filter_cb filter_idx = nullptr; - const bool needs_mem_idx = (arch == LLM_ARCH_QWEN4EXP); - if (arch == LLM_ARCH_FALCON_H1) { + const bool needs_mem_idx = (arch == LLM_ARCH_QWEN4EXP || arch == LLM_ARCH_KIMI_K31); + if (arch == LLM_ARCH_KIMI_K31) { + // DSA layers keep a K-only MLA cache plus a per-token indexer key cache, + // the KDA layers stay recurrent + filter_attn = [&](uint32_t il) { + return il < hparams.n_layer() && !hparams.is_recr(il); + }; + filter_recr = [&](uint32_t il) { + return il < hparams.n_layer() && hparams.is_recr(il); + }; + filter_idx = [&](uint32_t il) { + return il < hparams.n_layer() && !hparams.is_recr(il) && hparams.is_indexer_full(il); + }; + } else if (arch == LLM_ARCH_FALCON_H1) { filter_attn = [&](uint32_t) { return true; }; filter_recr = [&](uint32_t) { return true; }; } else if (arch == LLM_ARCH_NEMOTRON_H || arch == LLM_ARCH_NEMOTRON_H_MOE) { @@ -3400,6 +3536,10 @@ ggml_cgraph * llama_model::build_graph(const llm_graph_params & params) const { // TODO: move reranking logic here and generalize llm->build_dense_out(dense_2_out_layers, dense_2_out_layers_b, dense_3_out_layers); + llm->build_layer_inp_dev(); + + llm->build_h_nextn_dev(); + llm->res->set_outputs(params); return llm->res->get_gf(); @@ -3438,10 +3578,6 @@ const llama_vocab * llama_model_get_vocab(const llama_model * model) { return &model->vocab; } -void llama_free_model(llama_model * model) { - llama_model_free(model); -} - void llama_model_free(llama_model * model) { delete model; } @@ -3512,26 +3648,6 @@ const char * llama_model_decision_label(const struct llama_model * model, uint32 return nullptr; } -// deprecated -int32_t llama_n_ctx_train(const llama_model * model) { - return llama_model_n_ctx_train(model); -} - -// deprecated -int32_t llama_n_embd(const llama_model * model) { - return llama_model_n_embd(model); -} - -// deprecated -int32_t llama_n_layer(const llama_model * model) { - return llama_model_n_layer(model); -} - -// deprecated -int32_t llama_n_head(const llama_model * model) { - return llama_model_n_head(model); -} - llama_rope_type llama_model_rope_type(const llama_model * model) { switch (model->arch) { // these models do not use RoPE @@ -3555,9 +3671,12 @@ llama_rope_type llama_model_rope_type(const llama_model * model) { case LLM_ARCH_WAVTOKENIZER_DEC: case LLM_ARCH_NEMOTRON_H: case LLM_ARCH_NEMOTRON_H_MOE: + case LLM_ARCH_METIS16: case LLM_ARCH_KIMI_LINEAR: case LLM_ARCH_KIMI_K3: + case LLM_ARCH_KIMI_K31: case LLM_ARCH_GLM5_NEXT: + case LLM_ARCH_STARCODER: return LLAMA_ROPE_TYPE_NONE; // use what we call a normal RoPE, operating on pairs of consecutive head values @@ -3566,7 +3685,6 @@ llama_rope_type llama_model_rope_type(const llama_model * model) { case LLM_ARCH_LLAMA4: case LLM_ARCH_DECI: case LLM_ARCH_BAICHUAN: - case LLM_ARCH_STARCODER: case LLM_ARCH_INTERNLM2: case LLM_ARCH_MINICPM: case LLM_ARCH_XVERSE: @@ -3582,6 +3700,7 @@ llama_rope_type llama_model_rope_type(const llama_model * model) { case LLM_ARCH_DEEPSEEK4: case LLM_ARCH_DEEPSEEK41: case LLM_ARCH_MUSE_GLIMMER: + case LLM_ARCH_XING4_0: case LLM_ARCH_MUSE_SPARK: case LLM_ARCH_PLM: case LLM_ARCH_CHATGLM: @@ -3639,6 +3758,7 @@ llama_rope_type llama_model_rope_type(const llama_model * model) { case LLM_ARCH_QWEN3MOE: case LLM_ARCH_LLADA_MOE: case LLM_ARCH_RND1: + case LLM_ARCH_RICURSIVE: case LLM_ARCH_OLMO2: case LLM_ARCH_OLMOE: case LLM_ARCH_PHI2: @@ -4049,6 +4169,10 @@ uint32_t llama_model_n_tokens_delay(const struct llama_model * model) { return model->hparams.n_tokens_delay; } +bool llama_model_classifier_free_guidance(const struct llama_model * model) { + return model->hparams.classifier_free_guidance; +} + uint32_t llama_model_get_tok_embd_rows(const struct llama_model * model, const llama_token * tokens, uint32_t n_tokens, float * out) { if (model->vocab.n_tokens() == 0 || model->tok_embd == nullptr || tokens == nullptr || out == nullptr) { return 0; diff --git a/src/llama-model.h b/src/llama-model.h index da4964c98f7a..1b7ea1ce75fc 100644 --- a/src/llama-model.h +++ b/src/llama-model.h @@ -76,6 +76,7 @@ enum llm_type { LLM_TYPE_7B, LLM_TYPE_8B, LLM_TYPE_9B, + LLM_TYPE_10B, LLM_TYPE_11B, LLM_TYPE_12B, LLM_TYPE_13B, @@ -289,6 +290,8 @@ struct llama_layer { struct ggml_tensor * wq = nullptr; struct ggml_tensor * wk = nullptr; struct ggml_tensor * wv = nullptr; + struct ggml_tensor * k_cache_s = nullptr; + struct ggml_tensor * v_cache_s = nullptr; struct ggml_tensor * wo = nullptr; struct ggml_tensor * wqkv = nullptr; struct ggml_tensor * wg = nullptr; @@ -376,6 +379,9 @@ struct llama_layer { struct ggml_tensor * ffn_resid_alpha = nullptr; struct ggml_tensor * ffn_resid_beta = nullptr; + // value residual mix + struct ggml_tensor * attn_vres_lambda = nullptr; + // ff shared expert (shexp) struct ggml_tensor * ffn_gate_inp_shexp = nullptr; struct ggml_tensor * ffn_gate_shexp = nullptr; @@ -646,6 +652,18 @@ struct llama_layer { // gemma4 layer output scale, reused for talkie embedding skip scale struct ggml_tensor * out_scale = nullptr; + // metis16 per-pass hyper-connection mixes, attention pass LoRA, expert route embeddings + struct ggml_tensor * hc_attn_mix = nullptr; + struct ggml_tensor * hc_attn_read = nullptr; + struct ggml_tensor * hc_attn_write = nullptr; + struct ggml_tensor * hc_ffn_mix = nullptr; + struct ggml_tensor * hc_ffn_read = nullptr; + struct ggml_tensor * hc_ffn_write = nullptr; + struct ggml_tensor * attn_pass_lora_down = nullptr; + struct ggml_tensor * attn_pass_lora_up = nullptr; + struct ggml_tensor * attn_pass_gate = nullptr; + struct ggml_tensor * ffn_exp_embd = nullptr; + // limite struct ggml_tensor * attn_ve_gate = nullptr; struct ggml_tensor * attn_xsa_alpha = nullptr; @@ -730,6 +748,23 @@ struct llama_hadamard { llama_hadamard_rotations inv; // latent lookup table -> inverse transform }; +struct llama_rope_params { + int32_t n_rot = 0; // 0 if the layer has no RoPE + int32_t n_offs = 0; // first rotated dim in a cached K head + int32_t type = LLAMA_ROPE_TYPE_NONE; + int32_t n_ctx_orig = 0; + + float freq_base = 0.0f; + float freq_scale = 0.0f; + float ext_factor = 0.0f; + float attn_factor = 0.0f; + float beta_fast = 0.0f; + float beta_slow = 0.0f; + + ggml_tensor * factors = nullptr; + ggml_tensor * k_norm = nullptr; // K norm weight applied after RoPE, a K-shift has to undo it +}; + struct llama_model { llm_type type = LLM_TYPE_UNKNOWN; llm_arch arch = LLM_ARCH_UNKNOWN; @@ -779,6 +814,34 @@ struct llama_model { // hrm-text initial low-cycle state struct ggml_tensor * hrm_z_l_init = nullptr; + // metis16 hyper-connection stream embeddings, depth memory and continuation head + struct ggml_tensor * stream_embd = nullptr; + struct ggml_tensor * depth_mem_state_write = nullptr; + struct ggml_tensor * depth_mem_meta_write = nullptr; + struct ggml_tensor * depth_mem_entry_bias = nullptr; + struct ggml_tensor * depth_mem_query = nullptr; + struct ggml_tensor * depth_mem_query_b = nullptr; + struct ggml_tensor * depth_mem_key = nullptr; + struct ggml_tensor * depth_mem_key_b = nullptr; + struct ggml_tensor * depth_mem_value = nullptr; + struct ggml_tensor * depth_mem_value_b = nullptr; + struct ggml_tensor * depth_mem_out = nullptr; + struct ggml_tensor * depth_mem_out_b = nullptr; + struct ggml_tensor * depth_mem_route_state = nullptr; + struct ggml_tensor * depth_mem_route_retr = nullptr; + struct ggml_tensor * depth_mem_route_diff = nullptr; + struct ggml_tensor * depth_mem_route_hist = nullptr; + struct ggml_tensor * depth_mem_route_b = nullptr; + struct ggml_tensor * depth_mem_stream_gate = nullptr; + struct ggml_tensor * depth_mem_stream_gate_b = nullptr; + struct ggml_tensor * cont_hidden_state = nullptr; + struct ggml_tensor * cont_hidden_retr = nullptr; + struct ggml_tensor * cont_hidden_diff = nullptr; + struct ggml_tensor * cont_hidden_route = nullptr; + struct ggml_tensor * cont_hidden_b = nullptr; + struct ggml_tensor * cont_output = nullptr; + struct ggml_tensor * cont_output_b = nullptr; + // DeepSeek-V4 struct ggml_tensor * hc_head_fn = nullptr; struct ggml_tensor * hc_head_base = nullptr; @@ -927,6 +990,12 @@ struct llama_model { const struct ggml_tensor * get_tensor(const char * name) const; + // the RoPE of a layer, used by both the model graph and the K-shift + virtual llama_rope_params get_rope_params(const llama_cparams & cparams, int il) const; + + // same, for the keys of the lightning indexer + virtual llama_rope_params get_rope_params_indexer(const llama_cparams & cparams, int il) const; + float get_rope_freq_base (const llama_cparams & cparams, int il) const; float get_rope_freq_scale(const llama_cparams & cparams, int il) const; diff --git a/src/llama-moe-cache.cpp b/src/llama-moe-cache.cpp index 6a343a76033c..965f6a177612 100644 --- a/src/llama-moe-cache.cpp +++ b/src/llama-moe-cache.cpp @@ -7,35 +7,57 @@ #include "ggml-cpp.h" #include +#include +#include +#include +#include #include #include #include namespace { -// SLRU of the experts of a group of layers, the slot of each expert is kept in the slot map of its layer -struct moe_cache_slru { +// ClassLRU: one LRU list per frequency class, over the experts of a group of layers +// slot_map[il][e] is the slot of expert e of layer il, n_slots or later if not cached, the host slot map tensor +// CRF (combined recency and frequency) = uses of an expert, decayed with a half-life of N tokens, retained across evictions +// class = floor(log2(CRF)), at most n_class - 1; each slot is in the LRU list of the class, as of its last use +// victim: an empty slot, or the least recently used slot of each list not in the current plan() with the max age/2^class +struct moe_cache_lru { + static constexpr int n_class = 8; // floor(log2(1/(1 - 2^(-1/crf_half_life)))) = 7, class of CRF_max for half-life of 128 + static constexpr float crf_half_life = 128.0f; // tokens + int32_t n_expert = 0; int32_t n_slots = 0; - - std::vector slot_map; // [n_layer] data of the slot maps, -1 if the expert is not cached - std::vector key_of; // [n_slots] il*n_expert + expert, -1 if empty - - // probation and protected LRU lists; hits promote slots to the protected half - std::vector prev; - std::vector next; - int32_t head[2] = { -1, -1 }; - int32_t tail[2] = { -1, -1 }; - std::vector is_protected; - int32_t n_protected = 0; - - std::vector seen; // [n_expert] + int32_t empty = 0; // slot of the experts that are not cached + + // [n_layer] data of the slot maps, the experts at slot n_slots or later are not cached + // the experts computed on the CPU read zero slots after the cached ones on the device + std::vector slot_map; + std::vector key_of; // [n_slots] key of the expert in the slot, -1 if empty + + // one LRU list per class, the head is the least recently used end + std::vector prev; // [n_slots] -1 at the ends + std::vector next; // [n_slots] + int32_t head[n_class]; + int32_t tail[n_class]; + std::vector list_of; // [n_slots] class list of the slot + int32_t n_filled = 0; // slots that hold an expert + + std::vector crf; // [n_layer*n_expert] + std::vector last_use; // [n_layer*n_expert] layer clock at the last use, -1 if not used + std::vector clock; // [n_layer] tokens seen by the layer (as added by plan() for each ubatch) + + std::vector seen; // [n_expert] seen[e] == seen_gen if expert e is selected by the current plan() uint32_t seen_gen = 0; - std::vector uniq; + std::vector uniq; // the distinct selected experts + std::vector misses; + std::vector token_zero; // [n_tokens] zero slots used by each token void init(int32_t n_layer, int32_t n_expert, int32_t n_slots) { + GGML_ASSERT(n_class == (int) std::log2(1.0f/(1.0f - std::exp2(-1.0f/crf_half_life))) + 1); this->n_expert = n_expert; this->n_slots = n_slots; + this->empty = n_slots; slot_map.assign(n_layer, nullptr); key_of.assign(n_slots, -1); prev.resize(n_slots); @@ -44,16 +66,23 @@ struct moe_cache_slru { prev[s] = s - 1; next[s] = s + 1 < n_slots ? s + 1 : -1; } + for (int c = 0; c < n_class; ++c) { + head[c] = -1; + tail[c] = -1; + } head[0] = 0; tail[0] = n_slots - 1; - is_protected.assign(n_slots, false); + list_of.assign(n_slots, 0); + crf.assign((size_t) n_layer*n_expert, 0.0f); + last_use.assign((size_t) n_layer*n_expert, -1); + clock.assign(n_layer, 0); seen.assign(n_expert, 0); } - // move slot s to the most recently used end of a segment - void move(int32_t s, bool protect) { - const bool from = is_protected[s]; - if (from == protect && s == tail[from]) { + // move slot s to the most recently used end of list c + void move(int32_t s, int c) { + const int from = list_of[s]; + if (from == c && s == tail[from]) { return; } if (prev[s] >= 0) { @@ -67,36 +96,82 @@ struct moe_cache_slru { tail[from] = prev[s]; } - prev[s] = tail[protect]; + prev[s] = tail[c]; next[s] = -1; - if (tail[protect] >= 0) { - next[tail[protect]] = s; + if (tail[c] >= 0) { + next[tail[c]] = s; } else { - head[protect] = s; + head[c] = s; } - tail[protect] = s; - is_protected[s] = protect; - n_protected += int(protect) - int(from); + tail[c] = s; + list_of[s] = c; } - void touch(int32_t s) { - move(s, true); - if (n_protected > n_slots/2) { - move(head[1], false); + // key = index of an expert of a layer in crf and last_use + int32_t make_key(int32_t il, int32_t e) const { return il*n_expert + e; } + int32_t key_layer (int32_t key) const { return key / n_expert; } + int32_t key_expert(int32_t key) const { return key % n_expert; } + + // for a key after use(): its CRF is >= 1, so log2 is >= 0 and the cast is the floor + int class_of(int32_t key) const { + return std::min(n_class - 1, (int) std::log2(crf[key])); + } + + // the CRF of a key, decayed by the time since its last use + float decayed_crf(int32_t il, int32_t key) const { + return crf[key]*std::exp2(-(clock[il] - last_use[key])/crf_half_life); + } + + // an unused key gets 1 because crf[key] is 0 prior to first use + void use(int32_t il, int32_t key) { + crf[key] = decayed_crf(il, key) + 1.0f; + last_use[key] = clock[il]; + } + + // the experts selected by this plan() must not be evicted, nor the slots this plan() filled already + bool is_selected(int32_t il, int32_t key) const { + if (key < 0) { + return false; // unused slot + } + if (key_layer(key) != il) { + return false; // seen is per expert index, an expert of another layer can have the same index + } + return seen[key_expert(key)] == seen_gen; + } + + // the least recently used slot of list c that this plan() does not select, -1 if none + int32_t candidate(int c, int32_t il) const { + for (int32_t s = head[c]; s >= 0; s = next[s]) { + if (!is_selected(il, key_of[s])) { + return s; + } } + return -1; } int32_t victim(int32_t il) const { - for (int segment = 0; segment < 2; ++segment) { - for (int32_t s = head[segment]; s >= 0; s = next[s]) { - const int32_t key = key_of[s]; - // keep every expert selected by this request, including earlier fills - if (key < 0 || key / n_expert != il || seen[key % n_expert] != seen_gen) { - return s; - } + int32_t res = -1; + double res_score = -1.0; + for (int c = 0; c < n_class; ++c) { + const int32_t s = candidate(c, il); + if (s < 0) { + continue; + } + const int32_t key = key_of[s]; + if (key < 0) { + return s; // unused slot } + const int64_t age = clock[key_layer(key)] - last_use[key]; + const double score = std::ldexp((double) age, -c); // age/2^class + if (score > res_score) { + res = s; + res_score = score; + } + } + if (res < 0) { + GGML_ABORT("no MoE cache victim"); } - GGML_ABORT("no SLRU victim"); + return res; } struct fill { @@ -104,10 +179,15 @@ struct moe_cache_slru { int32_t slot; }; - // give a slot to each expert selected by ids in layer il, evicting from probation first + // give a slot to each expert selected by ids (n_tokens tokens) in layer il + // n_cpu(n) of the n misses are left to the CPU and read a zero slot on the device, they are not cached + // a zero slot is used at most once per token, a miss that does not get one is cached instead // returns false if the ids select more distinct experts than there are slots - bool plan(int32_t il, const int32_t * ids, size_t n_ids, std::vector & fills, size_t & n_hit) { + template + bool plan(int32_t il, const int32_t * ids, size_t n_ids, int64_t n_tokens, int32_t n_used, int32_t n_zero, F && n_cpu, + std::vector & fills, std::vector & cpu, size_t & n_hit) { fills.clear(); + cpu.clear(); n_hit = 0; if (++seen_gen == 0) { @@ -128,28 +208,80 @@ struct moe_cache_slru { int32_t * slots = slot_map[il]; - // promote hits before choosing victims for misses + // update the CRF of all selected experts and promote hits before choosing victims for misses + misses.clear(); for (int32_t e : uniq) { - if (slots[e] >= 0) { - touch(slots[e]); + const int32_t key = make_key(il, e); + use(il, key); + if (slots[e] < n_slots) { + move(slots[e], class_of(key)); n_hit++; + } else { + misses.push_back(e); } } - // sorted misses usually get consecutive slots, so the uploads can be merged - std::sort(uniq.begin(), uniq.end()); - for (int32_t e : uniq) { - if (slots[e] >= 0) { - continue; + + // the misses with the lowest CRF are left to the CPU, a miss that keeps being used gains CRF and enters the cache + // all misses are used at the same clock, so their CRF compare without decay + const size_t n_c = std::min(n_cpu(misses.size()), misses.size()); + if (n_c > 0) { + std::sort(misses.begin(), misses.end(), [&](int32_t a, int32_t b) { + const float crf_a = crf[make_key(il, a)]; + const float crf_b = crf[make_key(il, b)]; + return crf_a != crf_b ? crf_a < crf_b : a < b; + }); + GGML_ASSERT((size_t) n_tokens*n_used == n_ids); + token_zero.assign(n_tokens, 0); + size_t n_keep = n_c; + for (size_t i = 0; i < n_keep; ) { + const int32_t e = misses[i]; + uint64_t used = 0; + for (int64_t it = 0; it < n_tokens; ++it) { + for (int32_t k = 0; k < n_used; ++k) { + if (ids[it*n_used + k] == e) { + used |= token_zero[it]; + } + } + } + int32_t z = 0; + while (z < 64 && (used >> z & 1)) { + z++; + } + if (z >= n_zero) { + std::swap(misses[i], misses[--n_keep]); + continue; + } + for (int64_t it = 0; it < n_tokens; ++it) { + for (int32_t k = 0; k < n_used; ++k) { + if (ids[it*n_used + k] == e) { + token_zero[it] |= 1ull << z; + } + } + } + slots[e] = n_slots + z; + cpu.push_back({ e, n_slots + z }); + i++; } + misses.erase(misses.begin(), misses.begin() + n_keep); + } + + // fill the misses in expert order: victim() takes unused slots, and slots filled by one plan(), in list order, + // so consecutive experts often get consecutive slots and one upload + std::sort(misses.begin(), misses.end()); + for (int32_t e : misses) { const int32_t s = victim(il); if (key_of[s] >= 0) { - slot_map[key_of[s] / n_expert][key_of[s] % n_expert] = -1; + slot_map[key_layer(key_of[s])][key_expert(key_of[s])] = empty; + } else { + n_filled++; } - key_of[s] = il*n_expert + e; - slots[e] = s; - move(s, false); + const int32_t key = make_key(il, e); + key_of[s] = key; + slots[e] = s; + move(s, class_of(key)); fills.push_back({ e, s }); } + clock[il] += n_tokens; return true; } }; @@ -207,12 +339,16 @@ struct llama_moe_cache::impl { std::vector banks; // device storage of all slots, one per expert tensor size_t host_bytes = 0; int32_t n_slots = 0; - moe_cache_slru slru; + moe_cache_lru lru; }; struct layer { int32_t ig = -1; // -1 if the layer is not cached ggml_tensor * slot_map = nullptr; // I32 [1, n_expert] in host memory + ggml_tensor * slot_map_cpu = nullptr; // looked up in place of slot_map by the graphs with a CPU part, if cpu_share > 0, its data is not used + ggml_tensor * cpu_map = nullptr; // I32 [1, n_expert] in host memory, if cpu_share > 0 + std::vector cpu_experts; // experts set in cpu_map + uint32_t cpu_acc = 0; // fraction of a miss carried over to the next plan, in 1/65536 std::vector experts; // host expert tensors, in the order of the banks }; @@ -225,6 +361,7 @@ struct llama_moe_cache::impl { struct stats { size_t hits = 0; size_t misses = 0; + size_t cpu = 0; // misses computed on the CPU size_t bytes = 0; }; @@ -240,10 +377,18 @@ struct llama_moe_cache::impl { std::vector groups; std::vector layers; std::unordered_map bindings; // host experts -> cached experts - std::unordered_map layer_of; // slot map -> layer + std::unordered_map> layer_of; // slot map -> layer, and if the graph that looks it up has a CPU part std::vector ids; - std::vector fills; + std::vector fills; + std::vector cpu; + + float cpu_share = 0.0f; + uint32_t cpu_share_q = 0; // cpu_share in 1/65536, integer so that a simulator can reproduce the rounding + int32_t n_zero = 0; // zero slots per group + + // LLAMA_MOE_CACHE_TRACE: the selected experts of each slot lookup, for a simulator of the cache + FILE * trace = nullptr; // slot maps in host memory ggml_context_ptr ctx_host; @@ -253,8 +398,8 @@ struct llama_moe_cache::impl { // views used by copy_experts ggml_context_ptr ctx_views; - impl(const llama_model & model, bool no_alloc, const std::vector & backends, const std::vector & bufts, size_t size) : - n_expert_used(model.hparams.n_expert_used_max()), layers(model.layers.size()) { + impl(const llama_model & model, bool no_alloc, const std::vector & backends, const std::vector & bufts, size_t size, float cpu_share) : + n_expert_used(model.hparams.n_expert_used_max()), layers(model.layers.size()), cpu_share(std::clamp(cpu_share, 0.0f, 1.0f)) { for (size_t i = 0; i < backends.size(); ++i) { const auto dev_type = ggml_backend_dev_type(ggml_backend_get_device(backends[i])); if (dev_type == GGML_BACKEND_DEVICE_TYPE_GPU || dev_type == GGML_BACKEND_DEVICE_TYPE_IGPU) { @@ -302,12 +447,14 @@ struct llama_moe_cache::impl { return; } + // with misses on the CPU, the n_zero slots after the cached ones stay zero: the device computes these misses as zero // one extra slot at the end, CUDA MMQ can read past the last expert + n_zero = cpu_share > 0.0f ? std::min(n_expert_used, 64) : 0; auto alloc_size = [&](const group & g, int32_t n_slots) { const size_t alignment = ggml_backend_buft_get_alignment(devices[g.id].buft); size_t res = 0; for (const ggml_tensor * t : g.ref) { - res += GGML_PAD(t->nb[2]*(n_slots + 1), alignment); + res += GGML_PAD(t->nb[2]*(n_slots + n_zero + 1), alignment); } return res; }; @@ -343,6 +490,14 @@ struct llama_moe_cache::impl { } // within a device the budget is split by the size of the experts, so each group caches the same fraction of its experts + cpu_share_q = (uint32_t) std::lround(cpu_share*65536.0f); + if (const char * path = getenv("LLAMA_MOE_CACHE_TRACE"); path != nullptr && *path != '\0') { + trace = fopen(path, "wb"); + if (trace == nullptr) { + throw std::runtime_error(std::string("failed to open the MoE cache trace ") + path); + } + write_trace_header(split_sum); + } std::vector n_tensors(devices.size(), 0); size_t n_tensors_host = 0; for (group & g : groups) { @@ -358,9 +513,9 @@ struct llama_moe_cache::impl { g.n_slots = 0; continue; } - g.slru.init(model.layers.size(), n_expert, g.n_slots); + g.lru.init(model.layers.size(), n_expert, g.n_slots); n_tensors[g.id] += g.ref.size()*(1 + g.layers.size()); - n_tensors_host += g.layers.size(); + n_tensors_host += g.layers.size()*(n_zero > 0 ? 3 : 1); // slot maps and CPU map } if (n_tensors_host == 0) { throw std::runtime_error("MoE cache is too small to hold the experts of one token"); @@ -396,7 +551,7 @@ struct llama_moe_cache::impl { } ggml_context * ctx = devices[g.id].ctx.get(); for (const ggml_tensor * t : g.ref) { - ggml_tensor * bank = ggml_new_tensor_3d(ctx, t->type, t->ne[0], t->ne[1], g.n_slots + 1); + ggml_tensor * bank = ggml_new_tensor_3d(ctx, t->type, t->ne[0], t->ne[1], g.n_slots + n_zero + 1); GGML_ASSERT(bank->nb[2] == t->nb[2]); ggml_format_name(bank, "moe_cache.%zu.%s", ig, t->name); g.banks.push_back(bank); @@ -407,14 +562,23 @@ struct llama_moe_cache::impl { l.experts = llama_moe_cache_layer_experts(model.layers[il]); for (size_t ip = 0; ip < l.experts.size(); ++ip) { ggml_tensor * bank = g.banks[ip]; - ggml_tensor * cached = ggml_view_3d(ctx, bank, bank->ne[0], bank->ne[1], g.n_slots, bank->nb[1], bank->nb[2], 0); + ggml_tensor * cached = ggml_view_3d(ctx, bank, bank->ne[0], bank->ne[1], g.n_slots + n_zero, bank->nb[1], bank->nb[2], 0); ggml_format_name(cached, "moe_cache.%s", l.experts[ip]->name); bindings[l.experts[ip]] = { il, (int32_t) ip, cached }; } l.slot_map = ggml_new_tensor_2d(ctx_host.get(), GGML_TYPE_I32, 1, g.ref[0]->ne[2]); ggml_format_name(l.slot_map, "moe_cache.slot_map-%d", il); - layer_of[l.slot_map] = il; + layer_of[l.slot_map] = { il, false }; buf_host_size += GGML_PAD(ggml_nbytes(l.slot_map), alignment_host); + if (n_zero > 0) { + l.slot_map_cpu = ggml_new_tensor_2d(ctx_host.get(), GGML_TYPE_I32, 1, g.ref[0]->ne[2]); + ggml_format_name(l.slot_map_cpu, "moe_cache.slot_map_cpu-%d", il); + layer_of[l.slot_map_cpu] = { il, true }; + buf_host_size += GGML_PAD(ggml_nbytes(l.slot_map_cpu), alignment_host); + l.cpu_map = ggml_new_tensor_2d(ctx_host.get(), GGML_TYPE_I32, 1, g.ref[0]->ne[2]); + ggml_format_name(l.cpu_map, "moe_cache.cpu_map-%d", il); + buf_host_size += GGML_PAD(ggml_nbytes(l.cpu_map), alignment_host); + } } devices[g.id].buf_size += alloc_size(g, g.n_slots); } @@ -432,13 +596,15 @@ struct llama_moe_cache::impl { } buf_host = llama_memory_bufs::alloc_buf(ctx_host.get(), buft_host, no_alloc, "MoE cache slot maps"); if (!no_alloc) { - ggml_backend_buffer_clear(buf_host.get(), 0xff); // all slots are -1 + ggml_backend_buffer_clear(buf_host.get(), 0xff); // the CPU maps are -1 buf_host_size = ggml_backend_buffer_get_size(buf_host.get()); for (group & g : groups) { for (int32_t il : g.layers) { if (layers[il].slot_map != nullptr) { - g.slru.slot_map[il] = (int32_t *) layers[il].slot_map->data; + int32_t * slots = (int32_t *) layers[il].slot_map->data; + std::fill(slots, slots + layers[il].slot_map->ne[1], g.lru.empty); + g.lru.slot_map[il] = slots; } } } @@ -457,12 +623,13 @@ struct llama_moe_cache::impl { if (d.host_bytes == 0) { continue; } - LLAMA_LOG_INFO("%s: %10s MoE cache size = %8.2f MiB for %.2f MiB of host experts\n", __func__, - ggml_backend_buft_name(d.buft), d.buf_size/1024.0/1024.0, d.host_bytes/1024.0/1024.0); + LLAMA_LOG_INFO("%s: %10s MoE cache size = %8.2f MiB for %.2f MiB of host experts, %.0f%% of the misses on the CPU\n", __func__, + ggml_backend_buft_name(d.buft), d.buf_size/1024.0/1024.0, d.host_bytes/1024.0/1024.0, 100.0*cpu_share); for (const group & g : groups) { if (g.id == (int32_t) id) { - LLAMA_LOG_INFO("%s: %2zu layers, %s: %5d slots (%.1f%%)\n", __func__, - g.layers.size(), ggml_type_name(g.ref.back()->type), g.n_slots, 100.0*g.n_slots/(g.layers.size()*g.ref[0]->ne[2])); + LLAMA_LOG_INFO("%s: %2zu layers, %s: %5d slots (%.1f%%, %.1f tokens worth)\n", __func__, + g.layers.size(), ggml_type_name(g.ref.back()->type), g.n_slots, 100.0*g.n_slots/(g.layers.size()*g.ref[0]->ne[2]), + (double) g.n_slots/(g.layers.size()*n_expert_used)); } } } @@ -472,11 +639,52 @@ struct llama_moe_cache::impl { return devices[groups[layers[il].ig].id].backend; } + // trace format, little endian: "MCT1", then u32 n_layer, n_expert_used, n_group + // per group: u32 device, n_expert, n_layers, the layers (u32), f64 split, split_sum, u64 group host bytes, device host bytes, alignment, + // u32 n_tensors, the bytes of one expert of each tensor (u64) + // then one record per slot lookup: i32 layer, experts per token, n_ids, the ids (i32) + void write_trace_header(double split_sum) { + auto u32 = [&](uint32_t v) { fwrite(&v, sizeof(v), 1, trace); }; + auto u64 = [&](uint64_t v) { fwrite(&v, sizeof(v), 1, trace); }; + auto f64 = [&](double v) { fwrite(&v, sizeof(v), 1, trace); }; + fwrite("MCT1", 1, 4, trace); + u32(layers.size()); + u32(n_expert_used); + u32(groups.size()); + for (const group & g : groups) { + const device & d = devices[g.id]; + u32(g.id); + u32(g.ref[0]->ne[2]); + u32(g.layers.size()); + for (int32_t il : g.layers) { + u32(il); + } + f64(d.split); + f64(split_sum); + u64(g.host_bytes); + u64(d.host_bytes); + u64(ggml_backend_buft_get_alignment(d.buft)); + u32(g.ref.size()); + for (const ggml_tensor * t : g.ref) { + u64(t->nb[2]); + } + } + } + + void write_trace(int32_t il, int32_t n_used) { + const int32_t rec[3] = { il, n_used, (int32_t) ids.size() }; + fwrite(rec, sizeof(int32_t), 3, trace); + fwrite(ids.data(), sizeof(int32_t), ids.size(), trace); + } + ~impl() { log_stats(); + if (trace != nullptr) { + fclose(trace); + } } - ggml_tensor * get_slot_map(int32_t il, int64_t n_tokens, int64_t n_expert_used) const { + ggml_tensor * get_slot_map(int32_t il, int64_t n_tokens, int64_t n_expert_used, bool cpu_part) const { if (il < 0 || il >= (int32_t) layers.size() || layers[il].ig < 0) { return nullptr; } @@ -486,7 +694,28 @@ struct llama_moe_cache::impl { if (n_tokens == 0 || n_tokens > max_batch || std::min(n_tokens*n_expert_used, l.slot_map->ne[1]) > groups[l.ig].n_slots) { return nullptr; } - return l.slot_map; + GGML_ASSERT(!cpu_part || l.slot_map_cpu != nullptr); + return cpu_part ? l.slot_map_cpu : l.slot_map; + } + + ggml_tensor * get_cpu_map(int32_t il, int64_t n_tokens) const { + if (il < 0 || il >= (int32_t) layers.size() || layers[il].cpu_map == nullptr) { + return nullptr; + } + // the scheduler would run the CPU part on a device that offloads such batches from host weights + ggml_tensor probe = {}; + probe.op = GGML_OP_MUL_MAT_ID; + probe.type = GGML_TYPE_F32; + probe.ne[0] = 1; + probe.ne[1] = n_expert_used; + probe.ne[2] = n_tokens; + probe.ne[3] = 1; + for (const device & d : devices) { + if (ggml_backend_offload_op(d.backend, &probe)) { + return nullptr; + } + } + return layers[il].cpu_map; } ggml_tensor * get_experts(const ggml_tensor * w) const { @@ -506,12 +735,12 @@ struct llama_moe_cache::impl { } // large batches only read the cache, so the experts used in generation stay in it - const int32_t * slots = g.slru.slot_map[b.il]; - if (slots == nullptr || slots[e] < 0) { + const int32_t * slots = g.lru.slot_map[b.il]; + if (slots == nullptr || slots[e] >= g.n_slots) { return 0; } int64_t n = 1; - while (e + n <= last && slots[e + n] == slots[e] + n) { + while (e + n <= last && slots[e + n] < g.n_slots && slots[e + n] == slots[e] + n) { n++; } @@ -534,8 +763,9 @@ struct llama_moe_cache::impl { if (it == layer_of.end()) { return false; } - const int32_t il = it->second; - const layer & l = layers[il]; + const int32_t il = it->second.first; + const bool has_cpu = it->second.second; + layer & l = layers[il]; group & g = groups[l.ig]; GGML_ASSERT(backend == devices[g.id].backend); @@ -561,15 +791,40 @@ struct llama_moe_cache::impl { } } GGML_ASSERT(ggml_is_contiguous(sel)); + GGML_ASSERT(sel->op == GGML_OP_RESHAPE); // of the [n_expert_used, n_tokens] ids, see build_moe_cache_slots + + const int32_t n_used = (int32_t) sel->src[0]->ne[0]; ids.resize(ggml_nelements(sel)); ggml_backend_tensor_get_async(backend, sel, ids.data(), 0, ggml_nbytes(sel)); ggml_backend_synchronize(backend); + if (trace != nullptr) { + write_trace(il, n_used); + } + + // part of the misses is computed on the CPU once the cache is full, the carry keeps the share right on average + const uint32_t share_q = has_cpu && g.lru.n_filled == g.lru.n_slots ? cpu_share_q : 0; + auto n_cpu = [&](size_t n_miss) { + const uint64_t acc = (uint64_t) n_miss*share_q + l.cpu_acc; + l.cpu_acc = acc & 0xffff; + return (size_t) (acc >> 16); + }; size_t n_hit = 0; - if (!g.slru.plan(il, ids.data(), ids.size(), fills, n_hit)) { + if (!g.lru.plan(il, ids.data(), ids.size(), sel->src[0]->ne[1], n_used, n_zero, n_cpu, fills, cpu, n_hit)) { GGML_ABORT("the MoE cache is too small for the experts selected in layer %d", il); } + if (has_cpu) { + int32_t * cpu_map = (int32_t *) l.cpu_map->data; + for (int32_t e : l.cpu_experts) { + cpu_map[e] = -1; + } + l.cpu_experts.clear(); + for (const auto & f : cpu) { + cpu_map[f.expert] = f.expert; + l.cpu_experts.push_back(f.expert); + } + } // upload the missing experts, consecutive experts going to consecutive slots are uploaded together size_t bytes = 0; @@ -590,11 +845,12 @@ struct llama_moe_cache::impl { stats & st = ids.size() <= (size_t) 8*n_expert_used ? stats_small : stats_large; st.hits += n_hit; - st.misses += fills.size(); + st.misses += fills.size() + cpu.size(); + st.cpu += cpu.size(); st.bytes += bytes; // the next copy synchronizes the backend before it changes the slot map again - ggml_backend_tensor_set_async(backend, dst, src->data, 0, ggml_nbytes(src)); + ggml_backend_tensor_set_async(backend, dst, l.slot_map->data, 0, ggml_nbytes(l.slot_map)); return true; } @@ -605,8 +861,8 @@ struct llama_moe_cache::impl { if (n == 0) { return; } - LLAMA_LOG_INFO("llama_moe_cache: %s: hits = %zu, misses = %zu, hit rate = %.2f%%, uploaded = %.2f MiB\n", - name, st.hits, st.misses, 100.0*st.hits/n, st.bytes/1024.0/1024.0); + LLAMA_LOG_INFO("llama_moe_cache: %s: hits = %zu, misses = %zu (%zu on the CPU), hit rate = %.2f%%, uploaded = %.2f MiB\n", + name, st.hits, st.misses, st.cpu, 100.0*st.hits/n, st.bytes/1024.0/1024.0); }; log("ubatch <= 8", stats_small); log("ubatch > 8", stats_large); @@ -616,8 +872,8 @@ struct llama_moe_cache::impl { } }; -llama_moe_cache::llama_moe_cache(const llama_model & model, bool no_alloc, const std::vector & backends, const std::vector & bufts, size_t size) : - pimpl(new impl(model, no_alloc, backends, bufts, size)) { +llama_moe_cache::llama_moe_cache(const llama_model & model, bool no_alloc, const std::vector & backends, const std::vector & bufts, size_t size, float cpu_share) : + pimpl(new impl(model, no_alloc, backends, bufts, size, cpu_share)) { } llama_moe_cache::~llama_moe_cache() = default; @@ -626,14 +882,18 @@ ggml_backend_t llama_moe_cache::backend(int32_t il) const { return pimpl->backend(il); } -ggml_tensor * llama_moe_cache::get_slot_map(int32_t il, int64_t n_tokens, int64_t n_expert_used) const { - return pimpl->get_slot_map(il, n_tokens, n_expert_used); +ggml_tensor * llama_moe_cache::get_slot_map(int32_t il, int64_t n_tokens, int64_t n_expert_used, bool cpu_part) const { + return pimpl->get_slot_map(il, n_tokens, n_expert_used, cpu_part); } ggml_tensor * llama_moe_cache::get_experts(const ggml_tensor * w) const { return pimpl->get_experts(w); } +ggml_tensor * llama_moe_cache::get_cpu_map(int32_t il, int64_t n_tokens) const { + return pimpl->get_cpu_map(il, n_tokens); +} + bool llama_moe_cache::copy(ggml_backend_t backend, const ggml_tensor * src, ggml_tensor * dst, ggml_cgraph * graph) { return pimpl->copy(backend, src, dst, graph); } diff --git a/src/llama-moe-cache.h b/src/llama-moe-cache.h index 6c8e36582a42..636bc4559c19 100644 --- a/src/llama-moe-cache.h +++ b/src/llama-moe-cache.h @@ -13,18 +13,24 @@ struct llama_model; class llama_moe_cache { public: // backends are all the backends of the context, each GPU gets its own cache of the given size for the layers assigned to it - llama_moe_cache(const llama_model & model, bool no_alloc, const std::vector & backends, const std::vector & bufts, size_t size); + // cpu_share is the share of the misses computed on the CPU instead of uploaded + llama_moe_cache(const llama_model & model, bool no_alloc, const std::vector & backends, const std::vector & bufts, size_t size, float cpu_share); ~llama_moe_cache(); // the device that caches layer il ggml_backend_t backend(int32_t il) const; // the slot map of layer il, if its experts can be read from the cache for n_tokens tokens, nullptr otherwise - ggml_tensor * get_slot_map(int32_t il, int64_t n_tokens, int64_t n_expert_used) const; + // cpu_part: the graph computes the misses in the CPU map of the layer (see get_cpu_map) + ggml_tensor * get_slot_map(int32_t il, int64_t n_tokens, int64_t n_expert_used, bool cpu_part) const; // the experts of w in the cache, nullptr if w is not cached ggml_tensor * get_experts(const ggml_tensor * w) const; + // the experts of layer il computed on the CPU: I32 [1, n_expert] in host memory, e for these experts, -1 for the others + // nullptr if no misses of layer il are computed on the CPU for ubatches of n_tokens tokens + ggml_tensor * get_cpu_map(int32_t il, int64_t n_tokens) const; + // ggml_backend_sched copy callback, returns false if src is not a slot map bool copy(ggml_backend_t backend, const ggml_tensor * src, ggml_tensor * dst, ggml_cgraph * graph); diff --git a/src/llama-quant.cpp b/src/llama-quant.cpp index 34c247858188..4ae396101fae 100644 --- a/src/llama-quant.cpp +++ b/src/llama-quant.cpp @@ -361,6 +361,23 @@ static bool tensor_allows_quantization(const llama_model_quantize_params * param quantize &= name.find("indexer.k_proj.weight") == std::string::npos; quantize &= name.find("indexer.q_proj.weight") == std::string::npos; + // metis16: small control tensors that steer every layer; the stream and hc_ tables (ne[0] = 4) would fall back to F16 + if (arch == LLM_ARCH_METIS16) { + quantize &= name.rfind("depth_mem_", 0) == std::string::npos; + quantize &= name.rfind("cont_", 0) == std::string::npos; + quantize &= name.find("stream_embd") == std::string::npos; + quantize &= name.find("attn_pass_lora") == std::string::npos; + quantize &= name.find("ffn_exp_embd") == std::string::npos; + quantize &= name.find("hc_") == std::string::npos; + } + + // kimi-k31: the indexer picks the top-k cells, so keep its projections accurate + if (arch == LLM_ARCH_KIMI_K31) { + quantize &= name.find("indexer.attn_q_b") == std::string::npos; + quantize &= name.find("indexer.attn_k") == std::string::npos; + quantize &= name.find("indexer.proj") == std::string::npos; + } + // glm5-next if (arch == LLM_ARCH_GLM5_NEXT) { quantize &= name.find("hc_") == std::string::npos; @@ -379,6 +396,12 @@ static bool tensor_allows_quantization(const llama_model_quantize_params * param quantize &= name.find("attn_v_b.weight") == std::string::npos; } + // xing4_0: do not quantize the MHC hyper-connection gate matrices + // (the 1D hc_*_base / hc_*_scale tensors are already excluded by the n_dims < 2 rule) + if (arch == LLM_ARCH_XING4_0) { + quantize &= name.find("hc_") == std::string::npos; + } + // the decision head of vllm-decision2 is small and computed in F32 if (arch == LLM_ARCH_VLLM_DECISION2) { quantize &= name.compare(0, 9, "decision.") != 0; diff --git a/src/llama-sampler.cpp b/src/llama-sampler.cpp index 9004efa20df7..d8cd441a5583 100644 --- a/src/llama-sampler.cpp +++ b/src/llama-sampler.cpp @@ -2697,8 +2697,6 @@ static struct llama_sampler * llama_sampler_init_grammar_impl( const char * grammar_str, const char * grammar_root, bool lazy, - const char ** trigger_words, - size_t num_trigger_words, const llama_token * trigger_tokens, size_t num_trigger_tokens, const char ** trigger_patterns, @@ -2727,7 +2725,7 @@ static void llama_sampler_grammar_reset(struct llama_sampler * smpl) { static struct llama_sampler * llama_sampler_grammar_clone(const struct llama_sampler * smpl) { const auto * ctx = (const llama_sampler_grammar *) smpl->ctx; - auto * result = llama_sampler_init_grammar_impl(ctx->vocab, nullptr, nullptr, false, nullptr, 0, nullptr, 0, nullptr, 0); + auto * result = llama_sampler_init_grammar_impl(ctx->vocab, nullptr, nullptr, false, nullptr, 0, nullptr, 0); GGML_ASSERT(result); // copy the state @@ -2775,8 +2773,6 @@ static struct llama_sampler * llama_sampler_init_grammar_impl( const char * grammar_str, const char * grammar_root, bool lazy, - const char ** trigger_words, - size_t num_trigger_words, const llama_token * trigger_tokens, size_t num_trigger_tokens, const char ** trigger_patterns, @@ -2784,26 +2780,7 @@ static struct llama_sampler * llama_sampler_init_grammar_impl( auto * ctx = new llama_sampler_grammar; if (grammar_str != nullptr && grammar_str[0] != '\0') { - std::string trigger_pattern; - llama_grammar * grammar = nullptr; - // TODO: remove trigger_words support. - if (trigger_words != nullptr && num_trigger_words > 0) { - GGML_ASSERT(trigger_patterns == nullptr && num_trigger_patterns == 0); - trigger_pattern = "[\\s\\S]*?("; - for (size_t i = 0; i < num_trigger_words; ++i) { - static const std::regex special_chars("[.^$|()*+?\\[\\]{}\\\\]"); - if (i > 0) { - trigger_pattern += "|"; - } - trigger_pattern += std::regex_replace(trigger_words[i], special_chars, "\\$0"); - } - trigger_pattern += ")[\\s\\S]*"; - - std::array tmp_trigger_patterns = { trigger_pattern.c_str() }; - grammar = llama_grammar_init_impl(vocab, grammar_str, grammar_root, lazy, tmp_trigger_patterns.data(), tmp_trigger_patterns.size(), trigger_tokens, num_trigger_tokens); - } else { - grammar = llama_grammar_init_impl(vocab, grammar_str, grammar_root, lazy, trigger_patterns, num_trigger_patterns, trigger_tokens, num_trigger_tokens); - } + llama_grammar * grammar = llama_grammar_init_impl(vocab, grammar_str, grammar_root, lazy, trigger_patterns, num_trigger_patterns, trigger_tokens, num_trigger_tokens); *ctx = { /* .vocab = */ vocab, /* .grammar_str = */ grammar_str, @@ -2833,18 +2810,7 @@ struct llama_sampler * llama_sampler_init_grammar( const struct llama_vocab * vocab, const char * grammar_str, const char * grammar_root) { - return llama_sampler_init_grammar_impl(vocab, grammar_str, grammar_root, /* lazy= */ false, nullptr, 0, nullptr, 0, nullptr, 0); -} - -struct llama_sampler * llama_sampler_init_grammar_lazy( - const struct llama_vocab * vocab, - const char * grammar_str, - const char * grammar_root, - const char ** trigger_words, - size_t num_trigger_words, - const llama_token * trigger_tokens, - size_t num_trigger_tokens) { - return llama_sampler_init_grammar_impl(vocab, grammar_str, grammar_root, /* lazy= */ true, trigger_words, num_trigger_words, trigger_tokens, num_trigger_tokens, nullptr, 0); + return llama_sampler_init_grammar_impl(vocab, grammar_str, grammar_root, /* lazy= */ false, nullptr, 0, nullptr, 0); } struct llama_sampler * llama_sampler_init_grammar_lazy_patterns( @@ -2855,7 +2821,7 @@ struct llama_sampler * llama_sampler_init_grammar_lazy_patterns( size_t num_trigger_patterns, const llama_token * trigger_tokens, size_t num_trigger_tokens) { - return llama_sampler_init_grammar_impl(vocab, grammar_str, grammar_root, /* lazy= */ true, nullptr, 0, trigger_tokens, num_trigger_tokens, trigger_patterns, num_trigger_patterns); + return llama_sampler_init_grammar_impl(vocab, grammar_str, grammar_root, /* lazy= */ true, trigger_tokens, num_trigger_tokens, trigger_patterns, num_trigger_patterns); } // penalties diff --git a/src/llama-vocab.cpp b/src/llama-vocab.cpp index 229a051e0676..e90f1bd117d4 100644 --- a/src/llama-vocab.cpp +++ b/src/llama-vocab.cpp @@ -4720,11 +4720,6 @@ int32_t llama_vocab_n_tokens(const struct llama_vocab * vocab) { return vocab->n_tokens(); } -// deprecated -int32_t llama_n_vocab(const struct llama_vocab * vocab) { - return llama_vocab_n_tokens(vocab); -} - enum llama_vocab_type llama_vocab_type(const struct llama_vocab * vocab) { return vocab->get_type(); } @@ -4761,11 +4756,6 @@ llama_token llama_vocab_eot(const struct llama_vocab * vocab) { return vocab->token_eot(); } -// deprecated -llama_token llama_vocab_cls(const struct llama_vocab * vocab) { - return vocab->token_bos(); -} - llama_token llama_vocab_sep(const struct llama_vocab * vocab) { return vocab->token_sep(); } @@ -4826,107 +4816,6 @@ llama_token llama_vocab_mask(const struct llama_vocab* vocab) { return vocab->token_mask(); } -// deprecated -const char * llama_token_get_text(const struct llama_vocab * vocab, llama_token token) { - return llama_vocab_get_text(vocab, token); -} - -// deprecated -float llama_token_get_score(const struct llama_vocab * vocab, llama_token token) { - return llama_vocab_get_score(vocab, token); -} - -// deprecated -enum llama_token_attr llama_token_get_attr(const struct llama_vocab * vocab, llama_token token) { - return llama_vocab_get_attr(vocab, token); -} - -// deprecated -bool llama_token_is_eog(const struct llama_vocab * vocab, llama_token token) { - return llama_vocab_is_eog(vocab, token); -} - -// deprecated -bool llama_token_is_control(const struct llama_vocab * vocab, llama_token token) { - return llama_vocab_is_control(vocab, token); -} - -// deprecated -llama_token llama_token_bos(const struct llama_vocab * vocab) { - return llama_vocab_bos(vocab); -} - -// deprecated -llama_token llama_token_eos(const struct llama_vocab * vocab) { - return llama_vocab_eos(vocab); -} - -// deprecated -llama_token llama_token_eot(const struct llama_vocab * vocab) { - return llama_vocab_eot(vocab); -} - -// deprecated -llama_token llama_token_cls(const struct llama_vocab * vocab) { - //return llama_vocab_cls(vocab); - return llama_vocab_bos(vocab); // avoid deprecation warning -} - -// deprecated -llama_token llama_token_sep(const struct llama_vocab * vocab) { - return llama_vocab_sep(vocab); -} - -// deprecated -llama_token llama_token_nl (const struct llama_vocab * vocab) { - return llama_vocab_nl(vocab); -} - -// deprecated -llama_token llama_token_pad(const struct llama_vocab * vocab) { - return llama_vocab_pad(vocab); -} - -// deprecated -bool llama_add_bos_token(const struct llama_vocab * vocab) { - return llama_vocab_get_add_bos(vocab); -} - -// deprecated -bool llama_add_eos_token(const struct llama_vocab * vocab) { - return llama_vocab_get_add_eos(vocab); -} - -// deprecated -llama_token llama_token_fim_pre(const struct llama_vocab * vocab) { - return llama_vocab_fim_pre(vocab); -} - -// deprecated -llama_token llama_token_fim_suf(const struct llama_vocab * vocab) { - return llama_vocab_fim_suf(vocab); -} - -// deprecated -llama_token llama_token_fim_mid(const struct llama_vocab * vocab) { - return llama_vocab_fim_mid(vocab); -} - -// deprecated -llama_token llama_token_fim_pad(const struct llama_vocab * vocab) { - return llama_vocab_fim_pad(vocab); -} - -// deprecated -llama_token llama_token_fim_rep(const struct llama_vocab * vocab) { - return llama_vocab_fim_rep(vocab); -} - -// deprecated -llama_token llama_token_fim_sep(const struct llama_vocab * vocab) { - return llama_vocab_fim_sep(vocab); -} - // // tokenization // diff --git a/src/llama.cpp b/src/llama.cpp index e3699ea1d534..cee6a7dc4428 100644 --- a/src/llama.cpp +++ b/src/llama.cpp @@ -455,12 +455,6 @@ struct llama_model * llama_model_init_from_user( params.use_extra_bufts = false; return llama_model_load_from_file_impl(metadata, set_tensor_data, set_tensor_data_ud, path_model, splits, /*file*/ nullptr, params); } -// deprecated -struct llama_model * llama_load_model_from_file( - const char * path_model, - struct llama_model_params params) { - return llama_model_load_from_file(path_model, params); -} struct llama_model * llama_model_load_from_file( const char * path_model, diff --git a/src/models/afmoe.cpp b/src/models/afmoe.cpp index 266597a3553a..0eb208f6338b 100644 --- a/src/models/afmoe.cpp +++ b/src/models/afmoe.cpp @@ -23,6 +23,10 @@ void llama_model_afmoe::load_arch_hparams(llama_model_loader & ml) { hparams.swa_type = LLAMA_SWA_TYPE_NONE; } + for (uint32_t il = 0; il < hparams.n_layer_all; ++il) { + hparams.rope_pattern[il] = (il + 1) % hparams.n_no_rope_layer_step != 0; + } + // Default to sigmoid if not set if (hparams.expert_gating_func == LLAMA_EXPERT_GATING_FUNC_TYPE_NONE) { hparams.expert_gating_func = LLAMA_EXPERT_GATING_FUNC_TYPE_SIGMOID; @@ -101,10 +105,10 @@ void llama_model_afmoe::load_arch_tensors(llama_model_loader &) { } std::unique_ptr llama_model_afmoe::build_arch_graph(const llm_graph_params & params) const { - return std::make_unique(*this, params); + return std::make_unique(params); } -llama_model_afmoe::graph::graph(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) { +llama_model_afmoe::graph::graph(const llm_graph_params & params) : llm_graph_context(params) { const int64_t n_embd_head = hparams.n_embd_head_v(); GGML_ASSERT(n_embd_head == hparams.n_embd_head_k()); @@ -126,15 +130,9 @@ llama_model_afmoe::graph::graph(const llama_model & model, const llm_graph_param const float kq_scale = 1.0f/sqrtf(float(n_embd_head)); for (int il = 0; il < n_layer; ++il) { - const float freq_base_l = model.get_rope_freq_base (cparams, il); - const float freq_scale_l = model.get_rope_freq_scale(cparams, il); ggml_tensor * inpSA = inpL; - // This overlaps with SWA layers in current models, so get_rope_freq_base/scale may be superfluous - const bool use_rope = hparams.n_no_rope_layer_step > 0 && - (il + 1) % hparams.n_no_rope_layer_step != 0; - // dual attention normalization (pre) cur = build_norm(inpL, model.layers[il].attn_norm, NULL, @@ -158,19 +156,11 @@ llama_model_afmoe::graph::graph(const llama_model & model, const llm_graph_param cb(Qcur, "Qcur_normed", il); cb(Kcur, "Kcur_normed", il); - if (use_rope) { - Qcur = ggml_rope_ext( - ctx0, Qcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base_l, freq_scale_l, - ext_factor, attn_factor, beta_fast, beta_slow); - cb(Qcur, "Qcur_rope", il); - - Kcur = ggml_rope_ext( - ctx0, Kcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base_l, freq_scale_l, - ext_factor, attn_factor, beta_fast, beta_slow); - cb(Kcur, "Kcur_rope", il); - } + Qcur = build_rope(Qcur, inp_pos, il); + cb(Qcur, "Qcur_rope", il); + + Kcur = build_rope(Kcur, inp_pos, il); + cb(Kcur, "Kcur_rope", il); cur = build_attn(inp_attn, NULL, NULL, NULL, // wo will be applied after gating diff --git a/src/models/apertus.cpp b/src/models/apertus.cpp index 130980e32a70..c0bb23659b89 100644 --- a/src/models/apertus.cpp +++ b/src/models/apertus.cpp @@ -54,10 +54,10 @@ void llama_model_apertus::load_arch_tensors(llama_model_loader &) { } std::unique_ptr llama_model_apertus::build_arch_graph(const llm_graph_params & params) const { - return std::make_unique(*this, params); + return std::make_unique(params); } -llama_model_apertus::graph::graph(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) { +llama_model_apertus::graph::graph(const llm_graph_params & params) : llm_graph_context(params) { const int64_t n_embd_head = hparams.n_embd_head_v(); GGML_ASSERT(n_embd_head == hparams.n_embd_head_k()); @@ -84,8 +84,6 @@ llama_model_apertus::graph::graph(const llama_model & model, const llm_graph_par // self-attention { - ggml_tensor * rope_factors = model.get_rope_factors(cparams, il); - // compute Q and K and RoPE them auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur, n_embd_head, n_head, n_head_kv, il); @@ -96,11 +94,8 @@ llama_model_apertus::graph::graph(const llama_model & model, const llm_graph_par Kcur = build_norm(Kcur, model.layers[il].attn_k_norm, NULL, LLM_NORM_RMS, il); cb(Kcur, "Kcur_normed", il); - Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, rope_factors, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); - - Kcur = ggml_rope_ext(ctx0, Kcur, inp_pos, rope_factors, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); + Qcur = build_rope(Qcur, inp_pos, il); + Kcur = build_rope(Kcur, inp_pos, il); cb(Qcur, "Qcur_pos", il); cb(Kcur, "Kcur_pos", il); diff --git a/src/models/arcee.cpp b/src/models/arcee.cpp index 81e3b828f7e2..58607d82e7cf 100644 --- a/src/models/arcee.cpp +++ b/src/models/arcee.cpp @@ -42,10 +42,10 @@ void llama_model_arcee::load_arch_tensors(llama_model_loader &) { } std::unique_ptr llama_model_arcee::build_arch_graph(const llm_graph_params & params) const { - return std::make_unique(*this, params); + return std::make_unique(params); } -llama_model_arcee::graph::graph(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) { +llama_model_arcee::graph::graph(const llm_graph_params & params) : llm_graph_context(params) { const int64_t n_embd_head = hparams.n_embd_head_v(); GGML_ASSERT(n_embd_head == hparams.n_embd_head_k()); @@ -76,24 +76,12 @@ llama_model_arcee::graph::graph(const llama_model & model, const llm_graph_param // self-attention { - // rope freq factors for llama3; may return nullptr for llama2 and other models - ggml_tensor * rope_factors = model.get_rope_factors(cparams, il); - // compute Q and K and RoPE them auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur, n_embd_head, n_head, n_head_kv, il); - Qcur = ggml_rope_ext( - ctx0, Qcur, inp_pos, rope_factors, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); - - Kcur = ggml_rope_ext( - ctx0, Kcur, inp_pos, rope_factors, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); + Qcur = build_rope(Qcur, inp_pos, il); + Kcur = build_rope(Kcur, inp_pos, il); cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); diff --git a/src/models/arctic.cpp b/src/models/arctic.cpp index 1a170f085612..a72f2043b61b 100644 --- a/src/models/arctic.cpp +++ b/src/models/arctic.cpp @@ -50,10 +50,10 @@ void llama_model_arctic::load_arch_tensors(llama_model_loader &) { } std::unique_ptr llama_model_arctic::build_arch_graph(const llm_graph_params & params) const { - return std::make_unique(*this, params); + return std::make_unique(params); } -llama_model_arctic::graph::graph(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) { +llama_model_arctic::graph::graph(const llm_graph_params & params) : llm_graph_context(params) { const int64_t n_embd_head = hparams.n_embd_head_v(); GGML_ASSERT(n_embd_head == hparams.n_embd_head_k()); @@ -86,17 +86,8 @@ llama_model_arctic::graph::graph(const llama_model & model, const llm_graph_para auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur, n_embd_head, n_head, n_head_kv, il); - Qcur = ggml_rope_ext( - ctx0, Qcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); - - Kcur = ggml_rope_ext( - ctx0, Kcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); + Qcur = build_rope(Qcur, inp_pos, il); + Kcur = build_rope(Kcur, inp_pos, il); cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); diff --git a/src/models/arwkv7.cpp b/src/models/arwkv7.cpp index 5f1ca4384d72..022a6738ff1d 100644 --- a/src/models/arwkv7.cpp +++ b/src/models/arwkv7.cpp @@ -114,10 +114,10 @@ void llama_model_arwkv7::load_arch_tensors(llama_model_loader &) { } std::unique_ptr llama_model_arwkv7::build_arch_graph(const llm_graph_params & params) const { - return std::make_unique(*this, params); + return std::make_unique(params); } -llama_model_arwkv7::graph::graph(const llama_model & model, const llm_graph_params & params) : llm_build_rwkv7_base(model, params) { +llama_model_arwkv7::graph::graph(const llm_graph_params & params) : llm_build_rwkv7_base(params) { GGML_ASSERT(n_embd == hparams.n_embd_r()); ggml_tensor * cur; diff --git a/src/models/baichuan.cpp b/src/models/baichuan.cpp index b7f5b05f8b62..3bccc0bffdc9 100644 --- a/src/models/baichuan.cpp +++ b/src/models/baichuan.cpp @@ -12,6 +12,11 @@ void llama_model_baichuan::load_arch_hparams(llama_model_loader & ml) { // TODO: become GGUF KV parameter hparams.f_max_alibi_bias = 8.0f; } + + // only the 7B uses RoPE + if (type != LLM_TYPE_7B) { + std::fill(hparams.rope_pattern.begin(), hparams.rope_pattern.end(), 0); + } } void llama_model_baichuan::load_arch_tensors(llama_model_loader &) { @@ -40,10 +45,10 @@ void llama_model_baichuan::load_arch_tensors(llama_model_loader &) { } std::unique_ptr llama_model_baichuan::build_arch_graph(const llm_graph_params & params) const { - return std::make_unique(*this, params); + return std::make_unique(params); } -llama_model_baichuan::graph::graph(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) { +llama_model_baichuan::graph::graph(const llm_graph_params & params) : llm_graph_context(params) { const int64_t n_embd_head = hparams.n_embd_head_v(); GGML_ASSERT(n_embd_head == hparams.n_embd_head_k()); @@ -74,25 +79,8 @@ llama_model_baichuan::graph::graph(const llama_model & model, const llm_graph_pa auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur, n_embd_head, n_head, n_head_kv, il); - switch (model.type) { - case LLM_TYPE_7B: - Qcur = ggml_rope_ext( - ctx0, Qcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); - Kcur = ggml_rope_ext( - ctx0, Kcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); - break; - case LLM_TYPE_13B: - case LLM_TYPE_UNKNOWN: - break; - default: - GGML_ABORT("fatal error"); - } + Qcur = build_rope(Qcur, inp_pos, il); + Kcur = build_rope(Kcur, inp_pos, il); cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); diff --git a/src/models/bailingmoe.cpp b/src/models/bailingmoe.cpp index 2da47f09fd75..890dcaa7b30c 100644 --- a/src/models/bailingmoe.cpp +++ b/src/models/bailingmoe.cpp @@ -56,10 +56,10 @@ void llama_model_bailingmoe::load_arch_tensors(llama_model_loader &) { } std::unique_ptr llama_model_bailingmoe::build_arch_graph(const llm_graph_params & params) const { - return std::make_unique(*this, params); + return std::make_unique(params); } -llama_model_bailingmoe::graph::graph(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) { +llama_model_bailingmoe::graph::graph(const llm_graph_params & params) : llm_graph_context(params) { ggml_tensor * cur; ggml_tensor * inpL; @@ -83,24 +83,12 @@ llama_model_bailingmoe::graph::graph(const llama_model & model, const llm_graph_ // self-attention { - // rope freq factors for llama3; may return nullptr for llama2 and other models - ggml_tensor * rope_factors = model.get_rope_factors(cparams, il); - // compute Q and K and RoPE them auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur, n_embd_head_k, n_head, n_head_kv, il); - Qcur = ggml_rope_ext( - ctx0, Qcur, inp_pos, rope_factors, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); - - Kcur = ggml_rope_ext( - ctx0, Kcur, inp_pos, rope_factors, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); + Qcur = build_rope(Qcur, inp_pos, il); + Kcur = build_rope(Kcur, inp_pos, il); cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); diff --git a/src/models/bailingmoe2.cpp b/src/models/bailingmoe2.cpp index d7003ef7e519..651623e0e384 100644 --- a/src/models/bailingmoe2.cpp +++ b/src/models/bailingmoe2.cpp @@ -84,10 +84,10 @@ void llama_model_bailingmoe2::load_arch_tensors(llama_model_loader &) { } std::unique_ptr llama_model_bailingmoe2::build_arch_graph(const llm_graph_params & params) const { - return std::make_unique(*this, params); + return std::make_unique(params); } -llama_model_bailingmoe2::graph::graph(const llama_model & model, const llm_graph_params & params) : +llama_model_bailingmoe2::graph::graph(const llm_graph_params & params) : llm_graph_context(params) { const int64_t n_embd_head = hparams.n_embd_head_v(); @@ -120,14 +120,12 @@ llama_model_bailingmoe2::graph::graph(const llama_model & model, const llm_graph Qcur = build_norm(Qcur, model.layers[il].attn_q_norm, NULL, LLM_NORM_RMS, il); cb(Qcur, "Qcur_normed", il); - Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); + Qcur = build_rope(Qcur, inp_pos, il); Kcur = build_norm(Kcur, model.layers[il].attn_k_norm, NULL, LLM_NORM_RMS, il); cb(Kcur, "Kcur_normed", il); - Kcur = ggml_rope_ext(ctx0, Kcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); + Kcur = build_rope(Kcur, inp_pos, il); cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); diff --git a/src/models/bailingmoe3.cpp b/src/models/bailingmoe3.cpp index 49c0f7feaccf..ad1f0ee27b1f 100644 --- a/src/models/bailingmoe3.cpp +++ b/src/models/bailingmoe3.cpp @@ -155,29 +155,15 @@ void llama_model_bailingmoe3::load_arch_tensors(llama_model_loader & ml) { std::unique_ptr llama_model_bailingmoe3::build_arch_graph(const llm_graph_params & params) const { if (params.gtype == LLM_GRAPH_TYPE_DECODER_MTP) { - return std::make_unique(*this, params); + return std::make_unique(params); } - return std::make_unique(*this, params); + return std::make_unique(params); } -static ggml_tensor * bailingmoe3_causal_conv1d( - ggml_cgraph * gf, - ggml_context * ctx0, - ggml_tensor * conv_states_all, - ggml_tensor * conv_state_all, - int64_t qkv, - ggml_tensor * x, - ggml_tensor * proj_w, - ggml_tensor * conv_w, - int64_t d_conv, - int64_t head_dim, - int64_t n_head, - int64_t n_seq_tokens, - int64_t n_seqs, - int64_t n_tokens, - int64_t cache_head, - uint32_t mem_size, - uint32_t n_rs_seq) { +ggml_tensor * llama_model_bailingmoe3::graph::build_causal_conv1d( + llm_graph_input_rs * inp_rs, ggml_tensor * conv_states_all, ggml_tensor * conv_state_all, + int64_t qkv, ggml_tensor * x, ggml_tensor * proj_w, ggml_tensor * conv_w, + int64_t d_conv, int64_t head_dim, int64_t n_head, int64_t n_seq_tokens, int64_t n_seqs) { const int64_t d_inner = head_dim * n_head; const int64_t conv_state_size = (d_conv - 1) * d_inner; const int64_t total_state_size = 3 * conv_state_size; @@ -191,18 +177,8 @@ static ggml_tensor * bailingmoe3_causal_conv1d( x_proj = ggml_reshape_3d(ctx0, x_proj, d_inner, n_seq_tokens, n_seqs); ggml_tensor * conv_x = ggml_concat(ctx0, conv_state, ggml_transpose(ctx0, x_proj), 0); - const int64_t K = (int64_t) n_rs_seq + 1; - const int64_t n_written = std::min(n_seq_tokens, K); - - for (int64_t slot = 0; slot < n_written; ++slot) { - ggml_tensor * conv_snap = ggml_view_3d(ctx0, conv_x, d_conv - 1, d_inner, n_seqs, - conv_x->nb[1], conv_x->nb[2], (conv_x->ne[0] - (d_conv - 1) - slot) * conv_x->nb[0]); - ggml_build_forward_expand(gf, ggml_cpy(ctx0, conv_snap, - ggml_view_3d(ctx0, conv_states_all, d_conv - 1, d_inner, n_seqs, - (d_conv - 1) * ggml_element_size(conv_states_all), - total_state_size * ggml_element_size(conv_states_all), - ((slot * mem_size + cache_head) * total_state_size + qkv * conv_state_size) * ggml_element_size(conv_states_all)))); - } + build_conv_state_store(inp_rs, conv_states_all, conv_x, d_conv - 1, + ggml_row_size(conv_states_all->type, qkv * conv_state_size)); ggml_tensor * conv_weight = ggml_reshape_2d(ctx0, conv_w, d_conv, d_inner); ggml_tensor * out = ggml_ssm_conv(ctx0, conv_x, conv_weight); @@ -210,8 +186,8 @@ static ggml_tensor * bailingmoe3_causal_conv1d( return ggml_reshape_4d(ctx0, out, head_dim, n_head, n_seq_tokens, n_seqs); } -llama_model_bailingmoe3::graph::graph(const llama_model & model, const llm_graph_params & params) : - llm_build_delta_net_base(params), model(model) { +llama_model_bailingmoe3::graph::graph(const llm_graph_params & params) : + llm_build_delta_net_base(params) { ggml_tensor * inpL = build_inp_embd(model.tok_embd); cb(inpL, "model.input_embed", -1); @@ -229,9 +205,6 @@ llama_model_bailingmoe3::graph::graph(const llama_model & model, const llm_graph const int64_t n_seqs = ubatch.n_seqs; const int64_t n_seq_tokens = ubatch.n_seq_tokens; - const bool use_mrope = hparams.use_mrope(); - int sections[4]; - std::copy(std::begin(hparams.rope_sections), std::begin(hparams.rope_sections) + 4, sections); const int64_t qk_head_dim = hparams.n_embd_head_k_mla(); const int64_t v_head_dim = hparams.n_embd_head_v_mla(); const int64_t qk_rope_head_dim = hparams.n_rot(); @@ -253,20 +226,18 @@ llama_model_bailingmoe3::graph::graph(const llama_model & model, const llm_graph if (hparams.is_recr(il)) { const auto * mctx_cur = inp_rs->mctx; - const auto cache_head = mctx_cur->get_head(); - const auto mem_size = mctx_cur->get_size(); ggml_tensor * conv_states_all = mctx_cur->get_r_l(il); ggml_tensor * conv_state_all = build_rs(inp_rs, conv_states_all, hparams.n_embd_r(), n_seqs); - ggml_tensor * q = bailingmoe3_causal_conv1d( - gf, ctx0, conv_states_all, conv_state_all, 0, cur, layer.wq, layer.ssm_q_conv, - d_conv, head_dim, n_head, n_seq_tokens, n_seqs, n_tokens, cache_head, mem_size, cparams.n_rs_seq); - ggml_tensor * k = bailingmoe3_causal_conv1d( - gf, ctx0, conv_states_all, conv_state_all, 1, cur, layer.wk, layer.ssm_k_conv, - d_conv, head_dim, n_head, n_seq_tokens, n_seqs, n_tokens, cache_head, mem_size, cparams.n_rs_seq); - ggml_tensor * v = bailingmoe3_causal_conv1d( - gf, ctx0, conv_states_all, conv_state_all, 2, cur, layer.wv, layer.ssm_v_conv, - d_conv, head_dim, n_head, n_seq_tokens, n_seqs, n_tokens, cache_head, mem_size, cparams.n_rs_seq); + ggml_tensor * q = build_causal_conv1d( + inp_rs, conv_states_all, conv_state_all, 0, cur, layer.wq, layer.ssm_q_conv, + d_conv, head_dim, n_head, n_seq_tokens, n_seqs); + ggml_tensor * k = build_causal_conv1d( + inp_rs, conv_states_all, conv_state_all, 1, cur, layer.wk, layer.ssm_k_conv, + d_conv, head_dim, n_head, n_seq_tokens, n_seqs); + ggml_tensor * v = build_causal_conv1d( + inp_rs, conv_states_all, conv_state_all, 2, cur, layer.wv, layer.ssm_v_conv, + d_conv, head_dim, n_head, n_seq_tokens, n_seqs); ggml_tensor * gate = ggml_mul_mat(ctx0, layer.ssm_f_a, cur); gate = ggml_add(ctx0, gate, layer.ssm_dt_b); @@ -325,17 +296,8 @@ llama_model_bailingmoe3::graph::graph(const llama_model & model, const llm_graph ggml_row_size(kv_all->type, kv_lora_rank + qk_rope_head_dim), ggml_row_size(kv_all->type, kv_lora_rank)); - if (use_mrope) { - q_pe = ggml_rope_multi(ctx0, q_pe, inp_pos, nullptr, n_rot, sections, rope_type, - n_ctx_orig, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow); - k_pe = ggml_rope_multi(ctx0, k_pe, inp_pos, nullptr, n_rot, sections, rope_type, - n_ctx_orig, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow); - } else { - q_pe = ggml_rope_ext(ctx0, q_pe, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); - k_pe = ggml_rope_ext(ctx0, k_pe, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); - } + q_pe = build_rope(q_pe, inp_pos, il); + k_pe = build_rope(k_pe, inp_pos, il); kv = build_norm(kv, layer.attn_kv_a_norm, nullptr, LLM_NORM_RMS, il); q_nope = ggml_permute(ctx0, q_nope, 0, 2, 1, 3); @@ -414,7 +376,7 @@ llama_model_bailingmoe3::graph::graph(const llama_model & model, const llm_graph res->t_logits = cur; } -llama_model_bailingmoe3::graph_mtp::graph_mtp(const llama_model & model, const llm_graph_params & params) : +llama_model_bailingmoe3::graph_mtp::graph_mtp(const llm_graph_params & params) : llm_graph_context(params) { GGML_ASSERT(hparams.n_layer_nextn == 1 && "BailingMoE3 MTP requires one NextN layer"); @@ -442,9 +404,7 @@ llama_model_bailingmoe3::graph_mtp::graph_mtp(const llama_model & model, const l ggml_set_input(inp->tokens); inp->embd = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, hparams.n_embd_inp(), n_tokens); ggml_set_input(inp->embd); - inp->h = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, hparams.n_embd, n_tokens); - ggml_set_input(inp->h); - ggml_set_name(inp->h, "mtp_h_input"); + inp->h = build_inp_embd_state(inp.get(), hparams.n_embd); ggml_tensor * tok_embd = ubatch.token ? ggml_get_rows(ctx0, model.tok_embd, inp->tokens) : inp->embd; ggml_tensor * h_norm = build_norm(inp->h, layer.nextn.hnorm, nullptr, LLM_NORM_RMS, il); @@ -489,21 +449,8 @@ llama_model_bailingmoe3::graph_mtp::graph_mtp(const llama_model & model, const l ggml_row_size(kv_all->type, kv_lora_rank + qk_rope_head_dim), ggml_row_size(kv_all->type, kv_lora_rank)); - const bool use_mrope = hparams.use_mrope(); - int sections[4]; - std::copy(std::begin(hparams.rope_sections), std::begin(hparams.rope_sections) + 4, sections); - - if (use_mrope) { - q_pe = ggml_rope_multi(ctx0, q_pe, inp_pos, nullptr, n_rot, sections, rope_type, - n_ctx_orig, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow); - k_pe = ggml_rope_multi(ctx0, k_pe, inp_pos, nullptr, n_rot, sections, rope_type, - n_ctx_orig, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow); - } else { - q_pe = ggml_rope_ext(ctx0, q_pe, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); - k_pe = ggml_rope_ext(ctx0, k_pe, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); - } + q_pe = build_rope(q_pe, inp_pos, il); + k_pe = build_rope(k_pe, inp_pos, il); kv = build_norm(kv, layer.attn_kv_a_norm, nullptr, LLM_NORM_RMS, il); q_nope = ggml_permute(ctx0, q_nope, 0, 2, 1, 3); diff --git a/src/models/beam.cpp b/src/models/beam.cpp index 55817dd4839a..8654e0c95a4f 100644 --- a/src/models/beam.cpp +++ b/src/models/beam.cpp @@ -103,7 +103,7 @@ void llama_model_beam::load_arch_tensors(llama_model_loader &) { } std::unique_ptr llama_model_beam::build_arch_graph(const llm_graph_params & params) const { - return std::make_unique(*this, params); + return std::make_unique(params); } static ggml_tensor * beam_residual_add(ggml_context * ctx, ggml_tensor * h, ggml_tensor * r, int i) { @@ -112,7 +112,7 @@ static ggml_tensor * beam_residual_add(ggml_context * ctx, ggml_tensor * h, ggml return ggml_add(ctx, ggml_scale(ctx, h, h_scale), ggml_scale(ctx, r, r_scale)); } -llama_model_beam::graph::graph(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) { +llama_model_beam::graph::graph(const llm_graph_params & params) : llm_graph_context(params) { const int64_t n_embd_head = hparams.n_embd_head_v(); GGML_ASSERT(n_embd_head == hparams.n_embd_head_k()); @@ -156,15 +156,9 @@ llama_model_beam::graph::graph(const llama_model & model, const llm_graph_params cb(Kcur, "Kcur_normed", il); if (is_swa) { - const float freq_base_l = model.get_rope_freq_base (cparams, il); - const float freq_scale_l = model.get_rope_freq_scale(cparams, il); - - Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base_l, freq_scale_l, - ext_factor, attn_factor, beta_fast, beta_slow); - Kcur = ggml_rope_ext(ctx0, Kcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base_l, freq_scale_l, - ext_factor, attn_factor, beta_fast, beta_slow); + + Qcur = build_rope(Qcur, inp_pos, il); + Kcur = build_rope(Kcur, inp_pos, il); cb(Qcur, "Qcur_rope", il); cb(Kcur, "Kcur_rope", il); } else { diff --git a/src/models/bert.cpp b/src/models/bert.cpp index 8f3aa316c73a..b98faffc90fd 100644 --- a/src/models/bert.cpp +++ b/src/models/bert.cpp @@ -62,10 +62,10 @@ void llama_model_bert::load_arch_tensors(llama_model_loader &) { } std::unique_ptr llama_model_bert::build_arch_graph(const llm_graph_params & params) const { - return std::make_unique(*this, params); + return std::make_unique(params); } -llama_model_bert::graph::graph(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) { +llama_model_bert::graph::graph(const llm_graph_params & params) : llm_graph_context(params) { const int64_t n_embd_head = hparams.n_embd_head_v(); GGML_ASSERT(n_embd_head == hparams.n_embd_head_k()); @@ -131,11 +131,8 @@ llama_model_bert::graph::graph(const llama_model & model, const llm_graph_params // RoPE if (model.arch == LLM_ARCH_NOMIC_BERT || model.arch == LLM_ARCH_NOMIC_BERT_MOE || model.arch == LLM_ARCH_JINA_BERT_V3) { - Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); - - Kcur = ggml_rope_ext(ctx0, Kcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); + Qcur = build_rope(Qcur, inp_pos, il); + Kcur = build_rope(Kcur, inp_pos, il); } cb(Qcur, "Qcur", il); diff --git a/src/models/bitnet.cpp b/src/models/bitnet.cpp index a08881b9f6b7..caee27a3ddae 100644 --- a/src/models/bitnet.cpp +++ b/src/models/bitnet.cpp @@ -45,10 +45,10 @@ void llama_model_bitnet::load_arch_tensors(llama_model_loader &) { } std::unique_ptr llama_model_bitnet::build_arch_graph(const llm_graph_params & params) const { - return std::make_unique(*this, params); + return std::make_unique(params); } -llama_model_bitnet::graph::graph(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) { +llama_model_bitnet::graph::graph(const llm_graph_params & params) : llm_graph_context(params) { const int64_t n_embd_head = hparams.n_embd_head_v(); GGML_ASSERT(n_embd_head == hparams.n_embd_head_k()); @@ -78,17 +78,8 @@ llama_model_bitnet::graph::graph(const llama_model & model, const llm_graph_para auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur, n_embd_head, n_head, n_head_kv, il); - Qcur = ggml_rope_ext( - ctx0, Qcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); - - Kcur = ggml_rope_ext( - ctx0, Kcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); + Qcur = build_rope(Qcur, inp_pos, il); + Kcur = build_rope(Kcur, inp_pos, il); cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); diff --git a/src/models/bloom.cpp b/src/models/bloom.cpp index 54ef3a03759f..076927e0d513 100644 --- a/src/models/bloom.cpp +++ b/src/models/bloom.cpp @@ -59,10 +59,10 @@ void llama_model_bloom::load_arch_tensors(llama_model_loader &) { } std::unique_ptr llama_model_bloom::build_arch_graph(const llm_graph_params & params) const { - return std::make_unique(*this, params); + return std::make_unique(params); } -llama_model_bloom::graph::graph(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) { +llama_model_bloom::graph::graph(const llm_graph_params & params) : llm_graph_context(params) { const int64_t n_embd_head = hparams.n_embd_head_v(); GGML_ASSERT(n_embd_head == hparams.n_embd_head_k()); diff --git a/src/models/chameleon.cpp b/src/models/chameleon.cpp index bb597afef20f..ab16927c7e66 100644 --- a/src/models/chameleon.cpp +++ b/src/models/chameleon.cpp @@ -47,10 +47,10 @@ void llama_model_chameleon::load_arch_tensors(llama_model_loader &) { } std::unique_ptr llama_model_chameleon::build_arch_graph(const llm_graph_params & params) const { - return std::make_unique(*this, params); + return std::make_unique(params); } -llama_model_chameleon::graph::graph(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) { +llama_model_chameleon::graph::graph(const llm_graph_params & params) : llm_graph_context(params) { const int64_t n_embd_head = hparams.n_embd_head_v(); GGML_ASSERT(n_embd_head == hparams.n_embd_head_k()); @@ -103,17 +103,8 @@ llama_model_chameleon::graph::graph(const llama_model & model, const llm_graph_p cb(Kcur, "Kcur", il); } - Qcur = ggml_rope_ext( - ctx0, Qcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); - - Kcur = ggml_rope_ext( - ctx0, Kcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); + Qcur = build_rope(Qcur, inp_pos, il); + Kcur = build_rope(Kcur, inp_pos, il); cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); diff --git a/src/models/chatglm.cpp b/src/models/chatglm.cpp index 54e3913c4a2d..e7442b75f981 100644 --- a/src/models/chatglm.cpp +++ b/src/models/chatglm.cpp @@ -52,10 +52,10 @@ void llama_model_chatglm::load_arch_tensors(llama_model_loader &) { } std::unique_ptr llama_model_chatglm::build_arch_graph(const llm_graph_params & params) const { - return std::make_unique(*this, params); + return std::make_unique(params); } -llama_model_chatglm::graph::graph(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) { +llama_model_chatglm::graph::graph(const llm_graph_params & params) : llm_graph_context(params) { const int64_t n_embd_head = hparams.n_embd_head_v(); GGML_ASSERT(n_embd_head == hparams.n_embd_head_k()); @@ -87,17 +87,8 @@ llama_model_chatglm::graph::graph(const llama_model & model, const llm_graph_par n_embd_head, n_head, n_head_kv, il); //printf("freq_base: %f freq_scale: %f ext_factor: %f attn_factor: %f\n", freq_base, freq_scale, ext_factor, attn_factor); - Qcur = ggml_rope_ext( - ctx0, Qcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); - - Kcur = ggml_rope_ext( - ctx0, Kcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); + Qcur = build_rope(Qcur, inp_pos, il); + Kcur = build_rope(Kcur, inp_pos, il); cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); diff --git a/src/models/clef.cpp b/src/models/clef.cpp index e9be0ec141af..86fed4e874a8 100644 --- a/src/models/clef.cpp +++ b/src/models/clef.cpp @@ -99,7 +99,7 @@ void llama_model_clef::load_arch_tensors(llama_model_loader & ml) { } std::unique_ptr llama_model_clef::build_arch_graph(const llm_graph_params & params) const { - return std::make_unique(*this, params); + return std::make_unique(params); } // spans read by the head, [start, end) in ubatch token indices @@ -243,15 +243,12 @@ class llama_model_clef::input_decision : public llm_graph_input_i { }; // the backbone is copied from llama_model_qwen35::graph, without the memory module -llama_model_clef::graph::graph(const llama_model & model_base, const llm_graph_params & params) : - llm_build_delta_net_base(params), model(static_cast(model_base)) { +llama_model_clef::graph::graph(const llm_graph_params & params) : + llm_build_delta_net_base(params), model(static_cast(*params.model)) { const int64_t n_embd_head = hparams.n_embd_head_v(); GGML_ASSERT(n_embd_head == hparams.n_embd_head_k()); - int sections[4]; - std::copy(std::begin(hparams.rope_sections), std::begin(hparams.rope_sections) + 4, sections); - ggml_tensor * cur; ggml_tensor * inpL; @@ -274,7 +271,7 @@ llama_model_clef::graph::graph(const llama_model & model_base, const llm_graph_p if (hparams.is_recr(il)) { cur = build_layer_attn_linear(cur, il); } else { - cur = build_layer_attn(inp_attn, cur, inp_pos, sections, il); + cur = build_layer_attn(inp_attn, cur, inp_pos, il); } cur = ggml_add(ctx0, cur, inpSA); @@ -352,8 +349,6 @@ llm_graph_input_attn_no_cache * llama_model_clef::graph::build_attn_inp_causal() ggml_set_input(inp->self_attn_mask.raw); cb(inp->self_attn_mask.raw, "self_kq_mask", -1); - inp->self_attn_mask.cnv = inp->self_attn_mask.raw; - return (llm_graph_input_attn_no_cache *) res->add_input(std::move(inp)); } @@ -361,7 +356,6 @@ ggml_tensor * llama_model_clef::graph::build_layer_attn( llm_graph_input_attn_no_cache * inp, ggml_tensor * cur, ggml_tensor * inp_pos, - int * sections, int il) { const int64_t n_embd_head = hparams.n_embd_head_v(); @@ -391,17 +385,8 @@ ggml_tensor * llama_model_clef::graph::build_layer_attn( Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens); - Qcur = ggml_rope_multi( - ctx0, Qcur, inp_pos, nullptr, - n_rot, sections, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); - - Kcur = ggml_rope_multi( - ctx0, Kcur, inp_pos, nullptr, - n_rot, sections, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); + Qcur = build_rope(Qcur, inp_pos, il); + Kcur = build_rope(Kcur, inp_pos, il); const float kq_scale = hparams.f_attention_scale == 0.0f ? 1.0f / sqrtf(float(n_embd_head)) : hparams.f_attention_scale; @@ -472,16 +457,9 @@ ggml_tensor * llama_model_clef::graph::build_layer_attn_linear( q_conv = build_gdn_l2_norm(ctx0, q_conv, hparams.f_norm_rms_eps); k_conv = build_gdn_l2_norm(ctx0, k_conv, hparams.f_norm_rms_eps); - // note: need explicit repeat only if we are not using the fused GDN - if (num_k_heads != num_v_heads && (!cparams.fused_gdn_ar || !cparams.fused_gdn_ch)) { - GGML_ASSERT(num_v_heads % num_k_heads == 0); - q_conv = ggml_repeat_4d(ctx0, q_conv, head_k_dim, num_v_heads, n_tokens, n_seqs); - k_conv = ggml_repeat_4d(ctx0, k_conv, head_k_dim, num_v_heads, n_tokens, n_seqs); - } - ggml_tensor * state = ggml_fill(ctx0, ggml_new_tensor_4d(ctx0, GGML_TYPE_F32, head_v_dim, head_v_dim, num_v_heads, n_seqs), 0.0f); - ggml_tensor * output = build_delta_net(q_conv, k_conv, v_conv, gate, beta, state, il).first; + ggml_tensor * output = build_delta_net(q_conv, k_conv, v_conv, gate, beta, state).first; cb(output, "attn_output", il); // gated normalization diff --git a/src/models/clip.cpp b/src/models/clip.cpp index 537766aeb15b..3bf7ac3b83ef 100644 --- a/src/models/clip.cpp +++ b/src/models/clip.cpp @@ -14,5 +14,5 @@ void llama_model_clip::load_arch_tensors(llama_model_loader &) { [[noreturn]] std::unique_ptr llama_model_clip::build_arch_graph(const llm_graph_params &) const { - GGML_ABORT("CLIP has no inference graph via llama_model dispatch; runtime lives in tools/mtmd/clip.cpp"); + GGML_ABORT("CLIP has no inference graph via llama_model dispatch; runtime lives in tools/mtmd"); } diff --git a/src/models/clover-moe.cpp b/src/models/clover-moe.cpp index e7450790a12f..8892521eee35 100644 --- a/src/models/clover-moe.cpp +++ b/src/models/clover-moe.cpp @@ -113,11 +113,11 @@ void llama_model_clover_moe::load_arch_tensors(llama_model_loader &) { } std::unique_ptr llama_model_clover_moe::build_arch_graph(const llm_graph_params & params) const { - return std::make_unique(*this, params); + return std::make_unique(params); } -llama_model_clover_moe::graph::graph(const llama_model & model, const llm_graph_params & params) : - llm_build_delta_net_base(params), model(model) { +llama_model_clover_moe::graph::graph(const llm_graph_params & params) : + llm_build_delta_net_base(params) { const int64_t n_embd_head = hparams.n_embd_head_v(); GGML_ASSERT(n_embd_head == hparams.n_embd_head_k()); @@ -252,13 +252,8 @@ ggml_tensor * llama_model_clover_moe::graph::build_layer_attn( Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens); - Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); - - Kcur = ggml_rope_ext(ctx0, Kcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); + Qcur = build_rope(Qcur, inp_pos, il); + Kcur = build_rope(Kcur, inp_pos, il); cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); @@ -374,11 +369,6 @@ ggml_tensor * llama_model_clover_moe::graph::build_layer_attn_linear( q_conv = build_gdn_l2_norm(ctx0, q_conv, 1e-6f); k_conv = build_gdn_l2_norm(ctx0, k_conv, 1e-6f); - if (num_k_heads != num_v_heads && (!cparams.fused_gdn_ar || !cparams.fused_gdn_ch)) { - q_conv = ggml_repeat_4d(ctx0, q_conv, head_k_dim, num_v_heads, n_seq_tokens, n_seqs); - k_conv = ggml_repeat_4d(ctx0, k_conv, head_k_dim, num_v_heads, n_seq_tokens, n_seqs); - } - ggml_tensor * output = build_recurrent_attn(inp, ssm_states_all, q_conv, k_conv, v_conv, gate, beta, state, il); ggml_tensor * z_2d = ggml_reshape_4d(ctx0, z, head_v_dim, num_v_heads, n_seq_tokens, n_seqs); diff --git a/src/models/codeshell.cpp b/src/models/codeshell.cpp index be1e576197bc..e316602f5a21 100644 --- a/src/models/codeshell.cpp +++ b/src/models/codeshell.cpp @@ -47,10 +47,10 @@ void llama_model_codeshell::load_arch_tensors(llama_model_loader &) { } std::unique_ptr llama_model_codeshell::build_arch_graph(const llm_graph_params & params) const { - return std::make_unique(*this, params); + return std::make_unique(params); } -llama_model_codeshell::graph::graph(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) { +llama_model_codeshell::graph::graph(const llm_graph_params & params) : llm_graph_context(params) { const int64_t n_embd_head = hparams.n_embd_head_v(); GGML_ASSERT(n_embd_head == hparams.n_embd_head_k()); @@ -80,17 +80,8 @@ llama_model_codeshell::graph::graph(const llama_model & model, const llm_graph_p auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur, n_embd_head, n_head, n_head_kv, il); - Qcur = ggml_rope_ext( - ctx0, Qcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); - - Kcur = ggml_rope_ext( - ctx0, Kcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); + Qcur = build_rope(Qcur, inp_pos, il); + Kcur = build_rope(Kcur, inp_pos, il); cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); diff --git a/src/models/cogvlm.cpp b/src/models/cogvlm.cpp index 2e1416be8987..657b7d89d1c8 100644 --- a/src/models/cogvlm.cpp +++ b/src/models/cogvlm.cpp @@ -46,11 +46,27 @@ void llama_model_cogvlm::load_arch_tensors(llama_model_loader &) { } } +llama_rope_params llama_model_cogvlm::get_rope_params(const llama_cparams & cparams, int il) const { + auto res = llama_model::get_rope_params(cparams, il); + + // plain ggml_rope + res.n_ctx_orig = 0; + res.freq_base = 10000.0f; + res.freq_scale = 1.0f; + res.ext_factor = 0.0f; + res.attn_factor = 1.0f; + res.beta_fast = 0.0f; + res.beta_slow = 0.0f; + res.factors = nullptr; + + return res; +} + std::unique_ptr llama_model_cogvlm::build_arch_graph(const llm_graph_params & params) const { - return std::make_unique(*this, params); + return std::make_unique(params); } -llama_model_cogvlm::graph::graph(const llama_model & model, const llm_graph_params & params) : +llama_model_cogvlm::graph::graph(const llm_graph_params & params) : llm_graph_context(params) { const int64_t n_embd_head = hparams.n_embd_head_v(); const float kq_scale = 1.0f / sqrtf(float(n_embd_head)); @@ -113,8 +129,8 @@ llama_model_cogvlm::graph::graph(const llama_model & model, const llm_graph_para ggml_tensor * Vcur = ggml_view_3d(ctx0, qkv, n_embd_head, n_head_kv, n_tokens, n_embd_head * sizeof(float), qkv->nb[1], 2 * n_embd * ggml_element_size(qkv)); - Qcur = ggml_rope(ctx0, Qcur, inp_pos, n_embd_head, rope_type); - Kcur = ggml_rope(ctx0, Kcur, inp_pos, n_embd_head, rope_type); + Qcur = build_rope(Qcur, inp_pos, il); + Kcur = build_rope(Kcur, inp_pos, il); cur = build_attn(inp_attn, wo, nullptr, wo_s, diff --git a/src/models/cohere2.cpp b/src/models/cohere2.cpp index 969b4ef05a8e..251b2da62d05 100644 --- a/src/models/cohere2.cpp +++ b/src/models/cohere2.cpp @@ -3,6 +3,7 @@ void llama_model_cohere2::load_arch_hparams(llama_model_loader & ml) { hparams.swa_type = LLAMA_SWA_TYPE_STANDARD; load_swa_pattern(ml, 4); + hparams.rope_pattern = hparams.is_swa_impl; hparams.rope_freq_base_train_swa = hparams.rope_freq_base_train; hparams.rope_freq_scale_train_swa = hparams.rope_freq_scale_train; @@ -43,11 +44,21 @@ void llama_model_cohere2::load_arch_tensors(llama_model_loader &) { } } +llama_rope_params llama_model_cohere2::get_rope_params(const llama_cparams & cparams, int il) const { + auto res = llama_model::get_rope_params(cparams, il); + + // the SWA layers use the global base and scale + res.freq_base = cparams.rope_freq_base; + res.freq_scale = cparams.rope_freq_scale; + + return res; +} + std::unique_ptr llama_model_cohere2::build_arch_graph(const llm_graph_params & params) const { - return std::make_unique(*this, params); + return std::make_unique(params); } -llama_model_cohere2::graph::graph(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) { +llama_model_cohere2::graph::graph(const llm_graph_params & params) : llm_graph_context(params) { const int64_t n_embd_head = hparams.n_embd_head_v(); GGML_ASSERT(n_embd_head == hparams.n_embd_head_k()); @@ -67,10 +78,7 @@ llama_model_cohere2::graph::graph(const llama_model & model, const llm_graph_par ggml_tensor * inp_out_ids = build_inp_out_ids(); for (int il = 0; il < n_layer; ++il) { - const bool is_swa = hparams.is_swa(il); // UNUSED: - // const float freq_base_l = model.get_rope_freq_base (cparams, il); - // const float freq_scale_l = model.get_rope_freq_scale(cparams, il); // norm cur = build_norm(inpL, model.layers[il].attn_norm, NULL, LLM_NORM, il); @@ -79,26 +87,12 @@ llama_model_cohere2::graph::graph(const llama_model & model, const llm_graph_par // self-attention { - // rope freq factors for 128k context - ggml_tensor * rope_factors = model.get_rope_factors(cparams, il); - // compute Q and K and RoPE them auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur, n_embd_head, n_head, n_head_kv, il); - if (is_swa) { - Qcur = ggml_rope_ext( - ctx0, Qcur, inp_pos, rope_factors, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); - - Kcur = ggml_rope_ext( - ctx0, Kcur, inp_pos, rope_factors, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); - } + Qcur = build_rope(Qcur, inp_pos, il); + Kcur = build_rope(Kcur, inp_pos, il); cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); diff --git a/src/models/cohere2moe.cpp b/src/models/cohere2moe.cpp index 649682610eaa..52b959672dfa 100644 --- a/src/models/cohere2moe.cpp +++ b/src/models/cohere2moe.cpp @@ -27,6 +27,11 @@ void llama_model_cohere2moe::load_arch_hparams(llama_model_loader & ml) { hparams.swa_type = LLAMA_SWA_TYPE_STANDARD; load_swa_pattern(ml, 4, true); + // the dense prefix and the MTP layers use RoPE even if they are not SWA + for (uint32_t il = 0; il < hparams.n_layer_all; ++il) { + hparams.rope_pattern[il] = hparams.is_swa(il) || il < hparams.n_layer_dense_lead || il >= hparams.n_layer(); + } + hparams.rope_freq_base_train_swa = hparams.rope_freq_base_train; hparams.rope_freq_scale_train_swa = hparams.rope_freq_scale_train; ml.get_key(LLM_KV_ROPE_FREQ_BASE_SWA, hparams.rope_freq_base_train_swa, false); @@ -133,14 +138,24 @@ void llama_model_cohere2moe::load_arch_tensors(llama_model_loader & ml) { } } +llama_rope_params llama_model_cohere2moe::get_rope_params(const llama_cparams & cparams, int il) const { + auto res = llama_model::get_rope_params(cparams, il); + + // the SWA layers use the global base and scale + res.freq_base = cparams.rope_freq_base; + res.freq_scale = cparams.rope_freq_scale; + + return res; +} + std::unique_ptr llama_model_cohere2moe::build_arch_graph(const llm_graph_params & params) const { if (params.gtype == LLM_GRAPH_TYPE_DECODER_MTP) { - return std::make_unique(*this, params); + return std::make_unique(params); } - return std::make_unique(*this, params); + return std::make_unique(params); } -llama_model_cohere2moe::graph::graph(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) { +llama_model_cohere2moe::graph::graph(const llm_graph_params & params) : llm_graph_context(params) { const int64_t n_embd_head = hparams.n_embd_head_v(); GGML_ASSERT(n_embd_head == hparams.n_embd_head_k()); @@ -157,9 +172,6 @@ llama_model_cohere2moe::graph::graph(const llama_model & model, const llm_graph_ // MTP/NextN layers are loaded as extra decoder blocks but not executed in the main pass. for (int il = 0; il < n_layer; ++il) { - const bool is_swa = hparams.is_swa(il); - // Dense-prefix full-attention layers use RoPE; later layers follow the SWA pattern. - const bool force_rope = static_cast(il) < hparams.n_layer_dense_lead; cur = build_norm(inpL, model.layers[il].attn_norm, nullptr, cohere2moe_norm_type, il); cb(cur, "attn_norm", il); @@ -172,19 +184,8 @@ llama_model_cohere2moe::graph::graph(const llama_model & model, const llm_graph_ auto [Qcur, Kcur, Vcur] = build_qkv(layer, cur, n_embd_head, n_head, n_head_kv, il); - if (is_swa || force_rope) { - ggml_tensor * rope_factors = model.get_rope_factors(cparams, il); - - Qcur = ggml_rope_ext( - ctx0, Qcur, inp_pos, rope_factors, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); - - Kcur = ggml_rope_ext( - ctx0, Kcur, inp_pos, rope_factors, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); - } + Qcur = build_rope(Qcur, inp_pos, il); + Kcur = build_rope(Kcur, inp_pos, il); cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); @@ -277,7 +278,7 @@ llama_model_cohere2moe::graph::graph(const llama_model & model, const llm_graph_ res->t_logits = cur; } -llama_model_cohere2moe::graph_mtp::graph_mtp(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) { +llama_model_cohere2moe::graph_mtp::graph_mtp(const llm_graph_params & params) : llm_graph_context(params) { GGML_ASSERT(hparams.n_layer_nextn > 0 && "COHERE2MOE MTP requires n_layer_nextn > 0"); GGML_ASSERT(hparams.n_layer_nextn == 1 && "COHERE2MOE MTP currently only supports a single MTP block"); @@ -315,9 +316,7 @@ llama_model_cohere2moe::graph_mtp::graph_mtp(const llama_model & model, const ll } cb(tok_embd, "mtp_tok_embd", il); - inp->h = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, hparams.n_embd, n_tokens); - ggml_set_input(inp->h); - ggml_set_name(inp->h, "mtp_h_input"); + inp->h = build_inp_embd_state(inp.get(), hparams.n_embd); ggml_tensor * h_embd = inp->h; @@ -346,15 +345,8 @@ llama_model_cohere2moe::graph_mtp::graph_mtp(const llama_model & model, const ll ggml_tensor * ffn_inp = cur; auto [Qcur, Kcur, Vcur] = build_qkv(layer, cur, n_embd_head, n_head, n_head_kv, il); - ggml_tensor * rope_factors = model.get_rope_factors(cparams, il); - Qcur = ggml_rope_ext( - ctx0, Qcur, inp_pos, rope_factors, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); - Kcur = ggml_rope_ext( - ctx0, Kcur, inp_pos, rope_factors, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); + Qcur = build_rope(Qcur, inp_pos, il); + Kcur = build_rope(Kcur, inp_pos, il); cb(Qcur, "mtp_Qcur", il); cb(Kcur, "mtp_Kcur", il); diff --git a/src/models/command-r.cpp b/src/models/command-r.cpp index edf74c57b3fc..bea97e6d81d8 100644 --- a/src/models/command-r.cpp +++ b/src/models/command-r.cpp @@ -40,10 +40,10 @@ void llama_model_command_r::load_arch_tensors(llama_model_loader &) { } std::unique_ptr llama_model_command_r::build_arch_graph(const llm_graph_params & params) const { - return std::make_unique(*this, params); + return std::make_unique(params); } -llama_model_command_r::graph::graph(const llama_model & model, const llm_graph_params & params) : +llama_model_command_r::graph::graph(const llm_graph_params & params) : llm_graph_context(params) { const int64_t n_embd_head = hparams.n_embd_head_v(); @@ -80,15 +80,13 @@ llama_model_command_r::graph::graph(const llama_model & model, const llm_graph_p Qcur = build_norm(Qcur, model.layers[il].attn_q_norm, NULL, LLM_NORM, il); cb(Qcur, "Qcur", il); } - Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); + Qcur = build_rope(Qcur, inp_pos, il); if (model.layers[il].attn_k_norm) { Kcur = build_norm(Kcur, model.layers[il].attn_k_norm, NULL, LLM_NORM, il); cb(Kcur, "Kcur", il); } - Kcur = ggml_rope_ext(ctx0, Kcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); + Kcur = build_rope(Kcur, inp_pos, il); cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); diff --git a/src/models/dbrx.cpp b/src/models/dbrx.cpp index 8d4f783fbdb0..343040e0bd33 100644 --- a/src/models/dbrx.cpp +++ b/src/models/dbrx.cpp @@ -41,10 +41,10 @@ void llama_model_dbrx::load_arch_tensors(llama_model_loader &) { } std::unique_ptr llama_model_dbrx::build_arch_graph(const llm_graph_params & params) const { - return std::make_unique(*this, params); + return std::make_unique(params); } -llama_model_dbrx::graph::graph(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) { +llama_model_dbrx::graph::graph(const llm_graph_params & params) : llm_graph_context(params) { const int64_t n_embd_head = hparams.n_embd_head_v(); GGML_ASSERT(n_embd_head == hparams.n_embd_head_k()); @@ -76,17 +76,8 @@ llama_model_dbrx::graph::graph(const llama_model & model, const llm_graph_params auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur, n_embd_head, n_head, n_head_kv, il); - Qcur = ggml_rope_ext( - ctx0, Qcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); - - Kcur = ggml_rope_ext( - ctx0, Kcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); + Qcur = build_rope(Qcur, inp_pos, il); + Kcur = build_rope(Kcur, inp_pos, il); cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); diff --git a/src/models/deci.cpp b/src/models/deci.cpp index ce5de887c70b..114b12640796 100644 --- a/src/models/deci.cpp +++ b/src/models/deci.cpp @@ -74,10 +74,10 @@ void llama_model_deci::load_arch_tensors(llama_model_loader &) { } std::unique_ptr llama_model_deci::build_arch_graph(const llm_graph_params & params) const { - return std::make_unique(*this, params); + return std::make_unique(params); } -llama_model_deci::graph::graph(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) { +llama_model_deci::graph::graph(const llm_graph_params & params) : llm_graph_context(params) { const int64_t n_embd_head = hparams.n_embd_head_v(); GGML_ASSERT(n_embd_head == hparams.n_embd_head_k()); @@ -118,18 +118,13 @@ llama_model_deci::graph::graph(const llama_model & model, const llm_graph_params cb(cur, "wo", il); } else if (n_head > 0) { // self-attention - // rope freq factors for llama3; may return nullptr for llama2 and other models - ggml_tensor * rope_factors = model.get_rope_factors(cparams, il); // compute Q and K and RoPE them auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur, n_embd_head, n_head, n_head_kv, il); - Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, rope_factors, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); - - Kcur = ggml_rope_ext(ctx0, Kcur, inp_pos, rope_factors, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); + Qcur = build_rope(Qcur, inp_pos, il); + Kcur = build_rope(Kcur, inp_pos, il); cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); diff --git a/src/models/deepseek.cpp b/src/models/deepseek.cpp index a066d4fb6081..c6a6d6a97521 100644 --- a/src/models/deepseek.cpp +++ b/src/models/deepseek.cpp @@ -68,10 +68,10 @@ void llama_model_deepseek::load_arch_tensors(llama_model_loader &) { } std::unique_ptr llama_model_deepseek::build_arch_graph(const llm_graph_params & params) const { - return std::make_unique(*this, params); + return std::make_unique(params); } -llama_model_deepseek::graph::graph(const llama_model & model, const llm_graph_params & params) : +llama_model_deepseek::graph::graph(const llm_graph_params & params) : llm_graph_context(params) { const int64_t n_embd_head = hparams.n_embd_head_v(); @@ -102,18 +102,12 @@ llama_model_deepseek::graph::graph(const llama_model & model, const llm_graph_pa // self-attention { - // rope freq factors for llama3; may return nullptr for llama2 and other models - ggml_tensor * rope_factors = model.get_rope_factors(cparams, il); - // compute Q and K and RoPE them auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur, n_embd_head, n_head, n_head_kv, il); - Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, rope_factors, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); - - Kcur = ggml_rope_ext(ctx0, Kcur, inp_pos, rope_factors, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); + Qcur = build_rope(Qcur, inp_pos, il); + Kcur = build_rope(Kcur, inp_pos, il); cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); diff --git a/src/models/deepseek2.cpp b/src/models/deepseek2.cpp index 5fd24d105f22..96c628ff73de 100644 --- a/src/models/deepseek2.cpp +++ b/src/models/deepseek2.cpp @@ -156,12 +156,12 @@ void llama_model_deepseek2::load_arch_tensors(llama_model_loader & ml) { std::unique_ptr llama_model_deepseek2::build_arch_graph(const llm_graph_params & params) const { if (params.gtype == LLM_GRAPH_TYPE_DECODER_MTP) { - return std::make_unique(*this, params); + return std::make_unique(params); } - return std::make_unique(*this, params); + return std::make_unique(params); } -llama_model_deepseek2::graph_mtp::graph_mtp(const llama_model & model, const llm_graph_params & params) : +llama_model_deepseek2::graph_mtp::graph_mtp(const llm_graph_params & params) : llm_graph_context(params) { GGML_ASSERT(hparams.n_layer_nextn > 0 && "GLM4 MTP requires n_layer_nextn > 0"); GGML_ASSERT(hparams.n_layer_nextn == 1 && "GLM4 MTP currently only supports a single MTP block"); @@ -227,9 +227,7 @@ llama_model_deepseek2::graph_mtp::graph_mtp(const llama_model & model, const llm } cb(tok_embd, "mtp_tok_embd", il); - inp->h = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, hparams.n_embd, n_tokens); - ggml_set_input(inp->h); - ggml_set_name(inp->h, "mtp_h_input"); + inp->h = build_inp_embd_state(inp.get(), hparams.n_embd); ggml_tensor * h_embd = inp->h; @@ -308,14 +306,10 @@ llama_model_deepseek2::graph_mtp::graph_mtp(const llama_model & model, const llm const float kq_scale = 1.0f * mscale * mscale / sqrtf(float(n_embd_head_k_mla)); - q_pe = ggml_rope_ext(ctx0, q_pe, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); + q_pe = build_rope(q_pe, inp_pos, il); cb(q_pe, "mtp_q_pe_rope", il); - k_pe = ggml_rope_ext(ctx0, k_pe, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); + k_pe = build_rope(k_pe, inp_pos, il); cb(k_pe, "mtp_k_pe_rope", il); q_nope = ggml_permute(ctx0, q_nope, 0, 2, 1, 3); @@ -408,7 +402,7 @@ llama_model_deepseek2::graph_mtp::graph_mtp(const llama_model & model, const llm res->t_logits = cur; } -llama_model_deepseek2::graph::graph(const llama_model & model, const llm_graph_params & params) : +llama_model_deepseek2::graph::graph(const llm_graph_params & params) : llm_graph_context(params) { // lite variants include DeepSeek-V2-Lite, GigaChat3-10B-A1.8B bool is_ocr = model.arch == LLM_ARCH_DEEPSEEK2OCR; @@ -466,7 +460,6 @@ llama_model_deepseek2::graph::graph(const llama_model & model, const llm_graph_p // self_attention if (is_ocr) { const int n_embed_head = hparams.n_embd / hparams.n_head(); - const int ocr_rope_type = GGML_ROPE_TYPE_NEOX; GGML_ASSERT(n_embed_head == n_embd_head_k && n_embed_head == n_embd_head_v); auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur, @@ -476,8 +469,8 @@ llama_model_deepseek2::graph::graph(const llama_model & model, const llm_graph_p cb(Vcur, "v", il); GGML_ASSERT(fabs(freq_base - 10000.0) < 1e-4); - Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, nullptr, n_embed_head, ocr_rope_type, 0, freq_base, 1, 0, 1, 0, 0); - Kcur = ggml_rope_ext(ctx0, Kcur, inp_pos, nullptr, n_embed_head, ocr_rope_type, 0, freq_base, 1, 0, 1, 0, 0); + Qcur = build_rope(Qcur, inp_pos, il); + Kcur = build_rope(Kcur, inp_pos, il); cb(Qcur, "q_pe", il); cb(Kcur, "k_pe", il); @@ -524,8 +517,7 @@ llama_model_deepseek2::graph::graph(const llama_model & model, const llm_graph_p ggml_row_size(kv_cmpr_pe->type, kv_lora_rank)); cb(k_pe, "k_pe", il); - k_pe = ggml_rope_ext(ctx0, k_pe, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); + k_pe = build_rope(k_pe, inp_pos, il); cb(k_pe, "k_pe", il); kv_cmpr = build_norm(kv_cmpr, model.layers[il].attn_kv_a_norm, nullptr, LLM_NORM_RMS, il); @@ -542,8 +534,7 @@ llama_model_deepseek2::graph::graph(const llama_model & model, const llm_graph_p q->nb[1], q->nb[2], ggml_row_size(q->type, n_embd_head_qk_nope)); cb(q_pe, "q_pe", il); - q_pe = ggml_rope_ext(ctx0, q_pe, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); + q_pe = build_rope(q_pe, inp_pos, il); cb(q_pe, "q_pe", il); // {n_embd_head_qk_nope, n_tokens, n_head} @@ -606,8 +597,7 @@ llama_model_deepseek2::graph::graph(const llama_model & model, const llm_graph_p cb(Vcur, "Vcur_cont", il); // RoPE is applied to the trailing dims only - ggml_tensor * Qcur = ggml_rope_ext(ctx0, q, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, - freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow); + ggml_tensor * Qcur = build_rope(q, inp_pos, il); Qcur = ggml_rope_set_offset(Qcur, n_embd_head_qk_nope); cb(Qcur, "Qcur", il); diff --git a/src/models/deepseek2ocr.cpp b/src/models/deepseek2ocr.cpp index 3d630699ef2d..3357f1b13a85 100644 --- a/src/models/deepseek2ocr.cpp +++ b/src/models/deepseek2ocr.cpp @@ -74,7 +74,24 @@ void llama_model_deepseek2ocr::load_arch_tensors(llama_model_loader &) { } } +llama_rope_params llama_model_deepseek2ocr::get_rope_params(const llama_cparams & cparams, int il) const { + auto res = llama_model::get_rope_params(cparams, il); + + res.n_rot = hparams.n_embd_head_k(il); + res.n_offs = 0; + res.type = LLAMA_ROPE_TYPE_NEOX; + res.n_ctx_orig = 0; + res.freq_scale = 1.0f; + res.ext_factor = 0.0f; + res.attn_factor = 1.0f; + res.beta_fast = 0.0f; + res.beta_slow = 0.0f; + res.factors = nullptr; + + return res; +} + std::unique_ptr llama_model_deepseek2ocr::build_arch_graph(const llm_graph_params & params) const { - return std::make_unique(*this, params); + return std::make_unique(params); } diff --git a/src/models/deepseek32.cpp b/src/models/deepseek32.cpp index 22f516a481ea..fc6207f8b3d0 100644 --- a/src/models/deepseek32.cpp +++ b/src/models/deepseek32.cpp @@ -148,12 +148,12 @@ void llama_model_deepseek32::load_arch_tensors(llama_model_loader & ml) { std::unique_ptr llama_model_deepseek32::build_arch_graph(const llm_graph_params & params) const { if (params.gtype == LLM_GRAPH_TYPE_DECODER_MTP) { - return std::make_unique(*this, params); + return std::make_unique(params); } - return std::make_unique(*this, params); + return std::make_unique(params); } -llama_model_deepseek32::graph::graph(const llama_model & model, const llm_graph_params & params) : +llama_model_deepseek32::graph::graph(const llm_graph_params & params) : llm_graph_context(params) { const bool is_mla = hparams.is_mla(); GGML_ASSERT(is_mla); @@ -224,9 +224,7 @@ llama_model_deepseek32::graph::graph(const llama_model & model, const llm_graph_ // {n_embd_indexer_head, n_indexer_head, n_tokens} indexer_q = ggml_reshape_3d(ctx0, indexer_q, n_embd_indexer_head, n_indexer_head, n_tokens); - indexer_q = ggml_rope_ext(ctx0, indexer_q, inp_pos, nullptr, n_rot, - LLAMA_ROPE_TYPE_NEOX, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); + indexer_q = build_rope_indexer(indexer_q, inp_pos, il); cb(indexer_q, "indexer_q", il); ggml_tensor * indexer_k = ggml_mul_mat(ctx0, model.layers[il].indexer_attn_k, cur); @@ -237,9 +235,7 @@ llama_model_deepseek32::graph::graph(const llama_model & model, const llm_graph_ // {n_embd_indexer_head, 1, n_tokens} indexer_k = ggml_reshape_3d(ctx0, indexer_k, n_embd_indexer_head, 1, n_tokens); - indexer_k = ggml_rope_ext(ctx0, indexer_k, inp_pos, nullptr, n_rot, - LLAMA_ROPE_TYPE_NEOX, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); + indexer_k = build_rope_indexer(indexer_k, inp_pos, il); cb(indexer_k, "indexer_k", il); // perform Hadamard transform on indexer q and k @@ -271,7 +267,7 @@ llama_model_deepseek32::graph::graph(const llama_model & model, const llm_graph_ ggml_tensor * indexer_score = nullptr; if (cparams.fused_lid) { - indexer_score = ggml_lightning_indexer(ctx0, indexer_q, indexer_k, indexer_weights, inp_attn_dsa->self_attn_mask_lid.cnv); + indexer_score = ggml_lightning_indexer(ctx0, indexer_q, indexer_k, indexer_weights, inp_attn_dsa->self_attn_mask_lid.raw); cb(indexer_score, "indexer_score", il); res->add_fused_node({LLM_FUSED_OP_LIGHTNING_INDEXER, indexer_score, il}); } else { @@ -305,7 +301,7 @@ llama_model_deepseek32::graph::graph(const llama_model & model, const llm_graph_ cb(indexer_score, "indexer_score", il); // mask indexer scores - ggml_tensor * indexer_kq_mask = inp_attn_dsa->self_attn_mask_lid.cnv; + ggml_tensor * indexer_kq_mask = inp_attn_dsa->self_attn_mask_lid.raw; indexer_score = ggml_add(ctx0, indexer_score, indexer_kq_mask); cb(indexer_score, "indexer_score", il); } @@ -347,12 +343,10 @@ llama_model_deepseek32::graph::graph(const llama_model & model, const llm_graph_ ggml_row_size(kv_cmpr_pe->type, kv_lora_rank)); cb(k_pe, "k_pe", il); - q_pe = ggml_rope_ext(ctx0, q_pe, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); + q_pe = build_rope(q_pe, inp_pos, il); cb(q_pe, "q_pe", il); - k_pe = ggml_rope_ext(ctx0, k_pe, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); + k_pe = build_rope(k_pe, inp_pos, il); cb(k_pe, "k_pe", il); kv_cmpr = build_norm(kv_cmpr, model.layers[il].attn_kv_a_norm, nullptr, LLM_NORM_RMS, il); @@ -484,7 +478,7 @@ llama_model_deepseek32::graph::graph(const llama_model & model, const llm_graph_ // with shared expert, exactly as the trunk deepseek2 graph builds it) -> // shared_head_norm (fallback output_norm) -> shared LM head. // The DSA indexer is not used at runtime. -llama_model_deepseek32::graph_mtp::graph_mtp(const llama_model & model, const llm_graph_params & params) +llama_model_deepseek32::graph_mtp::graph_mtp(const llm_graph_params & params) : llm_graph_context(params) { GGML_ASSERT(hparams.n_layer_nextn > 0 && "DEEPSEEK32 MTP requires n_layer_nextn > 0"); GGML_ASSERT(hparams.n_layer_nextn == 1 && "DEEPSEEK32 MTP currently only supports a single MTP block"); @@ -537,9 +531,7 @@ llama_model_deepseek32::graph_mtp::graph_mtp(const llama_model & model, const ll } cb(tok_embd, "mtp_tok_embd", il); - inp->h = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, hparams.n_embd, n_tokens); - ggml_set_input(inp->h); - ggml_set_name(inp->h, "mtp_h_input"); + inp->h = build_inp_embd_state(inp.get(), hparams.n_embd); ggml_tensor * h_embd = inp->h; @@ -607,12 +599,10 @@ llama_model_deepseek32::graph_mtp::graph_mtp(const llama_model & model, const ll ggml_row_size(kv_cmpr_pe->type, kv_lora_rank)); cb(k_pe, "mtp_k_pe", il); - q_pe = ggml_rope_ext(ctx0, q_pe, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); + q_pe = build_rope(q_pe, inp_pos, il); cb(q_pe, "mtp_q_pe", il); - k_pe = ggml_rope_ext(ctx0, k_pe, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); + k_pe = build_rope(k_pe, inp_pos, il); cb(k_pe, "mtp_k_pe", il); kv_cmpr = build_norm(kv_cmpr, layer.attn_kv_a_norm, nullptr, LLM_NORM_RMS, il); diff --git a/src/models/deepseek4.cpp b/src/models/deepseek4.cpp index 85fe141fd87a..0d5d9a469e64 100644 --- a/src/models/deepseek4.cpp +++ b/src/models/deepseek4.cpp @@ -176,9 +176,9 @@ void llama_model_deepseek4::load_arch_tensors(llama_model_loader & ml) { std::unique_ptr llama_model_deepseek4::build_arch_graph(const llm_graph_params & params) const { if (params.gtype == LLM_GRAPH_TYPE_DECODER_MTP) { - return std::make_unique(*this, params); + return std::make_unique(params); } - return std::make_unique(*this, params); + return std::make_unique(params); } static size_t dsv4_elem_offset(const ggml_tensor * t, int64_t i) { @@ -590,7 +590,6 @@ ggml_tensor * llama_model_deepseek4::graph::build_overlap_compressed_kv_from_sta } ggml_tensor * llama_model_deepseek4::graph::build_lid_top_k( - const llama_model & model, llm_graph_input_dsv4 * inp_dsv4, ggml_tensor * qr, ggml_tensor * cur, @@ -714,7 +713,6 @@ ggml_tensor * llama_model_deepseek4::graph::build_top_k_mask( } ggml_tensor * llama_model_deepseek4::graph::build_csa_lid_attention( - const llama_model & model, llm_graph_input_dsv4 * inp_dsv4, llm_graph_input_dsv4_raw * inp_attn, ggml_tensor * q, @@ -728,7 +726,7 @@ ggml_tensor * llama_model_deepseek4::graph::build_csa_lid_attention( const auto & inp_csa = inp_dsv4->get_csa(); GGML_ASSERT(inp_csa.kq_mask); - ggml_tensor * top_k = build_lid_top_k(model, inp_dsv4, qr, cur, inp_pos, il); + ggml_tensor * top_k = build_lid_top_k(inp_dsv4, qr, cur, inp_pos, il); ggml_tensor * k_rot = inp_attn->self_k_rot[il]; if (k_rot) { @@ -759,7 +757,7 @@ ggml_tensor * llama_model_deepseek4::graph::build_csa_lid_attention( ggml_tensor * k_all = ggml_concat(ctx0, raw_k, csa_k, 2); cb(k_all, "csa_k_all", il); - ggml_tensor * raw_mask = inp_attn->self_attn_mask.cnv; + ggml_tensor * raw_mask = inp_attn->self_attn_mask.raw; ggml_tensor * csa_mask = build_top_k_mask(inp_csa.kq_mask, top_k, "csa_top_k_mask", il); ggml_tensor * kq_mask = ggml_concat(ctx0, raw_mask, csa_mask, 0); @@ -815,7 +813,7 @@ ggml_tensor * llama_model_deepseek4::graph::build_hca_attention( ggml_tensor * k_all = ggml_concat(ctx0, raw_k, hca_k, 2); cb(k_all, "hca_k_all", il); - ggml_tensor * raw_mask = inp_attn->self_attn_mask.cnv; + ggml_tensor * raw_mask = inp_attn->self_attn_mask.raw; ggml_tensor * hca_mask = inp_hca.kq_mask; ggml_tensor * kq_mask = ggml_concat(ctx0, raw_mask, hca_mask, 0); @@ -853,7 +851,7 @@ ggml_tensor * llama_model_deepseek4::graph::build_raw_attention( ggml_build_forward_expand(gf, mctx_cur->cpy_k(ctx0, kv, inp_attn->get_k_idxs(), il)); - ggml_tensor * kq_mask = inp_attn->self_attn_mask.cnv; + ggml_tensor * kq_mask = inp_attn->self_attn_mask.raw; ggml_tensor * k = mctx_cur->get_k(ctx0, il); @@ -867,25 +865,22 @@ ggml_tensor * llama_model_deepseek4::graph::build_raw_attention( } ggml_tensor * llama_model_deepseek4::graph::build_attention( - const llama_model & model, llm_graph_input_dsv4 * inp_dsv4, ggml_tensor * cur, ggml_tensor * inp_pos, int il) const { - return build_attention_impl(model, inp_dsv4, nullptr, cur, inp_pos, il); + return build_attention_impl(inp_dsv4, nullptr, cur, inp_pos, il); } ggml_tensor * llama_model_deepseek4::graph::build_attention( - const llama_model & model, llm_graph_input_attn_k_iswa * inp_mtp, ggml_tensor * cur, ggml_tensor * inp_pos, int il) const { - return build_attention_impl(model, nullptr, inp_mtp, cur, inp_pos, il); + return build_attention_impl(nullptr, inp_mtp, cur, inp_pos, il); } ggml_tensor * llama_model_deepseek4::graph::build_attention_impl( - const llama_model & model, llm_graph_input_dsv4 * inp_dsv4, llm_graph_input_attn_k_iswa * inp_mtp, ggml_tensor * cur, @@ -1198,7 +1193,7 @@ ggml_tensor * llama_model_deepseek4::graph::build_attention_impl( inp_dsv4->get_csa().kq_mask && inp_dsv4->get_lid().kq_mask && inp_dsv4->get_lid().k_rot[il]) { - out = build_csa_lid_attention(model, inp_dsv4, inp_attn, q, kv, qr, cur, inp_pos, layer.attn_sinks, + out = build_csa_lid_attention(inp_dsv4, inp_attn, q, kv, qr, cur, inp_pos, layer.attn_sinks, 1.0f/sqrtf(float(n_embd_head)), il); } else if (ratio == DSV4_HCA_RATIO && inp_dsv4->get_hca().kq_mask) { @@ -1228,7 +1223,7 @@ ggml_tensor * llama_model_deepseek4::graph::build_attention_impl( return out; } -llama_model_deepseek4::graph::graph(const llama_model & model, const llm_graph_params & params) : +llama_model_deepseek4::graph::graph(const llm_graph_params & params) : llm_graph_context(params) { ggml_tensor * cur; @@ -1263,7 +1258,7 @@ llama_model_deepseek4::graph::graph(const llama_model & model, const llm_graph_p cur = build_norm(cur, model.layers[il].attn_norm, nullptr, LLM_NORM_RMS, il); cb(cur, "attn_norm", il); - cur = build_attention(model, inp_dsv4, cur, inp_pos, il); + cur = build_attention(inp_dsv4, cur, inp_pos, il); inpL = build_hc_post(cur, residual, attn_hc.post, attn_hc.comb, il); cb(inpL, "hc_attn_post", il); @@ -1364,8 +1359,8 @@ llama_model_deepseek4::graph::graph(const llama_model & model, const llm_graph_p } -llama_model_deepseek4::graph_mtp::graph_mtp(const llama_model & model, const llm_graph_params & params) : - graph(params) { +llama_model_deepseek4::graph_mtp::graph_mtp(const llm_graph_params & params) : + graph(params, no_build{}) { GGML_ASSERT(hparams.n_layer_nextn > 0 && "DEEPSEEK4 MTP requires n_layer_nextn > 0"); GGML_ASSERT(hparams.n_layer_nextn == 1 && "DEEPSEEK4 MTP currently only supports a single MTP block"); GGML_ASSERT(cparams.nextn_layer_offset >= 0 && @@ -1391,9 +1386,7 @@ llama_model_deepseek4::graph_mtp::graph_mtp(const llama_model & model, const llm inp->embd = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, hparams.n_embd_inp(), n_tokens); ggml_set_input(inp->embd); - inp->h = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, hparams.n_embd_out(), n_tokens); - ggml_set_input(inp->h); - ggml_set_name(inp->h, "mtp_h_input"); + inp->h = build_inp_embd_state(inp.get(), hparams.n_embd_out()); ggml_tensor * tok_embd; if (ubatch.token) { @@ -1440,7 +1433,7 @@ llama_model_deepseek4::graph_mtp::graph_mtp(const llama_model & model, const llm cur = build_norm(cur, layer.attn_norm, nullptr, LLM_NORM_RMS, il); cb(cur, "mtp_attn_norm", il); - cur = build_attention(model, inp_attn, cur, inp_pos, il); + cur = build_attention(inp_attn, cur, inp_pos, il); inpL = build_hc_post(cur, residual, attn_hc.post, attn_hc.comb, il); cb(inpL, "mtp_hc_attn_post", il); diff --git a/src/models/deepseek41.cpp b/src/models/deepseek41.cpp index d8299897043b..96de9041270c 100644 --- a/src/models/deepseek41.cpp +++ b/src/models/deepseek41.cpp @@ -222,7 +222,7 @@ std::unique_ptr llama_model_deepseek41::build_arch_graph(cons throw std::runtime_error("DeepSeek-V4.1 has no MTP graph: its DSpark draft is a separate DFLASH GGUF " "(convert_hf_to_gguf.py --dspark), used with --spec-type draft-dspark"); } - return std::make_unique(*this, params); + return std::make_unique(params); } // Engram n-gram hash (engram.py NgramHashState.forward) @@ -642,7 +642,7 @@ ggml_tensor * llama_model_deepseek41::graph::build_attention41( ggml_build_forward_expand(gf, inp_attn->mctx->cpy_k(ctx0, kv, inp_attn->get_k_idxs(), il)); ggml_tensor * raw_k = inp_attn->mctx->get_k(ctx0, il); - ggml_tensor * raw_mask = inp_attn->self_attn_mask.cnv; + ggml_tensor * raw_mask = inp_attn->self_attn_mask.raw; const int32_t topk_src = hparams.dsv41_topk_src[il]; const int64_t k_sel = top_k->ne[0]; @@ -765,9 +765,9 @@ ggml_tensor * llama_model_deepseek41::graph::build_attention41( return out; } -llama_model_deepseek41::graph::graph(const llama_model & model_base, const llm_graph_params & params) : - llama_model_deepseek4::graph(params) { - const auto & model = static_cast(model_base); +llama_model_deepseek41::graph::graph(const llm_graph_params & params) : + llama_model_deepseek4::graph(params, no_build{}) { + const auto & model = static_cast(this->model); static const bool sparse_env = dsv41_env_on("LLAMA_DSV41_SPARSE"); sparse = sparse_env; @@ -875,7 +875,7 @@ llama_model_deepseek41::graph::graph(const llama_model & model_base, const llm_g cur = build_norm(cur, layer.ffn_norm, nullptr, LLM_NORM_RMS, il); cb(cur, "ffn_norm", il); - // F32 router accumulation: expert selection is near-tie sensitive + // the reference computes the router in F32: https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash-1017/blob/1bae0e1088993b9b82de08f59de5be5a34e42170/inference/model.py#L813 ggml_tensor * logits = build_lora_mm(layer.ffn_gate_inp, cur); ggml_prec_set_acc(logits, GGML_PREC_F32); cb(logits, "ffn_moe_logits", il); diff --git a/src/models/delta-net-base.cpp b/src/models/delta-net-base.cpp index 2e15777ee156..45b2de4bbc9d 100644 --- a/src/models/delta-net-base.cpp +++ b/src/models/delta-net-base.cpp @@ -3,378 +3,15 @@ #include "llama-impl.h" #include "llama-memory-recurrent.h" -// utility to get one slice from the third dimension -// input dim: [x, y, c, b] -// output dim: [x, y, 1, b] -static ggml_tensor * get_slice_2d(ggml_context * ctx0, ggml_tensor * t, int64_t c) { - return ggml_view_4d(ctx0, t, t->ne[0], t->ne[1], 1, t->ne[3], - t->nb[1], t->nb[2], t->nb[3], t->nb[2] * c); -} - llm_build_delta_net_base::llm_build_delta_net_base(const llm_graph_params & params) : llm_graph_context(params) {} -std::pair llm_build_delta_net_base::build_delta_net_chunking( - ggml_tensor * q, - ggml_tensor * k, - ggml_tensor * v, - ggml_tensor * g, - ggml_tensor * b, - ggml_tensor * s, - int il) { - const int64_t S_k = q->ne[0]; - const int64_t H_k = q->ne[1]; - const int64_t n_tokens = q->ne[2]; - const int64_t n_seqs = q->ne[3]; - - const int64_t S_v = v->ne[0]; - const int64_t H_v = v->ne[1]; - const bool kda = (g->ne[0] == S_k && g->ne[1] == H_k); - - GGML_ASSERT(S_k == S_v); - GGML_ASSERT(H_v % H_k == 0); - - GGML_ASSERT(q->ne[0] == S_k && q->ne[1] == H_k && q->ne[2] == n_tokens && q->ne[3] == n_seqs); - GGML_ASSERT(k->ne[0] == S_k && k->ne[1] == H_k && k->ne[2] == n_tokens && k->ne[3] == n_seqs); - GGML_ASSERT(v->ne[0] == S_v && v->ne[1] == H_v && v->ne[2] == n_tokens && v->ne[3] == n_seqs); - - GGML_ASSERT(g->ne[0] == 1 || g->ne[0] == S_v); - GGML_ASSERT( g->ne[1] == H_v && g->ne[2] == n_tokens && g->ne[3] == n_seqs); - GGML_ASSERT(b->ne[0] == 1 && b->ne[1] == H_v && b->ne[2] == n_tokens && b->ne[3] == n_seqs); - GGML_ASSERT(s->ne[0] == S_v && s->ne[1] == S_v && s->ne[2] == H_v && s->ne[3] == n_seqs); - - const float scale = 1.0f / sqrtf(S_k); - - q = ggml_scale(ctx0, q, scale); - - cb(q, "q_in", il); - cb(k, "k_in", il); - cb(v, "v_in", il); - cb(b, "b_in", il); - cb(g, "g_in", il); - - q = ggml_permute(ctx0, q, 0, 2, 1, 3); // [S_k, n_tokens, H_k, n_seqs] - k = ggml_permute(ctx0, k, 0, 2, 1, 3); // [S_k, n_tokens, H_k, n_seqs] - v = ggml_permute(ctx0, v, 0, 2, 1, 3); // [S_v, n_tokens, H_v, n_seqs] - g = ggml_permute(ctx0, g, 0, 2, 1, 3); // [g_0, n_tokens, H_v, n_seqs] - b = ggml_permute(ctx0, b, 0, 2, 1, 3); // [ 1, n_tokens, H_v, n_seqs] - - const int CS = kda ? 16 : 64; // chunk size - - const int pad = (CS - n_tokens % CS) % CS; - const int n_chunks = (n_tokens + pad) / CS; - - q = ggml_pad(ctx0, q, 0, pad, 0, 0); - k = ggml_pad(ctx0, k, 0, pad, 0, 0); - v = ggml_pad(ctx0, v, 0, pad, 0, 0); - g = ggml_pad(ctx0, g, 0, pad, 0, 0); - b = ggml_pad(ctx0, b, 0, pad, 0, 0); - - ggml_tensor * v_b = ggml_mul(ctx0, v, b); - ggml_tensor * k_b = ggml_mul(ctx0, k, b); - - cb(v_b, "v_b", il); - cb(k_b, "k_b", il); - - q = ggml_reshape_4d(ctx0, q, S_k, CS, n_chunks, H_k * n_seqs); - k = ggml_reshape_4d(ctx0, k, S_k, CS, n_chunks, H_k * n_seqs); - k_b = ggml_reshape_4d(ctx0, k_b, S_k, CS, n_chunks, H_v * n_seqs); - v = ggml_reshape_4d(ctx0, v, S_v, CS, n_chunks, H_v * n_seqs); - v_b = ggml_reshape_4d(ctx0, v_b, S_v, CS, n_chunks, H_v * n_seqs); - - g = ggml_reshape_4d(ctx0, g, g->ne[0], CS, n_chunks, H_v * n_seqs); - b = ggml_reshape_4d(ctx0, b, 1, CS, n_chunks, H_v * n_seqs); - - // [CS, g_0, n_chunks, H_v * n_seqs] - // TODO: extend ggml_cumsum with axis parameter to avoid transpose - ggml_tensor * g_cs = ggml_cumsum(ctx0, ggml_cont(ctx0, ggml_transpose(ctx0, g))); - cb(g_cs, "g_cs", il); - - ggml_tensor * kb = nullptr; - ggml_tensor * kq = nullptr; - if (kda) { - const int64_t CHB = n_chunks * H_k * n_seqs; - - ggml_tensor * g_cs_i = ggml_reshape_4d(ctx0, g_cs, CS, 1, S_k, CHB); // [chunk_size, 1, S_k, CHB] - ggml_tensor * g_cs_j = ggml_reshape_4d(ctx0, g_cs, 1, CS, S_k, CHB); // [1, chunk_size, S_k, CHB] - - g_cs_j = ggml_repeat_4d(ctx0, g_cs_j, CS, CS, S_k, CHB); // [1, chunk_size, S_k, CHB] -> [chunk_size, chunk_size, S_k, CHB] - - // decay_mask [chunk_size,chunk_size,S_k,CHB] - ggml_tensor * decay_mask; - decay_mask = ggml_sub(ctx0, g_cs_j, g_cs_i); - decay_mask = ggml_tri(ctx0, decay_mask, GGML_TRI_TYPE_LOWER_DIAG); - decay_mask = ggml_exp(ctx0, decay_mask); - cb(decay_mask, "decay_mask", il); - - // decay_mask [S_k,BT_j,BT_i,CHB] *Note* second and third chunk_sizes are switched - decay_mask = ggml_cont_4d(ctx0, ggml_permute(ctx0, decay_mask, 2, 1, 0, 3), S_k, CS, CS, CHB); - - ggml_tensor * k_b_i = ggml_reshape_4d(ctx0, k_b, S_k, CS, 1, CHB); - ggml_tensor * k_j = ggml_reshape_4d(ctx0, k, S_k, 1, CS, CHB); - ggml_tensor * q_i = ggml_reshape_4d(ctx0, q, S_k, CS, 1, CHB); - - ggml_tensor * decay_k_b_i = ggml_mul(ctx0, decay_mask, k_b_i); - ggml_tensor * decay_q_i = ggml_mul(ctx0, decay_mask, q_i); - - // decay_k_b_i [S,BT,BT,CHB] @ k_j [S,1,BT,CHB] = Akk [BT,1,BT,CHB] - kb = ggml_mul_mat(ctx0, decay_k_b_i, k_j); - kq = ggml_mul_mat(ctx0, decay_q_i, k_j); - - kb = ggml_cont(ctx0, ggml_transpose(ctx0, ggml_reshape_4d(ctx0, kb, CS, CS, n_chunks, H_v * n_seqs))); - kq = ggml_cont(ctx0, ggml_transpose(ctx0, ggml_reshape_4d(ctx0, kq, CS, CS, n_chunks, H_v * n_seqs))); - } else { - ggml_tensor * g_cs_i = g_cs; - ggml_tensor * g_cs_j = ggml_reshape_4d(ctx0, g_cs, 1, CS, n_chunks, H_v * n_seqs); - - g_cs_j = ggml_repeat_4d(ctx0, g_cs_j, CS, CS, n_chunks, H_v * n_seqs); - - // [CS, CS, n_chunks, H_v * n_seqs] - ggml_tensor * decay_mask; - decay_mask = ggml_sub(ctx0, g_cs_j, g_cs_i); - decay_mask = ggml_tri(ctx0, decay_mask, GGML_TRI_TYPE_LOWER_DIAG); - decay_mask = ggml_exp(ctx0, decay_mask); - cb(decay_mask, "decay_mask", il); - - // [CS, CS, n_chunks, H_k * n_seqs] - kb = ggml_mul_mat(ctx0, k, k_b); - kb = ggml_mul (ctx0, kb, decay_mask); - - // [CS, CS, n_chunks, H_k * n_seqs] - kq = ggml_mul_mat(ctx0, k, q); - kq = ggml_mul(ctx0, kq, decay_mask); - } - - kq = ggml_tri(ctx0, kq, GGML_TRI_TYPE_LOWER_DIAG); - cb(kq, "kq", il); - - // [CS, CS, n_chunks, H_k * n_seqs] - ggml_tensor * attn; - attn = ggml_tri(ctx0, kb, GGML_TRI_TYPE_LOWER); - cb(attn, "attn", il); - - ggml_tensor * identity; - identity = ggml_view_1d(ctx0, attn, CS, 0); - identity = ggml_fill (ctx0, identity, 1.0f); - identity = ggml_diag (ctx0, identity); - - ggml_tensor * lhs = ggml_add(ctx0, attn, identity); - cb(lhs, "dnet_add_ch_lhs", il); - - attn = ggml_neg(ctx0, attn); - cb(attn, "attn_pre_solve", il); - - ggml_tensor * lin_solve = ggml_solve_tri(ctx0, lhs, attn, true, true, false); - attn = ggml_add(ctx0, lin_solve, identity); - cb(attn, "dnet_add_ch_attn_solved", il); // [CS, CS, n_chunks, H_k * n_seqs] - - // [S_v, CS, n_chunks, H_v * n_seqs] - v = ggml_mul_mat(ctx0, ggml_cont(ctx0, ggml_transpose(ctx0, v_b)), attn); - - // [CS, 1, n_chunks, H_v * n_seqs] KDA: [CS, S_k, n_chunks, H_v * n_seqs] - ggml_tensor * g_exp = ggml_exp(ctx0, g_cs); - - k_b = ggml_cont(ctx0, ggml_transpose(ctx0, k_b)); - - // [CS, S_k, n_chunks, H_k * n_seqs] - ggml_tensor * kbg = ggml_mul(ctx0, k_b, g_exp); - cb(kbg, "k_beta_g_exp", il); - - // [S_k, CS, n_chunks, H_k * n_seqs] - ggml_tensor * k_cd = ggml_mul_mat(ctx0, kbg, attn); - cb(k_cd, "k_cumdecay", il); - - // [1, CS, n_chunks, H_k * n_seqs] KDA: [S_k, CS, n_chunks, H_k * n_seqs] - ggml_tensor * g_exp_t = ggml_cont(ctx0, ggml_transpose(ctx0, g_exp)); - ggml_tensor * q_g_exp = ggml_mul(ctx0, q, g_exp_t); - - // vectorized calculation of key_gdiff - // improved from the chunked version: - // g_last = torch.clamp(g_cum[:, :, -1], max=50.0).exp().unsqueeze(-1).unsqueeze(-1) - // g_diff = torch.clamp(g_cum[:, :, -1:] - g_cum, max=50.0).exp() - // key_gdiff = key * g_diff.unsqueeze(-1) - // kgdmulvnew = (key_gdiff).transpose(-1, -2) @ v_new - // last_recurrent_state = last_recurrent_state * g_last + kgdmulvnew - - // get last element in g_cumsum along CS dimension (ne0) - // example: [[x, y, z, ..., last], ...] -> [[last], ...] - // [1, 1, n_chunks, H_v * n_seqs] KDA: [1, S_k, n_chunks, H_v * n_seqs] - ggml_tensor * g_last = ggml_view_4d(ctx0, g_cs, 1, g_cs->ne[1], g_cs->ne[2], g_cs->ne[3], - g_cs->nb[1], - g_cs->nb[2], - g_cs->nb[3], - ggml_row_size(g_cs->type, g_cs->ne[0] - 1)); - cb(g_last, "g_last", il); - - // [1, 1, n_chunks, H_v * n_seqs] KDA: [S_k, 1, n_chunks, H_v * n_seqs] - ggml_tensor * g_last_exp_t = ggml_transpose(ctx0, ggml_exp(ctx0, g_last)); - cb(g_last_exp_t, "g_last_exp_t", il); - - // [CS, 1, n_chunks, H_v * n_seqs] KDA: [CS, S_k, n_chunks, H_v * n_seqs] - ggml_tensor * g_diff = ggml_neg(ctx0, ggml_sub(ctx0, g_cs, g_last)); - cb(g_diff, "g_diff", il); - - ggml_tensor * g_diff_exp_t = ggml_cont(ctx0, ggml_transpose(ctx0, ggml_exp(ctx0, g_diff))); - - // [S_k, CS, n_chunks, H_v * n_seqs] - ggml_tensor * kg = ggml_mul(ctx0, k, g_diff_exp_t); - cb(kg, "key_gdiff", il); - - // [CS, S_k, n_chunks, H_v * n_seqs] - ggml_tensor * kg_t = ggml_cont(ctx0, ggml_transpose(ctx0, kg)); - cb(kg_t, "key_gdiff_t", il); - - s = ggml_reshape_4d(ctx0, s, S_v, S_v, 1, H_v * n_seqs); - cb(s, "dnet_add_ch_state", il); - - // [CS, S_v, n_chunks, H_v * n_seqs] - ggml_tensor * v_t = ggml_cont(ctx0, ggml_transpose(ctx0, v)); - - for (int64_t chunk = 0; chunk < n_chunks; chunk++) { - ggml_tensor * ch_k_cd = get_slice_2d(ctx0, k_cd, chunk); // [S_k, CS, 1, H_k * n_seqs] - ggml_tensor * ch_v_t = get_slice_2d(ctx0, v_t, chunk); // [ CS, S_v, 1, H_v * n_seqs] - ggml_tensor * ch_kq = get_slice_2d(ctx0, kq, chunk); // [ CS, CS, 1, H_k * n_seqs] - ggml_tensor * ch_q_g_exp = get_slice_2d(ctx0, q_g_exp, chunk); // [S_k, CS, 1, H_k * n_seqs] - ggml_tensor * ch_kg_t = get_slice_2d(ctx0, kg_t, chunk); // [ CS, S_k, 1, H_v * n_seqs] - - // [CS, S_v, 1, H_v * n_seqs] - ggml_tensor * v_t_p = ggml_mul_mat(ctx0, ch_k_cd, s); - cb(v_t_p, "v_prime", il); - - // [CS, S_v, 1, H_v * n_seqs] - ggml_tensor * v_t_new = ggml_sub(ctx0, ch_v_t, v_t_p); - cb(v_t_new, "v_t_new", il); - - // [S_v, CS, 1, H_v * n_seqs] - ggml_tensor * v_attn = ggml_mul_mat(ctx0, v_t_new, ch_kq); - cb(v_attn, "v_attn", il); - - // [S_v, CS, 1, H_v * n_seqs] - ggml_tensor * attn_inter = ggml_mul_mat(ctx0, s, ch_q_g_exp); - cb(attn_inter, "attn_inter", il); - - // [S_v, CS, 1, H_v * n_seqs] - ggml_tensor * o_ch = ggml_add(ctx0, attn_inter, v_attn); - cb(o_ch, "dnet_add_ch_attn_out", il); - - v = ggml_set_inplace(ctx0, v, o_ch, v->nb[1], v->nb[2], v->nb[3], chunk * v->nb[2]); - - // kgdmulvnew = (key_gdiff).transpose(-1, -2) @ v_new - // TODO: head broadcast might not work here - probably will need a transpose - ggml_tensor * kgv = ggml_mul_mat(ctx0, ch_kg_t, v_t_new); // [S_k, S_v, 1, H_k * n_seqs] - - // last_recurrent_state = last_recurrent_state * g_last + kgdmulvnew - ggml_tensor * ch_g_last_exp_t = get_slice_2d(ctx0, g_last_exp_t, chunk); - - s = ggml_mul(ctx0, s, ch_g_last_exp_t); - s = ggml_add(ctx0, s, kgv); - cb(s, "dnet_add_ch_state", il); - } - - // truncate padded tokens - ggml_tensor * o = ggml_view_4d(ctx0, v, - S_v, n_tokens, H_v, n_seqs, - ggml_row_size(v->type, S_v), - ggml_row_size(v->type, S_v * CS * n_chunks), - ggml_row_size(v->type, S_v * CS * n_chunks * H_v), 0); - o = ggml_permute (ctx0, o, 0, 2, 1, 3); // [S_v, H_v, n_tokens, n_seqs] - s = ggml_reshape_4d(ctx0, s, S_v, S_v, H_v, n_seqs); - cb(s, "output_state", il); - - return {o, s}; -} - -std::pair llm_build_delta_net_base::build_delta_net_autoregressive( - ggml_tensor * q, - ggml_tensor * k, - ggml_tensor * v, - ggml_tensor * g, - ggml_tensor * b, // beta - ggml_tensor * s, // state - int il) { - const int64_t S_k = q->ne[0]; - const int64_t H_k = q->ne[1]; - const int64_t n_tokens = q->ne[2]; - const int64_t n_seqs = q->ne[3]; - - const int64_t S_v = v->ne[0]; - const int64_t H_v = v->ne[1]; - - GGML_ASSERT(n_tokens == 1); - - GGML_ASSERT(S_k == S_v); - GGML_ASSERT(H_v % H_k == 0); - - GGML_ASSERT(q->ne[0] == S_k && q->ne[1] == H_k && q->ne[2] == n_tokens && q->ne[3] == n_seqs); - GGML_ASSERT(k->ne[0] == S_k && k->ne[1] == H_k && k->ne[2] == n_tokens && k->ne[3] == n_seqs); - GGML_ASSERT(v->ne[0] == S_v && v->ne[1] == H_v && v->ne[2] == n_tokens && v->ne[3] == n_seqs); - - GGML_ASSERT(g->ne[0] == 1 || g->ne[0] == S_v); - GGML_ASSERT( g->ne[1] == H_v && g->ne[2] == n_tokens && g->ne[3] == n_seqs); - GGML_ASSERT(b->ne[0] == 1 && b->ne[1] == H_v && b->ne[2] == n_tokens && b->ne[3] == n_seqs); - GGML_ASSERT(s->ne[0] == S_v && s->ne[1] == S_v && s->ne[2] == H_v && s->ne[3] == n_seqs); - - const float scale = 1.0f / sqrtf(S_k); - - q = ggml_scale(ctx0, q, scale); - - q = ggml_permute(ctx0, q, 0, 2, 1, 3); // [S_k, n_tokens, H_k, n_seqs] - k = ggml_permute(ctx0, k, 0, 2, 1, 3); // [S_k, n_tokens, H_k, n_seqs] - v = ggml_permute(ctx0, v, 0, 2, 1, 3); // [S_v, n_tokens, H_v, n_seqs] - - cb(q, "q_in", il); - cb(k, "k_in", il); - cb(v, "v_in", il); - cb(b, "b_in", il); - cb(g, "g_in", il); - - // GDA: [1, 1, H_v, n_seqs] - // KDA: [1, S_k, H_v, n_seqs] - g = ggml_reshape_4d(ctx0, g, 1, g->ne[0], H_v, n_seqs); - b = ggml_reshape_4d(ctx0, b, 1, 1, H_v, n_seqs); - - // [S_v, S_v, H_v, n_seqs] - g = ggml_exp(ctx0, g); - s = ggml_mul(ctx0, s, g); - - // [1, S_v, H_v, n_seqs] - ggml_tensor * sk; - sk = ggml_mul (ctx0, s, k); - sk = ggml_sum_rows(ctx0, sk); - - // [S_v, 1, H_v, n_seqs] - ggml_tensor * d; - d = ggml_sub(ctx0, v, ggml_transpose(ctx0, sk)); - d = ggml_mul(ctx0, d, b); - - // [1, S_v, H_v, n_seqs] - ggml_tensor * d_t; - d_t = ggml_transpose(ctx0, d); - - // [S_v, S_v, H_v, n_seqs] - ggml_tensor * kd; - k = ggml_repeat(ctx0, k, s); - kd = ggml_mul (ctx0, k, d_t); - - s = ggml_add(ctx0, s, kd); - - cb(s, "dnet_add_ar_state", il); - - ggml_tensor * s_q = ggml_mul (ctx0, s, q); - ggml_tensor * o = ggml_sum_rows(ctx0, s_q); - - o = ggml_permute (ctx0, o, 2, 0, 1, 3); // [S_v, H_v, n_tokens, n_seqs] - - return {o, s}; -} - -std::pair llm_build_delta_net_base::build_delta_net_fused( +std::pair llm_build_delta_net_base::build_delta_net( ggml_tensor * q, ggml_tensor * k, ggml_tensor * v, ggml_tensor * g, ggml_tensor * b, - ggml_tensor * s, - int il) { + ggml_tensor * s) { const int64_t S_k = q->ne[0]; const int64_t H_k = q->ne[1]; const int64_t n_tokens = q->ne[2]; @@ -397,12 +34,6 @@ std::pair llm_build_delta_net_base::build_delta_ne // K=1: output carries the final state only. state s is 4D [S_v, S_v, H_v, n_seqs]. ggml_tensor * result = ggml_gated_delta_net(ctx0, q, k, v, g, b, s, /*K=*/1); - if (n_tokens == 1) { - res->add_fused_node({LLM_FUSED_OP_GDN_AR, result, il}); - } else { - res->add_fused_node({LLM_FUSED_OP_GDN_CH, result, il}); - } - ggml_tensor * output = ggml_view_4d(ctx0, result, S_v, H_v, n_tokens, n_seqs, ggml_row_size(result->type, S_v), @@ -419,30 +50,6 @@ std::pair llm_build_delta_net_base::build_delta_ne return {output, new_state}; } -std::pair llm_build_delta_net_base::build_delta_net( - ggml_tensor * q, - ggml_tensor * k, - ggml_tensor * v, - ggml_tensor * g, - ggml_tensor * b, - ggml_tensor * s, - int il) { - const int64_t n_seq_tokens = q->ne[2]; - - if (n_seq_tokens == 1) { - if (cparams.fused_gdn_ar) { - return build_delta_net_fused(q, k, v, g, b, s, il); - } - return build_delta_net_autoregressive(q, k, v, g, b, s, il); - } - - if (cparams.fused_gdn_ch) { - return build_delta_net_fused(q, k, v, g, b, s, il); - } - - return build_delta_net_chunking(q, k, v, g, b, s, il); -} - ggml_tensor * llm_build_delta_net_base::build_conv_state( llm_graph_input_rs * inp, ggml_tensor * conv_states_all, @@ -450,11 +57,6 @@ ggml_tensor * llm_build_delta_net_base::build_conv_state( int64_t conv_kernel_size, int64_t conv_channels, int il) { - const auto * mctx_cur = inp->mctx; - - const auto kv_head = mctx_cur->get_head(); - const auto mem_size = mctx_cur->get_size(); - const int64_t n_seqs = ubatch.n_seqs; ggml_tensor * conv_states = build_rs(inp, conv_states_all, hparams.n_embd_r(), n_seqs); @@ -469,54 +71,7 @@ ggml_tensor * llm_build_delta_net_base::build_conv_state( ggml_tensor * conv_input = ggml_concat(ctx0, conv_states, qkv_mixed, 0); cb(conv_input, "conv_input", il); - const int64_t row_count = (conv_kernel_size - 1) * conv_channels; - - const size_t row_size = ggml_row_size(conv_states_all->type, row_count); - - if (cparams.n_rs_seq == 0) { - const int64_t s_idx = conv_input->ne[0] - conv_states->ne[0]; - const int64_t s_slot = 0; - - ggml_tensor * conv_state_last = - ggml_view_3d(ctx0, conv_input, - conv_kernel_size - 1, conv_channels, n_seqs, - conv_input->nb[1], conv_input->nb[2], - ggml_row_size(conv_input->type, s_idx)); - cb(conv_state_last, "conv_state_last", il); - - ggml_tensor * conv_state_update = - ggml_view_2d(ctx0, conv_states_all, - row_count, n_seqs, conv_states_all->nb[1], - (s_slot * mem_size + kv_head) * row_size); - cb(conv_state_update, "conv_state_update", il); - - ggml_build_forward_expand(gf, ggml_cpy(ctx0, conv_state_last, conv_state_update)); - } else { - // [TAG_RECURRENT_ROLLBACK_SPLITS] - // this logic assumes that the last (n_rs_seq + 1) tokens of a sequence in a batch are inside - // the same ubatch, which `split_equal()` guarantees via its n_keep_tail argument - - const int64_t K = (int64_t) cparams.n_rs_seq + 1; - - for (int64_t t = 1; t <= K; ++t) { - const int64_t s_idx = std::max(0, conv_input->ne[0] - conv_states->ne[0] - K + t); - const int64_t s_slot = K - t; - - ggml_tensor * conv_state_last = - ggml_view_3d(ctx0, conv_input, - conv_kernel_size - 1, conv_channels, n_seqs, - conv_input->nb[1], conv_input->nb[2], - ggml_row_size(conv_input->type, s_idx)); - - ggml_tensor * conv_state_update = - ggml_view_2d(ctx0, - conv_states_all, row_count, n_seqs, - conv_states_all->nb[1], - (s_slot * mem_size + kv_head) * row_size); - - ggml_build_forward_expand(gf, ggml_cpy(ctx0, conv_state_last, conv_state_update)); - } - } + build_conv_state_store(inp, conv_states_all, conv_input, conv_kernel_size - 1, 0); return conv_input; } @@ -543,7 +98,7 @@ ggml_tensor * llm_build_delta_net_base::build_recurrent_attn( const bool keep = cparams.n_rs_seq > 0; if (!keep) { - auto attn_out = build_delta_net(q, k, v, g, b, s, il); + auto attn_out = build_delta_net(q, k, v, g, b, s); ggml_tensor * output = attn_out.first; ggml_tensor * new_state = attn_out.second; cb(output, "attn_output", il); @@ -562,12 +117,6 @@ ggml_tensor * llm_build_delta_net_base::build_recurrent_attn( // state s is 4D [S_v, S_v, H_v, n_seqs]; K snapshot slots are written into the output. ggml_tensor * gdn_out = ggml_gated_delta_net(ctx0, q, k, v, g, b, s, K); - if (n_seq_tokens > 1) { - res->add_fused_node({LLM_FUSED_OP_GDN_CH, gdn_out, il}); - } else { - res->add_fused_node({LLM_FUSED_OP_GDN_AR, gdn_out, il}); - } - const int64_t attn_score_elems = S_v * H_v * n_seq_tokens * n_seqs; const int64_t state_size_per_snap = S_v * S_v * H_v * n_seqs; diff --git a/src/models/dflash.cpp b/src/models/dflash.cpp index 209498421cd4..13020bb32ca2 100644 --- a/src/models/dflash.cpp +++ b/src/models/dflash.cpp @@ -419,7 +419,7 @@ ggml_tensor * llama_model_dflash::graph::build_inp_embd_enc() const { // DFlash Encoder: processes target model features through feature fusion layer template <> -llama_model_dflash::graph::graph(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) { +llama_model_dflash::graph::graph(const llm_graph_params & params) : llm_graph_context(params) { ggml_tensor * cur = build_inp_embd_enc(); cur = build_lora_mm(model.fc, cur, model.fc_s); @@ -434,6 +434,23 @@ llama_model_dflash::graph::graph(const llama_model & model, const llm_grap ggml_build_forward_expand(gf, cur); } +// target features of an injection batch: host embd rows, or rows of the target's layer input device buffer +static ggml_tensor * build_inp_target_features(llm_graph_context & g, llm_graph_input_embd & inp, int64_t n_embd_inp) { + if (g.ubatch.embd_row) { + GGML_ASSERT(g.layer_inp_src != nullptr && g.layer_inp_src->ne[0] == n_embd_inp); + + inp.embd_rows = ggml_new_tensor_1d(g.ctx0, GGML_TYPE_I32, g.n_tokens); + ggml_set_input(inp.embd_rows); + + return ggml_get_rows(g.ctx0, g.layer_inp_src, inp.embd_rows); + } + + inp.embd = ggml_new_tensor_2d(g.ctx0, GGML_TYPE_F32, n_embd_inp, g.n_tokens); + ggml_set_input(inp.embd); + + return inp.embd; +} + // DSpark (DFlash + Markov & Confidence head): Markov bias on the draft logits, chained per block position static void build_dspark_markov_head(llm_graph_context & g, const llama_model & model, ggml_tensor * tokens, ggml_tensor * hidden = nullptr) { ggml_context * ctx0 = g.ctx0; @@ -902,7 +919,7 @@ static void build_lilicorr_scores(llm_graph_context & g, const llama_model & mod // * embd batch -> fused target features: project + inject K/V into the cache. // * token batch -> noise-block diffusion: attend over [committed, MASK...] to generate draft tokens template <> -llama_model_dflash::graph::graph(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) { +llama_model_dflash::graph::graph(const llm_graph_params & params) : llm_graph_context(params) { const int64_t n_embd_inp = hparams.n_embd_inp_enc(); const int64_t n_embd_head = hparams.n_embd_head_v(); @@ -939,13 +956,10 @@ llama_model_dflash::graph::graph(const llama_model & model, const llm_gra // KV cache injection ASSERT_EMBD_OR_TOKEN(ubatch); - if (ubatch.embd) { + if (ubatch.embd || ubatch.embd_row) { auto inp = std::make_unique(n_embd_inp); - inp->embd = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_embd_inp, n_tokens); - ggml_set_input(inp->embd); - - ggml_tensor * inp_target = inp->embd; + ggml_tensor * inp_target = build_inp_target_features(*this, *inp, n_embd_inp); cb(inp_target, "inp_target_features", -1); res->add_input(std::move(inp)); @@ -1206,8 +1220,8 @@ llama_model_dflash::graph::graph(const llama_model & model, const llm_gra // DSV4 DSpark decoder, dual-mode by batch type (see the DFlash decoder above): // * embd batch -> project main_x through each stage's wkv and inject K into the ring cache // * token batch -> noise block through 3 full DSV4 stages (hc + MLA + MoE), markov + confidence heads -llama_model_dflash::graph_dsv4::graph_dsv4(const llama_model & model, const llm_graph_params & params) : - llama_model_deepseek4::graph(params) { +llama_model_dflash::graph_dsv4::graph_dsv4(const llm_graph_params & params) : + llama_model_deepseek4::graph(params, no_build{}) { const int64_t n_embd_inp = hparams.n_embd_inp_enc(); const int64_t n_embd_head = hparams.n_embd_head_k(); const int64_t n_embd_head_rope = hparams.n_rot(); @@ -1219,13 +1233,10 @@ llama_model_dflash::graph_dsv4::graph_dsv4(const llama_model & model, const llm_ // KV cache injection: fused target features from the encoder ASSERT_EMBD_OR_TOKEN(ubatch); - if (ubatch.embd) { + if (ubatch.embd || ubatch.embd_row) { auto inp = std::make_unique(n_embd_inp); - inp->embd = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_embd_inp, n_tokens); - ggml_set_input(inp->embd); - - ggml_tensor * inp_target = inp->embd; + ggml_tensor * inp_target = build_inp_target_features(*this, *inp, n_embd_inp); cb(inp_target, "inp_target_features", -1); res->add_input(std::move(inp)); @@ -1303,7 +1314,7 @@ llama_model_dflash::graph_dsv4::graph_dsv4(const llama_model & model, const llm_ cur = build_norm(cur, layer.attn_norm, nullptr, LLM_NORM_RMS, il); cb(cur, "attn_norm", il); - cur = build_attention(model, inp_attn, cur, inp_pos, il); + cur = build_attention(inp_attn, cur, inp_pos, il); inpL = build_hc_post(cur, residual, attn_hc.post, attn_hc.comb, il); cb(inpL, "hc_attn_post", il); @@ -1380,25 +1391,9 @@ llama_model_dflash::graph_dsv4::graph_dsv4(const llama_model & model, const llm_ // * embd batch -> writes every stage's kv_norm(wkv(main_proj(target features))) into its window ring // * token batch -> [anchor, noise...] block through the stages, single-pass mHC, no per-head q norm // window of a block = window of its first query, see n_swa -// TODO: move this to a new type of `llm_graph_input` and implement this in `set_input` [TAG_ATTN_MASK_CNV] -static void build_dspark_block_mask(llm_graph_context & g, llm_graph_input_attn_k_iswa * inp_attn) { - ggml_context * ctx0 = g.ctx0; - - ggml_tensor * m = inp_attn->self_attn_mask_swa.cnv; - const int64_t n_tokens = g.n_tokens; - const int64_t n_blocks = g.ubatch.n_seqs_unq; - const int64_t bd = n_blocks > 0 ? n_tokens / n_blocks : 0; - if (m && bd > 1 && n_tokens % n_blocks == 0 && m->ne[2] == 1 && m->ne[1] % bd == 0 && ggml_is_contiguous(m)) { - ggml_tensor * m4 = ggml_reshape_4d(ctx0, m, m->ne[0], bd, m->ne[1]/bd, m->ne[3]); - ggml_tensor * r0 = ggml_view_4d(ctx0, m4, m4->ne[0], 1, m4->ne[2], m4->ne[3], m4->nb[1], m4->nb[2], m4->nb[3], 0); - m4 = ggml_repeat(ctx0, r0, m4); - inp_attn->self_attn_mask_swa.cnv = ggml_reshape_4d(ctx0, m4, m->ne[0], m->ne[1], 1, m->ne[3]); - g.cb(inp_attn->self_attn_mask_swa.cnv, "dspark_block_mask", -1); - } -} -llama_model_dflash::graph_dsv41::graph_dsv41(const llama_model & model, const llm_graph_params & params) : - llama_model_deepseek41::graph(params) { +llama_model_dflash::graph_dsv41::graph_dsv41(const llm_graph_params & params) : + llama_model_deepseek41::graph(params, no_build{}) { const int64_t n_embd_inp = hparams.n_embd_inp_enc(); const int64_t n_embd_head = hparams.n_embd_head_k(); const int64_t n_embd_head_rope = hparams.n_rot(); @@ -1418,13 +1413,10 @@ llama_model_dflash::graph_dsv41::graph_dsv41(const llama_model & model, const ll }; ASSERT_EMBD_OR_TOKEN(ubatch); - if (ubatch.embd) { + if (ubatch.embd || ubatch.embd_row) { auto inp = std::make_unique(n_embd_inp); - inp->embd = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_embd_inp, n_tokens); - ggml_set_input(inp->embd); - - ggml_tensor * inp_target = inp->embd; + ggml_tensor * inp_target = build_inp_target_features(*this, *inp, n_embd_inp); cb(inp_target, "inp_target_features", -1); res->add_input(std::move(inp)); @@ -1488,7 +1480,8 @@ llama_model_dflash::graph_dsv41::graph_dsv41(const llama_model & model, const ll inpL = ggml_repeat_4d(ctx0, inpL, n_embd, hc, n_tokens, 1); cb(inpL, "hc_init", -1); - build_dspark_block_mask(*this, inp_attn); + // the whole block shares the mask row of its first query (the anchor) + inp_attn->self_attn_mask_swa.block_uniform = true; ggml_tensor * pre_prev = nullptr; @@ -1626,7 +1619,7 @@ llama_model_dflash::graph_dsv41::graph_dsv41(const llama_model & model, const ll // Mistral Large 4 DSpark, dual-mode like graph_dsv41: // - embd batch: write latent K of target_norm(fc(features)) to the cache, no attn norm // - token batch: decode [anchor, mask_1, ...] block, non-causal -llama_model_dflash::graph_mistral_large4::graph_mistral_large4(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) { +llama_model_dflash::graph_mistral_large4::graph_mistral_large4(const llm_graph_params & params) : llm_graph_context(params) { const int64_t n_embd_inp = hparams.n_embd_inp_enc(); const int64_t kv_lora_rank = hparams.n_lora_kv; const int64_t n_embd_head_k_mla = hparams.n_embd_head_k_mla(); @@ -1668,13 +1661,10 @@ llama_model_dflash::graph_mistral_large4::graph_mistral_large4(const llama_model }; ASSERT_EMBD_OR_TOKEN(ubatch); - if (ubatch.embd) { + if (ubatch.embd || ubatch.embd_row) { auto inp = std::make_unique(n_embd_inp); - inp->embd = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_embd_inp, n_tokens); - ggml_set_input(inp->embd); - - ggml_tensor * inp_target = inp->embd; + ggml_tensor * inp_target = build_inp_target_features(*this, *inp, n_embd_inp); cb(inp_target, "inp_target_features", -1); res->add_input(std::move(inp)); @@ -1738,7 +1728,8 @@ llama_model_dflash::graph_mistral_large4::graph_mistral_large4(const llama_model } cb(inpL, "inp_noise_embd", -1); - build_dspark_block_mask(*this, inp_attn); + // the whole block shares the mask row of its first query (the anchor) + inp_attn->self_attn_mask_swa.block_uniform = true; for (int il = 0; il < n_layer; ++il) { const auto & layer = model.layers[il]; @@ -1841,19 +1832,19 @@ llama_model_dflash::graph_mistral_large4::graph_mistral_large4(const llama_model std::unique_ptr llama_model_dflash::build_arch_graph(const llm_graph_params & params) const { switch (params.gtype) { case LLM_GRAPH_TYPE_ENCODER: - return std::make_unique>(*this, params); + return std::make_unique>(params); case LLM_GRAPH_TYPE_DEFAULT: case LLM_GRAPH_TYPE_DECODER: if (hparams.n_lora_kv > 0) { - return std::make_unique(*this, params); + return std::make_unique(params); } if (hparams.dsv4_hc_mult > 0 && hparams.dsv4_hc_single_pass) { - return std::make_unique(*this, params); + return std::make_unique(params); } if (hparams.dsv4_hc_mult > 0) { - return std::make_unique(*this, params); + return std::make_unique(params); } - return std::make_unique>(*this, params); + return std::make_unique>(params); default: GGML_ABORT("invalid graph type"); }; diff --git a/src/models/dots1.cpp b/src/models/dots1.cpp index f29f669c21f0..c2fa965608cb 100644 --- a/src/models/dots1.cpp +++ b/src/models/dots1.cpp @@ -68,10 +68,10 @@ void llama_model_dots1::load_arch_tensors(llama_model_loader &) { } std::unique_ptr llama_model_dots1::build_arch_graph(const llm_graph_params & params) const { - return std::make_unique(*this, params); + return std::make_unique(params); } -llama_model_dots1::graph::graph(const llama_model & model, const llm_graph_params & params) : +llama_model_dots1::graph::graph(const llm_graph_params & params) : llm_graph_context(params) { const int64_t n_embd_head = hparams.n_embd_head_v(); @@ -106,14 +106,12 @@ llama_model_dots1::graph::graph(const llama_model & model, const llm_graph_param Qcur = build_norm(Qcur, model.layers[il].attn_q_norm, NULL, LLM_NORM_RMS, il); cb(Qcur, "Qcur_normed", il); - Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); + Qcur = build_rope(Qcur, inp_pos, il); Kcur = build_norm(Kcur, model.layers[il].attn_k_norm, NULL, LLM_NORM_RMS, il); cb(Kcur, "Kcur_normed", il); - Kcur = ggml_rope_ext(ctx0, Kcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); + Kcur = build_rope(Kcur, inp_pos, il); cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); diff --git a/src/models/dots3note.cpp b/src/models/dots3note.cpp index df6fee7e4b50..a6fccd58f1d6 100644 --- a/src/models/dots3note.cpp +++ b/src/models/dots3note.cpp @@ -145,11 +145,19 @@ void llama_model_dots3note::load_arch_tensors(llama_model_loader & ml) { } } +llama_rope_params llama_model_dots3note::get_rope_params(const llama_cparams & cparams, int il) const { + auto res = llama_model::get_rope_params(cparams, il); + + res.freq_scale = cparams.rope_freq_scale; + + return res; +} + std::unique_ptr llama_model_dots3note::build_arch_graph(const llm_graph_params & params) const { - return std::make_unique(*this, params); + return std::make_unique(params); } -llama_model_dots3note::graph::graph(const llama_model & model, const llm_graph_params & params) : +llama_model_dots3note::graph::graph(const llm_graph_params & params) : llm_graph_context(params) { GGML_ASSERT(hparams.is_mla()); @@ -186,7 +194,6 @@ llama_model_dots3note::graph::graph(const llama_model & model, const llm_graph_p const int64_t n_embd_head_qk_nope = n_embd_head_k_mla - n_embd_head_qk_rope; const float kq_scale = 1.0f/sqrtf(float(n_embd_head_k_mla)); - const float freq_base_l = model.get_rope_freq_base(cparams, il); // norm cur = build_norm(inpL, model.layers[il].attn_norm, NULL, LLM_NORM_RMS, il); @@ -211,9 +218,7 @@ llama_model_dots3note::graph::graph(const llama_model & model, const llm_graph_p // {n_embd_indexer_head, n_indexer_head, n_tokens} indexer_q = ggml_reshape_3d(ctx0, indexer_q, n_embd_indexer_head, n_indexer_head, n_tokens); - indexer_q = ggml_rope_ext(ctx0, indexer_q, inp_pos, nullptr, n_rot, - LLAMA_ROPE_TYPE_NEOX, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); + indexer_q = build_rope_indexer(indexer_q, inp_pos, il); cb(indexer_q, "indexer_q", il); ggml_tensor * indexer_k = ggml_mul_mat(ctx0, model.layers[il].indexer_attn_k, cur); @@ -224,9 +229,7 @@ llama_model_dots3note::graph::graph(const llama_model & model, const llm_graph_p // {n_embd_indexer_head, 1, n_tokens} indexer_k = ggml_reshape_3d(ctx0, indexer_k, n_embd_indexer_head, 1, n_tokens); - indexer_k = ggml_rope_ext(ctx0, indexer_k, inp_pos, nullptr, n_rot, - LLAMA_ROPE_TYPE_NEOX, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); + indexer_k = build_rope_indexer(indexer_k, inp_pos, il); cb(indexer_k, "indexer_k", il); // perform Hadamard transform on indexer q and k @@ -256,7 +259,7 @@ llama_model_dots3note::graph::graph(const llama_model & model, const llm_graph_p ggml_tensor * indexer_score = nullptr; if (cparams.fused_lid) { - indexer_score = ggml_lightning_indexer(ctx0, indexer_q, indexer_k, indexer_weights, inp_attn->get_dsa()->self_attn_mask_lid.cnv); + indexer_score = ggml_lightning_indexer(ctx0, indexer_q, indexer_k, indexer_weights, inp_attn->get_dsa()->self_attn_mask_lid.raw); cb(indexer_score, "indexer_score", il); res->add_fused_node({LLM_FUSED_OP_LIGHTNING_INDEXER, indexer_score, il}); } else { @@ -286,7 +289,7 @@ llama_model_dots3note::graph::graph(const llama_model & model, const llm_graph_p indexer_score = ggml_cont(ctx0, ggml_permute(ctx0, indexer_score, 2, 1, 0, 3)); cb(indexer_score, "indexer_score", il); - ggml_tensor * indexer_kq_mask = inp_attn->get_dsa()->self_attn_mask_lid.cnv; + ggml_tensor * indexer_kq_mask = inp_attn->get_dsa()->self_attn_mask_lid.raw; indexer_score = ggml_add(ctx0, indexer_score, indexer_kq_mask); cb(indexer_score, "indexer_score", il); } @@ -332,12 +335,10 @@ llama_model_dots3note::graph::graph(const llama_model & model, const llm_graph_p k_pe = build_norm(k_pe, model.layers[il].attn_k_norm, nullptr, LLM_NORM_RMS, il); cb(k_pe, "k_pe", il); - q_pe = ggml_rope_ext(ctx0, q_pe, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base_l, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); + q_pe = build_rope(q_pe, inp_pos, il); cb(q_pe, "q_pe", il); - k_pe = ggml_rope_ext(ctx0, k_pe, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base_l, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); + k_pe = build_rope(k_pe, inp_pos, il); cb(k_pe, "k_pe", il); kv_cmpr = build_norm(kv_cmpr, model.layers[il].attn_kv_a_norm, nullptr, LLM_NORM_RMS, il); diff --git a/src/models/dream.cpp b/src/models/dream.cpp index 7d5ffa56a59a..76c0b03c86a5 100644 --- a/src/models/dream.cpp +++ b/src/models/dream.cpp @@ -46,10 +46,10 @@ void llama_model_dream::load_arch_tensors(llama_model_loader &) { } std::unique_ptr llama_model_dream::build_arch_graph(const llm_graph_params & params) const { - return std::make_unique(*this, params); + return std::make_unique(params); } -llama_model_dream::graph::graph(const llama_model & model, const llm_graph_params & params) : +llama_model_dream::graph::graph(const llm_graph_params & params) : llm_graph_context(params) { //copied from qwen2 const int64_t n_embd_head = hparams.n_embd_head_v(); @@ -81,11 +81,8 @@ llama_model_dream::graph::graph(const llama_model & model, const llm_graph_param auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur, n_embd_head, n_head, n_head_kv, il); - Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); - - Kcur = ggml_rope_ext(ctx0, Kcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); + Qcur = build_rope(Qcur, inp_pos, il); + Kcur = build_rope(Kcur, inp_pos, il); cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); diff --git a/src/models/eagle3.cpp b/src/models/eagle3.cpp index d47340759009..990b9fe5de0c 100644 --- a/src/models/eagle3.cpp +++ b/src/models/eagle3.cpp @@ -122,7 +122,7 @@ ggml_tensor * llama_model_eagle3::graph::build_inp_embd_enc() const { // Input: target_features e.g. [12288, n_tokens] from target model layers low, middle, high // Output: g_embeddings e.g. [4096, n_tokens] stored in context template <> -llama_model_eagle3::graph::graph(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) { +llama_model_eagle3::graph::graph(const llm_graph_params & params) : llm_graph_context(params) { ggml_tensor * cur = nullptr; cur = build_inp_embd_enc(); @@ -149,7 +149,7 @@ llama_model_eagle3::graph::graph(const llama_model & model, const llm_grap // Input: draft tokens + g_embeddings from encoder // Output: draft logits template <> -llama_model_eagle3::graph::graph(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) { +llama_model_eagle3::graph::graph(const llm_graph_params & params) : llm_graph_context(params) { const int64_t n_embd_head = hparams.n_embd_head_v(); GGML_ASSERT(n_embd_head == hparams.n_embd_head_k()); @@ -234,20 +234,9 @@ llama_model_eagle3::graph::graph(const llama_model & model, const llm_gra Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens); Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens); - // rope freq factors, returns nullptr if not available - ggml_tensor * rope_factors = model.get_rope_factors(cparams, il); - // RoPE - Qcur = ggml_rope_ext( - ctx0, Qcur, inp_pos, rope_factors, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); - Kcur = ggml_rope_ext( - ctx0, Kcur, inp_pos, rope_factors, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); + Qcur = build_rope(Qcur, inp_pos, il); + Kcur = build_rope(Kcur, inp_pos, il); cb(Qcur, "Qcur_rope", il); cb(Kcur, "Kcur_rope", il); @@ -326,10 +315,10 @@ llama_model_eagle3::graph::graph(const llama_model & model, const llm_gra std::unique_ptr llama_model_eagle3::build_arch_graph(const llm_graph_params & params) const { switch (params.gtype) { case LLM_GRAPH_TYPE_ENCODER: - return std::make_unique>(*this, params); + return std::make_unique>(params); case LLM_GRAPH_TYPE_DEFAULT: case LLM_GRAPH_TYPE_DECODER: - return std::make_unique>(*this, params); + return std::make_unique>(params); default: GGML_ABORT("invalid graph type"); }; diff --git a/src/models/ernie4-5-moe.cpp b/src/models/ernie4-5-moe.cpp index 6bf539d21331..7edf9f508bcc 100644 --- a/src/models/ernie4-5-moe.cpp +++ b/src/models/ernie4-5-moe.cpp @@ -1,10 +1,10 @@ #include "models.h" std::unique_ptr llama_model_ernie4_5_moe::build_arch_graph(const llm_graph_params & params) const { - return std::make_unique(*this, params); + return std::make_unique(params); } -llama_model_ernie4_5_moe::graph::graph(const llama_model & model, const llm_graph_params & params) : +llama_model_ernie4_5_moe::graph::graph(const llm_graph_params & params) : llm_graph_context(params) { const int64_t n_embd_head = hparams.n_embd_head_v(); @@ -37,11 +37,8 @@ llama_model_ernie4_5_moe::graph::graph(const llama_model & model, const llm_grap auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur, n_embd_head, n_head, n_head_kv, il); - Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); - - Kcur = ggml_rope_ext(ctx0, Kcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); + Qcur = build_rope(Qcur, inp_pos, il); + Kcur = build_rope(Kcur, inp_pos, il); cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); diff --git a/src/models/ernie4-5.cpp b/src/models/ernie4-5.cpp index b7a6e9629c6c..932cc74cefde 100644 --- a/src/models/ernie4-5.cpp +++ b/src/models/ernie4-5.cpp @@ -70,10 +70,10 @@ void llama_model_ernie4_5::load_arch_tensors(llama_model_loader &) { } std::unique_ptr llama_model_ernie4_5::build_arch_graph(const llm_graph_params & params) const { - return std::make_unique(*this, params); + return std::make_unique(params); } -llama_model_ernie4_5::graph::graph(const llama_model & model, const llm_graph_params & params) : +llama_model_ernie4_5::graph::graph(const llm_graph_params & params) : llm_graph_context(params) { const int64_t n_embd_head = hparams.n_embd_head_v(); @@ -105,11 +105,8 @@ llama_model_ernie4_5::graph::graph(const llama_model & model, const llm_graph_pa auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur, n_embd_head, n_head, n_head_kv, il); - Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); - - Kcur = ggml_rope_ext(ctx0, Kcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); + Qcur = build_rope(Qcur, inp_pos, il); + Kcur = build_rope(Kcur, inp_pos, il); cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); diff --git a/src/models/eurobert.cpp b/src/models/eurobert.cpp index 13bae8b71608..7a42f0dbf2e1 100644 --- a/src/models/eurobert.cpp +++ b/src/models/eurobert.cpp @@ -32,10 +32,10 @@ void llama_model_eurobert::load_arch_tensors(llama_model_loader &) { } std::unique_ptr llama_model_eurobert::build_arch_graph(const llm_graph_params & params) const { - return std::make_unique(*this, params); + return std::make_unique(params); } -llama_model_eurobert::graph::graph(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) { +llama_model_eurobert::graph::graph(const llm_graph_params & params) : llm_graph_context(params) { const int64_t n_embd_head = hparams.n_embd_head_v(); GGML_ASSERT(n_embd_head == hparams.n_embd_head_k()); @@ -62,17 +62,8 @@ llama_model_eurobert::graph::graph(const llama_model & model, const llm_graph_pa auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur, n_embd_head, n_head, n_head_kv, il); - Qcur = ggml_rope_ext( - ctx0, Qcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); - - Kcur = ggml_rope_ext( - ctx0, Kcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); + Qcur = build_rope(Qcur, inp_pos, il); + Kcur = build_rope(Kcur, inp_pos, il); cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); diff --git a/src/models/exaone-moe.cpp b/src/models/exaone-moe.cpp index 1a09d544d539..15bbd5a3f6e8 100644 --- a/src/models/exaone-moe.cpp +++ b/src/models/exaone-moe.cpp @@ -4,6 +4,7 @@ void llama_model_exaone_moe::load_arch_hparams(llama_model_loader & ml) { hparams.swa_type = LLAMA_SWA_TYPE_STANDARD; hparams.n_swa = 128; load_swa_pattern(ml, 4); + hparams.rope_pattern = hparams.is_swa_impl; hparams.rope_freq_base_train_swa = hparams.rope_freq_base_train; hparams.rope_freq_scale_train_swa = hparams.rope_freq_scale_train; @@ -101,11 +102,21 @@ void llama_model_exaone_moe::load_arch_tensors(llama_model_loader &) { } } +llama_rope_params llama_model_exaone_moe::get_rope_params(const llama_cparams & cparams, int il) const { + auto res = llama_model::get_rope_params(cparams, il); + + // the SWA layers use the global base and scale + res.freq_base = cparams.rope_freq_base; + res.freq_scale = cparams.rope_freq_scale; + + return res; +} + std::unique_ptr llama_model_exaone_moe::build_arch_graph(const llm_graph_params & params) const { - return std::make_unique(*this, params); + return std::make_unique(params); } -llama_model_exaone_moe::graph::graph(const llama_model & model, const llm_graph_params & params) : +llama_model_exaone_moe::graph::graph(const llm_graph_params & params) : llm_graph_context(params) { const int64_t n_embd_head = hparams.n_embd_head_k(); @@ -127,17 +138,12 @@ llama_model_exaone_moe::graph::graph(const llama_model & model, const llm_graph_ for (int il = 0; il < n_layer; ++il) { ggml_tensor * inpSA = inpL; - // use RoPE for SWA layers - const bool is_local_layer = hparams.is_swa(il); - // norm cur = build_norm(inpL, model.layers[il].attn_norm, NULL, LLM_NORM_RMS, il); cb(cur, "attn_norm", il); // self-attention { - ggml_tensor * rope_factors = model.get_rope_factors(cparams, il); - // compute Q and K and RoPE them auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur, n_embd_head, n_head, n_head_kv, il); @@ -147,13 +153,8 @@ llama_model_exaone_moe::graph::graph(const llama_model & model, const llm_graph_ cb(Qcur, "Qcur_normed", il); cb(Kcur, "Kcur_normed", il); - if (is_local_layer) { - Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, rope_factors, n_rot, rope_type, n_ctx_orig, freq_base, - freq_scale, ext_factor, attn_factor, beta_fast, beta_slow); - - Kcur = ggml_rope_ext(ctx0, Kcur, inp_pos, rope_factors, n_rot, rope_type, n_ctx_orig, freq_base, - freq_scale, ext_factor, attn_factor, beta_fast, beta_slow); - } + Qcur = build_rope(Qcur, inp_pos, il); + Kcur = build_rope(Kcur, inp_pos, il); cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); cb(Vcur, "Vcur", il); diff --git a/src/models/exaone.cpp b/src/models/exaone.cpp index 070bfd568eea..780f34f04571 100644 --- a/src/models/exaone.cpp +++ b/src/models/exaone.cpp @@ -40,10 +40,10 @@ void llama_model_exaone::load_arch_tensors(llama_model_loader &) { } std::unique_ptr llama_model_exaone::build_arch_graph(const llm_graph_params & params) const { - return std::make_unique(*this, params); + return std::make_unique(params); } -llama_model_exaone::graph::graph(const llama_model & model, const llm_graph_params & params) : +llama_model_exaone::graph::graph(const llm_graph_params & params) : llm_graph_context(params) { const int64_t n_embd_head = hparams.n_embd_head_v(); @@ -71,18 +71,12 @@ llama_model_exaone::graph::graph(const llama_model & model, const llm_graph_para // self-attention { - // rope freq factors for llama3; may return nullptr for llama2 and other models - ggml_tensor * rope_factors = model.get_rope_factors(cparams, il); - // compute Q and K and RoPE them auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur, n_embd_head, n_head, n_head_kv, il); - Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, rope_factors, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); - - Kcur = ggml_rope_ext(ctx0, Kcur, inp_pos, rope_factors, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); + Qcur = build_rope(Qcur, inp_pos, il); + Kcur = build_rope(Kcur, inp_pos, il); cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); diff --git a/src/models/exaone4.cpp b/src/models/exaone4.cpp index 80447908e142..bca08e0616ca 100644 --- a/src/models/exaone4.cpp +++ b/src/models/exaone4.cpp @@ -5,6 +5,7 @@ void llama_model_exaone4::load_arch_hparams(llama_model_loader & ml) { hparams.swa_type = LLAMA_SWA_TYPE_STANDARD; hparams.n_swa = 4096; load_swa_pattern(ml, 4); + hparams.rope_pattern = hparams.is_swa_impl; hparams.rope_freq_base_train_swa = hparams.rope_freq_base_train; hparams.rope_freq_scale_train_swa = hparams.rope_freq_scale_train; @@ -70,16 +71,26 @@ void llama_model_exaone4::load_arch_tensors(llama_model_loader &) { } } +llama_rope_params llama_model_exaone4::get_rope_params(const llama_cparams & cparams, int il) const { + auto res = llama_model::get_rope_params(cparams, il); + + // the SWA layers use the global base and scale + res.freq_base = cparams.rope_freq_base; + res.freq_scale = cparams.rope_freq_scale; + + return res; +} + std::unique_ptr llama_model_exaone4::build_arch_graph(const llm_graph_params & params) const { if (hparams.swa_type == LLAMA_SWA_TYPE_STANDARD) { - return std::make_unique>(*this, params); + return std::make_unique>(params); } else { - return std::make_unique>(*this, params); + return std::make_unique>(params); } } template -llama_model_exaone4::graph::graph(const llama_model & model, const llm_graph_params & params) : +llama_model_exaone4::graph::graph(const llm_graph_params & params) : llm_graph_context(params) { const int64_t n_embd_head = hparams.n_embd_head_k(); @@ -108,13 +119,11 @@ llama_model_exaone4::graph::graph(const llama_model & model, const llm_gra ggml_tensor * inpSA = inpL; // use RoPE for SWA layers or non-SWA models - const bool use_rope = hparams.is_swa(il) || hparams.swa_type == LLAMA_SWA_TYPE_NONE; cur = inpL; // self-attention { - ggml_tensor * rope_factors = model.get_rope_factors(cparams, il); auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur, n_embd_head, n_head, n_head_kv, il); @@ -124,13 +133,8 @@ llama_model_exaone4::graph::graph(const llama_model & model, const llm_gra cb(Qcur, "Qcur_normed", il); cb(Kcur, "Kcur_normed", il); - if (use_rope) { - Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, rope_factors, n_rot, rope_type, n_ctx_orig, freq_base, - freq_scale, ext_factor, attn_factor, beta_fast, beta_slow); - - Kcur = ggml_rope_ext(ctx0, Kcur, inp_pos, rope_factors, n_rot, rope_type, n_ctx_orig, freq_base, - freq_scale, ext_factor, attn_factor, beta_fast, beta_slow); - } + Qcur = build_rope(Qcur, inp_pos, il); + Kcur = build_rope(Kcur, inp_pos, il); cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); cb(Vcur, "Vcur", il); diff --git a/src/models/falcon-h1.cpp b/src/models/falcon-h1.cpp index a7f9e55765f2..22c677b475f1 100644 --- a/src/models/falcon-h1.cpp +++ b/src/models/falcon-h1.cpp @@ -105,11 +105,19 @@ void llama_model_falcon_h1::load_arch_tensors(llama_model_loader &) { } } +llama_rope_params llama_model_falcon_h1::get_rope_params(const llama_cparams & cparams, int il) const { + auto res = llama_model::get_rope_params(cparams, il); + + res.factors = nullptr; + + return res; +} + std::unique_ptr llama_model_falcon_h1::build_arch_graph(const llm_graph_params & params) const { - return std::make_unique(*this, params); + return std::make_unique(params); } -llama_model_falcon_h1::graph::graph(const llama_model & model, const llm_graph_params & params) : +llama_model_falcon_h1::graph::graph(const llm_graph_params & params) : llm_build_mamba_base(params) { const int64_t n_embd_head = hparams.n_embd_head_v(); @@ -139,11 +147,8 @@ llama_model_falcon_h1::graph::graph(const llama_model & model, const llm_graph_p auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur, n_embd_head, n_head, n_head_kv, il); - Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, nullptr, n_rot, hparams.rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); - - Kcur = ggml_rope_ext(ctx0, Kcur, inp_pos, nullptr, n_rot, hparams.rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); + Qcur = build_rope(Qcur, inp_pos, il); + Kcur = build_rope(Kcur, inp_pos, il); cb(Qcur, "Qcur-post-rope", il); cb(Kcur, "Kcur-post-rope", il); @@ -158,7 +163,7 @@ llama_model_falcon_h1::graph::graph(const llama_model & model, const llm_graph_p // Mamba2 layer cb(cur, "ssm_in", il); - ggml_tensor * ssm_out = build_mamba2_layer(inp->get_recr(), cur, model, ubatch, il); + ggml_tensor * ssm_out = build_mamba2_layer(inp->get_recr(), cur, ubatch, il); cb(ssm_out, "ssm_out", il); // // Aggregation diff --git a/src/models/falcon.cpp b/src/models/falcon.cpp index 555468a47532..6269e633c150 100644 --- a/src/models/falcon.cpp +++ b/src/models/falcon.cpp @@ -44,10 +44,10 @@ void llama_model_falcon::load_arch_tensors(llama_model_loader &) { } std::unique_ptr llama_model_falcon::build_arch_graph(const llm_graph_params & params) const { - return std::make_unique(*this, params); + return std::make_unique(params); } -llama_model_falcon::graph::graph(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) { +llama_model_falcon::graph::graph(const llm_graph_params & params) : llm_graph_context(params) { const int64_t n_embd_head = hparams.n_embd_head_v(); GGML_ASSERT(n_embd_head == hparams.n_embd_head_k()); @@ -91,17 +91,8 @@ llama_model_falcon::graph::graph(const llama_model & model, const llm_graph_para n_embd_head, n_head, n_head_kv, il); // using mode = 2 for neox mode - Qcur = ggml_rope_ext( - ctx0, Qcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); - - Kcur = ggml_rope_ext( - ctx0, Kcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); + Qcur = build_rope(Qcur, inp_pos, il); + Kcur = build_rope(Kcur, inp_pos, il); cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); diff --git a/src/models/flint.cpp b/src/models/flint.cpp index e7e6f6b792df..520334ce656f 100644 --- a/src/models/flint.cpp +++ b/src/models/flint.cpp @@ -70,11 +70,21 @@ void llama_model_flint::load_arch_tensors(llama_model_loader &) { } } +llama_rope_params llama_model_flint::get_rope_params(const llama_cparams & cparams, int il) const { + auto res = llama_model::get_rope_params(cparams, il); + + // the SWA layers use the global base and scale + res.freq_base = cparams.rope_freq_base; + res.freq_scale = cparams.rope_freq_scale; + + return res; +} + std::unique_ptr llama_model_flint::build_arch_graph(const llm_graph_params & params) const { - return std::make_unique(*this, params); + return std::make_unique(params); } -llama_model_flint::graph::graph(const llama_model & model, const llm_graph_params & params) : +llama_model_flint::graph::graph(const llm_graph_params & params) : llm_graph_context(params) { const int64_t n_embd_head = hparams.n_embd_head_v(); @@ -109,12 +119,8 @@ llama_model_flint::graph::graph(const llama_model & model, const llm_graph_param Qcur = build_norm(Qcur, model.layers[il].attn_q_norm, nullptr, LLM_NORM_RMS, il); Kcur = build_norm(Kcur, model.layers[il].attn_k_norm, nullptr, LLM_NORM_RMS, il); - if (hparams.has_rope(il)) { - Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, - freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow); - Kcur = ggml_rope_ext(ctx0, Kcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, - freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow); - } + Qcur = build_rope(Qcur, inp_pos, il); + Kcur = build_rope(Kcur, inp_pos, il); cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); cb(Vcur, "Vcur", il); diff --git a/src/models/gemma-embedding.cpp b/src/models/gemma-embedding.cpp index bb4e964e81ec..aa43910be0c0 100644 --- a/src/models/gemma-embedding.cpp +++ b/src/models/gemma-embedding.cpp @@ -67,10 +67,10 @@ void llama_model_gemma_embedding::load_arch_tensors(llama_model_loader &) { } std::unique_ptr llama_model_gemma_embedding::build_arch_graph(const llm_graph_params & params) const { - return std::make_unique(*this, params); + return std::make_unique(params); } -llama_model_gemma_embedding::graph::graph(const llama_model & model, const llm_graph_params & params) : +llama_model_gemma_embedding::graph::graph(const llm_graph_params & params) : llm_graph_context(params) { const int64_t n_embd_head = hparams.n_embd_head_k(); @@ -89,8 +89,6 @@ llama_model_gemma_embedding::graph::graph(const llama_model & model, const llm_g ggml_tensor * inp_out_ids = build_inp_out_ids(); for (int il = 0; il < n_layer; ++il) { - const float freq_base_l = model.get_rope_freq_base(cparams, il); - const float freq_scale_l = model.get_rope_freq_scale(cparams, il); // norm cur = build_norm(inpL, model.layers[il].attn_norm, NULL, LLM_NORM_RMS, il); @@ -105,14 +103,12 @@ llama_model_gemma_embedding::graph::graph(const llama_model & model, const llm_g Qcur = build_norm(Qcur, model.layers[il].attn_q_norm, NULL, LLM_NORM_RMS, il); cb(Qcur, "Qcur_normed", il); - Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base_l, freq_scale_l, - ext_factor, attn_factor, beta_fast, beta_slow); + Qcur = build_rope(Qcur, inp_pos, il); Kcur = build_norm(Kcur, model.layers[il].attn_k_norm, NULL, LLM_NORM_RMS, il); cb(Kcur, "Kcur_normed", il); - Kcur = ggml_rope_ext(ctx0, Kcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base_l, freq_scale_l, - ext_factor, attn_factor, beta_fast, beta_slow); + Kcur = build_rope(Kcur, inp_pos, il); cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); diff --git a/src/models/gemma-embedding2.cpp b/src/models/gemma-embedding2.cpp index 2c7fa6750941..920827dc58d8 100644 --- a/src/models/gemma-embedding2.cpp +++ b/src/models/gemma-embedding2.cpp @@ -73,12 +73,11 @@ void llama_model_gemma_embedding2::load_arch_tensors(llama_model_loader &) { } std::unique_ptr llama_model_gemma_embedding2::build_arch_graph(const llm_graph_params & params) const { - return std::make_unique(*this, params); + return std::make_unique(params); } -llama_model_gemma_embedding2::graph::graph(const llama_model & model, const llm_graph_params & params) : - llm_graph_context(params), - model(model) { +llama_model_gemma_embedding2::graph::graph(const llm_graph_params & params) : + llm_graph_context(params) { ggml_tensor * cur; ggml_tensor * inpL; @@ -101,10 +100,6 @@ llama_model_gemma_embedding2::graph::graph(const llama_model & model, const llm_ const int64_t n_head = hparams.n_head(il); const int64_t n_head_kv = hparams.n_head_kv(il); - const float freq_base_l = model.get_rope_freq_base(cparams, il); - const float freq_scale_l = model.get_rope_freq_scale(cparams, il); - const int n_rot_l = hparams.n_rot(il); - // norm cur = build_norm(inpL, model.layers[il].attn_norm, nullptr, LLM_NORM_RMS, il); cb(cur, "attn_norm", il); @@ -129,10 +124,8 @@ llama_model_gemma_embedding2::graph::graph(const llama_model & model, const llm_ cb(Kcur, "Kcur_normed", il); cb(Vcur, "Vcur_normed", il); - Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, nullptr, n_rot_l, rope_type, n_ctx_orig, freq_base_l, freq_scale_l, - ext_factor, attn_factor, beta_fast, beta_slow); - Kcur = ggml_rope_ext(ctx0, Kcur, inp_pos, nullptr, n_rot_l, rope_type, n_ctx_orig, freq_base_l, freq_scale_l, - ext_factor, attn_factor, beta_fast, beta_slow); + Qcur = build_rope(Qcur, inp_pos, il); + Kcur = build_rope(Kcur, inp_pos, il); cb(Qcur, "Qcur_pos", il); cb(Kcur, "Kcur_pos", il); diff --git a/src/models/gemma.cpp b/src/models/gemma.cpp index 1da4ad815033..1d42acd0127c 100644 --- a/src/models/gemma.cpp +++ b/src/models/gemma.cpp @@ -35,10 +35,10 @@ void llama_model_gemma::load_arch_tensors(llama_model_loader &) { } std::unique_ptr llama_model_gemma::build_arch_graph(const llm_graph_params & params) const { - return std::make_unique(*this, params); + return std::make_unique(params); } -llama_model_gemma::graph::graph(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) { +llama_model_gemma::graph::graph(const llm_graph_params & params) : llm_graph_context(params) { const int64_t n_embd_head = hparams.n_embd_head_v(); ggml_tensor * cur; @@ -69,15 +69,8 @@ llama_model_gemma::graph::graph(const llama_model & model, const llm_graph_param auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur, n_embd_head, n_head, n_head_kv, il); - Qcur = ggml_rope_ext( - ctx0, Qcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); - - Kcur = ggml_rope_ext( - ctx0, Kcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); + Qcur = build_rope(Qcur, inp_pos, il); + Kcur = build_rope(Kcur, inp_pos, il); cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); diff --git a/src/models/gemma2.cpp b/src/models/gemma2.cpp index c405e6400f3a..826396ce1a59 100644 --- a/src/models/gemma2.cpp +++ b/src/models/gemma2.cpp @@ -54,10 +54,10 @@ void llama_model_gemma2::load_arch_tensors(llama_model_loader &) { } std::unique_ptr llama_model_gemma2::build_arch_graph(const llm_graph_params & params) const { - return std::make_unique(*this, params); + return std::make_unique(params); } -llama_model_gemma2::graph::graph(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) { +llama_model_gemma2::graph::graph(const llm_graph_params & params) : llm_graph_context(params) { const int64_t n_embd_head = hparams.n_embd_head_k(); ggml_tensor * cur; @@ -76,8 +76,6 @@ llama_model_gemma2::graph::graph(const llama_model & model, const llm_graph_para ggml_tensor * inp_out_ids = build_inp_out_ids(); for (int il = 0; il < n_layer; ++il) { - const float freq_base_l = model.get_rope_freq_base (cparams, il); - const float freq_scale_l = model.get_rope_freq_scale(cparams, il); // norm cur = build_norm(inpL, @@ -91,15 +89,8 @@ llama_model_gemma2::graph::graph(const llama_model & model, const llm_graph_para auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur, n_embd_head, n_head, n_head_kv, il); - Qcur = ggml_rope_ext( - ctx0, Qcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base_l, freq_scale_l, - ext_factor, attn_factor, beta_fast, beta_slow); - - Kcur = ggml_rope_ext( - ctx0, Kcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base_l, freq_scale_l, - ext_factor, attn_factor, beta_fast, beta_slow); + Qcur = build_rope(Qcur, inp_pos, il); + Kcur = build_rope(Kcur, inp_pos, il); cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); diff --git a/src/models/gemma3.cpp b/src/models/gemma3.cpp index 4c8183cce904..9520c3fcfc72 100644 --- a/src/models/gemma3.cpp +++ b/src/models/gemma3.cpp @@ -72,14 +72,14 @@ void llama_model_gemma3::load_arch_tensors(llama_model_loader &) { std::unique_ptr llama_model_gemma3::build_arch_graph(const llm_graph_params & params) const { if (hparams.swa_type == LLAMA_SWA_TYPE_STANDARD) { - return std::make_unique>(*this, params); + return std::make_unique>(params); } else { - return std::make_unique>(*this, params); + return std::make_unique>(params); } } template -llama_model_gemma3::graph::graph(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) { +llama_model_gemma3::graph::graph(const llm_graph_params & params) : llm_graph_context(params) { const int64_t n_embd_head = hparams.n_embd_head_k(); ggml_tensor * cur; @@ -105,17 +105,6 @@ llama_model_gemma3::graph::graph(const llama_model & model, const llm_grap ggml_tensor * inp_out_ids = build_inp_out_ids(); for (int il = 0; il < n_layer; ++il) { - float freq_base_l = 0.0f; - float freq_scale_l = 0.0f; - - if constexpr (iswa) { - freq_base_l = model.get_rope_freq_base (cparams, il); - freq_scale_l = model.get_rope_freq_scale(cparams, il); - } else { - freq_base_l = freq_base; - freq_scale_l = freq_scale; - } - // norm cur = build_norm(inpL, model.layers[il].attn_norm, NULL, LLM_NORM_RMS, il); cb(cur, "attn_norm", il); @@ -129,18 +118,12 @@ llama_model_gemma3::graph::graph(const llama_model & model, const llm_grap Qcur = build_norm(Qcur, model.layers[il].attn_q_norm, NULL, LLM_NORM_RMS, il); cb(Qcur, "Qcur_normed", il); - Qcur = ggml_rope_ext( - ctx0, Qcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base_l, freq_scale_l, - ext_factor, attn_factor, beta_fast, beta_slow); + Qcur = build_rope(Qcur, inp_pos, il); Kcur = build_norm(Kcur, model.layers[il].attn_k_norm, NULL, LLM_NORM_RMS, il); cb(Kcur, "Kcur_normed", il); - Kcur = ggml_rope_ext( - ctx0, Kcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base_l, freq_scale_l, - ext_factor, attn_factor, beta_fast, beta_slow); + Kcur = build_rope(Kcur, inp_pos, il); cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); diff --git a/src/models/gemma3n.cpp b/src/models/gemma3n.cpp index 7f12b1747547..64809def800c 100644 --- a/src/models/gemma3n.cpp +++ b/src/models/gemma3n.cpp @@ -81,7 +81,7 @@ void llama_model_gemma3n::load_arch_tensors(llama_model_loader &) { } std::unique_ptr llama_model_gemma3n::build_arch_graph(const llm_graph_params & params) const { - return std::make_unique(*this, params); + return std::make_unique(params); } // get 2D slice view from a 3D tensor, the idx corresponds to the 3rd dim @@ -91,9 +91,8 @@ static ggml_tensor * gemma3n_view_2d_slice(ggml_context * ctx0, ggml_tensor * x, idx * x->ne[0] * x->ne[1] * ggml_element_size(x)); } -llama_model_gemma3n::graph::graph(const llama_model & model, const llm_graph_params & params) : +llama_model_gemma3n::graph::graph(const llm_graph_params & params) : llm_graph_context(params), - model(model), n_embd_head(model.hparams.n_embd_head_k()), n_embd_altup(model.hparams.n_embd_altup), n_altup(model.hparams.n_altup), @@ -133,8 +132,6 @@ llama_model_gemma3n::graph::graph(const llama_model & model, const llm_graph_par for (int il = 0; il < n_layer; ++il) { // this block is made to be closely resemble Gemma3p5DecoderLayer on python code - const float freq_base_l = model.get_rope_freq_base(cparams, il); - const float freq_scale_l = model.get_rope_freq_scale(cparams, il); ggml_tensor * cur = inpL; // [n_embd, n_tokens, n_altup] ggml_tensor * predictions = altup_predict(cur, il); // [n_embd, n_tokens, n_altup] @@ -163,11 +160,8 @@ llama_model_gemma3n::graph::graph(const llama_model & model, const llm_graph_par cb(Kcur, "Kcur_normed", il); cb(Vcur, "Vcur_normed", il); - Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base_l, freq_scale_l, - ext_factor, attn_factor, beta_fast, beta_slow); - - Kcur = ggml_rope_ext(ctx0, Kcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base_l, freq_scale_l, - ext_factor, attn_factor, beta_fast, beta_slow); + Qcur = build_rope(Qcur, inp_pos, il); + Kcur = build_rope(Kcur, inp_pos, il); cb(Qcur, "Qcur_pos", il); cb(Kcur, "Kcur_pos", il); @@ -191,8 +185,7 @@ llama_model_gemma3n::graph::graph(const llama_model & model, const llm_graph_par Qcur = build_norm(Qcur, model.layers[il].attn_q_norm, NULL, LLM_NORM_RMS, il); cb(Qcur, "Qcur_normed", il); - Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base_l, freq_scale_l, - ext_factor, attn_factor, beta_fast, beta_slow); + Qcur = build_rope(Qcur, inp_pos, il); cb(Qcur, "Qcur_pos", il); cur = build_attn(inp_attn, diff --git a/src/models/gemma4-assistant.cpp b/src/models/gemma4-assistant.cpp index 6ebc3c3e8e35..48b7aa864aca 100644 --- a/src/models/gemma4-assistant.cpp +++ b/src/models/gemma4-assistant.cpp @@ -78,10 +78,10 @@ void llama_model_gemma4_assistant::load_arch_tensors(llama_model_loader &) { } std::unique_ptr llama_model_gemma4_assistant::build_arch_graph(const llm_graph_params & params) const { - return std::make_unique(*this, params); + return std::make_unique(params); } -llama_model_gemma4_assistant::graph::graph(const llama_model & model, const llm_graph_params & params) : +llama_model_gemma4_assistant::graph::graph(const llm_graph_params & params) : llm_graph_context(params) { const int64_t n_embd_backbone = hparams.n_embd_inp(); @@ -103,9 +103,8 @@ llama_model_gemma4_assistant::graph::graph(const llama_model & model, const llm_ inp_embd = inp->embd; res->t_inp_embd = inp->embd; - inp->h = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_embd_backbone, ubatch.n_tokens); + inp->h = build_inp_embd_state(inp.get(), n_embd_backbone); cb(inp->h, "inp_h", -1); - ggml_set_input(inp->h); inp_h = inp->h; res->add_input(std::move(inp)); @@ -131,15 +130,10 @@ llama_model_gemma4_assistant::graph::graph(const llama_model & model, const llm_ ggml_tensor * inpL = cur; for (int il = 0; il < n_layer_nextn; ++il) { - const bool is_swa = hparams.is_swa(il); const int64_t n_embd_head = hparams.n_embd_head_k(il); const int64_t n_head = hparams.n_head(il); - const float freq_base_l = model.get_rope_freq_base(cparams, il); - const float freq_scale_l = model.get_rope_freq_scale(cparams, il); - const int n_rot_l = hparams.n_rot(il); - ggml_tensor * cur_norm = build_norm(inpL, model.layers[il].attn_norm, nullptr, LLM_NORM_RMS, il); cb(cur_norm, "attn_norm", il); @@ -148,9 +142,7 @@ llama_model_gemma4_assistant::graph::graph(const llama_model & model, const llm_ Qcur = build_norm(Qcur, model.layers[il].attn_q_norm, nullptr, LLM_NORM_RMS, il); cb(Qcur, "Qcur_normed", il); - ggml_tensor * freq_factors = is_swa ? nullptr : model.layers[il].rope_freqs; - Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, freq_factors, n_rot_l, rope_type, n_ctx_orig, - freq_base_l, freq_scale_l, ext_factor, attn_factor, beta_fast, beta_slow); + Qcur = build_rope(Qcur, inp_pos, il); cb(Qcur, "Qcur_pos", il); cur = build_attn(inp_attn, model.layers[il].wo, nullptr, nullptr, diff --git a/src/models/gemma4.cpp b/src/models/gemma4.cpp index d0b2b3a5b473..9c061310ded8 100644 --- a/src/models/gemma4.cpp +++ b/src/models/gemma4.cpp @@ -148,19 +148,17 @@ void llama_model_gemma4::load_arch_tensors(llama_model_loader &) { } std::unique_ptr llama_model_gemma4::build_arch_graph(const llm_graph_params & params) const { - return std::make_unique(*this, params); + return std::make_unique(params); } // get 2D slice view from a 3D tensor, the idx corresponds to the 3rd dim static ggml_tensor * gemma4_view_2d_slice(ggml_context * ctx0, ggml_tensor * x, int idx) { GGML_ASSERT(idx < (int) x->ne[2]); - return ggml_view_2d(ctx0, x, x->ne[0], x->ne[1], ggml_row_size(x->type, x->ne[0]), - idx * x->ne[0] * x->ne[1] * ggml_element_size(x)); + return ggml_view_2d(ctx0, x, x->ne[0], x->ne[1], x->nb[1], idx * x->nb[2]); } -llama_model_gemma4::graph::graph(const llama_model & model, const llm_graph_params & params) : +llama_model_gemma4::graph::graph(const llm_graph_params & params) : llm_graph_context(params), - model(model), n_embd_per_layer(model.hparams.n_embd_per_layer) { ggml_tensor * cur; ggml_tensor * inpL; @@ -183,10 +181,14 @@ llama_model_gemma4::graph::graph(const llama_model & model, const llm_graph_para // TODO: is causal == true correct? might need some changes auto * inp_attn = build_attn_inp_kv_iswa(); + auto inp_attn_full = *inp_attn; + auto inp_attn_prune = *inp_attn; + ggml_tensor * inp_out_ids = build_inp_out_ids(); if (model.per_layer_tok_embd) { - const float tok_embd_scale = sqrtf((float) n_embd_per_layer); + // fold per_layer_input_scale (1/sqrt(2)) from project_per_layer_inputs() into tok_embd_scale + const float tok_embd_scale = sqrtf((float) n_embd_per_layer) / sqrtf(2.0f); inp_per_layer = ggml_scale (ctx0, inp_per_layer, tok_embd_scale); inp_per_layer = ggml_reshape_3d(ctx0, inp_per_layer, n_embd_per_layer, n_layer, inp_per_layer->ne[1]); @@ -195,29 +197,48 @@ llama_model_gemma4::graph::graph(const llama_model & model, const llm_graph_para inp_per_layer = project_per_layer_inputs(inpL, inp_per_layer); } + // KV-shared layers do not write to the KV cache, so prune non-output token rows before the first KV-shared layer + bool prune_shared = crop_before_nextn(inp_out_ids) && + hparams.n_layer_kv_from_start > 0 && hparams.n_layer_kv_from_start < n_layer && + !cparams.embeddings && cparams.n_seq_max == 1; + for (int il = (int) hparams.n_layer_kv_from_start; prune_shared && il <= n_layer; ++il) { + prune_shared = !cparams.embeddings_layer_inp[il]; + } + + int64_t n_rows = n_tokens; + for (int il = 0; il < n_layer; ++il) { + if (prune_shared && il == (int) hparams.n_layer_kv_from_start) { + inpL = ggml_get_rows(ctx0, inpL, inp_out_ids); + inp_pos = ggml_reshape_1d(ctx0, ggml_get_rows(ctx0, ggml_reshape_2d(ctx0, inp_pos, 1, n_tokens), inp_out_ids), n_outputs); + + // TODO: instead of doing get_rows here, extend llm_attn_mask to have a `out` mask that + // is construted during set_input to contain just the pruned output cells + // this can be used to enable pruning with more than 1 sequence, both with unified and split KV cache + // ref: https://github.com/ggml-org/llama.cpp/pull/31356#discussion_r4455286628 + inp_attn_prune.self_attn_mask.raw = ggml_get_rows(ctx0, inp_attn_full.self_attn_mask.raw, inp_out_ids); + inp_attn_prune.self_attn_mask_swa.raw = ggml_get_rows(ctx0, inp_attn_full.self_attn_mask_swa.raw, inp_out_ids); + + inp_attn_prune.self_attn_mask.raw = ggml_cast(ctx0, inp_attn_prune.self_attn_mask.raw, GGML_TYPE_F16); + inp_attn_prune.self_attn_mask_swa.raw = ggml_cast(ctx0, inp_attn_prune.self_attn_mask_swa.raw, GGML_TYPE_F16); + + n_rows = n_outputs; + + inp_attn = &inp_attn_prune; + } + const int64_t n_embd_head = hparams.n_embd_head_k(il); GGML_ASSERT(n_embd_head == hparams.n_embd_head_v(il)); const int64_t n_head = hparams.n_head(il); const int64_t n_head_kv = hparams.n_head_kv(il); - const float freq_base_l = model.get_rope_freq_base(cparams, il); - const float freq_scale_l = model.get_rope_freq_scale(cparams, il); - const int n_rot_l = hparams.n_rot(il); - res->t_layer_inp[il] = inpL; // norm cur = build_norm(inpL, model.layers[il].attn_norm, nullptr, LLM_NORM_RMS, il); cb(cur, "attn_norm", il); - ggml_tensor * freq_factors = nullptr; - if (!hparams.is_swa(il)) { - // full_attention layers use rope_freqs for proportional rope - freq_factors = model.layers[il].rope_freqs; - } - // Q projection (shared for both non-KV and KV layers) // this is to mirror Gemma4Attention in pytorch code ggml_tensor * qkv_fused = nullptr; @@ -226,20 +247,19 @@ llama_model_gemma4::graph::graph(const llama_model & model, const llm_graph_para qkv_fused = build_lora_mm(model.layers[il].wqkv, cur, model.layers[il].wqkv_s); cb(qkv_fused, "wqkv", il); const int64_t q_dim = n_embd_head * n_head; - Qcur = ggml_cont(ctx0, ggml_view_2d(ctx0, qkv_fused, q_dim, n_tokens, qkv_fused->nb[1], 0)); + Qcur = ggml_cont(ctx0, ggml_view_2d(ctx0, qkv_fused, q_dim, n_rows, qkv_fused->nb[1], 0)); } else { Qcur = build_lora_mm(model.layers[il].wq, cur, model.layers[il].wq_s); } { cb(Qcur, "Qcur", il); - Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens); + Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_rows); Qcur = build_norm(Qcur, model.layers[il].attn_q_norm, nullptr, LLM_NORM_RMS, il); cb(Qcur, "Qcur_normed", il); - Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, freq_factors, n_rot_l, rope_type, n_ctx_orig, freq_base_l, freq_scale_l, - ext_factor, attn_factor, beta_fast, beta_slow); + Qcur = build_rope(Qcur, inp_pos, il); cb(Qcur, "Qcur_pos", il); } @@ -272,8 +292,7 @@ llama_model_gemma4::graph::graph(const llama_model & model, const llm_graph_para cb(Kcur, "Kcur_normed", il); cb(Vcur, "Vcur_normed", il); - Kcur = ggml_rope_ext(ctx0, Kcur, inp_pos, freq_factors, n_rot_l, rope_type, n_ctx_orig, freq_base_l, freq_scale_l, - ext_factor, attn_factor, beta_fast, beta_slow); + Kcur = build_rope(Kcur, inp_pos, il); cb(Kcur, "Kcur_pos", il); @@ -287,9 +306,8 @@ llama_model_gemma4::graph::graph(const llama_model & model, const llm_graph_para Qcur, nullptr, nullptr, nullptr, nullptr, nullptr, hparams.f_attention_scale, il); } - // TODO @ngxson : strip unused token right after the last KV layer to speed up prompt processing // keep all rows when extracting unmasked nextn embeddings (MTP target needs the hidden state for every token) - if (il == n_layer - 1 && crop_before_nextn(inp_out_ids)) { + if (!prune_shared && il == n_layer - 1 && crop_before_nextn(inp_out_ids)) { cur = ggml_get_rows(ctx0, cur, inp_out_ids); inpL = ggml_get_rows(ctx0, inpL, inp_out_ids); } @@ -331,7 +349,7 @@ llama_model_gemma4::graph::graph(const llama_model & model, const llm_graph_para ggml_tensor * tmp = ggml_rms_norm(ctx0, attn_out, hparams.f_norm_rms_eps); tmp = ggml_scale(ctx0, tmp, 1.0f / sqrtf((float) n_embd)); tmp = ggml_mul(ctx0, tmp, model.layers[il].ffn_gate_inp_s); - ggml_tensor * logits = build_lora_mm(model.layers[il].ffn_gate_inp, tmp); // [n_expert, n_tokens] + ggml_tensor * logits = build_lora_mm(model.layers[il].ffn_gate_inp, tmp); // [n_expert, n_rows] cb(logits, "ffn_moe_logits", il); cur_moe = build_moe_ffn(cur_moe, @@ -383,18 +401,17 @@ llama_model_gemma4::graph::graph(const llama_model & model, const llm_graph_para ggml_tensor * pe_in = cur; cb(cur, "pe_in", il); - cur = build_lora_mm(model.layers[il].per_layer_inp_gate, cur); // [n_embd_per_layer, n_tokens] + cur = build_lora_mm(model.layers[il].per_layer_inp_gate, cur); // [n_embd_per_layer, n_rows] cur = ggml_gelu(ctx0, cur); ggml_tensor * inp_this_layer = gemma4_view_2d_slice(ctx0, inp_per_layer, il); // [n_embd_per_layer, n_tokens] - // TODO @ngxson : improve this - if (il == n_layer - 1 && crop_before_nextn(inp_out_ids)) { + if ((prune_shared && !hparams.has_kv(il)) || (il == n_layer - 1 && crop_before_nextn(inp_out_ids))) { inp_this_layer = ggml_get_rows(ctx0, inp_this_layer, inp_out_ids); } cur = ggml_mul(ctx0, cur, inp_this_layer); - cur = build_lora_mm(model.layers[il].per_layer_proj, cur); // [n_embd, n_tokens] + cur = build_lora_mm(model.layers[il].per_layer_proj, cur); // [n_embd, n_rows] cur = build_norm(cur, model.layers[il].per_layer_post_norm, nullptr, LLM_NORM_RMS, il); cb(cur, "per_layer_embd_out", il); @@ -522,19 +539,22 @@ ggml_tensor * llama_model_gemma4::graph::project_per_layer_inputs(ggml_tensor * const float per_layer_input_scale = 1.0f / sqrtf(2.0f); // note: this matrix multiplication will be performed in the input layer (i.e. on the CPU) + // fold per_layer_input_scale into proj_norm_w and per_layer_projection_scale into proj_eps + ggml_tensor * proj_norm_w = ggml_scale(ctx0, model.per_layer_proj_norm, per_layer_input_scale); + ggml_build_forward_expand(gf, proj_norm_w); + ggml_tensor * per_layer_proj; per_layer_proj = ggml_mul_mat (ctx0, model.per_layer_model_proj, inp_batch); - per_layer_proj = ggml_scale (ctx0, per_layer_proj, per_layer_projection_scale); per_layer_proj = ggml_reshape_3d(ctx0, per_layer_proj, n_embd_per_layer, n_layer, n_tokens); - per_layer_proj = build_norm(per_layer_proj, model.per_layer_proj_norm, nullptr, LLM_NORM_RMS, -1); + const float proj_eps = hparams.f_norm_rms_eps / (per_layer_projection_scale * per_layer_projection_scale); + per_layer_proj = ggml_rms_norm(ctx0, per_layer_proj, proj_eps); + per_layer_proj = ggml_mul(ctx0, per_layer_proj, proj_norm_w); cb(per_layer_proj, "per_layer_proj", -1); - inp_per_layer = ggml_add (ctx0, per_layer_proj, inp_per_layer); - inp_per_layer = ggml_scale(ctx0, inp_per_layer, per_layer_input_scale); + inp_per_layer = ggml_add(ctx0, per_layer_proj, inp_per_layer); cb(inp_per_layer, "inp_per_layer", -1); // permute to shape: [n_embd_per_layer, n_tokens, n_layer] - inp_per_layer = ggml_cont(ctx0, ggml_permute(ctx0, inp_per_layer, 0, 2, 1, 3)); - return inp_per_layer; + return ggml_permute(ctx0, inp_per_layer, 0, 2, 1, 3); } diff --git a/src/models/glm-dsa.cpp b/src/models/glm-dsa.cpp index 0bf7a7c853a3..7007f427faeb 100644 --- a/src/models/glm-dsa.cpp +++ b/src/models/glm-dsa.cpp @@ -176,14 +176,22 @@ void llama_model_glm_dsa::load_arch_tensors(llama_model_loader & ml) { } } +llama_rope_params llama_model_glm_dsa::get_rope_params_indexer(const llama_cparams & cparams, int il) const { + auto res = llama_model::get_rope_params_indexer(cparams, il); + + res.type = LLAMA_ROPE_TYPE_NORM; + + return res; +} + std::unique_ptr llama_model_glm_dsa::build_arch_graph(const llm_graph_params & params) const { if (params.gtype == LLM_GRAPH_TYPE_DECODER_MTP) { - return std::make_unique(*this, params); + return std::make_unique(params); } - return std::make_unique(*this, params); + return std::make_unique(params); } -llama_model_glm_dsa::graph::graph(const llama_model & model, const llm_graph_params & params) : +llama_model_glm_dsa::graph::graph(const llm_graph_params & params) : llm_graph_context(params) { const bool is_mla = hparams.is_mla(); GGML_ASSERT(is_mla); @@ -258,9 +266,7 @@ llama_model_glm_dsa::graph::graph(const llama_model & model, const llm_graph_par // {n_embd_indexer_head, n_indexer_head, n_tokens} indexer_q = ggml_reshape_3d(ctx0, indexer_q, n_embd_indexer_head, n_indexer_head, n_tokens); - indexer_q = ggml_rope_ext(ctx0, indexer_q, inp_pos, nullptr, n_rot, - LLAMA_ROPE_TYPE_NORM, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); + indexer_q = build_rope_indexer(indexer_q, inp_pos, il); cb(indexer_q, "indexer_q", il); ggml_tensor * indexer_k = ggml_mul_mat(ctx0, model.layers[il].indexer_attn_k, cur); @@ -271,9 +277,7 @@ llama_model_glm_dsa::graph::graph(const llama_model & model, const llm_graph_par // {n_embd_indexer_head, 1, n_tokens} indexer_k = ggml_reshape_3d(ctx0, indexer_k, n_embd_indexer_head, 1, n_tokens); - indexer_k = ggml_rope_ext(ctx0, indexer_k, inp_pos, nullptr, n_rot, - LLAMA_ROPE_TYPE_NORM, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); + indexer_k = build_rope_indexer(indexer_k, inp_pos, il); cb(indexer_k, "indexer_k", il); // perform Hadamard transform on indexer q and k @@ -305,7 +309,7 @@ llama_model_glm_dsa::graph::graph(const llama_model & model, const llm_graph_par ggml_tensor * indexer_score = nullptr; if (cparams.fused_lid) { - indexer_score = ggml_lightning_indexer(ctx0, indexer_q, indexer_k, indexer_weights, inp_attn_dsa->self_attn_mask_lid.cnv); + indexer_score = ggml_lightning_indexer(ctx0, indexer_q, indexer_k, indexer_weights, inp_attn_dsa->self_attn_mask_lid.raw); cb(indexer_score, "indexer_score", il); res->add_fused_node({LLM_FUSED_OP_LIGHTNING_INDEXER, indexer_score, il}); } else { @@ -339,7 +343,7 @@ llama_model_glm_dsa::graph::graph(const llama_model & model, const llm_graph_par cb(indexer_score, "indexer_score", il); // mask indexer scores - ggml_tensor * indexer_kq_mask = inp_attn_dsa->self_attn_mask_lid.cnv; + ggml_tensor * indexer_kq_mask = inp_attn_dsa->self_attn_mask_lid.raw; indexer_score = ggml_add(ctx0, indexer_score, indexer_kq_mask); cb(indexer_score, "indexer_score", il); } @@ -387,12 +391,10 @@ llama_model_glm_dsa::graph::graph(const llama_model & model, const llm_graph_par ggml_row_size(kv_cmpr_pe->type, kv_lora_rank)); cb(k_pe, "k_pe", il); - q_pe = ggml_rope_ext(ctx0, q_pe, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); + q_pe = build_rope(q_pe, inp_pos, il); cb(q_pe, "q_pe", il); - k_pe = ggml_rope_ext(ctx0, k_pe, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); + k_pe = build_rope(k_pe, inp_pos, il); cb(k_pe, "k_pe", il); kv_cmpr = build_norm(kv_cmpr, model.layers[il].attn_kv_a_norm, nullptr, LLM_NORM_RMS, il); @@ -524,7 +526,7 @@ llama_model_glm_dsa::graph::graph(const llama_model & model, const llm_graph_par // with shared expert, exactly as the trunk deepseek2 graph builds it) -> // shared_head_norm (fallback output_norm) -> shared LM head. // The DSA indexer is not used at runtime (same as the trunk graph). -llama_model_glm_dsa::graph_mtp::graph_mtp(const llama_model & model, const llm_graph_params & params) +llama_model_glm_dsa::graph_mtp::graph_mtp(const llm_graph_params & params) : llm_graph_context(params) { GGML_ASSERT(hparams.n_layer_nextn > 0 && "GLM_DSA MTP requires n_layer_nextn > 0"); GGML_ASSERT(hparams.n_layer_nextn == 1 && "GLM_DSA MTP currently only supports a single MTP block"); @@ -577,9 +579,7 @@ llama_model_glm_dsa::graph_mtp::graph_mtp(const llama_model & model, const llm_g } cb(tok_embd, "mtp_tok_embd", il); - inp->h = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, hparams.n_embd, n_tokens); - ggml_set_input(inp->h); - ggml_set_name(inp->h, "mtp_h_input"); + inp->h = build_inp_embd_state(inp.get(), hparams.n_embd); ggml_tensor * h_embd = inp->h; @@ -647,12 +647,10 @@ llama_model_glm_dsa::graph_mtp::graph_mtp(const llama_model & model, const llm_g ggml_row_size(kv_cmpr_pe->type, kv_lora_rank)); cb(k_pe, "mtp_k_pe", il); - q_pe = ggml_rope_ext(ctx0, q_pe, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); + q_pe = build_rope(q_pe, inp_pos, il); cb(q_pe, "mtp_q_pe", il); - k_pe = ggml_rope_ext(ctx0, k_pe, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); + k_pe = build_rope(k_pe, inp_pos, il); cb(k_pe, "mtp_k_pe", il); kv_cmpr = build_norm(kv_cmpr, layer.attn_kv_a_norm, nullptr, LLM_NORM_RMS, il); diff --git a/src/models/glm4-moe.cpp b/src/models/glm4-moe.cpp index d19eb553f841..9560fd3d0429 100644 --- a/src/models/glm4-moe.cpp +++ b/src/models/glm4-moe.cpp @@ -118,12 +118,12 @@ void llama_model_glm4_moe::load_arch_tensors(llama_model_loader & ml) { std::unique_ptr llama_model_glm4_moe::build_arch_graph(const llm_graph_params & params) const { if (params.gtype == LLM_GRAPH_TYPE_DECODER_MTP) { - return std::make_unique(*this, params); + return std::make_unique(params); } - return std::make_unique(*this, params); + return std::make_unique(params); } -llama_model_glm4_moe::graph_mtp::graph_mtp(const llama_model & model, const llm_graph_params & params) +llama_model_glm4_moe::graph_mtp::graph_mtp(const llm_graph_params & params) : llm_graph_context(params) { GGML_ASSERT(hparams.n_layer_nextn > 0 && "GLM4_MOE MTP requires n_layer_nextn > 0"); GGML_ASSERT(hparams.n_layer_nextn == 1 && "GLM4_MOE MTP currently only supports a single MTP block"); @@ -161,9 +161,7 @@ llama_model_glm4_moe::graph_mtp::graph_mtp(const llama_model & model, const llm_ } cb(tok_embd, "mtp_tok_embd", il); - inp->h = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, hparams.n_embd, n_tokens); - ggml_set_input(inp->h); - ggml_set_name(inp->h, "mtp_h_input"); + inp->h = build_inp_embd_state(inp.get(), hparams.n_embd); ggml_tensor * h_embd = inp->h; @@ -203,13 +201,8 @@ llama_model_glm4_moe::graph_mtp::graph_mtp(const llama_model & model, const llm_ cb(Kcur, "mtp_Kcur_normed", il); } - Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, nullptr, n_rot, - rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); - - Kcur = ggml_rope_ext(ctx0, Kcur, inp_pos, nullptr, n_rot, - rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); + Qcur = build_rope(Qcur, inp_pos, il); + Kcur = build_rope(Kcur, inp_pos, il); cb(Qcur, "mtp_Qcur", il); cb(Kcur, "mtp_Kcur", il); @@ -282,14 +275,11 @@ llama_model_glm4_moe::graph_mtp::graph_mtp(const llama_model & model, const llm_ res->t_logits = cur; } -llama_model_glm4_moe::graph::graph(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) { +llama_model_glm4_moe::graph::graph(const llm_graph_params & params) : llm_graph_context(params) { const int64_t n_embd_head = hparams.n_embd_head_v(); GGML_ASSERT(n_embd_head == hparams.n_embd_head_k()); - int sections[4]; - std::copy(std::begin(hparams.rope_sections), std::begin(hparams.rope_sections) + 4, sections); - ggml_tensor * cur; ggml_tensor * inpL; @@ -331,24 +321,9 @@ llama_model_glm4_moe::graph::graph(const llama_model & model, const llm_graph_pa cb(Kcur, "Kcur_normed", il); } - if (use_mrope) { - Qcur = ggml_rope_multi(ctx0, Qcur, inp_pos, nullptr, - n_rot, sections, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); - - Kcur = ggml_rope_multi(ctx0, Kcur, inp_pos, nullptr, - n_rot, sections, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); - } else { - // Normal RoPE - Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, nullptr, n_rot, - rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); - - Kcur = ggml_rope_ext(ctx0, Kcur, inp_pos, nullptr, n_rot, - rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); - } + // Normal RoPE + Qcur = build_rope(Qcur, inp_pos, il); + Kcur = build_rope(Kcur, inp_pos, il); cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); diff --git a/src/models/glm4.cpp b/src/models/glm4.cpp index a6cb0889b66b..e57c51cbe78e 100644 --- a/src/models/glm4.cpp +++ b/src/models/glm4.cpp @@ -62,17 +62,14 @@ void llama_model_glm4::load_arch_tensors(llama_model_loader &) { } std::unique_ptr llama_model_glm4::build_arch_graph(const llm_graph_params & params) const { - return std::make_unique(*this, params); + return std::make_unique(params); } -llama_model_glm4::graph::graph(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) { +llama_model_glm4::graph::graph(const llm_graph_params & params) : llm_graph_context(params) { const int64_t n_embd_head = hparams.n_embd_head_v(); GGML_ASSERT(n_embd_head == hparams.n_embd_head_k()); - int sections[4]; - std::copy(std::begin(hparams.rope_sections), std::begin(hparams.rope_sections) + 4, sections); - ggml_tensor * cur; ggml_tensor * inpL; @@ -105,24 +102,9 @@ llama_model_glm4::graph::graph(const llama_model & model, const llm_graph_params auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur, n_embd_head, n_head, n_head_kv, il); - if (use_mrope) { - Qcur = ggml_rope_multi(ctx0, Qcur, inp_pos, nullptr, - n_rot, sections, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); - - Kcur = ggml_rope_multi(ctx0, Kcur, inp_pos, nullptr, - n_rot, sections, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); - } else { - // Normal RoPE - Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, nullptr, n_rot, - rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); - - Kcur = ggml_rope_ext(ctx0, Kcur, inp_pos, nullptr, n_rot, - rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); - } + // Normal RoPE + Qcur = build_rope(Qcur, inp_pos, il); + Kcur = build_rope(Kcur, inp_pos, il); cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); diff --git a/src/models/glm5-next.cpp b/src/models/glm5-next.cpp index 100af6d23284..844a327b3dd2 100644 --- a/src/models/glm5-next.cpp +++ b/src/models/glm5-next.cpp @@ -185,19 +185,17 @@ void llama_model_glm5_next::load_arch_tensors(llama_model_loader & ml) { std::unique_ptr llama_model_glm5_next::build_arch_graph(const llm_graph_params & params) const { if (params.gtype == LLM_GRAPH_TYPE_DECODER_MTP) { - return std::make_unique(*this, params); + return std::make_unique(params); } - return std::make_unique(*this, params); + return std::make_unique(params); } // Causal conv1d over one of Q/K/V -static ggml_tensor * glm5_conv1d(ggml_cgraph * gf, ggml_context * ctx0, - ggml_tensor * conv_states_all, ggml_tensor * conv_state_all, - int64_t qkv, ggml_tensor * x, ggml_tensor * proj_w, ggml_tensor * conv_w, - int64_t d_conv, int64_t head_dim, int64_t n_head, - int64_t n_seq_tokens, int64_t n_seqs, int64_t n_tokens, int64_t kv_head, - int64_t mem_size, int64_t K_rs) { - const int64_t d_inner = head_dim * n_head; +ggml_tensor * llama_model_glm5_next::graph::build_kda_conv1d( + llm_graph_input_rs * inp_rs, ggml_tensor * conv_states_all, ggml_tensor * conv_state_all, + int64_t qkv, ggml_tensor * x, ggml_tensor * proj_w, ggml_tensor * conv_w, + int64_t d_conv, int64_t head_dim, int64_t n_head_kda, int64_t n_seq_tokens, int64_t n_seqs) { + const int64_t d_inner = head_dim * n_head_kda; const int64_t conv_state_size = (d_conv - 1) * d_inner; const int64_t n_embd_r_total = 3 * conv_state_size; @@ -210,26 +208,15 @@ static ggml_tensor * glm5_conv1d(ggml_cgraph * gf, ggml_context * ctx0, ggml_tensor * x_3d = ggml_reshape_3d(ctx0, x_proj, d_inner, n_seq_tokens, n_seqs); ggml_tensor * conv_x = ggml_concat(ctx0, conv_state_x, ggml_transpose(ctx0, x_3d), 0); - // group s holds the conv window s tokens back. - // [TAG_RECURRENT_ROLLBACK_SPLITS]: the last K_rs tokens must share one ubatch. - for (int64_t s = 0; s < K_rs; ++s) { - const int64_t s_idx = std::max(0, n_seq_tokens - s); - ggml_tensor * conv_x_s = ggml_view_3d(ctx0, conv_x, d_conv - 1, d_inner, n_seqs, - conv_x->nb[1], conv_x->nb[2], s_idx * conv_x->nb[0]); - ggml_build_forward_expand(gf, - ggml_cpy(ctx0, conv_x_s, - ggml_view_3d(ctx0, conv_states_all, d_conv - 1, d_inner, n_seqs, - (d_conv - 1) * ggml_element_size(conv_states_all), - n_embd_r_total * ggml_element_size(conv_states_all), - ((s * mem_size + kv_head) * n_embd_r_total + qkv * conv_state_size) * ggml_element_size(conv_states_all)))); - } + build_conv_state_store(inp_rs, conv_states_all, conv_x, d_conv - 1, + ggml_row_size(conv_states_all->type, qkv * conv_state_size)); ggml_tensor * conv_weight = ggml_reshape_2d(ctx0, conv_w, d_conv, d_inner); ggml_tensor * Xcur = ggml_ssm_conv(ctx0, conv_x, conv_weight); Xcur = ggml_reshape_2d(ctx0, Xcur, d_inner, n_tokens); Xcur = ggml_silu(ctx0, Xcur); - return ggml_reshape_4d(ctx0, Xcur, head_dim, n_head, n_seq_tokens, n_seqs); + return ggml_reshape_4d(ctx0, Xcur, head_dim, n_head_kda, n_seq_tokens, n_seqs); } @@ -537,12 +524,12 @@ ggml_tensor * llama_model_glm5_next::graph::build_hc_post( } // construct the graph helpers without building the trunk, so graph_mtp can reuse them -llama_model_glm5_next::graph::graph(const llama_model & model, const llm_graph_params & params, no_build) : - llm_build_delta_net_base(params), model(model) { +llama_model_glm5_next::graph::graph(const llm_graph_params & params, no_build) : + llm_build_delta_net_base(params) { } -llama_model_glm5_next::graph_mtp::graph_mtp(const llama_model & model, const llm_graph_params & params) : - graph(model, params, no_build{}) { +llama_model_glm5_next::graph_mtp::graph_mtp(const llm_graph_params & params) : + graph(params, no_build{}) { GGML_ASSERT(hparams.n_layer_nextn > 0 && "GLM5-Next MTP requires n_layer_nextn > 0"); GGML_ASSERT(hparams.n_layer_nextn == 1 && "GLM5-Next MTP currently supports a single NextN block"); @@ -577,9 +564,7 @@ llama_model_glm5_next::graph_mtp::graph_mtp(const llama_model & model, const llm inp->embd = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, hparams.n_embd_inp(), n_tokens); ggml_set_input(inp->embd); - inp->h = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_embd, n_tokens); - ggml_set_input(inp->h); - ggml_set_name(inp->h, "mtp_h_input"); + inp->h = build_inp_embd_state(inp.get(), n_embd); ggml_tensor * tok_embd; if (ubatch.token) { @@ -671,8 +656,8 @@ llama_model_glm5_next::graph_mtp::graph_mtp(const llama_model & model, const llm res->t_logits = cur; } -llama_model_glm5_next::graph::graph(const llama_model & model, const llm_graph_params & params) : - llm_build_delta_net_base(params), model(model) { +llama_model_glm5_next::graph::graph(const llm_graph_params & params) : + llm_build_delta_net_base(params) { ggml_tensor * cur; @@ -812,17 +797,13 @@ ggml_tensor * llama_model_glm5_next::graph::build_kda_layer( int64_t d_inner, int64_t n_seq_tokens, int64_t n_seqs, int il) { const auto * mctx_cur = inp_rs->mctx; - const auto kv_head = mctx_cur->get_head(); ggml_tensor * conv_states_all = mctx_cur->get_r_l(il); ggml_tensor * conv_state_all = build_rs(inp_rs, conv_states_all, hparams.n_embd_r(), n_seqs); - const int64_t mem_size = mctx_cur->get_size(); - const int64_t K_rs = (int64_t) cparams.n_rs_seq + 1; - - ggml_tensor * Qcur = glm5_conv1d(gf, ctx0, conv_states_all, conv_state_all, 0, cur, layer.wq, layer.ssm_q_conv, d_conv, head_dim, n_head_kda, n_seq_tokens, n_seqs, n_tokens, kv_head, mem_size, K_rs); - ggml_tensor * Kcur = glm5_conv1d(gf, ctx0, conv_states_all, conv_state_all, 1, cur, layer.wk, layer.ssm_k_conv, d_conv, head_dim, n_head_kda, n_seq_tokens, n_seqs, n_tokens, kv_head, mem_size, K_rs); - ggml_tensor * Vcur = glm5_conv1d(gf, ctx0, conv_states_all, conv_state_all, 2, cur, layer.wv, layer.ssm_v_conv, d_conv, head_dim, n_head_kda, n_seq_tokens, n_seqs, n_tokens, kv_head, mem_size, K_rs); + ggml_tensor * Qcur = build_kda_conv1d(inp_rs, conv_states_all, conv_state_all, 0, cur, layer.wq, layer.ssm_q_conv, d_conv, head_dim, n_head_kda, n_seq_tokens, n_seqs); + ggml_tensor * Kcur = build_kda_conv1d(inp_rs, conv_states_all, conv_state_all, 1, cur, layer.wk, layer.ssm_k_conv, d_conv, head_dim, n_head_kda, n_seq_tokens, n_seqs); + ggml_tensor * Vcur = build_kda_conv1d(inp_rs, conv_states_all, conv_state_all, 2, cur, layer.wv, layer.ssm_v_conv, d_conv, head_dim, n_head_kda, n_seq_tokens, n_seqs); cb(Qcur, "kda_q_conv", il); cb(Kcur, "kda_k_conv", il); cb(Vcur, "kda_v_conv", il); diff --git a/src/models/gpt2.cpp b/src/models/gpt2.cpp index d6ca64744f49..711a148a28bd 100644 --- a/src/models/gpt2.cpp +++ b/src/models/gpt2.cpp @@ -22,6 +22,7 @@ void llama_model_gpt2::load_arch_tensors(llama_model_loader &) { output_norm = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd}, 0); output_norm_b = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, "bias"), {n_embd}, 0); output = create_tensor(tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab}, TENSOR_NOT_REQUIRED); + output_b = create_tensor(tn(LLM_TENSOR_OUTPUT, "bias"), {n_vocab}, TENSOR_NOT_REQUIRED); // if output is NULL, init from the input tok embed if (output == NULL) { @@ -52,10 +53,10 @@ void llama_model_gpt2::load_arch_tensors(llama_model_loader &) { } std::unique_ptr llama_model_gpt2::build_arch_graph(const llm_graph_params & params) const { - return std::make_unique(*this, params); + return std::make_unique(params); } -llama_model_gpt2::graph::graph(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) { +llama_model_gpt2::graph::graph(const llm_graph_params & params) : llm_graph_context(params) { const int64_t n_embd_head = hparams.n_embd_head_v(); GGML_ASSERT(n_embd_head == hparams.n_embd_head_k()); @@ -140,6 +141,9 @@ llama_model_gpt2::graph::graph(const llama_model & model, const llm_graph_params res->t_embd = cur; cur = build_lora_mm(model.output, cur, model.output_s); + if (model.output_b) { + cur = ggml_add(ctx0, cur, model.output_b); + } cb(cur, "result_output", -1); res->t_logits = cur; diff --git a/src/models/gptneox.cpp b/src/models/gptneox.cpp index bc8a9fd4a993..8217e5a8a71d 100644 --- a/src/models/gptneox.cpp +++ b/src/models/gptneox.cpp @@ -81,10 +81,10 @@ void llama_model_gptneox::load_arch_tensors(llama_model_loader &) { } std::unique_ptr llama_model_gptneox::build_arch_graph(const llm_graph_params & params) const { - return std::make_unique(*this, params); + return std::make_unique(params); } -llama_model_gptneox::graph::graph(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) { +llama_model_gptneox::graph::graph(const llm_graph_params & params) : llm_graph_context(params) { const int64_t n_embd_head = hparams.n_embd_head_v(); GGML_ASSERT(n_embd_head == hparams.n_embd_head_k()); @@ -113,17 +113,8 @@ llama_model_gptneox::graph::graph(const llama_model & model, const llm_graph_par auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur, n_embd_head, n_head, n_head_kv, il); - Qcur = ggml_rope_ext( - ctx0, Qcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); - - Kcur = ggml_rope_ext( - ctx0, Kcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); + Qcur = build_rope(Qcur, inp_pos, il); + Kcur = build_rope(Kcur, inp_pos, il); cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); diff --git a/src/models/granite-hybrid.cpp b/src/models/granite-hybrid.cpp index a752a365cfd1..be352dfd770d 100644 --- a/src/models/granite-hybrid.cpp +++ b/src/models/granite-hybrid.cpp @@ -129,10 +129,10 @@ void llama_model_granite_hybrid::load_arch_tensors(llama_model_loader &) { } std::unique_ptr llama_model_granite_hybrid::build_arch_graph(const llm_graph_params & params) const { - return std::make_unique(*this, params); + return std::make_unique(params); } -llama_model_granite_hybrid::graph::graph(const llama_model & model, const llm_graph_params & params) : +llama_model_granite_hybrid::graph::graph(const llm_graph_params & params) : llm_build_mamba_base(params) { const int64_t n_embd_head = hparams.n_embd_head_v(); GGML_ASSERT(n_embd_head == hparams.n_embd_head_k()); @@ -163,10 +163,10 @@ llama_model_granite_hybrid::graph::graph(const llama_model & model, const llm_gr if (hparams.is_recr(il)) { // ssm layer // - cur = build_mamba2_layer(inp->get_recr(), cur, model, ubatch, il); + cur = build_mamba2_layer(inp->get_recr(), cur, ubatch, il); } else { // attention layer // - cur = build_attention_layer(cur, inp_pos, inp->get_attn(), model, n_embd_head, il); + cur = build_attention_layer(cur, inp_pos, inp->get_attn(), n_embd_head, il); } if (il == n_layer - 1 && inp_out_ids) { @@ -175,7 +175,7 @@ llama_model_granite_hybrid::graph::graph(const llama_model & model, const llm_gr } // ffn - cur = build_layer_ffn(cur, inpSA, model, il); + cur = build_layer_ffn(cur, inpSA, il); // input for next layer inpL = cur; @@ -202,19 +202,12 @@ llama_model_granite_hybrid::graph::graph(const llama_model & model, const llm_gr ggml_tensor * llama_model_granite_hybrid::graph::build_attention_layer(ggml_tensor * cur, ggml_tensor * inp_pos, llm_graph_input_attn_kv * inp_attn, - const llama_model & model, const int64_t n_embd_head, const int il) { auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur, n_embd_head, hparams.n_head(il), hparams.n_head_kv(il), il); - if (hparams.has_rope(il)) { - ggml_tensor * rope_factors = model.get_rope_factors(cparams, il); - Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, rope_factors, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); - - Kcur = ggml_rope_ext(ctx0, Kcur, inp_pos, rope_factors, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); - } + Qcur = build_rope(Qcur, inp_pos, il); + Kcur = build_rope(Kcur, inp_pos, il); cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); @@ -231,7 +224,6 @@ ggml_tensor * llama_model_granite_hybrid::graph::build_attention_layer(ggml_tens ggml_tensor * llama_model_granite_hybrid::graph::build_layer_ffn(ggml_tensor * cur, ggml_tensor * inpSA, - const llama_model & model, const int il) { // For Granite architectures - scale residual if (hparams.f_residual_scale) { @@ -246,9 +238,9 @@ ggml_tensor * llama_model_granite_hybrid::graph::build_layer_ffn(ggml_tensor * cb(cur, "ffn_norm", il); cur = build_ffn(cur, - model.layers[il].ffn_up, model.layers[il].ffn_up_b, NULL, - model.layers[il].ffn_gate, model.layers[il].ffn_gate_b, NULL, - model.layers[il].ffn_down, model.layers[il].ffn_down_b, NULL, + model.layers[il].ffn_up, model.layers[il].ffn_up_b, model.layers[il].ffn_up_s, + model.layers[il].ffn_gate, model.layers[il].ffn_gate_b, model.layers[il].ffn_gate_s, + model.layers[il].ffn_down, model.layers[il].ffn_down_b, model.layers[il].ffn_down_s, NULL, LLM_FFN_SILU, LLM_FFN_PAR, il); cb(cur, "ffn_out", il); @@ -268,16 +260,20 @@ ggml_tensor * llama_model_granite_hybrid::graph::build_layer_ffn(ggml_tensor * LLM_FFN_SILU, true, hparams.expert_weights_scale, LLAMA_EXPERT_GATING_FUNC_TYPE_SOFTMAX, - il); + il, + nullptr, nullptr, + model.layers[il].ffn_up_exps_s, + model.layers[il].ffn_gate_exps_s, + model.layers[il].ffn_down_exps_s); cb(moe_out, "ffn_moe_out", il); // For Granite MoE Shared if (hparams.n_ff_shexp > 0) { ggml_tensor * ffn_shexp = build_ffn(cur, - model.layers[il].ffn_up_shexp, NULL, NULL, - model.layers[il].ffn_gate_shexp, NULL, NULL, - model.layers[il].ffn_down_shexp, NULL, NULL, + model.layers[il].ffn_up_shexp, NULL, model.layers[il].ffn_up_shexp_s, + model.layers[il].ffn_gate_shexp, NULL, model.layers[il].ffn_gate_shexp_s, + model.layers[il].ffn_down_shexp, NULL, model.layers[il].ffn_down_shexp_s, NULL, LLM_FFN_SILU, LLM_FFN_PAR, il); cb(ffn_shexp, "ffn_shexp", il); diff --git a/src/models/granite-moe.cpp b/src/models/granite-moe.cpp index febe1bfa7aa4..108b681662a1 100644 --- a/src/models/granite-moe.cpp +++ b/src/models/granite-moe.cpp @@ -79,6 +79,6 @@ void llama_model_granite_moe::load_arch_tensors(llama_model_loader &) { } std::unique_ptr llama_model_granite_moe::build_arch_graph(const llm_graph_params & params) const { - return std::make_unique(*this, params); + return std::make_unique(params); } diff --git a/src/models/granite-swa.cpp b/src/models/granite-swa.cpp index 60c1eb8ba821..be9e5577856d 100644 --- a/src/models/granite-swa.cpp +++ b/src/models/granite-swa.cpp @@ -114,14 +114,21 @@ void llama_model_granite_swa::load_arch_tensors(llama_model_loader &) { } } +llama_rope_params llama_model_granite_swa::get_rope_params(const llama_cparams & cparams, int il) const { + auto res = llama_model::get_rope_params(cparams, il); + + // the SWA layers use the global base and scale + res.freq_base = cparams.rope_freq_base; + res.freq_scale = cparams.rope_freq_scale; + + return res; +} + std::unique_ptr llama_model_granite_swa::build_arch_graph(const llm_graph_params & params) const { - return std::make_unique(*this, params); + return std::make_unique(params); } -llama_model_granite_swa::graph::graph( - const llama_model & model, - const llm_graph_params & params) - : llm_graph_context(params) { +llama_model_granite_swa::graph::graph(const llm_graph_params & params) : llm_graph_context(params) { const int64_t n_embd_head = hparams.n_embd_head_v(); @@ -167,14 +174,14 @@ llama_model_granite_swa::graph::graph( // self-attention cur = build_attention_layer( cur, inp_pos, inp_attn, - model, n_embd_head, il); + n_embd_head, il); if (il == n_layer - 1 && inp_out_ids) { cur = ggml_get_rows(ctx0, cur, inp_out_ids); inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids); } // ffn - cur = build_layer_ffn(cur, inpSA, model, il); + cur = build_layer_ffn(cur, inpSA, il); // input for next layer inpL = cur; @@ -201,28 +208,14 @@ ggml_tensor * llama_model_granite_swa::graph::build_attention_layer( ggml_tensor * cur, ggml_tensor * inp_pos, llm_graph_input_attn_kv_iswa * inp_attn, - const llama_model & model, const int64_t n_embd_head, const int il) { auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur, n_embd_head, hparams.n_head(il), hparams.n_head_kv(il), il); - const bool use_rope = hparams.has_rope(il); - if (use_rope) { - ggml_tensor * rope_factors = model.get_rope_factors(cparams, il); - Qcur = ggml_rope_ext( - ctx0, Qcur, inp_pos, rope_factors, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); - - Kcur = ggml_rope_ext( - ctx0, Kcur, inp_pos, rope_factors, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); - } + Qcur = build_rope(Qcur, inp_pos, il); + Kcur = build_rope(Kcur, inp_pos, il); cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); @@ -241,7 +234,6 @@ ggml_tensor * llama_model_granite_swa::graph::build_attention_layer( ggml_tensor * llama_model_granite_swa::graph::build_layer_ffn( ggml_tensor * cur, ggml_tensor * inpSA, - const llama_model & model, const int il) { // For Granite architectures - scale residual @@ -260,9 +252,9 @@ ggml_tensor * llama_model_granite_swa::graph::build_layer_ffn( cb(cur, "ffn_norm", il); cur = build_ffn(cur, - model.layers[il].ffn_up, model.layers[il].ffn_up_b, NULL, - model.layers[il].ffn_gate, model.layers[il].ffn_gate_b, NULL, - model.layers[il].ffn_down, model.layers[il].ffn_down_b, NULL, + model.layers[il].ffn_up, model.layers[il].ffn_up_b, model.layers[il].ffn_up_s, + model.layers[il].ffn_gate, model.layers[il].ffn_gate_b, model.layers[il].ffn_gate_s, + model.layers[il].ffn_down, model.layers[il].ffn_down_b, model.layers[il].ffn_down_s, NULL, LLM_FFN_SILU, LLM_FFN_PAR, il); cb(cur, "ffn_out", il); @@ -285,15 +277,18 @@ ggml_tensor * llama_model_granite_swa::graph::build_layer_ffn( hparams.expert_weights_scale, LLAMA_EXPERT_GATING_FUNC_TYPE_SOFTMAX, il, - nullptr, model.layers[il].ffn_gate_up_exps); + nullptr, model.layers[il].ffn_gate_up_exps, + model.layers[il].ffn_up_exps_s, + model.layers[il].ffn_gate_exps_s, + model.layers[il].ffn_down_exps_s); cb(moe_out, "ffn_moe_out", il); // For Granite MoE Shared - gate+up kept fused in ffn_up_shexp if (hparams.n_ff_shexp > 0) { ggml_tensor * ffn_shexp = build_ffn(cur, - model.layers[il].ffn_up_shexp, NULL, NULL, + model.layers[il].ffn_up_shexp, NULL, model.layers[il].ffn_up_shexp_s, NULL, NULL, NULL, - model.layers[il].ffn_down_shexp, NULL, NULL, + model.layers[il].ffn_down_shexp, NULL, model.layers[il].ffn_down_shexp_s, NULL, LLM_FFN_SWIGLU, LLM_FFN_SEQ, il); cb(ffn_shexp, "ffn_shexp", il); diff --git a/src/models/granite-switch.cpp b/src/models/granite-switch.cpp index f67bf6055f76..53195c97c5e1 100644 --- a/src/models/granite-switch.cpp +++ b/src/models/granite-switch.cpp @@ -186,7 +186,7 @@ void llm_graph_input_switch::set_input(const llama_ubatch * ubatch) { } std::unique_ptr llama_model_granite_switch::build_arch_graph(const llm_graph_params & params) const { - return std::make_unique(*this, params); + return std::make_unique(params); } // per-token switched LoRA delta: B_a*(A_a*x), adapter selected per token via ids. @@ -219,10 +219,7 @@ ggml_tensor * llama_model_granite_switch::graph::build_switched_lora_mm( return ggml_add(ctx0, base, delta); } -llama_model_granite_switch::graph::graph( - const llama_model & model, - const llm_graph_params & params) - : llm_graph_context(params) { +llama_model_granite_switch::graph::graph(const llm_graph_params & params) : llm_graph_context(params) { const auto & smodel = static_cast(model); @@ -300,7 +297,7 @@ llama_model_granite_switch::graph::graph( cur = build_norm(inpL, model.layers[il].attn_norm, NULL, LLM_NORM_RMS, il); cb(cur, "attn_norm", il); - cur = build_attention_layer(cur, inp_pos, adapter_ids, inp_attn, model, n_embd_head, il); + cur = build_attention_layer(cur, inp_pos, adapter_ids, inp_attn, n_embd_head, il); if (il == n_layer - 1 && inp_out_ids) { cur = ggml_get_rows(ctx0, cur, inp_out_ids); @@ -312,7 +309,7 @@ llama_model_granite_switch::graph::graph( adapter_ids = ggml_reshape_1d(ctx0, adapter_ids, n_out); } - cur = build_layer_ffn(cur, inpSA, adapter_ids, model, il); + cur = build_layer_ffn(cur, inpSA, adapter_ids, il); inpL = cur; } @@ -335,7 +332,6 @@ ggml_tensor * llama_model_granite_switch::graph::build_attention_layer( ggml_tensor * inp_pos, ggml_tensor * adapter_ids, llm_graph_input_attn_kv * inp_attn, - const llama_model & model, const int64_t n_embd_head, const int il) { @@ -364,15 +360,8 @@ ggml_tensor * llama_model_granite_switch::graph::build_attention_layer( Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens); Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens); - if (hparams.has_rope(il)) { - ggml_tensor * rope_factors = model.get_rope_factors(cparams, il); - Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, rope_factors, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); - Kcur = ggml_rope_ext(ctx0, Kcur, inp_pos, rope_factors, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); - } + Qcur = build_rope(Qcur, inp_pos, il); + Kcur = build_rope(Kcur, inp_pos, il); cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); cb(Vcur, "Vcur", il); @@ -395,7 +384,6 @@ ggml_tensor * llama_model_granite_switch::graph::build_layer_ffn( ggml_tensor * cur, ggml_tensor * inpSA, ggml_tensor * adapter_ids, - const llama_model & model, const int il) { const auto & layer = model.layers[il]; diff --git a/src/models/granite.cpp b/src/models/granite.cpp index aa4d01095675..0ab9084dae8a 100644 --- a/src/models/granite.cpp +++ b/src/models/granite.cpp @@ -117,13 +117,10 @@ void llama_model_granite::load_arch_tensors(llama_model_loader &) { } std::unique_ptr llama_model_granite::build_arch_graph(const llm_graph_params & params) const { - return std::make_unique(*this, params); + return std::make_unique(params); } -llama_model_granite::graph::graph( - const llama_model & model, - const llm_graph_params & params) - : llm_graph_context(params) { +llama_model_granite::graph::graph(const llm_graph_params & params) : llm_graph_context(params) { const int64_t n_embd_head = hparams.n_embd_head_v(); @@ -172,14 +169,14 @@ llama_model_granite::graph::graph( // self-attention cur = build_attention_layer( cur, inp_pos, inp_attn, - model, n_embd_head, il); + n_embd_head, il); if (il == n_layer - 1 && inp_out_ids) { cur = ggml_get_rows(ctx0, cur, inp_out_ids); inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids); } // ffn - cur = build_layer_ffn(cur, inpSA, model, il); + cur = build_layer_ffn(cur, inpSA, il); // input for next layer inpL = cur; @@ -206,27 +203,14 @@ ggml_tensor * llama_model_granite::graph::build_attention_layer( ggml_tensor * cur, ggml_tensor * inp_pos, llm_graph_input_attn_kv * inp_attn, - const llama_model & model, const int64_t n_embd_head, const int il) { auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur, n_embd_head, hparams.n_head(il), hparams.n_head_kv(il), il); - if (hparams.has_rope(il)) { - ggml_tensor * rope_factors = model.get_rope_factors(cparams, il); - Qcur = ggml_rope_ext( - ctx0, Qcur, inp_pos, rope_factors, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); - - Kcur = ggml_rope_ext( - ctx0, Kcur, inp_pos, rope_factors, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); - } + Qcur = build_rope(Qcur, inp_pos, il); + Kcur = build_rope(Kcur, inp_pos, il); cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); @@ -243,7 +227,6 @@ ggml_tensor * llama_model_granite::graph::build_attention_layer( ggml_tensor * llama_model_granite::graph::build_layer_ffn( ggml_tensor * cur, ggml_tensor * inpSA, - const llama_model & model, const int il) { // For Granite architectures - scale residual @@ -262,9 +245,9 @@ ggml_tensor * llama_model_granite::graph::build_layer_ffn( cb(cur, "ffn_norm", il); cur = build_ffn(cur, - model.layers[il].ffn_up, model.layers[il].ffn_up_b, NULL, - model.layers[il].ffn_gate, model.layers[il].ffn_gate_b, NULL, - model.layers[il].ffn_down, model.layers[il].ffn_down_b, NULL, + model.layers[il].ffn_up, model.layers[il].ffn_up_b, model.layers[il].ffn_up_s, + model.layers[il].ffn_gate, model.layers[il].ffn_gate_b, model.layers[il].ffn_gate_s, + model.layers[il].ffn_down, model.layers[il].ffn_down_b, model.layers[il].ffn_down_s, NULL, LLM_FFN_SILU, LLM_FFN_PAR, il); cb(cur, "ffn_out", il); @@ -286,15 +269,19 @@ ggml_tensor * llama_model_granite::graph::build_layer_ffn( LLM_FFN_SILU, true, hparams.expert_weights_scale, LLAMA_EXPERT_GATING_FUNC_TYPE_SOFTMAX, - il); + il, + nullptr, nullptr, + model.layers[il].ffn_up_exps_s, + model.layers[il].ffn_gate_exps_s, + model.layers[il].ffn_down_exps_s); cb(moe_out, "ffn_moe_out", il); // For Granite MoE Shared if (hparams.n_ff_shexp > 0) { ggml_tensor * ffn_shexp = build_ffn(cur, - model.layers[il].ffn_up_shexp, NULL, NULL, - model.layers[il].ffn_gate_shexp, NULL, NULL, - model.layers[il].ffn_down_shexp, NULL, NULL, + model.layers[il].ffn_up_shexp, NULL, model.layers[il].ffn_up_shexp_s, + model.layers[il].ffn_gate_shexp, NULL, model.layers[il].ffn_gate_shexp_s, + model.layers[il].ffn_down_shexp, NULL, model.layers[il].ffn_down_shexp_s, NULL, LLM_FFN_SILU, LLM_FFN_PAR, il); cb(ffn_shexp, "ffn_shexp", il); diff --git a/src/models/grok.cpp b/src/models/grok.cpp index 1a125d94e3c7..42486e60bead 100644 --- a/src/models/grok.cpp +++ b/src/models/grok.cpp @@ -80,10 +80,10 @@ void llama_model_grok::load_arch_tensors(llama_model_loader &) { } std::unique_ptr llama_model_grok::build_arch_graph(const llm_graph_params & params) const { - return std::make_unique(*this, params); + return std::make_unique(params); } -llama_model_grok::graph::graph(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) { +llama_model_grok::graph::graph(const llm_graph_params & params) : llm_graph_context(params) { const int64_t n_embd_head = hparams.n_embd_head_v(); GGML_ASSERT(n_embd_head == hparams.n_embd_head_k()); @@ -116,17 +116,8 @@ llama_model_grok::graph::graph(const llama_model & model, const llm_graph_params auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur, n_embd_head, n_head, n_head_kv, il); - Qcur = ggml_rope_ext( - ctx0, Qcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); - - Kcur = ggml_rope_ext( - ctx0, Kcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); + Qcur = build_rope(Qcur, inp_pos, il); + Kcur = build_rope(Kcur, inp_pos, il); cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); diff --git a/src/models/grovemoe.cpp b/src/models/grovemoe.cpp index b2648a13a81c..c86ddbbf5b3f 100644 --- a/src/models/grovemoe.cpp +++ b/src/models/grovemoe.cpp @@ -61,10 +61,10 @@ void llama_model_grovemoe::load_arch_tensors(llama_model_loader &) { } std::unique_ptr llama_model_grovemoe::build_arch_graph(const llm_graph_params & params) const { - return std::make_unique(*this, params); + return std::make_unique(params); } -llama_model_grovemoe::graph::graph(const llama_model & model, const llm_graph_params & params) : +llama_model_grovemoe::graph::graph(const llm_graph_params & params) : llm_graph_context(params) { const int64_t n_embd_head = hparams.n_embd_head_v(); const int64_t n_chunk_expert = n_expert / hparams.n_group_experts; @@ -100,14 +100,12 @@ llama_model_grovemoe::graph::graph(const llama_model & model, const llm_graph_pa Qcur = build_norm(Qcur, model.layers[il].attn_q_norm, NULL, LLM_NORM_RMS, il); cb(Qcur, "Qcur_normed", il); - Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); + Qcur = build_rope(Qcur, inp_pos, il); Kcur = build_norm(Kcur, model.layers[il].attn_k_norm, NULL, LLM_NORM_RMS, il); cb(Kcur, "Kcur_normed", il); - Kcur = ggml_rope_ext(ctx0, Kcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); + Kcur = build_rope(Kcur, inp_pos, il); cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); diff --git a/src/models/hrm-text.cpp b/src/models/hrm-text.cpp index 912a430f4047..8e3b34c57580 100644 --- a/src/models/hrm-text.cpp +++ b/src/models/hrm-text.cpp @@ -86,7 +86,7 @@ void llama_model_hrm_text::load_arch_tensors(llama_model_loader &) { } std::unique_ptr llama_model_hrm_text::build_arch_graph(const llm_graph_params & params) const { - return std::make_unique(*this, params); + return std::make_unique(params); } // one stack invocation: lps pre-norm decoder layers, then the parameterless final norm @@ -114,12 +114,10 @@ ggml_tensor * llama_model_hrm_text::graph::build_stack(llm_graph_input_attn_kv * auto [Qcur, Kcur, Vcur] = build_qkv(layer, cur, n_embd_head_k, n_head, n_head_kv, s); - Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); + Qcur = build_rope(Qcur, inp_pos, il); cb(Qcur, "Qcur", s); - Kcur = ggml_rope_ext(ctx0, Kcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); + Kcur = build_rope(Kcur, inp_pos, il); cb(Kcur, "Kcur", s); cur = build_attn(inp_attn, @@ -165,9 +163,8 @@ ggml_tensor * llama_model_hrm_text::graph::build_stack(llm_graph_input_attn_kv * return cur; } -llama_model_hrm_text::graph::graph(const llama_model & model, const llm_graph_params & params) : - llm_graph_context(params), - model(model) { +llama_model_hrm_text::graph::graph(const llm_graph_params & params) : + llm_graph_context(params) { ggml_tensor * cur; // {n_embd, n_tokens}, scaled by hparams.f_embedding_scale inside build_inp_embd diff --git a/src/models/hunyuan-dense.cpp b/src/models/hunyuan-dense.cpp index c137bd37c027..574a6412f558 100644 --- a/src/models/hunyuan-dense.cpp +++ b/src/models/hunyuan-dense.cpp @@ -1,6 +1,14 @@ #include "models.h" +llama_rope_params llama_model_hunyuan_dense::get_rope_params(const llama_cparams & cparams, int il) const { + auto res = llama_model::get_rope_params(cparams, il); + + res.k_norm = layers[il].attn_k_norm; + + return res; +} + std::unique_ptr llama_model_hunyuan_dense::build_arch_graph(const llm_graph_params & params) const { - return std::make_unique(*this, params); + return std::make_unique(params); } diff --git a/src/models/hunyuan-moe.cpp b/src/models/hunyuan-moe.cpp index 779575eee061..00659c88eb53 100644 --- a/src/models/hunyuan-moe.cpp +++ b/src/models/hunyuan-moe.cpp @@ -49,11 +49,19 @@ void llama_model_hunyuan_moe::load_arch_tensors(llama_model_loader &) { } } +llama_rope_params llama_model_hunyuan_moe::get_rope_params(const llama_cparams & cparams, int il) const { + auto res = llama_model::get_rope_params(cparams, il); + + res.k_norm = layers[il].attn_k_norm; + + return res; +} + std::unique_ptr llama_model_hunyuan_moe::build_arch_graph(const llm_graph_params & params) const { - return std::make_unique(*this, params); + return std::make_unique(params); } -llama_model_hunyuan_moe::graph::graph(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) { +llama_model_hunyuan_moe::graph::graph(const llm_graph_params & params) : llm_graph_context(params) { const int64_t n_embd_head = hparams.n_embd_head_v(); GGML_ASSERT(n_embd_head == hparams.n_embd_head_k()); @@ -84,28 +92,17 @@ llama_model_hunyuan_moe::graph::graph(const llama_model & model, const llm_graph // self-attention { - // rope freq factors for llama3; may return nullptr for llama2 and other models - ggml_tensor * rope_factors = model.get_rope_factors(cparams, il); - // compute Q and K and RoPE them auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur, n_embd_head, n_head, n_head_kv, il); - Qcur = ggml_rope_ext( - ctx0, Qcur, inp_pos, rope_factors, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); + Qcur = build_rope(Qcur, inp_pos, il); cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); cb(Vcur, "Vcur", il); - Kcur = ggml_rope_ext( - ctx0, Kcur, inp_pos, rope_factors, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); + Kcur = build_rope(Kcur, inp_pos, il); Kcur = build_norm(Kcur, model.layers[il].attn_k_norm, nullptr, diff --git a/src/models/hunyuan-vl.cpp b/src/models/hunyuan-vl.cpp index fa7183190e47..219ba39bda1a 100644 --- a/src/models/hunyuan-vl.cpp +++ b/src/models/hunyuan-vl.cpp @@ -54,20 +54,15 @@ void llama_model_hunyuan_vl::load_arch_tensors(llama_model_loader &) { } std::unique_ptr llama_model_hunyuan_vl::build_arch_graph(const llm_graph_params & params) const { - return std::make_unique(*this, params); + return std::make_unique(params); } -llama_model_hunyuan_vl::graph::graph(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) { +llama_model_hunyuan_vl::graph::graph(const llm_graph_params & params) : llm_graph_context(params) { const int64_t n_embd_head = hparams.n_embd_head_v(); GGML_ASSERT(n_embd_head == hparams.n_embd_head_k()); GGML_ASSERT(n_embd_head == n_rot); - const bool use_mrope = hparams.use_mrope(); - - int sections[4]; - std::copy(std::begin(hparams.rope_sections), std::begin(hparams.rope_sections) + 4, sections); - ggml_tensor * cur; ggml_tensor * inpL; @@ -94,38 +89,12 @@ llama_model_hunyuan_vl::graph::graph(const llama_model & model, const llm_graph_ cb(cur, "attn_norm", il); // self-attention { - // rope freq factors for llama3; may return nullptr for llama2 and other models - ggml_tensor * rope_factors = model.get_rope_factors(cparams, il); - // compute Q and K and RoPE them auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur, n_embd_head, n_head, n_head_kv, il); - if (use_mrope) { - Qcur = ggml_rope_multi( - ctx0, Qcur, inp_pos, rope_factors, - n_rot, sections, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); - - Kcur = ggml_rope_multi( - ctx0, Kcur, inp_pos, rope_factors, - n_rot, sections, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); - } else { - Qcur = ggml_rope_ext( - ctx0, Qcur, inp_pos, rope_factors, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); - - Kcur = ggml_rope_ext( - ctx0, Kcur, inp_pos, rope_factors, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); - } + Qcur = build_rope(Qcur, inp_pos, il); + Kcur = build_rope(Kcur, inp_pos, il); cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); diff --git a/src/models/hy-v3.cpp b/src/models/hy-v3.cpp index 080b30c1a5d1..c47c373f89ea 100644 --- a/src/models/hy-v3.cpp +++ b/src/models/hy-v3.cpp @@ -89,12 +89,12 @@ void llama_model_hy_v3::load_arch_tensors(llama_model_loader & ml) { std::unique_ptr llama_model_hy_v3::build_arch_graph(const llm_graph_params & params) const { if (params.gtype == LLM_GRAPH_TYPE_DECODER_MTP) { - return std::make_unique(*this, params); + return std::make_unique(params); } - return std::make_unique(*this, params); + return std::make_unique(params); } -llama_model_hy_v3::graph::graph(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) { +llama_model_hy_v3::graph::graph(const llm_graph_params & params) : llm_graph_context(params) { const int64_t n_embd_head = hparams.n_embd_head_v(); GGML_ASSERT(n_embd_head == hparams.n_embd_head_k()); @@ -119,19 +119,14 @@ llama_model_hy_v3::graph::graph(const llama_model & model, const llm_graph_param // self-attention { - ggml_tensor * rope_factors = model.get_rope_factors(cparams, il); auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur, n_embd_head, n_head, n_head_kv, il); Qcur = build_norm(Qcur, model.layers[il].attn_q_norm, nullptr, LLM_NORM_RMS, il); Kcur = build_norm(Kcur, model.layers[il].attn_k_norm, nullptr, LLM_NORM_RMS, il); - Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, rope_factors, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); - Kcur = ggml_rope_ext(ctx0, Kcur, inp_pos, rope_factors, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); + Qcur = build_rope(Qcur, inp_pos, il); + Kcur = build_rope(Kcur, inp_pos, il); cur = build_attn(inp_attn, model.layers[il].wo, model.layers[il].wo_b, model.layers[il].wo_s, @@ -225,7 +220,7 @@ llama_model_hy_v3::graph::graph(const llama_model & model, const llm_graph_param // hy_v3 decoder block -> final_layernorm (stored as nextn.shared_head_norm) -> // shared LM head (the main model's lm_head; the checkpoint has no separate // MTP head or MTP embeddings). -llama_model_hy_v3::graph_mtp::graph_mtp(const llama_model & model, const llm_graph_params & params) +llama_model_hy_v3::graph_mtp::graph_mtp(const llm_graph_params & params) : llm_graph_context(params) { GGML_ASSERT(hparams.n_layer_nextn > 0 && "HY_V3 MTP requires n_layer_nextn > 0"); @@ -251,9 +246,7 @@ llama_model_hy_v3::graph_mtp::graph_mtp(const llama_model & model, const llm_gra inp->embd = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, hparams.n_embd_inp(), n_tokens); ggml_set_input(inp->embd); - inp->h = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, hparams.n_embd, n_tokens); - ggml_set_input(inp->h); - ggml_set_name(inp->h, "mtp_h_input"); + inp->h = build_inp_embd_state(inp.get(), hparams.n_embd); ggml_tensor * tok_embd_w = layer.nextn.embed_tokens ? layer.nextn.embed_tokens : model.tok_embd; @@ -286,19 +279,14 @@ llama_model_hy_v3::graph_mtp::graph_mtp(const llama_model & model, const llm_gra cb(cur, "mtp_attn_norm", il); { - ggml_tensor * rope_factors = model.get_rope_factors(cparams, il); auto [Qcur, Kcur, Vcur] = build_qkv(layer, cur, n_embd_head, n_head, n_head_kv, il); Qcur = build_norm(Qcur, layer.attn_q_norm, nullptr, LLM_NORM_RMS, il); Kcur = build_norm(Kcur, layer.attn_k_norm, nullptr, LLM_NORM_RMS, il); - Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, rope_factors, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); - Kcur = ggml_rope_ext(ctx0, Kcur, inp_pos, rope_factors, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); + Qcur = build_rope(Qcur, inp_pos, il); + Kcur = build_rope(Kcur, inp_pos, il); const float kq_scale = 1.0f / sqrtf(float(n_embd_head)); diff --git a/src/models/hy-v4.cpp b/src/models/hy-v4.cpp index 06833e20bb42..386d504bcde7 100644 --- a/src/models/hy-v4.cpp +++ b/src/models/hy-v4.cpp @@ -154,8 +154,18 @@ void llama_model_hy_v4::load_arch_tensors(llama_model_loader &) { } } +llama_rope_params llama_model_hy_v4::get_rope_params_indexer(const llama_cparams & cparams, int il) const { + auto res = llama_model::get_rope_params_indexer(cparams, il); + + // [nope | rope], the same as the MLA keys + res.n_offs = hparams.indexer_head_size - res.n_rot; + res.type = hparams.rope_type; + + return res; +} + std::unique_ptr llama_model_hy_v4::build_arch_graph(const llm_graph_params & params) const { - return std::make_unique(*this, params); + return std::make_unique(params); } // reduce hc streams x[:,i,:] weighted by w[i,:] -> [n_embd, n_tokens] @@ -267,7 +277,6 @@ ggml_tensor * llama_model_hy_v4::graph::build_hc_head( } ggml_tensor * llama_model_hy_v4::graph::build_attention( - const llama_model & model, llm_graph_input_attn_k * inp_attn, ggml_tensor * cur, ggml_tensor * inp_pos, @@ -298,11 +307,9 @@ ggml_tensor * llama_model_hy_v4::graph::build_attention( ggml_row_size(kv_cmpr_pe->type, kv_lora_rank + n_embd_head_qk_rope), ggml_row_size(kv_cmpr_pe->type, kv_lora_rank)); - q_pe = ggml_rope_ext(ctx0, q_pe, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); + q_pe = build_rope(q_pe, inp_pos, il); cb(q_pe, "q_pe", il); - k_pe = ggml_rope_ext(ctx0, k_pe, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); + k_pe = build_rope(k_pe, inp_pos, il); cb(k_pe, "k_pe", il); kv_cmpr = build_norm(kv_cmpr, layer.attn_kv_a_norm, nullptr, LLM_NORM_RMS, il); @@ -339,7 +346,6 @@ ggml_tensor * llama_model_hy_v4::graph::build_attention( } ggml_tensor * llama_model_hy_v4::graph::build_indexer_top_k( - const llama_model & model, llm_graph_input_attn_k_dsa * inp_attn_dsa, ggml_tensor * cur, ggml_tensor * qr, @@ -349,17 +355,13 @@ ggml_tensor * llama_model_hy_v4::graph::build_indexer_top_k( const int64_t n_indexer_head = hparams.indexer_n_head; const int64_t n_embd_indexer = hparams.indexer_head_size; - const int64_t n_embd_indexer_rope = hparams.n_rot(); - const int64_t n_embd_indexer_nope = n_embd_indexer - n_embd_indexer_rope; // nope rows come first, so rope only the last n_embd_indexer_rope rows, same as the MLA path ggml_tensor * iq = ggml_mul_mat(ctx0, layer.indexer_attn_q_b, qr); iq = ggml_reshape_3d(ctx0, iq, n_embd_indexer, n_indexer_head, n_tokens); - iq = ggml_rope_ext(ctx0, iq, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, - freq_scale, ext_factor, attn_factor, beta_fast, beta_slow); - iq = ggml_rope_set_offset(iq, n_embd_indexer_nope); + iq = build_rope_indexer(iq, inp_pos, il); cb(iq, "indexer_q", il); ggml_tensor * ik = ggml_mul_mat(ctx0, layer.indexer_attn_k, cur); @@ -368,9 +370,7 @@ ggml_tensor * llama_model_hy_v4::graph::build_indexer_top_k( ik = ggml_reshape_3d(ctx0, ik, n_embd_indexer, 1, n_tokens); - ik = ggml_rope_ext(ctx0, ik, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, - freq_scale, ext_factor, attn_factor, beta_fast, beta_slow); - ik = ggml_rope_set_offset(ik, n_embd_indexer_nope); + ik = build_rope_indexer(ik, inp_pos, il); cb(ik, "indexer_k", il); // the reference applies a Hadamard rotation here, but it only helps its FP8 kernels. @@ -395,7 +395,7 @@ ggml_tensor * llama_model_hy_v4::graph::build_indexer_top_k( ggml_tensor * score = nullptr; if (cparams.fused_lid) { - score = ggml_lightning_indexer(ctx0, iq, ik, iw, inp_attn_dsa->self_attn_mask_lid.cnv); + score = ggml_lightning_indexer(ctx0, iq, ik, iw, inp_attn_dsa->self_attn_mask_lid.raw); cb(score, "indexer_score", il); res->add_fused_node({LLM_FUSED_OP_LIGHTNING_INDEXER, score, il}); } else { @@ -408,7 +408,7 @@ ggml_tensor * llama_model_hy_v4::graph::build_indexer_top_k( score = ggml_mul(ctx0, score, iw); score = ggml_sum_rows(ctx0, score); score = ggml_cont(ctx0, ggml_permute(ctx0, score, 2, 1, 0, 3)); - score = ggml_add(ctx0, score, inp_attn_dsa->self_attn_mask_lid.cnv); + score = ggml_add(ctx0, score, inp_attn_dsa->self_attn_mask_lid.raw); cb(score, "indexer_score", il); } @@ -419,7 +419,6 @@ ggml_tensor * llama_model_hy_v4::graph::build_indexer_top_k( llama_model_hy_v4::graph::attn_dsa llama_model_hy_v4::graph::build_attention_dsa( - const llama_model & model, llm_graph_input_attn_k_dsa * inp_attn_dsa, ggml_tensor * cur, ggml_tensor * inp_pos, @@ -438,7 +437,7 @@ llama_model_hy_v4::graph::build_attention_dsa( ggml_tensor * top_k = last_top_k; if (hparams.is_indexer_full(il)) { - top_k = build_indexer_top_k(model, inp_attn_dsa, cur, qr, inp_pos, il); + top_k = build_indexer_top_k(inp_attn_dsa, cur, qr, inp_pos, il); cb(top_k, "top_k", il); } GGML_ASSERT(top_k != nullptr); @@ -459,11 +458,9 @@ llama_model_hy_v4::graph::build_attention_dsa( ggml_row_size(kv_cmpr_pe->type, kv_lora_rank + n_embd_head_qk_rope), ggml_row_size(kv_cmpr_pe->type, kv_lora_rank)); - q_pe = ggml_rope_ext(ctx0, q_pe, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); + q_pe = build_rope(q_pe, inp_pos, il); cb(q_pe, "q_pe", il); - k_pe = ggml_rope_ext(ctx0, k_pe, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); + k_pe = build_rope(k_pe, inp_pos, il); cb(k_pe, "k_pe", il); kv_cmpr = build_norm(kv_cmpr, layer.attn_kv_a_norm, nullptr, LLM_NORM_RMS, il); @@ -495,7 +492,7 @@ llama_model_hy_v4::graph::build_attention_dsa( return { out, top_k }; } -llama_model_hy_v4::graph::graph(const llama_model & model, const llm_graph_params & params) : +llama_model_hy_v4::graph::graph(const llm_graph_params & params) : llm_graph_context(params) { const int64_t hc = hparams.dsv4_hc_mult; const int64_t n_embd_head_k = hparams.n_embd_head_k_mla(); @@ -528,11 +525,11 @@ llama_model_hy_v4::graph::graph(const llama_model & model, const llm_graph_param cb(cur, "attn_norm", il); if (is_dsa) { - attn_dsa attn = build_attention_dsa(model, inp_attn_dsa, cur, inp_pos, last_top_k, kq_scale, il); + attn_dsa attn = build_attention_dsa(inp_attn_dsa, cur, inp_pos, last_top_k, kq_scale, il); cur = attn.cur; last_top_k = attn.top_k; } else { - cur = build_attention(model, inp_attn, cur, inp_pos, kq_scale, il); + cur = build_attention(inp_attn, cur, inp_pos, kq_scale, il); } inpL = build_hc_post(cur, residual, attn_hc.post, il); diff --git a/src/models/internlm2.cpp b/src/models/internlm2.cpp index 8fc99856a28d..39ce67af6683 100644 --- a/src/models/internlm2.cpp +++ b/src/models/internlm2.cpp @@ -35,10 +35,10 @@ void llama_model_internlm2::load_arch_tensors(llama_model_loader &) { } std::unique_ptr llama_model_internlm2::build_arch_graph(const llm_graph_params & params) const { - return std::make_unique(*this, params); + return std::make_unique(params); } -llama_model_internlm2::graph::graph(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) { +llama_model_internlm2::graph::graph(const llm_graph_params & params) : llm_graph_context(params) { const int64_t n_embd_head = hparams.n_embd_head_v(); GGML_ASSERT(n_embd_head == hparams.n_embd_head_k()); @@ -71,17 +71,8 @@ llama_model_internlm2::graph::graph(const llama_model & model, const llm_graph_p auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur, n_embd_head, n_head, n_head_kv, il); - Qcur = ggml_rope_ext( - ctx0, Qcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); - - Kcur = ggml_rope_ext( - ctx0, Kcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); + Qcur = build_rope(Qcur, inp_pos, il); + Kcur = build_rope(Kcur, inp_pos, il); cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); diff --git a/src/models/jais.cpp b/src/models/jais.cpp index 446b56617555..daa3315bfc30 100644 --- a/src/models/jais.cpp +++ b/src/models/jais.cpp @@ -49,10 +49,10 @@ void llama_model_jais::load_arch_tensors(llama_model_loader &) { } std::unique_ptr llama_model_jais::build_arch_graph(const llm_graph_params & params) const { - return std::make_unique(*this, params); + return std::make_unique(params); } -llama_model_jais::graph::graph(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) { +llama_model_jais::graph::graph(const llm_graph_params & params) : llm_graph_context(params) { const int64_t n_embd_head = hparams.n_embd_head_v(); GGML_ASSERT(n_embd_head == hparams.n_embd_head_k()); diff --git a/src/models/jais2.cpp b/src/models/jais2.cpp index 4b24e697f5c6..8376423ea5fd 100644 --- a/src/models/jais2.cpp +++ b/src/models/jais2.cpp @@ -46,12 +46,12 @@ void llama_model_jais2::load_arch_tensors(llama_model_loader &) { } std::unique_ptr llama_model_jais2::build_arch_graph(const llm_graph_params & params) const { - return std::make_unique(*this, params); + return std::make_unique(params); } // JAIS-2 model graph builder // Uses: LayerNorm (not RMSNorm), relu2 activation, separate Q/K/V, RoPE embeddings -llama_model_jais2::graph::graph(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) { +llama_model_jais2::graph::graph(const llm_graph_params & params) : llm_graph_context(params) { const int64_t n_embd_head = hparams.n_embd_head_v(); GGML_ASSERT(n_embd_head == hparams.n_embd_head_k()); @@ -84,17 +84,8 @@ llama_model_jais2::graph::graph(const llama_model & model, const llm_graph_param n_embd_head, n_head, n_head_kv, il); // Apply RoPE - Qcur = ggml_rope_ext( - ctx0, Qcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); - - Kcur = ggml_rope_ext( - ctx0, Kcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); + Qcur = build_rope(Qcur, inp_pos, il); + Kcur = build_rope(Kcur, inp_pos, il); cb(Qcur, "Qcur_rope", il); cb(Kcur, "Kcur_rope", il); diff --git a/src/models/jamba.cpp b/src/models/jamba.cpp index ed4c9f13e594..842df63b6a80 100644 --- a/src/models/jamba.cpp +++ b/src/models/jamba.cpp @@ -102,10 +102,10 @@ void llama_model_jamba::load_arch_tensors(llama_model_loader &) { } std::unique_ptr llama_model_jamba::build_arch_graph(const llm_graph_params & params) const { - return std::make_unique(*this, params); + return std::make_unique(params); } -llama_model_jamba::graph::graph(const llama_model & model, const llm_graph_params & params) : llm_build_mamba_base(params) { +llama_model_jamba::graph::graph(const llm_graph_params & params) : llm_build_mamba_base(params) { const int64_t n_embd_head = hparams.n_embd_head_v(); ggml_tensor * cur; @@ -125,7 +125,7 @@ llama_model_jamba::graph::graph(const llama_model & model, const llm_graph_param cb(cur, "attn_norm", il); if (n_head_kv == 0) { - cur = build_mamba_layer(inp_hybrid->get_recr(), cur, model, ubatch, il); + cur = build_mamba_layer(inp_hybrid->get_recr(), cur, ubatch, il); } else { // Attention diff --git a/src/models/jina-bert-v2.cpp b/src/models/jina-bert-v2.cpp index 86ff1c84d1a6..e036018d2a90 100644 --- a/src/models/jina-bert-v2.cpp +++ b/src/models/jina-bert-v2.cpp @@ -61,6 +61,6 @@ void llama_model_jina_bert_v2::load_arch_tensors(llama_model_loader & ml) { } std::unique_ptr llama_model_jina_bert_v2::build_arch_graph(const llm_graph_params & params) const { - return std::make_unique(*this, params); + return std::make_unique(params); } diff --git a/src/models/jina-bert-v3.cpp b/src/models/jina-bert-v3.cpp index 78cf9d835976..a4b7ef4283c0 100644 --- a/src/models/jina-bert-v3.cpp +++ b/src/models/jina-bert-v3.cpp @@ -44,6 +44,6 @@ void llama_model_jina_bert_v3::load_arch_tensors(llama_model_loader &) { } std::unique_ptr llama_model_jina_bert_v3::build_arch_graph(const llm_graph_params & params) const { - return std::make_unique(*this, params); + return std::make_unique(params); } diff --git a/src/models/k2-horizon.cpp b/src/models/k2-horizon.cpp index 845940913f62..534efa523a0c 100644 --- a/src/models/k2-horizon.cpp +++ b/src/models/k2-horizon.cpp @@ -117,7 +117,7 @@ void llama_model_k2_horizon::load_arch_tensors(llama_model_loader &) { } std::unique_ptr llama_model_k2_horizon::build_arch_graph(const llm_graph_params & params) const { - return std::make_unique(*this, params); + return std::make_unique(params); } // RMS norm over n_groups equal slices of ne[0], then one full-width weight @@ -210,7 +210,7 @@ ggml_tensor * llama_model_k2_horizon::graph::build_routed_value(const llama_laye return value_out; } -llama_model_k2_horizon::graph::graph(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) { +llama_model_k2_horizon::graph::graph(const llm_graph_params & params) : llm_graph_context(params) { const int64_t n_embd_head = hparams.n_embd_head_v(); GGML_ASSERT(n_embd_head == hparams.n_embd_head_k()); @@ -261,12 +261,8 @@ llama_model_k2_horizon::graph::graph(const llama_model & model, const llm_graph_ Kcur = build_norm(Kcur, layer.attn_k_norm, NULL, LLM_NORM_RMS, il); } - Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); - Kcur = ggml_rope_ext(ctx0, Kcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); + Qcur = build_rope(Qcur, inp_pos, il); + Kcur = build_rope(Kcur, inp_pos, il); cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); diff --git a/src/models/kimi-k3.cpp b/src/models/kimi-k3.cpp index 27ceffe923d0..2e91c5243a64 100644 --- a/src/models/kimi-k3.cpp +++ b/src/models/kimi-k3.cpp @@ -1,6 +1,5 @@ #include "models.h" -#include #include "llama-memory-recurrent.h" // @@ -167,7 +166,7 @@ void llama_model_kimi_k3::load_arch_tensors(llama_model_loader &) { } std::unique_ptr llama_model_kimi_k3::build_arch_graph(const llm_graph_params & params) const { - return std::make_unique(*this, params); + return std::make_unique(params); } // situ(gate, up) = beta*tanh(gate/beta)*sigmoid(gate) * linear_beta*tanh(up/linear_beta) @@ -233,8 +232,8 @@ ggml_tensor * llama_model_kimi_k3::graph::res_mix(ggml_tensor * cur, ggml_tensor return out; } -llama_model_kimi_k3::graph::graph(const llama_model & model, const llm_graph_params & params) : - llm_build_delta_net_base(params), model(model) { +llama_model_kimi_k3::graph::graph(const llm_graph_params & params) : + llm_build_delta_net_base(params) { ggml_tensor * cur; ggml_tensor * inpL; @@ -353,13 +352,11 @@ llama_model_kimi_k3::graph::graph(const llama_model & model, const llm_graph_par // // causal conv1d over one of Q/K/V. `qkv` selects which third of the conv state to use -static ggml_tensor * kimi_k3_conv1d(ggml_cgraph * gf, ggml_context * ctx0, - ggml_tensor * conv_states_all, ggml_tensor * conv_state_all, - int64_t qkv, ggml_tensor * x, ggml_tensor * proj_w, ggml_tensor * conv_w, - int64_t d_conv, int64_t head_dim, int64_t n_head, - int64_t n_seq_tokens, int64_t n_seqs, int64_t n_tokens, int64_t kv_head, - int64_t mem_size, int64_t K_rs) { - const int64_t d_inner = head_dim * n_head; +ggml_tensor * llama_model_kimi_k3::graph::build_kda_conv1d( + llm_graph_input_rs * inp_rs, ggml_tensor * conv_states_all, ggml_tensor * conv_state_all, + int64_t qkv, ggml_tensor * x, ggml_tensor * proj_w, ggml_tensor * conv_w, + int64_t d_conv, int64_t head_dim, int64_t n_head_kda, int64_t n_seq_tokens, int64_t n_seqs) { + const int64_t d_inner = head_dim * n_head_kda; const int64_t conv_state_size = (d_conv - 1) * d_inner; const int64_t n_embd_r_total = 3 * conv_state_size; @@ -372,26 +369,15 @@ static ggml_tensor * kimi_k3_conv1d(ggml_cgraph * gf, ggml_context * ctx0, ggml_tensor * x_3d = ggml_reshape_3d(ctx0, x_proj, d_inner, n_seq_tokens, n_seqs); ggml_tensor * conv_x = ggml_concat(ctx0, conv_state_x, ggml_transpose(ctx0, x_3d), 0); - // group s holds the conv window s tokens back. - // [TAG_RECURRENT_ROLLBACK_SPLITS]: the last K_rs tokens must share one ubatch. - for (int64_t s = 0; s < K_rs; ++s) { - const int64_t s_idx = std::max(0, n_seq_tokens - s); - ggml_tensor * conv_x_s = ggml_view_3d(ctx0, conv_x, d_conv - 1, d_inner, n_seqs, - conv_x->nb[1], conv_x->nb[2], s_idx * conv_x->nb[0]); - ggml_build_forward_expand(gf, - ggml_cpy(ctx0, conv_x_s, - ggml_view_3d(ctx0, conv_states_all, d_conv - 1, d_inner, n_seqs, - (d_conv - 1) * ggml_element_size(conv_states_all), - n_embd_r_total * ggml_element_size(conv_states_all), - ((s * mem_size + kv_head) * n_embd_r_total + qkv * conv_state_size) * ggml_element_size(conv_states_all)))); - } + build_conv_state_store(inp_rs, conv_states_all, conv_x, d_conv - 1, + ggml_row_size(conv_states_all->type, qkv * conv_state_size)); ggml_tensor * conv_weight = ggml_reshape_2d(ctx0, conv_w, d_conv, d_inner); ggml_tensor * Xcur = ggml_ssm_conv(ctx0, conv_x, conv_weight); Xcur = ggml_reshape_2d(ctx0, Xcur, d_inner, n_tokens); Xcur = ggml_silu(ctx0, Xcur); - return ggml_reshape_4d(ctx0, Xcur, head_dim, n_head, n_seq_tokens, n_seqs); + return ggml_reshape_4d(ctx0, Xcur, head_dim, n_head_kda, n_seq_tokens, n_seqs); } ggml_tensor * llama_model_kimi_k3::graph::build_kda_layer( @@ -400,17 +386,13 @@ ggml_tensor * llama_model_kimi_k3::graph::build_kda_layer( int64_t d_inner, int64_t n_seq_tokens, int64_t n_seqs, int il) { const auto * mctx_cur = inp_rs->mctx; - const auto kv_head = mctx_cur->get_head(); ggml_tensor * conv_states_all = mctx_cur->get_r_l(il); ggml_tensor * conv_state_all = build_rs(inp_rs, conv_states_all, hparams.n_embd_r(), n_seqs); - const int64_t mem_size = mctx_cur->get_size(); - const int64_t K_rs = (int64_t) cparams.n_rs_seq + 1; - - ggml_tensor * Qcur = kimi_k3_conv1d(gf, ctx0, conv_states_all, conv_state_all, 0, cur, layer.wq, layer.ssm_q_conv, d_conv, head_dim, n_head_kda, n_seq_tokens, n_seqs, n_tokens, kv_head, mem_size, K_rs); - ggml_tensor * Kcur = kimi_k3_conv1d(gf, ctx0, conv_states_all, conv_state_all, 1, cur, layer.wk, layer.ssm_k_conv, d_conv, head_dim, n_head_kda, n_seq_tokens, n_seqs, n_tokens, kv_head, mem_size, K_rs); - ggml_tensor * Vcur = kimi_k3_conv1d(gf, ctx0, conv_states_all, conv_state_all, 2, cur, layer.wv, layer.ssm_v_conv, d_conv, head_dim, n_head_kda, n_seq_tokens, n_seqs, n_tokens, kv_head, mem_size, K_rs); + ggml_tensor * Qcur = build_kda_conv1d(inp_rs, conv_states_all, conv_state_all, 0, cur, layer.wq, layer.ssm_q_conv, d_conv, head_dim, n_head_kda, n_seq_tokens, n_seqs); + ggml_tensor * Kcur = build_kda_conv1d(inp_rs, conv_states_all, conv_state_all, 1, cur, layer.wk, layer.ssm_k_conv, d_conv, head_dim, n_head_kda, n_seq_tokens, n_seqs); + ggml_tensor * Vcur = build_kda_conv1d(inp_rs, conv_states_all, conv_state_all, 2, cur, layer.wv, layer.ssm_v_conv, d_conv, head_dim, n_head_kda, n_seq_tokens, n_seqs); cb(Qcur, "kda_q_conv", il); cb(Kcur, "kda_k_conv", il); cb(Vcur, "kda_v_conv", il); diff --git a/src/models/kimi-k31.cpp b/src/models/kimi-k31.cpp new file mode 100644 index 000000000000..0c0ca2af259d --- /dev/null +++ b/src/models/kimi-k31.cpp @@ -0,0 +1,701 @@ +#include "models.h" + +#include +#include "llama-memory-recurrent.h" +#include "llama-memory-hybrid-idx.h" + +// +// Kimi-K3.1 text model: Kimi-K3 with DSA (indexer + top-k) on the full-attention layers. +// KDA layers, cross-layer residuals, latent MoE, situ and the MLA shapes are unchanged. +// 1. full-attention layers run a lightning indexer over the cached latent, keep the +// top-k cells and attend only to those (the "Gated DSA" of the model card) +// 2. the indexer keeps its own per-token key cache, one cell per attention cell +// + +void llama_model_kimi_k31::load_arch_hparams(llama_model_loader & ml) { + ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps); + ml.get_key(LLM_KV_ATTENTION_KEY_LENGTH_MLA, hparams.n_embd_head_k_mla_impl); + ml.get_key(LLM_KV_ATTENTION_VALUE_LENGTH_MLA, hparams.n_embd_head_v_mla_impl); + ml.get_key(LLM_KV_ATTENTION_Q_LORA_RANK, hparams.n_lora_q, false); + ml.get_key(LLM_KV_ATTENTION_KV_LORA_RANK, hparams.n_lora_kv); + ml.get_key(LLM_KV_SSM_CONV_KERNEL, hparams.ssm_d_conv); + ml.get_key(LLM_KV_KDA_HEAD_DIM, hparams.n_embd_head_kda); + ml.get_key(LLM_KV_KDA_GATE_LOWER_BOUND, hparams.kda_gate_lower_bound, false); + + // the MLA cache holds the compressed latent + // set it here too, as older GGUFs have no value_length key + hparams.n_embd_head_v_full = hparams.n_lora_kv; + + // n_head_kv == 0 marks a KDA (recurrent) layer, as in kimi-linear + for (uint32_t i = 0; i < hparams.n_layer(); ++i) { + hparams.is_recr_impl[i] = hparams.n_head_kv(i) == 0; + } + + ml.get_key_or_arr(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp_arr, hparams.n_layer_all); + ml.get_key(LLM_KV_EXPERT_SHARED_COUNT, hparams.n_expert_shared); + ml.get_key(LLM_KV_LEADING_DENSE_BLOCK_COUNT, hparams.n_layer_dense_lead, false); + ml.get_key(LLM_KV_EXPERT_WEIGHTS_SCALE, hparams.expert_weights_scale, false); + ml.get_key(LLM_KV_EXPERT_WEIGHTS_NORM, hparams.expert_weights_norm, false); + ml.get_key(LLM_KV_EXPERT_GATING_FUNC, hparams.expert_gating_func); + ml.get_key(LLM_KV_EXPERT_LATENT_LENGTH, hparams.n_expert_latent, false); + + ml.get_key(LLM_KV_ATTN_RES_BLOCK_SIZE, hparams.attn_res_block_size); + ml.get_key(LLM_KV_ACTIVATION_SITU_BETA, hparams.situ_beta); + ml.get_key(LLM_KV_ACTIVATION_SITU_LINEAR_BETA, hparams.situ_linear_beta); + + // DSA indexer, present on the dense-attention layers only + hparams.f_norm_eps = 1e-5f; // eps of the indexer key LayerNorm + ml.get_key(LLM_KV_ATTENTION_LAYERNORM_EPS, hparams.f_norm_eps, false); + ml.get_key(LLM_KV_ATTENTION_INDEXER_HEAD_COUNT, hparams.indexer_n_head); + ml.get_key(LLM_KV_ATTENTION_INDEXER_KEY_LENGTH, hparams.indexer_head_size); + ml.get_key(LLM_KV_ATTENTION_INDEXER_TOP_K, hparams.indexer_top_k); + // every DSA layer scores its own selection unless the GGUF says otherwise + std::fill(hparams.is_indexer_full_impl.begin(), hparams.is_indexer_full_impl.end(), 1); + ml.get_key_or_arr(LLM_KV_ATTENTION_INDEXER_TYPES, hparams.is_indexer_full_impl, hparams.n_layer(), false); + + if (hparams.indexer_top_k > 0) { + GGML_ASSERT(hparams.indexer_n_head > 0 && hparams.indexer_head_size > 0); + GGML_ASSERT(hparams.n_lora_q > 0 && "the indexer queries come from the MLA q latent"); + } + + switch (hparams.n_layer()) { + case 93: type = LLM_TYPE_2_8T_A50B; break; // Kimi-K3.1-Pro + default: type = LLM_TYPE_UNKNOWN; + } +} + +void llama_model_kimi_k31::load_arch_tensors(llama_model_loader &) { + LLAMA_LOAD_LOCALS; + + const int64_t n_embd_latent = hparams.n_expert_latent > 0 ? hparams.n_expert_latent : n_embd; + + tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, 0); + + output_norm = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd}, 0); + output = create_tensor(tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab}, 0); + + if (hparams.attn_res_block_size > 0) { + output_res_score = create_tensor(tn(LLM_TENSOR_OUTPUT_RES_SCORE, "weight"), {n_embd}, 0); + } + + for (int i = 0; i < n_layer; ++i) { + auto & layer = layers[i]; + + layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd}, 0); + layer.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd}, 0); + + if (hparams.attn_res_block_size > 0) { + layer.attn_res_score = create_tensor(tn(LLM_TENSOR_ATTN_RES_SCORE, "weight", i), {n_embd}, 0); + layer.ffn_res_score = create_tensor(tn(LLM_TENSOR_FFN_RES_SCORE, "weight", i), {n_embd}, 0); + } + + const int64_t head_dim = hparams.n_embd_head_kda; + const int64_t d_conv = hparams.ssm_d_conv; + const int64_t d_inner = head_dim * n_head; + + if (hparams.is_recr(i)) { + // conv1d may be stored 4D [d_conv, 1, d_inner, 1] or 3D (quantization drops the trailing 1) + auto conv = [&](llm_tensor tid) { + ggml_tensor * t = create_tensor(tn(tid, "weight", i), {d_conv, 1, d_inner, 1}, TENSOR_NOT_REQUIRED); + return t ? t : create_tensor(tn(tid, "weight", i), {d_conv, 1, d_inner}, 0); + }; + layer.ssm_q_conv = conv(LLM_TENSOR_SSM_CONV1D_Q); + layer.ssm_k_conv = conv(LLM_TENSOR_SSM_CONV1D_K); + layer.ssm_v_conv = conv(LLM_TENSOR_SSM_CONV1D_V); + + create_tensor_qkv(layer, i, n_embd, d_inner, d_inner, d_inner, 0); + + layer.ssm_f_a = create_tensor(tn(LLM_TENSOR_SSM_F_A, "weight", i), {n_embd, head_dim}, 0); + layer.ssm_f_b = create_tensor(tn(LLM_TENSOR_SSM_F_B, "weight", i), {head_dim, d_inner}, 0); + layer.ssm_beta = create_tensor(tn(LLM_TENSOR_SSM_BETA, "weight", i), {n_embd, n_head}, 0); + + // K3's A_log is a plain 1-D [n_head] tensor (kimi-linear's is padded) + layer.ssm_a = create_tensor(tn(LLM_TENSOR_SSM_A_NOSCAN, i), {n_head}, 0); + layer.ssm_dt_b = create_tensor(tn(LLM_TENSOR_SSM_DT, "bias", i), {d_inner}, 0); + + // K3 uses a single full-rank gate instead of kimi-linear's g_a/g_b pair + layer.ssm_g = create_tensor(tn(LLM_TENSOR_SSM_G, "weight", i), {n_embd, d_inner}, 0); + layer.ssm_o_norm = create_tensor(tn(LLM_TENSOR_SSM_NORM, "weight", i), {head_dim}, 0); + layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), {d_inner, n_embd}, 0); + } else { + const int64_t q_lora_rank = hparams.n_lora_q; + const int64_t kv_lora_rank = hparams.n_lora_kv; + const int64_t n_embd_head_k = hparams.n_embd_head_k_mla(); + const int64_t n_embd_head_v = hparams.n_embd_head_v_mla(); + const int64_t qk_rope_head_dim = hparams.n_rot(); + const int64_t qk_nope_head_dim = n_embd_head_k - qk_rope_head_dim; + + layer.attn_q_a_norm = create_tensor(tn(LLM_TENSOR_ATTN_Q_A_NORM, "weight", i), {q_lora_rank}, TENSOR_NOT_REQUIRED); + layer.attn_kv_a_norm = create_tensor(tn(LLM_TENSOR_ATTN_KV_A_NORM, "weight", i), {kv_lora_rank}, 0); + + if (layer.attn_q_a_norm) { + layer.wq_a = create_tensor(tn(LLM_TENSOR_ATTN_Q_A, "weight", i), {n_embd, q_lora_rank}, 0); + layer.wq_b = create_tensor(tn(LLM_TENSOR_ATTN_Q_B, "weight", i), {q_lora_rank, n_head * n_embd_head_k}, 0); + } else { + layer.wq = create_tensor(tn(LLM_TENSOR_ATTN_Q, "weight", i), {n_embd, n_head * n_embd_head_k}, 0); + } + + layer.wkv_a_mqa = create_tensor(tn(LLM_TENSOR_ATTN_KV_A_MQA, "weight", i), {n_embd, kv_lora_rank + qk_rope_head_dim}, 0); + layer.wkv_b = create_tensor(tn(LLM_TENSOR_ATTN_KV_B, "weight", i), + {kv_lora_rank, n_head * (qk_nope_head_dim + n_embd_head_v)}, + TENSOR_NOT_REQUIRED | TENSOR_SKIP_IF_VIRTUAL); + if (!layer.wkv_b) { + layer.wk_b = create_tensor(tn(LLM_TENSOR_ATTN_K_B, "weight", i), {qk_nope_head_dim, kv_lora_rank, n_head}, 0); + layer.wv_b = create_tensor(tn(LLM_TENSOR_ATTN_V_B, "weight", i), {kv_lora_rank, n_embd_head_v, n_head}, 0); + } + + // K3: sigmoid output gate applied to the attention output before o_proj + layer.wqkv_gate = create_tensor(tn(LLM_TENSOR_ATTN_GATE, "weight", i), {n_embd, n_head * n_embd_head_v}, TENSOR_NOT_REQUIRED); + + // DSA lightning indexer, present on the layers the GGUF marks as indexer layers + if (hparams.indexer_top_k > 0 && hparams.is_indexer_full(i)) { + const int64_t n_idx_head = hparams.indexer_n_head; + const int64_t n_idx_dim = hparams.indexer_head_size; + + layer.indexer_attn_q_b = create_tensor(tn(LLM_TENSOR_INDEXER_ATTN_Q_B, "weight", i), {q_lora_rank, n_idx_head * n_idx_dim}, 0); + layer.indexer_attn_k = create_tensor(tn(LLM_TENSOR_INDEXER_ATTN_K, "weight", i), {n_embd, n_idx_dim}, 0); + layer.indexer_k_norm = create_tensor(tn(LLM_TENSOR_INDEXER_K_NORM, "weight", i), {n_idx_dim}, 0); + layer.indexer_k_norm_b = create_tensor(tn(LLM_TENSOR_INDEXER_K_NORM, "bias", i), {n_idx_dim}, 0); + layer.indexer_proj = create_tensor(tn(LLM_TENSOR_INDEXER_PROJ, "weight", i), {n_embd, n_idx_head}, 0); + } + + layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_head * n_embd_head_v, n_embd}, 0); + } + + if (i < (int) hparams.n_layer_dense_lead) { + layer.ffn_gate = create_tensor(tn(LLM_TENSOR_FFN_GATE, "weight", i), {n_embd, n_ff}, 0); + layer.ffn_down = create_tensor(tn(LLM_TENSOR_FFN_DOWN, "weight", i), {n_ff, n_embd}, 0); + layer.ffn_up = create_tensor(tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff}, 0); + } else { + const int64_t n_ff_exp = hparams.n_ff_exp(); + + layer.ffn_gate_inp = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP, "weight", i), {n_embd, n_expert}, 0); + layer.ffn_exp_probs_b = create_tensor(tn(LLM_TENSOR_FFN_EXP_PROBS_B, "bias", i), {n_expert}, 0); + + // routed experts live in the latent space + layer.ffn_gate_exps = create_tensor(tn(LLM_TENSOR_FFN_GATE_EXPS, "weight", i), {n_embd_latent, n_ff_exp, n_expert}, 0); + layer.ffn_down_exps = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS, "weight", i), {n_ff_exp, n_embd_latent, n_expert}, 0); + layer.ffn_up_exps = create_tensor(tn(LLM_TENSOR_FFN_UP_EXPS, "weight", i), {n_embd_latent, n_ff_exp, n_expert}, 0); + + if (hparams.n_expert_latent > 0) { + layer.ffn_routed_down = create_tensor(tn(LLM_TENSOR_FFN_ROUTED_DOWN, "weight", i), {n_embd, n_embd_latent}, 0); + layer.ffn_routed_up = create_tensor(tn(LLM_TENSOR_FFN_ROUTED_UP, "weight", i), {n_embd_latent, n_embd}, 0); + layer.ffn_routed_norm = create_tensor(tn(LLM_TENSOR_FFN_ROUTED_NORM, "weight", i), {n_embd_latent}, TENSOR_NOT_REQUIRED); + } + + // shared experts stay at n_embd, width = moe_intermediate_size * n_expert_shared + const int64_t n_ff_shexp = n_ff_exp * (hparams.n_expert_shared > 0 ? hparams.n_expert_shared : 1); + layer.ffn_gate_shexp = create_tensor(tn(LLM_TENSOR_FFN_GATE_SHEXP, "weight", i), {n_embd, n_ff_shexp}, TENSOR_NOT_REQUIRED); + layer.ffn_down_shexp = create_tensor(tn(LLM_TENSOR_FFN_DOWN_SHEXP, "weight", i), {n_ff_shexp, n_embd}, TENSOR_NOT_REQUIRED); + layer.ffn_up_shexp = create_tensor(tn(LLM_TENSOR_FFN_UP_SHEXP, "weight", i), {n_embd, n_ff_shexp}, TENSOR_NOT_REQUIRED); + } + } +} + +std::unique_ptr llama_model_kimi_k31::build_arch_graph(const llm_graph_params & params) const { + return std::make_unique(params); +} + +// situ(gate, up) = beta*tanh(gate/beta)*sigmoid(gate) * linear_beta*tanh(up/linear_beta) +// linear_beta <= 0 disables the transform on the up branch +static ggml_tensor * kimi_k31_situ(ggml_context * ctx0, ggml_tensor * gate, ggml_tensor * up, + float beta, float linear_beta) { + ggml_tensor * a = ggml_scale(ctx0, ggml_tanh(ctx0, ggml_scale(ctx0, gate, 1.0f/beta)), beta); + a = ggml_mul(ctx0, a, ggml_sigmoid(ctx0, gate)); + + if (linear_beta > 0.0f) { + up = ggml_scale(ctx0, ggml_tanh(ctx0, ggml_scale(ctx0, up, 1.0f/linear_beta)), linear_beta); + } + return ggml_mul(ctx0, a, up); +} + +// +// cross-layer residual attention +// + +// layout is [n_embd, n_ckpt, n_tokens]: rms_norm reduces over ne0, dsv4_hc_pre over ne1 +// append the new checkpoint, do not re-fold the whole chain +void llama_model_kimi_k31::graph::res_push(ggml_tensor * cur, int64_t n_embd, int64_t n_tokens) { + ggml_tensor * ckpt = ggml_reshape_3d(ctx0, cur, n_embd, 1, n_tokens); + + resi_stack = resi_stack ? ggml_concat(ctx0, resi_stack, ckpt, 1) : ckpt; +} + +ggml_tensor * llama_model_kimi_k31::graph::res_mix(ggml_tensor * cur, ggml_tensor * score_w, + int64_t n_tokens, int il) { + if (!resi_stack) { + return cur; // layer 0: nothing banked yet + } + + const int n_ckpt = (int) resi_stack->ne[1]; + const float eps = hparams.f_norm_rms_eps; + + ggml_tensor * src = resi_stack; // [n_embd, n_ckpt, n_tokens] + + // one rms_norm scores all checkpoints at once + // note: the scores use the normalized values, but the sum below uses the raw ones + ggml_tensor * sc_src = ggml_rms_norm(ctx0, src, eps); + sc_src = ggml_mul(ctx0, sc_src, score_w); + sc_src = ggml_sum_rows(ctx0, sc_src); // [1, n_ckpt, n_tokens] + sc_src = ggml_reshape_2d(ctx0, sc_src, n_ckpt, n_tokens); + + // the current residual stream is scored apart, so the stack stays append-only + ggml_tensor * sc_cur = ggml_rms_norm(ctx0, cur, eps); + sc_cur = ggml_mul(ctx0, sc_cur, score_w); + sc_cur = ggml_sum_rows(ctx0, sc_cur); // [1, n_tokens] + + ggml_tensor * scores = ggml_concat(ctx0, sc_src, sc_cur, 0); // [n_ckpt+1, n_tokens] + ggml_tensor * probs = ggml_soft_max(ctx0, scores); // over ne0 = n_ckpt+1 + cb(probs, "res_probs", il); + + // split the sum: hc_pre handles the stack, a broadcast-multiply the current stream + ggml_tensor * p_src = ggml_cont(ctx0, ggml_view_2d(ctx0, probs, n_ckpt, n_tokens, probs->nb[1], 0)); + ggml_tensor * p_cur = ggml_cont(ctx0, ggml_view_2d(ctx0, probs, 1, n_tokens, probs->nb[1], + probs->nb[0] * n_ckpt)); + + ggml_tensor * out = ggml_dsv4_hc_pre(ctx0, src, p_src); + out = ggml_add(ctx0, out, ggml_mul(ctx0, cur, p_cur)); + + return out; +} + +llama_model_kimi_k31::graph::graph(const llm_graph_params & params) : + llm_build_delta_net_base(params) { + + ggml_tensor * cur; + ggml_tensor * inpL; + + inpL = build_inp_embd(model.tok_embd); + cb(inpL, "inp_embd", -1); + + // K3.1 MLA is nope-only, so there is no position input + + auto * inp_k = build_inp_mem_hybrid_k(); + auto * inp_rs = inp_k->get_recr(); + auto * inp_attn = inp_k->get_attn(); + + // the indexer cache sits next to the attention cache and shares its cells + const auto * mctx_hyb = static_cast(inp_k->mctx); + const auto * mctx_idx = mctx_hyb->get_idx(); + if (mctx_idx) { + GGML_ASSERT(mctx_idx->get_n_kv() == inp_attn->mctx->get_n_kv() && + "the indexer cache must track the attention cache cell for cell"); + } + + ggml_tensor * inp_out_ids = build_inp_out_ids(); + + const int64_t n_head_kda = hparams.n_head(); + const int64_t head_dim = hparams.n_embd_head_kda; + const int64_t d_conv = hparams.ssm_d_conv; + const int64_t d_inner = n_head_kda * head_dim; + const int64_t n_seqs = ubatch.n_seqs; + const int64_t n_seq_tokens = ubatch.n_seq_tokens; + + GGML_ASSERT(n_seqs != 0); + GGML_ASSERT(ubatch.equal_seqs()); + GGML_ASSERT(ubatch.n_tokens == n_seq_tokens * n_seqs); + + const int64_t n_embd_head_k_mla = hparams.n_embd_head_k_mla(); + const int64_t kv_lora_rank = hparams.n_lora_kv; + const int64_t n_embd_head_qk_rope = hparams.n_rot(); + const int64_t n_embd_head_qk_nope = n_embd_head_k_mla - n_embd_head_qk_rope; + const float kq_scale_mla = 1.0f / sqrtf((float) n_embd_head_k_mla); + + const uint32_t res_bs = hparams.attn_res_block_size; + const bool use_attn_res = res_bs > 0; + const int64_t n_embd_latent = hparams.n_expert_latent > 0 ? hparams.n_expert_latent : n_embd; + + ggml_tensor * prev_sel = nullptr; + + for (int il = 0; il < n_layer; ++il) { + const auto & layer = model.layers[il]; + + // the residual stream, banked on checkpoint layers and then restarted + // from the attention output alone + ggml_tensor * prefix_sum = inpL; + + cur = use_attn_res ? res_mix(prefix_sum, layer.attn_res_score, n_tokens, il) + : prefix_sum; + + bool banked = false; + if (use_attn_res && (uint32_t) il % res_bs == 0) { + res_push(prefix_sum, n_embd, n_tokens); // banks the RAW layer input, not `cur` + banked = true; + } + + cur = build_norm(cur, layer.attn_norm, NULL, LLM_NORM_RMS, il); + cb(cur, "attn_norm", il); + ggml_build_forward_expand(gf, cur); + + if (hparams.is_recr(il)) { + cur = build_kda_layer(cur, layer, inp_rs, d_conv, head_dim, n_head_kda, + d_inner, n_seq_tokens, n_seqs, il); + } else { + dsa_out dsa = build_dsa_layer(cur, layer, mctx_idx, inp_attn, prev_sel, + n_embd_head_k_mla, kv_lora_rank, + n_embd_head_qk_rope, n_embd_head_qk_nope, kq_scale_mla, il); + cur = dsa.out; + prev_sel = dsa.sel; + } + + prefix_sum = banked ? cur : ggml_add(ctx0, prefix_sum, cur); + cb(prefix_sum, "prefix_sum_attn", il); + + cur = use_attn_res ? res_mix(prefix_sum, layer.ffn_res_score, n_tokens, il) + : prefix_sum; + + cur = build_norm(cur, layer.ffn_norm, NULL, LLM_NORM_RMS, il); + cb(cur, "ffn_norm", il); + + if ((uint32_t) il < hparams.n_layer_dense_lead) { + ggml_tensor * g = ggml_mul_mat(ctx0, layer.ffn_gate, cur); + ggml_tensor * u = ggml_mul_mat(ctx0, layer.ffn_up, cur); + cur = kimi_k31_situ(ctx0, g, u, hparams.situ_beta, hparams.situ_linear_beta); + cur = ggml_mul_mat(ctx0, layer.ffn_down, cur); + cb(cur, "ffn_out", il); + } else { + cur = build_latent_moe(cur, layer, n_embd_latent, il); + } + + prefix_sum = ggml_add(ctx0, prefix_sum, cur); + prefix_sum = build_cvec(prefix_sum, il); + cb(prefix_sum, "l_out", il); + + inpL = prefix_sum; + } + + cur = inpL; + + // final mix, then narrow to the output tokens + if (use_attn_res) { + cur = res_mix(cur, model.output_res_score, n_tokens, -1); + } + if (inp_out_ids) { + cur = ggml_get_rows(ctx0, cur, inp_out_ids); + } + + cur = build_norm(cur, model.output_norm, NULL, LLM_NORM_RMS, -1); + cb(cur, "result_norm", -1); + res->t_embd = cur; + + cur = ggml_mul_mat(ctx0, model.output, cur); + cb(cur, "result_output", -1); + res->t_logits = cur; +} + +// +// KDA layer +// + +// causal conv1d over one of Q/K/V. `qkv` selects which third of the conv state to use +ggml_tensor * llama_model_kimi_k31::graph::build_kda_conv1d( + llm_graph_input_rs * inp_rs, ggml_tensor * conv_states_all, ggml_tensor * conv_state_all, + int64_t qkv, ggml_tensor * x, ggml_tensor * proj_w, ggml_tensor * conv_w, + int64_t d_conv, int64_t head_dim, int64_t n_head_kda, int64_t n_seq_tokens, int64_t n_seqs) { + const int64_t d_inner = head_dim * n_head_kda; + const int64_t conv_state_size = (d_conv - 1) * d_inner; + const int64_t n_embd_r_total = 3 * conv_state_size; + + ggml_tensor * conv_state_x = ggml_view_3d(ctx0, conv_state_all, d_conv - 1, d_inner, n_seqs, + (d_conv - 1) * ggml_element_size(conv_state_all), + n_embd_r_total * ggml_element_size(conv_state_all), + qkv * conv_state_size * ggml_element_size(conv_state_all)); + + ggml_tensor * x_proj = ggml_mul_mat(ctx0, proj_w, x); + ggml_tensor * x_3d = ggml_reshape_3d(ctx0, x_proj, d_inner, n_seq_tokens, n_seqs); + ggml_tensor * conv_x = ggml_concat(ctx0, conv_state_x, ggml_transpose(ctx0, x_3d), 0); + + build_conv_state_store(inp_rs, conv_states_all, conv_x, d_conv - 1, + ggml_row_size(conv_states_all->type, qkv * conv_state_size)); + + ggml_tensor * conv_weight = ggml_reshape_2d(ctx0, conv_w, d_conv, d_inner); + ggml_tensor * Xcur = ggml_ssm_conv(ctx0, conv_x, conv_weight); + Xcur = ggml_reshape_2d(ctx0, Xcur, d_inner, n_tokens); + Xcur = ggml_silu(ctx0, Xcur); + + return ggml_reshape_4d(ctx0, Xcur, head_dim, n_head_kda, n_seq_tokens, n_seqs); +} + +ggml_tensor * llama_model_kimi_k31::graph::build_kda_layer( + ggml_tensor * cur, const llama_layer & layer, llm_graph_input_rs * inp_rs, + int64_t d_conv, int64_t head_dim, int64_t n_head_kda, + int64_t d_inner, int64_t n_seq_tokens, int64_t n_seqs, int il) { + + const auto * mctx_cur = inp_rs->mctx; + + ggml_tensor * conv_states_all = mctx_cur->get_r_l(il); + ggml_tensor * conv_state_all = build_rs(inp_rs, conv_states_all, hparams.n_embd_r(), n_seqs); + + ggml_tensor * Qcur = build_kda_conv1d(inp_rs, conv_states_all, conv_state_all, 0, cur, layer.wq, layer.ssm_q_conv, d_conv, head_dim, n_head_kda, n_seq_tokens, n_seqs); + ggml_tensor * Kcur = build_kda_conv1d(inp_rs, conv_states_all, conv_state_all, 1, cur, layer.wk, layer.ssm_k_conv, d_conv, head_dim, n_head_kda, n_seq_tokens, n_seqs); + ggml_tensor * Vcur = build_kda_conv1d(inp_rs, conv_states_all, conv_state_all, 2, cur, layer.wv, layer.ssm_v_conv, d_conv, head_dim, n_head_kda, n_seq_tokens, n_seqs); + cb(Qcur, "kda_q_conv", il); + cb(Kcur, "kda_k_conv", il); + cb(Vcur, "kda_v_conv", il); + + // gate_lower_bound is not a clamp - when set, it swaps the decay gate activation: + // unset (kimi-linear): g = -exp(A_log) * softplus(f_b(f_a(x)) + dt_bias) + // set (K3, -5.0): g = lower_bound * sigmoid(exp(A_log) * (f_b(f_a(x)) + dt_bias)) + // ssm_a holds -exp(A_log) (folded at conversion time), so exp(A_log) == -ssm_a + ggml_tensor * f_a = ggml_mul_mat(ctx0, layer.ssm_f_a, cur); + ggml_tensor * g1 = ggml_mul_mat(ctx0, layer.ssm_f_b, f_a); + g1 = ggml_add(ctx0, g1, layer.ssm_dt_b); + + ggml_tensor * A = ggml_reshape_3d(ctx0, layer.ssm_a, 1, n_head_kda, 1); + + if (hparams.kda_gate_lower_bound > -INFINITY) { + g1 = ggml_reshape_3d(ctx0, g1, head_dim, n_head_kda, n_tokens); + g1 = ggml_mul(ctx0, g1, A); // -exp(A_log) * (...) + g1 = ggml_sigmoid(ctx0, ggml_scale(ctx0, g1, -1.0f)); + g1 = ggml_scale(ctx0, g1, hparams.kda_gate_lower_bound); + } else { + g1 = ggml_softplus(ctx0, g1); + g1 = ggml_reshape_3d(ctx0, g1, head_dim, n_head_kda, n_tokens); + g1 = ggml_mul(ctx0, g1, A); + } + cb(g1, "kda_g1", il); + + g1 = ggml_reshape_4d(ctx0, g1, head_dim, n_head_kda, n_seq_tokens, n_seqs); + + ggml_tensor * beta = ggml_mul_mat(ctx0, layer.ssm_beta, cur); + beta = ggml_reshape_4d(ctx0, beta, 1, n_head_kda, n_seq_tokens, n_seqs); + beta = ggml_sigmoid(ctx0, beta); + cb(beta, "kda_beta", il); + + ggml_tensor * cur_3d = ggml_reshape_3d(ctx0, cur, cur->ne[0], n_seq_tokens, n_seqs); + + ggml_tensor * ssm_states_all = mctx_cur->get_s_l(il); + ggml_tensor * state = build_rs(inp_rs, ssm_states_all, hparams.n_embd_s(), n_seqs); + state = ggml_reshape_4d(ctx0, state, head_dim, head_dim, n_head_kda, n_seqs); + + const float eps_norm = hparams.f_norm_rms_eps; + Qcur = build_gdn_l2_norm(ctx0, Qcur, eps_norm); + Kcur = build_gdn_l2_norm(ctx0, Kcur, eps_norm); + + ggml_tensor * output = build_recurrent_attn(inp_rs, ssm_states_all, Qcur, Kcur, Vcur, g1, beta, state, il); + output = ggml_cont(ctx0, output); + cb(output, "kda_scan_out", il); + + // K3: single full-rank gate (kimi-linear factors this as g_b(g_a(x))) + ggml_tensor * cur_2d = ggml_reshape_2d(ctx0, cur_3d, cur_3d->ne[0], n_seq_tokens * n_seqs); + ggml_tensor * g2 = ggml_mul_mat(ctx0, layer.ssm_g, cur_2d); + g2 = ggml_reshape_3d(ctx0, g2, head_dim, n_head_kda, n_seq_tokens * n_seqs); + + ggml_tensor * o = ggml_reshape_3d(ctx0, output, head_dim, n_head_kda, n_seq_tokens * n_seqs); + ggml_tensor * normed = build_norm(o, layer.ssm_o_norm, nullptr, LLM_NORM_RMS, il); + cb(g2, "kda_g2", il); + cb(normed, "kda_normed", il); + ggml_tensor * gated = ggml_mul(ctx0, normed, ggml_sigmoid(ctx0, g2)); + + gated = ggml_cont_2d(ctx0, gated, d_inner, n_tokens); + cur = ggml_mul_mat(ctx0, layer.wo, gated); + cb(cur, "kda_out", il); + + return cur; +} + +// +// Gated DSA layer: nope MLA over the top-k cells picked by the lightning indexer +// + +llama_model_kimi_k31::graph::dsa_out +llama_model_kimi_k31::graph::build_dsa_layer( + ggml_tensor * cur, const llama_layer & layer, + const llama_kv_cache_context * mctx_idx, llm_graph_input_attn_k * inp_attn, + ggml_tensor * prev_sel, + int64_t n_embd_head_k_mla, int64_t kv_lora_rank, + int64_t n_embd_head_qk_rope, int64_t n_embd_head_qk_nope, float kq_scale, int il) { + + const auto * mctx_mla = inp_attn->mctx; + + const int64_t n_idx_head = hparams.indexer_n_head; + const int64_t n_idx_dim = hparams.indexer_head_size; + + ggml_tensor * inp_gate = cur; // the output gate reads the *normed* layer input + + // --- MLA query; the indexer reads the same q latent --- + GGML_ASSERT(layer.wq_a && layer.wq_b && "K3.1 MLA is served from the q latent"); + + ggml_tensor * qr = ggml_mul_mat(ctx0, layer.wq_a, cur); + qr = build_norm(qr, layer.attn_q_a_norm, nullptr, LLM_NORM_RMS, il); + ggml_tensor * Qcur = ggml_mul_mat(ctx0, layer.wq_b, qr); + + ggml_tensor * kv_cmpr_pe = ggml_mul_mat(ctx0, layer.wkv_a_mqa, cur); + + ggml_tensor * kv_cmpr = ggml_view_2d(ctx0, kv_cmpr_pe, kv_lora_rank, n_tokens, + ggml_row_size(kv_cmpr_pe->type, kv_lora_rank + n_embd_head_qk_rope), 0); + ggml_tensor * k_pe = ggml_view_3d(ctx0, kv_cmpr_pe, n_embd_head_qk_rope, 1, n_tokens, + ggml_row_size(kv_cmpr_pe->type, kv_lora_rank + n_embd_head_qk_rope), + ggml_row_size(kv_cmpr_pe->type, kv_lora_rank + n_embd_head_qk_rope), + ggml_row_size(kv_cmpr_pe->type, kv_lora_rank)); + + // no RoPE: mla_use_nope is asserted at conversion time + kv_cmpr = build_norm(kv_cmpr, layer.attn_kv_a_norm, nullptr, LLM_NORM_RMS, il); + + // the cache holds the compressed latent, so the MLA is served absorbed + GGML_ASSERT(layer.wk_b && layer.wv_b); + + ggml_tensor * q_nope = ggml_view_3d(ctx0, Qcur, n_embd_head_qk_nope, n_head, n_tokens, + ggml_row_size(Qcur->type, n_embd_head_k_mla), + ggml_row_size(Qcur->type, n_embd_head_k_mla) * n_head, 0); + ggml_tensor * q_pe = ggml_view_3d(ctx0, Qcur, n_embd_head_qk_rope, n_head, n_tokens, + ggml_row_size(Qcur->type, n_embd_head_k_mla), + ggml_row_size(Qcur->type, n_embd_head_k_mla) * n_head, + ggml_row_size(Qcur->type, n_embd_head_qk_nope)); + + q_nope = ggml_permute(ctx0, q_nope, 0, 2, 1, 3); + ggml_tensor * q_nope_absorbed = ggml_mul_mat(ctx0, layer.wk_b, q_nope); + q_nope_absorbed = ggml_permute(ctx0, q_nope_absorbed, 0, 2, 1, 3); + + ggml_tensor * Q = ggml_concat(ctx0, q_nope_absorbed, q_pe, 0); + ggml_tensor * kv_cmpr_3d = ggml_reshape_3d(ctx0, kv_cmpr, kv_lora_rank, 1, n_tokens); + ggml_tensor * K = ggml_concat(ctx0, kv_cmpr_3d, k_pe, 0); + + // --- DSA: keep only the top-k cells of this layer --- + ggml_tensor * sel = prev_sel; + if (hparams.is_indexer_full(il) && layer.indexer_attn_q_b) { + GGML_ASSERT(mctx_idx != nullptr); + + ggml_tensor * iq = ggml_mul_mat(ctx0, layer.indexer_attn_q_b, qr); + iq = ggml_reshape_3d(ctx0, iq, n_idx_dim, n_idx_head, n_tokens); + + ggml_tensor * ik = ggml_mul_mat(ctx0, layer.indexer_attn_k, cur); + ik = build_norm(ik, layer.indexer_k_norm, layer.indexer_k_norm_b, LLM_NORM, il); + ik = ggml_reshape_3d(ctx0, ik, n_idx_dim, 1, n_tokens); + cb(ik, "indexer_k", il); + + // the indexer keys go to the cells of the matching attention layer + ggml_build_forward_expand(gf, mctx_idx->cpy_k(ctx0, ik, inp_attn->get_k_idxs(), il)); + + ggml_tensor * iw = ggml_mul_mat(ctx0, layer.indexer_proj, cur); + // pre-scale so the huge score tensor is not scaled afterwards + iw = ggml_scale(ctx0, iw, 1.0f / sqrtf(float(n_idx_dim * n_idx_head))); + + ik = mctx_idx->get_k(ctx0, il); + + // split the ubatch into its streams, k carries one row per stream + const int64_t n_stream = ik->ne[3]; + iq = ggml_view_4d(ctx0, iq, n_idx_dim, n_idx_head, n_tokens/n_stream, n_stream, + iq->nb[1], iq->nb[2], iq->nb[3]/n_stream, 0); + iw = ggml_view_4d(ctx0, iw, n_idx_head, n_tokens/n_stream, 1, n_stream, + iw->nb[1], iw->nb[2]/n_stream, iw->nb[2]/n_stream, 0); + + ggml_tensor * kq_mask_lid = inp_attn->self_attn_mask.raw; + if (cparams.fused_lid && kq_mask_lid->type != GGML_TYPE_F16) { + kq_mask_lid = ggml_cast(ctx0, kq_mask_lid, GGML_TYPE_F16); + } + + ggml_tensor * score; + if (cparams.fused_lid) { + score = ggml_lightning_indexer(ctx0, iq, ik, iw, kq_mask_lid); + res->add_fused_node({LLM_FUSED_OP_LIGHTNING_INDEXER, score, il}); + } else { + ggml_tensor * q_p = ggml_permute(ctx0, iq, 0, 2, 1, 3); // [n_idx_dim, n_tokens, n_idx_head] + ggml_tensor * k_p = ggml_permute(ctx0, ik, 0, 2, 1, 3); // [n_idx_dim, n_kv, 1] + + ggml_tensor * kq = ggml_mul_mat(ctx0, k_p, q_p); // [n_kv, n_tokens, n_idx_head] + kq = ggml_cont(ctx0, ggml_permute(ctx0, kq, 2, 1, 0, 3)); // [n_idx_head, n_tokens, n_kv] + + score = ggml_relu(ctx0, kq); + score = ggml_mul(ctx0, score, iw); + score = ggml_sum_rows(ctx0, score); // [1, n_tokens, n_kv] + score = ggml_cont(ctx0, ggml_permute(ctx0, score, 2, 1, 0, 3)); // [n_kv, n_tokens, 1] + + score = ggml_add(ctx0, score, kq_mask_lid); + } + cb(score, "indexer_score", il); + + const int64_t n_top_k = std::min(score->ne[0], hparams.indexer_top_k); + ggml_tensor * top_k = ggml_cont(ctx0, ggml_top_k(ctx0, score, n_top_k)); + cb(top_k, "indexer_top_k", il); + + sel = build_top_k_mask(inp_attn->self_attn_mask.raw, top_k, il); + cb(sel, "kq_mask_dsa", il); + } else { + GGML_ASSERT(sel != nullptr && "shared indexer layer must follow a full indexer layer"); + } + + ggml_build_forward_expand(gf, Q); + ggml_build_forward_expand(gf, K); + ggml_build_forward_expand(gf, mctx_mla->cpy_k(ctx0, K, inp_attn->get_k_idxs(), il)); + + ggml_tensor * k = mctx_mla->get_k(ctx0, il); + ggml_tensor * v = ggml_view_4d(ctx0, k, kv_lora_rank, k->ne[1], k->ne[2], k->ne[3], + k->nb[1], k->nb[2], k->nb[3], 0); + + // the selection is applied as an additive mask, so attention only gets cheaper when the backend + // can skip masked columns: cuda flash attention gathers the visible cells when n_kv_max > 0 and + // the cache holds at least 4096 cells. on the dense fallbacks the mask keeps the result correct + // but the kq and kqv products still run over the whole cache, and the indexer is pure overhead + const int64_t n_sel = std::min(mctx_mla->get_n_kv(), hparams.indexer_top_k); + ggml_tensor * out = build_attn_mha(Q, k, v, nullptr, sel, nullptr, nullptr, layer.wv_b, + n_sel, kq_scale, il); + cb(out, "kqv_out", il); + + // K3.1: attn_output *= sigmoid(g_proj(x)), then o_proj + if (layer.wqkv_gate) { + ggml_tensor * g = ggml_sigmoid(ctx0, ggml_mul_mat(ctx0, layer.wqkv_gate, inp_gate)); + out = ggml_mul(ctx0, out, g); + cb(out, "mla_gated", il); + } + + out = ggml_mul_mat(ctx0, layer.wo, out); + cb(out, "mla_out", il); + + return { out, sel }; +} + +// +// latent MoE: down-project, run the routed experts in the latent space, norm, up-project; +// shared experts stay at n_embd and read the un-projected input. +// + +ggml_tensor * llama_model_kimi_k31::graph::build_latent_moe( + ggml_tensor * cur, const llama_layer & layer, int64_t n_embd_latent, int il) { + + ggml_tensor * identity = cur; + + ggml_tensor * routed_in = layer.ffn_routed_down + ? ggml_mul_mat(ctx0, layer.ffn_routed_down, cur) + : cur; + + // the router scores the full-width input while the experts take the latent one, + // so the logits are computed here and passed to build_moe_ffn + ggml_tensor * logits = ggml_mul_mat(ctx0, layer.ffn_gate_inp, identity); + cb(logits, "ffn_moe_logits", il); + + ggml_tensor * moe_out = build_moe_ffn(routed_in, + nullptr, // gate_inp unused: the logits above are passed instead + layer.ffn_up_exps, + layer.ffn_gate_exps, + layer.ffn_down_exps, + layer.ffn_exp_probs_b, + hparams.n_expert, + hparams.n_expert_used(), + LLM_FFN_SITU, hparams.expert_weights_norm, + hparams.expert_weights_scale, + (llama_expert_gating_func_type) hparams.expert_gating_func, + il, + logits); + cb(moe_out, "ffn_moe_out", il); + + if (layer.ffn_routed_norm) { + moe_out = build_norm(moe_out, layer.ffn_routed_norm, NULL, LLM_NORM_RMS, il); + } + if (layer.ffn_routed_up) { + moe_out = ggml_mul_mat(ctx0, layer.ffn_routed_up, moe_out); + } + GGML_UNUSED(n_embd_latent); + + if (layer.ffn_gate_shexp) { + ggml_tensor * g = ggml_mul_mat(ctx0, layer.ffn_gate_shexp, identity); + ggml_tensor * u = ggml_mul_mat(ctx0, layer.ffn_up_shexp, identity); + ggml_tensor * sh = kimi_k31_situ(ctx0, g, u, hparams.situ_beta, hparams.situ_linear_beta); + sh = ggml_mul_mat(ctx0, layer.ffn_down_shexp, sh); + cb(sh, "ffn_shexp", il); + moe_out = ggml_add(ctx0, moe_out, sh); + } + + cb(moe_out, "ffn_out", il); + return moe_out; +} diff --git a/src/models/kimi-linear.cpp b/src/models/kimi-linear.cpp index 7ce088ef698b..2190df8c95ab 100644 --- a/src/models/kimi-linear.cpp +++ b/src/models/kimi-linear.cpp @@ -167,7 +167,7 @@ void llama_model_kimi_linear::load_arch_tensors(llama_model_loader &) { } std::unique_ptr llama_model_kimi_linear::build_arch_graph(const llm_graph_params & params) const { - return std::make_unique(*this, params); + return std::make_unique(params); } // Causal Conv1d function for Q,K,V @@ -231,8 +231,8 @@ static ggml_tensor * causal_conv1d(ggml_cgraph * gf, ggml_context * ctx0, ggml_t return ggml_reshape_4d(ctx0, Xcur, head_dim, n_head, n_seq_tokens, n_seqs); } -llama_model_kimi_linear::graph::graph(const llama_model & model, const llm_graph_params & params) : - llm_build_delta_net_base(params), model(model) { +llama_model_kimi_linear::graph::graph(const llm_graph_params & params) : + llm_build_delta_net_base(params) { ggml_tensor * cur; ggml_tensor * inpL; @@ -348,8 +348,7 @@ llama_model_kimi_linear::graph::graph(const llama_model & model, const llm_graph Qcur = build_gdn_l2_norm(ctx0, Qcur, eps_norm); Kcur = build_gdn_l2_norm(ctx0, Kcur, eps_norm); - // Choose between build_delta_net_chunking and build_delta_net_recurrent based on n_tokens - auto attn_out = build_delta_net(Qcur, Kcur, Vcur, g1, beta, state, il); + auto attn_out = build_delta_net(Qcur, Kcur, Vcur, g1, beta, state); ggml_tensor * output = ggml_cont(ctx0, attn_out.first); ggml_tensor * new_state = attn_out.second; diff --git a/src/models/kolibri1.cpp b/src/models/kolibri1.cpp index 2301ef272acf..c4d8eca1bed4 100644 --- a/src/models/kolibri1.cpp +++ b/src/models/kolibri1.cpp @@ -15,6 +15,7 @@ void llama_model_kolibri1::load_arch_hparams(llama_model_loader & ml) { hparams.swa_type = LLAMA_SWA_TYPE_STANDARD; ml.get_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, hparams.is_swa_impl); + hparams.rope_pattern = hparams.is_swa_impl; hparams.rope_freq_base_train_swa = hparams.rope_freq_base_train; hparams.rope_freq_scale_train_swa = hparams.rope_freq_scale_train; @@ -64,7 +65,7 @@ void llama_model_kolibri1::load_arch_tensors(llama_model_loader &) { } } -llama_model_kolibri1::graph::graph(const llama_model & model, const llm_graph_params & params) +llama_model_kolibri1::graph::graph(const llm_graph_params & params) : llm_graph_context(params) { const int64_t n_embd_head = hparams.n_embd_head_v(); GGML_ASSERT(n_embd_head == hparams.n_embd_head_k()); @@ -95,22 +96,11 @@ llama_model_kolibri1::graph::graph(const llama_model & model, const llm_graph_pa cb(Qcur, "Qcur_normed", il); cb(Kcur, "Kcur_normed", il); - if (hparams.is_swa(il)) { - const float freq_base_l = model.get_rope_freq_base (cparams, il); - const float freq_scale_l = model.get_rope_freq_scale(cparams, il); - - Qcur = ggml_rope_ext( - ctx0, Qcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base_l, freq_scale_l, - ext_factor, attn_factor, beta_fast, beta_slow); - cb(Qcur, "Qcur_rope", il); - - Kcur = ggml_rope_ext( - ctx0, Kcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base_l, freq_scale_l, - ext_factor, attn_factor, beta_fast, beta_slow); - cb(Kcur, "Kcur_rope", il); - } + Qcur = build_rope(Qcur, inp_pos, il); + cb(Qcur, "Qcur_rope", il); + + Kcur = build_rope(Kcur, inp_pos, il); + cb(Kcur, "Kcur_rope", il); cur = build_attn(inp_attn, layer.wo, NULL, layer.wo_s, @@ -180,5 +170,5 @@ llama_model_kolibri1::graph::graph(const llama_model & model, const llm_graph_pa } std::unique_ptr llama_model_kolibri1::build_arch_graph(const llm_graph_params & params) const { - return std::make_unique(*this, params); + return std::make_unique(params); } diff --git a/src/models/laguna.cpp b/src/models/laguna.cpp index cbc7b26d7cec..1bf25eacf730 100644 --- a/src/models/laguna.cpp +++ b/src/models/laguna.cpp @@ -148,11 +148,26 @@ void llama_model_laguna::load_arch_tensors(llama_model_loader & ml) { } } +llama_rope_params llama_model_laguna::get_rope_params(const llama_cparams & cparams, int il) const { + auto res = llama_model::get_rope_params(cparams, il); + + // plain RoPE on the SWA layers + if (hparams.is_swa(il)) { + res.n_ctx_orig = hparams.n_ctx_train; + res.ext_factor = 0.0f; + res.attn_factor = 1.0f; + res.beta_fast = 0.0f; + res.beta_slow = 0.0f; + } + + return res; +} + std::unique_ptr llama_model_laguna::build_arch_graph(const llm_graph_params & params) const { - return std::make_unique(*this, params); + return std::make_unique(params); } -llama_model_laguna::graph::graph(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) { +llama_model_laguna::graph::graph(const llm_graph_params & params) : llm_graph_context(params) { const int64_t n_embd_head = hparams.n_embd_head_v(); GGML_ASSERT(n_embd_head == hparams.n_embd_head_k()); @@ -173,26 +188,11 @@ llama_model_laguna::graph::graph(const llama_model & model, const llm_graph_para const float kq_scale = 1.0f / sqrtf(float(n_embd_head)); for (int il = 0; il < n_layer; ++il) { - const bool is_swa_il = hparams.is_swa(il); const int64_t n_head_il = hparams.n_head(il); const int64_t n_head_kv_il = hparams.n_head_kv(il); // Per-layer-type RoPE config. SWA layers run plain rope (no YaRN), // achieved by zeroing the YaRN ext/beta params for those layers. - const int n_rot_l = is_swa_il ? hparams.n_rot_swa : n_rot; - const float freq_base_l = is_swa_il ? hparams.rope_freq_base_train_swa : freq_base; - const float freq_scale_l = is_swa_il ? hparams.rope_freq_scale_train_swa : freq_scale; - const float ext_factor_l = is_swa_il ? 0.0f : ext_factor; - // YaRN magnitude scaling (mscale) is already handled by the framework: - // llama_context pre-divides cparams.yarn_attn_factor by (1 + 0.1*ln(factor)) - // to cancel ggml rope_yarn's internal mscale *= 1 + 0.1*ln(1/freq_scale). - // Pass attn_factor straight through (like every other arch); SWA layers run - // plain RoPE (ext_factor 0, no mscale) so force 1.0 there. - const float attn_factor_l = is_swa_il ? 1.0f : attn_factor; - const float beta_fast_l = is_swa_il ? 0.0f : beta_fast; - const float beta_slow_l = is_swa_il ? 0.0f : beta_slow; - const int n_ctx_orig_l = is_swa_il ? hparams.n_ctx_train : n_ctx_orig; - ggml_tensor * inpSA = inpL; // Pre-norm @@ -218,12 +218,8 @@ llama_model_laguna::graph::graph(const llama_model & model, const llm_graph_para cb(Qcur, "Qcur_normed", il); cb(Kcur, "Kcur_normed", il); - Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, nullptr, - n_rot_l, rope_type, n_ctx_orig_l, freq_base_l, freq_scale_l, - ext_factor_l, attn_factor_l, beta_fast_l, beta_slow_l); - Kcur = ggml_rope_ext(ctx0, Kcur, inp_pos, nullptr, - n_rot_l, rope_type, n_ctx_orig_l, freq_base_l, freq_scale_l, - ext_factor_l, attn_factor_l, beta_fast_l, beta_slow_l); + Qcur = build_rope(Qcur, inp_pos, il); + Kcur = build_rope(Kcur, inp_pos, il); cb(Qcur, "Qcur_rope", il); cb(Kcur, "Kcur_rope", il); diff --git a/src/models/lfm-base.cpp b/src/models/lfm-base.cpp index 0dabf114bde1..3627c527dd7a 100644 --- a/src/models/lfm-base.cpp +++ b/src/models/lfm-base.cpp @@ -2,16 +2,12 @@ #include "llama-memory-recurrent.h" -#include - llm_build_lfm_base::llm_build_lfm_base(const llm_graph_params & params) : llm_graph_context(params) {} -ggml_tensor * llm_build_lfm_base::build_lfm_shortconv(const llama_model & model, - const llama_memory_recurrent_context * mctx_recr, +ggml_tensor * llm_build_lfm_base::build_lfm_shortconv(const llama_memory_recurrent_context * mctx_recr, llm_graph_input_rs * inp_recr, ggml_tensor * cur, int il) { - const uint32_t kv_head = mctx_recr->get_head(); const int64_t n_seq_tokens = ubatch.n_seq_tokens; const int64_t n_seqs = ubatch.n_seqs; GGML_ASSERT(n_seqs != 0); @@ -55,20 +51,7 @@ ggml_tensor * llm_build_lfm_base::build_lfm_shortconv(const llama_model & model, } GGML_ASSERT(bx->ne[0] > conv->ne[0]); - // write conv states: slot 0 = the final state, slot s = the state s tokens back (partial rollback) - const int64_t K = hparams.causal_attn && cparams.n_rs_seq > 0 ? (int64_t) cparams.n_rs_seq + 1 : 1; - const int64_t n_written = std::min(n_seq_tokens, K); - const auto mem_size = mctx_recr->get_size(); - const size_t row_size = ggml_row_size(conv_state->type, (int64_t) d_conv * n_embd); - - for (int64_t slot = 0; slot < n_written; ++slot) { - auto * conv_snap = ggml_view_3d(ctx0, bx, d_conv, bx->ne[1], bx->ne[2], bx->nb[1], bx->nb[2], - (bx->ne[0] - d_conv - slot) * ggml_element_size(bx)); - ggml_build_forward_expand(gf, ggml_cpy(ctx0, conv_snap, - ggml_view_2d(ctx0, conv_state, (int64_t) d_conv * n_embd, n_seqs, - conv_state->nb[1], - ((size_t) slot * mem_size + kv_head) * row_size))); - } + build_conv_state_store(inp_recr, conv_state, bx, d_conv, 0); auto * conv_kernel = model.layers[il].shortconv.conv; auto * conv_out = ggml_ssm_conv(ctx0, bx, conv_kernel); @@ -83,8 +66,7 @@ ggml_tensor * llm_build_lfm_base::build_lfm_shortconv(const llama_model & model, return y; } -ggml_tensor * llm_build_lfm_base::build_lfm_shortconv(const llama_model & model, - ggml_tensor * cur, +ggml_tensor * llm_build_lfm_base::build_lfm_shortconv(ggml_tensor * cur, int il) { const int64_t d_conv = hparams.n_shortconv_l_cache - 1; @@ -110,21 +92,19 @@ ggml_tensor * llm_build_lfm_base::build_lfm_shortconv(const llama_model & model, return y; } -ggml_tensor * llm_build_lfm_base::build_lfm_dense_ffn(const llama_model & model, - ggml_tensor * cur, +ggml_tensor * llm_build_lfm_base::build_lfm_dense_ffn(ggml_tensor * cur, int il) { GGML_ASSERT(!model.layers[il].ffn_up_b); GGML_ASSERT(!model.layers[il].ffn_gate_b); GGML_ASSERT(!model.layers[il].ffn_down_b); return build_ffn(cur, - model.layers[il].ffn_up, NULL, NULL, - model.layers[il].ffn_gate, NULL, NULL, - model.layers[il].ffn_down, NULL, NULL, + model.layers[il].ffn_up, NULL, model.layers[il].ffn_up_s, + model.layers[il].ffn_gate, NULL, model.layers[il].ffn_gate_s, + model.layers[il].ffn_down, NULL, model.layers[il].ffn_down_s, NULL, LLM_FFN_SILU, LLM_FFN_PAR, il); } -ggml_tensor * llm_build_lfm_base::build_lfm_moe_ffn(const llama_model & model, - ggml_tensor * cur, +ggml_tensor * llm_build_lfm_base::build_lfm_moe_ffn(ggml_tensor * cur, int il) { return build_moe_ffn(cur, model.layers[il].ffn_gate_inp, @@ -136,5 +116,10 @@ ggml_tensor * llm_build_lfm_base::build_lfm_moe_ffn(const llama_model & model, LLM_FFN_SILU, true, hparams.expert_weights_scale, static_cast(hparams.expert_gating_func), - il); + il, + nullptr, + nullptr, + model.layers[il].ffn_up_exps_s, + model.layers[il].ffn_gate_exps_s, + model.layers[il].ffn_down_exps_s); } diff --git a/src/models/lfm2.cpp b/src/models/lfm2.cpp index 3488b6c775a8..86b948cf3de9 100644 --- a/src/models/lfm2.cpp +++ b/src/models/lfm2.cpp @@ -134,28 +134,38 @@ void llama_model_lfm2::load_arch_tensors(llama_model_loader &) { dense_2_out_layers_b = create_tensor(tn(LLM_TENSOR_DENSE_2_OUT, "bias"), {hparams.n_embd_out() }, TENSOR_NOT_REQUIRED); } +llama_rope_params llama_model_lfm2::get_rope_params(const llama_cparams & cparams, int il) const { + auto res = llama_model::get_rope_params(cparams, il); + + // the SWA layers use the global base and scale + res.freq_base = cparams.rope_freq_base; + res.freq_scale = cparams.rope_freq_scale; + + return res; +} + std::unique_ptr llama_model_lfm2::build_arch_graph(const llm_graph_params & params) const { if (hparams.n_layer_decision > 0) { - return std::make_unique(*this, params); + return std::make_unique(params); } if (hparams.swa_type == LLAMA_SWA_TYPE_STANDARD) { - return std::make_unique>(*this, params); + return std::make_unique>(params); } else { - return std::make_unique>(*this, params); + return std::make_unique>(params); } } template -llama_model_lfm2::graph::graph(const llama_model & model, const llm_graph_params & params) : +llama_model_lfm2::graph::graph(const llm_graph_params & params) : llm_build_lfm_base(params) { using inp_hybrid_type = std::conditional_t; using inp_attn_type = std::conditional_t; using mem_hybrid_ctx = std::conditional_t; - auto build_attn_block = [&model, this](ggml_tensor * cur, - ggml_tensor * inp_pos, - inp_attn_type * inp_attn, - int il) -> ggml_tensor * { + auto build_attn_block = [this](ggml_tensor * cur, + ggml_tensor * inp_pos, + inp_attn_type * inp_attn, + int il) -> ggml_tensor * { GGML_ASSERT(hparams.n_embd_v_gqa(il) == hparams.n_embd_k_gqa(il)); const auto n_embd_head = hparams.n_embd_head_v(); const auto n_head_kv = hparams.n_head_kv(il); @@ -170,10 +180,8 @@ llama_model_lfm2::graph::graph(const llama_model & model, const llm_graph_ cb(k, "model.layers.{}.self_attn.k_layernorm", il); // RoPE - q = ggml_rope_ext(ctx0, q, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, ext_factor, - attn_factor, beta_fast, beta_slow); - k = ggml_rope_ext(ctx0, k, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, ext_factor, - attn_factor, beta_fast, beta_slow); + q = build_rope(q, inp_pos, il); + k = build_rope(k, inp_pos, il); cur = build_attn(inp_attn, model.layers[il].wo, NULL, model.layers[il].wo_s, @@ -210,7 +218,7 @@ llama_model_lfm2::graph::graph(const llama_model & model, const llm_graph_ if (hparams.is_recr(il)) { const auto * mctx_recr = static_cast(mctx)->get_recr(); - cur = build_lfm_shortconv(model, mctx_recr, inp_hybrid->get_recr(), cur, il); + cur = build_lfm_shortconv(mctx_recr, inp_hybrid->get_recr(), cur, il); } else { cur = build_attn_block(cur, inp_pos, inp_hybrid->get_attn(), il); } @@ -226,7 +234,7 @@ llama_model_lfm2::graph::graph(const llama_model & model, const llm_graph_ cb(ffn_norm_out, "model.layers.{}.ffn_norm", il); ggml_tensor * ffn_out = - is_moe_layer ? build_lfm_moe_ffn(model, ffn_norm_out, il) : build_lfm_dense_ffn(model, ffn_norm_out, il); + is_moe_layer ? build_lfm_moe_ffn(ffn_norm_out, il) : build_lfm_dense_ffn(ffn_norm_out, il); cb(ffn_norm_out, "model.layers.{}.ffn_out", il); cur = ggml_add(ctx0, cur, ffn_out); @@ -314,7 +322,7 @@ class llm_graph_input_conv_mask : public llm_graph_input_i { ggml_tensor * right = nullptr; // F32 [1, n_tokens] }; -llama_model_lfm2::graph_decision::graph_decision(const llama_model & model, const llm_graph_params & params) : +llama_model_lfm2::graph_decision::graph_decision(const llm_graph_params & params) : llm_graph_context(params) { const int64_t n_embd_head = hparams.n_embd_head_v(); const int n_layer_enc = n_layer - hparams.n_layer_decision; @@ -332,7 +340,6 @@ llama_model_lfm2::graph_decision::graph_decision(const llama_model & model, cons auto inp = std::make_unique(hparams, cparams, is_head); inp->self_attn_mask.raw = ggml_new_tensor_4d(ctx0, type_mask, n_tokens, n_tokens, 1, 1); ggml_set_input(inp->self_attn_mask.raw); - inp->self_attn_mask.cnv = inp->self_attn_mask.raw; inp_attn[is_head] = (llm_graph_input_attn_no_cache *) res->add_input(std::move(inp)); } @@ -385,10 +392,8 @@ llama_model_lfm2::graph_decision::graph_decision(const llama_model & model, cons q = build_norm(q, layer.attn_q_norm, NULL, LLM_NORM_RMS, il); k = build_norm(k, layer.attn_k_norm, NULL, LLM_NORM_RMS, il); - q = ggml_rope_ext(ctx0, q, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, ext_factor, - attn_factor, beta_fast, beta_slow); - k = ggml_rope_ext(ctx0, k, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, ext_factor, - attn_factor, beta_fast, beta_slow); + q = build_rope(q, inp_pos, il); + k = build_rope(k, inp_pos, il); cur = build_attn(inp_attn[0], layer.wo, NULL, layer.wo_s, @@ -400,9 +405,9 @@ llama_model_lfm2::graph_decision::graph_decision(const llama_model & model, cons ggml_tensor * ffn_out = build_norm(cur, layer.ffn_norm, NULL, LLM_NORM_RMS, il); ffn_out = build_ffn(ffn_out, - layer.ffn_up, NULL, NULL, - layer.ffn_gate, NULL, NULL, - layer.ffn_down, NULL, NULL, + layer.ffn_up, NULL, layer.ffn_up_s, + layer.ffn_gate, NULL, layer.ffn_gate_s, + layer.ffn_down, NULL, layer.ffn_down_s, NULL, LLM_FFN_SILU, LLM_FFN_PAR, il); cur = ggml_add(ctx0, cur, ffn_out); @@ -412,7 +417,7 @@ llama_model_lfm2::graph_decision::graph_decision(const llama_model & model, cons cur = build_norm(cur, model.output_norm, NULL, LLM_NORM_RMS, -1); cb(cur, "result_norm", -1); - cur = build_decision_head(model, cur, inp_attn[1], inp_out_ids); + cur = build_decision_head(cur, inp_attn[1], inp_out_ids); res->t_embd = cur; ggml_build_forward_expand(gf, cur); @@ -420,7 +425,6 @@ llama_model_lfm2::graph_decision::graph_decision(const llama_model & model, cons // same as llama_model_modern_bert::graph::build_decision_head(), with the head counts of the head layers ggml_tensor * llama_model_lfm2::graph_decision::build_decision_head( - const llama_model & model, ggml_tensor * inp, llm_graph_input_attn_no_cache * inp_attn, ggml_tensor * inp_out_ids) { diff --git a/src/models/lfm2moe.cpp b/src/models/lfm2moe.cpp index f8d47f9b85ae..642fe88996b9 100644 --- a/src/models/lfm2moe.cpp +++ b/src/models/lfm2moe.cpp @@ -77,9 +77,9 @@ void llama_model_lfm2moe::load_arch_tensors(llama_model_loader &) { std::unique_ptr llama_model_lfm2moe::build_arch_graph(const llm_graph_params & params) const { if (hparams.swa_type == LLAMA_SWA_TYPE_STANDARD) { - return std::make_unique>(*this, params); + return std::make_unique>(params); } else { - return std::make_unique>(*this, params); + return std::make_unique>(params); } } diff --git a/src/models/lfm3.cpp b/src/models/lfm3.cpp index 174ce9cf7e38..0746394ce31c 100644 --- a/src/models/lfm3.cpp +++ b/src/models/lfm3.cpp @@ -90,28 +90,22 @@ void llama_model_lfm3::load_arch_tensors(llama_model_loader &) { std::unique_ptr llama_model_lfm3::build_arch_graph(const llm_graph_params & params) const { if (!hparams.causal_attn) { - return std::make_unique>(*this, params); + return std::make_unique>(params); } - return std::make_unique>(*this, params); + return std::make_unique>(params); } template -llama_model_lfm3::graph::graph(const llama_model & model, const llm_graph_params & params) : +llama_model_lfm3::graph::graph(const llm_graph_params & params) : llm_build_lfm_base(params) { - auto build_attn_block = [&model, this](ggml_tensor * cur_in, - ggml_tensor * inp_pos, - auto * inp_attn, - int il) -> ggml_tensor * { + auto build_attn_block = [this](ggml_tensor * cur_in, + ggml_tensor * inp_pos, + auto * inp_attn, + int il) -> ggml_tensor * { const int64_t n_embd_head = hparams.n_embd_head_k(il); const int64_t n_head_il = hparams.n_head(il); const int64_t n_head_kv = hparams.n_head_kv(il); - const int n_rot_l = hparams.n_rot(il); - const float freq_base_l = model.get_rope_freq_base (cparams, il); - const float freq_scale_l = model.get_rope_freq_scale(cparams, il); - - ggml_tensor * rope_factors = hparams.is_swa(il) ? nullptr : model.layers[il].rope_freqs; - auto [q, k, v] = build_qkv(model.layers[il], cur_in, n_embd_head, n_head_il, n_head_kv, il); q = build_norm(q, model.layers[il].attn_q_norm, NULL, LLM_NORM_RMS, il); @@ -119,17 +113,15 @@ llama_model_lfm3::graph::graph(const llama_model & model, const llm_gr k = build_norm(k, model.layers[il].attn_k_norm, NULL, LLM_NORM_RMS, il); cb(k, "model.layers.{}.self_attn.k_layernorm", il); - q = ggml_rope_ext(ctx0, q, inp_pos, rope_factors, n_rot_l, rope_type, n_ctx_orig, - freq_base_l, freq_scale_l, ext_factor, attn_factor, beta_fast, beta_slow); - k = ggml_rope_ext(ctx0, k, inp_pos, rope_factors, n_rot_l, rope_type, n_ctx_orig, - freq_base_l, freq_scale_l, ext_factor, attn_factor, beta_fast, beta_slow); + q = build_rope(q, inp_pos, il); + k = build_rope(k, inp_pos, il); ggml_tensor * attn_out = build_attn(inp_attn, NULL, NULL, NULL, q, k, v, nullptr, nullptr, nullptr, 1.0f / sqrtf(float(n_embd_head)), il); cb(attn_out, "attn_out", il); - ggml_tensor * gate = build_lora_mm(model.layers[il].wqkv_gate, cur_in); + ggml_tensor * gate = build_lora_mm(model.layers[il].wqkv_gate, cur_in, model.layers[il].wqkv_gate_s); gate = ggml_sigmoid(ctx0, gate); cb(gate, "model.layers.{}.self_attn.gate", il); @@ -157,6 +149,8 @@ llama_model_lfm3::graph::graph(const llama_model & model, const llm_gr ggml_tensor * inp_out_ids = build_inp_out_ids(); for (int il = 0; il < n_layer; ++il) { + res->t_layer_inp[il] = cur; + auto * prev_cur = cur; cur = build_norm(cur, model.layers[il].attn_norm, NULL, LLM_NORM_RMS, il); @@ -164,10 +158,10 @@ llama_model_lfm3::graph::graph(const llama_model & model, const llm_gr if (hparams.is_recr(il)) { if constexpr (decision) { - cur = build_lfm_shortconv(model, cur, il); + cur = build_lfm_shortconv(cur, il); } else { const auto * mctx_recr = static_cast(mctx)->get_recr(); - cur = build_lfm_shortconv(model, mctx_recr, inp_hybrid->get_recr(), cur, il); + cur = build_lfm_shortconv(mctx_recr, inp_hybrid->get_recr(), cur, il); } } else if constexpr (decision) { cur = build_attn_block(cur, inp_pos, inp_attn, il); @@ -190,7 +184,7 @@ llama_model_lfm3::graph::graph(const llama_model & model, const llm_gr auto * ffn_norm_out = build_norm(cur, model.layers[il].ffn_norm, NULL, LLM_NORM_RMS, il); cb(ffn_norm_out, "model.layers.{}.ffn_norm", il); - ggml_tensor * ffn_out = build_lfm_dense_ffn(model, ffn_norm_out, il); + ggml_tensor * ffn_out = build_lfm_dense_ffn(ffn_norm_out, il); if (model.layers[il].ffn_post_norm) { ffn_out = build_norm(ffn_out, model.layers[il].ffn_post_norm, NULL, LLM_NORM_RMS, il); diff --git a/src/models/limite.cpp b/src/models/limite.cpp index 63f7c8288a34..2bd45561a124 100644 --- a/src/models/limite.cpp +++ b/src/models/limite.cpp @@ -13,6 +13,7 @@ void llama_model_limite::load_arch_hparams(llama_model_loader & ml) { hparams.swa_type = LLAMA_SWA_TYPE_STANDARD; ml.get_key(LLM_KV_ATTENTION_SLIDING_WINDOW, hparams.n_swa); load_swa_pattern(ml, 4); + hparams.rope_pattern = hparams.is_swa_impl; std::fill(hparams.mudd_tap_idx.begin(), hparams.mudd_tap_idx.end(), -1); ml.get_key(LLM_KV_MUDD_TAP_COUNT, hparams.n_mudd_taps, false); @@ -68,11 +69,21 @@ void llama_model_limite::load_arch_tensors(llama_model_loader &) { } } +llama_rope_params llama_model_limite::get_rope_params(const llama_cparams & cparams, int il) const { + auto res = llama_model::get_rope_params(cparams, il); + + // the SWA layers use the global base and scale + res.freq_base = cparams.rope_freq_base; + res.freq_scale = cparams.rope_freq_scale; + + return res; +} + std::unique_ptr llama_model_limite::build_arch_graph(const llm_graph_params & params) const { - return std::make_unique(*this, params); + return std::make_unique(params); } -llama_model_limite::graph::graph(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) { +llama_model_limite::graph::graph(const llm_graph_params & params) : llm_graph_context(params) { const int64_t n_embd_head = hparams.n_embd_head_k(); const int64_t n_taps = hparams.n_mudd_taps; @@ -160,14 +171,10 @@ llama_model_limite::graph::graph(const llama_model & model, const llm_graph_para cb(Kcur, "Kcur_normed", il); // the full attention layers use no positional encoding - if (hparams.is_swa(il)) { - Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); - Kcur = ggml_rope_ext(ctx0, Kcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); - cb(Qcur, "Qcur", il); - cb(Kcur, "Kcur", il); - } + Qcur = build_rope(Qcur, inp_pos, il); + Kcur = build_rope(Kcur, inp_pos, il); + cb(Qcur, "Qcur", il); + cb(Kcur, "Kcur", il); cur = build_attn(inp_attn, nullptr, nullptr, nullptr, diff --git a/src/models/llada-moe.cpp b/src/models/llada-moe.cpp index 0930cfb3cf78..0086d52afd8a 100644 --- a/src/models/llada-moe.cpp +++ b/src/models/llada-moe.cpp @@ -48,10 +48,10 @@ void llama_model_llada_moe::load_arch_tensors(llama_model_loader &) { } std::unique_ptr llama_model_llada_moe::build_arch_graph(const llm_graph_params & params) const { - return std::make_unique(*this, params); + return std::make_unique(params); } -llama_model_llada_moe::graph::graph(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) { +llama_model_llada_moe::graph::graph(const llm_graph_params & params) : llm_graph_context(params) { const int64_t n_embd_head = hparams.n_embd_head_v(); GGML_ASSERT(n_embd_head == hparams.n_embd_head_k()); @@ -90,17 +90,8 @@ llama_model_llada_moe::graph::graph(const llama_model & model, const llm_graph_p Kcur = build_norm(Kcur, model.layers[il].attn_k_norm, NULL, LLM_NORM_RMS, il); cb(Kcur, "Kcur_normed", il); - Qcur = ggml_rope_ext( - ctx0, Qcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); - - Kcur = ggml_rope_ext( - ctx0, Kcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); + Qcur = build_rope(Qcur, inp_pos, il); + Kcur = build_rope(Kcur, inp_pos, il); cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); diff --git a/src/models/llada.cpp b/src/models/llada.cpp index 1277e9dd7cbe..8777cff642cc 100644 --- a/src/models/llada.cpp +++ b/src/models/llada.cpp @@ -59,11 +59,19 @@ void llama_model_llada::load_arch_tensors(llama_model_loader &) { } } +llama_rope_params llama_model_llada::get_rope_params(const llama_cparams & cparams, int il) const { + auto res = llama_model::get_rope_params(cparams, il); + + res.factors = nullptr; + + return res; +} + std::unique_ptr llama_model_llada::build_arch_graph(const llm_graph_params & params) const { - return std::make_unique(*this, params); + return std::make_unique(params); } -llama_model_llada::graph::graph(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) { +llama_model_llada::graph::graph(const llm_graph_params & params) : llm_graph_context(params) { // LLaDA is similar to LLaMA but uses non-causal attention for diffusion const int64_t n_embd_head = hparams.n_embd_head_v(); @@ -96,11 +104,8 @@ llama_model_llada::graph::graph(const llama_model & model, const llm_graph_param auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur, n_embd_head, n_head, n_head_kv, il); - Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); - - Kcur = ggml_rope_ext(ctx0, Kcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); + Qcur = build_rope(Qcur, inp_pos, il); + Kcur = build_rope(Kcur, inp_pos, il); cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); diff --git a/src/models/llama-embed.cpp b/src/models/llama-embed.cpp index 0699e744461b..691eb8956d02 100644 --- a/src/models/llama-embed.cpp +++ b/src/models/llama-embed.cpp @@ -1,6 +1,6 @@ #include "models.h" std::unique_ptr llama_model_llama_embed::build_arch_graph(const llm_graph_params & params) const { - return std::make_unique>(*this, params); + return std::make_unique>(params); } diff --git a/src/models/llama.cpp b/src/models/llama.cpp index f9c1251131a3..26d54968cabd 100644 --- a/src/models/llama.cpp +++ b/src/models/llama.cpp @@ -92,11 +92,11 @@ void llama_model_llama::load_arch_tensors(llama_model_loader &) { } std::unique_ptr llama_model_llama::build_arch_graph(const llm_graph_params & params) const { - return std::make_unique>(*this, params); + return std::make_unique>(params); } template -llama_model_llama::graph::graph(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) { +llama_model_llama::graph::graph(const llm_graph_params & params) : llm_graph_context(params) { const int64_t n_embd_head = hparams.n_embd_head_v(); GGML_ASSERT(n_embd_head == hparams.n_embd_head_k()); @@ -136,24 +136,12 @@ llama_model_llama::graph::graph(const llama_model & model, const llm_grap // self-attention { - // rope freq factors for llama3; may return nullptr for llama2 and other models - ggml_tensor * rope_factors = model.get_rope_factors(cparams, il); - // compute Q and K and RoPE them auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur, n_embd_head, n_head, n_head_kv, il); - Qcur = ggml_rope_ext( - ctx0, Qcur, inp_pos, rope_factors, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); - - Kcur = ggml_rope_ext( - ctx0, Kcur, inp_pos, rope_factors, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); + Qcur = build_rope(Qcur, inp_pos, il); + Kcur = build_rope(Kcur, inp_pos, il); cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); diff --git a/src/models/llama4.cpp b/src/models/llama4.cpp index 9a365394ce45..5fa2a623f743 100644 --- a/src/models/llama4.cpp +++ b/src/models/llama4.cpp @@ -23,6 +23,10 @@ void llama_model_llama4::load_arch_hparams(llama_model_loader & ml) { ml.get_key(LLM_KV_ROPE_FREQ_BASE_SWA, hparams.rope_freq_base_train_swa, false); } + for (uint32_t il = 0; il < hparams.n_layer_all; ++il) { + hparams.rope_pattern[il] = (il + 1) % hparams.n_no_rope_layer_step != 0; + } + switch (hparams.n_expert) { case 0: { // MobileLLM (no MoE) @@ -95,14 +99,14 @@ void llama_model_llama4::load_arch_tensors(llama_model_loader &) { std::unique_ptr llama_model_llama4::build_arch_graph(const llm_graph_params & params) const { if (hparams.swa_type == LLAMA_SWA_TYPE_NONE) { - return std::make_unique>(*this, params); + return std::make_unique>(params); } else { - return std::make_unique>(*this, params); + return std::make_unique>(params); } } template -llama_model_llama4::graph::graph(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) { +llama_model_llama4::graph::graph(const llm_graph_params & params) : llm_graph_context(params) { const int64_t n_embd_head = hparams.n_embd_head_v(); GGML_ASSERT(n_embd_head == hparams.n_embd_head_k()); @@ -134,8 +138,6 @@ llama_model_llama4::graph::graph(const llama_model & model, const llm_grap ggml_tensor * inp_out_ids = build_inp_out_ids(); for (int il = 0; il < n_layer; ++il) { - const float freq_base_l = model.get_rope_freq_base (cparams, il); - const float freq_scale_l = model.get_rope_freq_scale(cparams, il); ggml_tensor * inpSA = inpL; @@ -151,25 +153,13 @@ llama_model_llama4::graph::graph(const llama_model & model, const llm_grap // self-attention { - // rope freq factors for llama3; may return nullptr for llama2 and other models - ggml_tensor * rope_factors = model.get_rope_factors(cparams, il); - // compute Q and K and RoPE them auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur, n_embd_head, n_head, n_head_kv, il); if (use_rope) { - Qcur = ggml_rope_ext( - ctx0, Qcur, inp_pos, rope_factors, - n_rot, rope_type, n_ctx_orig, freq_base_l, freq_scale_l, - ext_factor, attn_factor, beta_fast, beta_slow - ); - - Kcur = ggml_rope_ext( - ctx0, Kcur, inp_pos, rope_factors, - n_rot, rope_type, n_ctx_orig, freq_base_l, freq_scale_l, - ext_factor, attn_factor, beta_fast, beta_slow - ); + Qcur = build_rope(Qcur, inp_pos, il); + Kcur = build_rope(Kcur, inp_pos, il); } else if (inp_attn_scale) { Qcur = ggml_mul(ctx0, Qcur, inp_attn_scale); } diff --git a/src/models/maincoder.cpp b/src/models/maincoder.cpp index 1eeab45a156e..1a2d2c695b6c 100644 --- a/src/models/maincoder.cpp +++ b/src/models/maincoder.cpp @@ -40,11 +40,19 @@ void llama_model_maincoder::load_arch_tensors(llama_model_loader &) { } } +llama_rope_params llama_model_maincoder::get_rope_params(const llama_cparams & cparams, int il) const { + auto res = llama_model::get_rope_params(cparams, il); + + res.k_norm = layers[il].attn_k_norm; + + return res; +} + std::unique_ptr llama_model_maincoder::build_arch_graph(const llm_graph_params & params) const { - return std::make_unique(*this, params); + return std::make_unique(params); } -llama_model_maincoder::graph::graph(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) { +llama_model_maincoder::graph::graph(const llm_graph_params & params) : llm_graph_context(params) { const int64_t n_embd_head = hparams.n_embd_head_v(); GGML_ASSERT(n_embd_head == hparams.n_embd_head_k()); @@ -77,17 +85,8 @@ llama_model_maincoder::graph::graph(const llama_model & model, const llm_graph_p auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur, n_embd_head, n_head, n_head_kv, il); - Qcur = ggml_rope_ext( - ctx0, Qcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); - - Kcur = ggml_rope_ext( - ctx0, Kcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); + Qcur = build_rope(Qcur, inp_pos, il); + Kcur = build_rope(Kcur, inp_pos, il); Qcur = build_norm(Qcur, model.layers[il].attn_q_norm, NULL, LLM_NORM_RMS, il); cb(Qcur, "Qcur_normed", il); diff --git a/src/models/maion-coder.cpp b/src/models/maion-coder.cpp index bb3ebabc7ed3..5c1e376c5d8c 100644 --- a/src/models/maion-coder.cpp +++ b/src/models/maion-coder.cpp @@ -65,10 +65,10 @@ void llama_model_maion_coder::load_arch_tensors(llama_model_loader &) { } std::unique_ptr llama_model_maion_coder::build_arch_graph(const llm_graph_params & params) const { - return std::make_unique(*this, params); + return std::make_unique(params); } -llama_model_maion_coder::graph::graph(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) { +llama_model_maion_coder::graph::graph(const llm_graph_params & params) : llm_graph_context(params) { const int64_t n_embd_head = hparams.n_embd_head_k(); GGML_ASSERT(n_embd_head == hparams.n_embd_head_v()); @@ -94,12 +94,8 @@ llama_model_maion_coder::graph::graph(const llama_model & model, const llm_graph Kcur = build_norm(Kcur, model.layers[il].attn_k_norm, NULL, LLM_NORM_RMS, il); // Full-attention layers do not use RoPE. - if (hparams.is_swa(il)) { - const float freq_base_l = model.get_rope_freq_base(cparams, il); - const float freq_scale_l = model.get_rope_freq_scale(cparams, il); - Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base_l, freq_scale_l, ext_factor, attn_factor, beta_fast, beta_slow); - Kcur = ggml_rope_ext(ctx0, Kcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base_l, freq_scale_l, ext_factor, attn_factor, beta_fast, beta_slow); - } + Qcur = build_rope(Qcur, inp_pos, il); + Kcur = build_rope(Kcur, inp_pos, il); cur = build_attn(inp_attn, model.layers[il].wo, NULL, model.layers[il].wo_s, Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, kq_scale, il); diff --git a/src/models/mamba-base.cpp b/src/models/mamba-base.cpp index 370de0f0141d..4e16f18aaaf0 100644 --- a/src/models/mamba-base.cpp +++ b/src/models/mamba-base.cpp @@ -8,7 +8,6 @@ llm_build_mamba_base::llm_build_mamba_base(const llm_graph_params & params) : ll ggml_tensor * llm_build_mamba_base::build_mamba_layer(llm_graph_input_rs * inp, ggml_tensor * cur, - const llama_model & model, const llama_ubatch & ubatch, int il) { const auto * mctx_cur = inp->mctx; @@ -152,7 +151,6 @@ ggml_tensor * llm_build_mamba_base::build_mamba_layer(llm_graph_input_rs * inp, ggml_tensor * llm_build_mamba_base::build_mamba2_layer(llm_graph_input_rs * inp, ggml_tensor * cur, - const llama_model & model, const llama_ubatch & ubatch, int il) const { const auto * mctx_cur = inp->mctx; @@ -206,19 +204,7 @@ ggml_tensor * llm_build_mamba_base::build_mamba2_layer(llm_graph_input_rs * inp, // => {d_conv - 1 + n_seq_tokens, d_inner + 2*n_group*d_state, n_seqs} ggml_tensor * conv_x = ggml_concat(ctx0, conv, ggml_transpose(ctx0, xBC), 0); - const int64_t row_count = (d_conv - 1) * (d_inner + 2 * n_group * d_state); - const size_t row_size = ggml_row_size(conv_states_all->type, row_count); - const int64_t n_written = std::min(n_seq_tokens, K); - - for (int64_t slot = 0; slot < n_written; ++slot) { - ggml_tensor * last_conv = ggml_view_3d(ctx0, conv_x, d_conv - 1, d_inner + 2 * n_group * d_state, n_seqs, - conv_x->nb[1], conv_x->nb[2], (n_seq_tokens - slot) * conv_x->nb[0]); - - ggml_build_forward_expand(gf, ggml_cpy(ctx0, last_conv, - ggml_view_2d(ctx0, conv_states_all, row_count, n_seqs, - conv_states_all->nb[1], - ((size_t) slot * mem_size + kv_head) * row_size))); - } + build_conv_state_store(inp, conv_states_all, conv_x, d_conv - 1, 0); // 1D convolution // The equivalent is to make a self-overlapping view of conv_x @@ -290,7 +276,11 @@ ggml_tensor * llm_build_mamba_base::build_mamba2_layer(llm_graph_input_rs * inp, // grouped RMS norm if (model.layers[il].ssm_norm) { y = ggml_reshape_4d(ctx0, y, d_inner / n_group, n_group, n_seq_tokens, n_seqs); - y = build_norm(y, model.layers[il].ssm_norm, NULL, LLM_NORM_RMS, il); + if (hparams.ssm_norm_eps > 0.0f) { + y = ggml_mul(ctx0, ggml_rms_norm(ctx0, y, hparams.ssm_norm_eps), model.layers[il].ssm_norm); + } else { + y = build_norm(y, model.layers[il].ssm_norm, NULL, LLM_NORM_RMS, il); + } } y = ggml_reshape_2d(ctx0, y, d_inner, n_seq_tokens * n_seqs); diff --git a/src/models/mamba.cpp b/src/models/mamba.cpp index d47e8e5e1252..13a9de6b0aea 100644 --- a/src/models/mamba.cpp +++ b/src/models/mamba.cpp @@ -81,10 +81,10 @@ void llama_model_mamba::load_arch_tensors(llama_model_loader &) { } std::unique_ptr llama_model_mamba::build_arch_graph(const llm_graph_params & params) const { - return std::make_unique(*this, params); + return std::make_unique(params); } -llama_model_mamba::graph::graph(const llama_model & model, const llm_graph_params & params) : llm_build_mamba_base(params) { +llama_model_mamba::graph::graph(const llm_graph_params & params) : llm_build_mamba_base(params) { ggml_tensor * cur; ggml_tensor * inpL; @@ -101,9 +101,9 @@ llama_model_mamba::graph::graph(const llama_model & model, const llm_graph_param cb(cur, "attn_norm", il); if (model.arch == LLM_ARCH_MAMBA2) { - cur = build_mamba2_layer(rs_inp, cur, model, ubatch, il); + cur = build_mamba2_layer(rs_inp, cur, ubatch, il); } else { - cur = build_mamba_layer(rs_inp, cur, model, ubatch, il); + cur = build_mamba_layer(rs_inp, cur, ubatch, il); } if (il == n_layer - 1 && inp_out_ids) { diff --git a/src/models/mamba2.cpp b/src/models/mamba2.cpp index d5c167cf0560..9088d941e8b0 100644 --- a/src/models/mamba2.cpp +++ b/src/models/mamba2.cpp @@ -83,6 +83,6 @@ void llama_model_mamba2::load_arch_tensors(llama_model_loader &) { } std::unique_ptr llama_model_mamba2::build_arch_graph(const llm_graph_params & params) const { - return std::make_unique(*this, params); + return std::make_unique(params); } diff --git a/src/models/maple.cpp b/src/models/maple.cpp index 9568c647589c..eb0b3b4ac823 100644 --- a/src/models/maple.cpp +++ b/src/models/maple.cpp @@ -8,6 +8,7 @@ void llama_model_maple::load_arch_hparams(llama_model_loader & ml) { ml.get_key_or_arr(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp_arr, hparams.n_layer_all); ml.get_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, hparams.is_swa_impl); + hparams.rope_pattern = hparams.is_swa_impl; hparams.rope_freq_base_train_swa = hparams.rope_freq_base_train; hparams.rope_freq_scale_train_swa = hparams.rope_freq_scale_train; @@ -59,10 +60,10 @@ void llama_model_maple::load_arch_tensors(llama_model_loader &) { } std::unique_ptr llama_model_maple::build_arch_graph(const llm_graph_params & params) const { - return std::make_unique(*this, params); + return std::make_unique(params); } -llama_model_maple::graph::graph(const llama_model & model, const llm_graph_params & params) : +llama_model_maple::graph::graph(const llm_graph_params & params) : llm_graph_context(params) { const int64_t n_embd_head = hparams.n_embd_head_k(); @@ -87,16 +88,8 @@ llama_model_maple::graph::graph(const llama_model & model, const llm_graph_param cb(Qcur, "Qcur_normed", il); cb(Kcur, "Kcur_normed", il); - if (hparams.is_swa(il)) { - const int64_t n_rot_l = hparams.n_rot(il); - const float freq_base_l = model.get_rope_freq_base(cparams, il); - const float freq_scale_l = model.get_rope_freq_scale(cparams, il); - - Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, nullptr, n_rot_l, rope_type, n_ctx_orig, freq_base_l, - freq_scale_l, ext_factor, attn_factor, beta_fast, beta_slow); - Kcur = ggml_rope_ext(ctx0, Kcur, inp_pos, nullptr, n_rot_l, rope_type, n_ctx_orig, freq_base_l, - freq_scale_l, ext_factor, attn_factor, beta_fast, beta_slow); - } + Qcur = build_rope(Qcur, inp_pos, il); + Kcur = build_rope(Kcur, inp_pos, il); cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); cb(Vcur, "Vcur", il); diff --git a/src/models/mellum.cpp b/src/models/mellum.cpp index eaa540310c7f..57dd5b79a28e 100644 --- a/src/models/mellum.cpp +++ b/src/models/mellum.cpp @@ -88,24 +88,37 @@ void llama_model_mellum::load_arch_tensors(llama_model_loader & ml) { } } +llama_rope_params llama_model_mellum::get_rope_params(const llama_cparams & cparams, int il) const { + auto res = llama_model::get_rope_params(cparams, il); + + if (hparams.is_swa(il)) { + res.ext_factor = 0.0f; + res.attn_factor = 1.0f; + res.beta_fast = 32.0f; + res.beta_slow = 1.0f; + } + + return res; +} + std::unique_ptr llama_model_mellum::build_arch_graph(const llm_graph_params & params) const { const bool iswa = hparams.swa_type == LLAMA_SWA_TYPE_STANDARD; if (params.gtype == LLM_GRAPH_TYPE_DECODER_MTP) { if (iswa) { - return std::make_unique>(*this, params); + return std::make_unique>(params); } - return std::make_unique>(*this, params); + return std::make_unique>(params); } if (iswa) { - return std::make_unique>(*this, params); + return std::make_unique>(params); } - return std::make_unique>(*this, params); + return std::make_unique>(params); } template -llama_model_mellum::graph::graph(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) { +llama_model_mellum::graph::graph(const llm_graph_params & params) : llm_graph_context(params) { const int64_t n_embd_head = hparams.n_embd_head_v(); GGML_ASSERT(n_embd_head == hparams.n_embd_head_k()); @@ -151,26 +164,8 @@ llama_model_mellum::graph::graph(const llama_model & model, const llm_grap Kcur = build_norm(Kcur, model.layers[il].attn_k_norm, nullptr, LLM_NORM_RMS, il); cb(Kcur, "Kcur_normed", il); - const bool is_swa = hparams.is_swa(il); - - const float freq_base_l = model.get_rope_freq_base(cparams, il); - const float freq_scale_l = model.get_rope_freq_scale(cparams, il); - const float ext_factor_l = is_swa ? 0.0f : ext_factor; - const float attn_factor_l = is_swa ? 1.0f : attn_factor; - const float beta_fast_l = is_swa ? 32.0f : beta_fast; - const float beta_slow_l = is_swa ? 1.0f : beta_slow; - - Qcur = ggml_rope_ext( - ctx0, Qcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base_l, freq_scale_l, - ext_factor_l, attn_factor_l, beta_fast_l, beta_slow_l - ); - - Kcur = ggml_rope_ext( - ctx0, Kcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base_l, freq_scale_l, - ext_factor_l, attn_factor_l, beta_fast_l, beta_slow_l - ); + Qcur = build_rope(Qcur, inp_pos, il); + Kcur = build_rope(Kcur, inp_pos, il); cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); @@ -246,7 +241,7 @@ llama_model_mellum::graph::graph(const llama_model & model, const llm_grap } template -llama_model_mellum::graph_mtp::graph_mtp(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) { +llama_model_mellum::graph_mtp::graph_mtp(const llm_graph_params & params) : llm_graph_context(params) { GGML_ASSERT(hparams.n_layer_nextn == 1 && "MELLUM MTP currently only supports a single MTP block"); const int64_t n_embd_head = hparams.n_embd_head_v(); @@ -271,9 +266,7 @@ llama_model_mellum::graph_mtp::graph_mtp(const llama_model & model, const inp->embd = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, hparams.n_embd_inp(), n_tokens); ggml_set_input(inp->embd); - inp->h = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, hparams.n_embd, n_tokens); - ggml_set_input(inp->h); - ggml_set_name(inp->h, "mtp_h_input"); + inp->h = build_inp_embd_state(inp.get(), hparams.n_embd); // TODO: make static using `ggml_build_forward_select()` // see llm_graph_context::build_inp_embd() for reference @@ -329,26 +322,8 @@ llama_model_mellum::graph_mtp::graph_mtp(const llama_model & model, const Kcur = build_norm(Kcur, layer.attn_k_norm, nullptr, LLM_NORM_RMS, il); cb(Kcur, "mtp_Kcur_normed", il); - const bool is_swa = hparams.is_swa(il); - - const float freq_base_l = model.get_rope_freq_base(cparams, il); - const float freq_scale_l = model.get_rope_freq_scale(cparams, il); - const float ext_factor_l = is_swa ? 0.0f : ext_factor; - const float attn_factor_l = is_swa ? 1.0f : attn_factor; - const float beta_fast_l = is_swa ? 32.0f : beta_fast; - const float beta_slow_l = is_swa ? 1.0f : beta_slow; - - Qcur = ggml_rope_ext( - ctx0, Qcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base_l, freq_scale_l, - ext_factor_l, attn_factor_l, beta_fast_l, beta_slow_l - ); - - Kcur = ggml_rope_ext( - ctx0, Kcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base_l, freq_scale_l, - ext_factor_l, attn_factor_l, beta_fast_l, beta_slow_l - ); + Qcur = build_rope(Qcur, inp_pos, il); + Kcur = build_rope(Kcur, inp_pos, il); cb(Qcur, "mtp_Qcur", il); cb(Kcur, "mtp_Kcur", il); diff --git a/src/models/metis16.cpp b/src/models/metis16.cpp new file mode 100644 index 000000000000..8a433f3cc0bb --- /dev/null +++ b/src/models/metis16.cpp @@ -0,0 +1,403 @@ +#include "models.h" + +#include + +void llama_model_metis16::load_arch_hparams(llama_model_loader & ml) { + ml.get_key(LLM_KV_SSM_CONV_KERNEL, hparams.ssm_d_conv); + ml.get_key(LLM_KV_SSM_INNER_SIZE, hparams.ssm_d_inner); + ml.get_key(LLM_KV_SSM_STATE_SIZE, hparams.ssm_d_state); + ml.get_key(LLM_KV_SSM_TIME_STEP_RANK, hparams.ssm_dt_rank); + ml.get_key(LLM_KV_SSM_GROUP_COUNT, hparams.ssm_n_group); + ml.get_key(LLM_KV_SSM_NORM_EPS, hparams.ssm_norm_eps); + + ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps); + + ml.get_key_or_arr(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp_arr, hparams.n_layer_all); + ml.get_key(LLM_KV_EXPERT_SHARED_FEED_FORWARD_LENGTH, hparams.n_ff_shexp); + ml.get_key(LLM_KV_MOE_LATENT_SIZE, hparams.moe_latent_size); + + ml.get_key(LLM_KV_HYPER_CONNECTION_COUNT, hparams.dsv4_hc_mult); + ml.get_key(LLM_KV_METIS_PASS_COUNT, hparams.metis_n_pass); + ml.get_key(LLM_KV_METIS_MEMORY_LENGTH, hparams.metis_n_mem); + ml.get_key(LLM_KV_METIS_ROUTE_LENGTH, hparams.metis_n_route); + ml.get_key(LLM_KV_ATTENTION_PASS_LORA_RANK, hparams.metis_pass_lora_rank); + ml.get_key(LLM_KV_METIS_CONTINUATION_LOGIT_LIMIT, hparams.metis_cont_limit); + + n_pass = hparams.metis_n_pass; + n_stream = hparams.dsv4_hc_mult; + n_mem = hparams.metis_n_mem; + n_route = hparams.metis_n_route; + cont_limit = hparams.metis_cont_limit; + + if (n_pass < 1 || hparams.n_layer() % n_pass != 0) { + throw std::runtime_error(format("metis16: %u layers do not split into %u passes", hparams.n_layer(), n_pass)); + } + n_layer_phys = hparams.n_layer() / n_pass; + + bool has_attn = false; + for (uint32_t il = 0; il < hparams.n_layer(); ++il) { + if (hparams.n_head_kv(il) != hparams.n_head_kv(il % n_layer_phys)) { + throw std::runtime_error(format("metis16: layer %u does not repeat the layout of layer %u", il, il % n_layer_phys)); + } + hparams.is_recr_impl[il] = hparams.n_head_kv(il) == 0; + has_attn = has_attn || !hparams.is_recr_impl[il]; + } + if (!has_attn) { + throw std::runtime_error("metis16: at least one attention layer is required"); + } + + switch (n_layer_phys) { + case 12: type = LLM_TYPE_3B; break; // Praxis + case 20: type = LLM_TYPE_10B; break; // Logos + default: type = LLM_TYPE_UNKNOWN; + } +} + +void llama_model_metis16::load_arch_tensors(llama_model_loader &) { + LLAMA_LOAD_LOCALS; + + const int64_t d_conv = hparams.ssm_d_conv; + const int64_t d_inner = hparams.ssm_d_inner; + const int64_t d_state = hparams.ssm_d_state; + const int64_t n_ssm_head = hparams.ssm_dt_rank; + const int64_t n_group = hparams.ssm_n_group; + const int64_t d_in_proj = 2*d_inner + 2*n_group*d_state + n_ssm_head; + const int64_t n_latent = hparams.moe_latent_size; + const int64_t n_ff_shexp = hparams.n_ff_shexp; + const int64_t n_s = n_stream; + const int64_t n_p = n_pass; + const int64_t n_m = n_mem; + const int64_t n_r = n_route; + + int64_t n_attn = 0; + for (uint32_t il = 0; il < n_layer_phys; ++il) { + n_attn += !hparams.is_recr(il); + } + + tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, 0); + output_norm = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd}, 0); + output = create_tensor(tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab}, TENSOR_NOT_REQUIRED); + if (output == NULL) { + output = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, TENSOR_DUPLICATED); + } + + stream_embd = create_tensor(tn(LLM_TENSOR_STREAM_EMBD, "weight"), {n_s, n_embd}, 0); + + depth_mem_state_write = create_tensor(tn(LLM_TENSOR_DEPTH_MEM_STATE_WRITE, "weight"), {n_embd, n_m}, 0); + depth_mem_meta_write = create_tensor(tn(LLM_TENSOR_DEPTH_MEM_META_WRITE, "weight"), {n_r + 3, n_m}, 0); + depth_mem_entry_bias = create_tensor(tn(LLM_TENSOR_DEPTH_MEM_ENTRY_BIAS, "weight"), {n_m, n_p*(n_attn + 1)}, 0); + depth_mem_query = create_tensor(tn(LLM_TENSOR_DEPTH_MEM_QUERY, "weight"), {n_embd, n_m}, 0); + depth_mem_query_b = create_tensor(tn(LLM_TENSOR_DEPTH_MEM_QUERY, "bias"), {n_m}, 0); + depth_mem_key = create_tensor(tn(LLM_TENSOR_DEPTH_MEM_KEY, "weight"), {n_m, n_m}, 0); + depth_mem_key_b = create_tensor(tn(LLM_TENSOR_DEPTH_MEM_KEY, "bias"), {n_m}, 0); + depth_mem_value = create_tensor(tn(LLM_TENSOR_DEPTH_MEM_VALUE, "weight"), {n_m, n_m}, 0); + depth_mem_value_b = create_tensor(tn(LLM_TENSOR_DEPTH_MEM_VALUE, "bias"), {n_m}, 0); + depth_mem_out = create_tensor(tn(LLM_TENSOR_DEPTH_MEM_OUT, "weight"), {n_m, n_embd}, 0); + depth_mem_out_b = create_tensor(tn(LLM_TENSOR_DEPTH_MEM_OUT, "bias"), {n_embd}, 0); + depth_mem_route_state = create_tensor(tn(LLM_TENSOR_DEPTH_MEM_ROUTE_STATE, "weight"), {n_embd, n_r}, 0); + depth_mem_route_retr = create_tensor(tn(LLM_TENSOR_DEPTH_MEM_ROUTE_RETR, "weight"), {n_embd, n_r}, 0); + depth_mem_route_diff = create_tensor(tn(LLM_TENSOR_DEPTH_MEM_ROUTE_DIFF, "weight"), {n_embd, n_r}, 0); + depth_mem_route_hist = create_tensor(tn(LLM_TENSOR_DEPTH_MEM_ROUTE_HIST, "weight"), {n_r, n_r}, 0); + depth_mem_route_b = create_tensor(tn(LLM_TENSOR_DEPTH_MEM_ROUTE, "bias"), {n_r}, 0); + depth_mem_stream_gate = create_tensor(tn(LLM_TENSOR_DEPTH_MEM_STREAM_GATE, "weight"), {n_embd, n_s}, 0); + depth_mem_stream_gate_b = create_tensor(tn(LLM_TENSOR_DEPTH_MEM_STREAM_GATE, "bias"), {n_s}, 0); + + cont_hidden_state = create_tensor(tn(LLM_TENSOR_CONT_HIDDEN_STATE, "weight"), {n_embd, n_r}, 0); + cont_hidden_retr = create_tensor(tn(LLM_TENSOR_CONT_HIDDEN_RETR, "weight"), {n_embd, n_r}, 0); + cont_hidden_diff = create_tensor(tn(LLM_TENSOR_CONT_HIDDEN_DIFF, "weight"), {n_embd, n_r}, 0); + cont_hidden_route = create_tensor(tn(LLM_TENSOR_CONT_HIDDEN_ROUTE, "weight"), {n_r, n_r}, 0); + cont_hidden_b = create_tensor(tn(LLM_TENSOR_CONT_HIDDEN, "bias"), {n_r}, 0); + cont_output = create_tensor(tn(LLM_TENSOR_CONT_OUTPUT, "weight"), {n_r, 1}, 0); + cont_output_b = create_tensor(tn(LLM_TENSOR_CONT_OUTPUT, "bias"), {1}, 0); + + for (int il = 0; il < (int) n_layer_phys; ++il) { + auto & layer = layers[il]; + + layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, "weight", il), {n_embd}, 0); + layer.hc_attn_mix = create_tensor(tn(LLM_TENSOR_HC_ATTN_MIX, "weight", il), {n_s, n_s, n_p}, 0); + layer.hc_attn_read = create_tensor(tn(LLM_TENSOR_HC_ATTN_READ, "weight", il), {n_s, n_p}, 0); + layer.hc_attn_write = create_tensor(tn(LLM_TENSOR_HC_ATTN_WRITE, "weight", il), {n_s, n_p}, 0); + layer.hc_ffn_mix = create_tensor(tn(LLM_TENSOR_HC_FFN_MIX, "weight", il), {n_s, n_s, n_p}, 0); + layer.hc_ffn_read = create_tensor(tn(LLM_TENSOR_HC_FFN_READ, "weight", il), {n_s, n_p}, 0); + layer.hc_ffn_write = create_tensor(tn(LLM_TENSOR_HC_FFN_WRITE, "weight", il), {n_s, n_p}, 0); + layer.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, "weight", il), {n_embd}, 0); + + if (hparams.is_recr(il)) { + layer.ssm_in = create_tensor(tn(LLM_TENSOR_SSM_IN, "weight", il), {n_embd, d_in_proj}, 0); + layer.ssm_conv1d = create_tensor(tn(LLM_TENSOR_SSM_CONV1D, "weight", il), {d_conv, d_inner + 2*n_group*d_state}, 0); + layer.ssm_conv1d_b = create_tensor(tn(LLM_TENSOR_SSM_CONV1D, "bias", il), {d_inner + 2*n_group*d_state}, 0); + layer.ssm_dt_b = create_tensor(tn(LLM_TENSOR_SSM_DT, "bias", il), {n_ssm_head}, 0); + layer.ssm_norm = create_tensor(tn(LLM_TENSOR_SSM_NORM, "weight", il), {d_inner / n_group, n_group}, 0); + layer.ssm_out = create_tensor(tn(LLM_TENSOR_SSM_OUT, "weight", il), {d_inner, n_embd}, 0); + + // no "weight" suffix for these + layer.ssm_a = create_tensor(tn(LLM_TENSOR_SSM_A, il), {1, n_ssm_head}, 0); + layer.ssm_d = create_tensor(tn(LLM_TENSOR_SSM_D, il), {1, n_ssm_head}, 0); + } else { + const int64_t n_qkv = n_embd_head_k * (hparams.n_head(il) + 2*hparams.n_head_kv(il)); + const int64_t rank = hparams.metis_pass_lora_rank; + + layer.wqkv = create_tensor(tn(LLM_TENSOR_ATTN_QKV, "weight", il), {n_embd, n_qkv}, 0); + layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", il), {n_embd_head_k * hparams.n_head(il), n_embd}, 0); + layer.attn_pass_lora_down = create_tensor(tn(LLM_TENSOR_ATTN_PASS_LORA_DOWN, "weight", il), {n_embd, rank}, 0); + layer.attn_pass_lora_up = create_tensor(tn(LLM_TENSOR_ATTN_PASS_LORA_UP, "weight", il), {rank, n_qkv}, 0); + layer.attn_pass_gate = create_tensor(tn(LLM_TENSOR_ATTN_PASS_GATE, "weight", il), {n_p}, 0); + } + + const int64_t n_ff_exp = hparams.n_ff_exp(il); + + layer.ffn_latent_down = create_tensor(tn(LLM_TENSOR_FFN_LATENT_DOWN, "weight", il), {n_embd, n_latent}, 0); + layer.ffn_latent_up = create_tensor(tn(LLM_TENSOR_FFN_LATENT_UP, "weight", il), {n_latent, n_embd}, 0); + layer.ffn_gate_inp = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP, "weight", il), {n_latent + n_r, n_expert}, 0); + layer.ffn_gate_inp_b = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP, "bias", il), {n_expert}, 0); + layer.ffn_exp_probs_b = create_tensor(tn(LLM_TENSOR_FFN_EXP_PROBS_B, "bias", il), {n_expert}, 0); + layer.ffn_exp_embd = create_tensor(tn(LLM_TENSOR_FFN_EXP_EMBD, "weight", il), {n_r, n_expert}, 0); + layer.ffn_gate_up_exps = create_tensor(tn(LLM_TENSOR_FFN_GATE_UP_EXPS, "weight", il), {n_latent, 2*n_ff_exp, n_expert}, 0); + layer.ffn_down_exps = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS, "weight", il), {n_ff_exp, n_latent, n_expert}, 0); + layer.ffn_gate_shexp = create_tensor(tn(LLM_TENSOR_FFN_GATE_SHEXP, "weight", il), {n_latent, n_ff_shexp}, 0); + layer.ffn_up_shexp = create_tensor(tn(LLM_TENSOR_FFN_UP_SHEXP, "weight", il), {n_latent, n_ff_shexp}, 0); + layer.ffn_down_shexp = create_tensor(tn(LLM_TENSOR_FFN_DOWN_SHEXP, "weight", il), {n_ff_shexp, n_latent}, 0); + } + + // the virtual layers of later passes share the physical layer's weights + for (int il = n_layer_phys; il < n_layer; ++il) { + layers[il] = layers[il % n_layer_phys]; + } +} + +std::unique_ptr llama_model_metis16::build_arch_graph(const llm_graph_params & params) const { + return std::make_unique(params); +} + +// streams are kept stream-innermost: [n_stream, n_embd, n_tokens] +ggml_tensor * llama_model_metis16::graph::stream_mean(ggml_tensor * x) const { + ggml_tensor * mean = ggml_mean(ctx0, ggml_reshape_2d(ctx0, x, x->ne[0], x->ne[1]*x->ne[2])); + return ggml_reshape_2d(ctx0, mean, x->ne[1], x->ne[2]); +} + +// returns the normalized sublayer input +ggml_tensor * llama_model_metis16::graph::hc_read(ggml_tensor * x, ggml_tensor * mix, ggml_tensor * read, ggml_tensor * norm, + int p, ggml_tensor ** mixed, int il) const { + const int64_t n_s = x->ne[0]; + ggml_tensor * x2 = ggml_reshape_2d(ctx0, x, n_s, x->ne[1]*x->ne[2]); + + ggml_tensor * m = ggml_view_2d(ctx0, mix, n_s, n_s, mix->nb[1], p*mix->nb[2]); + *mixed = ggml_mul_mat(ctx0, m, x2); + + ggml_tensor * r = ggml_reshape_2d(ctx0, ggml_view_1d(ctx0, read, n_s, p*read->nb[1]), n_s, 1); + ggml_tensor * src = ggml_reshape_2d(ctx0, ggml_mul_mat(ctx0, r, x2), x->ne[1], x->ne[2]); + return build_norm(src, norm, NULL, LLM_NORM_RMS, il); +} + +ggml_tensor * llama_model_metis16::graph::hc_write(ggml_tensor * mixed, ggml_tensor * write, ggml_tensor * update, int p) const { + const int64_t n_s = mixed->ne[0]; + ggml_tensor * w = ggml_reshape_2d(ctx0, ggml_view_1d(ctx0, write, n_s, p*write->nb[1]), 1, n_s); + ggml_tensor * u = ggml_reshape_2d(ctx0, update, 1, update->ne[0]*update->ne[1]); + ggml_tensor * out = ggml_add(ctx0, mixed, ggml_mul_mat(ctx0, w, u)); + return ggml_reshape_3d(ctx0, out, n_s, update->ne[0], update->ne[1]); +} + +// blends the retrieved entries into the streams; returns their summary +ggml_tensor * llama_model_metis16::graph::mem_read(const llama_model_metis16 & model, ggml_tensor ** x) const { + const int64_t n_s = (*x)->ne[0]; + const int64_t n_m = model.n_mem; + + ggml_tensor * x_e = ggml_cont(ctx0, ggml_permute(ctx0, *x, 1, 0, 2, 3)); + + ggml_tensor * q = ggml_mul_mat(ctx0, model.depth_mem_query, ggml_reshape_2d(ctx0, x_e, n_embd, n_s*n_tokens)); + q = ggml_reshape_3d(ctx0, ggml_add(ctx0, q, model.depth_mem_query_b), n_m, n_s, n_tokens); + + ggml_tensor * kq = ggml_mul_mat(ctx0, bank_k, q); + kq = ggml_soft_max_ext(ctx0, kq, nullptr, 1.0f/sqrtf(float(n_m)), 0.0f); + + ggml_tensor * ret = ggml_mul_mat(ctx0, bank_vt, kq); + ggml_tensor * proj = ggml_mul_mat(ctx0, model.depth_mem_out, ggml_reshape_2d(ctx0, ret, n_m, n_s*n_tokens)); + proj = ggml_reshape_3d(ctx0, ggml_add(ctx0, proj, model.depth_mem_out_b), n_embd, n_s, n_tokens); + + ggml_tensor * gate = ggml_sum_rows(ctx0, ggml_mul(ctx0, x_e, model.depth_mem_stream_gate)); + gate = ggml_sigmoid(ctx0, ggml_add(ctx0, gate, ggml_reshape_2d(ctx0, model.depth_mem_stream_gate_b, 1, n_s))); + + ggml_tensor * proj_s = ggml_cont(ctx0, ggml_permute(ctx0, proj, 1, 0, 2, 3)); + *x = ggml_add(ctx0, *x, ggml_mul(ctx0, proj_s, ggml_reshape_3d(ctx0, gate, n_s, 1, n_tokens))); + + return stream_mean(proj_s); +} + +void llama_model_metis16::graph::mem_write(const llama_model_metis16 & model, ggml_tensor * x, ggml_tensor * summary, + ggml_tensor * entropy, ggml_tensor * confidence, ggml_tensor * cont, int entry) { + const int64_t n_m = model.n_mem; + + if (cont == nullptr) { + cont = ggml_scale(ctx0, confidence, 0.0f); + } + ggml_tensor * meta = ggml_concat(ctx0, ggml_concat(ctx0, summary, entropy, 0), ggml_concat(ctx0, confidence, cont, 0), 0); + + ggml_tensor * e = ggml_add(ctx0, ggml_mul_mat(ctx0, model.depth_mem_state_write, stream_mean(x)), ggml_mul_mat(ctx0, model.depth_mem_meta_write, meta)); + e = ggml_add(ctx0, e, ggml_view_1d(ctx0, model.depth_mem_entry_bias, n_m, entry*model.depth_mem_entry_bias->nb[1])); + + ggml_tensor * k = ggml_add(ctx0, ggml_mul_mat(ctx0, model.depth_mem_key, e), model.depth_mem_key_b); + ggml_tensor * v = ggml_add(ctx0, ggml_mul_mat(ctx0, model.depth_mem_value, e), model.depth_mem_value_b); + k = ggml_reshape_3d(ctx0, k, n_m, 1, n_tokens); + v = ggml_reshape_3d(ctx0, v, 1, n_m, n_tokens); + bank_k = bank_k ? ggml_concat(ctx0, bank_k, k, 1) : k; + bank_vt = bank_vt ? ggml_concat(ctx0, bank_vt, v, 0) : v; +} + +ggml_tensor * llama_model_metis16::graph::route_features(const llama_model_metis16 & model, ggml_tensor * state, + ggml_tensor * summary, ggml_tensor * diff, ggml_tensor * history) const { + ggml_tensor * f = ggml_mul_mat(ctx0, model.depth_mem_route_state, state); + if (summary) { + f = ggml_add(ctx0, f, ggml_mul_mat(ctx0, model.depth_mem_route_retr, summary)); + } + if (diff) { + f = ggml_add(ctx0, f, ggml_mul_mat(ctx0, model.depth_mem_route_diff, diff)); + } + if (history) { + f = ggml_add(ctx0, f, ggml_mul_mat(ctx0, model.depth_mem_route_hist, history)); + } + return ggml_tanh(ctx0, ggml_add(ctx0, f, model.depth_mem_route_b)); +} + +llama_model_metis16::graph::graph(const llm_graph_params & params) : + llm_build_mamba_base(params) { + const auto & model = static_cast(this->model); + + const int64_t n_s = model.n_stream; + const int n_p = model.n_pass; + const int n_phys = model.n_layer_phys; + const int64_t n_r = model.n_route; + const int64_t n_hd = hparams.n_embd_head_k(); + const int64_t n_exp = hparams.n_expert; + const int64_t n_used = hparams.n_expert_used(); + + ggml_tensor * inpL = build_inp_embd(model.tok_embd); + auto * inp = build_inp_mem_hybrid(); + ggml_tensor * inp_out_ids = build_inp_out_ids(); + + ggml_tensor * x = ggml_repeat_4d(ctx0, ggml_reshape_3d(ctx0, inpL, 1, n_embd, n_tokens), n_s, n_embd, n_tokens, 1); + x = ggml_add(ctx0, x, model.stream_embd); + cb(x, "streams_in", -1); + + ggml_tensor * prev_state = stream_mean(x); + ggml_tensor * history = nullptr; // routing history, [n_route, n_tokens] + ggml_tensor * cont = nullptr; // depth head reading of the last pass, [1, n_tokens] + int n_entry = 0; + + for (int p = 0; p < n_p; ++p) { + ggml_tensor * summ = nullptr; + ggml_tensor * ent = nullptr; + ggml_tensor * conf = nullptr; + + for (int il_p = 0; il_p < n_phys; ++il_p) { + const int il = p*n_phys + il_p; + const auto & layer = model.layers[il]; + const bool is_attn = !hparams.is_recr(il); + const bool writes = is_attn || il_p == n_phys - 1; + + ggml_tensor * retrieved = bank_k ? mem_read(model, &x) : nullptr; + ggml_tensor * feats = route_features(model, stream_mean(x), retrieved, nullptr, history); + + ggml_tensor * mixed = nullptr; + ggml_tensor * cur = hc_read(x, layer.hc_attn_mix, layer.hc_attn_read, layer.attn_norm, p, &mixed, il); + if (is_attn) { + ggml_tensor * qkv = build_lora_mm(layer.wqkv, cur); + ggml_tensor * lora = ggml_mul_mat(ctx0, layer.attn_pass_lora_up, ggml_mul_mat(ctx0, layer.attn_pass_lora_down, cur)); + qkv = ggml_add(ctx0, qkv, ggml_mul(ctx0, lora, ggml_view_1d(ctx0, layer.attn_pass_gate, 1, p*layer.attn_pass_gate->nb[0]))); + + const int64_t n_head = hparams.n_head(il); + const int64_t n_head_kv = hparams.n_head_kv(il); + const size_t es = ggml_element_size(qkv); + ggml_tensor * Q = ggml_cont(ctx0, ggml_view_3d(ctx0, qkv, n_hd, n_head, n_tokens, n_hd*es, qkv->nb[1], 0)); + ggml_tensor * K = ggml_cont(ctx0, ggml_view_3d(ctx0, qkv, n_hd, n_head_kv, n_tokens, n_hd*es, qkv->nb[1], n_hd*n_head*es)); + ggml_tensor * V = ggml_cont(ctx0, ggml_view_3d(ctx0, qkv, n_hd, n_head_kv, n_tokens, n_hd*es, qkv->nb[1], n_hd*(n_head + n_head_kv)*es)); + + cur = build_attn(inp->get_attn(), layer.wo, nullptr, nullptr, Q, K, V, nullptr, nullptr, nullptr, 1.0f/sqrtf(float(n_hd)), il); + } else { + cur = build_mamba2_layer(inp->get_recr(), cur, ubatch, il); + } + cb(cur, "mixer_out", il); + x = hc_write(mixed, layer.hc_attn_write, cur, p); + + cur = hc_read(x, layer.hc_ffn_mix, layer.hc_ffn_read, layer.ffn_norm, p, &mixed, il); + { + ggml_tensor * latent = ggml_mul_mat(ctx0, layer.ffn_latent_down, cur); + ggml_tensor * logits = ggml_mul_mat(ctx0, layer.ffn_gate_inp, ggml_concat(ctx0, latent, feats, 0)); + logits = ggml_add(ctx0, logits, layer.ffn_gate_inp_b); + + // the balance bias picks the experts; the weights come from the unbiased logits + ggml_tensor * selected = ggml_argsort_top_k(ctx0, ggml_add(ctx0, logits, layer.ffn_exp_probs_b), n_used); + + ggml_tensor * moe_out = build_moe_ffn(latent, nullptr, nullptr, nullptr, layer.ffn_down_exps, nullptr, + n_exp, n_used, LLM_FFN_SILU, false, 0.0f, LLAMA_EXPERT_GATING_FUNC_TYPE_SOFTMAX_WEIGHT, il_p, + logits, layer.ffn_gate_up_exps, nullptr, nullptr, nullptr, selected); + ggml_tensor * shexp = build_ffn(latent, + layer.ffn_up_shexp, NULL, NULL, + layer.ffn_gate_shexp, NULL, NULL, + layer.ffn_down_shexp, NULL, NULL, + NULL, LLM_FFN_SILU, LLM_FFN_PAR, il_p); + cur = ggml_mul_mat(ctx0, layer.ffn_latent_up, ggml_add(ctx0, moe_out, shexp)); + cb(cur, "ffn_out", il); + + // routing summary: the chosen experts' identity vectors blended by their weights + ggml_tensor * logits3 = ggml_reshape_3d(ctx0, logits, 1, n_exp, n_tokens); + ggml_tensor * weights = ggml_soft_max(ctx0, ggml_reshape_2d(ctx0, ggml_get_rows(ctx0, logits3, selected), n_used, n_tokens)); + ggml_tensor * rows = ggml_get_rows(ctx0, layer.ffn_exp_embd, ggml_reshape_1d(ctx0, ggml_cont(ctx0, selected), n_used*n_tokens)); + rows = ggml_cont(ctx0, ggml_permute(ctx0, ggml_reshape_3d(ctx0, rows, n_r, n_used, n_tokens), 1, 0, 2, 3)); + summ = ggml_reshape_2d(ctx0, ggml_mul_mat(ctx0, rows, ggml_reshape_3d(ctx0, weights, n_used, 1, n_tokens)), n_r, n_tokens); + + if (writes) { + ggml_tensor * probs = ggml_soft_max(ctx0, logits); + ent = ggml_sum_rows(ctx0, ggml_mul(ctx0, probs, ggml_log(ctx0, ggml_clamp(ctx0, probs, 1.0e-9f, 1.0f)))); + ent = ggml_scale(ctx0, ent, -1.0f); + conf = ggml_get_rows(ctx0, ggml_reshape_3d(ctx0, probs, 1, n_exp, n_tokens), + ggml_reshape_2d(ctx0, ggml_argmax(ctx0, probs), 1, n_tokens)); + conf = ggml_reshape_2d(ctx0, conf, 1, n_tokens); + } + } + x = hc_write(mixed, layer.hc_ffn_write, cur, p); + cb(x, "streams", il); + + history = history ? ggml_add(ctx0, ggml_scale(ctx0, history, 0.5f), ggml_scale(ctx0, summ, 0.5f)) : ggml_scale(ctx0, summ, 0.5f); + + if (is_attn) { + mem_write(model, x, summ, ent, conf, cont, n_entry++); + } + } + + // end of the pass: one more memory read, the depth head, and an entry recording where the pass left the token + ggml_tensor * retrieved = mem_read(model, &x); + ggml_tensor * state = stream_mean(x); + ggml_tensor * diff = ggml_sub(ctx0, state, prev_state); + ggml_tensor * feats = route_features(model, state, retrieved, diff, history); + + ggml_tensor * h = ggml_mul_mat(ctx0, model.cont_hidden_state, state); + h = ggml_add(ctx0, h, ggml_mul_mat(ctx0, model.cont_hidden_retr, retrieved)); + h = ggml_add(ctx0, h, ggml_mul_mat(ctx0, model.cont_hidden_diff, diff)); + h = ggml_add(ctx0, h, ggml_mul_mat(ctx0, model.cont_hidden_route, feats)); + h = ggml_silu(ctx0, ggml_add(ctx0, h, model.cont_hidden_b)); + + ggml_tensor * logit = ggml_add(ctx0, ggml_mul_mat(ctx0, model.cont_output, h), model.cont_output_b); + ggml_tensor * scale = ggml_clamp(ctx0, ggml_sqrt(ctx0, ggml_mean(ctx0, ggml_sqr(ctx0, state))), 1.0e-6f, INFINITY); + ggml_tensor * innov = ggml_sqrt(ctx0, ggml_mean(ctx0, ggml_sqr(ctx0, diff))); + logit = ggml_add(ctx0, logit, ggml_scale_bias(ctx0, ggml_tanh(ctx0, ggml_div(ctx0, innov, scale)), 1.0f, -0.5f)); + cont = ggml_sigmoid(ctx0, ggml_clamp(ctx0, logit, -model.cont_limit, model.cont_limit)); + cb(cont, "continuation", p); + + mem_write(model, x, summ, ent, conf, cont, n_entry++); + prev_state = state; + } + + ggml_tensor * cur = stream_mean(x); + if (inp_out_ids) { + cur = ggml_get_rows(ctx0, cur, inp_out_ids); + } + + cur = build_norm(cur, model.output_norm, NULL, LLM_NORM_RMS, -1); + cb(cur, "result_norm", -1); + res->t_embd = cur; + + cur = build_lora_mm(model.output, cur); + cb(cur, "result_output", -1); + res->t_logits = cur; + +} diff --git a/src/models/mimo2.cpp b/src/models/mimo2.cpp index fdf5d00af19e..4fa97ff5b645 100644 --- a/src/models/mimo2.cpp +++ b/src/models/mimo2.cpp @@ -79,12 +79,12 @@ void llama_model_mimo2::load_arch_tensors(llama_model_loader & ml) { std::unique_ptr llama_model_mimo2::build_arch_graph(const llm_graph_params & params) const { if (params.gtype == LLM_GRAPH_TYPE_DECODER_MTP) { - return std::make_unique(*this, params); + return std::make_unique(params); } - return std::make_unique(*this, params); + return std::make_unique(params); } -llama_model_mimo2::graph::graph(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) { +llama_model_mimo2::graph::graph(const llm_graph_params & params) : llm_graph_context(params) { ggml_tensor * cur; ggml_tensor * inpL; @@ -104,8 +104,6 @@ llama_model_mimo2::graph::graph(const llama_model & model, const llm_graph_param uint32_t n_head_l = hparams.n_head(il); uint32_t n_head_kv_l = hparams.n_head_kv(il); - const float freq_base_l = model.get_rope_freq_base(cparams, il); - const float freq_scale_l = model.get_rope_freq_scale(cparams, il); cur = inpL; @@ -148,17 +146,8 @@ llama_model_mimo2::graph::graph(const llama_model & model, const llm_graph_param Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head_v, n_head_kv_l, n_tokens); } - Qcur = ggml_rope_ext( - ctx0, Qcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base_l, freq_scale_l, - ext_factor, attn_factor, beta_fast, beta_slow - ); - - Kcur = ggml_rope_ext( - ctx0, Kcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base_l, freq_scale_l, - ext_factor, attn_factor, beta_fast, beta_slow - ); + Qcur = build_rope(Qcur, inp_pos, il); + Kcur = build_rope(Kcur, inp_pos, il); cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); @@ -258,7 +247,7 @@ llama_model_mimo2::graph::graph(const llama_model & model, const llm_graph_param // Mirrors MiMo's appended NextN block: normalize and fuse token and hidden inputs, run the decoder block, // expose its pre-head-norm state to the next draft step, then apply the shared output norm and LM head. // Converted checkpoints may store that shared norm as layer_out_norm, so it remains in the fallback chain. -llama_model_mimo2::graph_mtp::graph_mtp(const llama_model & model, const llm_graph_params & params) +llama_model_mimo2::graph_mtp::graph_mtp(const llm_graph_params & params) : llm_graph_context(params) { GGML_ASSERT(hparams.n_layer_nextn > 0 && "MIMO2 MTP requires n_layer_nextn > 0"); @@ -276,8 +265,6 @@ llama_model_mimo2::graph_mtp::graph_mtp(const llama_model & model, const llm_gra const uint32_t n_head_l = hparams.n_head(il); const uint32_t n_head_kv_l = hparams.n_head_kv(il); - const float freq_base_l = model.get_rope_freq_base(cparams, il); - const float freq_scale_l = model.get_rope_freq_scale(cparams, il); const float v_scale = hparams.f_attn_value_scale; auto inp = std::make_unique(hparams.n_embd_inp(), hparams.n_embd); @@ -288,9 +275,7 @@ llama_model_mimo2::graph_mtp::graph_mtp(const llama_model & model, const llm_gra inp->embd = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, hparams.n_embd_inp(), n_tokens); ggml_set_input(inp->embd); - inp->h = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, hparams.n_embd, n_tokens); - ggml_set_input(inp->h); - ggml_set_name(inp->h, "mtp_h_input"); + inp->h = build_inp_embd_state(inp.get(), hparams.n_embd); ggml_tensor * tok_embd_w = layer.nextn.embed_tokens ? layer.nextn.embed_tokens : model.tok_embd; ggml_tensor * h_input = inp->h; @@ -333,15 +318,8 @@ llama_model_mimo2::graph_mtp::graph_mtp(const llama_model & model, const llm_gra ggml_tensor * Kcur = ggml_view_3d(ctx0, qkv, n_embd_head_k, n_head_kv_l, n_tokens, row_k, row_full, k_off); ggml_tensor * Vcur = ggml_view_3d(ctx0, qkv, n_embd_head_v, n_head_kv_l, n_tokens, row_v, row_full, v_off); - Qcur = ggml_rope_ext( - ctx0, Qcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base_l, freq_scale_l, - ext_factor, attn_factor, beta_fast, beta_slow); - - Kcur = ggml_rope_ext( - ctx0, Kcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base_l, freq_scale_l, - ext_factor, attn_factor, beta_fast, beta_slow); + Qcur = build_rope(Qcur, inp_pos, il); + Kcur = build_rope(Kcur, inp_pos, il); cb(Qcur, "mtp_Qcur", il); cb(Kcur, "mtp_Kcur", il); diff --git a/src/models/minicpm.cpp b/src/models/minicpm.cpp index fc3e5b171d52..56ab9249f71b 100644 --- a/src/models/minicpm.cpp +++ b/src/models/minicpm.cpp @@ -84,6 +84,6 @@ void llama_model_minicpm::load_arch_tensors(llama_model_loader &) { } std::unique_ptr llama_model_minicpm::build_arch_graph(const llm_graph_params & params) const { - return std::make_unique(*this, params); + return std::make_unique(params); } diff --git a/src/models/minicpm3.cpp b/src/models/minicpm3.cpp index 7136921dc392..67c8d6708557 100644 --- a/src/models/minicpm3.cpp +++ b/src/models/minicpm3.cpp @@ -57,10 +57,10 @@ void llama_model_minicpm3::load_arch_tensors(llama_model_loader &) { } std::unique_ptr llama_model_minicpm3::build_arch_graph(const llm_graph_params & params) const { - return std::make_unique(*this, params); + return std::make_unique(params); } -llama_model_minicpm3::graph::graph(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) { +llama_model_minicpm3::graph::graph(const llm_graph_params & params) : llm_graph_context(params) { //TODO: if the model varies, these parameters need to be read from the model const int64_t n_embd_base = 256; const float scale_embd = 12.0f; @@ -91,8 +91,6 @@ llama_model_minicpm3::graph::graph(const llama_model & model, const llm_graph_pa for (int il = 0; il < n_layer; ++il) { ggml_tensor * inpSA = inpL; - ggml_tensor * rope_factors = model.get_rope_factors(cparams, il); - // norm cur = build_norm(inpL, model.layers[il].attn_norm, NULL, @@ -162,20 +160,12 @@ llama_model_minicpm3::graph::graph(const llama_model & model, const llm_graph_pa v_states = ggml_cont(ctx0, v_states); cb(v_states, "v_states", il); - q = ggml_rope_ext( - ctx0, q, inp_pos, rope_factors, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); + q = build_rope(q, inp_pos, il); q = ggml_rope_set_offset(q, n_embd_head_qk_nope); cb(q, "q_rope", il); // shared RoPE key - k_pe = ggml_rope_ext( - ctx0, k_pe, inp_pos, rope_factors, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); + k_pe = build_rope(k_pe, inp_pos, il); cb(k_pe, "k_pe", il); ggml_tensor * q_states = q; diff --git a/src/models/minimax-01.cpp b/src/models/minimax-01.cpp index 27fe5291b069..08268e0f3e7d 100644 --- a/src/models/minimax-01.cpp +++ b/src/models/minimax-01.cpp @@ -61,7 +61,7 @@ void llama_model_minimax_01::load_arch_tensors(llama_model_loader &) { } std::unique_ptr llama_model_minimax_01::build_arch_graph(const llm_graph_params & params) const { - return std::make_unique(*this, params); + return std::make_unique(params); } class llm_graph_input_la : public llm_graph_input_i { @@ -189,7 +189,7 @@ class llm_graph_input_la : public llm_graph_input_i { ggml_tensor * inp_diag_decay = nullptr; // F32 [n_batch, n_batch, n_head] }; -llama_model_minimax_01::graph::graph(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) { +llama_model_minimax_01::graph::graph(const llm_graph_params & params) : llm_graph_context(params) { const int64_t n_embd_head = hparams.n_embd_head_v(); GGML_ASSERT(n_embd_head == hparams.n_embd_head_k()); @@ -255,17 +255,8 @@ llama_model_minimax_01::graph::graph(const llama_model & model, const llm_graph_ auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur, n_embd_head, n_head, n_head_kv, il); - Qcur = ggml_rope_ext( - ctx0, Qcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); - - Kcur = ggml_rope_ext( - ctx0, Kcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); + Qcur = build_rope(Qcur, inp_pos, il); + Kcur = build_rope(Kcur, inp_pos, il); cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); diff --git a/src/models/minimax-m2.cpp b/src/models/minimax-m2.cpp index 8b80eab2da19..5b8258255f09 100644 --- a/src/models/minimax-m2.cpp +++ b/src/models/minimax-m2.cpp @@ -41,10 +41,10 @@ void llama_model_minimax_m2::load_arch_tensors(llama_model_loader &) { } std::unique_ptr llama_model_minimax_m2::build_arch_graph(const llm_graph_params & params) const { - return std::make_unique(*this, params); + return std::make_unique(params); } -llama_model_minimax_m2::graph::graph(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) { +llama_model_minimax_m2::graph::graph(const llm_graph_params & params) : llm_graph_context(params) { const int64_t n_embd_head = hparams.n_embd_head_v(); GGML_ASSERT(n_embd_head == hparams.n_embd_head_k()); @@ -92,17 +92,8 @@ llama_model_minimax_m2::graph::graph(const llama_model & model, const llm_graph_ Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens); Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens); - Qcur = ggml_rope_ext( - ctx0, Qcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); - - Kcur = ggml_rope_ext( - ctx0, Kcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); + Qcur = build_rope(Qcur, inp_pos, il); + Kcur = build_rope(Kcur, inp_pos, il); cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); diff --git a/src/models/minimax-m3.cpp b/src/models/minimax-m3.cpp index 0da9d5a44aa8..466985cc430a 100644 --- a/src/models/minimax-m3.cpp +++ b/src/models/minimax-m3.cpp @@ -90,7 +90,7 @@ void llama_model_minimax_m3::load_arch_tensors(llama_model_loader &) { } std::unique_ptr llama_model_minimax_m3::build_arch_graph(const llm_graph_params & params) const { - return std::make_unique(*this, params); + return std::make_unique(params); } class llm_graph_input_msa : public llm_graph_input_i { @@ -207,7 +207,7 @@ ggml_tensor * llama_model_minimax_m3::graph::build_attn_msa_fa( return ggml_reshape_2d(ctx0, o, D*HQ, T); } -llama_model_minimax_m3::graph::graph(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) { +llama_model_minimax_m3::graph::graph(const llm_graph_params & params) : llm_graph_context(params) { const int64_t n_embd_head = hparams.n_embd_head_v(); const auto & mm = static_cast(model); @@ -262,7 +262,7 @@ llama_model_minimax_m3::graph::graph(const llama_model & model, const llm_graph_ if (msa_enabled) { const auto * mctx_msa = static_cast(mctx); - msa_kqm = inp_attn->self_attn_mask.cnv; + msa_kqm = inp_attn->self_attn_mask.raw; n_kv = msa_kqm->ne[0]; n_tps = msa_kqm->ne[1]; // tokens per stream ns = msa_kqm->ne[3]; // streams in this ubatch @@ -318,14 +318,8 @@ llama_model_minimax_m3::graph::graph(const llama_model & model, const llm_graph_ cb(Kcur, "Kcur_normed", il); // partial rotary: only the first n_rot dims are rotated - Qcur = ggml_rope_ext( - ctx0, Qcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); - Kcur = ggml_rope_ext( - ctx0, Kcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); + Qcur = build_rope(Qcur, inp_pos, il); + Kcur = build_rope(Kcur, inp_pos, il); cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); @@ -347,10 +341,8 @@ llama_model_minimax_m3::graph::graph(const llama_model & model, const llm_graph_ ik = ggml_reshape_3d(ctx0, ik, n_idx_dim, 1, n_tokens); iq = build_norm(iq, model.layers[il].index_q_norm, NULL, LLM_NORM_RMS, il); // +1 baked ik = build_norm(ik, model.layers[il].index_k_norm, NULL, LLM_NORM_RMS, il); - iq = ggml_rope_ext(ctx0, iq, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, - freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow); - ik = ggml_rope_ext(ctx0, ik, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, - freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow); + iq = build_rope(iq, inp_pos, il); + ik = build_rope(ik, inp_pos, il); const auto * mctx_msa_l = static_cast(mctx); const auto * mctx_cur = mctx_msa_l->get_base(); @@ -394,6 +386,7 @@ llama_model_minimax_m3::graph::graph(const llama_model & model, const llm_graph_ ggml_tensor * iq4 = ggml_reshape_4d(ctx0, iq, n_idx_dim, Hd, 1, ns); ggml_tensor * sc = ggml_mul_mat(ctx0, ggml_reshape_4d(ctx0, ikp, n_idx_dim, n_ps, 1, ns), iq4); + // the reference computes the indexer scores in F32: https://github.com/huggingface/transformers/blob/0f53828e83f2212060233d2ea552cc76617b8712/src/transformers/models/minimax_m3_vl/modeling_minimax_m3_vl.py#L562 ggml_prec_set_acc(sc, GGML_PREC_F32); // unmapped positions come out -inf, so they can never rank into the top-k sc = ggml_add_inplace(ctx0, sc, @@ -475,7 +468,7 @@ llama_model_minimax_m3::graph::graph(const llama_model & model, const llm_graph_ ggml_tensor * ikp = ggml_get_rows(ctx0, ik_s, psl_s); // [n_idx_dim, n_ps] ggml_tensor * sc = ggml_mul_mat(ctx0, ikp, ggml_reshape_2d(ctx0, iq_s, n_idx_dim, Hd*n_tps)); - // indexer scores run in F32 + // the reference computes the indexer scores in F32: https://github.com/huggingface/transformers/blob/0f53828e83f2212060233d2ea552cc76617b8712/src/transformers/models/minimax_m3_vl/modeling_minimax_m3_vl.py#L562 ggml_prec_set_acc(sc, GGML_PREC_F32); sc = ggml_reshape_3d(ctx0, sc, n_ps, Hd, n_tps); // unmapped positions (holes, padding, empty cells) come out -inf diff --git a/src/models/mistral-large4.cpp b/src/models/mistral-large4.cpp index dbd1907a4b61..a4315a9919e3 100644 --- a/src/models/mistral-large4.cpp +++ b/src/models/mistral-large4.cpp @@ -148,11 +148,20 @@ void llama_model_mistral_large4::load_arch_tensors(llama_model_loader &) { } } +llama_rope_params llama_model_mistral_large4::get_rope_params_indexer(const llama_cparams & cparams, int il) const { + auto res = llama_model::get_rope_params_indexer(cparams, il); + + res.type = hparams.rope_type; + res.factors = get_rope_factors(cparams, il); + + return res; +} + std::unique_ptr llama_model_mistral_large4::build_arch_graph(const llm_graph_params & params) const { - return std::make_unique(*this, params); + return std::make_unique(params); } -llama_model_mistral_large4::graph::graph(const llama_model & model, const llm_graph_params & params) : +llama_model_mistral_large4::graph::graph(const llm_graph_params & params) : llm_graph_context(params) { GGML_ASSERT(hparams.is_mla()); @@ -216,8 +225,6 @@ llama_model_mistral_large4::graph::graph(const llama_model & model, const llm_gr const int64_t n_rot_l = use_rope ? n_rot : 0; const int64_t n_embd_head_qk_nope = n_embd_head_k - n_rot_l; - ggml_tensor * rope_factors = model.get_rope_factors(cparams, il); - ggml_tensor * inpSA = inpL; cur = build_norm(inpL, layer.attn_norm, NULL, LLM_NORM_RMS, il); @@ -250,14 +257,10 @@ llama_model_mistral_large4::graph::graph(const llama_model & model, const llm_gr indexer_k = ggml_reshape_3d(ctx0, indexer_k, n_embd_indexer_head, 1, n_tokens); if (use_rope) { - indexer_q = ggml_rope_ext(ctx0, indexer_q, inp_pos, rope_factors, n_rot, - rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); + indexer_q = build_rope_indexer(indexer_q, inp_pos, il); cb(indexer_q, "indexer_q", il); - indexer_k = ggml_rope_ext(ctx0, indexer_k, inp_pos, rope_factors, n_rot, - rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); + indexer_k = build_rope_indexer(indexer_k, inp_pos, il); cb(indexer_k, "indexer_k", il); } @@ -289,7 +292,7 @@ llama_model_mistral_large4::graph::graph(const llama_model & model, const llm_gr ggml_tensor * indexer_score = nullptr; if (cparams.fused_lid) { - indexer_score = ggml_lightning_indexer(ctx0, indexer_q, indexer_k, indexer_weights, inp_attn_dsa->self_attn_mask_lid.cnv); + indexer_score = ggml_lightning_indexer(ctx0, indexer_q, indexer_k, indexer_weights, inp_attn_dsa->self_attn_mask_lid.raw); cb(indexer_score, "indexer_score", il); res->add_fused_node({LLM_FUSED_OP_LIGHTNING_INDEXER, indexer_score, il}); } else { @@ -323,7 +326,7 @@ llama_model_mistral_large4::graph::graph(const llama_model & model, const llm_gr cb(indexer_score, "indexer_score", il); // mask indexer scores - ggml_tensor * indexer_kq_mask = inp_attn_dsa->self_attn_mask_lid.cnv; + ggml_tensor * indexer_kq_mask = inp_attn_dsa->self_attn_mask_lid.raw; indexer_score = ggml_add(ctx0, indexer_score, indexer_kq_mask); cb(indexer_score, "indexer_score", il); } @@ -368,8 +371,7 @@ llama_model_mistral_large4::graph::graph(const llama_model & model, const llm_gr ggml_row_size(q->type, n_embd_head_k), ggml_row_size(q->type, n_embd_head_k) * n_head, ggml_row_size(q->type, n_embd_head_qk_nope)); - q_pe = ggml_rope_ext(ctx0, q_pe, inp_pos, rope_factors, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); + q_pe = build_rope(q_pe, inp_pos, il); cb(q_pe, "q_pe", il); Qcur = ggml_concat(ctx0, q_nope_absorbed, q_pe, 0); @@ -406,8 +408,7 @@ llama_model_mistral_large4::graph::graph(const llama_model & model, const llm_gr ggml_row_size(kv_cmpr_pe->type, kv_lora_rank + n_rot), ggml_row_size(kv_cmpr_pe->type, kv_lora_rank + n_rot), ggml_row_size(kv_cmpr_pe->type, kv_lora_rank)); - k_pe = ggml_rope_ext(ctx0, k_pe, inp_pos, rope_factors, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); + k_pe = build_rope(k_pe, inp_pos, il); cb(k_pe, "k_pe", il); Kcur = ggml_concat(ctx0, kv_cmpr, k_pe, 0); diff --git a/src/models/mistral3.cpp b/src/models/mistral3.cpp index af058d558d93..ab81218635da 100644 --- a/src/models/mistral3.cpp +++ b/src/models/mistral3.cpp @@ -124,10 +124,10 @@ class llama_model_mistral3::input_t_cond : public llm_graph_input_i { }; std::unique_ptr llama_model_mistral3::build_arch_graph(const llm_graph_params & params) const { - return std::make_unique(*this, params); + return std::make_unique(params); } -llama_model_mistral3::graph::graph(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) { +llama_model_mistral3::graph::graph(const llm_graph_params & params) : llm_graph_context(params) { const int64_t n_embd_head = hparams.n_embd_head_v(); GGML_ASSERT(n_embd_head == hparams.n_embd_head_k()); @@ -175,24 +175,12 @@ llama_model_mistral3::graph::graph(const llama_model & model, const llm_graph_pa // self-attention { - // rope freq factors for llama3; may return nullptr for llama2 and other models - ggml_tensor * rope_factors = model.get_rope_factors(cparams, il); - // compute Q and K and RoPE them auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur, n_embd_head, n_head, n_head_kv, il); - Qcur = ggml_rope_ext( - ctx0, Qcur, inp_pos, rope_factors, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); - - Kcur = ggml_rope_ext( - ctx0, Kcur, inp_pos, rope_factors, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); + Qcur = build_rope(Qcur, inp_pos, il); + Kcur = build_rope(Kcur, inp_pos, il); cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); diff --git a/src/models/mistral4.cpp b/src/models/mistral4.cpp index 3d9190650e37..a25041b441f1 100644 --- a/src/models/mistral4.cpp +++ b/src/models/mistral4.cpp @@ -1,6 +1,6 @@ #include "models.h" std::unique_ptr llama_model_mistral4::build_arch_graph(const llm_graph_params & params) const { - return std::make_unique(*this, params); + return std::make_unique(params); } diff --git a/src/models/models.h b/src/models/models.h index 1fc7cae1dff0..cfef363b6a10 100644 --- a/src/models/models.h +++ b/src/models/models.h @@ -26,8 +26,8 @@ struct llm_build_mamba_base : public llm_graph_context { virtual ~llm_build_mamba_base() = default; - ggml_tensor * build_mamba_layer(llm_graph_input_rs * inp, ggml_tensor * cur, const llama_model & model, const llama_ubatch & ubatch, int il); - ggml_tensor * build_mamba2_layer(llm_graph_input_rs * inp, ggml_tensor * cur, const llama_model & model, const llama_ubatch & ubatch, int il) const; + ggml_tensor * build_mamba_layer(llm_graph_input_rs * inp, ggml_tensor * cur, const llama_ubatch & ubatch, int il); + ggml_tensor * build_mamba2_layer(llm_graph_input_rs * inp, ggml_tensor * cur, const llama_ubatch & ubatch, int il) const; }; @@ -37,44 +37,13 @@ struct llm_build_delta_net_base : public llm_graph_context { virtual ~llm_build_delta_net_base() = default; // returns pair of output and new state - std::pair build_delta_net_chunking( - ggml_tensor * q, - ggml_tensor * k, - ggml_tensor * v, - ggml_tensor * g, - ggml_tensor * b, - ggml_tensor * s, - int il); - - // returns pair of output and new state - std::pair build_delta_net_autoregressive( - ggml_tensor * q, - ggml_tensor * k, - ggml_tensor * v, - ggml_tensor * g, - ggml_tensor * b, - ggml_tensor * s, - int il); - - // use the ggml_gated_delta_net fused operator (K=1; state has shape [S_v, S_v, H_v, n_seqs]) - std::pair build_delta_net_fused( - ggml_tensor * q, - ggml_tensor * k, - ggml_tensor * v, - ggml_tensor * g, - ggml_tensor * b, - ggml_tensor * s, - int il); - - // choose one of two implementations above based on the number of tokens std::pair build_delta_net( ggml_tensor * q, ggml_tensor * k, ggml_tensor * v, ggml_tensor * g, ggml_tensor * b, - ggml_tensor * s, - int il); + ggml_tensor * s); // read conv state from cache, concat with qkv_mixed, write back (single slot or per-token) // qkv_mixed: (qkv_dim, n_seq_tokens, n_seqs); returns conv_input: (kernel_size + n_seq_tokens - 1, channels, n_seqs) @@ -105,30 +74,24 @@ struct llm_build_lfm_base : public llm_graph_context { virtual ~llm_build_lfm_base() = default; - ggml_tensor * build_lfm_shortconv(const llama_model & model, - const llama_memory_recurrent_context * mctx_recr, + ggml_tensor * build_lfm_shortconv(const llama_memory_recurrent_context * mctx_recr, llm_graph_input_rs * inp_recr, ggml_tensor * cur, int il); // no recurrent state (decision graphs): causal over the ubatch from a zero state - ggml_tensor * build_lfm_shortconv(const llama_model & model, - ggml_tensor * cur, + ggml_tensor * build_lfm_shortconv(ggml_tensor * cur, int il); - ggml_tensor * build_lfm_dense_ffn(const llama_model & model, - ggml_tensor * cur, + ggml_tensor * build_lfm_dense_ffn(ggml_tensor * cur, int il); - ggml_tensor * build_lfm_moe_ffn(const llama_model & model, - ggml_tensor * cur, + ggml_tensor * build_lfm_moe_ffn(ggml_tensor * cur, int il); }; struct llm_build_rwkv6_base : public llm_graph_context { - const llama_model & model; - - llm_build_rwkv6_base(const llama_model & model, const llm_graph_params & params); + llm_build_rwkv6_base(const llm_graph_params & params); virtual ~llm_build_rwkv6_base() = default; @@ -146,9 +109,7 @@ struct llm_build_rwkv6_base : public llm_graph_context { // Base class for RWKV7-related models struct llm_build_rwkv7_base : public llm_graph_context { - const llama_model & model; - - llm_build_rwkv7_base(const llama_model & model, const llm_graph_params & params); + llm_build_rwkv7_base(const llm_graph_params & params); virtual ~llm_build_rwkv7_base() = default; @@ -176,13 +137,12 @@ struct llama_model_llama : public llama_model_base { template struct graph : public llm_graph_context { - graph(const llama_model & model, const llm_graph_params & params); + graph(const llm_graph_params & params); }; std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_llama4 : public llama_model_base { llama_model_llama4(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; @@ -190,13 +150,12 @@ struct llama_model_llama4 : public llama_model_base { template struct graph : public llm_graph_context { - graph(const llama_model & model, const llm_graph_params & params); + graph(const llm_graph_params & params); }; std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_llama_embed : public llama_model_llama { llama_model_llama_embed(const struct llama_model_params & params) : llama_model_llama(params) {} // reuse load_arch_hparams and load_arch_tensors from llama_model_llama @@ -207,137 +166,127 @@ struct llama_model_llama_embed : public llama_model_llama { std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_maincoder : public llama_model_base { llama_model_maincoder(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; void load_arch_tensors(llama_model_loader & ml) override; struct graph : public llm_graph_context { - graph(const llama_model & model, const llm_graph_params & params); + graph(const llm_graph_params & params); }; + llama_rope_params get_rope_params(const llama_cparams & cparams, int il) const override; std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_maion_coder : public llama_model_base { llama_model_maion_coder(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; void load_arch_tensors(llama_model_loader & ml) override; struct graph : public llm_graph_context { - graph(const llama_model & model, const llm_graph_params & params); + graph(const llm_graph_params & params); }; std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_talkie : public llama_model_base { llama_model_talkie(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; void load_arch_tensors(llama_model_loader & ml) override; struct graph : public llm_graph_context { - graph(const llama_model & model, const llm_graph_params & params); + graph(const llm_graph_params & params); }; std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_deci : public llama_model_base { llama_model_deci(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; void load_arch_tensors(llama_model_loader & ml) override; struct graph : public llm_graph_context { - graph(const llama_model & model, const llm_graph_params & params); + graph(const llm_graph_params & params); }; std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_baichuan : public llama_model_base { llama_model_baichuan(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; void load_arch_tensors(llama_model_loader & ml) override; struct graph : public llm_graph_context { - graph(const llama_model & model, const llm_graph_params & params); + graph(const llm_graph_params & params); }; std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_falcon : public llama_model_base { llama_model_falcon(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; void load_arch_tensors(llama_model_loader & ml) override; struct graph : public llm_graph_context { - graph(const llama_model & model, const llm_graph_params & params); + graph(const llm_graph_params & params); }; std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_grok : public llama_model_base { llama_model_grok(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; void load_arch_tensors(llama_model_loader & ml) override; struct graph : public llm_graph_context { - graph(const llama_model & model, const llm_graph_params & params); + graph(const llm_graph_params & params); }; std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_starcoder : public llama_model_base { llama_model_starcoder(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; void load_arch_tensors(llama_model_loader & ml) override; struct graph : public llm_graph_context { - graph(const llama_model & model, const llm_graph_params & params); + graph(const llm_graph_params & params); }; std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_refact : public llama_model_base { llama_model_refact(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; void load_arch_tensors(llama_model_loader & ml) override; struct graph : public llm_graph_context { - graph(const llama_model & model, const llm_graph_params & params); + graph(const llm_graph_params & params); }; std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_bert : public llama_model_base { llama_model_bert(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; void load_arch_tensors(llama_model_loader & ml) override; struct graph : public llm_graph_context { - graph(const llama_model & model, const llm_graph_params & params); + graph(const llm_graph_params & params); }; std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_jina_bert_v2 : public llama_model_base { llama_model_jina_bert_v2(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; @@ -348,7 +297,6 @@ struct llama_model_jina_bert_v2 : public llama_model_base { std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_jina_bert_v3 : public llama_model_base { llama_model_jina_bert_v3(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; @@ -359,7 +307,6 @@ struct llama_model_jina_bert_v3 : public llama_model_base { std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_nomic_bert : public llama_model_base { llama_model_nomic_bert(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; @@ -370,7 +317,6 @@ struct llama_model_nomic_bert : public llama_model_base { std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_nomic_bert_moe : public llama_model_base { llama_model_nomic_bert_moe(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; @@ -381,17 +327,15 @@ struct llama_model_nomic_bert_moe : public llama_model_base { std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_modern_bert : public llama_model_base { llama_model_modern_bert(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; void load_arch_tensors(llama_model_loader & ml) override; struct graph : public llm_graph_context { - graph(const llama_model & model, const llm_graph_params & params); + graph(const llm_graph_params & params); ggml_tensor * build_decision_head( - const llama_model & model, ggml_tensor * inp, llm_graph_input_attn_no_cache * inp_attn, ggml_tensor * inp_out_ids); @@ -400,46 +344,42 @@ struct llama_model_modern_bert : public llama_model_base { std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_neo_bert : public llama_model_base { llama_model_neo_bert(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; void load_arch_tensors(llama_model_loader & ml) override; struct graph : public llm_graph_context { - graph(const llama_model & model, const llm_graph_params & params); + graph(const llm_graph_params & params); }; std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_eurobert : public llama_model_base { llama_model_eurobert(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; void load_arch_tensors(llama_model_loader & ml) override; struct graph : public llm_graph_context { - graph(const llama_model & model, const llm_graph_params & params); + graph(const llm_graph_params & params); }; std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_bloom : public llama_model_base { llama_model_bloom(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; void load_arch_tensors(llama_model_loader & ml) override; struct graph : public llm_graph_context { - graph(const llama_model & model, const llm_graph_params & params); + graph(const llm_graph_params & params); }; std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - // Quant-only stub for mmproj GGUFs // none of these are ever called, they only exist to satisfy the llama_model_base interface struct llama_model_clip : public llama_model_base { @@ -455,27 +395,25 @@ struct llama_model_clip : public llama_model_base { std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_mpt : public llama_model_base { llama_model_mpt(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; void load_arch_tensors(llama_model_loader & ml) override; struct graph : public llm_graph_context { - graph(const llama_model & model, const llm_graph_params & params); + graph(const llm_graph_params & params); }; std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_stablelm : public llama_model_base { llama_model_stablelm(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; void load_arch_tensors(llama_model_loader & ml) override; struct graph : public llm_graph_context { - graph(const llama_model & model, const llm_graph_params & params); + graph(const llm_graph_params & params); }; std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; @@ -488,14 +426,15 @@ struct llama_model_mellum : public llama_model_base { template struct graph : public llm_graph_context { - graph(const llama_model & model, const llm_graph_params & params); + graph(const llm_graph_params & params); }; template struct graph_mtp : public llm_graph_context { - graph_mtp(const llama_model & model, const llm_graph_params & params); + graph_mtp(const llm_graph_params & params); }; + llama_rope_params get_rope_params(const llama_cparams & cparams, int il) const override; std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; @@ -509,7 +448,7 @@ struct llama_model_nanbeige : public llama_model_base { bool skip_loop_final_norm = false; struct graph : public llm_graph_context { - graph(const llama_model & model, const llm_graph_params & params); + graph(const llm_graph_params & params); }; std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; @@ -521,104 +460,97 @@ struct llama_model_qwen : public llama_model_base { void load_arch_tensors(llama_model_loader & ml) override; struct graph : public llm_graph_context { - graph(const llama_model & model, const llm_graph_params & params); + graph(const llm_graph_params & params); }; std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_qwen2 : public llama_model_base { llama_model_qwen2(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; void load_arch_tensors(llama_model_loader & ml) override; struct graph : public llm_graph_context { - graph(const llama_model & model, const llm_graph_params & params); + graph(const llm_graph_params & params); }; std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_dream : public llama_model_base { llama_model_dream(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; void load_arch_tensors(llama_model_loader & ml) override; struct graph : public llm_graph_context { - graph(const llama_model & model, const llm_graph_params & params); + graph(const llm_graph_params & params); }; std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_llada : public llama_model_base { llama_model_llada(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; void load_arch_tensors(llama_model_loader & ml) override; struct graph : public llm_graph_context { - graph(const llama_model & model, const llm_graph_params & params); + graph(const llm_graph_params & params); }; + llama_rope_params get_rope_params(const llama_cparams & cparams, int il) const override; std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_llada_moe : public llama_model_base { llama_model_llada_moe(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; void load_arch_tensors(llama_model_loader & ml) override; struct graph : public llm_graph_context { - graph(const llama_model & model, const llm_graph_params & params); + graph(const llm_graph_params & params); }; std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_rnd1 : public llama_model_base { llama_model_rnd1(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; void load_arch_tensors(llama_model_loader & ml) override; struct graph : public llm_graph_context { - graph(const llama_model & model, const llm_graph_params & params); + graph(const llm_graph_params & params); }; std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_qwen2vl : public llama_model_base { llama_model_qwen2vl(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; void load_arch_tensors(llama_model_loader & ml) override; struct graph : public llm_graph_context { - graph(const llama_model & model, const llm_graph_params & params); + graph(const llm_graph_params & params); }; std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_qwen2moe : public llama_model_base { llama_model_qwen2moe(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; void load_arch_tensors(llama_model_loader & ml) override; struct graph : public llm_graph_context { - graph(const llama_model & model, const llm_graph_params & params); + graph(const llm_graph_params & params); }; std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_sensenova_u1 : public llama_model_base { llama_model_sensenova_u1(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; @@ -627,27 +559,26 @@ struct llama_model_sensenova_u1 : public llama_model_base { float rope_freq_base_spatial = 10000.0f; struct graph : public llm_graph_context { - graph(const llama_model_sensenova_u1 & model, const llm_graph_params & params); + graph(const llm_graph_params & params); }; std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_qwen3 : public llama_model_base { llama_model_qwen3(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; void load_arch_tensors(llama_model_loader & ml) override; struct graph : public llm_graph_context { - graph(const llama_model & model, const llm_graph_params & params); + graph(const llm_graph_params & params); protected: // the helpers alone, a derived graph builds its own body struct no_build {}; graph(const llm_graph_params & params, no_build) : llm_graph_context(params) {} // the layers and the final norm: [n_embd, n_outputs] - ggml_tensor * build_backbone(const llama_model & model); + ggml_tensor * build_backbone(); }; std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; @@ -673,7 +604,7 @@ struct llama_model_vllm_decision2 : public llama_model_qwen3 { class input_decision; struct graph : public llama_model_qwen3::graph { - graph(const llama_model & model, const llm_graph_params & params); + graph(const llm_graph_params & params); private: // hidden: [n_embd, n_outputs], returns one score per option: [1, n_options] ggml_tensor * build_head(ggml_tensor * hidden, input_decision * inp); @@ -684,77 +615,71 @@ struct llama_model_vllm_decision2 : public llama_model_qwen3 { std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_qwen3moe : public llama_model_base { llama_model_qwen3moe(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; void load_arch_tensors(llama_model_loader & ml) override; struct graph : public llm_graph_context { - graph(const llama_model & model, const llm_graph_params & params); + graph(const llm_graph_params & params); }; std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_flint : public llama_model_base { llama_model_flint(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; void load_arch_tensors(llama_model_loader & ml) override; struct graph : public llm_graph_context { - graph(const llama_model & model, const llm_graph_params & params); + graph(const llm_graph_params & params); }; + llama_rope_params get_rope_params(const llama_cparams & cparams, int il) const override; std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_qwen3vl : public llama_model_base { llama_model_qwen3vl(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; void load_arch_tensors(llama_model_loader & ml) override; struct graph : public llm_graph_context { - graph(const llama_model & model, const llm_graph_params & params); + graph(const llm_graph_params & params); }; std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_qwen3vlmoe : public llama_model_base { llama_model_qwen3vlmoe(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; void load_arch_tensors(llama_model_loader & ml) override; struct graph : public llm_graph_context { - graph(const llama_model & model, const llm_graph_params & params); + graph(const llm_graph_params & params); }; std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_qwen3tts : public llama_model_qwen3vl { llama_model_qwen3tts(const struct llama_model_params & params) : llama_model_qwen3vl(params) {} }; - struct llama_model_phi2 : public llama_model_base { llama_model_phi2(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; void load_arch_tensors(llama_model_loader & ml) override; struct graph : public llm_graph_context { - graph(const llama_model & model, const llm_graph_params & params); + graph(const llm_graph_params & params); }; std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_phi3 : public llama_model_base { llama_model_phi3(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; @@ -762,13 +687,12 @@ struct llama_model_phi3 : public llama_model_base { template struct graph : public llm_graph_context { - graph(const llama_model & model, const llm_graph_params & params); + graph(const llm_graph_params & params); }; std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_phimoe : public llama_model_base { llama_model_phimoe(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; @@ -780,37 +704,34 @@ struct llama_model_phimoe : public llama_model_base { std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_plamo : public llama_model_base { llama_model_plamo(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; void load_arch_tensors(llama_model_loader & ml) override; struct graph : public llm_graph_context { - graph(const llama_model & model, const llm_graph_params & params); + graph(const llm_graph_params & params); }; std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_plamo2 : public llama_model_base { llama_model_plamo2(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; void load_arch_tensors(llama_model_loader & ml) override; struct graph : public llm_build_mamba_base { - graph(const llama_model & model, const llm_graph_params & params); + graph(const llm_graph_params & params); private: - ggml_tensor * build_plamo2_mamba_layer(llm_graph_input_rs * inp, ggml_tensor * cur, const llama_model & model, const llama_ubatch & ubatch, int il); + ggml_tensor * build_plamo2_mamba_layer(llm_graph_input_rs * inp, ggml_tensor * cur, const llama_ubatch & ubatch, int il); ggml_tensor * build_plamo2_attn_layer(llm_graph_input_attn_kv * inp, ggml_tensor * inp_pos, ggml_tensor * cur, - const llama_model & model, int il); + int il); }; std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_plamo3 : public llama_model_base { llama_model_plamo3(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; @@ -818,117 +739,108 @@ struct llama_model_plamo3 : public llama_model_base { template struct graph : public llm_graph_context { - graph(const llama_model & model, const llm_graph_params & params); + graph(const llm_graph_params & params); }; std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_gpt2 : public llama_model_base { llama_model_gpt2(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; void load_arch_tensors(llama_model_loader & ml) override; struct graph : public llm_graph_context { - graph(const llama_model & model, const llm_graph_params & params); + graph(const llm_graph_params & params); }; std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_pockettts : public llama_model_base { llama_model_pockettts(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; void load_arch_tensors(llama_model_loader & ml) override; struct graph : public llm_graph_context { - graph(const llama_model & model, const llm_graph_params & params); + graph(const llm_graph_params & params); }; std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_codeshell : public llama_model_base { llama_model_codeshell(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; void load_arch_tensors(llama_model_loader & ml) override; struct graph : public llm_graph_context { - graph(const llama_model & model, const llm_graph_params & params); + graph(const llm_graph_params & params); }; std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_orion : public llama_model_base { llama_model_orion(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; void load_arch_tensors(llama_model_loader & ml) override; struct graph : public llm_graph_context { - graph(const llama_model & model, const llm_graph_params & params); + graph(const llm_graph_params & params); }; std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_internlm2 : public llama_model_base { llama_model_internlm2(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; void load_arch_tensors(llama_model_loader & ml) override; struct graph : public llm_graph_context { - graph(const llama_model & model, const llm_graph_params & params); + graph(const llm_graph_params & params); }; std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_minicpm3 : public llama_model_base { llama_model_minicpm3(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; void load_arch_tensors(llama_model_loader & ml) override; struct graph : public llm_graph_context { - graph(const llama_model & model, const llm_graph_params & params); + graph(const llm_graph_params & params); }; std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_gemma : public llama_model_base { llama_model_gemma(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; void load_arch_tensors(llama_model_loader & ml) override; struct graph : public llm_graph_context { - graph(const llama_model & model, const llm_graph_params & params); + graph(const llm_graph_params & params); }; std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_gemma2 : public llama_model_base { llama_model_gemma2(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; void load_arch_tensors(llama_model_loader & ml) override; struct graph : public llm_graph_context { - graph(const llama_model & model, const llm_graph_params & params); + graph(const llm_graph_params & params); }; std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_gemma3 : public llama_model_base { llama_model_gemma3(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; @@ -936,21 +848,18 @@ struct llama_model_gemma3 : public llama_model_base { template struct graph : public llm_graph_context { - graph(const llama_model & model, const llm_graph_params & params); + graph(const llm_graph_params & params); }; std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_gemma3n : public llama_model_base { llama_model_gemma3n(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; void load_arch_tensors(llama_model_loader & ml) override; struct graph : public llm_graph_context { - const llama_model & model; - const int64_t n_embd_head; const int64_t n_embd_altup; const int64_t n_altup; @@ -958,7 +867,7 @@ struct llama_model_gemma3n : public llama_model_base { const int n_layer_sparsity = 10; // number of layers using activation sparsity const float f_sparsity_std_mul = 1.6448533535003662f; // std_multiplier = normal_dist.icdf(0.95) - graph(const llama_model & model, const llm_graph_params & params); + graph(const llm_graph_params & params); ggml_tensor * calc_magnitude(ggml_tensor * x); // TODO: refactor in common "per-layer" functionality [TAG_PER_LAYER] @@ -975,18 +884,15 @@ struct llama_model_gemma3n : public llama_model_base { std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_gemma4 : public llama_model_base { llama_model_gemma4(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; void load_arch_tensors(llama_model_loader & ml) override; struct graph : public llm_graph_context { - const llama_model & model; - const int64_t n_embd_per_layer; - graph(const llama_model & model, const llm_graph_params & params); + graph(const llm_graph_params & params); // TODO: refactor in common "per-layer" functionality [TAG_PER_LAYER] ggml_tensor * build_inp_per_layer(); @@ -996,42 +902,37 @@ struct llama_model_gemma4 : public llama_model_base { std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_gemma4_assistant : public llama_model_base { llama_model_gemma4_assistant(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; void load_arch_tensors(llama_model_loader & ml) override; struct graph : public llm_graph_context { - graph(const llama_model & model, const llm_graph_params & params); + graph(const llm_graph_params & params); }; std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_gemma_embedding : public llama_model_base { llama_model_gemma_embedding(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; void load_arch_tensors(llama_model_loader & ml) override; struct graph : public llm_graph_context { - graph(const llama_model & model, const llm_graph_params & params); + graph(const llm_graph_params & params); }; std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_gemma_embedding2 : public llama_model_base { llama_model_gemma_embedding2(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; void load_arch_tensors(llama_model_loader & ml) override; struct graph : public llm_graph_context { - const llama_model & model; - - graph(const llama_model & model, const llm_graph_params & params); + graph(const llm_graph_params & params); ggml_tensor * build_inp_per_layer(ggml_tensor * inpL); }; @@ -1039,33 +940,30 @@ struct llama_model_gemma_embedding2 : public llama_model_base { std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_starcoder2 : public llama_model_base { llama_model_starcoder2(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; void load_arch_tensors(llama_model_loader & ml) override; struct graph : public llm_graph_context { - graph(const llama_model & model, const llm_graph_params & params); + graph(const llm_graph_params & params); }; std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_mamba : public llama_model_base { llama_model_mamba(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; void load_arch_tensors(llama_model_loader & ml) override; struct graph : public llm_build_mamba_base { - graph(const llama_model & model, const llm_graph_params & params); + graph(const llm_graph_params & params); }; std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_mamba2 : public llama_model_base { llama_model_mamba2(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; @@ -1076,115 +974,108 @@ struct llama_model_mamba2 : public llama_model_base { std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_maple : public llama_model_base { llama_model_maple(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; void load_arch_tensors(llama_model_loader & ml) override; struct graph : public llm_graph_context { - graph(const llama_model & model, const llm_graph_params & params); + graph(const llm_graph_params & params); }; std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_jamba : public llama_model_base { llama_model_jamba(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; void load_arch_tensors(llama_model_loader & ml) override; struct graph : public llm_build_mamba_base { - graph(const llama_model & model, const llm_graph_params & params); + graph(const llm_graph_params & params); }; std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_xverse : public llama_model_base { llama_model_xverse(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; void load_arch_tensors(llama_model_loader & ml) override; struct graph : public llm_graph_context { - graph(const llama_model & model, const llm_graph_params & params); + graph(const llm_graph_params & params); }; std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_command_r : public llama_model_base { llama_model_command_r(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; void load_arch_tensors(llama_model_loader & ml) override; struct graph : public llm_graph_context { - graph(const llama_model & model, const llm_graph_params & params); + graph(const llm_graph_params & params); }; std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_cohere2 : public llama_model_base { llama_model_cohere2(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; void load_arch_tensors(llama_model_loader & ml) override; struct graph : public llm_graph_context { - graph(const llama_model & model, const llm_graph_params & params); + graph(const llm_graph_params & params); }; + llama_rope_params get_rope_params(const llama_cparams & cparams, int il) const override; std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_cohere2moe : public llama_model_base { llama_model_cohere2moe(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; void load_arch_tensors(llama_model_loader & ml) override; struct graph : public llm_graph_context { - graph(const llama_model & model, const llm_graph_params & params); + graph(const llm_graph_params & params); }; struct graph_mtp : public llm_graph_context { - graph_mtp(const llama_model & model, const llm_graph_params & params); + graph_mtp(const llm_graph_params & params); }; + llama_rope_params get_rope_params(const llama_cparams & cparams, int il) const override; std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_dbrx : public llama_model_base { llama_model_dbrx(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; void load_arch_tensors(llama_model_loader & ml) override; struct graph : public llm_graph_context { - graph(const llama_model & model, const llm_graph_params & params); + graph(const llm_graph_params & params); }; std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_olmo : public llama_model_base { llama_model_olmo(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; void load_arch_tensors(llama_model_loader & ml) override; struct graph : public llm_graph_context { - graph(const llama_model & model, const llm_graph_params & params); + graph(const llm_graph_params & params); }; std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_olmo2 : public llama_model_base { llama_model_olmo2(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; @@ -1192,147 +1083,139 @@ struct llama_model_olmo2 : public llama_model_base { template struct graph : public llm_graph_context { - graph(const llama_model & model, const llm_graph_params & params); + graph(const llm_graph_params & params); }; + llama_rope_params get_rope_params(const llama_cparams & cparams, int il) const override; std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_olmoe : public llama_model_base { llama_model_olmoe(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; void load_arch_tensors(llama_model_loader & ml) override; struct graph : public llm_graph_context { - graph(const llama_model & model, const llm_graph_params & params); + graph(const llm_graph_params & params); }; std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_muse_spark : public llama_model_base { llama_model_muse_spark(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; void load_arch_tensors(llama_model_loader & ml) override; struct graph : public llm_graph_context { - graph(const llama_model & model, const llm_graph_params & params); + graph(const llm_graph_params & params); }; std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_muse_glimmer : public llama_model_base { llama_model_muse_glimmer(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; void load_arch_tensors(llama_model_loader & ml) override; struct graph : public llm_graph_context { - graph(const llama_model & model, const llm_graph_params & params); + graph(const llm_graph_params & params); }; std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_openelm : public llama_model_base { llama_model_openelm(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; void load_arch_tensors(llama_model_loader & ml) override; struct graph : public llm_graph_context { - graph(const llama_model & model, const llm_graph_params & params); + graph(const llm_graph_params & params); }; std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_gptneox : public llama_model_base { llama_model_gptneox(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; void load_arch_tensors(llama_model_loader & ml) override; struct graph : public llm_graph_context { - graph(const llama_model & model, const llm_graph_params & params); + graph(const llm_graph_params & params); }; std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_arctic : public llama_model_base { llama_model_arctic(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; void load_arch_tensors(llama_model_loader & ml) override; struct graph : public llm_graph_context { - graph(const llama_model & model, const llm_graph_params & params); + graph(const llm_graph_params & params); }; std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_deepseek : public llama_model_base { llama_model_deepseek(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; void load_arch_tensors(llama_model_loader & ml) override; struct graph : public llm_graph_context { - graph(const llama_model & model, const llm_graph_params & params); + graph(const llm_graph_params & params); }; std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_deepseek2 : public llama_model_base { llama_model_deepseek2(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; void load_arch_tensors(llama_model_loader & ml) override; struct graph : public llm_graph_context { - graph(const llama_model & model, const llm_graph_params & params); + graph(const llm_graph_params & params); }; struct graph_mtp : public llm_graph_context { - graph_mtp(const llama_model & model, const llm_graph_params & params); + graph_mtp(const llm_graph_params & params); }; std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_deepseek32 : public llama_model_base { llama_model_deepseek32(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; void load_arch_tensors(llama_model_loader & ml) override; struct graph : public llm_graph_context { - graph(const llama_model & model, const llm_graph_params & params); + graph(const llm_graph_params & params); }; struct graph_mtp : public llm_graph_context { - graph_mtp(const llama_model & model, const llm_graph_params & params); + graph_mtp(const llm_graph_params & params); }; std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_dots3note : public llama_model_base { llama_model_dots3note(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; void load_arch_tensors(llama_model_loader & ml) override; struct graph : public llm_graph_context { - graph(const llama_model & model, const llm_graph_params & params); + graph(const llm_graph_params & params); }; + llama_rope_params get_rope_params(const llama_cparams & cparams, int il) const override; std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; @@ -1351,8 +1234,9 @@ struct llama_model_deepseek4 : public llama_model_base { void load_arch_tensors(llama_model_loader & ml) override; struct graph : public llm_graph_context { - graph(const llm_graph_params & params) : llm_graph_context(params) {} - graph(const llama_model & model, const llm_graph_params & params); + struct no_build {}; + graph(const llm_graph_params & params, no_build) : llm_graph_context(params) {} + graph(const llm_graph_params & params); // collapse the hc streams with per-stream weights ggml_tensor * build_hc_pre( @@ -1410,21 +1294,18 @@ struct llama_model_deepseek4 : public llama_model_base { ggml_tensor * hc_base) const; ggml_tensor * build_attention( - const llama_model & model, llm_graph_input_dsv4 * inp_dsv4, ggml_tensor * cur, ggml_tensor * inp_pos, int il) const; ggml_tensor * build_attention( - const llama_model & model, llm_graph_input_attn_k_iswa * inp_mtp, ggml_tensor * cur, ggml_tensor * inp_pos, int il) const; ggml_tensor * build_attention_impl( - const llama_model & model, llm_graph_input_dsv4 * inp_dsv4, llm_graph_input_attn_k_iswa * inp_mtp, ggml_tensor * cur, @@ -1453,7 +1334,6 @@ struct llama_model_deepseek4 : public llama_model_base { int il) const; ggml_tensor * build_lid_top_k( - const llama_model & model, llm_graph_input_dsv4 * inp_dsv4, ggml_tensor * qr, ggml_tensor * cur, @@ -1467,7 +1347,6 @@ struct llama_model_deepseek4 : public llama_model_base { int il) const; ggml_tensor * build_csa_lid_attention( - const llama_model & model, llm_graph_input_dsv4 * inp_dsv4, llm_graph_input_dsv4_raw * inp_attn, ggml_tensor * q, @@ -1499,6 +1378,48 @@ struct llama_model_deepseek4 : public llama_model_base { }; struct graph_mtp : public graph { + graph_mtp(const llm_graph_params & params); + }; + + std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; +}; + + +struct llama_model_xing4_0 : public llama_model_base { + llama_model_xing4_0(const struct llama_model_params & params) : llama_model_base(params) {} + void load_arch_hparams(llama_model_loader & ml) override; + void load_arch_tensors(llama_model_loader & ml) override; + + struct graph : public llm_graph_context { + graph(const llama_model & model, const llm_graph_params & params); + + ggml_tensor * build_hc_pre( + ggml_tensor * x, + ggml_tensor * hc_fn, + ggml_tensor * hc_scale, + ggml_tensor * hc_base, + ggml_tensor ** post, + ggml_tensor ** comb, + int il) const; + + ggml_tensor * build_hc_pre( + ggml_tensor * x, + ggml_tensor * weights, + int il) const; + + ggml_tensor * build_hc_sinkhorn( + ggml_tensor * comb, + int il) const; + + ggml_tensor * build_hc_post( + ggml_tensor * x, + ggml_tensor * residual, + ggml_tensor * post, + ggml_tensor * comb, + int il) const; + }; + + struct graph_mtp : public llm_graph_context { graph_mtp(const llama_model & model, const llm_graph_params & params); }; @@ -1517,9 +1438,9 @@ struct llama_model_deepseek41 : public llama_model_deepseek4 { void engram_hash(int e, const llama_ubatch & ubatch, const std::vector & prev, std::vector & out) const; struct graph : public llama_model_deepseek4::graph { - graph(const llama_model & model, const llm_graph_params & params); + graph(const llm_graph_params & params); // helpers only (no graph built): base of graph_dsv41 - explicit graph(const llm_graph_params & params) : llama_model_deepseek4::graph(params) {} + graph(const llm_graph_params & params, no_build) : llama_model_deepseek4::graph(params, no_build{}) {} ggml_tensor * build_engram( const llama_model_deepseek41 & model, @@ -1560,23 +1481,24 @@ struct llama_model_deepseek2ocr : public llama_model_base { using graph = llama_model_deepseek2::graph; + llama_rope_params get_rope_params(const llama_cparams & cparams, int il) const override; std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_glm_dsa : public llama_model_base { llama_model_glm_dsa(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; void load_arch_tensors(llama_model_loader & ml) override; struct graph : public llm_graph_context { - graph(const llama_model & model, const llm_graph_params & params); + graph(const llm_graph_params & params); }; struct graph_mtp : public llm_graph_context { - graph_mtp(const llama_model & model, const llm_graph_params & params); + graph_mtp(const llm_graph_params & params); }; + llama_rope_params get_rope_params_indexer(const llama_cparams & cparams, int il) const override; std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; @@ -1587,7 +1509,7 @@ struct llama_model_eagle3 : public llama_model_base { template struct graph : public llm_graph_context { - graph(const llama_model & model, const llm_graph_params & params); + graph(const llm_graph_params & params); ggml_tensor * build_inp_embd_enc() const; }; @@ -1595,7 +1517,6 @@ struct llama_model_eagle3 : public llama_model_base { std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_dflash : public llama_model_base { llama_model_dflash(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; @@ -1603,42 +1524,41 @@ struct llama_model_dflash : public llama_model_base { template struct graph : public llm_graph_context { - graph(const llama_model & model, const llm_graph_params & params); + graph(const llm_graph_params & params); ggml_tensor * build_inp_embd_enc() const; }; struct graph_dsv4 : public llama_model_deepseek4::graph { - graph_dsv4(const llama_model & model, const llm_graph_params & params); + graph_dsv4(const llm_graph_params & params); }; // DeepSeek-V4.1 DSpark: single-pass mHC (lagged pre-mix, no hc_head), no per-head q norm struct graph_dsv41 : public llama_model_deepseek41::graph { - graph_dsv41(const llama_model & model, const llm_graph_params & params); + graph_dsv41(const llm_graph_params & params); }; // Mistral Large 4 DSpark: dense MLA stages, non-causal, rnn markov head struct graph_mistral_large4 : public llm_graph_context { - graph_mistral_large4(const llama_model & model, const llm_graph_params & params); + graph_mistral_large4(const llm_graph_params & params); }; std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_mistral_large4 : public llama_model_base { llama_model_mistral_large4(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; void load_arch_tensors(llama_model_loader & ml) override; struct graph : public llm_graph_context { - graph(const llama_model & model, const llm_graph_params & params); + graph(const llm_graph_params & params); }; + llama_rope_params get_rope_params_indexer(const llama_cparams & cparams, int il) const override; std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_mistral4 : public llama_model_deepseek2 { llama_model_mistral4(const struct llama_model_params & params) : llama_model_deepseek2(params) {} // reuse load_arch_hparams and load_arch_tensors from llama_model_deepseek2 @@ -1648,63 +1568,58 @@ struct llama_model_mistral4 : public llama_model_deepseek2 { std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_chatglm : public llama_model_base { llama_model_chatglm(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; void load_arch_tensors(llama_model_loader & ml) override; struct graph : public llm_graph_context { - graph(const llama_model & model, const llm_graph_params & params); + graph(const llm_graph_params & params); }; std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_glm4 : public llama_model_base { llama_model_glm4(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; void load_arch_tensors(llama_model_loader & ml) override; struct graph : public llm_graph_context { - graph(const llama_model & model, const llm_graph_params & params); + graph(const llm_graph_params & params); }; std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_glm4_moe : public llama_model_base { llama_model_glm4_moe(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; void load_arch_tensors(llama_model_loader & ml) override; struct graph : public llm_graph_context { - graph(const llama_model & model, const llm_graph_params & params); + graph(const llm_graph_params & params); }; struct graph_mtp : public llm_graph_context { - graph_mtp(const llama_model & model, const llm_graph_params & params); + graph_mtp(const llm_graph_params & params); }; std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_bitnet : public llama_model_base { llama_model_bitnet(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; void load_arch_tensors(llama_model_loader & ml) override; struct graph : public llm_graph_context { - graph(const llama_model & model, const llm_graph_params & params); + graph(const llm_graph_params & params); }; std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_t5 : public llama_model_base { llama_model_t5(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; @@ -1712,13 +1627,12 @@ struct llama_model_t5 : public llama_model_base { template struct graph : public llm_graph_context { - graph(const llama_model & model, const llm_graph_params & params); + graph(const llm_graph_params & params); }; std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_t5encoder : public llama_model_base { llama_model_t5encoder(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; @@ -1729,62 +1643,57 @@ struct llama_model_t5encoder : public llama_model_base { std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_jais : public llama_model_base { llama_model_jais(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; void load_arch_tensors(llama_model_loader & ml) override; struct graph : public llm_graph_context { - graph(const llama_model & model, const llm_graph_params & params); + graph(const llm_graph_params & params); }; std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_jais2 : public llama_model_base { llama_model_jais2(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; void load_arch_tensors(llama_model_loader & ml) override; struct graph : public llm_graph_context { - graph(const llama_model & model, const llm_graph_params & params); + graph(const llm_graph_params & params); }; std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_nemotron : public llama_model_base { llama_model_nemotron(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; void load_arch_tensors(llama_model_loader & ml) override; struct graph : public llm_graph_context { - graph(const llama_model & model, const llm_graph_params & params); + graph(const llm_graph_params & params); }; std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_nemotron_h : public llama_model_base { llama_model_nemotron_h(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; void load_arch_tensors(llama_model_loader & ml) override; struct graph : public llm_build_mamba_base { - graph(const llama_model & model, const llm_graph_params & params); - ggml_tensor * build_ffn_layer(ggml_tensor * cur, const llama_model & model, int il); + graph(const llm_graph_params & params); + ggml_tensor * build_ffn_layer(ggml_tensor * cur, int il); ggml_tensor * build_attention_layer(ggml_tensor * cur, llm_graph_input_attn_kv * inp_attn, - const llama_model & model, int64_t n_embd_head, int il); + int64_t n_embd_head, int il); }; std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_nemotron_h_moe : public llama_model_nemotron_h { llama_model_nemotron_h_moe(const struct llama_model_params & params) : llama_model_nemotron_h(params) {} // reuse load_arch_hparams and load_arch_tensors from llama_model_nemotron_h @@ -1792,12 +1701,43 @@ struct llama_model_nemotron_h_moe : public llama_model_nemotron_h { using graph = llama_model_nemotron_h::graph; struct graph_mtp : public llm_graph_context { - graph_mtp(const llama_model & model, const llm_graph_params & params); + graph_mtp(const llm_graph_params & params); }; std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; +struct llama_model_metis16 : public llama_model_base { + llama_model_metis16(const struct llama_model_params & params) : llama_model_base(params) {} + void load_arch_hparams(llama_model_loader & ml) override; + void load_arch_tensors(llama_model_loader & ml) override; + + uint32_t n_pass = 1; + uint32_t n_stream = 1; + uint32_t n_mem = 0; + uint32_t n_route = 0; + uint32_t n_layer_phys = 0; + float cont_limit = 0.0f; + + struct graph : public llm_build_mamba_base { + graph(const llm_graph_params & params); + + // memory bank of the current forward: keys [n_mem, n_entry, n_tokens], values [n_entry, n_mem, n_tokens] + ggml_tensor * bank_k = nullptr; + ggml_tensor * bank_vt = nullptr; + + ggml_tensor * stream_mean(ggml_tensor * x) const; + ggml_tensor * hc_read(ggml_tensor * x, ggml_tensor * mix, ggml_tensor * read, ggml_tensor * norm, int p, ggml_tensor ** mixed, int il) const; + ggml_tensor * hc_write(ggml_tensor * mixed, ggml_tensor * write, ggml_tensor * update, int p) const; + ggml_tensor * mem_read(const llama_model_metis16 & model, ggml_tensor ** x) const; + void mem_write(const llama_model_metis16 & model, ggml_tensor * x, ggml_tensor * summary, + ggml_tensor * entropy, ggml_tensor * confidence, ggml_tensor * cont, int entry); + ggml_tensor * route_features(const llama_model_metis16 & model, ggml_tensor * state, ggml_tensor * summary, + ggml_tensor * diff, ggml_tensor * history) const; + }; + + std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; +}; struct llama_model_exaone : public llama_model_base { llama_model_exaone(const struct llama_model_params & params) : llama_model_base(params) {} @@ -1805,13 +1745,12 @@ struct llama_model_exaone : public llama_model_base { void load_arch_tensors(llama_model_loader & ml) override; struct graph : public llm_graph_context { - graph(const llama_model & model, const llm_graph_params & params); + graph(const llm_graph_params & params); }; std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_exaone4 : public llama_model_base { llama_model_exaone4(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; @@ -1819,106 +1758,99 @@ struct llama_model_exaone4 : public llama_model_base { template struct graph : public llm_graph_context { - graph(const llama_model & model, const llm_graph_params & params); + graph(const llm_graph_params & params); }; + llama_rope_params get_rope_params(const llama_cparams & cparams, int il) const override; std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_exaone_moe : public llama_model_base { llama_model_exaone_moe(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; void load_arch_tensors(llama_model_loader & ml) override; struct graph : public llm_graph_context { - graph(const llama_model & model, const llm_graph_params & params); + graph(const llm_graph_params & params); }; + llama_rope_params get_rope_params(const llama_cparams & cparams, int il) const override; std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_rwkv6 : public llama_model_base { llama_model_rwkv6(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; void load_arch_tensors(llama_model_loader & ml) override; struct graph : public llm_build_rwkv6_base { - graph(const llama_model & model, const llm_graph_params & params); + graph(const llm_graph_params & params); }; std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_rwkv6qwen2 : public llama_model_base { llama_model_rwkv6qwen2(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; void load_arch_tensors(llama_model_loader & ml) override; struct graph : public llm_build_rwkv6_base { - graph(const llama_model & model, const llm_graph_params & params); + graph(const llm_graph_params & params); }; std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_rwkv7 : public llama_model_base { llama_model_rwkv7(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; void load_arch_tensors(llama_model_loader & ml) override; struct graph : public llm_build_rwkv7_base { - graph(const llama_model & model, const llm_graph_params & params); + graph(const llm_graph_params & params); }; std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_arwkv7 : public llama_model_base { llama_model_arwkv7(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; void load_arch_tensors(llama_model_loader & ml) override; struct graph : public llm_build_rwkv7_base { - graph(const llama_model & model, const llm_graph_params & params); + graph(const llm_graph_params & params); }; std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_granite : public llama_model_base { llama_model_granite(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; void load_arch_tensors(llama_model_loader & ml) override; struct graph : public llm_graph_context { - graph(const llama_model & model, const llm_graph_params & params); + graph(const llm_graph_params & params); private: ggml_tensor * build_attention_layer( ggml_tensor * cur, ggml_tensor * inp_pos, llm_graph_input_attn_kv * inp_attn, - const llama_model & model, const int64_t n_embd_head, const int il); ggml_tensor * build_layer_ffn( ggml_tensor * cur, ggml_tensor * inpSA, - const llama_model & model, const int il); }; std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_granite_moe : public llama_model_base { llama_model_granite_moe(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; @@ -1929,7 +1861,6 @@ struct llama_model_granite_moe : public llama_model_base { std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_granite_switch : public llama_model_base { llama_model_granite_switch(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; @@ -1943,7 +1874,7 @@ struct llama_model_granite_switch : public llama_model_base { std::unordered_map adapter_token_to_substitute; struct graph : public llm_graph_context { - graph(const llama_model & model, const llm_graph_params & params); + graph(const llm_graph_params & params); private: ggml_tensor * build_switched_lora_delta( @@ -1964,7 +1895,6 @@ struct llama_model_granite_switch : public llama_model_base { ggml_tensor * inp_pos, ggml_tensor * adapter_ids, llm_graph_input_attn_kv * inp_attn, - const llama_model & model, const int64_t n_embd_head, const int il); @@ -1972,14 +1902,12 @@ struct llama_model_granite_switch : public llama_model_base { ggml_tensor * cur, ggml_tensor * inpSA, ggml_tensor * adapter_ids, - const llama_model & model, const int il); }; std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_minicpm : public llama_model_base { llama_model_minicpm(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; @@ -1990,112 +1918,103 @@ struct llama_model_minicpm : public llama_model_base { std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_granite_hybrid : public llama_model_base { llama_model_granite_hybrid(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; void load_arch_tensors(llama_model_loader & ml) override; struct graph : public llm_build_mamba_base { - graph(const llama_model & model, const llm_graph_params & params); - ggml_tensor * build_layer_ffn(ggml_tensor * cur, ggml_tensor * inpSA, const llama_model & model, const int il); + graph(const llm_graph_params & params); + ggml_tensor * build_layer_ffn(ggml_tensor * cur, ggml_tensor * inpSA, const int il); ggml_tensor * build_attention_layer(ggml_tensor * cur, ggml_tensor * inp_pos, llm_graph_input_attn_kv * inp_attn, - const llama_model & model,const int64_t n_embd_head, const int il); + const int64_t n_embd_head, const int il); }; std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_granite_swa : public llama_model_base { llama_model_granite_swa(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; void load_arch_tensors(llama_model_loader & ml) override; struct graph : public llm_graph_context { - graph(const llama_model & model, const llm_graph_params & params); + graph(const llm_graph_params & params); private: ggml_tensor * build_attention_layer( ggml_tensor * cur, ggml_tensor * inp_pos, llm_graph_input_attn_kv_iswa * inp_attn, - const llama_model & model, const int64_t n_embd_head, const int il); ggml_tensor * build_layer_ffn( ggml_tensor * cur, ggml_tensor * inpSA, - const llama_model & model, const int il); }; + llama_rope_params get_rope_params(const llama_cparams & cparams, int il) const override; std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_chameleon : public llama_model_base { llama_model_chameleon(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; void load_arch_tensors(llama_model_loader & ml) override; struct graph : public llm_graph_context { - graph(const llama_model & model, const llm_graph_params & params); + graph(const llm_graph_params & params); }; std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_wavtokenizer_dec : public llama_model_base { llama_model_wavtokenizer_dec(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; void load_arch_tensors(llama_model_loader & ml) override; struct graph : public llm_graph_context { - graph(const llama_model & model, const llm_graph_params & params); + graph(const llm_graph_params & params); }; std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_plm : public llama_model_base { llama_model_plm(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; void load_arch_tensors(llama_model_loader & ml) override; struct graph : public llm_graph_context { - graph(const llama_model & model, const llm_graph_params & params); + graph(const llm_graph_params & params); }; std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_limite : public llama_model_base { llama_model_limite(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; void load_arch_tensors(llama_model_loader & ml) override; struct graph : public llm_graph_context { - graph(const llama_model & model, const llm_graph_params & params); + graph(const llm_graph_params & params); }; + llama_rope_params get_rope_params(const llama_cparams & cparams, int il) const override; std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_hrm_text : public llama_model_base { llama_model_hrm_text(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; void load_arch_tensors(llama_model_loader & ml) override; struct graph : public llm_graph_context { - graph(const llama_model & model, const llm_graph_params & params); - - const llama_model & model; + graph(const llm_graph_params & params); ggml_tensor * build_stack( llm_graph_input_attn_kv * inp_attn, @@ -2107,176 +2026,181 @@ struct llama_model_hrm_text : public llama_model_base { std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_bailingmoe : public llama_model_base { llama_model_bailingmoe(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; void load_arch_tensors(llama_model_loader & ml) override; struct graph : public llm_graph_context { - graph(const llama_model & model, const llm_graph_params & params); + graph(const llm_graph_params & params); }; std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_bailingmoe2 : public llama_model_base { llama_model_bailingmoe2(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; void load_arch_tensors(llama_model_loader & ml) override; struct graph : public llm_graph_context { - graph(const llama_model & model, const llm_graph_params & params); + graph(const llm_graph_params & params); }; std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_bailingmoe3 : public llama_model_base { llama_model_bailingmoe3(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; void load_arch_tensors(llama_model_loader & ml) override; struct graph : public llm_build_delta_net_base { - graph(const llama_model & model, const llm_graph_params & params); + graph(const llm_graph_params & params); - const llama_model & model; + // causal conv1d over one of Q/K/V, qkv selects which third of the conv state row is used + ggml_tensor * build_causal_conv1d(llm_graph_input_rs * inp_rs, ggml_tensor * conv_states_all, ggml_tensor * conv_state_all, + int64_t qkv, ggml_tensor * x, ggml_tensor * proj_w, ggml_tensor * conv_w, + int64_t d_conv, int64_t head_dim, int64_t n_head, int64_t n_seq_tokens, int64_t n_seqs); }; struct graph_mtp : public llm_graph_context { - graph_mtp(const llama_model & model, const llm_graph_params & params); + graph_mtp(const llm_graph_params & params); }; std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_seed_oss : public llama_model_base { llama_model_seed_oss(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; void load_arch_tensors(llama_model_loader & ml) override; struct graph : public llm_graph_context { - graph(const llama_model & model, const llm_graph_params & params); + graph(const llm_graph_params & params); }; std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_dots1 : public llama_model_base { llama_model_dots1(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; void load_arch_tensors(llama_model_loader & ml) override; struct graph : public llm_graph_context { - graph(const llama_model & model, const llm_graph_params & params); + graph(const llm_graph_params & params); }; std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_arcee : public llama_model_base { llama_model_arcee(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; void load_arch_tensors(llama_model_loader & ml) override; struct graph : public llm_graph_context { - graph(const llama_model & model, const llm_graph_params & params); + graph(const llm_graph_params & params); }; std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_afmoe : public llama_model_base { llama_model_afmoe(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; void load_arch_tensors(llama_model_loader & ml) override; struct graph : public llm_graph_context { - graph(const llama_model & model, const llm_graph_params & params); + graph(const llm_graph_params & params); }; std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_beam : public llama_model_base { llama_model_beam(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; void load_arch_tensors(llama_model_loader & ml) override; struct graph : public llm_graph_context { - graph(const llama_model & model, const llm_graph_params & params); + graph(const llm_graph_params & params); }; std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; +struct llama_model_ricursive : public llama_model_base { + llama_model_ricursive(const struct llama_model_params & params) : llama_model_base(params) {} + void load_arch_hparams(llama_model_loader & ml) override; + void load_arch_tensors(llama_model_loader & ml) override; + + struct graph : public llm_graph_context { + graph(const llm_graph_params & params); + }; + + std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; +}; + struct llama_model_laguna : public llama_model_base { llama_model_laguna(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; void load_arch_tensors(llama_model_loader & ml) override; struct graph : public llm_graph_context { - graph(const llama_model & model, const llm_graph_params & params); + graph(const llm_graph_params & params); }; + llama_rope_params get_rope_params(const llama_cparams & cparams, int il) const override; std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_ernie4_5 : public llama_model_base { llama_model_ernie4_5(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; void load_arch_tensors(llama_model_loader & ml) override; struct graph : public llm_graph_context { - graph(const llama_model & model, const llm_graph_params & params); + graph(const llm_graph_params & params); }; std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_ernie4_5_moe : public llama_model_ernie4_5 { llama_model_ernie4_5_moe(const struct llama_model_params & params) : llama_model_ernie4_5(params) {} // reuse load_arch_hparams and load_arch_tensors from llama_model_ernie4_5 struct graph : public llm_graph_context { - graph(const llama_model & model, const llm_graph_params & params); + graph(const llm_graph_params & params); }; std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_paddleocr : public llama_model_ernie4_5 { llama_model_paddleocr(const struct llama_model_params & params) : llama_model_ernie4_5(params) {} // reuse load_arch_hparams and load_arch_tensors from llama_model_ernie4_5 struct graph : public llm_graph_context { - graph(const llama_model & model, const llm_graph_params & params); + graph(const llm_graph_params & params); }; std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_hunyuan_moe : public llama_model_base { llama_model_hunyuan_moe(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; void load_arch_tensors(llama_model_loader & ml) override; struct graph : public llm_graph_context { - graph(const llama_model & model, const llm_graph_params & params); + graph(const llm_graph_params & params); }; + llama_rope_params get_rope_params(const llama_cparams & cparams, int il) const override; std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; @@ -2286,24 +2210,23 @@ struct llama_model_hy_v3 : public llama_model_base { void load_arch_tensors(llama_model_loader & ml) override; struct graph : public llm_graph_context { - graph(const llama_model & model, const llm_graph_params & params); + graph(const llm_graph_params & params); }; struct graph_mtp : public llm_graph_context { - graph_mtp(const llama_model & model, const llm_graph_params & params); + graph_mtp(const llm_graph_params & params); }; std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_hy_v4 : public llama_model_base { llama_model_hy_v4(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; void load_arch_tensors(llama_model_loader & ml) override; struct graph : public llm_graph_context { - graph(const llama_model & model, const llm_graph_params & params); + graph(const llm_graph_params & params); // iHC (independent Hyper-Connections): pre reduces the hc streams to one and returns the // per-stream post gates, post writes the sublayer output back into the streams, head @@ -2333,7 +2256,6 @@ struct llama_model_hy_v4 : public llama_model_base { ggml_tensor * hc_base) const; ggml_tensor * build_attention( - const llama_model & model, llm_graph_input_attn_k * inp_attn, ggml_tensor * cur, ggml_tensor * inp_pos, @@ -2343,7 +2265,6 @@ struct llama_model_hy_v4 : public llama_model_base { // DSA lightning indexer: top-k KV positions for this layer. Only "full" layers compute // it, "shared" layers reuse the last preceding full layer result through last_top_k. ggml_tensor * build_indexer_top_k( - const llama_model & model, llm_graph_input_attn_k_dsa * inp_attn_dsa, ggml_tensor * cur, ggml_tensor * qr, @@ -2357,7 +2278,6 @@ struct llama_model_hy_v4 : public llama_model_base { }; attn_dsa build_attention_dsa( - const llama_model & model, llm_graph_input_attn_k_dsa * inp_attn_dsa, ggml_tensor * cur, ggml_tensor * inp_pos, @@ -2366,72 +2286,69 @@ struct llama_model_hy_v4 : public llama_model_base { int il) const; }; + llama_rope_params get_rope_params_indexer(const llama_cparams & cparams, int il) const override; std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_hunyuan_vl : public llama_model_base { llama_model_hunyuan_vl(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; void load_arch_tensors(llama_model_loader & ml) override; struct graph : public llm_graph_context { - graph(const llama_model & model, const llm_graph_params & params); + graph(const llm_graph_params & params); }; std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_hunyuan_dense : public llama_model_hunyuan_vl { llama_model_hunyuan_dense(const struct llama_model_params & params) : llama_model_hunyuan_vl(params) {} // reuse load_arch_hparams and load_arch_tensors from llama_model_hunyuan_vl using graph = llama_model_hunyuan_vl::graph; + llama_rope_params get_rope_params(const llama_cparams & cparams, int il) const override; std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_smollm3 : public llama_model_base { llama_model_smollm3(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; void load_arch_tensors(llama_model_loader & ml) override; struct graph : public llm_graph_context { - graph(const llama_model & model, const llm_graph_params & params); + graph(const llm_graph_params & params); }; std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_openai_moe : public llama_model_base { llama_model_openai_moe(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; void load_arch_tensors(llama_model_loader & ml) override; struct graph : public llm_graph_context { - graph(const llama_model & model, const llm_graph_params & params); + graph(const llm_graph_params & params); }; std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_falcon_h1 : public llama_model_base { llama_model_falcon_h1(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; void load_arch_tensors(llama_model_loader & ml) override; struct graph : public llm_build_mamba_base { - graph(const llama_model & model, const llm_graph_params & params); + graph(const llm_graph_params & params); }; + llama_rope_params get_rope_params(const llama_cparams & cparams, int il) const override; std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_lfm2 : public llama_model_base { llama_model_lfm2(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; @@ -2439,24 +2356,23 @@ struct llama_model_lfm2 : public llama_model_base { template struct graph : public llm_build_lfm_base { - graph(const llama_model & model, const llm_graph_params & params); + graph(const llm_graph_params & params); }; // non-causal trunk without memory, then the decision head struct graph_decision : public llm_graph_context { - graph_decision(const llama_model & model, const llm_graph_params & params); + graph_decision(const llm_graph_params & params); ggml_tensor * build_decision_head( - const llama_model & model, ggml_tensor * inp, llm_graph_input_attn_no_cache * inp_attn, ggml_tensor * inp_out_ids); }; + llama_rope_params get_rope_params(const llama_cparams & cparams, int il) const override; std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_lfm2moe : public llama_model_base { llama_model_lfm2moe(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; @@ -2468,7 +2384,6 @@ struct llama_model_lfm2moe : public llama_model_base { std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_lfm3 : public llama_model_base { llama_model_lfm3(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; @@ -2477,13 +2392,12 @@ struct llama_model_lfm3 : public llama_model_base { // decision: d1, the prompt read in one batch, no memory template struct graph : public llm_build_lfm_base { - graph(const llama_model & model, const llm_graph_params & params); + graph(const llm_graph_params & params); }; std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_smallthinker : public llama_model_base { llama_model_smallthinker(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; @@ -2491,59 +2405,55 @@ struct llama_model_smallthinker : public llama_model_base { template struct graph : public llm_graph_context { - graph(const llama_model & model, const llm_graph_params & params); + graph(const llm_graph_params & params); }; std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_grovemoe : public llama_model_base { llama_model_grovemoe(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; void load_arch_tensors(llama_model_loader & ml) override; struct graph : public llm_graph_context { - graph(const llama_model & model, const llm_graph_params & params); + graph(const llm_graph_params & params); }; std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_apertus : public llama_model_base { llama_model_apertus(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; void load_arch_tensors(llama_model_loader & ml) override; struct graph : public llm_graph_context { - graph(const llama_model & model, const llm_graph_params & params); + graph(const llm_graph_params & params); }; std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_minimax_01 : public llama_model_base { llama_model_minimax_01(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; void load_arch_tensors(llama_model_loader & ml) override; struct graph : public llm_graph_context { - graph(const llama_model & model, const llm_graph_params & params); + graph(const llm_graph_params & params); }; std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_minimax_m2 : public llama_model_base { llama_model_minimax_m2(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; void load_arch_tensors(llama_model_loader & ml) override; struct graph : public llm_graph_context { - graph(const llama_model & model, const llm_graph_params & params); + graph(const llm_graph_params & params); }; std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; @@ -2561,7 +2471,7 @@ struct llama_model_minimax_m3 : public llama_model_base { void load_arch_tensors(llama_model_loader & ml) override; msa_params msa_p; struct graph : public llm_graph_context { - graph(const llama_model & model, const llm_graph_params & params); + graph(const llm_graph_params & params); ggml_tensor * build_attn_msa_fa( ggml_tensor * q_cur, // [D, HQ, S] f32 @@ -2579,33 +2489,33 @@ struct llama_model_cogvlm : public llama_model_base { void load_arch_tensors(llama_model_loader & ml) override; struct graph : public llm_graph_context { - graph(const llama_model & model, const llm_graph_params & params); + graph(const llm_graph_params & params); }; + llama_rope_params get_rope_params(const llama_cparams & cparams, int il) const override; std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_pangu_embed : public llama_model_base { llama_model_pangu_embed(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; void load_arch_tensors(llama_model_loader & ml) override; struct graph : public llm_graph_context { - graph(const llama_model & model, const llm_graph_params & params); + graph(const llm_graph_params & params); }; + llama_rope_params get_rope_params(const llama_cparams & cparams, int il) const override; std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_qwen3next : public llama_model_base { llama_model_qwen3next(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; void load_arch_tensors(llama_model_loader & ml) override; struct graph : public llm_build_delta_net_base { - graph(const llama_model & model, const llm_graph_params & params); + graph(const llm_graph_params & params); private: ggml_tensor * build_layer_attn( llm_graph_input_attn_kv * inp_attn, @@ -2632,31 +2542,27 @@ struct llama_model_qwen3next : public llama_model_base { std::pair build_qkvz( ggml_tensor * input, int il); - - const llama_model & model; }; struct graph_mtp : public llm_graph_context { - graph_mtp(const llama_model & model, const llm_graph_params & params); + graph_mtp(const llm_graph_params & params); }; std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_qwen35 : public llama_model_base { llama_model_qwen35(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; void load_arch_tensors(llama_model_loader & ml) override; struct graph : public llm_build_delta_net_base { - graph(const llama_model & model, const llm_graph_params & params); + graph(const llm_graph_params & params); private: ggml_tensor * build_layer_attn( llm_graph_input_attn_kv * inp_attn, ggml_tensor * cur, ggml_tensor * inp_pos, - int * sections, int il); ggml_tensor * build_layer_attn_linear( @@ -2678,18 +2584,15 @@ struct llama_model_qwen35 : public llama_model_base { std::pair build_qkvz( ggml_tensor * input, int il); - - const llama_model & model; }; struct graph_mtp : public llm_graph_context { - graph_mtp(const llama_model & model, const llm_graph_params & params); + graph_mtp(const llm_graph_params & params); }; std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - // Qwen3.5 backbone evaluated without memory, with a joint decision head on top of its hidden states struct llama_model_clef : public llama_model_qwen35 { llama_model_clef(const struct llama_model_params & params) : llama_model_qwen35(params) {} @@ -2752,7 +2655,7 @@ struct llama_model_clef : public llama_model_qwen35 { class input_decision; struct graph : public llm_build_delta_net_base { - graph(const llama_model & model, const llm_graph_params & params); + graph(const llm_graph_params & params); private: llm_graph_input_attn_no_cache * build_attn_inp_causal(); @@ -2760,7 +2663,6 @@ struct llama_model_clef : public llama_model_qwen35 { llm_graph_input_attn_no_cache * inp_attn, ggml_tensor * cur, ggml_tensor * inp_pos, - int * sections, int il); ggml_tensor * build_layer_attn_linear( @@ -2782,7 +2684,6 @@ struct llama_model_clef : public llama_model_qwen35 { std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_qwen4exp : public llama_model_base { llama_model_qwen4exp(const struct llama_model_params & params) : llama_model_base(params) {} @@ -2792,12 +2693,12 @@ struct llama_model_qwen4exp : public llama_model_base { void load_arch_tensors(llama_model_loader & ml) override; struct graph : public llm_build_delta_net_base { - graph(const llama_model & model, const llm_graph_params & params); + graph(const llm_graph_params & params); protected: // the helpers alone, graph_mtp builds its own body struct no_build {}; - graph(const llama_model & model, const llm_graph_params & params, no_build) : - llm_build_delta_net_base(params), model(model) {} + graph(const llm_graph_params & params, no_build) : + llm_build_delta_net_base(params) {} // HC replaces every layer norm: residual is [n_embd, hc, n_tokens] struct hc_mix { @@ -2826,7 +2727,6 @@ struct llama_model_qwen4exp : public llama_model_base { llm_graph_input_kpool * inp_kpool, ggml_tensor * cur, ggml_tensor * inp_pos, - int * sections, int il); // dense self-attention over the cells the QSA mask keeps @@ -2849,7 +2749,6 @@ struct llama_model_qwen4exp : public llama_model_base { llm_graph_input_kpool * inp_kpool, ggml_tensor * cur, ggml_tensor * inp_pos, - int * sections, int il); ggml_tensor * build_layer_attn_linear( @@ -2892,13 +2791,11 @@ struct llama_model_qwen4exp : public llama_model_base { std::pair build_qkvz( ggml_tensor * input, int il); - - const llama_model & model; }; // MTP draft head: one QSA block after the trunk, fed by the trunk's hc-wide residual struct graph_mtp : public graph { - graph_mtp(const llama_model & model, const llm_graph_params & params); + graph_mtp(const llm_graph_params & params); }; std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; @@ -2910,13 +2807,12 @@ struct llama_model_qwen35moe : public llama_model_base { void load_arch_tensors(llama_model_loader & ml) override; struct graph : public llm_build_delta_net_base { - graph(const llama_model & model, const llm_graph_params & params); + graph(const llm_graph_params & params); private: ggml_tensor * build_layer_attn( llm_graph_input_attn_kv * inp_attn, ggml_tensor * cur, ggml_tensor * inp_pos, - int * sections, int il); ggml_tensor * build_layer_attn_linear( @@ -2938,33 +2834,28 @@ struct llama_model_qwen35moe : public llama_model_base { std::pair build_qkvz( ggml_tensor * input, int il); - - const llama_model & model; }; struct graph_mtp : public llm_graph_context { - graph_mtp(const llama_model & model, const llm_graph_params & params); + graph_mtp(const llm_graph_params & params); }; std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_clover_moe : public llama_model_base { llama_model_clover_moe(const llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; void load_arch_tensors(llama_model_loader & ml) override; struct graph : public llm_build_delta_net_base { - graph(const llama_model & model, const llm_graph_params & params); + graph(const llm_graph_params & params); private: ggml_tensor * build_layer_attn(llm_graph_input_attn_kv * inp_attn, ggml_tensor * cur, ggml_tensor * inp_pos, int il); ggml_tensor * build_layer_attn_linear(llm_graph_input_rs * inp, ggml_tensor * cur, int il); ggml_tensor * build_layer_ffn(ggml_tensor * cur, int il); ggml_tensor * build_norm_gated(ggml_tensor * input, ggml_tensor * weights, ggml_tensor * gate, int il); std::pair build_qkvz(ggml_tensor * input, int il); - - const llama_model & model; }; std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; @@ -2978,39 +2869,35 @@ struct llama_model_mistral3 : public llama_model_base { class input_t_cond; // voxtral realtime: time conditioning of the ada rms norm struct graph : public llm_graph_context { - graph(const llama_model & model, const llm_graph_params & params); + graph(const llm_graph_params & params); }; std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_mimo2 : public llama_model_base { llama_model_mimo2(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; void load_arch_tensors(llama_model_loader & ml) override; struct graph : public llm_graph_context { - graph(const llama_model & model, const llm_graph_params & params); + graph(const llm_graph_params & params); }; struct graph_mtp : public llm_graph_context { - graph_mtp(const llama_model & model, const llm_graph_params & params); + graph_mtp(const llm_graph_params & params); }; std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_kimi_k3 : public llama_model_base { llama_model_kimi_k3(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; void load_arch_tensors(llama_model_loader & ml) override; struct graph : public llm_build_delta_net_base { - graph(const llama_model & model, const llm_graph_params & params); - - const llama_model & model; + graph(const llm_graph_params & params); // Cross-layer residual attention (K3's `_apply_attn_res`). ggml_tensor * resi_stack = nullptr; @@ -3024,6 +2911,11 @@ struct llama_model_kimi_k3 : public llama_model_base { int64_t d_conv, int64_t head_dim, int64_t n_head_kda, int64_t d_inner, int64_t n_seq_tokens, int64_t n_seqs, int il); + // causal conv1d over one of Q/K/V, qkv selects which third of the conv state row is used + ggml_tensor * build_kda_conv1d(llm_graph_input_rs * inp_rs, ggml_tensor * conv_states_all, ggml_tensor * conv_state_all, + int64_t qkv, ggml_tensor * x, ggml_tensor * proj_w, ggml_tensor * conv_w, + int64_t d_conv, int64_t head_dim, int64_t n_head_kda, int64_t n_seq_tokens, int64_t n_seqs); + ggml_tensor * build_mla_layer(ggml_tensor * cur, const llama_layer & layer, llm_graph_input_attn_k * inp_attn_k, llm_graph_input_attn_kv * inp_attn_kv, @@ -3038,6 +2930,53 @@ struct llama_model_kimi_k3 : public llama_model_base { std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; +// Kimi-K3.1: Kimi-K3 with a lightning indexer and top-k sparse attention on the full-attention layers +struct llama_model_kimi_k31 : public llama_model_base { + llama_model_kimi_k31(const struct llama_model_params & params) : llama_model_base(params) {} + void load_arch_hparams(llama_model_loader & ml) override; + void load_arch_tensors(llama_model_loader & ml) override; + + struct graph : public llm_build_delta_net_base { + graph(const llm_graph_params & params); + + // Cross-layer residual attention (kimi-k3's `_apply_attn_res`). + ggml_tensor * resi_stack = nullptr; + + void res_push(ggml_tensor * cur, int64_t n_embd, int64_t n_tokens); + ggml_tensor * res_mix(ggml_tensor * cur, ggml_tensor * score_w, + int64_t n_tokens, int il); + + ggml_tensor * build_kda_conv1d(llm_graph_input_rs * inp_rs, + ggml_tensor * conv_states_all, ggml_tensor * conv_state_all, + int64_t qkv, ggml_tensor * x, ggml_tensor * proj_w, ggml_tensor * conv_w, + int64_t d_conv, int64_t head_dim, int64_t n_head_kda, + int64_t n_seq_tokens, int64_t n_seqs); + + ggml_tensor * build_kda_layer(ggml_tensor * cur, const llama_layer & layer, + llm_graph_input_rs * inp_rs, + int64_t d_conv, int64_t head_dim, int64_t n_head_kda, + int64_t d_inner, int64_t n_seq_tokens, int64_t n_seqs, int il); + + struct dsa_out { + ggml_tensor * out; + ggml_tensor * sel; + }; + + dsa_out build_dsa_layer(ggml_tensor * cur, const llama_layer & layer, + const llama_kv_cache_context * mctx_idx, + llm_graph_input_attn_k * inp_attn, + ggml_tensor * prev_sel, + int64_t n_embd_head_k_mla, int64_t kv_lora_rank, + int64_t n_embd_head_qk_rope, + int64_t n_embd_head_qk_nope, float kq_scale, int il); + + ggml_tensor * build_latent_moe(ggml_tensor * cur, const llama_layer & layer, + int64_t n_embd_latent, int il); + }; + + std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; +}; + struct llama_model_glm5_next : public llama_model_base { llama_model_glm5_next(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; @@ -3047,11 +2986,11 @@ struct llama_model_glm5_next : public llama_model_base { class llm_graph_input_kpool; struct graph : public llm_build_delta_net_base { - graph(const llama_model & model, const llm_graph_params & params); + graph(const llm_graph_params & params); // build the helpers without the trunk, so graph_mtp can reuse them struct no_build {}; - graph(const llama_model & model, const llm_graph_params & params, no_build); + graph(const llm_graph_params & params, no_build); // collapse the hc streams with per-stream weights ggml_tensor * build_hc_pre( @@ -3087,8 +3026,6 @@ struct llama_model_glm5_next : public llama_model_base { ggml_tensor * comb, int il) const; - const llama_model & model; - llm_graph_input_kpool * build_inp_kpool(const llama_memory_hybrid_idx_context * mctx_hyb); ggml_tensor * build_kda_layer(ggml_tensor * cur, const llama_layer & layer, @@ -3096,6 +3033,11 @@ struct llama_model_glm5_next : public llama_model_base { int64_t d_conv, int64_t head_dim, int64_t n_head_kda, int64_t d_inner, int64_t n_seq_tokens, int64_t n_seqs, int il); + // causal conv1d over one of Q/K/V, qkv selects which third of the conv state row is used + ggml_tensor * build_kda_conv1d(llm_graph_input_rs * inp_rs, ggml_tensor * conv_states_all, ggml_tensor * conv_state_all, + int64_t qkv, ggml_tensor * x, ggml_tensor * proj_w, ggml_tensor * conv_w, + int64_t d_conv, int64_t head_dim, int64_t n_head_kda, int64_t n_seq_tokens, int64_t n_seqs); + ggml_tensor * build_kpool_select(ggml_tensor * cur, ggml_tensor * qr, const llama_layer & layer, const llama_memory_hybrid_idx_context * mctx_hyb, llm_graph_input_kpool * inp_kpool, int il); @@ -3113,7 +3055,7 @@ struct llama_model_glm5_next : public llama_model_base { // the NextN / MTP draft head: one DSA block appended after the trunk struct graph_mtp : public graph { - graph_mtp(const llama_model & model, const llm_graph_params & params); + graph_mtp(const llm_graph_params & params); }; std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; @@ -3125,7 +3067,7 @@ struct llama_model_kimi_linear : public llama_model_base { void load_arch_tensors(llama_model_loader & ml) override; struct graph : public llm_build_delta_net_base { - graph(const llama_model & model, const llm_graph_params & params); + graph(const llm_graph_params & params); std::pair build_kda_autoregressive( ggml_tensor * q, @@ -3147,38 +3089,35 @@ struct llama_model_kimi_linear : public llama_model_base { ggml_tensor * identity, ggml_tensor * diag_mask, int il); - - const llama_model & model; }; std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_step35 : public llama_model_base { llama_model_step35(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; void load_arch_tensors(llama_model_loader & ml) override; struct graph : public llm_graph_context { - graph(const llama_model & model, const llm_graph_params & params); + graph(const llm_graph_params & params); }; struct graph_mtp : public llm_graph_context { - graph_mtp(const llama_model & model, const llm_graph_params & params); + graph_mtp(const llm_graph_params & params); }; + llama_rope_params get_rope_params(const llama_cparams & cparams, int il) const override; std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_spark2_5 : public llama_model_base { llama_model_spark2_5(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; void load_arch_tensors(llama_model_loader & ml) override; struct graph : public llm_graph_context { - graph(const llama_model & model, const llm_graph_params & params); + graph(const llm_graph_params & params); }; std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; @@ -3190,7 +3129,7 @@ struct llama_model_k2_horizon : public llama_model_base { void load_arch_tensors(llama_model_loader & ml) override; struct graph : public llm_graph_context { - graph(const llama_model & model, const llm_graph_params & params); + graph(const llm_graph_params & params); ggml_tensor * build_routed_value(const llama_layer & layer, ggml_tensor * cur, int il) const; }; @@ -3198,14 +3137,13 @@ struct llama_model_k2_horizon : public llama_model_base { std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_kolibri1 : public llama_model_base { llama_model_kolibri1(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; void load_arch_tensors(llama_model_loader & ml) override; struct graph : public llm_graph_context { - graph(const llama_model & model, const llm_graph_params & params); + graph(const llm_graph_params & params); }; std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; diff --git a/src/models/modern-bert.cpp b/src/models/modern-bert.cpp index 84c85f869959..17c9c5ab6b91 100644 --- a/src/models/modern-bert.cpp +++ b/src/models/modern-bert.cpp @@ -132,10 +132,10 @@ void llama_model_modern_bert::load_arch_tensors(llama_model_loader &) { } std::unique_ptr llama_model_modern_bert::build_arch_graph(const llm_graph_params & params) const { - return std::make_unique(*this, params); + return std::make_unique(params); } -llama_model_modern_bert::graph::graph(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) { +llama_model_modern_bert::graph::graph(const llm_graph_params & params) : llm_graph_context(params) { const int64_t n_embd_head = hparams.n_embd_head_v(); GGML_ASSERT(n_embd_head == hparams.n_embd_head_k()); @@ -160,8 +160,6 @@ llama_model_modern_bert::graph::graph(const llama_model & model, const llm_graph const int n_layer_enc = n_layer - n_layer_dec; for (int il = 0; il < n_layer_enc; ++il) { - const float freq_base_l = model.get_rope_freq_base(cparams, il); - const float freq_scale_l = model.get_rope_freq_scale(cparams, il); cur = inpL; @@ -178,17 +176,8 @@ llama_model_modern_bert::graph::graph(const llama_model & model, const llm_graph n_embd_head, n_head, n_head_kv, il); // RoPE - Qcur = ggml_rope_ext( - ctx0, Qcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base_l, freq_scale_l, - ext_factor, attn_factor, beta_fast, beta_slow - ); - - Kcur = ggml_rope_ext( - ctx0, Kcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base_l, freq_scale_l, - ext_factor, attn_factor, beta_fast, beta_slow - ); + Qcur = build_rope(Qcur, inp_pos, il); + Kcur = build_rope(Kcur, inp_pos, il); cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); @@ -237,7 +226,7 @@ llama_model_modern_bert::graph::graph(const llama_model & model, const llm_graph cb(cur, "final_norm_out", -1); if (n_layer_dec > 0) { - cur = build_decision_head(model, cur, inp_attn, inp_out_ids); + cur = build_decision_head(cur, inp_attn, inp_out_ids); } res->t_embd = cur; @@ -246,7 +235,6 @@ llama_model_modern_bert::graph::graph(const llama_model & model, const llm_graph // returns one score per question type for each output token, the caller reads them at the option markers ggml_tensor * llama_model_modern_bert::graph::build_decision_head( - const llama_model & model, ggml_tensor * inp, llm_graph_input_attn_no_cache * inp_attn, ggml_tensor * inp_out_ids) { diff --git a/src/models/mpt.cpp b/src/models/mpt.cpp index 69d8b8c66bca..774907b4de64 100644 --- a/src/models/mpt.cpp +++ b/src/models/mpt.cpp @@ -61,10 +61,10 @@ void llama_model_mpt::load_arch_tensors(llama_model_loader &) { } std::unique_ptr llama_model_mpt::build_arch_graph(const llm_graph_params & params) const { - return std::make_unique(*this, params); + return std::make_unique(params); } -llama_model_mpt::graph::graph(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) { +llama_model_mpt::graph::graph(const llm_graph_params & params) : llm_graph_context(params) { const int64_t n_embd_head = hparams.n_embd_head_v(); GGML_ASSERT(n_embd_head == hparams.n_embd_head_k()); diff --git a/src/models/muse-glimmer.cpp b/src/models/muse-glimmer.cpp index 2f84d6f98013..e3f478fe3480 100644 --- a/src/models/muse-glimmer.cpp +++ b/src/models/muse-glimmer.cpp @@ -11,6 +11,7 @@ void llama_model_muse_glimmer::load_arch_hparams(llama_model_loader & ml) { hparams.swa_type = LLAMA_SWA_TYPE_STANDARD; load_swa_pattern(ml, 4); + hparams.rope_pattern = hparams.is_swa_impl; switch (hparams.n_layer()) { case 52: type = LLM_TYPE_30B; break; @@ -54,7 +55,7 @@ void llama_model_muse_glimmer::load_arch_tensors(llama_model_loader &) { } } -llama_model_muse_glimmer::graph::graph(const llama_model & model, const llm_graph_params & params) +llama_model_muse_glimmer::graph::graph(const llm_graph_params & params) : llm_graph_context(params) { const int64_t n_embd_head = hparams.n_embd_head_v(); GGML_ASSERT(n_embd_head == hparams.n_embd_head_k()); @@ -79,14 +80,8 @@ llama_model_muse_glimmer::graph::graph(const llama_model & model, const llm_grap // expose per-layer residual for speculative drafts (see LLM_KV_TARGET_LAYERS). res->t_layer_inp[il] = inpL; - const float freq_base_l = model.get_rope_freq_base (cparams, il); - const float freq_scale_l = model.get_rope_freq_scale(cparams, il); - ggml_tensor * inpSA = inpL; - // RoPE runs on the SWA layers, NoPE on full ones. - const bool use_rope = hparams.is_swa(il); - // pre-attention norm (weight+1 folded at conversion time) cur = build_norm(inpL, model.layers[il].attn_norm, NULL, LLM_NORM_RMS, il); cb(cur, "attn_norm", il); @@ -109,19 +104,11 @@ llama_model_muse_glimmer::graph::graph(const llama_model & model, const llm_grap cb(Qcur, "Qcur_normed", il); cb(Kcur, "Kcur_normed", il); - if (use_rope) { - Qcur = ggml_rope_ext( - ctx0, Qcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base_l, freq_scale_l, - ext_factor, attn_factor, beta_fast, beta_slow); - cb(Qcur, "Qcur_rope", il); + Qcur = build_rope(Qcur, inp_pos, il); + cb(Qcur, "Qcur_rope", il); - Kcur = ggml_rope_ext( - ctx0, Kcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base_l, freq_scale_l, - ext_factor, attn_factor, beta_fast, beta_slow); - cb(Kcur, "Kcur_rope", il); - } + Kcur = build_rope(Kcur, inp_pos, il); + cb(Kcur, "Kcur_rope", il); // SDPA. wo is deferred; the gate goes between attn_out and o_proj. cur = build_attn(inp_attn, @@ -197,5 +184,5 @@ llama_model_muse_glimmer::graph::graph(const llama_model & model, const llm_grap } std::unique_ptr llama_model_muse_glimmer::build_arch_graph(const llm_graph_params & params) const { - return std::make_unique(*this, params); + return std::make_unique(params); } diff --git a/src/models/muse-spark.cpp b/src/models/muse-spark.cpp index 5842ecb98110..ea6ab84abd94 100644 --- a/src/models/muse-spark.cpp +++ b/src/models/muse-spark.cpp @@ -15,6 +15,7 @@ void llama_model_muse_spark::load_arch_hparams(llama_model_loader & ml) { hparams.swa_type = LLAMA_SWA_TYPE_STANDARD; ml.get_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, hparams.is_swa_impl); + hparams.rope_pattern = hparams.is_swa_impl; switch (hparams.n_layer()) { case 62: type = LLM_TYPE_816B_A42B; break; @@ -75,7 +76,7 @@ void llama_model_muse_spark::load_arch_tensors(llama_model_loader &) { } } -llama_model_muse_spark::graph::graph(const llama_model & model, const llm_graph_params & params) +llama_model_muse_spark::graph::graph(const llm_graph_params & params) : llm_graph_context(params) { const int64_t n_embd_head = hparams.n_embd_head_v(); GGML_ASSERT(n_embd_head == hparams.n_embd_head_k()); @@ -96,14 +97,8 @@ llama_model_muse_spark::graph::graph(const llama_model & model, const llm_graph_ for (int il = 0; il < n_layer; ++il) { res->t_layer_inp[il] = inpL; - const float freq_base_l = model.get_rope_freq_base (cparams, il); - const float freq_scale_l = model.get_rope_freq_scale(cparams, il); - ggml_tensor * inpSA = inpL; - // RoPE runs on the SWA layers, NoPE on the global ones. - const bool use_rope = hparams.is_swa(il); - cur = build_norm(inpL, model.layers[il].attn_norm, NULL, LLM_NORM_RMS, il); cb(cur, "attn_norm", il); @@ -120,19 +115,11 @@ llama_model_muse_spark::graph::graph(const llama_model & model, const llm_graph_ cb(Qcur, "Qcur_normed", il); cb(Kcur, "Kcur_normed", il); - if (use_rope) { - Qcur = ggml_rope_ext( - ctx0, Qcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base_l, freq_scale_l, - ext_factor, attn_factor, beta_fast, beta_slow); - cb(Qcur, "Qcur_rope", il); + Qcur = build_rope(Qcur, inp_pos, il); + cb(Qcur, "Qcur_rope", il); - Kcur = ggml_rope_ext( - ctx0, Kcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base_l, freq_scale_l, - ext_factor, attn_factor, beta_fast, beta_slow); - cb(Kcur, "Kcur_rope", il); - } + Kcur = build_rope(Kcur, inp_pos, il); + cb(Kcur, "Kcur_rope", il); cur = build_attn(inp_attn, NULL, NULL, NULL, @@ -234,5 +221,5 @@ llama_model_muse_spark::graph::graph(const llama_model & model, const llm_graph_ } std::unique_ptr llama_model_muse_spark::build_arch_graph(const llm_graph_params & params) const { - return std::make_unique(*this, params); + return std::make_unique(params); } diff --git a/src/models/nanbeige.cpp b/src/models/nanbeige.cpp index e2d0a7b13349..2f3e0ff1c3e7 100644 --- a/src/models/nanbeige.cpp +++ b/src/models/nanbeige.cpp @@ -74,10 +74,10 @@ void llama_model_nanbeige::load_arch_tensors(llama_model_loader &) { } std::unique_ptr llama_model_nanbeige::build_arch_graph(const llm_graph_params & params) const { - return std::make_unique(*this, params); + return std::make_unique(params); } -llama_model_nanbeige::graph::graph(const llama_model & model, const llm_graph_params & params) : +llama_model_nanbeige::graph::graph(const llm_graph_params & params) : llm_graph_context(params) { const auto & nb = static_cast(model); @@ -110,20 +110,12 @@ llama_model_nanbeige::graph::graph(const llama_model & model, const llm_graph_pa cb(cur, "attn_norm", il); { - ggml_tensor * rope_factors = model.get_rope_factors(cparams, il); auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur, n_embd_head, n_head, n_head_kv, il); - Qcur = ggml_rope_ext( - ctx0, Qcur, inp_pos, rope_factors, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); - - Kcur = ggml_rope_ext( - ctx0, Kcur, inp_pos, rope_factors, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); + Qcur = build_rope(Qcur, inp_pos, il); + Kcur = build_rope(Kcur, inp_pos, il); cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); diff --git a/src/models/nemotron-h-moe.cpp b/src/models/nemotron-h-moe.cpp index c6ce58a7a671..4753558327d1 100644 --- a/src/models/nemotron-h-moe.cpp +++ b/src/models/nemotron-h-moe.cpp @@ -2,13 +2,13 @@ std::unique_ptr llama_model_nemotron_h_moe::build_arch_graph(const llm_graph_params & params) const { if (params.gtype == LLM_GRAPH_TYPE_DECODER_MTP) { - return std::make_unique(*this, params); + return std::make_unique(params); } - return std::make_unique(*this, params); + return std::make_unique(params); } // MTP draft head for Nemotron-H MoE -llama_model_nemotron_h_moe::graph_mtp::graph_mtp(const llama_model & model, const llm_graph_params & params) +llama_model_nemotron_h_moe::graph_mtp::graph_mtp(const llm_graph_params & params) : llm_graph_context(params) { GGML_ASSERT(hparams.n_layer_nextn == 1 && "NEMOTRON_H_MOE MTP currently supports a single MTP block"); @@ -42,9 +42,7 @@ llama_model_nemotron_h_moe::graph_mtp::graph_mtp(const llama_model & model, cons } cb(tok_embd, "mtp_tok_embd", il); - inp->h = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, hparams.n_embd, n_tokens); - ggml_set_input(inp->h); - ggml_set_name(inp->h, "mtp_h_input"); + inp->h = build_inp_embd_state(inp.get(), hparams.n_embd); ggml_tensor * h_embd = inp->h; diff --git a/src/models/nemotron-h.cpp b/src/models/nemotron-h.cpp index e9c61308feb1..313f42f1c7a0 100644 --- a/src/models/nemotron-h.cpp +++ b/src/models/nemotron-h.cpp @@ -196,10 +196,10 @@ void llama_model_nemotron_h::load_arch_tensors(llama_model_loader & ml) { } std::unique_ptr llama_model_nemotron_h::build_arch_graph(const llm_graph_params & params) const { - return std::make_unique(*this, params); + return std::make_unique(params); } -llama_model_nemotron_h::graph::graph(const llama_model & model, const llm_graph_params & params) : +llama_model_nemotron_h::graph::graph(const llm_graph_params & params) : llm_build_mamba_base(params) { const int64_t n_embd_head = hparams.n_embd_head_v(); GGML_ASSERT(n_embd_head == hparams.n_embd_head_k()); @@ -226,12 +226,12 @@ llama_model_nemotron_h::graph::graph(const llama_model & model, const llm_graph_ if (hparams.is_recr(il)) { // ssm layer // - cur = build_mamba2_layer(inp->get_recr(), cur, model, ubatch, il); + cur = build_mamba2_layer(inp->get_recr(), cur, ubatch, il); } else if (hparams.n_ff(il) == 0) { // attention layer // - cur = build_attention_layer(cur, inp->get_attn(), model, n_embd_head, il); + cur = build_attention_layer(cur, inp->get_attn(), n_embd_head, il); } else { - cur = build_ffn_layer(cur, model, il); + cur = build_ffn_layer(cur, il); } if (il == n_layer - 1 && crop_before_nextn(inp_out_ids) && !extract_final_inp) { @@ -277,8 +277,7 @@ llama_model_nemotron_h::graph::graph(const llama_model & model, const llm_graph_ ggml_tensor * llama_model_nemotron_h::graph::build_attention_layer(ggml_tensor * cur, llm_graph_input_attn_kv * inp_attn, - const llama_model & model, - int64_t n_embd_head, + int64_t n_embd_head, int il) { auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur, n_embd_head, hparams.n_head(il), hparams.n_head_kv(il), il); @@ -291,7 +290,7 @@ ggml_tensor * llama_model_nemotron_h::graph::build_attention_layer(ggml_tensor * return cur; } -ggml_tensor * llama_model_nemotron_h::graph::build_ffn_layer(ggml_tensor * cur, const llama_model & model, int il) { +ggml_tensor * llama_model_nemotron_h::graph::build_ffn_layer(ggml_tensor * cur, int il) { if (model.layers[il].ffn_gate_inp == nullptr) { cur = build_ffn(cur, model.layers[il].ffn_up, model.layers[il].ffn_up_b, model.layers[il].ffn_up_s, diff --git a/src/models/nemotron.cpp b/src/models/nemotron.cpp index 6b5b771074b7..f769b25375a9 100644 --- a/src/models/nemotron.cpp +++ b/src/models/nemotron.cpp @@ -47,10 +47,10 @@ void llama_model_nemotron::load_arch_tensors(llama_model_loader &) { } std::unique_ptr llama_model_nemotron::build_arch_graph(const llm_graph_params & params) const { - return std::make_unique(*this, params); + return std::make_unique(params); } -llama_model_nemotron::graph::graph(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) { +llama_model_nemotron::graph::graph(const llm_graph_params & params) : llm_graph_context(params) { const int64_t n_embd_head = hparams.n_embd_head_v(); GGML_ASSERT(n_embd_head == hparams.n_embd_head_k()); @@ -84,17 +84,8 @@ llama_model_nemotron::graph::graph(const llama_model & model, const llm_graph_pa auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur, n_embd_head, n_head, n_head_kv, il); - Qcur = ggml_rope_ext( - ctx0, Qcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); - - Kcur = ggml_rope_ext( - ctx0, Kcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); + Qcur = build_rope(Qcur, inp_pos, il); + Kcur = build_rope(Kcur, inp_pos, il); cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); diff --git a/src/models/neo-bert.cpp b/src/models/neo-bert.cpp index 9b141c3b70ae..58a012a68950 100644 --- a/src/models/neo-bert.cpp +++ b/src/models/neo-bert.cpp @@ -37,10 +37,10 @@ void llama_model_neo_bert::load_arch_tensors(llama_model_loader &) { } std::unique_ptr llama_model_neo_bert::build_arch_graph(const llm_graph_params & params) const { - return std::make_unique(*this, params); + return std::make_unique(params); } -llama_model_neo_bert::graph::graph(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) { +llama_model_neo_bert::graph::graph(const llm_graph_params & params) : llm_graph_context(params) { const int64_t n_embd_head = hparams.n_embd_head_v(); GGML_ASSERT(n_embd_head == hparams.n_embd_head_k()); @@ -70,17 +70,8 @@ llama_model_neo_bert::graph::graph(const llama_model & model, const llm_graph_pa n_embd_head, n_head, n_head_kv, il); // RoPE - Qcur = ggml_rope_ext( - ctx0, Qcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); - - Kcur = ggml_rope_ext( - ctx0, Kcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); + Qcur = build_rope(Qcur, inp_pos, il); + Kcur = build_rope(Kcur, inp_pos, il); cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); diff --git a/src/models/nomic-bert-moe.cpp b/src/models/nomic-bert-moe.cpp index 924af5e0bc40..afecce745339 100644 --- a/src/models/nomic-bert-moe.cpp +++ b/src/models/nomic-bert-moe.cpp @@ -51,6 +51,6 @@ void llama_model_nomic_bert_moe::load_arch_tensors(llama_model_loader &) { } std::unique_ptr llama_model_nomic_bert_moe::build_arch_graph(const llm_graph_params & params) const { - return std::make_unique(*this, params); + return std::make_unique(params); } diff --git a/src/models/nomic-bert.cpp b/src/models/nomic-bert.cpp index 509787286efa..2dc40567fac6 100644 --- a/src/models/nomic-bert.cpp +++ b/src/models/nomic-bert.cpp @@ -46,6 +46,6 @@ void llama_model_nomic_bert::load_arch_tensors(llama_model_loader &) { } std::unique_ptr llama_model_nomic_bert::build_arch_graph(const llm_graph_params & params) const { - return std::make_unique(*this, params); + return std::make_unique(params); } diff --git a/src/models/olmo.cpp b/src/models/olmo.cpp index 261aa98f7c23..2afead631bac 100644 --- a/src/models/olmo.cpp +++ b/src/models/olmo.cpp @@ -37,10 +37,10 @@ void llama_model_olmo::load_arch_tensors(llama_model_loader &) { } std::unique_ptr llama_model_olmo::build_arch_graph(const llm_graph_params & params) const { - return std::make_unique(*this, params); + return std::make_unique(params); } -llama_model_olmo::graph::graph(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) { +llama_model_olmo::graph::graph(const llm_graph_params & params) : llm_graph_context(params) { const int64_t n_embd_head = hparams.n_embd_head_v(); GGML_ASSERT(n_embd_head == hparams.n_embd_head_k()); @@ -73,17 +73,8 @@ llama_model_olmo::graph::graph(const llama_model & model, const llm_graph_params auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur, n_embd_head, n_head, n_head_kv, il); - Qcur = ggml_rope_ext( - ctx0, Qcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); - - Kcur = ggml_rope_ext( - ctx0, Kcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); + Qcur = build_rope(Qcur, inp_pos, il); + Kcur = build_rope(Kcur, inp_pos, il); cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); diff --git a/src/models/olmo2.cpp b/src/models/olmo2.cpp index db2de3afd72a..c0a3e8295da9 100644 --- a/src/models/olmo2.cpp +++ b/src/models/olmo2.cpp @@ -51,16 +51,31 @@ void llama_model_olmo2::load_arch_tensors(llama_model_loader &) { } } +llama_rope_params llama_model_olmo2::get_rope_params(const llama_cparams & cparams, int il) const { + auto res = llama_model::get_rope_params(cparams, il); + + res.freq_base = cparams.rope_freq_base; + + // regular RoPE on the SWA layers, without YaRN + if (hparams.is_swa(il)) { + res.freq_scale = 1.0f; + res.ext_factor = 0.0f; + res.attn_factor = 1.0f; + } + + return res; +} + std::unique_ptr llama_model_olmo2::build_arch_graph(const llm_graph_params & params) const { if (hparams.swa_type == LLAMA_SWA_TYPE_STANDARD) { - return std::make_unique>(*this, params); + return std::make_unique>(params); } else { - return std::make_unique>(*this, params); + return std::make_unique>(params); } } template -llama_model_olmo2::graph::graph(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) { +llama_model_olmo2::graph::graph(const llm_graph_params & params) : llm_graph_context(params) { const int64_t n_embd_head = hparams.n_embd_head_v(); GGML_ASSERT(n_embd_head == hparams.n_embd_head_k()); @@ -112,36 +127,8 @@ llama_model_olmo2::graph::graph(const llama_model & model, const llm_graph Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens); Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens); - const bool is_swa = hparams.is_swa(il); - - if (is_swa) { - // For sliding window layers, Olmo3 use regular rope with no yarn rope scaling. - // This is achieved here by setting freq_scale and attn_factor to 1. - // We also set ext_factor to 0 to avoid a few unnecessary computations. - Qcur = ggml_rope_ext( - ctx0, Qcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, 1.0, - 0.0, 1.0, beta_fast, beta_slow - ); - - Kcur = ggml_rope_ext( - ctx0, Kcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, 1.0, - 0.0, 1.0, beta_fast, beta_slow - ); - } else { - Qcur = ggml_rope_ext( - ctx0, Qcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); - - Kcur = ggml_rope_ext( - ctx0, Kcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); - } + Qcur = build_rope(Qcur, inp_pos, il); + Kcur = build_rope(Kcur, inp_pos, il); cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); cb(Vcur, "Vcur", il); diff --git a/src/models/olmoe.cpp b/src/models/olmoe.cpp index f447838216d0..9ebdd9c94aeb 100644 --- a/src/models/olmoe.cpp +++ b/src/models/olmoe.cpp @@ -47,10 +47,10 @@ void llama_model_olmoe::load_arch_tensors(llama_model_loader &) { } std::unique_ptr llama_model_olmoe::build_arch_graph(const llm_graph_params & params) const { - return std::make_unique(*this, params); + return std::make_unique(params); } -llama_model_olmoe::graph::graph(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) { +llama_model_olmoe::graph::graph(const llm_graph_params & params) : llm_graph_context(params) { const int64_t n_embd_head = hparams.n_embd_head_v(); GGML_ASSERT(n_embd_head == hparams.n_embd_head_k()); @@ -100,17 +100,8 @@ llama_model_olmoe::graph::graph(const llama_model & model, const llm_graph_param Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens); Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens); - Qcur = ggml_rope_ext( - ctx0, Qcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); - - Kcur = ggml_rope_ext( - ctx0, Kcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); + Qcur = build_rope(Qcur, inp_pos, il); + Kcur = build_rope(Kcur, inp_pos, il); cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); diff --git a/src/models/openai-moe.cpp b/src/models/openai-moe.cpp index 8e42278039f9..920acbb64c00 100644 --- a/src/models/openai-moe.cpp +++ b/src/models/openai-moe.cpp @@ -56,10 +56,10 @@ void llama_model_openai_moe::load_arch_tensors(llama_model_loader &) { } std::unique_ptr llama_model_openai_moe::build_arch_graph(const llm_graph_params & params) const { - return std::make_unique(*this, params); + return std::make_unique(params); } -llama_model_openai_moe::graph::graph(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) { +llama_model_openai_moe::graph::graph(const llm_graph_params & params) : llm_graph_context(params) { ggml_tensor * cur; ggml_tensor * inpL; @@ -75,9 +75,6 @@ llama_model_openai_moe::graph::graph(const llama_model & model, const llm_graph_ for (int il = 0; il < n_layer; ++il) { res->t_layer_inp[il] = inpL; - const float freq_base_l = model.get_rope_freq_base (cparams, il); - const float freq_scale_l = model.get_rope_freq_scale(cparams, il); - ggml_tensor * inpSA = inpL; // norm @@ -92,17 +89,8 @@ llama_model_openai_moe::graph::graph(const llama_model & model, const llm_graph_ auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur, n_rot, n_head, n_head_kv, il); - Qcur = ggml_rope_ext( - ctx0, Qcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base_l, freq_scale_l, - ext_factor, attn_factor, beta_fast, beta_slow - ); - - Kcur = ggml_rope_ext( - ctx0, Kcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base_l, freq_scale_l, - ext_factor, attn_factor, beta_fast, beta_slow - ); + Qcur = build_rope(Qcur, inp_pos, il); + Kcur = build_rope(Kcur, inp_pos, il); cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); diff --git a/src/models/openelm.cpp b/src/models/openelm.cpp index 1d7d4e5ec6b4..3328bd54c683 100644 --- a/src/models/openelm.cpp +++ b/src/models/openelm.cpp @@ -44,10 +44,10 @@ void llama_model_openelm::load_arch_tensors(llama_model_loader &) { } std::unique_ptr llama_model_openelm::build_arch_graph(const llm_graph_params & params) const { - return std::make_unique(*this, params); + return std::make_unique(params); } -llama_model_openelm::graph::graph(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) { +llama_model_openelm::graph::graph(const llm_graph_params & params) : llm_graph_context(params) { const int64_t n_embd_head = hparams.n_embd_head_v(); GGML_ASSERT(n_embd_head == hparams.n_embd_head_k()); @@ -103,17 +103,8 @@ llama_model_openelm::graph::graph(const llama_model & model, const llm_graph_par LLM_NORM_RMS, il); cb(Kcur, "Kcur", il); - Qcur = ggml_rope_ext( - ctx0, Qcur, inp_pos, NULL, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); - - Kcur = ggml_rope_ext( - ctx0, Kcur, inp_pos, NULL, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); + Qcur = build_rope(Qcur, inp_pos, il); + Kcur = build_rope(Kcur, inp_pos, il); cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); diff --git a/src/models/orion.cpp b/src/models/orion.cpp index 06ed3bd31c3f..4fe921189571 100644 --- a/src/models/orion.cpp +++ b/src/models/orion.cpp @@ -37,10 +37,10 @@ void llama_model_orion::load_arch_tensors(llama_model_loader &) { } std::unique_ptr llama_model_orion::build_arch_graph(const llm_graph_params & params) const { - return std::make_unique(*this, params); + return std::make_unique(params); } -llama_model_orion::graph::graph(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) { +llama_model_orion::graph::graph(const llm_graph_params & params) : llm_graph_context(params) { const int64_t n_embd_head = hparams.n_embd_head_v(); GGML_ASSERT(n_embd_head == hparams.n_embd_head_k()); @@ -73,17 +73,8 @@ llama_model_orion::graph::graph(const llama_model & model, const llm_graph_param auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur, n_embd_head, n_head, n_head_kv, il); - Qcur = ggml_rope_ext( - ctx0, Qcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); - - Kcur = ggml_rope_ext( - ctx0, Kcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); + Qcur = build_rope(Qcur, inp_pos, il); + Kcur = build_rope(Kcur, inp_pos, il); cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); diff --git a/src/models/paddleocr.cpp b/src/models/paddleocr.cpp index 68030b9af0d0..4dbd9e215542 100644 --- a/src/models/paddleocr.cpp +++ b/src/models/paddleocr.cpp @@ -1,10 +1,10 @@ #include "models.h" std::unique_ptr llama_model_paddleocr::build_arch_graph(const llm_graph_params & params) const { - return std::make_unique(*this, params); + return std::make_unique(params); } -llama_model_paddleocr::graph::graph(const llama_model & model, const llm_graph_params & params) : +llama_model_paddleocr::graph::graph(const llm_graph_params & params) : llm_graph_context(params) { // NOTE: same with qwen2vl.cpp, but bias tensors are optional @@ -19,9 +19,6 @@ llama_model_paddleocr::graph::graph(const llama_model & model, const llm_graph_p inpL = build_inp_embd(model.tok_embd); - int sections[4]; - std::copy(std::begin(hparams.rope_sections), std::begin(hparams.rope_sections) + 4, sections); - // inp_pos - contains the positions ggml_tensor * inp_pos = build_inp_pos(); @@ -42,17 +39,8 @@ llama_model_paddleocr::graph::graph(const llama_model & model, const llm_graph_p auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur, n_embd_head, n_head, n_head_kv, il); - Qcur = ggml_rope_multi( - ctx0, Qcur, inp_pos, nullptr, - n_rot, sections, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); - - Kcur = ggml_rope_multi( - ctx0, Kcur, inp_pos, nullptr, - n_rot, sections, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); + Qcur = build_rope(Qcur, inp_pos, il); + Kcur = build_rope(Kcur, inp_pos, il); cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); diff --git a/src/models/pangu-embed.cpp b/src/models/pangu-embed.cpp index cf34cd16a75f..336c71a41a8b 100644 --- a/src/models/pangu-embed.cpp +++ b/src/models/pangu-embed.cpp @@ -51,11 +51,19 @@ void llama_model_pangu_embed::load_arch_tensors(llama_model_loader &) { } } +llama_rope_params llama_model_pangu_embed::get_rope_params(const llama_cparams & cparams, int il) const { + auto res = llama_model::get_rope_params(cparams, il); + + res.factors = nullptr; + + return res; +} + std::unique_ptr llama_model_pangu_embed::build_arch_graph(const llm_graph_params & params) const { - return std::make_unique(*this, params); + return std::make_unique(params); } -llama_model_pangu_embed::graph::graph(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) { +llama_model_pangu_embed::graph::graph(const llm_graph_params & params) : llm_graph_context(params) { const int64_t n_embd_head = hparams.n_embd_head_v(); GGML_ASSERT(n_embd_head == hparams.n_embd_head_k()); @@ -88,16 +96,8 @@ llama_model_pangu_embed::graph::graph(const llama_model & model, const llm_graph auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur, n_embd_head, n_head, n_head_kv, il); - Qcur = ggml_rope_ext( - ctx0, Qcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); - - Kcur = ggml_rope_ext(ctx0, Kcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); + Qcur = build_rope(Qcur, inp_pos, il); + Kcur = build_rope(Kcur, inp_pos, il); cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); diff --git a/src/models/phi2.cpp b/src/models/phi2.cpp index de14cfe37b84..90c502cfdee4 100644 --- a/src/models/phi2.cpp +++ b/src/models/phi2.cpp @@ -41,10 +41,10 @@ void llama_model_phi2::load_arch_tensors(llama_model_loader &) { } std::unique_ptr llama_model_phi2::build_arch_graph(const llm_graph_params & params) const { - return std::make_unique(*this, params); + return std::make_unique(params); } -llama_model_phi2::graph::graph(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) { +llama_model_phi2::graph::graph(const llm_graph_params & params) : llm_graph_context(params) { const int64_t n_embd_head = hparams.n_embd_head_v(); GGML_ASSERT(n_embd_head == hparams.n_embd_head_k()); @@ -74,17 +74,8 @@ llama_model_phi2::graph::graph(const llama_model & model, const llm_graph_params { auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], attn_norm_output, n_embd_head, n_head, n_head_kv, il); - Qcur = ggml_rope_ext( - ctx0, Qcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); - - Kcur = ggml_rope_ext( - ctx0, Kcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); + Qcur = build_rope(Qcur, inp_pos, il); + Kcur = build_rope(Kcur, inp_pos, il); cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); diff --git a/src/models/phi3.cpp b/src/models/phi3.cpp index fc808484097d..815ab0465b2a 100644 --- a/src/models/phi3.cpp +++ b/src/models/phi3.cpp @@ -58,14 +58,14 @@ void llama_model_phi3::load_arch_tensors(llama_model_loader &) { std::unique_ptr llama_model_phi3::build_arch_graph(const llm_graph_params & params) const { if (hparams.swa_type != LLAMA_SWA_TYPE_NONE) { - return std::make_unique> (*this, params); + return std::make_unique> (params); } else { - return std::make_unique>(*this, params); + return std::make_unique>(params); } } template -llama_model_phi3::graph::graph(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) { +llama_model_phi3::graph::graph(const llm_graph_params & params) : llm_graph_context(params) { const int64_t n_embd_head = hparams.n_embd_head_v(); GGML_ASSERT(n_embd_head == hparams.n_embd_head_k()); @@ -93,8 +93,6 @@ llama_model_phi3::graph::graph(const llama_model & model, const llm_graph_ // self-attention { - // rope freq factors for 128k context - ggml_tensor * rope_factors = model.get_rope_factors(cparams, il); ggml_tensor* attn_norm_output = build_norm(inpL, model.layers[il].attn_norm, @@ -104,17 +102,8 @@ llama_model_phi3::graph::graph(const llama_model & model, const llm_graph_ auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], attn_norm_output, n_embd_head, n_head, n_head_kv, il); - Qcur = ggml_rope_ext( - ctx0, Qcur, inp_pos, rope_factors, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); - - Kcur = ggml_rope_ext( - ctx0, Kcur, inp_pos, rope_factors, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); + Qcur = build_rope(Qcur, inp_pos, il); + Kcur = build_rope(Kcur, inp_pos, il); cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); diff --git a/src/models/phimoe.cpp b/src/models/phimoe.cpp index c332553bc7d1..ce27a92995c5 100644 --- a/src/models/phimoe.cpp +++ b/src/models/phimoe.cpp @@ -47,9 +47,9 @@ void llama_model_phimoe::load_arch_tensors(llama_model_loader &) { std::unique_ptr llama_model_phimoe::build_arch_graph(const llm_graph_params & params) const { if (hparams.swa_type != LLAMA_SWA_TYPE_NONE) { - return std::make_unique> (*this, params); + return std::make_unique> (params); } else { - return std::make_unique>(*this, params); + return std::make_unique>(params); } } diff --git a/src/models/plamo.cpp b/src/models/plamo.cpp index 401ffe48af4e..10348f291e00 100644 --- a/src/models/plamo.cpp +++ b/src/models/plamo.cpp @@ -33,10 +33,10 @@ void llama_model_plamo::load_arch_tensors(llama_model_loader &) { } std::unique_ptr llama_model_plamo::build_arch_graph(const llm_graph_params & params) const { - return std::make_unique(*this, params); + return std::make_unique(params); } -llama_model_plamo::graph::graph(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) { +llama_model_plamo::graph::graph(const llm_graph_params & params) : llm_graph_context(params) { const int64_t n_embd_head = hparams.n_embd_head_v(); GGML_ASSERT(n_embd_head == hparams.n_embd_head_k()); @@ -69,17 +69,8 @@ llama_model_plamo::graph::graph(const llama_model & model, const llm_graph_param auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur, n_embd_head, n_head, n_head_kv, il); - Qcur = ggml_rope_ext( - ctx0, Qcur, inp_pos, nullptr, - n_embd_head, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); - - Kcur = ggml_rope_ext( - ctx0, Kcur, inp_pos, nullptr, - n_embd_head, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); + Qcur = build_rope(Qcur, inp_pos, il); + Kcur = build_rope(Kcur, inp_pos, il); cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); diff --git a/src/models/plamo2.cpp b/src/models/plamo2.cpp index 9ce229b19416..f019bd0b6955 100644 --- a/src/models/plamo2.cpp +++ b/src/models/plamo2.cpp @@ -104,10 +104,10 @@ void llama_model_plamo2::load_arch_tensors(llama_model_loader &) { } std::unique_ptr llama_model_plamo2::build_arch_graph(const llm_graph_params & params) const { - return std::make_unique(*this, params); + return std::make_unique(params); } -llama_model_plamo2::graph::graph(const llama_model & model, const llm_graph_params & params) : +llama_model_plamo2::graph::graph(const llm_graph_params & params) : llm_build_mamba_base(params) { ggml_tensor * cur; ggml_tensor * inpL; @@ -136,10 +136,10 @@ llama_model_plamo2::graph::graph(const llama_model & model, const llm_graph_para if (is_mamba_layer) { // PLaMo-2 Mamba layer - cur = build_plamo2_mamba_layer(inp_hybrid->get_recr(), cur, model, ubatch, il); + cur = build_plamo2_mamba_layer(inp_hybrid->get_recr(), cur, ubatch, il); } else { // PLaMo-2 Attention layer - cur = build_plamo2_attn_layer(inp_hybrid->get_attn(), inp_pos, cur, model, il); + cur = build_plamo2_attn_layer(inp_hybrid->get_attn(), inp_pos, cur, il); } if (il == n_layer - 1 && inp_out_ids) { @@ -201,7 +201,6 @@ llama_model_plamo2::graph::graph(const llama_model & model, const llm_graph_para ggml_tensor * llama_model_plamo2::graph::build_plamo2_attn_layer(llm_graph_input_attn_kv * inp, ggml_tensor * inp_pos, ggml_tensor * cur, - const llama_model & model, int il) { // self-attention { @@ -234,14 +233,12 @@ ggml_tensor * llama_model_plamo2::graph::build_plamo2_attn_layer(llm_graph_input Qcur = build_norm(Qcur, model.layers[il].attn_q_norm, NULL, LLM_NORM_RMS, il); cb(Qcur, "Qcur_normed", il); - Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); + Qcur = build_rope(Qcur, inp_pos, il); Kcur = build_norm(Kcur, model.layers[il].attn_k_norm, NULL, LLM_NORM_RMS, il); cb(Kcur, "Kcur_normed", il); - Kcur = ggml_rope_ext(ctx0, Kcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); + Kcur = build_rope(Kcur, inp_pos, il); cur = build_attn(inp, model.layers[il].wo, NULL, model.layers[il].wo_s, @@ -255,7 +252,6 @@ ggml_tensor * llama_model_plamo2::graph::build_plamo2_attn_layer(llm_graph_input ggml_tensor * llama_model_plamo2::graph::build_plamo2_mamba_layer(llm_graph_input_rs * inp, ggml_tensor * cur, - const llama_model & model, const llama_ubatch & ubatch, int il) { const auto * mctx_cur = inp->mctx; @@ -322,7 +318,6 @@ ggml_tensor * llama_model_plamo2::graph::build_plamo2_mamba_layer(llm_graph_inpu (d_conv - 1) * (d_inner + 2 * n_group * d_state) * (n_seqs), kv_head * (d_conv - 1) * (d_inner + 2 * n_group * d_state) * ggml_element_size(conv_states_all)))); - cb(conv_states_all, "mamba_conv1d_state", il); // 1D convolution x = ggml_ssm_conv(ctx0, conv_x, model.layers[il].ssm_conv1d); @@ -394,7 +389,6 @@ ggml_tensor * llama_model_plamo2::graph::build_plamo2_mamba_layer(llm_graph_inpu n_heads * head_dim * n_seq_tokens * n_seqs * ggml_element_size(y_ssm)), ggml_view_1d(ctx0, ssm_states_all, n_heads * head_dim * d_state * n_seqs, kv_head * n_seqs * n_heads * head_dim * d_state * ggml_element_size(ssm_states_all)))); - cb(ssm_states_all, "mamba_ssm_states", il); ggml_tensor * y = ggml_view_4d(ctx0, y_ssm, head_dim, n_heads, n_seq_tokens, n_seqs, head_dim * ggml_element_size(x), head_dim * n_heads * ggml_element_size(x), diff --git a/src/models/plamo3.cpp b/src/models/plamo3.cpp index 4dcbcad81a71..d6de1a356446 100644 --- a/src/models/plamo3.cpp +++ b/src/models/plamo3.cpp @@ -59,14 +59,14 @@ void llama_model_plamo3::load_arch_tensors(llama_model_loader &) { std::unique_ptr llama_model_plamo3::build_arch_graph(const llm_graph_params & params) const { if (hparams.swa_type != LLAMA_SWA_TYPE_NONE) { - return std::make_unique> (*this, params); + return std::make_unique> (params); } else { - return std::make_unique>(*this, params); + return std::make_unique>(params); } } template -llama_model_plamo3::graph::graph(const llama_model & model, const llm_graph_params & params) : +llama_model_plamo3::graph::graph(const llm_graph_params & params) : llm_graph_context(params) { const int64_t head_dim_q = hparams.n_embd_head_k(); const int64_t head_dim_v = hparams.n_embd_head_v(); @@ -89,16 +89,6 @@ llama_model_plamo3::graph::graph(const llama_model & model, const llm_grap for (int il = 0; il < n_layer; ++il) { ggml_tensor * residual = inpL; - float freq_base_l = 0.0f; - float freq_scale_l = 0.0f; - if constexpr (iswa) { - freq_base_l = model.get_rope_freq_base (cparams, il); - freq_scale_l = model.get_rope_freq_scale(cparams, il); - } else { - freq_base_l = freq_base; - freq_scale_l = freq_scale; - } - cur = build_norm(inpL, model.layers[il].attn_norm, NULL, LLM_NORM_RMS, il); cb(cur, "attn_norm", il); @@ -128,12 +118,8 @@ llama_model_plamo3::graph::graph(const llama_model & model, const llm_grap Kcur = build_norm(Kcur, model.layers[il].attn_k_norm, NULL, LLM_NORM_RMS, il); cb(Kcur, "attn_k_norm", il); - Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base_l, freq_scale_l, - ext_factor, attn_factor, beta_fast, beta_slow); - Kcur = ggml_rope_ext(ctx0, Kcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base_l, freq_scale_l, - ext_factor, attn_factor, beta_fast, beta_slow); + Qcur = build_rope(Qcur, inp_pos, il); + Kcur = build_rope(Kcur, inp_pos, il); const float attn_scale = 1.0f / sqrtf(float(head_dim_q)); diff --git a/src/models/plm.cpp b/src/models/plm.cpp index 64528c3727fa..b0069e49b954 100644 --- a/src/models/plm.cpp +++ b/src/models/plm.cpp @@ -42,10 +42,10 @@ void llama_model_plm::load_arch_tensors(llama_model_loader &) { } std::unique_ptr llama_model_plm::build_arch_graph(const llm_graph_params & params) const { - return std::make_unique(*this, params); + return std::make_unique(params); } -llama_model_plm::graph::graph(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) { +llama_model_plm::graph::graph(const llm_graph_params & params) : llm_graph_context(params) { const float kq_scale = 1.0f/sqrtf(float(hparams.n_embd_head_k())); const uint32_t n_embd_head_qk_rope = hparams.n_rot(); @@ -133,20 +133,12 @@ llama_model_plm::graph::graph(const llama_model & model, const llm_graph_params 0); cb(v_states, "v_states", il); - q = ggml_rope_ext( - ctx0, q, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); + q = build_rope(q, inp_pos, il); q = ggml_rope_set_offset(q, n_embd_head_qk_nope); cb(q, "q_rope", il); // shared RoPE key - k_pe = ggml_rope_ext( - ctx0, k_pe, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); + k_pe = build_rope(k_pe, inp_pos, il); cb(k_pe, "k_pe", il); ggml_tensor * q_states = q; diff --git a/src/models/pockettts.cpp b/src/models/pockettts.cpp index ad1835893aad..a7dcb5366a12 100644 --- a/src/models/pockettts.cpp +++ b/src/models/pockettts.cpp @@ -42,10 +42,10 @@ void llama_model_pockettts::load_arch_tensors(llama_model_loader &) { } std::unique_ptr llama_model_pockettts::build_arch_graph(const llm_graph_params & params) const { - return std::make_unique(*this, params); + return std::make_unique(params); } -llama_model_pockettts::graph::graph(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) { +llama_model_pockettts::graph::graph(const llm_graph_params & params) : llm_graph_context(params) { const int64_t n_embd_head = hparams.n_embd_head_v(); GGML_ASSERT(n_embd_head == hparams.n_embd_head_k()); @@ -74,17 +74,8 @@ llama_model_pockettts::graph::graph(const llama_model & model, const llm_graph_p auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur, n_embd_head, n_head, n_head_kv, il); - Qcur = ggml_rope_ext( - ctx0, Qcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); - - Kcur = ggml_rope_ext( - ctx0, Kcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); + Qcur = build_rope(Qcur, inp_pos, il); + Kcur = build_rope(Kcur, inp_pos, il); cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); diff --git a/src/models/qwen.cpp b/src/models/qwen.cpp index 3c42cad3cbe0..4e0b1986da88 100644 --- a/src/models/qwen.cpp +++ b/src/models/qwen.cpp @@ -37,10 +37,10 @@ void llama_model_qwen::load_arch_tensors(llama_model_loader &) { } std::unique_ptr llama_model_qwen::build_arch_graph(const llm_graph_params & params) const { - return std::make_unique(*this, params); + return std::make_unique(params); } -llama_model_qwen::graph::graph(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) { +llama_model_qwen::graph::graph(const llm_graph_params & params) : llm_graph_context(params) { const int64_t n_embd_head = hparams.n_embd_head_v(); GGML_ASSERT(n_embd_head == hparams.n_embd_head_k()); @@ -71,17 +71,8 @@ llama_model_qwen::graph::graph(const llama_model & model, const llm_graph_params n_embd_head, n_head, n_head_kv, il); // using mode = 2 for neox mode - Qcur = ggml_rope_ext( - ctx0, Qcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); - - Kcur = ggml_rope_ext( - ctx0, Kcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); + Qcur = build_rope(Qcur, inp_pos, il); + Kcur = build_rope(Kcur, inp_pos, il); cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); diff --git a/src/models/qwen2.cpp b/src/models/qwen2.cpp index d2cd8e22dd81..bb93ca9b153c 100644 --- a/src/models/qwen2.cpp +++ b/src/models/qwen2.cpp @@ -47,10 +47,10 @@ void llama_model_qwen2::load_arch_tensors(llama_model_loader &) { } std::unique_ptr llama_model_qwen2::build_arch_graph(const llm_graph_params & params) const { - return std::make_unique(*this, params); + return std::make_unique(params); } -llama_model_qwen2::graph::graph(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) { +llama_model_qwen2::graph::graph(const llm_graph_params & params) : llm_graph_context(params) { const int64_t n_embd_head = hparams.n_embd_head_v(); GGML_ASSERT(n_embd_head == hparams.n_embd_head_k()); @@ -83,17 +83,8 @@ llama_model_qwen2::graph::graph(const llama_model & model, const llm_graph_param auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur, n_embd_head, n_head, n_head_kv, il); - Qcur = ggml_rope_ext( - ctx0, Qcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); - - Kcur = ggml_rope_ext( - ctx0, Kcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); + Qcur = build_rope(Qcur, inp_pos, il); + Kcur = build_rope(Kcur, inp_pos, il); cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); @@ -117,9 +108,9 @@ llama_model_qwen2::graph::graph(const llama_model & model, const llm_graph_param cb(cur, "ffn_norm", il); cur = build_ffn(cur, - model.layers[il].ffn_up, NULL, NULL, - model.layers[il].ffn_gate, NULL, NULL, - model.layers[il].ffn_down, NULL, NULL, + model.layers[il].ffn_up, NULL, model.layers[il].ffn_up_s, + model.layers[il].ffn_gate, NULL, model.layers[il].ffn_gate_s, + model.layers[il].ffn_down, NULL, model.layers[il].ffn_down_s, NULL, LLM_FFN_SILU, LLM_FFN_PAR, il); cb(cur, "ffn_out", il); diff --git a/src/models/qwen2moe.cpp b/src/models/qwen2moe.cpp index 5f9222f1ce42..e24919bc679c 100644 --- a/src/models/qwen2moe.cpp +++ b/src/models/qwen2moe.cpp @@ -59,10 +59,10 @@ void llama_model_qwen2moe::load_arch_tensors(llama_model_loader &) { } std::unique_ptr llama_model_qwen2moe::build_arch_graph(const llm_graph_params & params) const { - return std::make_unique(*this, params); + return std::make_unique(params); } -llama_model_qwen2moe::graph::graph(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) { +llama_model_qwen2moe::graph::graph(const llm_graph_params & params) : llm_graph_context(params) { const int64_t n_embd_head = hparams.n_embd_head_v(); GGML_ASSERT(n_embd_head == hparams.n_embd_head_k()); @@ -95,17 +95,8 @@ llama_model_qwen2moe::graph::graph(const llama_model & model, const llm_graph_pa auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur, n_embd_head, n_head, n_head_kv, il); - Qcur = ggml_rope_ext( - ctx0, Qcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); - - Kcur = ggml_rope_ext( - ctx0, Kcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); + Qcur = build_rope(Qcur, inp_pos, il); + Kcur = build_rope(Kcur, inp_pos, il); cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); diff --git a/src/models/qwen2vl.cpp b/src/models/qwen2vl.cpp index 778d1c80a3bf..5831fc435ea7 100644 --- a/src/models/qwen2vl.cpp +++ b/src/models/qwen2vl.cpp @@ -36,10 +36,10 @@ void llama_model_qwen2vl::load_arch_tensors(llama_model_loader &) { } std::unique_ptr llama_model_qwen2vl::build_arch_graph(const llm_graph_params & params) const { - return std::make_unique(*this, params); + return std::make_unique(params); } -llama_model_qwen2vl::graph::graph(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) { +llama_model_qwen2vl::graph::graph(const llm_graph_params & params) : llm_graph_context(params) { const int64_t n_embd_head = hparams.n_embd_head_v(); GGML_ASSERT(n_embd_head == hparams.n_embd_head_k()); @@ -55,9 +55,6 @@ llama_model_qwen2vl::graph::graph(const llama_model & model, const llm_graph_par auto * inp_attn = build_attn_inp_kv(); - int sections[4]; - std::copy(std::begin(hparams.rope_sections), std::begin(hparams.rope_sections) + 4, sections); - ggml_tensor * inp_out_ids = build_inp_out_ids(); for (int il = 0; il < n_layer; ++il) { @@ -75,17 +72,8 @@ llama_model_qwen2vl::graph::graph(const llama_model & model, const llm_graph_par auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur, n_embd_head, n_head, n_head_kv, il); - Qcur = ggml_rope_multi( - ctx0, Qcur, inp_pos, nullptr, - n_rot, sections, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); - - Kcur = ggml_rope_multi( - ctx0, Kcur, inp_pos, nullptr, - n_rot, sections, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); + Qcur = build_rope(Qcur, inp_pos, il); + Kcur = build_rope(Kcur, inp_pos, il); cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); diff --git a/src/models/qwen3.cpp b/src/models/qwen3.cpp index c01f8382921b..7a6052970d8c 100644 --- a/src/models/qwen3.cpp +++ b/src/models/qwen3.cpp @@ -51,11 +51,11 @@ void llama_model_qwen3::load_arch_tensors(llama_model_loader &) { } std::unique_ptr llama_model_qwen3::build_arch_graph(const llm_graph_params & params) const { - return std::make_unique(*this, params); + return std::make_unique(params); } -llama_model_qwen3::graph::graph(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) { - ggml_tensor * cur = build_backbone(model); +llama_model_qwen3::graph::graph(const llm_graph_params & params) : llm_graph_context(params) { + ggml_tensor * cur = build_backbone(); // lm_head cur = build_lora_mm(model.output, cur, model.output_s); @@ -64,7 +64,7 @@ llama_model_qwen3::graph::graph(const llama_model & model, const llm_graph_param res->t_logits = cur; } -ggml_tensor * llama_model_qwen3::graph::build_backbone(const llama_model & model) { +ggml_tensor * llama_model_qwen3::graph::build_backbone() { const int64_t n_embd_head = hparams.n_embd_head_v(); GGML_ASSERT(n_embd_head == hparams.n_embd_head_k()); @@ -102,20 +102,12 @@ ggml_tensor * llama_model_qwen3::graph::build_backbone(const llama_model & model Qcur = build_norm(Qcur, model.layers[il].attn_q_norm, NULL, LLM_NORM_RMS, il); cb(Qcur, "Qcur_normed", il); - Qcur = ggml_rope_ext( - ctx0, Qcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); + Qcur = build_rope(Qcur, inp_pos, il); Kcur = build_norm(Kcur, model.layers[il].attn_k_norm, NULL, LLM_NORM_RMS, il); cb(Kcur, "Kcur_normed", il); - Kcur = ggml_rope_ext( - ctx0, Kcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); + Kcur = build_rope(Kcur, inp_pos, il); cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); diff --git a/src/models/qwen35.cpp b/src/models/qwen35.cpp index 8a4e434117fa..5b05fc63fdd9 100644 --- a/src/models/qwen35.cpp +++ b/src/models/qwen35.cpp @@ -78,13 +78,20 @@ void llama_model_qwen35::load_arch_tensors(llama_model_loader & ml) { } else { // Linear attention (gated delta net) specific tensors // Create tensors with calculated dimensions - layer.wqkv = create_tensor(tn(LLM_TENSOR_ATTN_QKV, "weight", il), { n_embd, key_dim * 2 + value_dim }, TENSOR_NOT_REQUIRED); - layer.wqkv_gate = create_tensor(tn(LLM_TENSOR_ATTN_GATE, "weight", il), { n_embd, value_dim }, TENSOR_NOT_REQUIRED); + const int separate_flags = flags & TENSOR_SKIP ? flags | TENSOR_NOT_REQUIRED : flags; + layer.wqkv_z = create_tensor(tn(LLM_TENSOR_ATTN_QKV_GATE, "weight", il), { n_embd, key_dim * 2 + value_dim * 2 }, flags | TENSOR_NOT_REQUIRED | TENSOR_SKIP_IF_VIRTUAL); + if (!layer.wqkv_z) { + layer.wqkv = create_tensor(tn(LLM_TENSOR_ATTN_QKV, "weight", il), { n_embd, key_dim * 2 + value_dim }, separate_flags); + layer.wqkv_gate = create_tensor(tn(LLM_TENSOR_ATTN_GATE, "weight", il), { n_embd, value_dim }, separate_flags); + } layer.ssm_conv1d = create_tensor(tn(LLM_TENSOR_SSM_CONV1D, "weight", il), { hparams.ssm_d_conv, conv_dim }, flags); layer.ssm_dt = create_tensor(tn(LLM_TENSOR_SSM_DT, "bias", il), { hparams.ssm_dt_rank }, flags); layer.ssm_a = create_tensor(tn(LLM_TENSOR_SSM_A_NOSCAN, il), { hparams.ssm_dt_rank }, flags); - layer.ssm_beta = create_tensor(tn(LLM_TENSOR_SSM_BETA, "weight", il), { n_embd, n_v_heads }, flags); - layer.ssm_alpha = create_tensor(tn(LLM_TENSOR_SSM_ALPHA, "weight", il), { n_embd, n_v_heads }, flags); + layer.ssm_beta_alpha_cat = create_tensor(tn(LLM_TENSOR_SSM_BETA_ALPHA_CAT, "weight", il), { n_embd, n_v_heads * 2 }, flags | TENSOR_NOT_REQUIRED | TENSOR_SKIP_IF_VIRTUAL); + if (!layer.ssm_beta_alpha_cat) { + layer.ssm_beta = create_tensor(tn(LLM_TENSOR_SSM_BETA, "weight", il), { n_embd, n_v_heads }, separate_flags); + layer.ssm_alpha = create_tensor(tn(LLM_TENSOR_SSM_ALPHA, "weight", il), { n_embd, n_v_heads }, separate_flags); + } layer.ssm_norm = create_tensor(tn(LLM_TENSOR_SSM_NORM, "weight", il), { head_v_dim }, flags); layer.ssm_out = create_tensor(tn(LLM_TENSOR_SSM_OUT, "weight", il), { value_dim, n_embd }, flags); } @@ -129,20 +136,17 @@ void llama_model_qwen35::load_arch_tensors(llama_model_loader & ml) { std::unique_ptr llama_model_qwen35::build_arch_graph(const llm_graph_params & params) const { if (params.gtype == LLM_GRAPH_TYPE_DECODER_MTP) { - return std::make_unique(*this, params); + return std::make_unique(params); } - return std::make_unique(*this, params); + return std::make_unique(params); } -llama_model_qwen35::graph::graph(const llama_model & model, const llm_graph_params & params) : - llm_build_delta_net_base(params), model(model) { +llama_model_qwen35::graph::graph(const llm_graph_params & params) : + llm_build_delta_net_base(params) { const int64_t n_embd_head = hparams.n_embd_head_v(); GGML_ASSERT(n_embd_head == hparams.n_embd_head_k()); - int sections[4]; - std::copy(std::begin(hparams.rope_sections), std::begin(hparams.rope_sections) + 4, sections); - ggml_tensor * cur; ggml_tensor * inpL; @@ -172,7 +176,7 @@ llama_model_qwen35::graph::graph(const llama_model & model, const llm_graph_para cur = build_layer_attn_linear(inp->get_recr(), cur, il); } else { // Full attention layer - cur = build_layer_attn(inp->get_attn(), cur, inp_pos, sections, il); + cur = build_layer_attn(inp->get_attn(), cur, inp_pos, il); } if (il == n_layer - 1 && crop_before_nextn(inp_out_ids)) { @@ -242,6 +246,22 @@ std::pair llama_model_qwen35::graph::build_qkvz( const int64_t n_seqs = ubatch.n_seqs; const int64_t n_seq_tokens = ubatch.n_seq_tokens; + if (model.layers[il].wqkv_z) { + ggml_tensor * qkvz = build_lora_mm(model.layers[il].wqkv_z, input); + cb(qkvz, "linear_attn_qkvz", il); + + const int64_t n_z = hparams.ssm_d_state * hparams.ssm_dt_rank; + const int64_t n_qkv = qkvz->ne[0] - n_z; + + ggml_tensor * qkv_mixed = ggml_view_3d(ctx0, qkvz, n_qkv, n_seq_tokens, n_seqs, qkvz->nb[1], qkvz->nb[1] * n_seq_tokens, 0); + cb(qkv_mixed, "linear_attn_qkv_mixed", il); + + ggml_tensor * z = ggml_view_2d(ctx0, qkvz, n_z, n_seq_tokens * n_seqs, qkvz->nb[1], ggml_row_size(qkvz->type, n_qkv)); + cb(z, "z", il); + + return { qkv_mixed, z }; + } + ggml_tensor * qkv_mixed = build_lora_mm(model.layers[il].wqkv, input, model.layers[il].wqkv_s); qkv_mixed = ggml_reshape_3d(ctx0, qkv_mixed, qkv_mixed->ne[0], n_seq_tokens, n_seqs); cb(qkv_mixed, "linear_attn_qkv_mixed", il); @@ -267,7 +287,6 @@ ggml_tensor * llama_model_qwen35::graph::build_layer_attn( llm_graph_input_attn_kv * inp, ggml_tensor * cur, ggml_tensor * inp_pos, - int * sections, int il) { const int64_t n_embd_head = hparams.n_embd_head_v(); GGML_ASSERT(n_embd_head == hparams.n_embd_head_k()); @@ -308,17 +327,8 @@ ggml_tensor * llama_model_qwen35::graph::build_layer_attn( Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens); // Apply MRoPE - Qcur = ggml_rope_multi( - ctx0, Qcur, inp_pos, nullptr, - n_rot, sections, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); - - Kcur = ggml_rope_multi( - ctx0, Kcur, inp_pos, nullptr, - n_rot, sections, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); + Qcur = build_rope(Qcur, inp_pos, il); + Kcur = build_rope(Kcur, inp_pos, il); cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); @@ -367,17 +377,29 @@ ggml_tensor * llama_model_qwen35::graph::build_layer_attn_linear( ggml_tensor * qkv_mixed = qkvz.first; ggml_tensor * z = qkvz.second; - ggml_tensor * beta = build_lora_mm(model.layers[il].ssm_beta, cur, model.layers[il].ssm_beta_s); - beta = ggml_reshape_4d(ctx0, beta, 1, num_v_heads, n_seq_tokens, n_seqs); + ggml_tensor * beta = nullptr; + ggml_tensor * alpha = nullptr; + if (model.layers[il].ssm_beta_alpha_cat) { + ggml_tensor * ba = build_lora_mm(model.layers[il].ssm_beta_alpha_cat, cur); + cb(ba, "beta_alpha", il); + + beta = ggml_view_4d(ctx0, ba, 1, num_v_heads, n_seq_tokens, n_seqs, + ggml_element_size(ba), ba->nb[1], ba->nb[1] * n_seq_tokens, 0); + alpha = ggml_view_3d(ctx0, ba, num_v_heads, n_seq_tokens, n_seqs, + ba->nb[1], ba->nb[1] * n_seq_tokens, ggml_row_size(ba->type, num_v_heads)); + } else { + beta = build_lora_mm(model.layers[il].ssm_beta, cur, model.layers[il].ssm_beta_s); + beta = ggml_reshape_4d(ctx0, beta, 1, num_v_heads, n_seq_tokens, n_seqs); + + alpha = build_lora_mm(model.layers[il].ssm_alpha, cur, model.layers[il].ssm_alpha_s); + alpha = ggml_reshape_3d(ctx0, alpha, num_v_heads, n_seq_tokens, n_seqs); + } cb(beta, "beta", il); + cb(alpha, "alpha", il); beta = ggml_sigmoid(ctx0, beta); cb(beta, "beta_sigmoid", il); - ggml_tensor * alpha = build_lora_mm(model.layers[il].ssm_alpha, cur, model.layers[il].ssm_alpha_s); - alpha = ggml_reshape_3d(ctx0, alpha, num_v_heads, n_seq_tokens, n_seqs); - cb(alpha, "alpha", il); - ggml_tensor * alpha_biased = ggml_add(ctx0, alpha, model.layers[il].ssm_dt); ggml_tensor * alpha_softplus = ggml_softplus(ctx0, alpha_biased); cb(alpha_softplus, "a_softplus", il); @@ -445,14 +467,6 @@ ggml_tensor * llama_model_qwen35::graph::build_layer_attn_linear( //k_conv = ggml_cont_4d(ctx0, k_conv, head_k_dim, num_k_heads, n_seq_tokens, n_seqs); //v_conv = ggml_cont_4d(ctx0, v_conv, head_v_dim, num_v_heads, n_seq_tokens, n_seqs); - // if head keys and value keys are different, repeat to force tensors into matching shapes - // note: need explicit repeat only if we are not using the fused GDN. - if (num_k_heads != num_v_heads && (!cparams.fused_gdn_ar || !cparams.fused_gdn_ch)) { - GGML_ASSERT(num_v_heads % num_k_heads == 0); - q_conv = ggml_repeat_4d(ctx0, q_conv, head_k_dim, num_v_heads, n_seq_tokens, n_seqs); - k_conv = ggml_repeat_4d(ctx0, k_conv, head_k_dim, num_v_heads, n_seq_tokens, n_seqs); - } - cb(q_conv, "q_conv_predelta", il); cb(k_conv, "k_conv_predelta", il); cb(v_conv, "v_conv_predelta", il); @@ -460,7 +474,8 @@ ggml_tensor * llama_model_qwen35::graph::build_layer_attn_linear( ggml_tensor * output = build_recurrent_attn(inp, ssm_states_all, q_conv, k_conv, v_conv, gate, beta, state, il); // z: [head_dim, n_heads, n_tokens, n_seqs] -> [n_heads * n_tokens * n_seqs, head_dim] - ggml_tensor * z_2d = ggml_reshape_4d(ctx0, z, head_v_dim, num_v_heads, n_seq_tokens, n_seqs); + ggml_tensor * z_2d = ggml_view_4d(ctx0, z, head_v_dim, num_v_heads, n_seq_tokens, n_seqs, + ggml_row_size(z->type, head_v_dim), z->nb[1], z->nb[1] * n_seq_tokens, 0); // Apply gated normalization: self.norm(core_attn_out, z) ggml_tensor * attn_out_norm = build_norm_gated(output, model.layers[il].ssm_norm, z_2d, il); @@ -492,7 +507,7 @@ ggml_tensor * llama_model_qwen35::graph::build_layer_ffn(ggml_tensor * cur, cons } // LLM_GRAPH_TYPE_DECODER_MTP draft head for Qwen3.5/3.6 dense series -llama_model_qwen35::graph_mtp::graph_mtp(const llama_model & model, const llm_graph_params & params) +llama_model_qwen35::graph_mtp::graph_mtp(const llm_graph_params & params) : llm_graph_context(params) { GGML_ASSERT(hparams.n_layer_nextn > 0 && "QWEN35 MTP requires n_layer_nextn > 0"); GGML_ASSERT(hparams.n_layer_nextn == 1 && "QWEN35 MTP currently only supports a single MTP block"); @@ -509,9 +524,6 @@ llama_model_qwen35::graph_mtp::graph_mtp(const llama_model & model, const llm_gr GGML_ASSERT(layer.nextn.enorm && "MTP block missing nextn.enorm"); GGML_ASSERT(layer.nextn.hnorm && "MTP block missing nextn.hnorm"); - int sections[4]; - std::copy(std::begin(hparams.rope_sections), std::begin(hparams.rope_sections) + 4, sections); - // TODO: extract in a common llm_graph_context::build_inp_embd_h() auto inp = std::make_unique(hparams.n_embd_inp(), hparams.n_embd); @@ -544,9 +556,7 @@ llama_model_qwen35::graph_mtp::graph_mtp(const llama_model & model, const llm_gr } cb(tok_embd, "mtp_tok_embd", il); - inp->h = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, hparams.n_embd, n_tokens); - ggml_set_input(inp->h); - ggml_set_name(inp->h, "mtp_h_input"); + inp->h = build_inp_embd_state(inp.get(), hparams.n_embd); ggml_tensor * h_embd = inp->h; @@ -604,12 +614,8 @@ llama_model_qwen35::graph_mtp::graph_mtp(const llama_model & model, const llm_gr Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens); cb(Vcur, "mtp_Vcur", il); - Qcur = ggml_rope_multi(ctx0, Qcur, inp_pos, nullptr, - n_rot, sections, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); - Kcur = ggml_rope_multi(ctx0, Kcur, inp_pos, nullptr, - n_rot, sections, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); + Qcur = build_rope(Qcur, inp_pos, il); + Kcur = build_rope(Kcur, inp_pos, il); const float kq_scale = hparams.f_attention_scale == 0.0f ? 1.0f / sqrtf(float(n_embd_head)) : hparams.f_attention_scale; diff --git a/src/models/qwen35moe.cpp b/src/models/qwen35moe.cpp index 939f10f7f061..c0ee6fa03fc8 100644 --- a/src/models/qwen35moe.cpp +++ b/src/models/qwen35moe.cpp @@ -148,20 +148,17 @@ void llama_model_qwen35moe::load_arch_tensors(llama_model_loader & ml) { std::unique_ptr llama_model_qwen35moe::build_arch_graph(const llm_graph_params & params) const { if (params.gtype == LLM_GRAPH_TYPE_DECODER_MTP) { - return std::make_unique(*this, params); + return std::make_unique(params); } - return std::make_unique(*this, params); + return std::make_unique(params); } -llama_model_qwen35moe::graph::graph(const llama_model & model, const llm_graph_params & params) : - llm_build_delta_net_base(params), model(model) { +llama_model_qwen35moe::graph::graph(const llm_graph_params & params) : + llm_build_delta_net_base(params) { const int64_t n_embd_head = hparams.n_embd_head_v(); GGML_ASSERT(n_embd_head == hparams.n_embd_head_k()); - int sections[4]; - std::copy(std::begin(hparams.rope_sections), std::begin(hparams.rope_sections) + 4, sections); - ggml_tensor * cur; ggml_tensor * inpL; @@ -191,7 +188,7 @@ llama_model_qwen35moe::graph::graph(const llama_model & model, const llm_graph_p cur = build_layer_attn_linear(inp->get_recr(), cur, il); } else { // Full attention layer - cur = build_layer_attn(inp->get_attn(), cur, inp_pos, sections, il); + cur = build_layer_attn(inp->get_attn(), cur, inp_pos, il); } if (il == n_layer - 1 && crop_before_nextn(inp_out_ids)) { @@ -277,7 +274,6 @@ ggml_tensor * llama_model_qwen35moe::graph::build_layer_attn( llm_graph_input_attn_kv * inp, ggml_tensor * cur, ggml_tensor * inp_pos, - int * sections, int il) { const int64_t n_embd_head = hparams.n_embd_head_v(); GGML_ASSERT(n_embd_head == hparams.n_embd_head_k()); @@ -318,17 +314,8 @@ ggml_tensor * llama_model_qwen35moe::graph::build_layer_attn( Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens); // Apply IMRoPE - Qcur = ggml_rope_multi( - ctx0, Qcur, inp_pos, nullptr, - n_rot, sections, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); - - Kcur = ggml_rope_multi( - ctx0, Kcur, inp_pos, nullptr, - n_rot, sections, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); + Qcur = build_rope(Qcur, inp_pos, il); + Kcur = build_rope(Kcur, inp_pos, il); cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); @@ -455,14 +442,6 @@ ggml_tensor * llama_model_qwen35moe::graph::build_layer_attn_linear( //k_conv = ggml_cont_4d(ctx0, k_conv, head_k_dim, num_k_heads, n_seq_tokens, n_seqs); //v_conv = ggml_cont_4d(ctx0, v_conv, head_v_dim, num_v_heads, n_seq_tokens, n_seqs); - // if head keys and value keys are different, repeat to force tensors into matching shapes - // note: need explicit repeat only if we are not using the fused GDN. - if (num_k_heads != num_v_heads && (!cparams.fused_gdn_ar || !cparams.fused_gdn_ch)) { - GGML_ASSERT(num_v_heads % num_k_heads == 0); - q_conv = ggml_repeat_4d(ctx0, q_conv, head_k_dim, num_v_heads, n_seq_tokens, n_seqs); - k_conv = ggml_repeat_4d(ctx0, k_conv, head_k_dim, num_v_heads, n_seq_tokens, n_seqs); - } - cb(q_conv, "q_conv_predelta", il); cb(k_conv, "k_conv_predelta", il); cb(v_conv, "v_conv_predelta", il); @@ -542,7 +521,7 @@ ggml_tensor * llama_model_qwen35moe::graph::build_layer_ffn(ggml_tensor * cur, c } // LLM_GRAPH_TYPE_DECODER_MTP draft head for Qwen3.5/3.6 MoE -llama_model_qwen35moe::graph_mtp::graph_mtp(const llama_model & model, const llm_graph_params & params) +llama_model_qwen35moe::graph_mtp::graph_mtp(const llm_graph_params & params) : llm_graph_context(params) { GGML_ASSERT(hparams.n_layer_nextn > 0 && "QWEN35MOE MTP requires n_layer_nextn > 0"); GGML_ASSERT(hparams.n_layer_nextn == 1 && "QWEN35MOE MTP currently only supports a single MTP block"); @@ -558,9 +537,6 @@ llama_model_qwen35moe::graph_mtp::graph_mtp(const llama_model & model, const llm GGML_ASSERT(layer.nextn.hnorm && "MTP block missing nextn.hnorm"); GGML_ASSERT(layer.ffn_gate_inp && "MTP block missing ffn_gate_inp"); - int sections[4]; - std::copy(std::begin(hparams.rope_sections), std::begin(hparams.rope_sections) + 4, sections); - // TODO: extract in a common llm_graph_context::build_inp_embd_h() auto inp = std::make_unique(hparams.n_embd_inp(), hparams.n_embd); @@ -583,9 +559,7 @@ llama_model_qwen35moe::graph_mtp::graph_mtp(const llama_model & model, const llm } cb(tok_embd, "mtp_tok_embd", il); - inp->h = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, hparams.n_embd, n_tokens); - ggml_set_input(inp->h); - ggml_set_name(inp->h, "mtp_h_input"); + inp->h = build_inp_embd_state(inp.get(), hparams.n_embd); ggml_tensor * h_embd = inp->h; @@ -643,12 +617,8 @@ llama_model_qwen35moe::graph_mtp::graph_mtp(const llama_model & model, const llm Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens); cb(Vcur, "mtp_Vcur", il); - Qcur = ggml_rope_multi(ctx0, Qcur, inp_pos, nullptr, - n_rot, sections, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); - Kcur = ggml_rope_multi(ctx0, Kcur, inp_pos, nullptr, - n_rot, sections, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); + Qcur = build_rope(Qcur, inp_pos, il); + Kcur = build_rope(Kcur, inp_pos, il); const float kq_scale = hparams.f_attention_scale == 0.0f ? 1.0f / sqrtf(float(n_embd_head)) : hparams.f_attention_scale; diff --git a/src/models/qwen3moe.cpp b/src/models/qwen3moe.cpp index 2198fcfb5a55..ce7144a229b3 100644 --- a/src/models/qwen3moe.cpp +++ b/src/models/qwen3moe.cpp @@ -56,10 +56,10 @@ void llama_model_qwen3moe::load_arch_tensors(llama_model_loader &) { } std::unique_ptr llama_model_qwen3moe::build_arch_graph(const llm_graph_params & params) const { - return std::make_unique(*this, params); + return std::make_unique(params); } -llama_model_qwen3moe::graph::graph(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) { +llama_model_qwen3moe::graph::graph(const llm_graph_params & params) : llm_graph_context(params) { const int64_t n_embd_head = hparams.n_embd_head_v(); GGML_ASSERT(n_embd_head == hparams.n_embd_head_k()); @@ -97,20 +97,12 @@ llama_model_qwen3moe::graph::graph(const llama_model & model, const llm_graph_pa Qcur = build_norm(Qcur, model.layers[il].attn_q_norm, NULL, LLM_NORM_RMS, il); cb(Qcur, "Qcur_normed", il); - Qcur = ggml_rope_ext( - ctx0, Qcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); + Qcur = build_rope(Qcur, inp_pos, il); Kcur = build_norm(Kcur, model.layers[il].attn_k_norm, NULL, LLM_NORM_RMS, il); cb(Kcur, "Kcur_normed", il); - Kcur = ggml_rope_ext( - ctx0, Kcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); + Kcur = build_rope(Kcur, inp_pos, il); cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); diff --git a/src/models/qwen3next.cpp b/src/models/qwen3next.cpp index fd082cc7f216..110581baec0a 100644 --- a/src/models/qwen3next.cpp +++ b/src/models/qwen3next.cpp @@ -130,13 +130,13 @@ void llama_model_qwen3next::load_arch_tensors(llama_model_loader & ml) { std::unique_ptr llama_model_qwen3next::build_arch_graph(const llm_graph_params & params) const { if (params.gtype == LLM_GRAPH_TYPE_DECODER_MTP) { - return std::make_unique(*this, params); + return std::make_unique(params); } - return std::make_unique(*this, params); + return std::make_unique(params); } -llama_model_qwen3next::graph::graph(const llama_model & model, const llm_graph_params & params) : - llm_build_delta_net_base(params), model(model) { +llama_model_qwen3next::graph::graph(const llm_graph_params & params) : + llm_build_delta_net_base(params) { ggml_tensor * cur; ggml_tensor * inpL; @@ -273,15 +273,8 @@ ggml_tensor * llama_model_qwen3next::graph::build_layer_attn( Kcur = build_norm(Kcur, model.layers[il].attn_k_norm, nullptr, LLM_NORM_RMS, il); cb(Kcur, "Kcur_normed", il); - Qcur = ggml_rope_ext( - ctx0, Qcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); - - Kcur = ggml_rope_ext( - ctx0, Kcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, - freq_scale, ext_factor, attn_factor, beta_fast, beta_slow); + Qcur = build_rope(Qcur, inp_pos, il); + Kcur = build_rope(Kcur, inp_pos, il); cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); @@ -616,7 +609,7 @@ ggml_tensor * llama_model_qwen3next::graph::build_layer_ffn(ggml_tensor * cur, c } // LLM_GRAPH_TYPE_DECODER_MTP draft head for Qwen3-Next -llama_model_qwen3next::graph_mtp::graph_mtp(const llama_model & model, const llm_graph_params & params) +llama_model_qwen3next::graph_mtp::graph_mtp(const llm_graph_params & params) : llm_graph_context(params) { GGML_ASSERT(hparams.n_layer_nextn > 0 && "QWEN3NEXT MTP requires n_layer_nextn > 0"); GGML_ASSERT(hparams.n_layer_nextn == 1 && "QWEN3NEXT MTP currently only supports a single MTP block"); @@ -654,9 +647,7 @@ llama_model_qwen3next::graph_mtp::graph_mtp(const llama_model & model, const llm } cb(tok_embd, "mtp_tok_embd", il); - inp->h = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, hparams.n_embd, n_tokens); - ggml_set_input(inp->h); - ggml_set_name(inp->h, "mtp_h_input"); + inp->h = build_inp_embd_state(inp.get(), hparams.n_embd); ggml_tensor * h_embd = inp->h; @@ -705,12 +696,8 @@ llama_model_qwen3next::graph_mtp::graph_mtp(const llama_model & model, const llm Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens); - Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); - Kcur = ggml_rope_ext(ctx0, Kcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); + Qcur = build_rope(Qcur, inp_pos, il); + Kcur = build_rope(Kcur, inp_pos, il); cb(Qcur, "mtp_Qcur", il); cb(Kcur, "mtp_Kcur", il); diff --git a/src/models/qwen3vl.cpp b/src/models/qwen3vl.cpp index 8c06dcccaec9..480e570dcfb4 100644 --- a/src/models/qwen3vl.cpp +++ b/src/models/qwen3vl.cpp @@ -54,10 +54,10 @@ void llama_model_qwen3vl::load_arch_tensors(llama_model_loader &) { } std::unique_ptr llama_model_qwen3vl::build_arch_graph(const llm_graph_params & params) const { - return std::make_unique(*this, params); + return std::make_unique(params); } -llama_model_qwen3vl::graph::graph(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) { +llama_model_qwen3vl::graph::graph(const llm_graph_params & params) : llm_graph_context(params) { const size_t n_deepstack_layers = hparams.n_deepstack_layers; const int64_t n_embd = hparams.n_embd; @@ -71,9 +71,6 @@ llama_model_qwen3vl::graph::graph(const llama_model & model, const llm_graph_par inpL = build_inp_embd(model.tok_embd); - int sections[4]; - std::copy(std::begin(hparams.rope_sections), std::begin(hparams.rope_sections) + 4, sections); - // inp_pos - contains the positions ggml_tensor * inp_pos = build_inp_pos(); @@ -99,20 +96,12 @@ llama_model_qwen3vl::graph::graph(const llama_model & model, const llm_graph_par Qcur = build_norm(Qcur, model.layers[il].attn_q_norm, NULL, LLM_NORM_RMS, il); cb(Qcur, "Qcur_normed", il); - Qcur = ggml_rope_multi( - ctx0, Qcur, inp_pos, nullptr, - n_rot, sections, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); + Qcur = build_rope(Qcur, inp_pos, il); Kcur = build_norm(Kcur, model.layers[il].attn_k_norm, NULL, LLM_NORM_RMS, il); cb(Kcur, "Kcur_normed", il); - Kcur = ggml_rope_multi( - ctx0, Kcur, inp_pos, nullptr, - n_rot, sections, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); + Kcur = build_rope(Kcur, inp_pos, il); cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); diff --git a/src/models/qwen3vlmoe.cpp b/src/models/qwen3vlmoe.cpp index 94974d8e84bd..c13f619906ff 100644 --- a/src/models/qwen3vlmoe.cpp +++ b/src/models/qwen3vlmoe.cpp @@ -58,10 +58,10 @@ void llama_model_qwen3vlmoe::load_arch_tensors(llama_model_loader &) { } std::unique_ptr llama_model_qwen3vlmoe::build_arch_graph(const llm_graph_params & params) const { - return std::make_unique(*this, params); + return std::make_unique(params); } -llama_model_qwen3vlmoe::graph::graph(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) { +llama_model_qwen3vlmoe::graph::graph(const llm_graph_params & params) : llm_graph_context(params) { const size_t n_deepstack_layers = hparams.n_deepstack_layers; const int64_t n_embd = hparams.n_embd; @@ -75,9 +75,6 @@ llama_model_qwen3vlmoe::graph::graph(const llama_model & model, const llm_graph_ inpL = build_inp_embd(model.tok_embd); - int sections[4]; - std::copy(std::begin(hparams.rope_sections), std::begin(hparams.rope_sections) + 4, sections); - // inp_pos - contains the positions ggml_tensor * inp_pos = build_inp_pos(); @@ -103,20 +100,12 @@ llama_model_qwen3vlmoe::graph::graph(const llama_model & model, const llm_graph_ Qcur = build_norm(Qcur, model.layers[il].attn_q_norm, NULL, LLM_NORM_RMS, il); cb(Qcur, "Qcur_normed", il); - Qcur = ggml_rope_multi( - ctx0, Qcur, inp_pos, nullptr, - n_rot, sections, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); + Qcur = build_rope(Qcur, inp_pos, il); Kcur = build_norm(Kcur, model.layers[il].attn_k_norm, NULL, LLM_NORM_RMS, il); cb(Kcur, "Kcur_normed", il); - Kcur = ggml_rope_multi( - ctx0, Kcur, inp_pos, nullptr, - n_rot, sections, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); + Kcur = build_rope(Kcur, inp_pos, il); cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); diff --git a/src/models/qwen4exp.cpp b/src/models/qwen4exp.cpp index 60665381c660..181e4dca4047 100644 --- a/src/models/qwen4exp.cpp +++ b/src/models/qwen4exp.cpp @@ -323,9 +323,9 @@ void llama_model_qwen4exp::load_arch_tensors(llama_model_loader & ml) { std::unique_ptr llama_model_qwen4exp::build_arch_graph(const llm_graph_params & params) const { if (params.gtype == LLM_GRAPH_TYPE_DECODER_MTP) { - return std::make_unique(*this, params); + return std::make_unique(params); } - return std::make_unique(*this, params); + return std::make_unique(params); } // DFlash drafters read the layer inputs as [n_embd, T] rows, so collapse the streams @@ -444,15 +444,12 @@ ggml_tensor * llama_model_qwen4exp::graph::build_hc_combine( return cur; } -llama_model_qwen4exp::graph::graph(const llama_model & model, const llm_graph_params & params) : - llm_build_delta_net_base(params), model(model) { +llama_model_qwen4exp::graph::graph(const llm_graph_params & params) : + llm_build_delta_net_base(params) { const int64_t hc = hparams.dsv4_hc_mult; GGML_ASSERT(hparams.n_embd_head_v() == hparams.n_embd_head_k()); - int sections[4]; - std::copy(std::begin(hparams.rope_sections), std::begin(hparams.rope_sections) + 4, sections); - auto * inp = build_inp_mem_hybrid(); // qwen4exp always builds llama_memory_hybrid_idx, so this downcast is safe @@ -519,7 +516,7 @@ llama_model_qwen4exp::graph::graph(const llama_model & model, const llm_graph_pa if (hparams.is_recr(il)) { cur = build_layer_attn_linear(inp->get_recr(), cur, il); } else { - cur = build_layer_attn(inp->get_attn(), mctx_hyb, inp_kpool, cur, inp_pos, sections, il); + cur = build_layer_attn(inp->get_attn(), mctx_hyb, inp_kpool, cur, inp_pos, il); } if (il == n_layer - 1 && crop_before_nextn(inp_out_ids)) { @@ -577,8 +574,8 @@ llama_model_qwen4exp::graph::graph(const llama_model & model, const llm_graph_pa res->t_logits = cur; } -llama_model_qwen4exp::graph_mtp::graph_mtp(const llama_model & model, const llm_graph_params & params) : - graph(model, params, no_build{}) { +llama_model_qwen4exp::graph_mtp::graph_mtp(const llm_graph_params & params) : + graph(params, no_build{}) { GGML_ASSERT(hparams.n_layer_nextn == 1 && "qwen4exp MTP has a single block"); ASSERT_EMBD_OR_TOKEN(ubatch); @@ -591,9 +588,6 @@ llama_model_qwen4exp::graph_mtp::graph_mtp(const llama_model & model, const llm_ GGML_ASSERT(layer.nextn.eh_proj && layer.nextn.enorm && layer.nextn.hnorm && layer.nextn.hc_head_norm && "MTP block missing, load the model with MTP enabled"); - int sections[4]; - std::copy(std::begin(hparams.rope_sections), std::begin(hparams.rope_sections) + 4, sections); - auto inp = std::make_unique(hparams.n_embd_inp(), hparams.n_embd_out()); inp->tokens = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_tokens); @@ -602,9 +596,7 @@ llama_model_qwen4exp::graph_mtp::graph_mtp(const llama_model & model, const llm_ inp->embd = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, hparams.n_embd_inp(), n_tokens); ggml_set_input(inp->embd); - inp->h = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, hparams.n_embd_out(), n_tokens); - ggml_set_input(inp->h); - ggml_set_name(inp->h, "mtp_h_input"); + inp->h = build_inp_embd_state(inp.get(), hparams.n_embd_out()); ggml_tensor * tok_embd; if (ubatch.token) { @@ -646,7 +638,7 @@ llama_model_qwen4exp::graph_mtp::graph_mtp(const llama_model & model, const llm_ hc_mix attn_mix = build_hc_mix(res_hc, layer.hc_attn_norm, layer.hc_attn_down, layer.hc_attn_up, layer.hc_attn_inject, layer.hc_attn_down_inject, il); ggml_tensor * cur = attn_mix.cur; - cur = build_layer_attn(inp_hyb->get_attn(), mctx_hyb, inp_kpool, cur, inp_pos, sections, il); + cur = build_layer_attn(inp_hyb->get_attn(), mctx_hyb, inp_kpool, cur, inp_pos, il); res_hc = build_hc_combine(res_hc, cur, attn_mix.inject, il); hc_mix ffn_mix = build_hc_mix(res_hc, layer.hc_ffn_norm, layer.hc_ffn_down, layer.hc_ffn_up, layer.hc_ffn_inject, layer.hc_ffn_down_inject, il); @@ -818,7 +810,6 @@ ggml_tensor * llama_model_qwen4exp::graph::build_qsa_sel( llm_graph_input_kpool * inp_kpool, ggml_tensor * cur, ggml_tensor * inp_pos, - int * sections, int il) { const llama_kv_cache_context * mctx_idx = mctx_hyb->get_idx(); @@ -855,9 +846,7 @@ ggml_tensor * llama_model_qwen4exp::graph::build_qsa_sel( pooled_new = build_norm(pooled_new, model.layers[il].index_k_norm, nullptr, LLM_NORM_RMS, il); pooled_new = ggml_reshape_3d(ctx0, pooled_new, idx_dim, 1, n_new); - pooled_new = ggml_rope_multi(ctx0, pooled_new, inp_kpool->new_pool_pos, nullptr, - n_rot, sections, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); + pooled_new = build_rope(pooled_new, inp_kpool->new_pool_pos, il); pooled_new = ggml_reshape_2d(ctx0, pooled_new, idx_dim, n_new); cb(pooled_new, "indexer_pool_k_new", il); @@ -871,9 +860,7 @@ ggml_tensor * llama_model_qwen4exp::graph::build_qsa_sel( ggml_tensor * q = build_lora_mm(model.layers[il].index_q_proj, cur); q = ggml_reshape_3d(ctx0, q, idx_dim, n_idx_h, n_tokens); q = build_norm(q, model.layers[il].index_q_norm, nullptr, LLM_NORM_RMS, il); - q = ggml_rope_multi(ctx0, q, inp_pos, nullptr, - n_rot, sections, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); + q = build_rope(q, inp_pos, il); cb(q, "indexer_q", il); // the reference sums the rectified head scores unweighted, scaled by 1/sqrt(head_dim), @@ -1022,7 +1009,6 @@ ggml_tensor * llama_model_qwen4exp::graph::build_layer_attn( llm_graph_input_kpool * inp_kpool, ggml_tensor * cur, ggml_tensor * inp_pos, - int * sections, int il) { const int64_t n_embd_head = hparams.n_embd_head_v(); GGML_ASSERT(n_embd_head == hparams.n_embd_head_k()); @@ -1030,7 +1016,7 @@ ggml_tensor * llama_model_qwen4exp::graph::build_layer_attn( // indexer reads the same block input as q/k/v; no cache or no ratio means dense const bool qsa = inp_kpool != nullptr && hparams.dsv4_compress_ratios[il] > 0; - ggml_tensor * sel = qsa ? build_qsa_sel(mctx_hyb, inp_kpool, cur, inp_pos, sections, il) : nullptr; + ggml_tensor * sel = qsa ? build_qsa_sel(mctx_hyb, inp_kpool, cur, inp_pos, il) : nullptr; // Qwen3Next uses a single Q projection that outputs query + gate ggml_tensor * Qcur_full = build_lora_mm(model.layers[il].wq, cur, model.layers[il].wq_s); // [ (n_embd_head * 2) * n_head, n_tokens ] @@ -1064,17 +1050,8 @@ ggml_tensor * llama_model_qwen4exp::graph::build_layer_attn( Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens); // Apply IMRoPE - Qcur = ggml_rope_multi( - ctx0, Qcur, inp_pos, nullptr, - n_rot, sections, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); - - Kcur = ggml_rope_multi( - ctx0, Kcur, inp_pos, nullptr, - n_rot, sections, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); + Qcur = build_rope(Qcur, inp_pos, il); + Kcur = build_rope(Kcur, inp_pos, il); cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); @@ -1214,13 +1191,6 @@ ggml_tensor * llama_model_qwen4exp::graph::build_layer_attn_linear( q_conv = build_gdn_l2_norm(ctx0, q_conv, eps_norm); k_conv = build_gdn_l2_norm(ctx0, k_conv, eps_norm); - // repeat to match shapes when head keys != value keys; unneeded with the fused GDN - if (num_k_heads != num_v_heads && (!cparams.fused_gdn_ar || !cparams.fused_gdn_ch)) { - GGML_ASSERT(num_v_heads % num_k_heads == 0); - q_conv = ggml_repeat_4d(ctx0, q_conv, head_k_dim, num_v_heads, n_seq_tokens, n_seqs); - k_conv = ggml_repeat_4d(ctx0, k_conv, head_k_dim, num_v_heads, n_seq_tokens, n_seqs); - } - cb(q_conv, "q_conv_predelta", il); cb(k_conv, "k_conv_predelta", il); cb(v_conv, "v_conv_predelta", il); @@ -1402,10 +1372,6 @@ ggml_tensor * llama_model_qwen4exp::graph::build_conv_state_at( int64_t state_cols, int64_t channels, int il) { - const auto * mctx_cur = inp->mctx; - - const auto kv_head = mctx_cur->get_head(); - const int64_t n_seqs = ubatch.n_seqs; const int64_t row_total = conv_states_all->ne[0]; @@ -1423,28 +1389,7 @@ ggml_tensor * llama_model_qwen4exp::graph::build_conv_state_at( ggml_tensor * conv_input = ggml_concat(ctx0, state, ggml_transpose(ctx0, x), 0); - // [TAG_RECURRENT_ROLLBACK_SPLITS] keep the last state_cols columns once per rollback slot, - // slot s ending s tokens earlier so a rollback of s tokens reads a history that never saw them - const size_t row_size = ggml_row_size(conv_states_all->type, row_total); - const uint32_t mem_size = mctx_cur->get_size(); - - const int64_t n_slots = (int64_t) cparams.n_rs_seq + 1; - - for (int64_t slot = 0; slot < n_slots; ++slot) { - const int64_t s_idx = std::max(0, conv_input->ne[0] - state_cols - slot); - - ggml_tensor * tail = ggml_view_3d(ctx0, conv_input, - state_cols, channels, n_seqs, - conv_input->nb[1], conv_input->nb[2], - ggml_row_size(conv_input->type, s_idx)); - - ggml_tensor * dst = ggml_view_2d(ctx0, conv_states_all, - state_cols * channels, n_seqs, - conv_states_all->nb[1], - (slot * mem_size + kv_head) * row_size); - - ggml_build_forward_expand(gf, ggml_cpy(ctx0, ggml_cont(ctx0, tail), dst)); - } + build_conv_state_store(inp, conv_states_all, conv_input, state_cols, 0); return conv_input; } diff --git a/src/models/refact.cpp b/src/models/refact.cpp index 5336608a7457..51bb0bad6227 100644 --- a/src/models/refact.cpp +++ b/src/models/refact.cpp @@ -73,10 +73,10 @@ void llama_model_refact::load_arch_tensors(llama_model_loader &) { } std::unique_ptr llama_model_refact::build_arch_graph(const llm_graph_params & params) const { - return std::make_unique(*this, params); + return std::make_unique(params); } -llama_model_refact::graph::graph(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) { +llama_model_refact::graph::graph(const llm_graph_params & params) : llm_graph_context(params) { const int64_t n_embd_head = hparams.n_embd_head_v(); GGML_ASSERT(n_embd_head == hparams.n_embd_head_k()); diff --git a/src/models/ricursive.cpp b/src/models/ricursive.cpp new file mode 100644 index 000000000000..e14caa8b3c1a --- /dev/null +++ b/src/models/ricursive.cpp @@ -0,0 +1,227 @@ +#include "models.h" + +void llama_model_ricursive::load_arch_hparams(llama_model_loader & ml) { + ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps); + ml.get_key(LLM_KV_EMBEDDING_SCALE, hparams.f_embedding_scale, false); + // optional: absent or 0 means no clamp (the reference tests `if clamp_threshold:`); a scalar applies to every layer + ml.get_key_or_arr(LLM_KV_SWIGLU_CLAMP, hparams.swiglu_clamp, hparams.n_layer_all, false); + for (uint32_t il = 0; il < hparams.n_layer_all; ++il) { + if (!std::isfinite(hparams.swiglu_clamp[il]) || hparams.swiglu_clamp[il] < 0.0f) { + throw std::runtime_error(format("ricursive: swiglu_clamp for layer %u must be finite and non-negative, got %g", il, hparams.swiglu_clamp[il])); + } + } + + if (hparams.n_embd_head_k() != hparams.n_embd_head_v()) { + throw std::runtime_error("ricursive: key_length and value_length must be equal"); + } + + // RoPE is only applied on the SWA layers, so a model without SWA would have no positional encoding + ml.get_key(LLM_KV_ATTENTION_SLIDING_WINDOW, hparams.n_swa); + if (hparams.n_swa == 0) { + throw std::runtime_error("ricursive: sliding_window must be > 0"); + } + hparams.swa_type = LLAMA_SWA_TYPE_STANDARD; + if (ml.get_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, hparams.is_swa_impl, false)) { + // load_swa_pattern only bounds the array by LLAMA_MAX_LAYERS: a short one would leave the last layers dense and without RoPE + uint32_t n_entries = 0; + ml.get_arr_n(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, n_entries); + if (n_entries != hparams.n_layer()) { + throw std::runtime_error(format("ricursive: sliding_window_pattern has %u entries, expected %u", n_entries, hparams.n_layer())); + } + } else { + // default: every 5th layer is global (RicursiveConfig layer_types: full attention iff i % 5 == 4) + load_swa_pattern(ml, 5); + } + if (!hparams.is_swa_any()) { + throw std::runtime_error("ricursive: no sliding window layer, the model would have no positional encoding"); + } + + // RoPE exactly on the SWA layers; rope_pattern is optional because the first published GGUFs do not carry it + hparams.rope_pattern = hparams.is_swa_impl; + ml.get_key_or_arr(LLM_KV_ATTENTION_ROPE_PATTERN, hparams.rope_pattern, hparams.n_layer(), false); + for (uint32_t il = 0; il < hparams.n_layer(); ++il) { + if (hparams.has_rope(il) != hparams.is_swa(il)) { + throw std::runtime_error(format("ricursive: layer %u has %s but %s", il, + hparams.has_rope(il) ? "RoPE" : "no RoPE", hparams.is_swa(il) ? "a sliding window" : "no sliding window")); + } + } + + // the SWA layers are the only RoPE layers, they use the model RoPE parameters + hparams.rope_freq_base_train_swa = hparams.rope_freq_base_train; + hparams.rope_freq_scale_train_swa = hparams.rope_freq_scale_train; + ml.get_key(LLM_KV_ROPE_FREQ_BASE_SWA, hparams.rope_freq_base_train_swa, false); + + switch (hparams.n_layer()) { + case 80: type = LLM_TYPE_65B; break; + default: type = LLM_TYPE_UNKNOWN; + } +} + +void llama_model_ricursive::load_arch_tensors(llama_model_loader &) { + LLAMA_LOAD_LOCALS; + + tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, 0); + + output_norm = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd}, 0); + output = create_tensor(tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab}, TENSOR_NOT_REQUIRED); + if (output == NULL) { + output = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, TENSOR_DUPLICATED); + } + + const int64_t n_embd_q = n_embd_head_k * n_head; + + for (int i = 0; i < n_layer; ++i) { + auto & layer = layers[i]; + + layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd}, 0); + + layer.wq = create_tensor(tn(LLM_TENSOR_ATTN_Q, "weight", i), {n_embd, n_embd_q}, 0); + layer.wk = create_tensor(tn(LLM_TENSOR_ATTN_K, "weight", i), {n_embd, n_embd_k_gqa}, 0); + layer.wv = create_tensor(tn(LLM_TENSOR_ATTN_V, "weight", i), {n_embd, n_embd_v_gqa}, 0); + layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd_q, n_embd}, 0); + + layer.attn_q_norm = create_tensor(tn(LLM_TENSOR_ATTN_Q_NORM, "weight", i), {n_embd_head_k}, 0); + layer.attn_k_norm = create_tensor(tn(LLM_TENSOR_ATTN_K_NORM, "weight", i), {n_embd_head_k}, 0); + + layer.wqkv_gate = create_tensor(tn(LLM_TENSOR_ATTN_GATE, "weight", i), {n_embd, n_embd_q}, TENSOR_NOT_REQUIRED); + layer.attn_vres_lambda = create_tensor(tn(LLM_TENSOR_ATTN_VRES_LAMBDA, "weight", i), {1}, TENSOR_NOT_REQUIRED); + + layer.attn_post_norm = create_tensor(tn(LLM_TENSOR_ATTN_POST_NORM, "weight", i), {n_embd}, 0); + + layer.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd}, 0); + layer.ffn_gate = create_tensor(tn(LLM_TENSOR_FFN_GATE, "weight", i), {n_embd, n_ff}, 0); + layer.ffn_up = create_tensor(tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff}, 0); + layer.ffn_down = create_tensor(tn(LLM_TENSOR_FFN_DOWN, "weight", i), {n_ff, n_embd}, 0); + layer.ffn_post_norm = create_tensor(tn(LLM_TENSOR_FFN_POST_NORM, "weight", i), {n_embd}, 0); + } + + // the gate and the value residual are per-model switches (use_gated_attention, use_valres) + for (int i = 1; i < n_layer; ++i) { + if ((layers[i].wqkv_gate == nullptr) != (layers[0].wqkv_gate == nullptr)) { + throw std::runtime_error(format("ricursive: attn_gate must be present in all layers or in none (layers 0 and %d differ)", i)); + } + if ((layers[i].attn_vres_lambda == nullptr) != (layers[0].attn_vres_lambda == nullptr)) { + throw std::runtime_error(format("ricursive: attn_vres_lambda must be present in all layers or in none (layers 0 and %d differ)", i)); + } + } +} + +std::unique_ptr llama_model_ricursive::build_arch_graph(const llm_graph_params & params) const { + return std::make_unique(params); +} + +llama_model_ricursive::graph::graph(const llm_graph_params & params) : llm_graph_context(params) { + const int64_t n_embd_head = hparams.n_embd_head_v(); + + ggml_tensor * cur; + ggml_tensor * inpL; + + inpL = build_inp_embd(model.tok_embd); + + // value residual: x0 = token embeddings * embedding_scale, the same tensor for all layers (modeling_ricursive.py:380-381,405) + ggml_tensor * inp_vres = inpL; + + ggml_tensor * inp_pos = build_inp_pos(); + + auto * inp_attn = build_attn_inp_kv_iswa(); + + ggml_tensor * inp_out_ids = build_inp_out_ids(); + + const float kq_scale = 1.0f / sqrtf(float(n_embd_head)); + + for (int il = 0; il < n_layer; ++il) { + const auto & layer = model.layers[il]; + + ggml_tensor * inpSA = inpL; + + cur = build_norm(inpL, layer.attn_norm, NULL, LLM_NORM_RMS, il); + cb(cur, "attn_norm", il); + + ggml_tensor * attn_in = cur; + + { + ggml_tensor * Qcur = build_lora_mm(layer.wq, attn_in, layer.wq_s); + ggml_tensor * Kcur = build_lora_mm(layer.wk, attn_in, layer.wk_s); + + // v_in = lerp(x0, attn_norm(x), sigmoid(lambda)), lambda is learned per layer (modeling_ricursive.py:182-185) + ggml_tensor * v_in = attn_in; + if (layer.attn_vres_lambda) { + ggml_tensor * mix = ggml_sigmoid(ctx0, layer.attn_vres_lambda); + v_in = ggml_add(ctx0, inp_vres, ggml_mul(ctx0, ggml_sub(ctx0, attn_in, inp_vres), mix)); + cb(v_in, "vres_in", il); + } + ggml_tensor * Vcur = build_lora_mm(layer.wv, v_in, layer.wv_s); + + Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens); + Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens); + Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens); + + Qcur = build_norm(Qcur, layer.attn_q_norm, NULL, LLM_NORM_RMS, il); + Kcur = build_norm(Kcur, layer.attn_k_norm, NULL, LLM_NORM_RMS, il); + cb(Qcur, "Qcur_normed", il); + cb(Kcur, "Kcur_normed", il); + + // global layers use no RoPE (NoPE): the reference rotates only sliding_attention layers (modeling_ricursive.py:127-129,203-205) + Qcur = build_rope(Qcur, inp_pos, il); + Kcur = build_rope(Kcur, inp_pos, il); + cb(Qcur, "Qcur", il); + cb(Kcur, "Kcur", il); + cb(Vcur, "Vcur", il); + + cur = build_attn(inp_attn, NULL, NULL, NULL, Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, kq_scale, il); + cb(cur, "attn_out", il); + + if (layer.wqkv_gate) { + ggml_tensor * gate = build_lora_mm(layer.wqkv_gate, attn_in, layer.wqkv_gate_s); + gate = ggml_sigmoid(ctx0, gate); + cb(gate, "attn_gate", il); + cur = ggml_mul(ctx0, cur, gate); + cb(cur, "attn_gated", il); + } + + cur = build_lora_mm(layer.wo, cur, layer.wo_s); + } + + if (il == n_layer - 1 && inp_out_ids) { + cur = ggml_get_rows(ctx0, cur, inp_out_ids); + inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids); + } + + cur = build_norm(cur, layer.attn_post_norm, NULL, LLM_NORM_RMS, il); + cb(cur, "attn_post_norm", il); + + ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA); + cb(ffn_inp, "ffn_inp", il); + + cur = build_norm(ffn_inp, layer.ffn_norm, NULL, LLM_NORM_RMS, il); + cb(cur, "ffn_norm", il); + + { + ggml_tensor * gate = build_lora_mm(layer.ffn_gate, cur, layer.ffn_gate_s); + ggml_tensor * up = build_lora_mm(layer.ffn_up, cur, layer.ffn_up_s); + cur = hparams.swiglu_clamp[il] > 0.0f + ? ggml_swiglu_clamp(ctx0, gate, up, hparams.swiglu_clamp[il]) + : ggml_swiglu_split(ctx0, gate, up); + cb(cur, "ffn_swiglu", il); + cur = build_lora_mm(layer.ffn_down, cur, layer.ffn_down_s); + cb(cur, "ffn_out", il); + } + + cur = build_norm(cur, layer.ffn_post_norm, NULL, LLM_NORM_RMS, il); + cb(cur, "ffn_post_norm", il); + + cur = ggml_add(ctx0, cur, ffn_inp); + cur = build_cvec(cur, il); + cb(cur, "l_out", il); + + inpL = cur; + } + + cur = build_norm(inpL, model.output_norm, NULL, LLM_NORM_RMS, -1); + cb(cur, "result_norm", -1); + res->t_embd = cur; + + cur = build_lora_mm(model.output, cur, model.output_s); + cb(cur, "result_output", -1); + res->t_logits = cur; +} diff --git a/src/models/rnd1.cpp b/src/models/rnd1.cpp index 2b0b3e97ef0c..8ea8d23649a0 100644 --- a/src/models/rnd1.cpp +++ b/src/models/rnd1.cpp @@ -58,11 +58,11 @@ void llama_model_rnd1::load_arch_tensors(llama_model_loader &) { } std::unique_ptr llama_model_rnd1::build_arch_graph(const llm_graph_params & params) const { - return std::make_unique(*this, params); + return std::make_unique(params); } // RND1 is a Qwen3Moe AR model converted to diffusion model. -llama_model_rnd1::graph::graph(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) { +llama_model_rnd1::graph::graph(const llm_graph_params & params) : llm_graph_context(params) { const int64_t n_embd_head = hparams.n_embd_head_v(); GGML_ASSERT(n_embd_head == hparams.n_embd_head_k()); @@ -99,20 +99,12 @@ llama_model_rnd1::graph::graph(const llama_model & model, const llm_graph_params Qcur = build_norm(Qcur, model.layers[il].attn_q_norm, NULL, LLM_NORM_RMS, il); cb(Qcur, "Qcur_normed", il); - Qcur = ggml_rope_ext( - ctx0, Qcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); + Qcur = build_rope(Qcur, inp_pos, il); Kcur = build_norm(Kcur, model.layers[il].attn_k_norm, NULL, LLM_NORM_RMS, il); cb(Kcur, "Kcur_normed", il); - Kcur = ggml_rope_ext( - ctx0, Kcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); + Kcur = build_rope(Kcur, inp_pos, il); cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); diff --git a/src/models/rwkv6-base.cpp b/src/models/rwkv6-base.cpp index 83aeab7280b1..68efec666962 100644 --- a/src/models/rwkv6-base.cpp +++ b/src/models/rwkv6-base.cpp @@ -2,9 +2,8 @@ #include "llama-memory-recurrent.h" -llm_build_rwkv6_base::llm_build_rwkv6_base(const llama_model & model, const llm_graph_params & params) : - llm_graph_context(params), - model(model) {} +llm_build_rwkv6_base::llm_build_rwkv6_base(const llm_graph_params & params) : + llm_graph_context(params) {} ggml_tensor * llm_build_rwkv6_base::build_rwkv6_channel_mix(const llama_layer * layer, ggml_tensor * cur, diff --git a/src/models/rwkv6.cpp b/src/models/rwkv6.cpp index 84a18180b717..4cb5bce643fe 100644 --- a/src/models/rwkv6.cpp +++ b/src/models/rwkv6.cpp @@ -88,11 +88,11 @@ void llama_model_rwkv6::load_arch_tensors(llama_model_loader &) { } std::unique_ptr llama_model_rwkv6::build_arch_graph(const llm_graph_params & params) const { - return std::make_unique(*this, params); + return std::make_unique(params); } -llama_model_rwkv6::graph::graph(const llama_model & model, const llm_graph_params & params) : - llm_build_rwkv6_base(model, params) { +llama_model_rwkv6::graph::graph(const llm_graph_params & params) : + llm_build_rwkv6_base(params) { GGML_ASSERT(hparams.token_shift_count == 2); ggml_tensor * cur; diff --git a/src/models/rwkv6qwen2.cpp b/src/models/rwkv6qwen2.cpp index 0e5561c1957a..e81c0132f8cf 100644 --- a/src/models/rwkv6qwen2.cpp +++ b/src/models/rwkv6qwen2.cpp @@ -78,10 +78,10 @@ void llama_model_rwkv6qwen2::load_arch_tensors(llama_model_loader &) { } std::unique_ptr llama_model_rwkv6qwen2::build_arch_graph(const llm_graph_params & params) const { - return std::make_unique(*this, params); + return std::make_unique(params); } -llama_model_rwkv6qwen2::graph::graph(const llama_model & model, const llm_graph_params & params) : llm_build_rwkv6_base(model, params) { +llama_model_rwkv6qwen2::graph::graph(const llm_graph_params & params) : llm_build_rwkv6_base(params) { GGML_ASSERT(n_embd == hparams.n_embd_r()); ggml_tensor * cur; diff --git a/src/models/rwkv7-base.cpp b/src/models/rwkv7-base.cpp index 7fcab77745c7..3300822d2549 100644 --- a/src/models/rwkv7-base.cpp +++ b/src/models/rwkv7-base.cpp @@ -2,9 +2,8 @@ #include "llama-memory-recurrent.h" -llm_build_rwkv7_base::llm_build_rwkv7_base(const llama_model & model, const llm_graph_params & params) : - llm_graph_context(params), - model(model) {} +llm_build_rwkv7_base::llm_build_rwkv7_base(const llm_graph_params & params) : + llm_graph_context(params) {} ggml_tensor * llm_build_rwkv7_base::build_rwkv7_channel_mix(const llama_layer * layer, ggml_tensor * cur, diff --git a/src/models/rwkv7.cpp b/src/models/rwkv7.cpp index 6577a25f9216..77c97f953a86 100644 --- a/src/models/rwkv7.cpp +++ b/src/models/rwkv7.cpp @@ -118,11 +118,11 @@ void llama_model_rwkv7::load_arch_tensors(llama_model_loader &) { } std::unique_ptr llama_model_rwkv7::build_arch_graph(const llm_graph_params & params) const { - return std::make_unique(*this, params); + return std::make_unique(params); } -llama_model_rwkv7::graph::graph(const llama_model & model, const llm_graph_params & params) : - llm_build_rwkv7_base(model, params) { +llama_model_rwkv7::graph::graph(const llm_graph_params & params) : + llm_build_rwkv7_base(params) { GGML_ASSERT(hparams.token_shift_count == 2); ggml_tensor * cur; diff --git a/src/models/seed-oss.cpp b/src/models/seed-oss.cpp index 871f06cc5ed7..c7940dc806b4 100644 --- a/src/models/seed-oss.cpp +++ b/src/models/seed-oss.cpp @@ -43,10 +43,10 @@ void llama_model_seed_oss::load_arch_tensors(llama_model_loader &) { } std::unique_ptr llama_model_seed_oss::build_arch_graph(const llm_graph_params & params) const { - return std::make_unique(*this, params); + return std::make_unique(params); } -llama_model_seed_oss::graph::graph(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) { +llama_model_seed_oss::graph::graph(const llm_graph_params & params) : llm_graph_context(params) { const int64_t n_embd_head = hparams.n_embd_head_v(); GGML_ASSERT(n_embd_head == hparams.n_embd_head_k()); @@ -81,17 +81,8 @@ llama_model_seed_oss::graph::graph(const llama_model & model, const llm_graph_pa auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur, n_embd_head, n_head, n_head_kv, il); - Qcur = ggml_rope_ext( - ctx0, Qcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); - - Kcur = ggml_rope_ext( - ctx0, Kcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); + Qcur = build_rope(Qcur, inp_pos, il); + Kcur = build_rope(Kcur, inp_pos, il); cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); diff --git a/src/models/sensenova-u1.cpp b/src/models/sensenova-u1.cpp index c42c604e8ecd..7c8025703f18 100644 --- a/src/models/sensenova-u1.cpp +++ b/src/models/sensenova-u1.cpp @@ -70,10 +70,12 @@ void llama_model_sensenova_u1::load_arch_tensors(llama_model_loader &) { } std::unique_ptr llama_model_sensenova_u1::build_arch_graph(const llm_graph_params & params) const { - return std::make_unique(*this, params); + return std::make_unique(params); } -llama_model_sensenova_u1::graph::graph(const llama_model_sensenova_u1 & model, const llm_graph_params & params) : llm_graph_context(params) { +llama_model_sensenova_u1::graph::graph(const llm_graph_params & params) : llm_graph_context(params) { + const auto & model = static_cast(this->model); + const int64_t n_embd_head = hparams.n_embd_head_v(); GGML_ASSERT(n_embd_head == hparams.n_embd_head_k()); diff --git a/src/models/smallthinker.cpp b/src/models/smallthinker.cpp index 60803cb4fdb4..2c183e33c629 100644 --- a/src/models/smallthinker.cpp +++ b/src/models/smallthinker.cpp @@ -16,6 +16,10 @@ void llama_model_smallthinker::load_arch_hparams(llama_model_loader & ml) { hparams.n_no_rope_layer_step = hparams.n_layer(); } + for (uint32_t il = 0; il < hparams.n_layer_all; ++il) { + hparams.rope_pattern[il] = hparams.n_no_rope_layer_step == hparams.n_layer() || il % hparams.n_no_rope_layer_step != 0; + } + ml.get_key_or_arr(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp_arr, hparams.n_layer_all, false); ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps); ml.get_key(LLM_KV_EXPERT_GATING_FUNC, hparams.expert_gating_func, false); @@ -65,14 +69,14 @@ void llama_model_smallthinker::load_arch_tensors(llama_model_loader &) { std::unique_ptr llama_model_smallthinker::build_arch_graph(const llm_graph_params & params) const { if (hparams.swa_type == LLAMA_SWA_TYPE_STANDARD) { - return std::make_unique> (*this, params); + return std::make_unique> (params); } else { - return std::make_unique>(*this, params); + return std::make_unique>(params); } } template -llama_model_smallthinker::graph::graph(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params){ +llama_model_smallthinker::graph::graph(const llm_graph_params & params) : llm_graph_context(params){ const int64_t n_embd_head = hparams.n_embd_head_v(); GGML_ASSERT(n_embd_head == hparams.n_embd_head_k()); @@ -97,15 +101,9 @@ llama_model_smallthinker::graph::graph(const llama_model & model, const ll ggml_tensor * inp_out_ids = build_inp_out_ids(); for (int il = 0; il < n_layer; ++il) { - const float freq_base_l = model.get_rope_freq_base (cparams, il); - const float freq_scale_l = model.get_rope_freq_scale(cparams, il); ggml_tensor * inpSA = inpL; - // This overlaps with SWA layers in current models, so get_rope_freq_base/scale may be superfluous - const bool use_rope = hparams.n_no_rope_layer_step == n_layer || - il % hparams.n_no_rope_layer_step != 0; - ggml_tensor * probs = build_lora_mm(model.layers[il].ffn_gate_inp, inpL); // [n_expert, n_tokens] cb(probs, "ffn_moe_logits", il); @@ -119,13 +117,8 @@ llama_model_smallthinker::graph::graph(const llama_model & model, const ll auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur, n_embd_head, n_head, n_head_kv, il); - if (use_rope) { - Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base_l, freq_scale_l, - ext_factor, attn_factor, beta_fast, beta_slow); - - Kcur = ggml_rope_ext(ctx0, Kcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base_l, freq_scale_l, - ext_factor, attn_factor, beta_fast, beta_slow); - } + Qcur = build_rope(Qcur, inp_pos, il); + Kcur = build_rope(Kcur, inp_pos, il); cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); diff --git a/src/models/smollm3.cpp b/src/models/smollm3.cpp index 474093b68b9a..fa9e9c5395a7 100644 --- a/src/models/smollm3.cpp +++ b/src/models/smollm3.cpp @@ -3,6 +3,9 @@ void llama_model_smollm3::load_arch_hparams(llama_model_loader & ml) { ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps); hparams.n_no_rope_layer_step = 4; + for (uint32_t il = 0; il < hparams.n_layer_all; ++il) { + hparams.rope_pattern[il] = (il + 1) % hparams.n_no_rope_layer_step != 0; + } switch (hparams.n_layer()) { case 36: type = LLM_TYPE_3B; break; @@ -40,10 +43,10 @@ void llama_model_smollm3::load_arch_tensors(llama_model_loader &) { } std::unique_ptr llama_model_smollm3::build_arch_graph(const llm_graph_params & params) const { - return std::make_unique(*this, params); + return std::make_unique(params); } -llama_model_smollm3::graph::graph(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) { +llama_model_smollm3::graph::graph(const llm_graph_params & params) : llm_graph_context(params) { const int64_t n_embd_head = hparams.n_embd_head_v(); GGML_ASSERT(n_embd_head == hparams.n_embd_head_k()); @@ -66,8 +69,6 @@ llama_model_smollm3::graph::graph(const llama_model & model, const llm_graph_par for (int il = 0; il < n_layer; ++il) { ggml_tensor * inpSA = inpL; - const bool use_rope = (il + 1) % hparams.n_no_rope_layer_step != 0; - // norm cur = build_norm(inpL, model.layers[il].attn_norm, NULL, @@ -80,19 +81,8 @@ llama_model_smollm3::graph::graph(const llama_model & model, const llm_graph_par auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur, n_embd_head, n_head, n_head_kv, il); - if (use_rope) { - Qcur = ggml_rope_ext( - ctx0, Qcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); - - Kcur = ggml_rope_ext( - ctx0, Kcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); - } + Qcur = build_rope(Qcur, inp_pos, il); + Kcur = build_rope(Kcur, inp_pos, il); cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); cb(Vcur, "Vcur", il); diff --git a/src/models/spark2-5.cpp b/src/models/spark2-5.cpp index 970b32ba603c..e90184d8e866 100644 --- a/src/models/spark2-5.cpp +++ b/src/models/spark2-5.cpp @@ -50,10 +50,10 @@ void llama_model_spark2_5::load_arch_tensors(llama_model_loader &) { } std::unique_ptr llama_model_spark2_5::build_arch_graph(const llm_graph_params & params) const { - return std::make_unique(*this, params); + return std::make_unique(params); } -llama_model_spark2_5::graph::graph(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) { +llama_model_spark2_5::graph::graph(const llm_graph_params & params) : llm_graph_context(params) { const int64_t n_embd_head = hparams.n_embd_head_v(); GGML_ASSERT(n_embd_head == hparams.n_embd_head_k()); @@ -73,19 +73,12 @@ llama_model_spark2_5::graph::graph(const llama_model & model, const llm_graph_pa const int64_t n_head_i = hparams.n_head(il); const int64_t n_head_kv_i = hparams.n_head_kv(il); - const int64_t n_rot_i = hparams.n_rot(il); - const float freq_base_i = model.get_rope_freq_base(cparams, il); - const float freq_scale_i = model.get_rope_freq_scale(cparams, il); ggml_tensor * attn_inp = cur; auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur, n_embd_head, n_head_i, n_head_kv_i, il); - Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, nullptr, - n_rot_i, rope_type, n_ctx_orig, freq_base_i, freq_scale_i, - ext_factor, attn_factor, beta_fast, beta_slow); - Kcur = ggml_rope_ext(ctx0, Kcur, inp_pos, nullptr, - n_rot_i, rope_type, n_ctx_orig, freq_base_i, freq_scale_i, - ext_factor, attn_factor, beta_fast, beta_slow); + Qcur = build_rope(Qcur, inp_pos, il); + Kcur = build_rope(Kcur, inp_pos, il); cb(Qcur, "Qcur_rope", il); cb(Kcur, "Kcur_rope", il); diff --git a/src/models/stablelm.cpp b/src/models/stablelm.cpp index 5189a460687c..5aec7beee520 100644 --- a/src/models/stablelm.cpp +++ b/src/models/stablelm.cpp @@ -45,10 +45,10 @@ void llama_model_stablelm::load_arch_tensors(llama_model_loader &) { } std::unique_ptr llama_model_stablelm::build_arch_graph(const llm_graph_params & params) const { - return std::make_unique(*this, params); + return std::make_unique(params); } -llama_model_stablelm::graph::graph(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) { +llama_model_stablelm::graph::graph(const llm_graph_params & params) : llm_graph_context(params) { const int64_t n_embd_head = hparams.n_embd_head_v(); GGML_ASSERT(n_embd_head == hparams.n_embd_head_k()); @@ -96,17 +96,8 @@ llama_model_stablelm::graph::graph(const llama_model & model, const llm_graph_pa cb(Kcur, "Kcur", il); } - Qcur = ggml_rope_ext( - ctx0, Qcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); - - Kcur = ggml_rope_ext( - ctx0, Kcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); + Qcur = build_rope(Qcur, inp_pos, il); + Kcur = build_rope(Kcur, inp_pos, il); cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); diff --git a/src/models/starcoder.cpp b/src/models/starcoder.cpp index e68e24ffb818..7d73aa01bcbe 100644 --- a/src/models/starcoder.cpp +++ b/src/models/starcoder.cpp @@ -54,10 +54,10 @@ void llama_model_starcoder::load_arch_tensors(llama_model_loader &) { } std::unique_ptr llama_model_starcoder::build_arch_graph(const llm_graph_params & params) const { - return std::make_unique(*this, params); + return std::make_unique(params); } -llama_model_starcoder::graph::graph(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) { +llama_model_starcoder::graph::graph(const llm_graph_params & params) : llm_graph_context(params) { const int64_t n_embd_head = hparams.n_embd_head_v(); GGML_ASSERT(n_embd_head == hparams.n_embd_head_k()); diff --git a/src/models/starcoder2.cpp b/src/models/starcoder2.cpp index dcf6ed371408..2b670d5629db 100644 --- a/src/models/starcoder2.cpp +++ b/src/models/starcoder2.cpp @@ -53,10 +53,10 @@ void llama_model_starcoder2::load_arch_tensors(llama_model_loader &) { } std::unique_ptr llama_model_starcoder2::build_arch_graph(const llm_graph_params & params) const { - return std::make_unique(*this, params); + return std::make_unique(params); } -llama_model_starcoder2::graph::graph(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) { +llama_model_starcoder2::graph::graph(const llm_graph_params & params) : llm_graph_context(params) { const int64_t n_embd_head = hparams.n_embd_head_v(); GGML_ASSERT(n_embd_head == hparams.n_embd_head_k()); @@ -89,17 +89,8 @@ llama_model_starcoder2::graph::graph(const llama_model & model, const llm_graph_ auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur, n_embd_head, n_head, n_head_kv, il); - Qcur = ggml_rope_ext( - ctx0, Qcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); - - Kcur = ggml_rope_ext( - ctx0, Kcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); + Qcur = build_rope(Qcur, inp_pos, il); + Kcur = build_rope(Kcur, inp_pos, il); cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); diff --git a/src/models/step35.cpp b/src/models/step35.cpp index 3e225a11efa3..5bed70163a95 100644 --- a/src/models/step35.cpp +++ b/src/models/step35.cpp @@ -172,14 +172,24 @@ void llama_model_step35::load_arch_tensors(llama_model_loader & ml) { } } +llama_rope_params llama_model_step35::get_rope_params(const llama_cparams & cparams, int il) const { + auto res = llama_model::get_rope_params(cparams, il); + + if (hparams.is_swa(il)) { + res.factors = nullptr; + } + + return res; +} + std::unique_ptr llama_model_step35::build_arch_graph(const llm_graph_params & params) const { if (params.gtype == LLM_GRAPH_TYPE_DECODER_MTP) { - return std::make_unique(*this, params); + return std::make_unique(params); } - return std::make_unique(*this, params); + return std::make_unique(params); } -llama_model_step35::graph::graph(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) { +llama_model_step35::graph::graph(const llm_graph_params & params) : llm_graph_context(params) { ggml_tensor * cur; ggml_tensor * inpL; @@ -195,9 +205,6 @@ llama_model_step35::graph::graph(const llama_model & model, const llm_graph_para const uint32_t n_head_l = hparams.n_head(il); const uint32_t n_head_kv_l = hparams.n_head_kv(il); - const float freq_base_l = model.get_rope_freq_base(cparams, il); - const float freq_scale_l = model.get_rope_freq_scale(cparams, il); - cur = inpL; // dump pre-attn RMSNorm input to pinpoint layer boundary issues @@ -232,19 +239,8 @@ llama_model_step35::graph::graph(const llama_model & model, const llm_graph_para } // RoPE (partial rotary factors per layer) - const bool is_swa = hparams.is_swa(il); - ggml_tensor * rope_factors = is_swa ? nullptr : model.get_rope_factors(cparams, il); - const int64_t n_rot_l = hparams.n_rot(il); - Qcur = ggml_rope_ext( - ctx0, Qcur, inp_pos, rope_factors, - n_rot_l, rope_type, n_ctx_orig, freq_base_l, freq_scale_l, - ext_factor, attn_factor, beta_fast, beta_slow - ); - Kcur = ggml_rope_ext( - ctx0, Kcur, inp_pos, rope_factors, - n_rot_l, rope_type, n_ctx_orig, freq_base_l, freq_scale_l, - ext_factor, attn_factor, beta_fast, beta_slow - ); + Qcur = build_rope(Qcur, inp_pos, il); + Kcur = build_rope(Kcur, inp_pos, il); cb(Qcur, "Qcur_pos", il); cb(Kcur, "Kcur_pos", il); @@ -354,7 +350,7 @@ llama_model_step35::graph::graph(const llama_model & model, const llm_graph_para } // LLM_GRAPH_TYPE_DECODER_MTP draft head for Step3p5 (MoE) -llama_model_step35::graph_mtp::graph_mtp(const llama_model & model, const llm_graph_params & params) +llama_model_step35::graph_mtp::graph_mtp(const llm_graph_params & params) : llm_graph_context(params) { GGML_ASSERT(hparams.n_layer_nextn > 0 && "STEP35 MTP requires n_layer_nextn > 0"); @@ -375,9 +371,6 @@ llama_model_step35::graph_mtp::graph_mtp(const llama_model & model, const llm_gr const uint32_t n_head_l = hparams.n_head(il); const uint32_t n_head_kv_l = hparams.n_head_kv(il); - const float freq_base_l = model.get_rope_freq_base(cparams, il); - const float freq_scale_l = model.get_rope_freq_scale(cparams, il); - auto inp = std::make_unique(hparams.n_embd_inp(), hparams.n_embd); inp->tokens = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_tokens); @@ -386,9 +379,7 @@ llama_model_step35::graph_mtp::graph_mtp(const llama_model & model, const llm_gr inp->embd = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, hparams.n_embd_inp(), n_tokens); ggml_set_input(inp->embd); - inp->h = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, hparams.n_embd, n_tokens); - ggml_set_input(inp->h); - ggml_set_name(inp->h, "mtp_h_input"); + inp->h = build_inp_embd_state(inp.get(), hparams.n_embd); ggml_tensor * tok_embd_w = layer.nextn.embed_tokens ? layer.nextn.embed_tokens : model.tok_embd; @@ -441,18 +432,8 @@ llama_model_step35::graph_mtp::graph_mtp(const llama_model & model, const llm_gr cb(Kcur, "mtp_Kcur_normed", il); } - const bool is_swa = hparams.is_swa(il); - ggml_tensor * rope_factors = is_swa ? nullptr : model.get_rope_factors(cparams, il); - const int64_t n_rot_l = hparams.n_rot(il); - - Qcur = ggml_rope_ext( - ctx0, Qcur, inp_pos, rope_factors, - n_rot_l, rope_type, n_ctx_orig, freq_base_l, freq_scale_l, - ext_factor, attn_factor, beta_fast, beta_slow); - Kcur = ggml_rope_ext( - ctx0, Kcur, inp_pos, rope_factors, - n_rot_l, rope_type, n_ctx_orig, freq_base_l, freq_scale_l, - ext_factor, attn_factor, beta_fast, beta_slow); + Qcur = build_rope(Qcur, inp_pos, il); + Kcur = build_rope(Kcur, inp_pos, il); cb(Qcur, "mtp_Qcur_pos", il); cb(Kcur, "mtp_Kcur_pos", il); diff --git a/src/models/t5.cpp b/src/models/t5.cpp index a5ddbebf907b..8a0aac1c8004 100644 --- a/src/models/t5.cpp +++ b/src/models/t5.cpp @@ -107,7 +107,7 @@ void llama_model_t5::load_arch_tensors(llama_model_loader &) { } template <> -llama_model_t5::graph::graph(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) { +llama_model_t5::graph::graph(const llm_graph_params & params) : llm_graph_context(params) { const int64_t n_embd_head = hparams.n_embd_head_v(); //const int64_t n_embd_gqa = hparams.n_embd_v_gqa(); @@ -260,7 +260,7 @@ llama_model_t5::graph::graph(const llama_model & model, const llm_graph_p } template <> -llama_model_t5::graph::graph(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) { +llama_model_t5::graph::graph(const llm_graph_params & params) : llm_graph_context(params) { const int64_t n_embd_head = hparams.n_embd_head_v(); GGML_ASSERT(n_embd_head == hparams.n_embd_head_k()); @@ -358,10 +358,10 @@ llama_model_t5::graph::graph(const llama_model & model, const llm_graph_pa std::unique_ptr llama_model_t5::build_arch_graph(const llm_graph_params & params) const { switch (params.gtype) { case LLM_GRAPH_TYPE_ENCODER: - return std::make_unique>(*this, params); + return std::make_unique>(params); case LLM_GRAPH_TYPE_DEFAULT: case LLM_GRAPH_TYPE_DECODER: - return std::make_unique>(*this, params); + return std::make_unique>(params); default: GGML_ABORT("invalid graph type"); }; diff --git a/src/models/t5encoder.cpp b/src/models/t5encoder.cpp index 23c5f9b6a1ca..ccaa89e320c0 100644 --- a/src/models/t5encoder.cpp +++ b/src/models/t5encoder.cpp @@ -40,5 +40,5 @@ void llama_model_t5encoder::load_arch_tensors(llama_model_loader &) { } std::unique_ptr llama_model_t5encoder::build_arch_graph(const llm_graph_params & params) const { - return std::make_unique(*this, params); + return std::make_unique(params); } diff --git a/src/models/talkie.cpp b/src/models/talkie.cpp index c6a6b5abe8cb..5bf18f0665e3 100644 --- a/src/models/talkie.cpp +++ b/src/models/talkie.cpp @@ -34,10 +34,10 @@ void llama_model_talkie::load_arch_tensors(llama_model_loader &) { } std::unique_ptr llama_model_talkie::build_arch_graph(const llm_graph_params & params) const { - return std::make_unique(*this, params); + return std::make_unique(params); } -llama_model_talkie::graph::graph(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) { +llama_model_talkie::graph::graph(const llm_graph_params & params) : llm_graph_context(params) { const int64_t n_embd_head = hparams.n_embd_head_k(); GGML_ASSERT(n_embd_head == hparams.n_embd_head_v()); @@ -73,15 +73,8 @@ llama_model_talkie::graph::graph(const llama_model & model, const llm_graph_para auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur, n_embd_head, n_head, n_head_kv, il); - Qcur = ggml_rope_ext( - ctx0, Qcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); - - Kcur = ggml_rope_ext( - ctx0, Kcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); + Qcur = build_rope(Qcur, inp_pos, il); + Kcur = build_rope(Kcur, inp_pos, il); // reference applies qknorm after rope Qcur = build_norm(Qcur, model.layers[il].attn_q_norm, nullptr, LLM_NORM_RMS, il); diff --git a/src/models/vllm-decision2.cpp b/src/models/vllm-decision2.cpp index bbac808de132..2a3b9595c72d 100644 --- a/src/models/vllm-decision2.cpp +++ b/src/models/vllm-decision2.cpp @@ -39,7 +39,7 @@ void llama_model_vllm_decision2::load_arch_tensors(llama_model_loader & ml) { } std::unique_ptr llama_model_vllm_decision2::build_arch_graph(const llm_graph_params & params) const { - return std::make_unique(*this, params); + return std::make_unique(params); } // output rows read by the head: the last token of each option span and the last token of the question span @@ -127,13 +127,13 @@ class llama_model_vllm_decision2::input_decision : public llm_graph_input_i { const size_t n_options; }; -llama_model_vllm_decision2::graph::graph(const llama_model & model_base, const llm_graph_params & params) : - llama_model_qwen3::graph(params, no_build()), model(static_cast(model_base)) { +llama_model_vllm_decision2::graph::graph(const llm_graph_params & params) : + llama_model_qwen3::graph(params, no_build()), model(static_cast(*params.model)) { auto inp_decision_ptr = std::make_unique(ubatch); auto * inp_decision = inp_decision_ptr.get(); res->add_input(std::move(inp_decision_ptr)); - ggml_tensor * cur = build_backbone(model); + ggml_tensor * cur = build_backbone(); // the head is always evaluated, so that the graph has the same nodes for every batch cur = build_head(cur, inp_decision); diff --git a/src/models/wavtokenizer-dec.cpp b/src/models/wavtokenizer-dec.cpp index 881d2c3833e3..ac3094f9cc65 100644 --- a/src/models/wavtokenizer-dec.cpp +++ b/src/models/wavtokenizer-dec.cpp @@ -112,10 +112,10 @@ void llama_model_wavtokenizer_dec::load_arch_tensors(llama_model_loader &) { } std::unique_ptr llama_model_wavtokenizer_dec::build_arch_graph(const llm_graph_params & params) const { - return std::make_unique(*this, params); + return std::make_unique(params); } -llama_model_wavtokenizer_dec::graph::graph(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) { +llama_model_wavtokenizer_dec::graph::graph(const llm_graph_params & params) : llm_graph_context(params) { ggml_tensor * cur; ggml_tensor * inpL; diff --git a/src/models/xing4-0.cpp b/src/models/xing4-0.cpp new file mode 100644 index 000000000000..f405fcc9c9c5 --- /dev/null +++ b/src/models/xing4-0.cpp @@ -0,0 +1,820 @@ +#include "models.h" + +#include +#include +#include + +static size_t xing4_0_elem_offset(const ggml_tensor * t, int64_t i) { + return ggml_row_size(t->type, i); +} + +static ggml_tensor * xing4_0_view_1d(ggml_context * ctx, ggml_tensor * t, int64_t ne0, int64_t i0) { + return ggml_view_1d(ctx, t, ne0, xing4_0_elem_offset(t, i0)); +} + +static ggml_tensor * xing4_0_view_2d( + ggml_context * ctx, + ggml_tensor * t, + int64_t ne0, + int64_t ne1, + int64_t i0) { + return ggml_view_2d(ctx, t, ne0, ne1, t->nb[1], xing4_0_elem_offset(t, i0)); +} + +// mean over the hyper-connection streams: [n_embd, hc, n_tokens] -> [n_embd, n_tokens] +static ggml_tensor * xing4_0_hc_mean(ggml_context * ctx, ggml_tensor * x) { + const int64_t hc = x->ne[1]; + + ggml_tensor * acc = ggml_view_2d(ctx, x, x->ne[0], x->ne[2], x->nb[2], 0); + for (int64_t s = 1; s < hc; ++s) { + acc = ggml_add(ctx, acc, ggml_view_2d(ctx, x, x->ne[0], x->ne[2], x->nb[2], s*x->nb[1])); + } + return ggml_scale(ctx, acc, 1.0f/hc); +} + +static ggml_tensor * xing4_0_hc_affine( + ggml_context * ctx, + ggml_tensor * x, + ggml_tensor * scale, + ggml_tensor * base) { + x = ggml_mul(ctx, x, scale); + x = ggml_add(ctx, x, base); + return x; +} + +void llama_model_xing4_0::load_arch_hparams(llama_model_loader & ml) { + uint32_t n_vocab = 0; + ml.get_key(LLM_KV_VOCAB_SIZE, n_vocab, false) || ml.get_arr_n(LLM_KV_TOKENIZER_LIST, n_vocab, false); + + ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps); + ml.get_key(LLM_KV_LEADING_DENSE_BLOCK_COUNT, hparams.n_layer_dense_lead, false); + ml.get_key(LLM_KV_ATTENTION_Q_LORA_RANK, hparams.n_lora_q); + ml.get_key(LLM_KV_ATTENTION_KV_LORA_RANK, hparams.n_lora_kv); + ml.get_key(LLM_KV_ATTENTION_KEY_LENGTH_MLA, hparams.n_embd_head_k_mla_impl, false); + ml.get_key(LLM_KV_ATTENTION_VALUE_LENGTH_MLA, hparams.n_embd_head_v_mla_impl, false); + ml.get_key_or_arr(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp_arr, hparams.n_layer_all); + ml.get_key(LLM_KV_EXPERT_SHARED_COUNT, hparams.n_expert_shared); + ml.get_key(LLM_KV_EXPERT_WEIGHTS_SCALE, hparams.expert_weights_scale, false); + ml.get_key(LLM_KV_EXPERT_WEIGHTS_NORM, hparams.expert_weights_norm, false); + ml.get_key(LLM_KV_EXPERT_GATING_FUNC, hparams.expert_gating_func, false); + if (hparams.expert_gating_func == LLAMA_EXPERT_GATING_FUNC_TYPE_NONE) { + // fall back for GGUFs that have no expert_gating_func model parameter set + hparams.expert_gating_func = LLAMA_EXPERT_GATING_FUNC_TYPE_SOFTMAX; + } + + // Manifold-Constrained Hyper-Connections (MHC) + // Note: reuse the DSV4 hparams members, the saver writes them unconditionally + ml.get_key(LLM_KV_HYPER_CONNECTION_COUNT, hparams.dsv4_hc_mult); + ml.get_key(LLM_KV_HYPER_CONNECTION_SINKHORN_ITERATIONS, hparams.dsv4_hc_sinkhorn_iters); + ml.get_key(LLM_KV_HYPER_CONNECTION_EPSILON, hparams.dsv4_hc_eps); + GGML_ASSERT(hparams.dsv4_hc_mult == 4 && "xing4_0 requires 4 residual streams"); + + if (ml.get_key(LLM_KV_ROPE_SCALING_YARN_LOG_MUL, hparams.rope_yarn_log_mul, false)) { + // [TAG_DEEPSEEK2_YARN_LOG_MUL_FIX] + // cancel the factor from the convert script + hparams.rope_yarn_log_mul /= 0.1f; + } +} + +void llama_model_xing4_0::load_arch_tensors(llama_model_loader & ml) { + LLAMA_LOAD_LOCALS; + const int64_t n_expert_shared = hparams.n_expert_shared; + const int64_t xing4_0_hc_dim = hparams.dsv4_hc_mult * n_embd; + const int64_t xing4_0_hc_mix_dim = (2 + hparams.dsv4_hc_mult) * hparams.dsv4_hc_mult; + + const bool mtp_only = (hparams.n_layer_nextn > 0) && (ml.get_weight("blk.0.attn_norm.weight") == nullptr); + const std::string mtp_probe = "blk." + std::to_string(n_layer) + ".nextn.eh_proj.weight"; + const bool trunk_only = (hparams.n_layer_nextn > 0) && (ml.get_weight(mtp_probe.c_str()) == nullptr); + const int trunk_flags = mtp_only ? TENSOR_NOT_REQUIRED : 0; + int mtp_flags = trunk_only ? TENSOR_NOT_REQUIRED : 0; + + if (!ml.load_mtp) { + mtp_flags |= TENSOR_SKIP; + } + + GGML_ASSERT(hparams.is_mla() && "xing4_0 requires MLA"); + GGML_ASSERT(hparams.n_lora_q > 0 && "xing4_0 requires q_lora_rank"); + + const int64_t n_embd_head_k_mla = hparams.n_embd_head_k_mla(); + const int64_t n_embd_head_v_mla = hparams.n_embd_head_v_mla(); + + const int64_t n_embd_head_qk_rope = hparams.n_rot(); + const int64_t n_embd_head_qk_nope = n_embd_head_k_mla - n_embd_head_qk_rope; + GGML_ASSERT(n_embd_head_qk_nope >= 1); + + const int64_t q_lora_rank = hparams.n_lora_q; + const int64_t kv_lora_rank = hparams.n_lora_kv; + + const int64_t n_ff_exp = hparams.n_ff_exp(); + + tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, 0); + + // output + output_norm = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd}, 0); + // try to load output.weight, if not found, use token_embd (tied embeddings) + output = create_tensor(tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab}, TENSOR_NOT_REQUIRED); + if (!output) { + output = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, TENSOR_DUPLICATED); + } + + for (int i = 0; i < n_layer_all; ++i) { + auto & layer = layers[i]; + const int flags = i < n_layer ? trunk_flags : mtp_flags; + + layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd}, flags); + layer.attn_q_a_norm = create_tensor(tn(LLM_TENSOR_ATTN_Q_A_NORM, "weight", i), {q_lora_rank}, flags); + layer.attn_kv_a_norm = create_tensor(tn(LLM_TENSOR_ATTN_KV_A_NORM, "weight", i), {kv_lora_rank}, flags); + + layer.wq_a = create_tensor(tn(LLM_TENSOR_ATTN_Q_A, "weight", i), {n_embd, q_lora_rank}, flags); + layer.wq_b = create_tensor(tn(LLM_TENSOR_ATTN_Q_B, "weight", i), {q_lora_rank, n_head * n_embd_head_k_mla}, flags); + + layer.wkv_a_mqa = create_tensor(tn(LLM_TENSOR_ATTN_KV_A_MQA, "weight", i), {n_embd, kv_lora_rank + n_embd_head_qk_rope}, flags); + + // note: only old legacy GGUF files will have the unsplit wkv_b tensor in + layer.wk_b = create_tensor(tn(LLM_TENSOR_ATTN_K_B, "weight", i), {n_embd_head_qk_nope, kv_lora_rank, n_head}, flags); + layer.wv_b = create_tensor(tn(LLM_TENSOR_ATTN_V_B, "weight", i), {kv_lora_rank, n_embd_head_v_mla, n_head}, flags); + + layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_head * n_embd_head_v_mla, n_embd}, flags); + + // MHC tensors (trunk layers only) + if (i < n_layer) { + layer.hc_attn_fn = create_tensor(tn(LLM_TENSOR_HC_ATTN_FN, "weight", i), {xing4_0_hc_dim, xing4_0_hc_mix_dim}, flags); + layer.hc_attn_base = create_tensor(tn(LLM_TENSOR_HC_ATTN_BASE, "weight", i), {xing4_0_hc_mix_dim}, flags); + layer.hc_attn_scale = create_tensor(tn(LLM_TENSOR_HC_ATTN_SCALE, "weight", i), {3}, flags); + layer.hc_ffn_fn = create_tensor(tn(LLM_TENSOR_HC_FFN_FN, "weight", i), {xing4_0_hc_dim, xing4_0_hc_mix_dim}, flags); + layer.hc_ffn_base = create_tensor(tn(LLM_TENSOR_HC_FFN_BASE, "weight", i), {xing4_0_hc_mix_dim}, flags); + layer.hc_ffn_scale = create_tensor(tn(LLM_TENSOR_HC_FFN_SCALE, "weight", i), {3}, flags); + } + + layer.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd}, flags); + + if (i < (int) hparams.n_layer_dense_lead) { + layer.ffn_gate = create_tensor(tn(LLM_TENSOR_FFN_GATE, "weight", i), {n_embd, n_ff}, flags); + layer.ffn_down = create_tensor(tn(LLM_TENSOR_FFN_DOWN, "weight", i), { n_ff, n_embd}, flags); + layer.ffn_up = create_tensor(tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff}, flags); + } else { + layer.ffn_gate_inp = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP, "weight", i), {n_embd, n_expert}, flags); + layer.ffn_exp_probs_b = create_tensor(tn(LLM_TENSOR_FFN_EXP_PROBS_B, "bias", i), {n_expert}, TENSOR_NOT_REQUIRED | flags); + + if (n_expert == 0) { + throw std::runtime_error("n_expert must be > 0"); + } + if (n_expert_used == 0) { + throw std::runtime_error("n_expert_used must be > 0"); + } + + // MoE branch + layer.ffn_down_exps = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS, "weight", i), {n_ff_exp, n_embd, n_expert}, flags); + create_tensor_gate_up_exps(layer, i, n_embd, n_ff_exp, n_expert, flags); + + // Shared expert branch + layer.ffn_gate_shexp = create_tensor(tn(LLM_TENSOR_FFN_GATE_SHEXP, "weight", i), {n_embd, n_ff_exp * n_expert_shared}, flags); + layer.ffn_down_shexp = create_tensor(tn(LLM_TENSOR_FFN_DOWN_SHEXP, "weight", i), { n_ff_exp * n_expert_shared, n_embd}, flags); + layer.ffn_up_shexp = create_tensor(tn(LLM_TENSOR_FFN_UP_SHEXP, "weight", i), {n_embd, n_ff_exp * n_expert_shared}, flags); + } + + // NextN/MTP tensors + if (i >= n_layer) { + layer.nextn.eh_proj = create_tensor(tn(LLM_TENSOR_NEXTN_EH_PROJ, "weight", i), { 2 * n_embd, n_embd }, flags); + layer.nextn.enorm = create_tensor(tn(LLM_TENSOR_NEXTN_ENORM, "weight", i), { n_embd }, flags); + layer.nextn.hnorm = create_tensor(tn(LLM_TENSOR_NEXTN_HNORM, "weight", i), { n_embd }, flags); + layer.nextn.embed_tokens = create_tensor(tn(LLM_TENSOR_NEXTN_EMBED_TOKENS, "weight", i), { n_embd, n_vocab }, TENSOR_NOT_REQUIRED | flags); + layer.nextn.shared_head_head = create_tensor(tn(LLM_TENSOR_NEXTN_SHARED_HEAD_HEAD, "weight", i), { n_embd, n_vocab }, TENSOR_NOT_REQUIRED | flags); + layer.nextn.shared_head_norm = create_tensor(tn(LLM_TENSOR_NEXTN_SHARED_HEAD_NORM, "weight", i), { n_embd }, TENSOR_NOT_REQUIRED | flags); + } + } +} + +std::unique_ptr llama_model_xing4_0::build_arch_graph(const llm_graph_params & params) const { + if (params.gtype == LLM_GRAPH_TYPE_DECODER_MTP) { + return std::make_unique(*this, params); + } + return std::make_unique(*this, params); +} + +llama_model_xing4_0::graph_mtp::graph_mtp(const llama_model & model, const llm_graph_params & params) : + llm_graph_context(params) { + GGML_ASSERT(hparams.n_layer_nextn > 0 && "xing4_0 MTP requires n_layer_nextn > 0"); + GGML_ASSERT(hparams.n_layer_nextn == 1 && "xing4_0 MTP currently only supports a single MTP block"); + GGML_ASSERT(hparams.is_mla() && "xing4_0 MTP requires MLA"); + + // The appended MTP block is stored immediately after the main decoder layers. + const int il = hparams.n_layer(); + const auto & layer = model.layers[il]; + + GGML_ASSERT(layer.nextn.eh_proj && "MTP block missing nextn.eh_proj"); + GGML_ASSERT(layer.nextn.enorm && "MTP block missing nextn.enorm"); + GGML_ASSERT(layer.nextn.hnorm && "MTP block missing nextn.hnorm"); + + GGML_ASSERT((uint32_t) il >= hparams.n_layer_dense_lead && "xing4_0 MTP block expected to use MoE FFN"); + + const int64_t n_embd_head_k_mla = hparams.n_embd_head_k_mla(); + const int64_t n_embd_head_qk_rope = hparams.n_rot(); + const int64_t n_embd_head_qk_nope = n_embd_head_k_mla - n_embd_head_qk_rope; + const int64_t kv_lora_rank = hparams.n_lora_kv; + + GGML_ASSERT(n_embd_head_qk_nope >= 1); + GGML_ASSERT(hparams.n_lora_q > 0); + + auto inp = std::make_unique(hparams.n_embd_inp(), hparams.n_embd); + + inp->tokens = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_tokens); + ggml_set_input(inp->tokens); + + inp->embd = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, hparams.n_embd_inp(), n_tokens); + ggml_set_input(inp->embd); + + ggml_tensor * tok_embd; + ASSERT_EMBD_OR_TOKEN(ubatch); + if (ubatch.token) { + ggml_tensor * tok_embd_w = layer.nextn.embed_tokens + ? layer.nextn.embed_tokens + : model.tok_embd; + + tok_embd = ggml_get_rows(ctx0, tok_embd_w, inp->tokens); + } else { + tok_embd = inp->embd; + } + cb(tok_embd, "mtp_tok_embd", il); + + inp->h = build_inp_embd_state(inp.get(), hparams.n_embd); + + ggml_tensor * h_embd = inp->h; + + res->add_input(std::move(inp)); + + ggml_tensor * inp_pos = build_inp_pos(); + ggml_tensor * inp_out_ids = build_inp_out_ids(); + + auto * inp_attn_k = build_attn_inp_k(); + + ggml_tensor * h_norm = build_norm(h_embd, layer.nextn.hnorm, nullptr, LLM_NORM_RMS, il); + cb(h_norm, "mtp_hnorm", il); + + ggml_tensor * e_norm = build_norm(tok_embd, layer.nextn.enorm, nullptr, LLM_NORM_RMS, il); + cb(e_norm, "mtp_enorm", il); + + ggml_tensor * concat = ggml_concat(ctx0, e_norm, h_norm, 0); + cb(concat, "mtp_concat", il); + + ggml_tensor * cur = build_lora_mm(layer.nextn.eh_proj, concat, layer.nextn.eh_proj_s); + cb(cur, "mtp_eh_proj", il); + + ggml_tensor * inpSA = cur; + + cur = build_norm(cur, layer.attn_norm, nullptr, LLM_NORM_RMS, il); + cb(cur, "mtp_attn_norm", il); + + ggml_tensor * q = ggml_mul_mat(ctx0, layer.wq_a, cur); + cb(q, "mtp_q_a", il); + + q = build_norm(q, layer.attn_q_a_norm, nullptr, LLM_NORM_RMS, il); + cb(q, "mtp_q_a_norm", il); + + q = ggml_mul_mat(ctx0, layer.wq_b, q); + cb(q, "mtp_q_b", il); + + ggml_tensor * q_nope = + ggml_view_3d(ctx0, q, n_embd_head_qk_nope, n_head, n_tokens, + ggml_row_size(q->type, n_embd_head_k_mla), + ggml_row_size(q->type, n_embd_head_k_mla) * n_head, 0); + cb(q_nope, "mtp_q_nope", il); + + ggml_tensor * q_pe = + ggml_view_3d(ctx0, q, n_embd_head_qk_rope, n_head, n_tokens, + ggml_row_size(q->type, n_embd_head_k_mla), + ggml_row_size(q->type, n_embd_head_k_mla) * n_head, + ggml_row_size(q->type, n_embd_head_qk_nope)); + cb(q_pe, "mtp_q_pe", il); + + ggml_tensor * kv_cmpr_pe = ggml_mul_mat(ctx0, layer.wkv_a_mqa, cur); + cb(kv_cmpr_pe, "mtp_kv_cmpr_pe", il); + + ggml_tensor * kv_cmpr = + ggml_view_2d(ctx0, kv_cmpr_pe, kv_lora_rank, n_tokens, + ggml_row_size(kv_cmpr_pe->type, kv_lora_rank + n_embd_head_qk_rope), 0); + cb(kv_cmpr, "mtp_kv_cmpr", il); + + ggml_tensor * k_pe = + ggml_view_3d(ctx0, kv_cmpr_pe, n_embd_head_qk_rope, 1, n_tokens, + ggml_row_size(kv_cmpr_pe->type, kv_lora_rank + n_embd_head_qk_rope), + ggml_row_size(kv_cmpr_pe->type, kv_lora_rank + n_embd_head_qk_rope), + ggml_row_size(kv_cmpr_pe->type, kv_lora_rank)); + cb(k_pe, "mtp_k_pe", il); + + kv_cmpr = build_norm(kv_cmpr, layer.attn_kv_a_norm, nullptr, LLM_NORM_RMS, il); + cb(kv_cmpr, "mtp_kv_cmpr_norm", il); + + GGML_ASSERT(ext_factor >= 0.0f); + + const float attn_factor_org = + attn_factor * (1.0f + 0.1f * logf(1.0f / freq_scale)); + + const float mscale = + attn_factor_org * (1.0f + 0.1f * hparams.rope_yarn_log_mul * logf(1.0f / freq_scale)); + + const float kq_scale = + 1.0f * mscale * mscale / sqrtf(float(n_embd_head_k_mla)); + + q_pe = build_rope(q_pe, inp_pos, il); + cb(q_pe, "mtp_q_pe_rope", il); + + k_pe = build_rope(k_pe, inp_pos, il); + cb(k_pe, "mtp_k_pe_rope", il); + + q_nope = ggml_permute(ctx0, q_nope, 0, 2, 1, 3); + cb(q_nope, "mtp_q_nope_perm", il); + + ggml_tensor * q_nope_absorbed = ggml_mul_mat(ctx0, layer.wk_b, q_nope); + cb(q_nope_absorbed, "mtp_q_nope_absorbed", il); + + q_nope_absorbed = ggml_permute(ctx0, q_nope_absorbed, 0, 2, 1, 3); + cb(q_nope_absorbed, "mtp_q_nope_absorbed_perm", il); + + ggml_tensor * Qcur = ggml_concat(ctx0, q_nope_absorbed, q_pe, 0); + cb(Qcur, "mtp_Qcur", il); + + kv_cmpr = ggml_reshape_3d(ctx0, kv_cmpr, hparams.n_lora_kv, 1, n_tokens); + cb(kv_cmpr, "mtp_kv_cmpr_reshape", il); + + ggml_tensor * Kcur = ggml_concat(ctx0, kv_cmpr, k_pe, 0); + cb(Kcur, "mtp_Kcur", il); + + ggml_tensor * Vcur = kv_cmpr; + cb(Vcur, "mtp_Vcur", il); + + cur = build_attn(inp_attn_k, + layer.wo, nullptr, layer.wo_s, + Qcur, Kcur, Vcur, nullptr, nullptr, layer.wv_b, kq_scale, il); + cb(cur, "mtp_attn_out", il); + + ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA); + cb(ffn_inp, "mtp_ffn_inp", il); + + cur = build_norm(ffn_inp, layer.ffn_norm, nullptr, LLM_NORM_RMS, il); + cb(cur, "mtp_ffn_norm", il); + + ggml_tensor * moe_out = build_moe_ffn(cur, + layer.ffn_gate_inp, + layer.ffn_up_exps, + layer.ffn_gate_exps, + layer.ffn_down_exps, + layer.ffn_exp_probs_b, + n_expert, n_expert_used, + LLM_FFN_SILU, hparams.expert_weights_norm, + hparams.expert_weights_scale, + (llama_expert_gating_func_type) hparams.expert_gating_func, + il, + nullptr, + layer.ffn_gate_up_exps); + cb(moe_out, "mtp_ffn_moe_out", il); + + ggml_tensor * ffn_shexp = build_ffn(cur, + layer.ffn_up_shexp, nullptr, nullptr, + layer.ffn_gate_shexp, nullptr, nullptr, + layer.ffn_down_shexp, nullptr, nullptr, + nullptr, LLM_FFN_SILU, LLM_FFN_PAR, il); + cb(ffn_shexp, "mtp_ffn_shexp", il); + + cur = ggml_add(ctx0, moe_out, ffn_shexp); + cb(cur, "mtp_ffn_out", il); + + cur = ggml_add(ctx0, cur, ffn_inp); + cb(cur, "mtp_post_ffn", il); + + ggml_tensor * head_norm_w = layer.nextn.shared_head_norm + ? layer.nextn.shared_head_norm + : model.output_norm; + GGML_ASSERT(head_norm_w && "xing4_0 MTP: missing both nextn.shared_head_norm and output_norm"); + + cur = build_norm(cur, head_norm_w, nullptr, LLM_NORM_RMS, -1); + cb(cur, "h_nextn", -1); + res->t_h_nextn = cur; + + if (inp_out_ids) { + cur = ggml_get_rows(ctx0, cur, inp_out_ids); + } + cb(cur, "mtp_shared_head_norm", -1); + + ggml_tensor * head_w = layer.nextn.shared_head_head + ? layer.nextn.shared_head_head + : model.output; + + ggml_tensor * head_s = layer.nextn.shared_head_head + ? layer.nextn.shared_head_head_s + : model.output_s; + + GGML_ASSERT(head_w && "xing4_0 MTP: missing LM head (nextn.shared_head_head or model.output)"); + + cur = build_lora_mm(head_w, cur, head_s); + cb(cur, "result_output", -1); + + res->t_logits = cur; +} + +ggml_tensor * llama_model_xing4_0::graph::build_hc_pre( + ggml_tensor * x, + ggml_tensor * weights, + int il) const { + GGML_UNUSED(il); + + GGML_ASSERT(x->ne[0] == n_embd); + GGML_ASSERT(x->ne[1] == hparams.dsv4_hc_mult); + + const int64_t hc = hparams.dsv4_hc_mult; + const int64_t nt = x->ne[2]; + + if (cparams.fused_dsv4_hc_pre && il >= 0) { + ggml_tensor * result = ggml_dsv4_hc_pre(ctx0, x, weights); + res->add_fused_node({LLM_FUSED_OP_DSV4_HC_PRE, result, il}); + return result; + } + + ggml_tensor * result = nullptr; + for (int64_t ih = 0; ih < hc; ++ih) { + ggml_tensor * xh = ggml_view_2d(ctx0, x, n_embd, nt, x->nb[2], ih*x->nb[1]); + ggml_tensor * wh = ggml_view_2d(ctx0, weights, 1, nt, weights->nb[1], ih*weights->nb[0]); + ggml_tensor * cur = ggml_mul(ctx0, xh, wh); + result = result ? ggml_add(ctx0, result, cur) : cur; + } + + return result; +} + +ggml_tensor * llama_model_xing4_0::graph::build_hc_sinkhorn( + ggml_tensor * comb, + int il) const { + GGML_UNUSED(il); + + // comb is [src_hc, dst_hc, n_tokens]. Mirrors the reference mhc_pre_big_fuse_with_clamp_tilelang kernel. + // Note: This decomposed fallback (used when fused_dsv4_hc_comb is off) keeps exact reference semantics. + comb = ggml_soft_max(ctx0, comb); + + ggml_tensor * eps = ggml_new_tensor_1d(ctx0, GGML_TYPE_F32, 1); + eps = ggml_fill(ctx0, eps, hparams.dsv4_hc_eps); + + // normalize over ne1 (dst): each src row sums to 1 + auto norm_dst = [&]() { + ggml_tensor * sum = ggml_cont(ctx0, ggml_permute(ctx0, comb, 1, 0, 2, 3)); + sum = ggml_sum_rows(ctx0, sum); + sum = ggml_add(ctx0, sum, eps); + sum = ggml_permute(ctx0, sum, 1, 0, 2, 3); + comb = ggml_div(ctx0, comb, sum); + }; + + // normalize over ne0 (src): each dst column sums to 1 + auto norm_src = [&]() { + ggml_tensor * sum = ggml_sum_rows(ctx0, comb); + sum = ggml_add(ctx0, sum, eps); + comb = ggml_div(ctx0, comb, sum); + }; + + norm_dst(); + for (uint32_t i = 1; i < hparams.dsv4_hc_sinkhorn_iters; ++i) { + norm_src(); + norm_dst(); + } + + return comb; +} + +ggml_tensor * llama_model_xing4_0::graph::build_hc_pre( + ggml_tensor * x, + ggml_tensor * hc_fn, + ggml_tensor * hc_scale, + ggml_tensor * hc_base, + ggml_tensor ** post, + ggml_tensor ** comb, + int il) const { + const int64_t hc = hparams.dsv4_hc_mult; + const int64_t hc_dim = hc*n_embd; + const int64_t hc_mix_dim = (2 + hc)*hc; + const int64_t nt = x->ne[2]; + + GGML_ASSERT(hc == 4); + GGML_ASSERT(hc_fn->ne[1] == hc_mix_dim); + + ggml_tensor * flat = ggml_reshape_2d(ctx0, x, hc_dim, nt); + ggml_tensor * flat_norm = ggml_rms_norm(ctx0, flat, norm_rms_eps); + ggml_tensor * mixes = ggml_mul_mat(ctx0, hc_fn, flat_norm); + cb(mixes, "hc_mixes", il); + + ggml_tensor * scale_pre = xing4_0_view_1d(ctx0, hc_scale, 1, 0); + ggml_tensor * scale_post = xing4_0_view_1d(ctx0, hc_scale, 1, 1); + + ggml_tensor * base_pre = xing4_0_view_1d(ctx0, hc_base, hc, 0); + ggml_tensor * base_post = xing4_0_view_1d(ctx0, hc_base, hc, hc); + + ggml_tensor * pre = xing4_0_view_2d(ctx0, mixes, hc, nt, 0); + pre = xing4_0_hc_affine(ctx0, pre, scale_pre, base_pre); + // the reference with_clamp kernel has no hc_pre_eps on the pre gate + pre = ggml_sigmoid(ctx0, pre); + cb(pre, "hc_pre", il); + + *post = xing4_0_view_2d(ctx0, mixes, hc, nt, hc); + *post = xing4_0_hc_affine(ctx0, *post, scale_post, base_post); + *post = ggml_sigmoid(ctx0, *post); + *post = ggml_scale(ctx0, *post, 2.0f); + cb(*post, "hc_post", il); + + if (cparams.fused_dsv4_hc_comb) { + // feed the xing4_0 mixes buffer unchanged: the flat [src + 4*dst] layout + // makes dsv4's dst-indexed softmax run over xing4_0's src axis, so the + // Sinkhorn phases align and the output is [src, dst] as the decomposed + // path expects (see the file header for the precision notes) + *comb = ggml_dsv4_hc_comb(ctx0, mixes, hc_scale, hc_base, hparams.dsv4_hc_eps, + (int32_t) hparams.dsv4_hc_sinkhorn_iters); + res->add_fused_node({LLM_FUSED_OP_DSV4_HC_COMB, *comb, il}); + } else { + ggml_tensor * scale_comb = xing4_0_view_1d(ctx0, hc_scale, 1, 2); + ggml_tensor * base_comb = xing4_0_view_1d(ctx0, hc_base, hc*hc, 2*hc); + + *comb = xing4_0_view_2d(ctx0, mixes, hc*hc, nt, 2*hc); + *comb = xing4_0_hc_affine(ctx0, *comb, scale_comb, base_comb); + // the reference MHC kernel clamps the raw comb logits before Sinkhorn + *comb = ggml_clamp(ctx0, *comb, -30.0f, 30.0f); + *comb = ggml_reshape_3d(ctx0, *comb, hc, hc, nt); + *comb = build_hc_sinkhorn(*comb, il); + } + cb(*comb, "hc_comb", il); + + ggml_tensor * result = build_hc_pre(x, pre, il); + return result; +} + +ggml_tensor * llama_model_xing4_0::graph::build_hc_post( + ggml_tensor * x, + ggml_tensor * residual, + ggml_tensor * post, + ggml_tensor * comb, + int il) const { + GGML_UNUSED(il); + + GGML_ASSERT(x->ne[0] == n_embd); + GGML_ASSERT(residual->ne[1] == hparams.dsv4_hc_mult); + + if (cparams.fused_dsv4_hc_post) { + // dsv4 reads comb[dst, src]; xing4_0's comb is [src, dst] -- transpose + ggml_tensor * comb_t = ggml_cont(ctx0, ggml_permute(ctx0, comb, 1, 0, 2, 3)); + ggml_tensor * result = ggml_dsv4_hc_post(ctx0, x, residual, post, comb_t); + res->add_fused_node({LLM_FUSED_OP_DSV4_HC_POST, result, il}); + return result; + } + + const int64_t hc = hparams.dsv4_hc_mult; + const int64_t nt = x->ne[1]; + + ggml_tensor * out = nullptr; + for (int64_t dst = 0; dst < hc; ++dst) { + ggml_tensor * post_dst = ggml_view_2d(ctx0, post, 1, nt, post->nb[1], dst*post->nb[0]); + ggml_tensor * cur = ggml_mul(ctx0, x, post_dst); + + for (int64_t src = 0; src < hc; ++src) { + ggml_tensor * res_src = ggml_view_2d(ctx0, residual, n_embd, nt, residual->nb[2], src*residual->nb[1]); + // comb is [src_hc, dst_hc, n_tokens]: the element [src, dst] holds the + // mixing weight from residual stream src to output stream dst. + // NOTE: xing4_0's vllm reference (with_clamp kernel) transposes the + // comb flat index relative to deepseek4's non-clamp kernel, so here we + // must read comb[src, dst] (deepseek4 reads comb[dst, src]). + ggml_tensor * comb_src_dst = ggml_view_2d(ctx0, comb, 1, nt, comb->nb[2], + src*comb->nb[0] + dst*comb->nb[1]); + cur = ggml_add(ctx0, cur, ggml_mul(ctx0, res_src, comb_src_dst)); + } + + cur = ggml_reshape_3d(ctx0, cur, n_embd, 1, nt); + out = out ? ggml_concat(ctx0, out, cur, 1) : cur; + } + + return out; +} + +llama_model_xing4_0::graph::graph(const llama_model & model, const llm_graph_params & params) : + llm_graph_context(params) { + const int64_t hc = hparams.dsv4_hc_mult; + + const int64_t n_embd_head_k = hparams.n_embd_head_k_mla(); + + const int64_t n_embd_head_qk_rope = hparams.n_rot(); + const int64_t n_embd_head_qk_nope = n_embd_head_k - n_embd_head_qk_rope; + + const uint32_t kv_lora_rank = hparams.n_lora_kv; + + // We have to pre-scale kq_scale and attn_factor to make the YaRN RoPE work correctly. + // See https://github.com/ggml-org/llama.cpp/discussions/7416 for detailed explanation. + // And also: https://github.com/ggml-org/llama.cpp/pull/17945 [TAG_DEEPSEEK2_YARN_LOG_MUL_FIX] + + // first cancel the adjustment from llama_hparams::yarn_attn_factor_adjust to get the original attn_factor + GGML_ASSERT(ext_factor >= 0.0f); + const float attn_factor_org = attn_factor * (1.0f + 0.1f * logf(1.0f / freq_scale)); + + // use the original attn_factor to pre-scale the kq_scale + const float mscale = attn_factor_org * (1.0f + 0.1f * hparams.rope_yarn_log_mul * logf(1.0f / freq_scale)); + const float kq_scale = 1.0f * mscale * mscale / sqrtf(float(n_embd_head_k)); + + ggml_tensor * cur; + ggml_tensor * inpL; + + ggml_tensor * inp = build_inp_embd(model.tok_embd); + ggml_tensor * inp_pos = build_inp_pos(); + + auto * inp_attn_k = build_attn_inp_k(); + + ggml_tensor * inp_out_ids = build_inp_out_ids(); + + inpL = ggml_reshape_3d(ctx0, inp, n_embd, 1, n_tokens); + inpL = ggml_repeat_4d(ctx0, inpL, n_embd, hc, n_tokens, 1); + cb(inpL, "hc_init", -1); + + for (int il = 0; il < n_layer; ++il) { + const auto & layer = model.layers[il]; + + ggml_tensor * residual = inpL; + ggml_tensor * post = nullptr; + ggml_tensor * comb = nullptr; + + cur = build_hc_pre(inpL, + layer.hc_attn_fn, + layer.hc_attn_scale, + layer.hc_attn_base, + &post, &comb, il); + cb(cur, "hc_attn_pre", il); + + cur = build_norm(cur, layer.attn_norm, nullptr, LLM_NORM_RMS, il); + cb(cur, "attn_norm", il); + + // MLA self-attention (identical to deepseek2) + ggml_tensor * q = ggml_mul_mat(ctx0, layer.wq_a, cur); + cb(q, "q_a", il); + + q = build_norm(q, layer.attn_q_a_norm, nullptr, LLM_NORM_RMS, il); + cb(q, "q_anorm", il); + + q = ggml_mul_mat(ctx0, layer.wq_b, q); + cb(q, "q_b", il); + + // split into {n_embd_head_qk_nope, n_head, n_tokens} + ggml_tensor * q_nope = + ggml_view_3d(ctx0, q, n_embd_head_qk_nope, n_head, n_tokens, ggml_row_size(q->type, n_embd_head_k), + ggml_row_size(q->type, n_embd_head_k) * n_head, 0); + cb(q_nope, "q_nope", il); + + // and {n_embd_head_qk_rope, n_head, n_tokens} + ggml_tensor * q_pe = ggml_view_3d( + ctx0, q, n_embd_head_qk_rope, n_head, n_tokens, ggml_row_size(q->type, n_embd_head_k), + ggml_row_size(q->type, n_embd_head_k) * n_head, ggml_row_size(q->type, n_embd_head_qk_nope)); + cb(q_pe, "q_pe_in", il); + + ggml_tensor * kv_cmpr_pe = ggml_mul_mat(ctx0, layer.wkv_a_mqa, cur); + cb(kv_cmpr_pe, "kv_cmpr_pe", il); + + // split into {kv_lora_rank, n_tokens} + ggml_tensor * kv_cmpr = + ggml_view_2d(ctx0, kv_cmpr_pe, kv_lora_rank, n_tokens, + ggml_row_size(kv_cmpr_pe->type, kv_lora_rank + n_embd_head_qk_rope), 0); + cb(kv_cmpr, "kv_cmpr_in", il); + + // and {n_embd_head_qk_rope, 1, n_tokens} + ggml_tensor * k_pe = ggml_view_3d(ctx0, kv_cmpr_pe, n_embd_head_qk_rope, 1, n_tokens, + ggml_row_size(kv_cmpr_pe->type, kv_lora_rank + n_embd_head_qk_rope), + ggml_row_size(kv_cmpr_pe->type, kv_lora_rank + n_embd_head_qk_rope), + ggml_row_size(kv_cmpr_pe->type, kv_lora_rank)); + cb(k_pe, "k_pe_in", il); + + q_pe = build_rope(q_pe, inp_pos, il); + cb(q_pe, "q_pe", il); + + k_pe = build_rope(k_pe, inp_pos, il); + cb(k_pe, "k_pe", il); + + kv_cmpr = build_norm(kv_cmpr, layer.attn_kv_a_norm, nullptr, LLM_NORM_RMS, il); + cb(kv_cmpr, "kv_cmpr", il); + + // {n_embd_head_qk_nope, n_tokens, n_head} + q_nope = ggml_permute(ctx0, q_nope, 0, 2, 1, 3); + cb(q_nope, "q_nope_perm", il); + + // {n_embd_head_qk_nope, kv_lora_rank, n_head} x {n_embd_head_qk_nope, n_tokens, n_head} + ggml_tensor * q_nope_absorbed = ggml_mul_mat(ctx0, layer.wk_b, q_nope); + cb(q_nope_absorbed, "q_nope_absorbed", il); + + // {kv_lora_rank, n_head, n_tokens} + q_nope_absorbed = ggml_permute(ctx0, q_nope_absorbed, 0, 2, 1, 3); + cb(q_nope_absorbed, "q_nope_absorbed_perm", il); + + // {n_embd_head_qk_rope + kv_lora_rank, n_head, n_tokens} + // note: rope must go first for in-place context shifting in build_rope_shift() + ggml_tensor * Qcur = ggml_concat(ctx0, q_nope_absorbed, q_pe, 0); + cb(Qcur, "Qcur", il); + + kv_cmpr = ggml_reshape_3d(ctx0, kv_cmpr, kv_lora_rank, 1, n_tokens); + cb(kv_cmpr, "kv_cmpr_reshape", il); + + // {n_embd_head_qk_rope + kv_lora_rank, 1, n_tokens} + ggml_tensor * Kcur = ggml_concat(ctx0, kv_cmpr, k_pe, 0); + cb(Kcur, "Kcur", il); + + // {kv_lora_rank, 1, n_tokens} + ggml_tensor * Vcur = kv_cmpr; + cb(Vcur, "Vcur", il); + + // note: MLA with the absorption optimization converts into MQA (ie: GQA with 1 group) + cur = build_attn(inp_attn_k, + layer.wo, nullptr, layer.wo_s, + Qcur, Kcur, Vcur, nullptr, nullptr, layer.wv_b, kq_scale, il); + cb(cur, "attn_out", il); + + if (il == n_layer - 1 && inp_out_ids && (!cparams.embeddings_nextn || cparams.embeddings_nextn_masked)) { + cur = ggml_get_rows(ctx0, cur, inp_out_ids); + residual = ggml_reshape_3d(ctx0, + ggml_get_rows(ctx0, ggml_reshape_2d(ctx0, residual, n_embd*hc, n_tokens), inp_out_ids), + n_embd, hc, n_outputs); + post = ggml_get_rows(ctx0, post, inp_out_ids); + comb = ggml_reshape_3d(ctx0, + ggml_get_rows(ctx0, ggml_reshape_2d(ctx0, comb, hc*hc, n_tokens), inp_out_ids), + hc, hc, n_outputs); + } + + inpL = build_hc_post(cur, residual, post, comb, il); + cb(inpL, "hc_attn_post", il); + + residual = inpL; + cur = build_hc_pre(inpL, + layer.hc_ffn_fn, + layer.hc_ffn_scale, + layer.hc_ffn_base, + &post, &comb, il); + cb(cur, "hc_ffn_pre", il); + + ggml_build_forward_expand(gf, residual); + ggml_build_forward_expand(gf, post); + ggml_build_forward_expand(gf, comb); + + cur = build_norm(cur, layer.ffn_norm, nullptr, LLM_NORM_RMS, il); + cb(cur, "ffn_norm", il); + + if ((uint32_t) il < hparams.n_layer_dense_lead) { + cur = build_ffn(cur, + layer.ffn_up, nullptr, nullptr, + layer.ffn_gate, nullptr, nullptr, + layer.ffn_down, nullptr, nullptr, + nullptr, LLM_FFN_SILU, LLM_FFN_PAR, il); + cb(cur, "ffn_out", il); + } else { + // MoE branch + ggml_tensor * moe_out = build_moe_ffn(cur, + layer.ffn_gate_inp, + layer.ffn_up_exps, + layer.ffn_gate_exps, + layer.ffn_down_exps, + layer.ffn_exp_probs_b, + n_expert, n_expert_used, + LLM_FFN_SILU, hparams.expert_weights_norm, + hparams.expert_weights_scale, + (llama_expert_gating_func_type) hparams.expert_gating_func, + il, + nullptr, + layer.ffn_gate_up_exps); + cb(moe_out, "ffn_moe_out", il); + + // FFN shared expert + { + ggml_tensor * ffn_shexp = + build_ffn(cur, + layer.ffn_up_shexp, nullptr, nullptr, + layer.ffn_gate_shexp, nullptr, nullptr, + layer.ffn_down_shexp, nullptr, nullptr, + nullptr, LLM_FFN_SILU, LLM_FFN_PAR, il); + cb(ffn_shexp, "ffn_shexp", il); + + cur = ggml_add(ctx0, moe_out, ffn_shexp); + cb(cur, "ffn_out", il); + } + } + + inpL = build_hc_post(cur, residual, post, comb, il); + inpL = build_cvec(inpL, il); + cb(inpL, "l_out", il); + } + cur = inpL; + + // average the residual streams -> single stream for the head + cur = xing4_0_hc_mean(ctx0, cur); + cb(cur, "hc_mean", -1); + + cur = build_norm(cur, model.output_norm, nullptr, LLM_NORM_RMS, -1); + cb(cur, "h_nextn", -1); + res->t_h_nextn = cur; + + if (cparams.embeddings_nextn && !cparams.embeddings_nextn_masked && inp_out_ids) { + cur = ggml_get_rows(ctx0, cur, inp_out_ids); + } + + cb(cur, "result_norm", -1); + res->t_embd = cur; + + // lm_head + cur = ggml_mul_mat(ctx0, model.output, cur); + + cb(cur, "result_output", -1); + res->t_logits = cur; +} diff --git a/src/models/xverse.cpp b/src/models/xverse.cpp index bfc539e8da1e..e0710bd9cecd 100644 --- a/src/models/xverse.cpp +++ b/src/models/xverse.cpp @@ -35,10 +35,10 @@ void llama_model_xverse::load_arch_tensors(llama_model_loader &) { } std::unique_ptr llama_model_xverse::build_arch_graph(const llm_graph_params & params) const { - return std::make_unique(*this, params); + return std::make_unique(params); } -llama_model_xverse::graph::graph(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) { +llama_model_xverse::graph::graph(const llm_graph_params & params) : llm_graph_context(params) { const int64_t n_embd_head = hparams.n_embd_head_v(); GGML_ASSERT(n_embd_head == hparams.n_embd_head_k()); @@ -69,17 +69,8 @@ llama_model_xverse::graph::graph(const llama_model & model, const llm_graph_para auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur, n_embd_head, n_head, n_head_kv, il); - Qcur = ggml_rope_ext( - ctx0, Qcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); - - Kcur = ggml_rope_ext( - ctx0, Kcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); + Qcur = build_rope(Qcur, inp_pos, il); + Kcur = build_rope(Kcur, inp_pos, il); cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 4224a15aef60..36e48a0f2b44 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -161,7 +161,7 @@ if (NOT WIN32) NAME test-tokenizers-ggml-vocabs LABEL behavior WORKING_DIRECTORY ${CMAKE_RUNTIME_OUTPUT_DIRECTORY} - ARGS https://huggingface.co/ggml-org/vocabs ${PROJECT_SOURCE_DIR}/models/ggml-vocabs + ARGS https://huggingface.co/ggml-org-dev/vocabs ${PROJECT_SOURCE_DIR}/models/ggml-vocabs ) endif() @@ -190,6 +190,8 @@ if (NOT WIN32 OR NOT BUILD_SHARED_LIBS) llama_build_and_test(test-grammar-integration.cpp LABEL behavior) llama_build_and_test(test-llama-grammar.cpp LABEL behavior LINK llama) llama_build_and_test(test-batch-alloc.cpp LABEL behavior LINK llama) + llama_build_and_test(test-kv-cache.cpp LABEL behavior) + target_include_directories(test-kv-cache PRIVATE ${PROJECT_SOURCE_DIR}/src) llama_build_and_test(test-chat.cpp LABEL behavior WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}) target_include_directories(test-chat PRIVATE ${PROJECT_SOURCE_DIR}/tools/server) target_link_libraries(test-chat PRIVATE server-context) @@ -237,7 +239,9 @@ if (GGML_CPU AND (NOT WIN32 OR NOT BUILD_SHARED_LIBS)) set_tests_properties(test-generate-fp8-models PROPERTIES FIXTURES_SETUP fp8-models) llama_test(test-fp8 NAME test-fp8-requantize LABEL "main behavior backend" ARGS --models "${FP8_MODEL_DIR}") - set_tests_properties(test-fp8-requantize PROPERTIES FIXTURES_REQUIRED fp8-models) + llama_test(test-fp8 NAME test-fp8-kv LABEL "main behavior backend" + ARGS --models "${FP8_MODEL_DIR}" --kv) + set_tests_properties(test-fp8-requantize test-fp8-kv PROPERTIES FIXTURES_REQUIRED fp8-models) endif() set(MODEL_DIR "${CMAKE_CURRENT_BINARY_DIR}/test-models/") @@ -296,6 +300,12 @@ llama_test_cmd(python3 WORKING_DIRECTORY ${PROJECT_SOURCE_DIR} ARGS -m unittest discover -s tests -p test_conversion_fp8.py -v ) +llama_test_cmd(python3 + NAME test-kv-cache-scales + LABEL "python behavior" + WORKING_DIRECTORY ${PROJECT_SOURCE_DIR} + ARGS -m unittest discover -s tests -p test_kv_cache_scales.py -v +) llama_build_and_test(test-json-schema.cpp LABEL behavior) llama_build_and_test(test-chat-auto-parser.cpp LABEL behavior WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}) llama_build_and_test(test-chat-template.cpp LABEL behavior) @@ -315,15 +325,16 @@ llama_build_and_test( ) if (NOT ${CMAKE_SYSTEM_PROCESSOR} MATCHES "s390x") - set(MODEL_NAME "tinyllamas/stories15M-q4_0.gguf") + set(MODEL_NAME "stories15M-q4_0.gguf") set(MODEL_HASH "SHA256=66967fbece6dbe97886593fdbb73589584927e29119ec31f08090732d1861739") else() - set(MODEL_NAME "tinyllamas/stories15M-be.Q4_0.gguf") + set(MODEL_NAME "stories15M-be.Q4_0.gguf") set(MODEL_HASH "SHA256=9aec857937849d976f30397e97eb1cabb53eb9dcb1ce4611ba8247fb5f44c65d") endif() set(MODEL_DEST "${CMAKE_BINARY_DIR}/${MODEL_NAME}") add_test(NAME test-download-model COMMAND ${CMAKE_COMMAND} + -DREPO=ggml-org-dev/stories15M -DDEST=${MODEL_DEST} -DNAME=${MODEL_NAME} -DHASH=${MODEL_HASH} @@ -357,6 +368,13 @@ llama_build(test-backend-ops.cpp LINK ggml Threads::Threads) llama_build_and_test(test-backend-tensor.cpp LINK ggml) +if (BUILD_SHARED_LIBS) + llama_build_and_test(test-llama-mmap.cpp ${PROJECT_SOURCE_DIR}/src/llama-impl.cpp ${PROJECT_SOURCE_DIR}/src/llama-mmap.cpp LABEL behavior) + target_compile_definitions(test-llama-mmap PRIVATE LLAMA_BUILD) +else() + llama_build_and_test(test-llama-mmap.cpp LABEL behavior) +endif() + llama_build_and_test(test-tiled-mulmat.cpp LABEL "main behavior backend" LINK ggml) llama_build_and_test(test-model-load-cancel.cpp LABEL "model behavior") llama_build_and_test(test-autorelease.cpp LABEL "model behavior") diff --git a/tests/fusion/MTL.csv b/tests/fusion/MTL.csv index 254b8a03ae96..89e88418305f 100644 --- a/tests/fusion/MTL.csv +++ b/tests/fusion/MTL.csv @@ -3,6 +3,7 @@ afmoe ,1 ,any ,MUL+ADD , 2 afmoe ,1 ,any ,RMS_NORM+MUL , 10 afmoe ,1 ,any ,RMS_NORM+MUL+ADD , 3 +afmoe ,1 ,any ,UNARY+ADD+ARGSORT+GET_ROWS , 1 arcee ,0 ,any ,RMS_NORM+MUL , 5 arctic ,0 ,any ,MUL+ADD , 4 arctic ,0 ,any ,RMS_NORM+MUL , 7 @@ -15,11 +16,13 @@ bailingmoe ,1 ,any ,SOFT_MAX+ARGSORT+GET_ROWS , 2 bailingmoe2 ,1 ,any ,ADD+ADD , 1 bailingmoe2 ,1 ,any ,MUL+ADD , 2 bailingmoe2 ,1 ,any ,RMS_NORM+MUL , 9 +bailingmoe2 ,1 ,any ,UNARY+ADD+ARGSORT+GET_ROWS , 1 bailingmoe3 ,1 ,any ,ADD+ADD , 1 bailingmoe3 ,1 ,any ,GATED_DELTA_NET+CPY , 1 bailingmoe3 ,1 ,any ,MUL+ADD , 2 bailingmoe3 ,1 ,any ,RMS_NORM+MUL , 8 bailingmoe3 ,1 ,any ,RMS_NORM+SCALE , 2 +bailingmoe3 ,1 ,any ,UNARY+ADD+ARGSORT+GET_ROWS , 1 beam ,0 ,any ,MUL+ADD , 6 beam ,0 ,any ,RMS_NORM+MUL , 25 bloom ,0 ,any ,NORM+MUL+ADD , 6 @@ -50,21 +53,27 @@ deepseek ,0 ,any ,SOFT_MAX+ARGSORT+GET_ROWS , 1 deepseek2 ,0 ,any ,ADD+ADD , 1 deepseek2 ,0 ,any ,MUL+ADD , 2 deepseek2 ,0 ,any ,RMS_NORM+MUL , 9 +deepseek2 ,0 ,any ,UNARY+ADD+ARGSORT+GET_ROWS , 1 deepseek32 ,0 ,any ,ADD+ADD , 1 deepseek32 ,0 ,any ,MUL+ADD , 2 deepseek32 ,0 ,any ,NORM+MUL+ADD , 2 deepseek32 ,0 ,any ,RMS_NORM+MUL , 9 +deepseek32 ,0 ,any ,UNARY+ADD+ARGSORT+GET_ROWS , 1 deepseek4 ,0 ,any ,MUL+ADD , 8 deepseek4 ,0 ,any ,RMS_NORM+MUL , 20 +deepseek4 ,0 ,any ,UNARY+SQRT+ADD+ARGSORT+GET_ROWS+SUM_ROWS+CLAMP+DIV, 4 deepseek41 ,0 ,any ,MUL+ADD , 12 deepseek41 ,0 ,any ,RMS_NORM+MUL , 29 +deepseek41 ,0 ,any ,UNARY+SQRT+ADD+ARGSORT+GET_ROWS+SUM_ROWS+SCALE+DIV, 6 dots1 ,0 ,any ,ADD+ADD , 1 dots1 ,0 ,any ,MUL+ADD , 2 dots1 ,0 ,any ,RMS_NORM+MUL , 9 +dots1 ,0 ,any ,UNARY+ADD+ARGSORT+GET_ROWS , 1 dots3note ,0 ,any ,ADD+ADD , 1 dots3note ,0 ,any ,MUL+ADD , 2 dots3note ,0 ,any ,NORM+MUL+ADD , 1 dots3note ,0 ,any ,RMS_NORM+MUL , 11 +dots3note ,0 ,any ,UNARY+ADD+ARGSORT+GET_ROWS , 1 dream ,0 ,any ,RMS_NORM+MUL , 5 ernie4_5-moe ,1 ,any ,ADD+ADD , 1 ernie4_5-moe ,1 ,any ,MUL+ADD , 2 @@ -74,6 +83,7 @@ exaone ,0 ,any ,RMS_NORM+MUL , 5 exaone-moe ,1 ,any ,ADD+ADD , 1 exaone-moe ,1 ,any ,MUL+ADD , 2 exaone-moe ,1 ,any ,RMS_NORM+MUL , 9 +exaone-moe ,1 ,any ,UNARY+ADD+ARGSORT+GET_ROWS , 1 exaone4 ,0 ,any ,RMS_NORM+MUL , 5 exaone4 ,0 ,any ,RMS_NORM+MUL+ADD , 4 falcon ,0 ,any ,ADD+ADD , 2 @@ -83,6 +93,7 @@ falcon-h1 ,0 ,any ,RMS_NORM+MUL , 9 flint ,0 ,any ,ADD+ADD , 1 flint ,0 ,any ,MUL+ADD , 2 flint ,0 ,any ,RMS_NORM+MUL , 9 +flint ,0 ,any ,UNARY+ADD+ARGSORT+GET_ROWS+SUM_ROWS+CLAMP+DIV, 1 gemma ,0 ,any ,RMS_NORM+MUL , 5 gemma2 ,0 ,any ,RMS_NORM+MUL , 5 gemma2 ,0 ,any ,RMS_NORM+MUL+ADD , 4 @@ -94,17 +105,20 @@ glm-dsa ,0 ,any ,ADD+ADD , 1 glm-dsa ,0 ,any ,MUL+ADD , 2 glm-dsa ,0 ,any ,NORM+MUL+ADD , 2 glm-dsa ,0 ,any ,RMS_NORM+MUL , 9 +glm-dsa ,0 ,any ,UNARY+ADD+ARGSORT+GET_ROWS , 1 glm4 ,0 ,any ,RMS_NORM+MUL , 5 glm4 ,0 ,any ,RMS_NORM+MUL+ADD , 4 glm4moe ,1 ,any ,ADD+ADD , 1 glm4moe ,1 ,any ,MUL+ADD , 2 glm4moe ,1 ,any ,RMS_NORM+MUL , 9 +glm4moe ,1 ,any ,UNARY+ADD+ARGSORT+GET_ROWS , 1 glm5-next ,0 ,any ,ADD+ADD+ADD , 1 glm5-next ,0 ,any ,GATED_DELTA_NET+CPY , 1 glm5-next ,0 ,any ,MUL+ADD , 2 glm5-next ,0 ,any ,NORM+MUL+ADD , 1 glm5-next ,0 ,any ,RMS_NORM+MUL , 8 glm5-next ,0 ,any ,RMS_NORM+SCALE , 2 +glm5-next ,0 ,any ,UNARY+ADD+ARGSORT+GET_ROWS , 1 gpt-oss ,0 ,any ,MUL+ADD , 4 gpt-oss ,0 ,any ,RMS_NORM+MUL , 5 gpt2 ,0 ,any ,NORM+MUL+ADD , 5 @@ -135,9 +149,11 @@ hunyuan_vl ,0 ,any ,RMS_NORM+MUL , 9 hy_v3 ,0 ,any ,ADD+ADD , 2 hy_v3 ,0 ,any ,MUL+ADD , 4 hy_v3 ,0 ,any ,RMS_NORM+MUL , 9 +hy_v3 ,0 ,any ,UNARY+ADD+ARGSORT+GET_ROWS , 2 hy_v4 ,0 ,any ,MUL+ADD , 2 hy_v4 ,0 ,any ,NORM+MUL+ADD , 1 hy_v4 ,0 ,any ,RMS_NORM+MUL , 9 +hy_v4 ,0 ,any ,UNARY+ADD+ARGSORT+GET_ROWS+SUM_ROWS+CLAMP+DIV, 1 internlm2 ,0 ,any ,RMS_NORM+MUL , 5 jais ,0 ,any ,NORM+MUL+ADD , 5 jais2 ,0 ,any ,NORM+MUL+ADD , 5 @@ -146,24 +162,35 @@ k2-horizon ,0 ,any ,RMS_NORM+MUL , 4 k2-horizon ,0 ,any ,ADD+ADD , 1 k2-horizon ,0 ,any ,MUL+ADD , 2 k2-horizon ,0 ,any ,RMS_NORM+MUL , 4 +k2-horizon ,0 ,any ,UNARY+ADD+ARGSORT+GET_ROWS , 1 kimi-k3 ,0 ,any ,GATED_DELTA_NET+CPY , 1 kimi-k3 ,0 ,any ,MUL+ADD , 2 kimi-k3 ,0 ,any ,RMS_NORM+MUL , 17 kimi-k3 ,0 ,any ,RMS_NORM+SCALE , 2 +kimi-k3 ,0 ,any ,UNARY+ADD+ARGSORT+GET_ROWS , 1 +kimi-k31 ,0 ,any ,GATED_DELTA_NET+CPY , 1 +kimi-k31 ,0 ,any ,MUL+ADD , 2 +kimi-k31 ,0 ,any ,NORM+MUL+ADD , 1 +kimi-k31 ,0 ,any ,RMS_NORM+MUL , 17 +kimi-k31 ,0 ,any ,RMS_NORM+SCALE , 2 +kimi-k31 ,0 ,any ,UNARY+ADD+ARGSORT+GET_ROWS , 1 kimi-linear ,0 ,any ,ADD+ADD , 1 kimi-linear ,0 ,any ,GATED_DELTA_NET+CPY , 1 kimi-linear ,0 ,any ,MUL+ADD , 2 kimi-linear ,0 ,any ,RMS_NORM+MUL , 7 kimi-linear ,0 ,any ,RMS_NORM+SCALE , 2 +kimi-linear ,0 ,any ,UNARY+ADD+ARGSORT+GET_ROWS+SUM_ROWS+CLAMP+DIV, 1 kolibri1 ,0 ,any ,MUL+ADD , 4 kolibri1 ,0 ,any ,RMS_NORM+MUL , 10 kolibri1 ,0 ,any ,RMS_NORM+MUL+ADD , 3 laguna ,0 ,any ,ADD+ADD , 1 laguna ,0 ,any ,MUL+ADD , 2 laguna ,0 ,any ,RMS_NORM+MUL , 9 +laguna ,0 ,any ,UNARY+ADD+ARGSORT+GET_ROWS , 1 lfm2 ,0 ,any ,RMS_NORM+MUL , 7 lfm2moe ,1 ,any ,MUL+ADD , 2 lfm2moe ,1 ,any ,RMS_NORM+MUL , 7 +lfm2moe ,1 ,any ,UNARY+ADD+ARGSORT+GET_ROWS+SUM_ROWS+CLAMP+DIV, 1 lfm3 ,0 ,any ,RMS_NORM+MUL , 17 llada ,0 ,any ,RMS_NORM+MUL , 5 llada-moe ,1 ,any ,MUL+ADD , 4 @@ -188,8 +215,11 @@ maple ,0 ,any ,SOFT_MAX+ARGSORT+GET_ROWS+SUM_ROWS+CLAMP+DIV, mellum ,0 ,any ,MUL+ADD , 4 mellum ,0 ,any ,RMS_NORM+MUL , 9 mellum ,0 ,any ,SOFT_MAX+ARGSORT+GET_ROWS+SUM_ROWS+CLAMP+DIV, 2 +metis16 ,0 ,any ,MUL+ADD , 8 +metis16 ,0 ,any ,RMS_NORM+MUL , 11 mimo2 ,0 ,any ,MUL+ADD , 4 mimo2 ,0 ,any ,RMS_NORM+MUL , 5 +mimo2 ,0 ,any ,UNARY+ADD+ARGSORT+GET_ROWS+SUM_ROWS+CLAMP+DIV, 2 minicpm ,0 ,any ,RMS_NORM+MUL , 5 minicpm ,0 ,any ,MUL+ADD , 4 minicpm ,0 ,any ,RMS_NORM+MUL , 5 @@ -200,9 +230,11 @@ minimax-01 ,0 ,any ,RMS_NORM+MUL , 6 minimax-01 ,0 ,any ,SOFT_MAX+ARGSORT+GET_ROWS+SUM_ROWS+CLAMP+DIV, 2 minimax-m2 ,0 ,any ,MUL+ADD , 4 minimax-m2 ,0 ,any ,RMS_NORM+MUL , 9 +minimax-m2 ,0 ,any ,UNARY+ADD+ARGSORT+GET_ROWS+SUM_ROWS+CLAMP+DIV, 2 minimax-m3 ,0 ,any ,ADD+ADD , 1 minimax-m3 ,0 ,any ,MUL+ADD , 2 minimax-m3 ,0 ,any ,RMS_NORM+MUL , 11 +minimax-m3 ,0 ,any ,UNARY+ADD+ARGSORT+GET_ROWS , 1 mistral-large4 ,0 ,any ,ADD+ADD , 3 mistral-large4 ,0 ,any ,MUL+ADD , 6 mistral-large4 ,0 ,any ,NORM+MUL+ADD , 6 @@ -214,11 +246,14 @@ mistral3 ,0 ,any ,SOFT_MAX+ARGSORT+GET_ROWS+SUM_ROWS+CLAMP+DIV, mistral4 ,0 ,any ,ADD+ADD , 1 mistral4 ,0 ,any ,MUL+ADD , 2 mistral4 ,0 ,any ,RMS_NORM+MUL , 9 +mistral4 ,0 ,any ,UNARY+ADD+ARGSORT+GET_ROWS , 1 mpt ,0 ,any ,NORM+MUL+ADD , 5 muse-glimmer ,0 ,any ,RMS_NORM+MUL , 10 muse-glimmer ,0 ,any ,RMS_NORM+MUL+ADD , 3 -muse-spark ,0 ,any ,MUL+ADD , 4 +muse-spark ,0 ,prefill ,MUL+ADD , 2 +muse-spark ,0 ,decode ,MUL+ADD , 4 muse-spark ,0 ,any ,RMS_NORM+MUL , 14 +muse-spark ,0 ,any ,UNARY+ADD+ARGSORT+GET_ROWS+SUM_ROWS+CLAMP+DIV, 2 nanbeige ,0 ,any ,RMS_NORM+MUL , 5 nemotron ,0 ,any ,NORM+MUL+ADD , 5 nemotron_h ,0 ,any ,RMS_NORM+MUL , 5 @@ -289,6 +324,8 @@ qwen4exp ,0 ,any ,SOFT_MAX+ARGSORT+GET_ROWS+SUM_ROWS+CLAMP+DIV, qwen4exp ,0 ,any ,SSM_CONV+UNARY , 1 refact ,0 ,any ,RMS_NORM+MUL , 5 refact ,0 ,any ,RMS_NORM+MUL , 5 +ricursive ,0 ,any ,RMS_NORM+MUL , 9 +ricursive ,0 ,any ,RMS_NORM+MUL+ADD , 4 rnd1 ,0 ,any ,MUL+ADD , 4 rnd1 ,0 ,any ,RMS_NORM+MUL , 9 rnd1 ,0 ,any ,SOFT_MAX+ARGSORT+GET_ROWS+SUM_ROWS+CLAMP+DIV, 2 @@ -302,4 +339,8 @@ stablelm ,0 ,any ,NORM+MUL+ADD , 5 starcoder ,0 ,any ,NORM+MUL+ADD , 5 starcoder2 ,0 ,any ,NORM+MUL+ADD , 5 talkie ,0 ,any ,ADD+ADD , 2 +xing4_0 ,0 ,any ,ADD+ADD+ADD , 1 +xing4_0 ,0 ,any ,MUL+ADD , 2 +xing4_0 ,0 ,any ,RMS_NORM+MUL , 9 +xing4_0 ,0 ,any ,UNARY+ADD+ARGSORT+GET_ROWS , 1 xverse ,0 ,any ,RMS_NORM+MUL , 5 diff --git a/tests/gguf-model-data.cpp b/tests/gguf-model-data.cpp index 99acada22996..99e289802c82 100644 --- a/tests/gguf-model-data.cpp +++ b/tests/gguf-model-data.cpp @@ -566,7 +566,7 @@ static std::optional fetch_and_parse( static std::filesystem::path get_cache_file_path(const std::filesystem::path & cdir, const std::string & repo_part, const std::string & filename) { std::string fname_part = sanitize_for_path(filename); - return cdir / std::filesystem::u8path(repo_part + "--" + fname_part + ".partial"); + return cdir / std::filesystem::path(repo_part + "--" + fname_part + ".partial"); } // Try cache first, then fetch and parse a single GGUF shard. @@ -601,7 +601,7 @@ std::optional gguf_fetch_model_meta( const std::string & quant, const std::string & cache_dir, bool verbose) { - const std::filesystem::path cdir = cache_dir.empty() ? get_default_cache_dir() : std::filesystem::u8path(cache_dir); + const std::filesystem::path cdir = cache_dir.empty() ? get_default_cache_dir() : std::filesystem::path(cache_dir); std::string repo_part = sanitize_for_path(repo); std::string split_prefix; @@ -662,7 +662,7 @@ gguf_context_ptr gguf_fetch_gguf_ctx( const std::string & quant, const std::string & cache_dir, bool verbose) { - const std::filesystem::path cdir = cache_dir.empty() ? get_default_cache_dir() : std::filesystem::u8path(cache_dir); + const std::filesystem::path cdir = cache_dir.empty() ? get_default_cache_dir() : std::filesystem::path(cache_dir); std::string repo_part = sanitize_for_path(repo); std::string split_prefix; diff --git a/tests/test-arg-parser.cpp b/tests/test-arg-parser.cpp index 543da2aed5d0..537c333949b1 100644 --- a/tests/test-arg-parser.cpp +++ b/tests/test-arg-parser.cpp @@ -245,6 +245,26 @@ static void test(void) { assert(true == common_params_parse(argv.size(), list_str_to_char(argv).data(), params, LLAMA_EXAMPLE_COMMON)); assert(params.model.path == "model_file.gguf"); + { + auto parse_shards = [&list_str_to_char](std::vector args) { + common_params shard_params; + args.insert(args.begin(), {"binary_name", "-m", "model_file.gguf"}); + assert(common_params_parse(args.size(), list_str_to_char(args).data(), shard_params, LLAMA_EXAMPLE_COMMON)); + auto paths = shard_params.model.get_paths(); + assert(paths.front() == "model_file.gguf"); + return std::vector(paths.begin() + 1, paths.end()); + }; + assert(parse_shards({"--shard", "a.gguf"}) == std::vector({"a.gguf"})); + const std::vector two = {"a.gguf", "b.gguf"}; + assert(parse_shards({"--shard", "a.gguf", "--shard", "b.gguf"}) == two); + assert(parse_shards({"--shard", "a.gguf,b.gguf"}) == two); + const std::vector three = {"a.gguf", "b.gguf", "c.gguf"}; + assert(parse_shards({"--shard", "a.gguf,b.gguf", "--shard", "c.gguf"}) == three); + assert(parse_shards({"--shard", "a.gguf", "--shard", "b.gguf", "--shard", "c.gguf"}) == three); + assert(parse_shards({"--ple", "p.gguf"}) == parse_shards({"--shard", "p.gguf"})); + assert(parse_shards({"--shard", "model_file.gguf,a.gguf,b.gguf,a.gguf"}) == two); + } + argv = {"binary_name", "-t", "1234"}; assert(true == common_params_parse(argv.size(), list_str_to_char(argv).data(), params, LLAMA_EXAMPLE_COMMON)); assert(params.cpuparams.n_threads == 1234); diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp index eec0423cbef5..3abdff5fd0da 100644 --- a/tests/test-backend-ops.cpp +++ b/tests/test-backend-ops.cpp @@ -48,9 +48,33 @@ #ifdef __EMSCRIPTEN__ # define N_THREADS 1 #else -# define N_THREADS std::thread::hardware_concurrency() +static unsigned test_hardware_concurrency() { + const char * env = std::getenv("TEST_ARG_HARDWARE_CONCURRENCY"); + if (env != nullptr) { + const long n = std::strtol(env, nullptr, 10); + if (n > 0) { + return (unsigned) n; + } + } + + return std::thread::hardware_concurrency(); +} + +# define N_THREADS test_hardware_concurrency() #endif +static int test_parallel_workers(int default_workers) { + const char * env = std::getenv("TEST_ARG_PARALLEL_WORKERS"); + if (env != nullptr) { + const long n = std::strtol(env, nullptr, 10); + if (n > 0) { + return (int) n; + } + } + + return default_workers; +} + static void init_tensor_uniform(ggml_tensor * tensor, float min = -1.0f, float max = 1.0f) { if (ggml_is_empty(tensor)) { return; @@ -418,7 +442,11 @@ static std::string var_to_str(ggml_type type) { } static std::string var_to_str(ggml_prec prec) { - return prec == GGML_PREC_F32 ? "f32" : "def"; + switch (prec) { + case GGML_PREC_F32: return "f32"; + case GGML_PREC_BF16: return "bf16"; + default: return "def"; + } } static std::string var_to_str(ggml_op_pool pool) { @@ -465,6 +493,7 @@ static std::string var_to_str(ggml_scale_mode mode) { #define VARS_TO_STR15(a, b, c, d, e, f, g, h, i, j, k, l, m, n, o) VAR_TO_STR(a) + "," + VARS_TO_STR14(b, c, d, e, f, g, h, i, j, k, l, m, n, o) #define VARS_TO_STR16(a, b, c, d, e, f, g, h, i, j, k, l, m, n, o, p) VAR_TO_STR(a) + "," + VARS_TO_STR15(b, c, d, e, f, g, h, i, j, k, l, m, n, o, p) #define VARS_TO_STR17(a, b, c, d, e, f, g, h, i, j, k, l, m, n, o, p, q) VAR_TO_STR(a) + "," + VARS_TO_STR16(b, c, d, e, f, g, h, i, j, k, l, m, n, o, p, q) +#define VARS_TO_STR18(a, b, c, d, e, f, g, h, i, j, k, l, m, n, o, p, q, r) VAR_TO_STR(a) + "," + VARS_TO_STR17(b, c, d, e, f, g, h, i, j, k, l, m, n, o, p, q, r) // accept FLT_MAX as infinity static bool isinf_or_max(float f) { @@ -1313,6 +1342,18 @@ struct test_case { } virtual bool run_whole_graph() { return false; } + // the output must be the same in each run on the same backend + virtual bool deterministic(ggml_backend_t backend) { + GGML_UNUSED(backend); + return false; + } + virtual bool same_output(ggml_tensor * t1, ggml_tensor * t2) { + std::vector t1_data(ggml_nbytes(t1)); + std::vector t2_data(ggml_nbytes(t2)); + ggml_backend_tensor_get(t1, t1_data.data(), 0, ggml_nbytes(t1)); + ggml_backend_tensor_get(t2, t2_data.data(), 0, ggml_nbytes(t2)); + return t1_data == t2_data; + } virtual std::vector fusion_test_nodes() { return {}; } virtual bool use_weight_context() { return false; } @@ -1588,6 +1629,24 @@ struct test_case { run_whole_graph() ? fused_nodes_to_verify.data() : nullptr, fused_nodes_to_verify.size()); + // compare backend1 with itself: a second copy of the graph must give the same bytes + if (deterministic(backend1)) { + auto callback_same = [](int index, ggml_tensor * t1, ggml_tensor * t2, void * user_data) -> bool { + callback_userdata * ud = (callback_userdata *) user_data; + if (!ud->tc->same_output(t1, t2)) { + printf("[%s] not deterministic ", ggml_op_desc(t1)); + ud->ok = false; + } + return true; + + GGML_UNUSED(index); + }; + + for (int i = 0; i < 4 && ud.ok; i++) { + ggml_backend_compare_graph_backend(backend1, backend1, gf, callback_same, &ud, &out, 1); + } + } + // Create test result bool test_passed = ud.ok && cmp_ok; std::string error_msg = test_passed ? "" : (!cmp_ok ? "compare failed" : "test failed"); @@ -2799,10 +2858,12 @@ struct test_rope_set_rows : public test_case { const std::array ne_a; int mode; const int n_dims; + const bool yarn; + const bool truncate; const int n_ctx{512}; std::string vars() override { - return VARS_TO_STR5(type, type_idx, ne_a, mode, n_dims); + return VARS_TO_STR7(type, type_idx, ne_a, mode, n_dims, yarn, truncate); } std::string op_desc(ggml_tensor * t) override { @@ -2816,8 +2877,8 @@ struct test_rope_set_rows : public test_case { ggml_type type_idx, std::array ne_a, int mode, - int n_dims = 128) - : type(type), type_idx(type_idx), ne_a(ne_a), mode(mode), n_dims(n_dims) {} + int n_dims = 128, bool yarn = false, bool truncate = true) + : type(type), type_idx(type_idx), ne_a(ne_a), mode(mode), n_dims(n_dims), yarn(yarn), truncate(truncate) {} ggml_tensor * build_graph(ggml_context * ctx) override { ggml_tensor * a = ggml_new_tensor_4d(ctx, GGML_TYPE_F32, ne_a[0], ne_a[1], ne_a[2], 1); @@ -2834,9 +2895,12 @@ struct test_rope_set_rows : public test_case { } ggml_set_name(pos, "pos"); - float fs = 1.4245f; - float ef = 0.7465f; - float af = 1.4245f; + const float fs = yarn ? 1.0f/256 : 1.4245f; + const float ef = yarn ? 1.0f : 0.7465f; + const float af = yarn ? 1.0f : 1.4245f; + const int n_ctx_orig = yarn ? 4096 : 0; + const float freq_base = yarn ? 1000000.0f : 10000.0f; + const float beta_fast = yarn ? 32.0f : 1.0f; ggml_tensor * freq = nullptr; ggml_tensor * rope = nullptr; @@ -2844,15 +2908,18 @@ struct test_rope_set_rows : public test_case { if (is_vision) { GGML_ASSERT(n_dims/4 > 0); int rope_sections[4] = {n_dims/4, n_dims/4, 0, 0}; // Vision-RoPE only use first two dimension for image (x, y) coordinate - rope = ggml_rope_multi(ctx, a, pos, freq, n_dims/2, rope_sections, mode, 0, 10000.0f, fs, ef, af, 1.0f, 1.0f); + rope = ggml_rope_multi(ctx, a, pos, freq, n_dims/2, rope_sections, mode, n_ctx_orig, freq_base, fs, ef, af, beta_fast, 1.0f); } else { GGML_ASSERT(n_dims/3 > 0); int rope_sections[4] = {n_dims/3, n_dims/3, n_dims/3, 0}; - rope = ggml_rope_multi(ctx, a, pos, freq, n_dims, rope_sections, mode, 0, 10000.0f, fs, ef, af, 1.0f, 1.0f); + rope = ggml_rope_multi(ctx, a, pos, freq, n_dims, rope_sections, mode, n_ctx_orig, freq_base, fs, ef, af, beta_fast, 1.0f); } + } else if (yarn) { + rope = ggml_rope_ext(ctx, a, pos, freq, n_dims, mode, n_ctx_orig, freq_base, fs, ef, af, beta_fast, 1.0f); } else { rope = ggml_rope(ctx, a, pos, n_dims, mode); } + ggml_rope_set_truncate(rope, truncate); ggml_tensor * view = ggml_view_2d(ctx, rope, ne_a[0] * ne_a[1], ne_a[2], rope->nb[2], 0); @@ -2880,7 +2947,7 @@ struct test_rope_set_rows : public test_case { const int num_pos_ids = (mode & GGML_ROPE_TYPE_MROPE) ? ne_a[2] * 4 : ne_a[2]; std::vector data(num_pos_ids); for (int i = 0; i < num_pos_ids; i++) { - data[i] = rand() % n_ctx; + data[i] = yarn ? 4096 + i * 67 : rand() % n_ctx; } ggml_backend_tensor_set(t, data.data(), 0, num_pos_ids * sizeof(int)); } else { @@ -2906,6 +2973,8 @@ struct test_rms_norm_mul_rope : public test_case { const bool broadcast; // multiply by a 1D [ne0] weight, as model norm weights are const ggml_type set_rows_type; int mode; + const bool yarn; + const bool truncate; std::string op_desc(ggml_tensor * t) override { GGML_UNUSED(t); @@ -2915,14 +2984,15 @@ struct test_rms_norm_mul_rope : public test_case { bool run_whole_graph() override { return true; } std::string vars() override { - return VARS_TO_STR9(ne, eps, multi_add, mul, rope, set_rows, broadcast, mode, set_rows_type); + return VARS_TO_STR11(ne, eps, multi_add, mul, rope, set_rows, broadcast, mode, set_rows_type, yarn, truncate); } test_rms_norm_mul_rope(std::array ne, float eps = 1e-6f, bool multi_add = false, bool set_rows = false, bool broadcast = false, int mode = GGML_ROPE_TYPE_NORMAL, - bool mul = true, bool rope = true, ggml_type set_rows_type = GGML_TYPE_F16) + bool mul = true, bool rope = true, ggml_type set_rows_type = GGML_TYPE_F16, + bool yarn = false, bool truncate = true) : ne(ne), eps(eps), multi_add(multi_add), mul(mul), rope(rope), set_rows(set_rows), broadcast(broadcast), - set_rows_type(set_rows_type), mode(mode) {} + set_rows_type(set_rows_type), mode(mode), yarn(yarn), truncate(truncate) {} ggml_tensor * build_graph(ggml_context * ctx) override { ggml_tensor * a = ggml_new_tensor_4d(ctx, GGML_TYPE_F32, ne[0], ne[1], ne[2], ne[3]); @@ -2954,14 +3024,21 @@ struct test_rms_norm_mul_rope : public test_case { if (rope) { const bool is_mrope = mode & GGML_ROPE_TYPE_MROPE; ggml_tensor * pos = ggml_new_tensor_1d(ctx, GGML_TYPE_I32, ne[2] * (is_mrope ? 4 : 1)); + const int n_ctx_orig = yarn ? 4096 : 0; + const float freq_base = yarn ? 1000000.0f : 10000.0f; + const float freq_scale = yarn ? 1.0f/256 : 1.0f; + const float ext_factor = yarn ? 1.0f : 0.0f; if (is_mrope) { const int n_dims = ne[0]; int sections[4] = { n_dims/3, n_dims/3, n_dims/3, 0 }; - a = ggml_rope_multi(ctx, a, pos, nullptr, n_dims, sections, mode, 0, 10000.0f, 1.0f, 0.0f, 1.0f, 32.0f, 1.0f); + a = ggml_rope_multi(ctx, a, pos, nullptr, n_dims, sections, mode, n_ctx_orig, freq_base, freq_scale, ext_factor, 1.0f, 32.0f, 1.0f); + } else if (yarn) { + a = ggml_rope_ext(ctx, a, pos, nullptr, ne[0], mode, n_ctx_orig, freq_base, freq_scale, ext_factor, 1.0f, 32.0f, 1.0f); } else { a = ggml_rope(ctx, a, pos, ne[0], mode); } + ggml_rope_set_truncate(a, truncate); } if (set_rows) { @@ -2986,8 +3063,8 @@ struct test_rms_norm_mul_rope : public test_case { init_set_rows_row_ids(t, ne[2] * 2); } else if (t->type == GGML_TYPE_I32) { std::vector data(ggml_nelements(t)); - for (int32_t & value : data) { - value = rand() % 512; + for (size_t i = 0; i < data.size(); i++) { + data[i] = yarn ? 127 + i * 67 : rand() % 512; } ggml_backend_tensor_set(t, data.data(), 0, ggml_nbytes(t)); } else { @@ -3060,14 +3137,15 @@ struct test_argmax : public test_case { struct test_count_equal : public test_case { const ggml_type type; const std::array ne; + const bool same_src; std::string vars() override { - return VARS_TO_STR2(type, ne); + return VARS_TO_STR3(type, ne, same_src); } test_count_equal(ggml_type type = GGML_TYPE_F32, - std::array ne = {4, 500, 1, 1}) - : type(type), ne(ne) {} + std::array ne = {4, 500, 1, 1}, bool same_src = false) + : type(type), ne(ne), same_src(same_src) {} ggml_tensor * build_graph(ggml_context * ctx) override { ggml_tensor * a = ggml_new_tensor(ctx, type, 4, ne.data()); @@ -3082,7 +3160,7 @@ struct test_count_equal : public test_case { ggml_tensor * b_argmax = ggml_argmax(ctx, b); ggml_set_name(b_argmax, "b_argmax"); - ggml_tensor * out = ggml_count_equal(ctx, a_argmax, b_argmax); + ggml_tensor * out = ggml_count_equal(ctx, a_argmax, same_src ? a_argmax : b_argmax); ggml_set_name(out, "out"); return out; @@ -5395,9 +5473,15 @@ struct test_mul_mat : public test_case { const bool src_overlap; // a and b are overlapping views of the same tensor const int64_t m_v; // rows of a in memory, the batches of a are strided for m_v > m, no view for m_v == 0 const int64_t pad; // bytes after the m_v rows of each batch of a, so nb[2] of a is not a multiple of nb[1] + const size_t offset_a; // byte offset of the a view when k_v > k or m_v > m + const bool weights; // a in a weights buffer (CUDA with PDL reads weights early) std::string vars() override { - return VARS_TO_STR13(type_a, type_b, m, n, k, bs, nr, per, k_v, o, src_overlap, m_v, pad); + std::string s = VARS_TO_STR14(type_a, type_b, m, n, k, bs, nr, per, k_v, o, src_overlap, m_v, pad, offset_a); + if (weights) { + s += ",weights=1"; + } + return s; } double max_nmse_err() override { @@ -5431,10 +5515,21 @@ struct test_mul_mat : public test_case { std::array bs = {10, 10}, std::array nr = {2, 2}, std::array per = {0, 1, 2, 3}, - int64_t k_v = 0, uint32_t o = 1, bool src_overlap = false, int64_t m_v = 0, int64_t pad = 0) - : type_a(type_a), type_b(type_b), m(m), n(n), k(k), bs(bs), nr(nr), per(per), k_v(k_v), o(o), src_overlap(src_overlap), m_v(m_v), pad(pad) {} + int64_t k_v = 0, uint32_t o = 1, bool src_overlap = false, int64_t m_v = 0, int64_t pad = 0, size_t offset_a = 0, bool weights = false) + : type_a(type_a), type_b(type_b), m(m), n(n), k(k), bs(bs), nr(nr), per(per), k_v(k_v), o(o), src_overlap(src_overlap), m_v(m_v), pad(pad), offset_a(offset_a), weights(weights) {} + + bool use_weight_context() override { + return weights; + } ggml_tensor * build_graph(ggml_context * ctx) override { + return build_graph(ctx, ctx); + } + + ggml_tensor * build_graph(ggml_context * ctx, ggml_context * ctx_weights) override { + ggml_context * ctx_a = weights ? ctx_weights : ctx; + GGML_ASSERT(ctx_a); + // C^T = A * B^T: (k, m) * (k, n) => (m, n) ggml_tensor * a; ggml_tensor * b; @@ -5451,7 +5546,7 @@ struct test_mul_mat : public test_case { const int64_t ne_a[4] = {k, m, bs[0], bs[1]}; const int64_t ne_b[4] = {k, n, bs[0]*nr[0], bs[1]*nr[1]}; - a = ggml_new_tensor_4d(ctx, type_a, ne_a[per[0]], ne_a[per[1]], ne_a[per[2]], ne_a[per[3]]); + a = ggml_new_tensor_4d(ctx_a, type_a, ne_a[per[0]], ne_a[per[1]], ne_a[per[2]], ne_a[per[3]]); b = ggml_new_tensor_4d(ctx, type_b, ne_b[per[0]], ne_b[per[1]], ne_b[per[2]], ne_b[per[3]]); if (!ggml_is_quantized(type_a)) { if (bs[1] == 1 && nr[1] == 1) { @@ -5470,6 +5565,7 @@ struct test_mul_mat : public test_case { GGML_ASSERT(type_a == type_b); GGML_ASSERT(k_v == 0); GGML_ASSERT(m_v == 0); + GGML_ASSERT(ctx_a == ctx); // not handled: b is a view of the same tensor // a and b are interleaved views of the same tensor: (e.g. fused QKV in MiniMax-01) ggml_tensor * base = ggml_new_tensor_4d(ctx, type_a, 2*k, std::max(m, n), bs[0]*nr[0], bs[1]*nr[1]); @@ -5482,7 +5578,7 @@ struct test_mul_mat : public test_case { } else { const int64_t k_physical = k_v == 0 ? k : k_v; const int64_t m_physical = m_v == 0 ? m : m_v + (pad != 0); - a = ggml_new_tensor_4d(ctx, type_a, k_physical, m_physical, bs[0], bs[1]); + a = ggml_new_tensor_4d(ctx_a, type_a, k_physical, m_physical, bs[0], bs[1]); b = ggml_new_tensor_4d(ctx, type_b, k_physical, n, bs[0]*nr[0], bs[1]*nr[1]); if (!ggml_is_quantized(type_a)) { @@ -5494,13 +5590,15 @@ struct test_mul_mat : public test_case { if (k_v != 0) { GGML_ASSERT(k_v > k); - a = ggml_view_4d(ctx, a, k, m, bs[0], bs[1], a->nb[1], a->nb[2], a->nb[3], 0); + GGML_ASSERT(offset_a <= ggml_row_size(type_a, k_v) - ggml_row_size(type_a, k)); + a = ggml_view_4d(ctx, a, k, m, bs[0], bs[1], a->nb[1], a->nb[2], a->nb[3], offset_a); b = ggml_view_4d(ctx, b, k, n, bs[0]*nr[0], bs[1]*nr[1], b->nb[1], b->nb[2], b->nb[3], 0); } if (m_v != 0) { GGML_ASSERT(m_v > m); GGML_ASSERT(pad < (int64_t) a->nb[1]); - a = ggml_view_4d(ctx, a, k, m, bs[0], bs[1], a->nb[1], m_v*a->nb[1] + pad, a->nb[3], 0); + GGML_ASSERT(k_v != 0 || offset_a <= (m_v - m)*a->nb[1]); + a = ggml_view_4d(ctx, a, k, m, bs[0], bs[1], a->nb[1], m_v*a->nb[1] + pad, a->nb[3], k_v == 0 ? offset_a : 0); } ggml_set_name(a, "a"); ggml_set_name(b, "b"); @@ -5535,8 +5633,8 @@ struct test_mul_mat_hadamard : public test_mul_mat { GGML_ASSERT(type_a == GGML_TYPE_F32); } - ggml_tensor * build_graph(ggml_context * ctx) override { - ggml_tensor * out = test_mul_mat::build_graph(ctx); + ggml_tensor * build_graph(ggml_context * ctx, ggml_context * ctx_weights) override { + ggml_tensor * out = test_mul_mat::build_graph(ctx, ctx_weights); // Find the mul_mat op in the graph and set the hint for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != NULL; t = ggml_get_next_tensor(ctx, t)) { if (t->op == GGML_OP_MUL_MAT) { @@ -5585,8 +5683,8 @@ struct test_mul_mat_w4a8 : public test_mul_mat { std::array bs = {1, 1}, std::array nr = {1, 1}) : test_mul_mat(type_a, type_b, m, n, k, bs, nr) {} - ggml_tensor * build_graph(ggml_context * ctx) override { - ggml_tensor * out = test_mul_mat::build_graph(ctx); + ggml_tensor * build_graph(ggml_context * ctx, ggml_context * ctx_weights) override { + ggml_tensor * out = test_mul_mat::build_graph(ctx, ctx_weights); for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != NULL; t = ggml_get_next_tensor(ctx, t)) { if (t->op == GGML_OP_MUL_MAT) { ggml_prec_set_src(t, GGML_PREC_Q8, 1); @@ -5600,6 +5698,91 @@ struct test_mul_mat_w4a8 : public test_mul_mat { } }; +// src1 GGML_PREC_BF16: the implementation may keep src1 in F32 or BF16, but must not convert it down to F16, +// which cannot hold the src1 values that the BF16 range allows +struct test_mul_mat_prec_src1_bf16 : public test_mul_mat { + test_mul_mat_prec_src1_bf16(ggml_type type_a = GGML_TYPE_F32, int64_t m = 64, int64_t n = 32, int64_t k = 256, + std::array bs = {1, 1}, + std::array nr = {1, 1}) + : test_mul_mat(type_a, GGML_TYPE_F32, m, n, k, bs, nr) {} + ggml_tensor * build_graph(ggml_context * ctx, ggml_context * ctx_weights) override { + ggml_tensor * out = test_mul_mat::build_graph(ctx, ctx_weights); + for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != NULL; t = ggml_get_next_tensor(ctx, t)) { + if (t->op == GGML_OP_MUL_MAT) { + ggml_prec_set_src(t, GGML_PREC_BF16, 1); + } + } + return out; + } + void initialize_tensors(ggml_context * ctx) override { + for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != NULL; t = ggml_get_next_tensor(ctx, t)) { + if (strcmp(t->name, "b") == 0) { + init_tensor_uniform(t, -1e6f, 1e6f); + } else { + init_tensor_uniform(t); + } + } + } + std::string op_desc(ggml_tensor * t) override { + GGML_UNUSED(t); + return "MUL_MAT_PREC_SRC1_BF16"; + } +}; + +// src1 GGML_PREC_F32: the implementation can not convert src1 or the F32 src0 tiles down to F16 or TF32 +struct test_mul_mat_prec_src1_f32 : public test_mul_mat { + test_mul_mat_prec_src1_f32(ggml_type type_a = GGML_TYPE_F32, int64_t m = 64, int64_t n = 32, int64_t k = 256, + std::array bs = {1, 1}, + std::array nr = {1, 1}) + : test_mul_mat(type_a, GGML_TYPE_F32, m, n, k, bs, nr) {} + double max_nmse_err() override { + return 1e-10; + } + ggml_tensor * build_graph(ggml_context * ctx, ggml_context * ctx_weights) override { + ggml_tensor * out = test_mul_mat::build_graph(ctx, ctx_weights); + for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != NULL; t = ggml_get_next_tensor(ctx, t)) { + if (t->op == GGML_OP_MUL_MAT) { + ggml_prec_set_src(t, GGML_PREC_F32, 1); + } + } + return out; + } + void initialize_tensors(ggml_context * ctx) override { + for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != NULL; t = ggml_get_next_tensor(ctx, t)) { + if (strcmp(t->name, "b") == 0) { + init_tensor_uniform(t, -1e6f, 1e6f); + } else { + init_tensor_uniform(t); + } + } + } + std::string op_desc(ggml_tensor * t) override { + GGML_UNUSED(t); + return "MUL_MAT_PREC_SRC1_F32"; + } +}; + +// src1 GGML_PREC_BF16 with src0 values outside the F16 range: the F32 src0 tiles must not be downcast to F16 +struct test_mul_mat_prec_src1_bf16_src0 : public test_mul_mat_prec_src1_bf16 { + test_mul_mat_prec_src1_bf16_src0(ggml_type type_a = GGML_TYPE_F32, int64_t m = 64, int64_t n = 32, int64_t k = 256, + std::array bs = {1, 1}, + std::array nr = {1, 1}) + : test_mul_mat_prec_src1_bf16(type_a, m, n, k, bs, nr) {} + void initialize_tensors(ggml_context * ctx) override { + for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != NULL; t = ggml_get_next_tensor(ctx, t)) { + if (strcmp(t->name, "a") == 0) { + init_tensor_uniform(t, -1e6f, 1e6f); + } else { + init_tensor_uniform(t); + } + } + } + std::string op_desc(ggml_tensor * t) override { + GGML_UNUSED(t); + return "MUL_MAT_PREC_SRC1_BF16_SRC0"; + } +}; + // FP4 native W4A4 path (default precision) struct test_mul_mat_w4a4 : public test_mul_mat { test_mul_mat_w4a4(ggml_type type_a = GGML_TYPE_NVFP4, ggml_type type_b = GGML_TYPE_F32, @@ -6312,33 +6495,6 @@ struct test_trunc : public test_case { } }; -// GGML_OP_DIAG_MASK_INF -struct test_diag_mask_inf : public test_case { - const ggml_type type; - const std::array ne; - const int n_past; - - std::string vars() override { - return VARS_TO_STR3(type, ne, n_past); - } - - test_diag_mask_inf(ggml_type type = GGML_TYPE_F32, - std::array ne = {10, 10, 3, 2}, - int n_past = 5) - : type(type), ne(ne), n_past(n_past) {} - - ggml_tensor * build_graph(ggml_context * ctx) override { - ggml_tensor * a = ggml_new_tensor(ctx, type, 4, ne.data()); - ggml_set_param(a); - ggml_set_name(a, "a"); - - ggml_tensor * out = ggml_diag_mask_inf(ctx, a, n_past); - ggml_set_name(out, "out"); - - return out; - } -}; - // GGML_OP_SOFT_MAX struct test_soft_max : public test_case { const ggml_type type; @@ -6452,18 +6608,23 @@ struct test_rope : public test_case { bool forward; bool inplace; int n_offs; // offset of the rotated dims window, set via ggml_rope_set_offset() + int n_ctx_orig; + float freq_base; + float beta_fast; + bool truncate; std::string vars() override { // forward can be inferred from the op, does not need to be printed - return VARS_TO_STR12(type, ne_a, n_dims, mode, n_ctx, fs, ef, af, ff, v, inplace, n_offs); + return VARS_TO_STR16(type, ne_a, n_dims, mode, n_ctx, fs, ef, af, ff, v, inplace, n_offs, n_ctx_orig, freq_base, beta_fast, truncate); } test_rope(ggml_type type = GGML_TYPE_F32, std::array ne_a = {10, 5, 3, 1}, int n_dims = 10, int mode = GGML_ROPE_TYPE_NORMAL, int n_ctx = 512, float fs = 1.0f, float ef = 0.0f, float af = 0.0f, bool ff = false, int v = 0, bool forward = true, bool inplace = false, - int n_offs = 0) - : type(type), ne_a(ne_a), n_dims(n_dims), mode(mode), n_ctx(n_ctx), fs(fs), ef(ef), af(af), ff(ff), v(v), forward(forward), inplace(inplace), n_offs(n_offs) {} + int n_offs = 0, int n_ctx_orig = 0, float freq_base = 10000.0f, float beta_fast = 1.0f, bool truncate = true) + : type(type), ne_a(ne_a), n_dims(n_dims), mode(mode), n_ctx(n_ctx), fs(fs), ef(ef), af(af), ff(ff), v(v), forward(forward), inplace(inplace), n_offs(n_offs), + n_ctx_orig(n_ctx_orig), freq_base(freq_base), beta_fast(beta_fast), truncate(truncate) {} ggml_tensor * build_graph(ggml_context * ctx) override { ggml_tensor * a; @@ -6479,7 +6640,7 @@ struct test_rope : public test_case { a = ggml_view_4d(ctx, a, ne_a[0], ne_a[1], ne_a[2], ne_a[3], a->nb[1], a->nb[2], a->nb[3], 0); ggml_set_name(a, "view_of_a"); } else if (v == 2) { - // second-half slice along dim 0 (mimics build_rope_2d in clip.cpp). + // second-half slice along dim 0 (mimics build_rope_2d in tools/mtmd/mtmd-graph.cpp). // The non-zero view offset (ne_a[0] * elem_size) often produces a // non-aligned buffer offset, which exercises backends' alignment paths. auto ne = ne_a; ne[0] *= 2; @@ -6527,37 +6688,38 @@ struct test_rope : public test_case { int rope_sections[4] = {n_dims/4, n_dims/4, 0, 0}; // Vision-RoPE only use first two dimension for image (x, y) coordinate if (forward) { if (inplace) { - out = ggml_rope_multi_inplace(ctx, a, pos, freq, n_dims/2, rope_sections, mode, 0, 10000.0f, fs, ef, af, 1.0f, 1.0f); + out = ggml_rope_multi_inplace(ctx, a, pos, freq, n_dims/2, rope_sections, mode, n_ctx_orig, freq_base, fs, ef, af, beta_fast, 1.0f); } else { - out = ggml_rope_multi(ctx, a, pos, freq, n_dims/2, rope_sections, mode, 0, 10000.0f, fs, ef, af, 1.0f, 1.0f); + out = ggml_rope_multi(ctx, a, pos, freq, n_dims/2, rope_sections, mode, n_ctx_orig, freq_base, fs, ef, af, beta_fast, 1.0f); } } else { - out = ggml_rope_multi_back(ctx, a, pos, freq, n_dims/2, rope_sections, mode, 0, 10000.0f, fs, ef, af, 1.0f, 1.0f); + out = ggml_rope_multi_back(ctx, a, pos, freq, n_dims/2, rope_sections, mode, n_ctx_orig, freq_base, fs, ef, af, beta_fast, 1.0f); } } else { GGML_ASSERT(n_dims/3 > 0); int rope_sections[4] = {n_dims/3, n_dims/3, n_dims/3, 0}; if (forward) { if (inplace) { - out = ggml_rope_multi_inplace(ctx, a, pos, freq, n_dims, rope_sections, mode, 0, 10000.0f, fs, ef, af, 1.0f, 1.0f); + out = ggml_rope_multi_inplace(ctx, a, pos, freq, n_dims, rope_sections, mode, n_ctx_orig, freq_base, fs, ef, af, beta_fast, 1.0f); } else { - out = ggml_rope_multi(ctx, a, pos, freq, n_dims, rope_sections, mode, 0, 10000.0f, fs, ef, af, 1.0f, 1.0f); + out = ggml_rope_multi(ctx, a, pos, freq, n_dims, rope_sections, mode, n_ctx_orig, freq_base, fs, ef, af, beta_fast, 1.0f); } } else { - out = ggml_rope_multi_back(ctx, a, pos, freq, n_dims, rope_sections, mode, 0, 10000.0f, fs, ef, af, 1.0f, 1.0f); + out = ggml_rope_multi_back(ctx, a, pos, freq, n_dims, rope_sections, mode, n_ctx_orig, freq_base, fs, ef, af, beta_fast, 1.0f); } } } else { if (forward) { if (inplace) { - out = ggml_rope_ext_inplace(ctx, a, pos, freq, n_dims, mode, 0, 10000.0f, fs, ef, af, 1.0f, 1.0f); + out = ggml_rope_ext_inplace(ctx, a, pos, freq, n_dims, mode, n_ctx_orig, freq_base, fs, ef, af, beta_fast, 1.0f); } else { - out = ggml_rope_ext(ctx, a, pos, freq, n_dims, mode, 0, 10000.0f, fs, ef, af, 1.0f, 1.0f); + out = ggml_rope_ext(ctx, a, pos, freq, n_dims, mode, n_ctx_orig, freq_base, fs, ef, af, beta_fast, 1.0f); } } else { - out = ggml_rope_ext_back(ctx, a, pos, freq, n_dims, mode, 0, 10000.0f, fs, ef, af, 1.0f, 1.0f); + out = ggml_rope_ext_back(ctx, a, pos, freq, n_dims, mode, n_ctx_orig, freq_base, fs, ef, af, beta_fast, 1.0f); } } + ggml_rope_set_truncate(out, truncate); if (n_offs != 0) { out = ggml_rope_set_offset(out, n_offs); } @@ -6587,6 +6749,13 @@ struct test_rope : public test_case { } } + double max_nmse_err() override { + // the f32 rotation angle of a position below n_ctx carries a rounding error of up to n_ctx*FLT_EPSILON radians + // backends compute the angle with different rounding sequences, so their agreement is bounded by that error + const double eps_theta = n_ctx * FLT_EPSILON; + return std::max(test_case::max_nmse_err(), eps_theta * eps_theta); + } + double max_maa_err() override { return 1e-3; } @@ -7293,6 +7462,34 @@ struct test_top_k : public test_case { // The logic in err can't handle the sentinel tensors. bool run_whole_graph() override { return ties; } + // the backends may break ties in different ways, but each backend must break them in the same way in each run + bool deterministic(ggml_backend_t backend) override { + // TODO: Vulkan is not deterministic yet, SYCL gathers the ties with an atomic counter as well + const char * name = ggml_backend_reg_name(ggml_backend_dev_backend_reg(ggml_backend_get_device(backend))); + // ROCm has no cub::DeviceTopK, every row goes through the radix or bitonic path + if (strcmp(name, "ROCm") == 0) { + return ties; + } + if (strcmp(name, "CUDA") != 0 && strcmp(name, "MTL") != 0) { + return false; + } + // CUDA uses cub::DeviceTopK for 1 or 2 rows, which does not guarantee deterministic ties (CCCL <= 3.5) + return ties && ne[1]*ne[2]*ne[3] > 2; + } + + // the order in a row is not specified, so compare the sets + bool same_output(ggml_tensor * t1, ggml_tensor * t2) override { + std::vector i1(ggml_nelements(t1)); + std::vector i2(ggml_nelements(t2)); + ggml_backend_tensor_get(t1, i1.data(), 0, ggml_nbytes(t1)); + ggml_backend_tensor_get(t2, i2.data(), 0, ggml_nbytes(t2)); + for (size_t r = 0; r < i1.size(); r += k) { + std::sort(i1.begin() + r, i1.begin() + r + k); + std::sort(i2.begin() + r, i2.begin() + r + k); + } + return i1 == i2; + } + double err(const float * a, const float * b, size_t n) override { // When there are no ties, we expect the exact same set of indices, // but possibly in a different order. When there are ties, the indices @@ -7525,6 +7722,9 @@ struct test_topk_moe : public test_case { const MoeGatingFunc gating_func; const float scale_w; const int n_expert_groups; + ggml_tensor * logits {}; + ggml_tensor * exp_probs_b {}; + const bool norm_bias; ggml_tensor * weights {}; ggml_tensor * selected_experts {}; @@ -7534,20 +7734,23 @@ struct test_topk_moe : public test_case { bool bias_probs = false, MoeGatingFunc gating_func = GATING_FUNC_SOFTMAX, float scale_w = 0.0f, - int n_expert_groups = 1) : + int n_expert_groups = 1, + bool norm_bias = false) : ne(ne), n_expert_used(n_expert_used), with_norm(with_norm), bias_probs(bias_probs), gating_func(gating_func), scale_w(scale_w), - n_expert_groups(n_expert_groups) { + n_expert_groups(n_expert_groups), + norm_bias(norm_bias) { GGML_ASSERT(n_expert_used <= ne[0]); GGML_ASSERT(ne[0] % n_expert_groups == 0); GGML_ASSERT(n_expert_used <= ne[0] / n_expert_groups); + GGML_ASSERT(!norm_bias || with_norm); } - std::string vars() override { return VARS_TO_STR7(ne, n_expert_used, with_norm, bias_probs, gating_func, scale_w, n_expert_groups); } + std::string vars() override { return VARS_TO_STR8(ne, n_expert_used, with_norm, norm_bias, bias_probs, gating_func, scale_w, n_expert_groups); } std::string op_desc(ggml_tensor * t) override { GGML_UNUSED(t); @@ -7556,13 +7759,106 @@ struct test_topk_moe : public test_case { bool run_whole_graph() override { return true; } + // the selection scores (probs plus bias) of a row are a random permutation of a grid, and the experts of + // the n_group_used best groups move up by half a step, so every expert and group decision is one step + // apart; the logits are the inverse of the gating function at these scores + void initialize_tensors(ggml_context * ctx) override { + const int n_expert = ne[0]; + const int n_exp_per_group = n_expert / n_expert_groups; + const int n_group_used = n_expert_groups / 2; + const bool softmax = gating_func == GATING_FUNC_SOFTMAX; + + // softmax probs sum to one, so its grid step and bias scale with 1/n_expert + const double step = softmax ? 1.0 / ((double) n_expert * n_expert) : 0.5 / n_expert; + const double bmax = softmax ? 0.125 / n_expert : 0.2; + + std::default_random_engine rng(std::random_device{}()); + std::uniform_real_distribution dist(-bmax, bmax); + + std::vector bias(n_expert, 0.0f); + if (exp_probs_b) { + for (float & v : bias) { + v = dist(rng); + } + ggml_backend_tensor_set(exp_probs_b, bias.data(), 0, ggml_nbytes(exp_probs_b)); + } + + std::vector perm(n_expert); + for (int i = 0; i < n_expert; ++i) { + perm[i] = i; + } + + std::vector rank(n_expert); + std::vector score(n_expert_groups); + std::vector order(n_expert_groups); + std::vector x(ggml_nelements(logits)); + for (int64_t r = 0; r < ggml_nrows(logits); ++r) { + std::shuffle(perm.begin(), perm.end(), rng); + for (int i = 0; i < n_expert; ++i) { + rank[i] = perm[i]; + } + + if (n_expert_groups > 1) { + for (int g = 0; g < n_expert_groups; ++g) { + const int * grp = perm.data() + g*n_exp_per_group; + int r0 = -1; + int r1 = -1; + for (int i = 0; i < n_exp_per_group; ++i) { + if (grp[i] > r0) { + r1 = r0; + r0 = grp[i]; + } else if (grp[i] > r1) { + r1 = grp[i]; + } + } + score[g] = r0 + r1; + order[g] = g; + } + std::sort(order.begin(), order.end(), [&](int a, int b) { return score[a] > score[b]; }); + for (int k = 0; k < n_group_used; ++k) { + for (int i = 0; i < n_exp_per_group; ++i) { + rank[order[k]*n_exp_per_group + i] += 0.5; + } + } + } + + // probs = scores - bias, with softmax probs shifted to sum to one + double c = 0.25; + if (softmax) { + double sum = 0.0; + for (int i = 0; i < n_expert; ++i) { + sum += rank[i]*step - bias[i]; + } + c = (1.0 - sum) / n_expert; + } + + float * row = x.data() + r*n_expert; + for (int i = 0; i < n_expert; ++i) { + const double p = c + rank[i]*step - bias[i]; + row[i] = gating_func == GATING_FUNC_SOFTMAX ? std::log(p) : + gating_func == GATING_FUNC_SIGMOID ? std::log(p / (1.0 - p)) : + gating_func == GATING_FUNC_SQRT_SOFTPLUS ? std::log(std::expm1(p*p)) : + p; + } + } + ggml_backend_tensor_set(logits, x.data(), 0, ggml_nbytes(logits)); + + for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != nullptr; t = ggml_get_next_tensor(ctx, t)) { + const ggml_tensor * src = t->view_src ? t->view_src : t; + if (src != logits && src != exp_probs_b) { + init_tensor_uniform(t); + } + } + } + ggml_tensor * build_graph(ggml_context * ctx) override { const int n_expert = ne[0]; const int n_tokens = ne[1]; const int n_exp_per_group = n_expert / n_expert_groups; const int n_group_used = n_expert_groups / 2; - ggml_tensor * logits = ggml_new_tensor(ctx, GGML_TYPE_F32, 4, ne.data()); + logits = ggml_new_tensor(ctx, GGML_TYPE_F32, 4, ne.data()); + exp_probs_b = nullptr; ggml_tensor * probs = (gating_func == GATING_FUNC_SOFTMAX) ? ggml_soft_max(ctx, logits) : (gating_func == GATING_FUNC_SIGMOID) ? ggml_sigmoid(ctx, logits) : @@ -7571,7 +7867,7 @@ struct test_topk_moe : public test_case { ggml_tensor * selection_probs = probs; if (bias_probs) { - ggml_tensor * exp_probs_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, ne[0]); + exp_probs_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, ne[0]); ggml_set_name(exp_probs_b, "exp_probs_b"); selection_probs = ggml_add(ctx, probs, exp_probs_b); ggml_set_name(selection_probs, "selection_probs"); @@ -7612,7 +7908,11 @@ struct test_topk_moe : public test_case { ggml_tensor * weights_sum = ggml_sum_rows(ctx, weights); // [1, n_tokens] ggml_set_name(weights_sum, "weights_sum"); - weights_sum = ggml_clamp(ctx, weights_sum, 6.103515625e-5, INFINITY); + if (norm_bias) { + weights_sum = ggml_scale_bias(ctx, weights_sum, 1.0f, 1e-20f); + } else { + weights_sum = ggml_clamp(ctx, weights_sum, 6.103515625e-5, INFINITY); + } weights = ggml_div(ctx, weights, weights_sum); // [n_expert_used, n_tokens] weights = ggml_reshape_3d(ctx, weights, 1, n_expert_used, n_tokens); } @@ -7808,7 +8108,7 @@ struct test_mul_mat_add : public test_case { } }; -struct test_mul_mat_vec_fusion : public test_case { +struct test_mul_mat_fusion : public test_case { const ggml_type type; const ggml_glu_op glu_op; const int64_t m; @@ -7824,7 +8124,7 @@ struct test_mul_mat_vec_fusion : public test_case { std::array batch_dims; const bool skip; // set some ids to a negative value (only for use_id) - test_mul_mat_vec_fusion(ggml_type type, ggml_glu_op op, int64_t m, int64_t n, int64_t k, + test_mul_mat_fusion(ggml_type type, ggml_glu_op op, int64_t m, int64_t n, int64_t k, bool use_id = false, int n_mats = 1, int n_used = 1, bool b = false, bool with_bias = false, bool with_gate = true, bool with_lane_scale = false, std::array batch_dims = {4, 2}, bool skip = false) : type(type), glu_op(op), m(m), n(n), k(k), use_id(use_id), n_mats(n_mats), n_used(n_used), b(b), with_bias(with_bias), @@ -7840,7 +8140,7 @@ struct test_mul_mat_vec_fusion : public test_case { std::string op_desc(ggml_tensor * t) override { GGML_UNUSED(t); - return "MUL_MAT_VEC_FUSION"; + return "MUL_MAT_FUSION"; } bool run_whole_graph() override { return true; } @@ -7993,6 +8293,36 @@ struct test_mul_mat_vec_fusion : public test_case { double max_nmse_err() override { return 5e-3; } + + double max_nmse_err(ggml_backend_t backend) override { + // 4-bit activations on Blackwell, see test_mul_mat + if (type == GGML_TYPE_NVFP4 && !graph_mul_mat_hi_prec_act(gf, use_id ? GGML_OP_MUL_MAT_ID : GGML_OP_MUL_MAT) && + backend_has_feature(backend, "BLACKWELL_NATIVE_FP4")) { + return 2e-2; + } + return max_nmse_err(); + } +}; + +// Fused gate/up/GLU with GGML_PREC_Q8 on src1, e.g. a W4A16 NVFP4 checkpoint (W4A8, see test_mul_mat_w4a8) +struct test_mul_mat_fusion_w4a8 : public test_mul_mat_fusion { + using test_mul_mat_fusion::test_mul_mat_fusion; + using test_mul_mat_fusion::build_graph; + + ggml_tensor * build_graph(ggml_context * ctx, ggml_context * ctx_weights) override { + ggml_tensor * out = test_mul_mat_fusion::build_graph(ctx, ctx_weights); + for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != NULL; t = ggml_get_next_tensor(ctx, t)) { + if (t->op == GGML_OP_MUL_MAT || t->op == GGML_OP_MUL_MAT_ID) { + ggml_prec_set_src(t, GGML_PREC_Q8, 1); + } + } + return out; + } + + std::string op_desc(ggml_tensor * t) override { + GGML_UNUSED(t); + return "MUL_MAT_FUSION_W4A8"; + } }; // GGML_OP_SUM @@ -8503,18 +8833,28 @@ struct test_pad_reflect_1d : public test_case { const std::array ne_a; const int pad_0; const int pad_1; + const bool view; std::string vars() override { - return VARS_TO_STR4(type, ne_a, pad_0, pad_1); + return VARS_TO_STR5(type, ne_a, pad_0, pad_1, view); } test_pad_reflect_1d(ggml_type type = GGML_TYPE_F32, std::array ne_a = {512, 34, 2, 1}, - int pad_0 = 10, int pad_1 = 9) - : type(type), ne_a(ne_a), pad_0(pad_0), pad_1(pad_1) {} + int pad_0 = 10, int pad_1 = 9, bool view = false) + : type(type), ne_a(ne_a), pad_0(pad_0), pad_1(pad_1), view(view) {} ggml_tensor * build_graph(ggml_context * ctx) override { - ggml_tensor * a = ggml_new_tensor(ctx, type, 2, ne_a.data()); + ggml_tensor * a; + if (view) { + const size_t nb1 = ggml_row_size(type, ne_a[0]); + const size_t nb2 = nb1 * ne_a[1]; + const size_t nb3 = nb2 * ne_a[2]; + ggml_tensor * base = ggml_new_tensor_1d(ctx, type, ne_a[0] * ne_a[1] * ne_a[2] * ne_a[3] + 1); + a = ggml_view_4d(ctx, base, ne_a[0], ne_a[1], ne_a[2], ne_a[3], nb1, nb2, nb3, ggml_type_size(type)); + } else { + a = ggml_new_tensor(ctx, type, 4, ne_a.data()); + } ggml_set_name(a, "a"); ggml_tensor * out = ggml_pad_reflect_1d(ctx, a, pad_0, pad_1); @@ -8655,9 +8995,10 @@ struct test_flash_attn_ext : public test_case { const bool kv_view; // create K/V as views of a larger buffer (like a KV cache) const bool v_is_view_of_k; const int64_t n_kv_max; + const int64_t q_head_padding; std::string vars() override { - return VARS_TO_STR17(hsk, hsv, nh, nr23, kv, nb, mask, sinks, max_bias, logit_softcap, prec, type_K, type_V, permute, kv_view, v_is_view_of_k, n_kv_max); + return VARS_TO_STR18(hsk, hsv, nh, nr23, kv, nb, mask, sinks, max_bias, logit_softcap, prec, type_K, type_V, permute, kv_view, v_is_view_of_k, n_kv_max, q_head_padding); } double max_nmse_err() override { @@ -8674,9 +9015,9 @@ struct test_flash_attn_ext : public test_case { test_flash_attn_ext(int64_t hsk = 128, int64_t hsv = 128, int64_t nh = 32, std::array nr23 = {1, 1}, int64_t kv = 96, int64_t nb = 8, bool mask = true, bool sinks = false, float max_bias = 0.0f, float logit_softcap = 0.0f, ggml_prec prec = GGML_PREC_F32, ggml_type type_K = GGML_TYPE_F16, ggml_type type_V = GGML_TYPE_F16, std::array permute = {0, 1, 2, 3}, - bool kv_view = true, bool v_is_view_of_k = false, int64_t n_kv_max = 0) + bool kv_view = true, bool v_is_view_of_k = false, int64_t n_kv_max = 0, int64_t q_head_padding = 0) : hsk(hsk), hsv(hsv), nh(nh), nr23(nr23), kv(kv), nb(nb), mask(mask), sinks(sinks), max_bias(max_bias), logit_softcap(logit_softcap), prec(prec), - type_K(type_K), type_V(type_V), permute(permute), kv_view(kv_view), v_is_view_of_k(v_is_view_of_k), n_kv_max(n_kv_max) {} + type_K(type_K), type_V(type_V), permute(permute), kv_view(kv_view), v_is_view_of_k(v_is_view_of_k), n_kv_max(n_kv_max), q_head_padding(q_head_padding) {} ggml_tensor * build_graph(ggml_context * ctx) override { const int64_t hsk_padded = GGML_PAD(hsk, ggml_blck_size(type_K)); @@ -8701,7 +9042,11 @@ struct test_flash_attn_ext : public test_case { return t; }; - ggml_tensor * q = create_permuted(GGML_TYPE_F32, hsk_padded, nb, nh*nr23[0], nr23[1], false); + ggml_tensor * q = create_permuted(GGML_TYPE_F32, hsk_padded + q_head_padding, nb, nh*nr23[0], nr23[1], false); + if (q_head_padding > 0) { + ggml_set_name(q, "q_padded"); + q = ggml_view_4d(ctx, q, hsk_padded, q->ne[1], q->ne[2], q->ne[3], q->nb[1], q->nb[2], q->nb[3], 0); + } ggml_set_name(q, "q"); ggml_tensor * k = create_permuted(type_K, hsk_padded, kv, nh, nr23[1], kv_view); // the K tensor is usually a view of the K cache @@ -9144,6 +9489,69 @@ struct test_tri : public test_case { } }; +// ggml_tri where the values that have to be discarded are not finite: they must come back as a plain +// zero. A backend that scales by the comparison instead of selecting turns them into NaN. +struct test_tri_special : public test_case { + const ggml_type type; + const std::array ne; + const ggml_tri_type tri_type; + + std::string vars() override { return VARS_TO_STR3(type, ne, tri_type); } + + test_tri_special(ggml_tri_type tri_type, ggml_type type = GGML_TYPE_F32, + std::array ne = { 16, 16, 2, 2 }) + : type(type), ne(ne), tri_type(tri_type) { + GGML_ASSERT(type == GGML_TYPE_F32); + GGML_ASSERT(ne[0] == ne[1]); + } + + bool keeps(int64_t i, int64_t r) const { + switch (tri_type) { + case GGML_TRI_TYPE_LOWER: return i < r; + case GGML_TRI_TYPE_LOWER_DIAG: return i <= r; + case GGML_TRI_TYPE_UPPER: return i > r; + case GGML_TRI_TYPE_UPPER_DIAG: return i >= r; + } + return false; + } + + ggml_tensor * build_graph(ggml_context * ctx) override { + ggml_tensor * a = ggml_new_tensor_4d(ctx, type, ne[0], ne[1], ne[2], ne[3]); + ggml_set_param(a); + ggml_set_name(a, "a"); + + ggml_tensor * out = ggml_tri(ctx, a, tri_type); + + ggml_set_name(out, "out"); + + return out; + } + + void initialize_tensors(ggml_context * ctx) override { + static const float discarded[] = { 0.0f, -0.0f, 1.0f, -1.0f, INFINITY, -INFINITY }; + + int64_t k = 0; + for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != NULL; t = ggml_get_next_tensor(ctx, t)) { + if (t->type != type || t->ne[0] != ne[0] || t->ne[1] != ne[1]) { + init_tensor_uniform(t, -1.0f, 1.0f); + continue; + } + std::vector data(ggml_nelements(t)); + for (int64_t i3 = 0; i3 < t->ne[3]; ++i3) { + for (int64_t i2 = 0; i2 < t->ne[2]; ++i2) { + for (int64_t i1 = 0; i1 < t->ne[1]; ++i1) { + for (int64_t i0 = 0; i0 < t->ne[0]; ++i0) { + const int64_t off = ((i3*t->ne[2] + i2)*t->ne[1] + i1)*t->ne[0] + i0; + data[off] = keeps(i0, i1) ? 1.0f : discarded[k++ % (int64_t) (sizeof(discarded)/sizeof(discarded[0]))]; + } + } + } + } + ggml_backend_tensor_set(t, data.data(), 0, data.size()*sizeof(float)); + } + } +}; + // GGML_OP_FILL struct test_fill : public test_case { const ggml_type type; @@ -9174,8 +9582,9 @@ struct test_solve_tri : public test_case { const ggml_type type; const std::array ne_lhs; const std::array ne_rhs; + const bool nan_out; // fill the output with NaN before the op runs - std::string vars() override { return VARS_TO_STR3(type, ne_lhs, ne_rhs); } + std::string vars() override { return VARS_TO_STR4(type, ne_lhs, ne_rhs, nan_out); } uint64_t op_flops(ggml_tensor * t) override { GGML_UNUSED(t); @@ -9188,9 +9597,10 @@ struct test_solve_tri : public test_case { test_solve_tri(ggml_type type = GGML_TYPE_F32, std::array ne_lhs = { 10, 10, 4, 3 }, - std::array ne_rhs = { 3, 10, 4, 3 } + std::array ne_rhs = { 3, 10, 4, 3 }, + bool nan_out = false ) - : type(type), ne_lhs(ne_lhs), ne_rhs(ne_rhs) {} + : type(type), ne_lhs(ne_lhs), ne_rhs(ne_rhs), nan_out(nan_out) {} ggml_tensor * build_graph(ggml_context * ctx) override { ggml_tensor * a = ggml_new_tensor_4d(ctx, type, ne_lhs[0], ne_lhs[1], ne_lhs[2], ne_lhs[3]); @@ -9212,6 +9622,9 @@ struct test_solve_tri : public test_case { if (strcmp(t->name, "a") == 0) { // note: avoid zeros in the diagonal init_tensor_tril(t, 0.1, 1.0f); + } else if (nan_out && strcmp(t->name, "out") == 0) { + std::vector data(ggml_nelements(t), NAN); + ggml_backend_tensor_set(t, data.data(), 0, ggml_nbytes(t)); } else { init_tensor_uniform(t, -1.0f, 1.0f); } @@ -10234,6 +10647,8 @@ static std::vector> make_test_cases_eval() { test_cases.emplace_back(new test_dsv4_hc_post(31, 17, true, true)); test_cases.emplace_back(new test_dsv4_hc_post(2560, 21, true, true)); test_cases.emplace_back(new test_dsv4_hc_post(31, 17, false, true)); + test_cases.emplace_back(new test_dsv4_hc_post(4096, 1)); // DeepSeek-V4-Flash decode + test_cases.emplace_back(new test_dsv4_hc_post(2560, 1, true, true)); // Qwen4-Exp decode for (ggml_type type : {GGML_TYPE_F16, GGML_TYPE_F32}) { // blocks with extra cells and scores (k-pool) @@ -10247,6 +10662,13 @@ static std::vector> make_test_cases_eval() { test_cases.emplace_back(new test_top_k_mask(type, 256, 0, 0, 16, 0, false, 32)); test_cases.emplace_back(new test_top_k_mask(type, 1001, 0, 0, 64, 0, false, 7, 1, 3)); test_cases.emplace_back(new test_top_k_mask(type, 4096, 0, 0, 2048, 0, false, 5)); + // one row, no blocks, fewer than 128 picks: no read past the pick list + test_cases.emplace_back(new test_top_k_mask(type, 1001, 0, 0, 16, 3, false, 1)); + test_cases.emplace_back(new test_top_k_mask(type, 256, 0, 0, 40, 0, true, 1)); + // one row (single-row kernel) with each optional input combination: no score and no extra, blocks only, blocks and extra without score + test_cases.emplace_back(new test_top_k_mask(type, 512, 0, 0, 16, 0, false, 1)); + test_cases.emplace_back(new test_top_k_mask(type, 1024, 4, 200, 32, 0, false, 1)); + test_cases.emplace_back(new test_top_k_mask(type, 1024, 4, 200, 32, 5, false, 1)); // cells with scores and extra cells test_cases.emplace_back(new test_top_k_mask(type, 512, 0, 0, 32, 5, true, 9)); // blocks picked per head, the mask broadcast over the heads @@ -10308,6 +10730,9 @@ static std::vector> make_test_cases_eval() { for (ggml_type type : {GGML_TYPE_IQ3_XXS, GGML_TYPE_IQ3_S}) { test_cases.emplace_back(new test_mul_mat_reorder_then_big_batch(type)); } + for (int n : {1, 3, 257}) { + test_cases.emplace_back(new test_get_rows(GGML_TYPE_F8_E4M3, n, 5, 3, 1, 1, false)); + } for (ggml_type type : all_types) { for (int b : {1, 7}) { for (bool v : {false, true}) { @@ -10378,6 +10803,14 @@ static std::vector> make_test_cases_eval() { } } + for (int mode : {GGML_ROPE_TYPE_NORMAL, GGML_ROPE_TYPE_NEOX}) { + for (ggml_type type : {GGML_TYPE_F32, GGML_TYPE_F16}) { + for (bool truncate : {false, true}) { + test_cases.emplace_back(new test_rope_set_rows(type, GGML_TYPE_I64, {128, 2, 3, 2}, mode, 128, true, truncate)); + } + } + } + for (ggml_type type_input : {GGML_TYPE_F32}) { for (ggml_op_pool pool_type : {GGML_OP_POOL_AVG, GGML_OP_POOL_MAX}) { for (int k0 : {1, 3}) { @@ -10709,6 +11142,13 @@ static std::vector> make_test_cases_eval() { test_cases.emplace_back(new test_conv_3d(1, 1, 2, 2, 2, 1, 1, 1, 0, 1, 1, 1, 0, 0, 0, 1, 1, 1, kernel_type)); test_cases.emplace_back(new test_conv_3d(1, 1, 2, 2, 2, 1, 1, 0, 1, 1, 1, 1, 0, 0, 0, 1, 1, 1, kernel_type)); test_cases.emplace_back(new test_conv_3d(1, 1, 2, 2, 2, 1, 0, 1, 1, 1, 1, 1, 0, 0, 0, 1, 1, 1, kernel_type)); + // input and output channel strides just over 16 MB, beyond the 24-bit DMA stride field + test_cases.emplace_back(new test_conv_3d(1, 2, 65, 256, 256, 2, 1, 1, 1, 1, 1, 1, 0, 0, 0, 1, 1, 1, kernel_type)); + // KD * KH = 127, 128, 255, 256 around the HTP DMA queue limits + test_cases.emplace_back(new test_conv_3d(1, 2, 127, 1, 40, 3, 127, 1, 1, 1, 1, 1, 0, 0, 0, 1, 1, 1, kernel_type)); + test_cases.emplace_back(new test_conv_3d(1, 2, 64, 2, 40, 3, 64, 2, 1, 1, 1, 1, 0, 0, 0, 1, 1, 1, kernel_type)); + test_cases.emplace_back(new test_conv_3d(1, 2, 85, 3, 40, 3, 85, 3, 1, 1, 1, 1, 0, 0, 0, 1, 1, 1, kernel_type)); + test_cases.emplace_back(new test_conv_3d(1, 2, 16, 16, 40, 3, 16, 16, 1, 1, 1, 1, 0, 0, 0, 1, 1, 1, kernel_type)); } for(uint32_t Cout : {1, 9}){ @@ -10754,8 +11194,10 @@ static std::vector> make_test_cases_eval() { test_cases.emplace_back(new test_conv_transpose_2d({10, 10, 9, 2}, {3, 3, 1, 9}, 2, kernel_type)); // for multiple batches } - test_cases.emplace_back(new test_count_equal(GGML_TYPE_F32, {4, 500, 1, 1})); - test_cases.emplace_back(new test_count_equal(GGML_TYPE_F32, {4, 5000, 1, 1})); + for (bool same_src : {false, true}) { + test_cases.emplace_back(new test_count_equal(GGML_TYPE_F32, {4, 500, 1, 1}, same_src)); + test_cases.emplace_back(new test_count_equal(GGML_TYPE_F32, {4, 5000, 1, 1}, same_src)); + } test_cases.emplace_back(new test_argmax(GGML_TYPE_F32, {32, 1, 1, 1})); test_cases.emplace_back(new test_argmax(GGML_TYPE_F32, {32, 513, 1, 1})); @@ -10865,6 +11307,8 @@ static std::vector> make_test_cases_eval() { test_cases.emplace_back(new test_cpy(GGML_TYPE_I32, GGML_TYPE_I32, {256, 1, 4, 1}, {-1,-1,-1,-1}, {1, 2, 0, 3}, {0, 0, 0, 0})); test_cases.emplace_back(new test_cpy(GGML_TYPE_F32, GGML_TYPE_F32, {256, 1, 4, 1}, {-1,-1,-1,-1}, {1, 2, 0, 3}, {0, 0, 0, 0})); test_cases.emplace_back(new test_cpy(GGML_TYPE_F32, GGML_TYPE_F32, {2, 2097121, 1, 1}, {-1,-1,-1,-1}, {1, 0, 2, 3})); + test_cases.emplace_back(new test_cpy(GGML_TYPE_F32, GGML_TYPE_F32, {1, 64, 48, 1}, {-1,-1,-1,-1}, {0, 2, 1, 3})); // ne[0] == 1 permute + test_cases.emplace_back(new test_cpy(GGML_TYPE_F16, GGML_TYPE_F16, {1, 64, 48, 1}, {-1,-1,-1,-1}, {0, 2, 1, 3})); // ne[0] == 1 permute test_cases.emplace_back(new test_cpy(GGML_TYPE_F32, GGML_TYPE_F32, {2, 2, 524281, 1}, {-1,-1,-1,-1}, {1, 0, 2, 3})); test_cases.emplace_back(new test_cpy(GGML_TYPE_F32, GGML_TYPE_F32, {128, 2, 3, 1}, {128, 2, 3, 1}, {0, 0, 0, 0}, {0, 0, 0, 0}, false, {128, 4, 3, 1})); // strided dst test_cases.emplace_back(new test_cpy(GGML_TYPE_F16, GGML_TYPE_F16, {128, 2, 3, 1}, {128, 2, 3, 1}, {0, 0, 0, 0}, {0, 0, 0, 0}, false, {128, 4, 3, 1})); // strided dst @@ -10931,6 +11375,31 @@ static std::vector> make_test_cases_eval() { test_cases.emplace_back(new test_cont(GGML_TYPE_F32, {512, 8193, 2, 1}, false, {2, 1, 0, 3})); test_cases.emplace_back(new test_cont(GGML_TYPE_F32, {256, 16385, 2, 1}, false, {0, 2, 1, 3})); + // channel swap with sizes not multiples of 32/64 tile + for (ggml_type type_dst : { GGML_TYPE_F32, GGML_TYPE_F16 }) { + test_cases.emplace_back(new test_cont(type_dst, {33, 17, 5, 19}, false, {1, 2, 3, 0})); + } + + // small channel dims and extent-1 dims + test_cases.emplace_back(new test_cont(GGML_TYPE_F32, {8, 104, 60, 13}, false, {1, 2, 3, 0})); + test_cases.emplace_back(new test_cont(GGML_TYPE_F16, {8, 104, 60, 13}, false, {1, 2, 3, 0})); + test_cases.emplace_back(new test_cont(GGML_TYPE_F32, {16, 1, 50, 7}, false, {1, 2, 3, 0})); + + // full reversal: contiguous src dim lands on dst dim 3 + test_cases.emplace_back(new test_cont(GGML_TYPE_F32, {33, 5, 7, 9}, false, {3, 2, 1, 0})); + test_cases.emplace_back(new test_cont(GGML_TYPE_F32, {33, 64, 32, 16}, false, {3, 2, 1, 0})); + + // permutes that move dense blocks (ne[0] stays dim 0) + for (ggml_type type_dst : { GGML_TYPE_F32, GGML_TYPE_F16 }) { + test_cases.emplace_back(new test_cont(type_dst, {4, 65, 9, 2}, false, {0, 2, 1, 3})); + } + test_cases.emplace_back(new test_cont(GGML_TYPE_F16, {64, 9, 7, 1}, false, {0, 2, 1, 3})); + + // transpose with i32 and CPY with reshape + test_cases.emplace_back(new test_cont(GGML_TYPE_I32, {33, 17, 5, 19}, false, {1, 2, 3, 0})); + test_cases.emplace_back(new test_cpy(GGML_TYPE_F32, GGML_TYPE_F32, {33, 17, 5, 1}, {17, 33, 5, 1}, {0, 0, 0, 0}, {1, 0, 2, 3})); + test_cases.emplace_back(new test_cpy(GGML_TYPE_F16, GGML_TYPE_F16, {8, 52, 39, 1}, {8, 39, 52, 1}, {0, 0, 0, 0}, {0, 2, 1, 3})); + auto add_test_bin_bcast = [&](ggml_type type, std::array ne, std::array nr, bool perm1 = false, bool src_overlap = false) { for (auto op : {ggml_add, ggml_sub, ggml_mul, ggml_div}) { test_cases.emplace_back(new test_bin_bcast(op, type, ne, nr, 1, perm1, src_overlap)); @@ -11141,6 +11610,18 @@ static std::vector> make_test_cases_eval() { // ne[2] > 65535 exceeds the CUDA gridDim.y limit (#27901) test_cases.emplace_back(new test_rms_norm_mul_rope({4, 1, 65536, 1}, 1e-6f, false, false, false, GGML_ROPE_TYPE_NORMAL)); test_cases.emplace_back(new test_rms_norm_mul_rope({4, 1, 65536, 1}, 1e-6f, false, true, false, GGML_ROPE_TYPE_NEOX)); + + for (int mode : {GGML_ROPE_TYPE_NORMAL, GGML_ROPE_TYPE_NEOX}) { + for (int64_t n : {128, 1024}) { + for (bool truncate : {false, true}) { + test_cases.emplace_back(new test_rms_norm_mul_rope({n, 2, 3, 1}, 1e-6f, false, false, true, mode, true, true, GGML_TYPE_F16, true, truncate)); + for (ggml_type type : {GGML_TYPE_F32, GGML_TYPE_F16}) { + test_cases.emplace_back(new test_rms_norm_mul_rope({n, 2, 3, 1}, 1e-6f, false, true, true, mode, true, true, type, true, truncate)); + } + } + } + } + for (int64_t d_conv : {3, 4, 7, 9}) { for (int64_t d_inner: {1024, 1536, 2048}) { test_cases.emplace_back(new test_ssm_conv(GGML_TYPE_F32, {d_conv, d_inner, 1, 1}, {d_conv, d_inner, 1, 1})); @@ -11174,6 +11655,8 @@ static std::vector> make_test_cases_eval() { } test_cases.emplace_back(new test_ssm_scan(GGML_TYPE_F32, 16, 1, 1024, 1, 32, 4)); // Mamba-1 + test_cases.emplace_back(new test_ssm_scan(GGML_TYPE_F32, 64, 64, 112, 2, 32, 4)); // Zamba2 (real hparams: d_inner=7168, dt_rank=112, n_group=2 -> head_dim=7168/112=64) + test_cases.emplace_back(new test_ssm_scan(GGML_TYPE_F32, 64, 64, 112, 2, 1, 1)); // Zamba2 decode (single-token, matches real per-token decode workload) test_cases.emplace_back(new test_ssm_scan(GGML_TYPE_F32, 96, 64, 128, 8, 1, 1)); // Nemotron-3-Puzzle decode (scan path, unused warp in last block) test_cases.emplace_back(new test_ssm_scan(GGML_TYPE_F32, 96, 64, 16, 8, 300, 2, false, /*K=*/1, /*weak_decay=*/true)); // d_state=96 SSD multi-chunk (partial 2nd chunk, 2 seqs) test_cases.emplace_back(new test_ssm_scan_rollback(GGML_TYPE_F32, 96, 64, 16, 8, 8, 2, /*K=*/3)); // d_state=96 rollback snapshots match prefix states @@ -11195,6 +11678,7 @@ static std::vector> make_test_cases_eval() { test_cases.emplace_back(new test_ssm_scan_cache_fusion(GGML_TYPE_F32, 128, 64, 16, 2, 4, 1, 4)); test_cases.emplace_back(new test_ssm_scan_cache_fusion(GGML_TYPE_F32, 128, 64, 16, 2, 1, 1, 4)); // n_seq_tokens < K test_cases.emplace_back(new test_ssm_scan_cache_fusion(GGML_TYPE_F32, 128, 64, 16, 2, 8, 1, 3)); // n_seq_tokens > K + test_cases.emplace_back(new test_ssm_scan_cache_fusion(GGML_TYPE_F32, 64, 64, 112, 2, 4, 1, 4)); // Zamba2 test_cases.emplace_back(new test_ssm_scan_cache_fusion(GGML_TYPE_F32, 96, 64, 16, 2, 4, 1, 4)); test_cases.emplace_back(new test_ssm_scan_cache_fusion(GGML_TYPE_F32, 256, 64, 8, 2, 4, 1, 4)); test_cases.emplace_back(new test_ssm_scan_cache_fusion(GGML_TYPE_F32, 128, 64, 16, 2, 1, 1, 1)); // K == 1, final state only @@ -11252,6 +11736,36 @@ static std::vector> make_test_cases_eval() { test_cases.emplace_back(new test_mul_mat_hadamard(GGML_TYPE_F32, GGML_TYPE_F16, 8192, 1, 8192)); test_cases.emplace_back(new test_mul_mat_hadamard(GGML_TYPE_F32, GGML_TYPE_F16, 1024, 7, 1024)); // many rows + // src1 precision: few-row paths, aligned tiles, partial tiles (K and output), broadcast batches + for (ggml_type type_a : {GGML_TYPE_F32, GGML_TYPE_F16, GGML_TYPE_Q8_0}) { + // Q8_0 requires K to be a multiple of its block size (32) + const int64_t k_partial = type_a == GGML_TYPE_Q8_0 ? 96 : 100; + test_cases.emplace_back(new test_mul_mat_prec_src1_bf16(type_a, 64, 1, 256)); + test_cases.emplace_back(new test_mul_mat_prec_src1_bf16(type_a, 64, 8, 256)); + test_cases.emplace_back(new test_mul_mat_prec_src1_bf16(type_a, 64, 32, 256)); + test_cases.emplace_back(new test_mul_mat_prec_src1_bf16(type_a, 128, 512, 512)); + test_cases.emplace_back(new test_mul_mat_prec_src1_bf16(type_a, 67, 33, k_partial)); + test_cases.emplace_back(new test_mul_mat_prec_src1_bf16(type_a, 64, 32, 256, {2, 3}, {2, 1})); + } + + // src1 precision: same shapes with GGML_PREC_F32, which the backends must not round down to F16 or TF32 + for (ggml_type type_a : {GGML_TYPE_F32, GGML_TYPE_F16}) { + test_cases.emplace_back(new test_mul_mat_prec_src1_f32(type_a, 64, 1, 256)); + test_cases.emplace_back(new test_mul_mat_prec_src1_f32(type_a, 64, 8, 256)); + test_cases.emplace_back(new test_mul_mat_prec_src1_f32(type_a, 64, 32, 256)); + test_cases.emplace_back(new test_mul_mat_prec_src1_f32(type_a, 128, 512, 512)); + test_cases.emplace_back(new test_mul_mat_prec_src1_f32(type_a, 67, 33, 100)); + test_cases.emplace_back(new test_mul_mat_prec_src1_f32(type_a, 64, 32, 256, {2, 3}, {2, 1})); + } + + // src0 range: an F32 src0 must not be downcast to F16 when src1 requests BF16 + test_cases.emplace_back(new test_mul_mat_prec_src1_bf16_src0(GGML_TYPE_F32, 64, 1, 256)); + test_cases.emplace_back(new test_mul_mat_prec_src1_bf16_src0(GGML_TYPE_F32, 64, 8, 256)); + test_cases.emplace_back(new test_mul_mat_prec_src1_bf16_src0(GGML_TYPE_F32, 64, 32, 256)); + test_cases.emplace_back(new test_mul_mat_prec_src1_bf16_src0(GGML_TYPE_F32, 128, 512, 512)); + test_cases.emplace_back(new test_mul_mat_prec_src1_bf16_src0(GGML_TYPE_F32, 67, 33, 100)); + test_cases.emplace_back(new test_mul_mat_prec_src1_bf16_src0(GGML_TYPE_F32, 64, 32, 256, {2, 3}, {2, 1})); + // FP4 activation precision (default = native W4A4, src1 GGML_PREC_Q8 = W4A8) test_cases.emplace_back(new test_mul_mat_w4a8(GGML_TYPE_NVFP4, GGML_TYPE_F32, 32, 1, 256)); test_cases.emplace_back(new test_mul_mat_w4a8(GGML_TYPE_NVFP4, GGML_TYPE_F32, 32, 32, 256)); @@ -11268,6 +11782,15 @@ static std::vector> make_test_cases_eval() { test_cases.emplace_back(new test_mul_mat_id_w4a8(GGML_TYPE_MXFP4, GGML_TYPE_F32, 8, 2, false, 32, 32, 256)); test_cases.emplace_back(new test_mul_mat_id_w4a4(GGML_TYPE_MXFP4, GGML_TYPE_F32, 8, 2, false, 32, 32, 256)); + // small K at batch > 8, e.g. LoRA B matrices with K = rank + for (ggml_type type_a : {GGML_TYPE_F16, GGML_TYPE_F32, GGML_TYPE_BF16}) { + for (int64_t k : {16, 32, 48}) { + for (int64_t n : {9, 33, 128}) { + test_cases.emplace_back(new test_mul_mat(type_a, GGML_TYPE_F32, 100, n, k, {1, 1}, {1, 1})); + } + } + } + #if 0 // > 2^31-element dst (8.9 GB), about 45 GB peak memory. Too large to be enabled by default. // kernel_mul_mm dst offset of the last batch is past INT32_MAX: rows 0-31 use full tiles, rows 32-39 use edge tiles. @@ -11335,6 +11858,17 @@ static std::vector> make_test_cases_eval() { test_cases.emplace_back(new test_mul_mat(GGML_TYPE_F16, GGML_TYPE_F16, 1, 512, 2048, {1, 1}, {1, 1})); test_cases.emplace_back(new test_mul_mat(GGML_TYPE_F16, GGML_TYPE_F32, 1, 509, 2051, {1, 1}, {1, 1})); test_cases.emplace_back(new test_mul_mat(GGML_TYPE_F16, GGML_TYPE_F16, 1, 509, 2051, {1, 1}, {1, 1})); + test_cases.emplace_back(new test_mul_mat(GGML_TYPE_F8_E4M3, GGML_TYPE_BF16, 16, 4, 3, {1, 1}, {1, 1})); + test_cases.emplace_back(new test_mul_mat(GGML_TYPE_F8_E4M3, GGML_TYPE_BF16, 16, 4, 256, {1, 1}, {1, 1})); + // FP8 views with an odd byte offset must use the cuBLAS fallback instead of MMVF. + for (int n : {1, 2}) { + test_cases.emplace_back(new test_mul_mat(GGML_TYPE_F8_E4M3, GGML_TYPE_F32, 2, n, 64, {1, 1}, {1, 1}, {0, 1, 2, 3}, 66, 1, false, 0, 0, 1)); + } + // Native FP8 cuBLASLt requires 16-byte alignment, including each batch pointer. + for (size_t offset_a : {1, 2, 4, 16}) { + test_cases.emplace_back(new test_mul_mat(GGML_TYPE_F8_E4M3, GGML_TYPE_F32, 16, 16, 64, {1, 1}, {1, 1}, {0, 1, 2, 3}, 0, 1, false, 17, 0, offset_a)); + } + test_cases.emplace_back(new test_mul_mat(GGML_TYPE_F8_E4M3, GGML_TYPE_F32, 16, 16, 64, {2, 1}, {1, 1}, {0, 1, 2, 3}, 0, 1, false, 17, 4)); // testing very large N test_cases.emplace_back(new test_mul_mat(GGML_TYPE_F16, GGML_TYPE_F32, 16, 2105344, 32, {1, 1}, {1, 1})); @@ -11388,6 +11922,28 @@ static std::vector> make_test_cases_eval() { test_cases.emplace_back(new test_mul_mat_pos(type_a, GGML_TYPE_F32, 1000, n, 1024, {1, 1}, {1, 1})); } } + // CUDA few-row MMQ: 1-3 column groups, 32-row tiles (6170 rows), K = 18 units (uneven over the warps) and 9 units (one unit per ring stage) + for (ggml_type type_a : {GGML_TYPE_Q8_0, GGML_TYPE_Q3_K, GGML_TYPE_Q4_K, GGML_TYPE_Q5_K, GGML_TYPE_Q6_K, + GGML_TYPE_IQ3_S, GGML_TYPE_IQ4_NL, GGML_TYPE_IQ4_XS}) { + for (int64_t k : {4608, 2304}) { + test_cases.emplace_back(new test_mul_mat(type_a, GGML_TYPE_F32, 1000, 8, k, {1, 1}, {1, 1})); + test_cases.emplace_back(new test_mul_mat(type_a, GGML_TYPE_F32, 6170, 13, k, {1, 1}, {1, 1})); + test_cases.emplace_back(new test_mul_mat(type_a, GGML_TYPE_F32, 1000, 21, k, {1, 1}, {1, 1})); + } + } + // permuted and broadcast src0/src1 for the 2 byte aligned types + test_cases.emplace_back(new test_mul_mat(GGML_TYPE_Q6_K, GGML_TYPE_F32, 64, 13, 2304, {2, 3}, {1, 1}, {0, 2, 1, 3})); + test_cases.emplace_back(new test_mul_mat(GGML_TYPE_Q6_K, GGML_TYPE_F32, 63, 5, 2304, {3, 2}, {2, 2})); + test_cases.emplace_back(new test_mul_mat(GGML_TYPE_Q3_K, GGML_TYPE_F32, 64, 13, 4608, {2, 3}, {1, 1}, {0, 2, 1, 3})); + test_cases.emplace_back(new test_mul_mat(GGML_TYPE_IQ3_S, GGML_TYPE_F32, 63, 5, 4608, {3, 2}, {2, 2})); + // partial column group, broadcast, padded rows, src1 in [0, 1], src0 in a weights buffer + test_cases.emplace_back(new test_mul_mat(GGML_TYPE_Q4_K, GGML_TYPE_F32, 1000, 5, 4608, {1, 1}, {1, 1})); + test_cases.emplace_back(new test_mul_mat(GGML_TYPE_Q4_K, GGML_TYPE_F32, 64, 13, 4608, {3, 2}, {2, 2})); + test_cases.emplace_back(new test_mul_mat(GGML_TYPE_Q4_K, GGML_TYPE_F32, 1000, 13, 4608, {1, 1}, {1, 1}, {0, 1, 2, 3}, 5120)); + test_cases.emplace_back(new test_mul_mat_pos(GGML_TYPE_Q4_K, GGML_TYPE_F32, 1000, 13, 4608, {1, 1}, {1, 1})); + test_cases.emplace_back(new test_mul_mat_pos(GGML_TYPE_Q5_K, GGML_TYPE_F32, 1000, 13, 4608, {1, 1}, {1, 1})); + test_cases.emplace_back(new test_mul_mat(GGML_TYPE_Q4_K, GGML_TYPE_F32, 6170, 13, 4608, {1, 1}, {1, 1}, {0, 1, 2, 3}, 0, 1, false, 0, 0, 0, true)); + test_cases.emplace_back(new test_mul_mat(GGML_TYPE_Q6_K, GGML_TYPE_F32, 1000, 21, 4608, {1, 1}, {1, 1}, {0, 1, 2, 3}, 0, 1, false, 0, 0, 0, true)); #if 0 { @@ -11462,6 +12018,13 @@ static std::vector> make_test_cases_eval() { } } + // BF16 matrices of few rows and a long K at small batch sizes (MMVF instead of MMF) + for (int n : {2, 3, 4, 5, 8}) { + test_cases.emplace_back(new test_mul_mat(GGML_TYPE_BF16, GGML_TYPE_F32, 320, n, 10240, {1, 1}, {1, 1})); + test_cases.emplace_back(new test_mul_mat(GGML_TYPE_BF16, GGML_TYPE_F32, 512, n, 2560, {1, 1}, {1, 1})); + test_cases.emplace_back(new test_mul_mat(GGML_TYPE_BF16, GGML_TYPE_F32, 128, n, 2560, {1, 1}, {1, 1})); + } + // BF16 is absent from base_types: add the 3 standard non-contig permutations explicitly test_cases.emplace_back(new test_mul_mat(GGML_TYPE_BF16, GGML_TYPE_F32, 16, 1, 256, {2, 3}, {1, 1}, {0, 2, 1, 3})); test_cases.emplace_back(new test_mul_mat(GGML_TYPE_BF16, GGML_TYPE_F32, 16, 1, 256, {2, 3}, {1, 1}, {0, 1, 3, 2})); @@ -11541,6 +12104,23 @@ static std::vector> make_test_cases_eval() { // the batches of a are padded, nb[2] is not a multiple of nb[1] test_cases.emplace_back(new test_mul_mat(GGML_TYPE_F32, GGML_TYPE_F32, 8, 1, 64, {8, 1}, {1, 1}, {0, 1, 2, 3}, 0, 1, false, 16, 16)); test_cases.emplace_back(new test_mul_mat(GGML_TYPE_F16, GGML_TYPE_F32, 8, 16, 64, {8, 1}, {1, 1}, {0, 1, 2, 3}, 0, 1, false, 16, 16)); + // Ragged expert slices, strided routing IDs and broadcast activations. + for (ggml_type type : {GGML_TYPE_Q8_0, GGML_TYPE_IQ4_NL, GGML_TYPE_IQ3_S, GGML_TYPE_Q2_K, GGML_TYPE_Q3_K}) { + for (int n : {3, 17, 33}) { + for (bool broadcast : {false, true}) { + test_cases.emplace_back(new test_mul_mat_id(type, GGML_TYPE_F32, 37, 2, broadcast, 65, n, 256)); + } + test_cases.emplace_back(new test_mul_mat_id(type, GGML_TYPE_F32, 37, 2, false, 65, n, 256, 0, false, true)); + } + } + + // Both experts receive every token + for (ggml_type type : {GGML_TYPE_Q2_K, GGML_TYPE_Q3_K, GGML_TYPE_Q4_K}) { + for (int n : {64, 65}) { + test_cases.emplace_back(new test_mul_mat_id(type, GGML_TYPE_F32, 2, 2, false, 33, n, 512)); + } + } + // as is a view whose experts are strided by more rows than it uses test_cases.emplace_back(new test_mul_mat_id(GGML_TYPE_F32, GGML_TYPE_F32, 4, 2, false, 8, 1, 64, 64)); test_cases.emplace_back(new test_mul_mat_id(GGML_TYPE_F16, GGML_TYPE_F32, 4, 2, false, 8, 16, 64, 64)); @@ -11811,10 +12391,6 @@ static std::vector> make_test_cases_eval() { test_cases.emplace_back(new test_trunc (type, {1024, 1024, 1, 1})); } - test_cases.emplace_back(new test_diag_mask_inf(GGML_TYPE_F32, {10, 10, 1, 1}, 5)); - test_cases.emplace_back(new test_diag_mask_inf(GGML_TYPE_F32, {10, 10, 3, 1}, 5)); - test_cases.emplace_back(new test_diag_mask_inf(GGML_TYPE_F32, {10, 10, 3, 2}, 5)); - #if 0 std::uniform_int_distribution<> dist_ne1(1, 50); int exponent = 1; @@ -11979,6 +12555,17 @@ static std::vector> make_test_cases_eval() { test_cases.emplace_back(new test_rope(type, {128, 32, 2, 1}, 32, GGML_ROPE_TYPE_NEOX, 512, 1.4245f, 0.7465f, 1.4245f, false, 0, true, true, 32)); } + for (ggml_type type : {GGML_TYPE_F32, GGML_TYPE_F16}) { + for (int mode : {GGML_ROPE_TYPE_NORMAL, GGML_ROPE_TYPE_NEOX, GGML_ROPE_TYPE_MROPE, GGML_ROPE_TYPE_IMROPE, GGML_ROPE_TYPE_VISION}) { + for (bool truncate : {false, true}) { + for (bool fw : {false, true}) { + test_cases.emplace_back(new test_rope(type, {128, 2, 3, 1}, 128, mode, 8192, 1.0f/256, 1.0f, 1.0f, false, 0, fw, false, 0, 4096, 1000000.0f, 32.0f, truncate)); + } + test_cases.emplace_back(new test_rope(type, {128, 2, 3, 1}, 128, mode, 8192, 1.0f/256, 1.0f, 1.0f, true, 1, true, true, 0, 4096, 1000000.0f, 32.0f, truncate)); + } + } + } + // Real-model RoPE: F32 forward, packed Q, 512-token prefill. test_cases.emplace_back(new test_rope(GGML_TYPE_F32, {256, 8, 512, 1}, 64, GGML_ROPE_TYPE_IMROPE, 512, 1.0f, 0.0f, 1.0f, false, 0, true)); // qwen3.5 0.8B test_cases.emplace_back(new test_rope(GGML_TYPE_F32, {256, 16, 512, 1}, 64, GGML_ROPE_TYPE_IMROPE, 512, 1.0f, 0.0f, 1.0f, false, 0, true)); // qwen3.5 4B @@ -12012,6 +12599,15 @@ static std::vector> make_test_cases_eval() { test_cases.emplace_back(new test_concat(GGML_TYPE_Q8_0, {8192, 2, 1, 1}, dim == 0 ? 4096 : 1, dim, 0)); } + // GDN conv_input + for (int64_t ne_b_d : { 31, 32, 33, 64 }) { + test_cases.emplace_back(new test_concat(GGML_TYPE_F32, {3, 10240, 1, 1}, ne_b_d, 0, 16)); + test_cases.emplace_back(new test_concat(GGML_TYPE_F32, {1, 37, 1, 1}, ne_b_d, 0, 16)); + } + test_cases.emplace_back(new test_concat(GGML_TYPE_F32, {64, 37, 1, 1}, 32, 0, 16)); + // Empty src0 + test_cases.emplace_back(new test_concat(GGML_TYPE_F32, {0, 64, 1, 1}, 33, 0, 16)); + for (ggml_sort_order order : {GGML_SORT_ORDER_ASC, GGML_SORT_ORDER_DESC}) { for (uint32_t i = 4; i <= 1024*1024; i *= 2) { test_cases.emplace_back(new test_argsort(GGML_TYPE_F32, {i-1, 1, 1, 1})); @@ -12039,6 +12635,8 @@ static std::vector> make_test_cases_eval() { test_cases.emplace_back(new test_top_k_inf({4096, 2, 1, 1}, k)); test_cases.emplace_back(new test_top_k_inf({248320, 1, 1, 1}, k)); } + // a selection of the indexer of sparse attention, with more ties at the threshold than slots + test_cases.emplace_back(new test_top_k(GGML_TYPE_F32, {1088, 2048, 1, 1}, 512, true)); for (int i = 0; i < 20; ++i) { for (int k : {1, 2, 3, 7, 15, 100, 500, 1023, 9999}) { @@ -12124,6 +12722,13 @@ static std::vector> make_test_cases_eval() { test_cases.emplace_back(new test_mean(GGML_TYPE_F32, { 11, 5, 6, 3 }, true, false)); test_cases.emplace_back(new test_mean(GGML_TYPE_F32, { 11, 5, 6, 3 }, false, true)); test_cases.emplace_back(new test_mean(GGML_TYPE_F32, { 11, 5, 6, 3 }, true, true)); + test_cases.emplace_back(new test_mean(GGML_TYPE_F32, { 1, 7, 3, 2 })); + for (int64_t n : {1023, 1024, 1025}) { + test_cases.emplace_back(new test_mean(GGML_TYPE_F32, { n, 5, 1, 1 })); + } + test_cases.emplace_back(new test_mean(GGML_TYPE_F32, { 17, 65537, 1, 1 })); + test_cases.emplace_back(new test_mean(GGML_TYPE_F32, { 512, 1, 1, 1 })); + test_cases.emplace_back(new test_mean(GGML_TYPE_F32, { 281, 1536, 1, 1 })); test_cases.emplace_back(new test_sum(GGML_TYPE_F32, { 33, 1, 1, 1 })); test_cases.emplace_back(new test_sum(GGML_TYPE_F32, { 33, 1024, 1, 1 })); test_cases.emplace_back(new test_sum(GGML_TYPE_F32, { 33, 256, 1, 1 })); @@ -12142,8 +12747,10 @@ static std::vector> make_test_cases_eval() { test_cases.emplace_back(new test_sum_rows(GGML_TYPE_F32, { 33, 1, 1, 1 })); test_cases.emplace_back(new test_sum_rows(GGML_TYPE_F32, { 33, 1024, 1, 1 })); test_cases.emplace_back(new test_sum_rows(GGML_TYPE_F32, { 33, 256, 1, 1 })); + test_cases.emplace_back(new test_sum_rows(GGML_TYPE_F32, { 17, 65537, 1, 1 })); test_cases.emplace_back(new test_group_norm(GGML_TYPE_F32, {64, 64, 320, 1})); test_cases.emplace_back(new test_group_norm(GGML_TYPE_F32, {9, 9, 1280, 1})); + test_cases.emplace_back(new test_group_norm(GGML_TYPE_F32, {3, 1, 1, 65537}, 1)); test_cases.emplace_back(new test_group_norm_mul_add(GGML_TYPE_F32, {64, 64, 320, 1})); test_cases.emplace_back(new test_group_norm_mul_add(GGML_TYPE_F32, {9, 9, 1280, 1})); test_cases.emplace_back(new test_acc(GGML_TYPE_F32, {256, 17, 1, 1}, {256, 16, 1, 1}, -1)); @@ -12175,10 +12782,20 @@ static std::vector> make_test_cases_eval() { test_cases.emplace_back(new test_pad_ext(GGML_TYPE_F32, {4, 2, 300, 300}, 1, 1, 0, 0, 0, 0, 0, 0, 0, false)); test_cases.emplace_back(new test_pad_reflect_1d()); + test_cases.emplace_back(new test_pad_reflect_1d(GGML_TYPE_F32, {1, 3, 2, 2}, 0, 0)); + test_cases.emplace_back(new test_pad_reflect_1d(GGML_TYPE_F32, {5, 3, 2, 2}, 4, 1, true)); + test_cases.emplace_back(new test_pad_reflect_1d(GGML_TYPE_F32, {281, 128, 1, 1}, 2, 2)); + test_cases.emplace_back(new test_pad_reflect_1d(GGML_TYPE_F32, {257, 7, 1, 1}, 0, 256)); + test_cases.emplace_back(new test_pad_reflect_1d(GGML_TYPE_F32, {2, 16777217, 1, 1}, 1, 1)); test_cases.emplace_back(new test_pad_reflect_1d(GGML_TYPE_F32, {3000, 384, 4, 1})); test_cases.emplace_back(new test_roll()); test_cases.emplace_back(new test_roll(3, -2, 1, -1, true)); test_cases.emplace_back(new test_arange()); + test_cases.emplace_back(new test_arange(GGML_TYPE_F32, 3.0f, 3.5f, 1.0f)); + test_cases.emplace_back(new test_arange(GGML_TYPE_F32, -10.0f, 10.0f, 0.3f)); + for (float n : {255.0f, 256.0f, 257.0f, 1023.0f, 1024.0f, 1025.0f}) { + test_cases.emplace_back(new test_arange(GGML_TYPE_F32, -1.0f, -1.0f + n * 0.25f, 0.25f)); + } test_cases.emplace_back(new test_arange(GGML_TYPE_F32, 0.0f, 1048576.0f, 1.0f)); test_cases.emplace_back(new test_timestep_embedding()); test_cases.emplace_back(new test_leaky_relu()); @@ -12210,6 +12827,10 @@ static std::vector> make_test_cases_eval() { test_cases.emplace_back(new test_tri(GGML_TRI_TYPE_LOWER_DIAG)); test_cases.emplace_back(new test_tri(GGML_TRI_TYPE_UPPER)); test_cases.emplace_back(new test_tri(GGML_TRI_TYPE_UPPER_DIAG)); + test_cases.emplace_back(new test_tri_special(GGML_TRI_TYPE_LOWER)); + test_cases.emplace_back(new test_tri_special(GGML_TRI_TYPE_LOWER_DIAG)); + test_cases.emplace_back(new test_tri_special(GGML_TRI_TYPE_UPPER)); + test_cases.emplace_back(new test_tri_special(GGML_TRI_TYPE_UPPER_DIAG)); test_cases.emplace_back(new test_fill(0.0f)); test_cases.emplace_back(new test_fill(2.0f, GGML_TYPE_F32, { 303, 207, 11, 3 })); @@ -12247,6 +12868,9 @@ static std::vector> make_test_cases_eval() { test_cases.emplace_back(new test_solve_tri(GGML_TYPE_F32, { 128, 128, 4, 1 }, { 32, 128, 4, 1 })); test_cases.emplace_back(new test_solve_tri(GGML_TYPE_F32, { 64, 64, 4, 4 }, { 200, 64, 4, 4 })); test_cases.emplace_back(new test_solve_tri(GGML_TYPE_F32, { 64, 64, 4, 4 }, { 384, 64, 4, 4 })); + // n % 32 != 0, output prefilled with NaN + test_cases.emplace_back(new test_solve_tri(GGML_TYPE_F32, { 34, 34, 1, 1 }, { 256, 34, 1, 1 }, true)); + test_cases.emplace_back(new test_solve_tri(GGML_TYPE_F32, { 79, 79, 5, 3 }, { 417, 79, 5, 3 }, true)); for (int tfrm : {0, 1, 2}) { for (bool circular : {false, true}) { @@ -12307,8 +12931,8 @@ static std::vector> make_test_cases_eval() { for (int kv : { 113, 512, 1024, }) { if (nr2 != 1 && kv != 512) continue; for (int nb : { 1, 3, 32, 75, }) { - for (ggml_prec prec : {GGML_PREC_F32, GGML_PREC_DEFAULT}) { - if (hsk != 128 && prec == GGML_PREC_DEFAULT) continue; + for (ggml_prec prec : {GGML_PREC_F32, GGML_PREC_BF16, GGML_PREC_UNDEFINED}) { + if (hsk != 128 && prec != GGML_PREC_F32) continue; for (ggml_type type_KV : {GGML_TYPE_F32, GGML_TYPE_F16, GGML_TYPE_BF16, GGML_TYPE_Q8_0, GGML_TYPE_Q5_1, GGML_TYPE_Q5_0, GGML_TYPE_Q4_1, GGML_TYPE_Q4_0, GGML_TYPE_IQ4_NL}) { if (type_KV != GGML_TYPE_F16 && hsk != 64 && hsk != 72) continue; // DeepSeek MLA: the V cache is a sub-view of the K cache @@ -12355,6 +12979,17 @@ static std::vector> make_test_cases_eval() { test_cases.emplace_back(new test_flash_attn_ext(128, 64, 4, {1, 1}, 128, 2, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q2_0, GGML_TYPE_Q4_0)); test_cases.emplace_back(new test_flash_attn_ext(64, 128, 4, {1, 1}, 128, 2, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q4_0, GGML_TYPE_Q2_0)); test_cases.emplace_back(new test_flash_attn_ext(128, 64, 4, {1, 1}, 64, 2, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q2_0, GGML_TYPE_F16)); + test_cases.emplace_back(new test_flash_attn_ext(64, 64, 4, {1, 1}, 128, 2, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F8_E4M3, GGML_TYPE_F8_E4M3)); + test_cases.emplace_back(new test_flash_attn_ext(128, 128, 4, {1, 1}, 128, 1, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F8_E4M3, GGML_TYPE_F8_E4M3)); + test_cases.emplace_back(new test_flash_attn_ext(256, 256, 4, {1, 1}, 128, 1, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F8_E4M3, GGML_TYPE_F8_E4M3)); + test_cases.emplace_back(new test_flash_attn_ext(64, 64, 4, {1, 1}, 128, 2, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F8_E4M3, GGML_TYPE_F16)); + test_cases.emplace_back(new test_flash_attn_ext(64, 64, 4, {1, 1}, 128, 2, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F8_E4M3)); + + for (int hs : {64, 128, 256}) { + for (ggml_type type_K : {GGML_TYPE_F16, GGML_TYPE_F8_E4M3}) { + test_cases.emplace_back(new test_flash_attn_ext(hs, hs, 4, {1, 1}, 256, 1, true, false, 0, 0, GGML_PREC_F32, type_K, GGML_TYPE_F8_E4M3)); + } + } // q8_0 KV cases: decode and prompt batches, KV pad, permuted KV, feature flags, and long context test_cases.emplace_back(new test_flash_attn_ext(256, 256, 2, {16, 1}, 113, 1, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q8_0, GGML_TYPE_Q8_0)); @@ -12364,6 +12999,18 @@ static std::vector> make_test_cases_eval() { test_cases.emplace_back(new test_flash_attn_ext(256, 256, 2, {16, 1}, 1025, 64, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q8_0, GGML_TYPE_Q8_0, {0, 2, 1, 3})); test_cases.emplace_back(new test_flash_attn_ext(256, 256, 2, {16, 1}, 16384, 1, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q8_0, GGML_TYPE_Q8_0)); + // Dense Q8 KV pre-dequantization: both query thresholds and a partial final tile. + test_cases.emplace_back(new test_flash_attn_ext(64, 64, 1, {8, 1}, 4096, 1024, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q8_0, GGML_TYPE_Q8_0, {0, 2, 1, 3}, false)); + test_cases.emplace_back(new test_flash_attn_ext(64, 64, 1, {8, 1}, 8192, 512, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q8_0, GGML_TYPE_Q8_0, {0, 2, 1, 3}, false)); + test_cases.emplace_back(new test_flash_attn_ext(64, 64, 1, {8, 1}, 8193, 513, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q8_0, GGML_TYPE_Q8_0, {0, 2, 1, 3}, false)); + + // Grouped decode uses Q's head stride, which is not eight-element aligned here. + for (int nb : {1, 3}) { + for (ggml_type type_KV : {GGML_TYPE_F16, GGML_TYPE_Q8_0}) { + test_cases.emplace_back(new test_flash_attn_ext(128, 128, 1, {2, 1}, 512, nb, true, false, 0, 0, GGML_PREC_F32, type_KV, type_KV, {0, 2, 1, 3}, false, false, 0, 4)); + } + } + // MLA shape: the V cache is a sub-view of the K cache, with quantized KV test_cases.emplace_back(new test_flash_attn_ext(576, 512, 1, {8, 1}, 113, 1, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q8_0, GGML_TYPE_Q8_0, {0, 1, 2, 3}, true, true)); test_cases.emplace_back(new test_flash_attn_ext(576, 512, 1, {8, 1}, 1024, 1, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q8_0, GGML_TYPE_Q8_0, {0, 1, 2, 3}, true, true)); @@ -12498,6 +13145,15 @@ static std::vector> make_test_cases_eval() { test_cases.emplace_back(new test_opt_step_adamw(GGML_TYPE_F32, {10, 5, 4, 3})); test_cases.emplace_back(new test_opt_step_sgd(GGML_TYPE_F32, {10, 5, 4, 3})); + for (bool with_bias : {false, true}) { + test_cases.emplace_back(new test_mul_mat_fusion(GGML_TYPE_F8_E4M3, GGML_GLU_OP_SWIGLU, 16, 32, 256, + false, 1, 1, false, with_bias, false, true, {1, 1})); + } + test_cases.emplace_back(new test_mul_mat_fusion(GGML_TYPE_F8_E4M3, GGML_GLU_OP_SWIGLU, 16, 32, 256, + false, 1, 1, false, false, false, true, {2, 1})); + test_cases.emplace_back(new test_mul_mat_fusion(GGML_TYPE_F8_E4M3, GGML_GLU_OP_SWIGLU, 16, 31, 257, + false, 1, 1, false, false, false, true, {1, 1})); + for (ggml_type type : base_types) { for (bool with_gate : {false, true}) { for (bool use_id : {false, true}) { @@ -12520,18 +13176,18 @@ static std::vector> make_test_cases_eval() { if (with_lane_scale && type != GGML_TYPE_NVFP4 && type != GGML_TYPE_F8_E4M3) { continue; } - test_cases.emplace_back(new test_mul_mat_vec_fusion(type, glu_op, 1, 32, 256, + test_cases.emplace_back(new test_mul_mat_fusion(type, glu_op, 1, 32, 256, use_id, 16, 8, b, with_bias, with_gate, with_lane_scale)); - test_cases.emplace_back(new test_mul_mat_vec_fusion(type, glu_op, 1, 32, 256, + test_cases.emplace_back(new test_mul_mat_fusion(type, glu_op, 1, 32, 256, use_id, 16, 8, b, with_bias, with_gate, with_lane_scale, {1, 1})); // multi-token batches (spec decoding) for (int64_t m_batch : { 2, 4, 8 }) { - test_cases.emplace_back(new test_mul_mat_vec_fusion(type, glu_op, m_batch, 32, 256, + test_cases.emplace_back(new test_mul_mat_fusion(type, glu_op, m_batch, 32, 256, use_id, 16, 8, b, with_bias, with_gate, with_lane_scale, {1, 1})); } // the lane scale does a GET_ROWS on the ids, which does not accept -1 if (use_id && !with_lane_scale) { - test_cases.emplace_back(new test_mul_mat_vec_fusion(type, glu_op, 8, 32, 256, + test_cases.emplace_back(new test_mul_mat_fusion(type, glu_op, 8, 32, 256, use_id, 16, 8, b, with_bias, with_gate, with_lane_scale, {1, 1}, true)); } } @@ -12543,7 +13199,7 @@ static std::vector> make_test_cases_eval() { } for (bool b : {false, true}) { - test_cases.emplace_back(new test_mul_mat_vec_fusion(GGML_TYPE_IQ2_S, GGML_GLU_OP_SWIGLU_CLAMP, 1, 32, 256, + test_cases.emplace_back(new test_mul_mat_fusion(GGML_TYPE_IQ2_S, GGML_GLU_OP_SWIGLU_CLAMP, 1, 32, 256, true, 16, 8, b, false, true, false)); } @@ -12553,7 +13209,7 @@ static std::vector> make_test_cases_eval() { GGML_TYPE_Q8_0}) { for (ggml_glu_op glu_op : {GGML_GLU_OP_SWIGLU, GGML_GLU_OP_GEGLU}) { for (int64_t m_batch : {1, 2, 3, 5, 8, 9, 16, 17}) { - test_cases.emplace_back(new test_mul_mat_vec_fusion(type, glu_op, m_batch, 1000, 1024, + test_cases.emplace_back(new test_mul_mat_fusion(type, glu_op, m_batch, 1000, 1024, false, 16, 8, false, false, true, false, { 1, 1 })); } } @@ -12561,7 +13217,7 @@ static std::vector> make_test_cases_eval() { for (int64_t k : {96, 896, 1024, 2880}) { for (ggml_glu_op glu_op : {GGML_GLU_OP_SWIGLU, GGML_GLU_OP_GEGLU}) { - test_cases.emplace_back(new test_mul_mat_vec_fusion(GGML_TYPE_Q8_0, glu_op, 1, 32, k, + test_cases.emplace_back(new test_mul_mat_fusion(GGML_TYPE_Q8_0, glu_op, 1, 32, k, false, 16, 8, false, false, true, false, {1, 1})); } } @@ -12571,7 +13227,7 @@ static std::vector> make_test_cases_eval() { //for (ggml_glu_op glu_op : { GGML_GLU_OP_SWIGLU, GGML_GLU_OP_GEGLU }) { // for (int64_t m_batch : { 2, 3, 4 }) { // for (int64_t rows : { 1, 2, 3 }) { - // test_cases.emplace_back(new test_mul_mat_vec_fusion(GGML_TYPE_Q4_K, glu_op, m_batch, rows, 256, + // test_cases.emplace_back(new test_mul_mat_fusion(GGML_TYPE_Q4_K, glu_op, m_batch, rows, 256, // false, 16, 8, false, false, true, false, { 1, 1 })); // } // } @@ -12579,19 +13235,42 @@ static std::vector> make_test_cases_eval() { // Both sides of the same row-count boundary as above, on the fused path. for (int64_t rows : {6271, 6272, 6273}) { - test_cases.emplace_back(new test_mul_mat_vec_fusion(GGML_TYPE_Q4_K, GGML_GLU_OP_SWIGLU, 2, rows, 256, + test_cases.emplace_back(new test_mul_mat_fusion(GGML_TYPE_Q4_K, GGML_GLU_OP_SWIGLU, 2, rows, 256, false, 16, 8, false, false, true, false, { 1, 1 })); } // Fused gate/up + GLU on the MMQ path, only used if its tiling is efficient: ~1280 tiles per case, one case per type and GLU op. - test_cases.emplace_back(new test_mul_mat_vec_fusion(GGML_TYPE_Q4_K, GGML_GLU_OP_SWIGLU, 17, 81920, 256, false, 1, 1, false, false, true, false, { 1, 1 })); - test_cases.emplace_back(new test_mul_mat_vec_fusion(GGML_TYPE_Q5_K, GGML_GLU_OP_GEGLU, 150, 40960, 256, false, 1, 1, false, false, true, false, { 1, 1 })); - test_cases.emplace_back(new test_mul_mat_vec_fusion(GGML_TYPE_Q6_K, GGML_GLU_OP_SWIGLU_OAI, 150, 40960, 256, false, 1, 1, false, false, true, false, { 1, 1 })); - test_cases.emplace_back(new test_mul_mat_vec_fusion(GGML_TYPE_Q8_0, GGML_GLU_OP_SWIGLU_CLAMP, 150, 6912, 256, false, 1, 1, false, false, true, false, { 2, 3 })); - test_cases.emplace_back(new test_mul_mat_vec_fusion(GGML_TYPE_IQ4_XS, GGML_GLU_OP_SWIGLU, 150, 40960, 512, false, 1, 1, false, false, true, false, { 1, 1 })); + test_cases.emplace_back(new test_mul_mat_fusion(GGML_TYPE_Q4_K, GGML_GLU_OP_SWIGLU, 17, 81920, 256, false, 1, 1, false, false, true, false, { 1, 1 })); + test_cases.emplace_back(new test_mul_mat_fusion(GGML_TYPE_Q5_K, GGML_GLU_OP_GEGLU, 150, 40960, 256, false, 1, 1, false, false, true, false, { 1, 1 })); + test_cases.emplace_back(new test_mul_mat_fusion(GGML_TYPE_Q6_K, GGML_GLU_OP_SWIGLU_OAI, 150, 40960, 256, false, 1, 1, false, false, true, false, { 1, 1 })); + test_cases.emplace_back(new test_mul_mat_fusion(GGML_TYPE_Q8_0, GGML_GLU_OP_SWIGLU_CLAMP, 150, 6912, 256, false, 1, 1, false, false, true, false, { 2, 3 })); + test_cases.emplace_back(new test_mul_mat_fusion(GGML_TYPE_IQ4_XS, GGML_GLU_OP_SWIGLU, 150, 40960, 512, false, 1, 1, false, false, true, false, { 1, 1 })); + test_cases.emplace_back(new test_mul_mat_fusion(GGML_TYPE_NVFP4, GGML_GLU_OP_SWIGLU, 150, 6912, 256, false, 1, 1, false, false, true, true, { 2, 3 })); + // NVFP4 weight scale without gate/up: MMQ applies it in the write-back, the trailing bias add stays a separate node + test_cases.emplace_back(new test_mul_mat_fusion(GGML_TYPE_NVFP4, GGML_GLU_OP_SWIGLU, 150, 2560, 256, false, 1, 1, false, false, false, true, { 1, 1 })); + test_cases.emplace_back(new test_mul_mat_fusion(GGML_TYPE_NVFP4, GGML_GLU_OP_SWIGLU, 150, 2560, 256, true, 16, 8, false, false, false, true)); + // Same pattern with broadcast input, 4 of 16 experts used, 8-bit activations (W4A8), and Q4_K, which keeps the scale as a separate node + test_cases.emplace_back(new test_mul_mat_fusion(GGML_TYPE_NVFP4, GGML_GLU_OP_SWIGLU, 150, 2560, 256, true, 16, 8, true, false, false, true)); + test_cases.emplace_back(new test_mul_mat_fusion(GGML_TYPE_NVFP4, GGML_GLU_OP_SWIGLU, 150, 2560, 256, true, 16, 4, false, false, false, true)); + test_cases.emplace_back(new test_mul_mat_fusion_w4a8(GGML_TYPE_NVFP4, GGML_GLU_OP_SWIGLU, 150, 2560, 256, false, 1, 1, false, false, false, true, { 1, 1 })); + test_cases.emplace_back(new test_mul_mat_fusion_w4a8(GGML_TYPE_NVFP4, GGML_GLU_OP_SWIGLU, 150, 2560, 256, true, 16, 8, false, false, false, true)); + // Few columns and tiles with a long K: split-K through the stream-k fixup, the scale is applied to the partial sums + test_cases.emplace_back(new test_mul_mat_fusion(GGML_TYPE_NVFP4, GGML_GLU_OP_SWIGLU, 11, 64, 2048, false, 1, 1, false, false, false, true, { 1, 1 })); + test_cases.emplace_back(new test_mul_mat_fusion_w4a8(GGML_TYPE_NVFP4, GGML_GLU_OP_SWIGLU, 11, 64, 2048, false, 1, 1, false, false, false, true, { 1, 1 })); + test_cases.emplace_back(new test_mul_mat_fusion(GGML_TYPE_Q4_K, GGML_GLU_OP_SWIGLU, 150, 2560, 256, false, 1, 1, false, false, false, true, { 1, 1 })); + test_cases.emplace_back(new test_mul_mat_fusion(GGML_TYPE_Q4_K, GGML_GLU_OP_SWIGLU, 150, 2560, 256, true, 16, 8, false, false, false, true)); // The same with experts (~1280 tiles each): 17 tokens with broadcast input (one MMA tile per warp), 2048 tokens without (two, several column tiles per expert). - test_cases.emplace_back(new test_mul_mat_vec_fusion(GGML_TYPE_Q4_K, GGML_GLU_OP_SWIGLU, 17, 5120, 256, true, 16, 8, true)); - test_cases.emplace_back(new test_mul_mat_vec_fusion(GGML_TYPE_Q8_0, GGML_GLU_OP_SWIGLU, 2048, 128, 256, true, 40, 8, false)); + test_cases.emplace_back(new test_mul_mat_fusion(GGML_TYPE_Q4_K, GGML_GLU_OP_SWIGLU, 17, 5120, 256, true, 16, 8, true)); + test_cases.emplace_back(new test_mul_mat_fusion(GGML_TYPE_Q8_0, GGML_GLU_OP_SWIGLU, 2048, 128, 256, true, 40, 8, false)); + // NVFP4 with experts, without and with per-expert weight scales. + test_cases.emplace_back(new test_mul_mat_fusion(GGML_TYPE_NVFP4, GGML_GLU_OP_SWIGLU, 150, 2560, 256, true, 16, 8, false)); + test_cases.emplace_back(new test_mul_mat_fusion(GGML_TYPE_NVFP4, GGML_GLU_OP_SWIGLU, 17, 5120, 256, true, 16, 8, true, false, true, true)); + // NVFP4 with 8-bit activations requested (W4A16 checkpoints), without and with weight scales. + test_cases.emplace_back(new test_mul_mat_fusion_w4a8(GGML_TYPE_NVFP4, GGML_GLU_OP_SWIGLU, 150, 2560, 256, true, 16, 8, false)); + test_cases.emplace_back(new test_mul_mat_fusion_w4a8(GGML_TYPE_NVFP4, GGML_GLU_OP_SWIGLU, 150, 40960, 256, false, 1, 1, false, false, true, true, { 1, 1 })); + test_cases.emplace_back(new test_mul_mat_fusion_w4a8(GGML_TYPE_NVFP4, GGML_GLU_OP_SWIGLU, 150, 2560, 256, true, 16, 8, false, false, true, true)); + // K = 2048: few-row MMQ, where enabled, takes the gate and up matmuls instead of the fused MMQ path + test_cases.emplace_back(new test_mul_mat_fusion(GGML_TYPE_Q4_K, GGML_GLU_OP_SWIGLU, 9, 128, 2048, false, 1, 1, false, false, true, false, { 1, 1 })); for (auto gate : {GATING_FUNC_SOFTMAX, GATING_FUNC_SIGMOID, GATING_FUNC_SOFTMAX_WEIGHT, GATING_FUNC_SQRT_SOFTPLUS}) { for (bool with_norm : {false, true}) { @@ -12626,6 +13305,22 @@ static std::vector> make_test_cases_eval() { } } + // the sum is adjusted with an additive bias instead of a lower clamp (DeepSeek-V4.1) + for (bool bias_probs : {false, true}) { + for (float scale_w : {0.0f, 2.0f}) { + test_cases.emplace_back(new test_topk_moe({8, 22, 1, 1}, 4, true, bias_probs, GATING_FUNC_SOFTMAX, scale_w, 1, true)); + test_cases.emplace_back(new test_topk_moe({384, 1, 1, 1}, 6, true, bias_probs, GATING_FUNC_SOFTMAX, scale_w, 1, true)); + test_cases.emplace_back(new test_topk_moe({8, 22, 1, 1}, 4, true, bias_probs, GATING_FUNC_SQRT_SOFTPLUS, scale_w, 1, true)); + test_cases.emplace_back(new test_topk_moe({384, 1, 1, 1}, 6, true, bias_probs, GATING_FUNC_SQRT_SOFTPLUS, scale_w, 1, true)); + } + } + + // sigmoid scoring always ranks a selection bias (GLM, DeepSeek-V2/V3.2) + for (float scale_w : {0.0f, 2.0f}) { + test_cases.emplace_back(new test_topk_moe({8, 22, 1, 1}, 4, true, true, GATING_FUNC_SIGMOID, scale_w, 1, true)); + test_cases.emplace_back(new test_topk_moe({384, 1, 1, 1}, 6, true, true, GATING_FUNC_SIGMOID, scale_w, 1, true)); + } + // Cover the supported boundaries, common k = 8 shapes, interleaved views and adds, and k = 16 fallback. test_cases.emplace_back(new test_moe_reduce(63, 2, 17)); test_cases.emplace_back(new test_moe_reduce(2048, 8, 128)); @@ -12732,9 +13427,33 @@ static std::vector> make_test_cases_eval() { static std::vector> make_test_cases_perf() { std::vector> test_cases; - // delta-net conv input: conv state followed by the transposed qkv of the ubatch - test_cases.emplace_back(new test_concat(GGML_TYPE_F32, {3, 8192, 1, 1}, 512, 0, 16)); + for (float n : {128.0f, 4096.0f, 32768.0f, 1048576.0f}) { + test_cases.emplace_back(new test_arange(GGML_TYPE_F32, 0.0f, n, 1.0f)); + } + + // LoRA at prefill: B is [rank, n_out], so K = rank + for (int64_t k : {16, 32, 48, 64}) { + test_cases.emplace_back(new test_mul_mat(GGML_TYPE_F16, GGML_TYPE_F32, 2048, 512, k, {1, 1}, {1, 1})); + } + + // DSV4_HC_PRE: n_tokens 1 is decode, 128 and 512 are prompt batches + for (int64_t n_tokens : {1, 128, 512}) { + test_cases.emplace_back(new test_dsv4_hc_pre(4096, 4, n_tokens)); // DeepSeek-V4-Flash + test_cases.emplace_back(new test_dsv4_hc_pre(4096, 4, n_tokens, true)); // synthetic, no gated model uses this width + test_cases.emplace_back(new test_dsv4_hc_pre(2560, 4, n_tokens, true)); // Qwen4-Exp (Qwen3.8-Flash-Next) + } + + // DSV4_HC_POST: n_tokens 1 is decode, 128 and 512 are prompt batches + for (int64_t n_tokens : {1, 128, 512}) { + test_cases.emplace_back(new test_dsv4_hc_post(4096, n_tokens)); // DeepSeek-V4-Flash + test_cases.emplace_back(new test_dsv4_hc_post(7168, n_tokens)); // DeepSeek-V4-Pro + test_cases.emplace_back(new test_dsv4_hc_post(2560, n_tokens, true, true)); // Qwen4-Exp, with the post gate chain + test_cases.emplace_back(new test_dsv4_hc_post(2560, n_tokens, true)); // synthetic, Qwen4-Exp kernel without the gate chain + } + // Qwen3.8-Flash-Next pooled indexer prefill. + test_cases.emplace_back(new test_lightning_indexer(128, 4, 1088, 1024, 1, 1, GGML_TYPE_F32)); + test_cases.emplace_back(new test_lightning_indexer(128, 4, 1088, 1024, 1, 1, GGML_TYPE_Q4_0)); // SWIGLU at a 27B-class FFN width, fused [gate|up] vs split operands // note: same bytes either way, so a backend that indexes them differently shows it here for (ggml_type type : {GGML_TYPE_F16, GGML_TYPE_F32, GGML_TYPE_BF16}) { @@ -12744,6 +13463,13 @@ static std::vector> make_test_cases_perf() { } } + // GDN conv_input shapes + for (int64_t channels : {8192, 10240}) { + for (int64_t n_tokens : {512, 2048, 4096}) { + test_cases.emplace_back(new test_concat(GGML_TYPE_F32, {3, channels, 1, 1}, n_tokens, 0, 16)); + } + } + // CONT of a 0<->2 permute at DeepSeek-V4 lightning-indexer shapes: // indexer_kq is [n_kv, n_tokens, n_head=64] and gets ggml_cont(ggml_permute(.., 2,1,0,3)). for (int64_t n_kv : { 1024, 1280, 2048, 2304 }) { @@ -12901,6 +13627,10 @@ static std::vector> make_test_cases_perf() { test_cases.emplace_back(new test_soft_max(GGML_TYPE_F32, {64, 64, 20, 1}, false, false, GGML_TYPE_F32, {1, 1}, 1.0f, 0.0f)); test_cases.emplace_back(new test_soft_max(GGML_TYPE_F32, {77, 64, 20, 1}, false, false, GGML_TYPE_F32, {1, 1}, 1.0f, 0.0f)); + for (int64_t n : {256, 1024}) { + test_cases.emplace_back(new test_count_equal(GGML_TYPE_F32, {4, n, 1, 1})); + } + test_cases.emplace_back(new test_argmax(GGML_TYPE_F32, {32, 10, 1, 1})); test_cases.emplace_back(new test_argmax(GGML_TYPE_F32, {1024, 10, 1, 1})); test_cases.emplace_back(new test_argmax(GGML_TYPE_F32, {32000, 512, 1, 1})); @@ -12911,6 +13641,10 @@ static std::vector> make_test_cases_perf() { test_cases.emplace_back(new test_pad_reflect_1d(GGML_TYPE_F32, {3000, 384, 1, 1})); test_cases.emplace_back(new test_pad_reflect_1d(GGML_TYPE_F32, {3000, 384, 4, 1})); + test_cases.emplace_back(new test_group_norm(GGML_TYPE_F32, {64, 64, 320, 1})); + test_cases.emplace_back(new test_group_norm(GGML_TYPE_F32, {9, 9, 1280, 1})); + test_cases.emplace_back(new test_group_norm(GGML_TYPE_F32, {3, 1, 1, 65537}, 1)); + // SNAKE activation fusion at BigVGAN scale (T=7680 = 24 kHz x 320 ms, C=192) test_cases.emplace_back(new test_snake_fuse(GGML_TYPE_F32, {7680, 192, 1, 1})); test_cases.emplace_back(new test_snake_fuse(GGML_TYPE_F16, {7680, 192, 1, 1})); @@ -12972,7 +13706,7 @@ static std::vector> make_test_cases_perf() { // qwen3-30b-a3b for (int bs : {1, 4, 8, 32, 64, 128, 256, 512}) { - for (ggml_type type_a : {GGML_TYPE_F32, GGML_TYPE_F16, GGML_TYPE_Q4_0, GGML_TYPE_Q8_0, GGML_TYPE_Q4_K, GGML_TYPE_Q6_K, GGML_TYPE_IQ2_XS, GGML_TYPE_IQ4_XS}) { + for (ggml_type type_a : {GGML_TYPE_F32, GGML_TYPE_F16, GGML_TYPE_Q4_0, GGML_TYPE_Q8_0, GGML_TYPE_Q2_K, GGML_TYPE_Q3_K, GGML_TYPE_Q4_K, GGML_TYPE_Q6_K, GGML_TYPE_IQ2_XS, GGML_TYPE_IQ4_XS}) { for (ggml_type type_b : {GGML_TYPE_F32}) { test_cases.emplace_back(new test_mul_mat_id(type_a, type_b, 128, 8, false, 768, bs, 2048)); test_cases.emplace_back(new test_mul_mat_id_fusion(type_a, type_b, 128, 8, false, 768, bs, 2048, 1)); @@ -13014,6 +13748,8 @@ static std::vector> make_test_cases_perf() { // Qwen3-VL-8B https://github.com/ggml-org/llama.cpp/issues/17012 test_cases.emplace_back(new test_flash_attn_ext(72, 72, 16, {1, 1}, 5776, 5776, false, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16)); + // Same non-causal vision-encoder shape for the other head size padded in-kernel by the CUDA MMA kernels + test_cases.emplace_back(new test_flash_attn_ext(40, 40, 16, {1, 1}, 5776, 5776, false, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16)); // Sparse flash attention (n_kv_max hint) decode across KV depths. // Shapes: 576/512 DeepSeek MLA, 512/512 DeepSeek-V4/GLM-5.2, 256/256 gqa12 Qwen QSA. @@ -13047,6 +13783,21 @@ static std::vector> make_test_cases_perf() { test_cases.emplace_back(new test_flash_attn_ext(576, 512, 1, {20, 1}, 8192, 32, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16, {0, 1, 2, 3}, true, true)); test_cases.emplace_back(new test_flash_attn_ext(576, 512, 1, {20, 1}, 8192, 512, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16, {0, 1, 2, 3}, true, true)); + test_cases.emplace_back(new test_flash_attn_ext(96, 96, 2, {8, 1}, 8192, 128, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q8_0, GGML_TYPE_Q8_0, {0, 2, 1, 3}, false)); + for (int64_t kv : {8192, 32768}) { + for (int64_t nq : {128, 256, 512, 1024}) { + test_cases.emplace_back(new test_flash_attn_ext(192, 192, 2, {8, 1}, kv, nq, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q8_0, GGML_TYPE_Q8_0, {0, 2, 1, 3}, false)); + } + } + // MLA uses V as a view of the compressed K cache. + test_cases.emplace_back(new test_flash_attn_ext(576, 512, 1, {20, 1}, 32768, 512, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q8_0, GGML_TYPE_Q8_0, {0, 2, 1, 3}, false, true)); + for (int64_t nq : {256, 512, 1024, 2048}) { + test_cases.emplace_back(new test_flash_attn_ext(256, 256, 2, {8, 1}, 32768, nq, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q8_0, GGML_TYPE_Q8_0, {0, 2, 1, 3}, false)); + } + for (int64_t nq : {512, 1024}) { + test_cases.emplace_back(new test_flash_attn_ext(128, 128, 2, {16, 1}, 4096, nq, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q8_0, GGML_TYPE_Q8_0, {0, 2, 1, 3}, false)); + } + // q8_0 KV cases with long context (decode and prompt) test_cases.emplace_back(new test_flash_attn_ext(256, 256, 2, {16, 1}, 128, 1, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q8_0, GGML_TYPE_Q8_0)); test_cases.emplace_back(new test_flash_attn_ext(256, 256, 2, {16, 1}, 512, 1, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q8_0, GGML_TYPE_Q8_0)); @@ -13108,6 +13859,8 @@ static std::vector> make_test_cases_perf() { test_cases.emplace_back(new test_col2im_1d(GGML_TYPE_F16, 16, 512, 2048, 8, 0)); test_cases.emplace_back(new test_mean(GGML_TYPE_F32, {256, 256, 3, 1})); + test_cases.emplace_back(new test_mean(GGML_TYPE_F32, {512, 1, 1, 1})); + test_cases.emplace_back(new test_mean(GGML_TYPE_F32, {281, 1536, 1, 1})); for (int n_token : {1, 512}) { @@ -13750,7 +14503,7 @@ int main(int argc, char ** argv) { const char * backend_filter = nullptr; const char * params_filter = nullptr; const char * test_file_path = nullptr; - int parallel_workers = 1; + int parallel_workers = test_parallel_workers(1); for (int i = 1; i < argc; i++) { if (strcmp(argv[i], "test") == 0) { diff --git a/tests/test-backend-tensor.cpp b/tests/test-backend-tensor.cpp index 3e8349c0b1b9..9685bc277789 100644 --- a/tests/test-backend-tensor.cpp +++ b/tests/test-backend-tensor.cpp @@ -148,6 +148,28 @@ static bool test_tensor_n(ggml_backend_buffer_type_t buft) { } } + { + uint8_t input[24], expected[128], out[128] = {}; + for (size_t i = 0; i < sizeof(input); i++) { + input[i] = (uint8_t) (i + 1); + } + ggml_backend_tensor_set(t_big, input, 0, sizeof(input)); + + // Verify scattered byte reads, including repeated source offsets. + std::vector tensors(128, t_big); + std::vector offsets(128), sizes(128, 1), data_offsets(128); + for (size_t i = 0; i < tensors.size(); i++) { + offsets[i] = (i * 5) % sizeof(input); + data_offsets[i] = i; + expected[i] = input[offsets[i]]; + } + + ggml_backend_tensor_get_n(tensors.data(), out, offsets.data(), sizes.data(), data_offsets.data(), tensors.size()); + if (memcmp(out, expected, sizeof(out)) != 0) { + return false; + } + } + return true; } diff --git a/tests/test-chat.cpp b/tests/test-chat.cpp index fa4525389976..2c59d1222a52 100644 --- a/tests/test-chat.cpp +++ b/tests/test-chat.cpp @@ -2043,6 +2043,55 @@ static void test_convert_responses_to_chatcmpl() { assert_equals(false, result.contains("tools")); } + + // Test JSON schema conversion + { + json input = json::parse(R"({ + "input": "Hello", + "model": "test-model", + "text": { + "format": { + "type": "json_schema", + "name": "TestOutput", + "schema": { + "type": "object", + "properties": { + "message": {"type": "string"} + }, + "required": ["message"], + "additionalProperties": false + }, + "strict": true + } + } + })"); + + json result = server_chat_convert_responses_to_chatcmpl(input); + + assert_equals(false, result.contains("text")); + assert_equals(true, result.contains("response_format")); + const auto & response_format = result.at("response_format"); + assert_equals(std::string("json_schema"), response_format.at("type").get()); + const auto & json_schema = response_format.at("json_schema"); + assert_equals(std::string("TestOutput"), json_schema.at("name").get()); + assert_equals(true, json_schema.at("strict").get()); + assert_equals(std::string("object"), json_schema.at("schema").at("type").get()); + assert_equals(false, json_schema.contains("type")); + + input["extra_params"] = {{"seed", 42}, {"min_p", 0.1}}; + json result_extra = server_chat_convert_responses_to_chatcmpl(input); + + assert_equals(false, result_extra.contains("text")); + assert_equals(false, result_extra.contains("extra_params")); + assert_equals(42, result_extra.at("seed").get()); + assert_equals(0.1, result_extra.at("min_p").get()); + assert_equals(response_format.dump(), result_extra.at("response_format").dump()); + + input["extra_params"]["response_format"] = {{"type", "text"}}; + result_extra = server_chat_convert_responses_to_chatcmpl(input); + + assert_equals(input.at("extra_params").at("response_format").dump(), result_extra.at("response_format").dump()); + } } // Shared LFM2 parser cases - all variants use one output format and parser @@ -2111,6 +2160,12 @@ static void test_lfm2_parser(const std::string & template_path, bool detailed_de }) .run(); + // Raw control characters in pythonic string arguments must be escaped into valid JSON. + tst.test("<|tool_call_start|>[python(code='a\nb')]<|tool_call_end|>") + .tools({ python_tool }) + .expect_tool_calls({ { "python", R"#({"code": "a\nb"})#", "" } }) + .run(); + // String escape sequences decode to their actual characters (newline + tab here), // so a "write a two line file" style call produces real line breaks, not literal "\n". tst.test("<|tool_call_start|>[python(code=\"First line\\nSecond line\\tindented\")]<|tool_call_end|>") @@ -2348,6 +2403,18 @@ static void test_template_output_peg_parsers(bool detailed_debug) { .expect_content(R"({"amount": 123.45, "date": "2025-12-03"})") .run(); + tst.test( + "I need to output the invoice details in JSON\n" + "
\n\n" + R"({"amount": 123.45, "date": "2025-12-03"})") + .reasoning_format(COMMON_REASONING_FORMAT_AUTO) + .enable_thinking(true) + .tools({ get_time_tool }) + .json_schema(invoice_schema) + .expect_reasoning("I need to output the invoice details in JSON") + .expect_content(R"({"amount": 123.45, "date": "2025-12-03"})") + .run(); + // a tool call ends the prefilled thinking block, with or without a closing tst.test( "\n" @@ -2692,6 +2759,18 @@ static void test_template_output_peg_parsers(bool detailed_debug) { .expect_content(R"({"amount": 123.45, "date": "2025-12-03"})") .run(); + tst.test( + "[THINK]I need to output the invoice details in JSON[/THINK]" + "```json\n" + R"({"amount": 123.45, "date": "2025-12-03"})" + "\n```") + .reasoning_format(COMMON_REASONING_FORMAT_AUTO) + .tools({ special_function_tool }) + .json_schema(invoice_schema) + .expect_reasoning("I need to output the invoice details in JSON") + .expect_content(R"({"amount": 123.45, "date": "2025-12-03"})") + .run(); + // fake tool call marker in reasoning tst.test( "[THINK]Let me think about [TOOL_CALLS]special_function[ARGS]{\"arg1\":1} and more[/THINK]" @@ -3188,6 +3267,61 @@ static void test_template_output_peg_parsers(bool detailed_debug) { .expect(message_with_tool_calls("amount", R"({"orig": 1.5e10})")) .run(); + // The argument grammar follows the shape of the parameter schema: keys in + // alphabetical order, optional keys skipped in place, nested objects and arrays + // shaped the same way, any other value accepted as written. + common_chat_tool shape_tool{ + /* .name = */ "shape", + /* .description = */ "Schema-shaped arguments", + /* .parameters = */ R"({ + "type": "object", + "properties": { + "path": {"type": "string"}, + "edits": { + "type": "array", + "items": { + "type": "object", + "properties": { + "oldText": {"type": "string"}, + "newText": {"type": "string"} + }, + "required": ["oldText", "newText"] + } + }, + "mode": {"type": "string", "enum": ["fast", "safe"]}, + "count": {"type": "integer"}, + "headers": { + "type": "object", + "properties": { + "content-type": {"type": "string"}, + "accept:lang": {"type": "string"} + } + }, + "options": {"type": "object"} + }, + "required": ["path", "edits"] + })", + }; + + tst.test( + "<|tool_call>call:shape{edits:[{newText:<|\"|>b<|\"|>,oldText:<|\"|>a<|\"|>}],path:<|\"|>f.c<|\"|>}") + .tools({ shape_tool }) + .expect(message_with_tool_calls("shape", R"({"edits":[{"newText":"b","oldText":"a"}],"path":"f.c"})")) + .run(); + + // Optional keys in place, quoted keys that are not identifiers + tst.test( + "<|tool_call>call:shape{count:3,edits:[],headers:{<|\"|>accept:lang<|\"|>:<|\"|>en<|\"|>,'content-type':<|\"|>text<|\"|>},mode:<|\"|>fast<|\"|>,options:{b:1,a:[true]},path:<|\"|>f<|\"|>}") + .tools({ shape_tool }) + .expect(message_with_tool_calls("shape", R"({"count":3,"edits":[],"headers":{"accept:lang":"en","content-type":"text"},"mode":"fast","options":{"b":1,"a":[true]},"path":"f"})")) + .run(); + + tst.test( + "<|tool_call>call:shape{edits:[],headers:{\"content-type\":<|\"|>text<|\"|>},path:<|\"|>f<|\"|>}") + .tools({ shape_tool }) + .expect(message_with_tool_calls("shape", R"({"edits":[],"headers":{"content-type":"text"},"path":"f"})")) + .run(); + // Edge cases tst.test( "<|channel>thought\nHello, world!\nWhat's up?") @@ -4277,6 +4411,17 @@ static void test_template_output_peg_parsers(bool detailed_debug) { .expect(message_assist) .run(); + tst.test( + "```json\n" + R"({"amount": 123.45, "date": "2025-12-03"})" + "\n```") + .enable_thinking(false) + .reasoning_format(COMMON_REASONING_FORMAT_DEEPSEEK) + .tools({ get_time_tool }) + .json_schema(invoice_schema) + .expect_content(R"({"amount": 123.45, "date": "2025-12-03"})") + .run(); + // Thinking + content tst.test("I'm\nthinkingHello, world!\nWhat's up?") .enable_thinking(true) @@ -6928,6 +7073,22 @@ static void test_template_output_peg_parsers(bool detailed_debug) { .expect_reasoning("Thinking.") .expect_content("Working on it.Hello!") .run(); + + // Structured output, straight to the final answer + tst.test(" to=user<|message|>" R"({"amount": 123.45, "date": "2025-12-03"})") + .reasoning_format(COMMON_REASONING_FORMAT_AUTO) + .json_schema(invoice_schema) + .expect_content(R"({"amount": 123.45, "date": "2025-12-03"})") + .run(); + + // Structured output after a reasoning message: reasoning stays free-form + tst.test(" to=self<|message|>I need to output the invoice details in JSON<|eom|>" + "<|start|>assistant to=user<|message|>" R"({"amount": 123.45, "date": "2025-12-03"})") + .reasoning_format(COMMON_REASONING_FORMAT_AUTO) + .json_schema(invoice_schema) + .expect_reasoning("I need to output the invoice details in JSON") + .expect_content(R"({"amount": 123.45, "date": "2025-12-03"})") + .run(); } // GPT-OSS format tests @@ -6937,6 +7098,10 @@ static void test_template_output_peg_parsers(bool detailed_debug) { // Basic content only - final channel tst.test("<|channel|>final<|message|>Hello, world!\nWhat's up?").expect(message_assist).run(); + // Final channel with a constraint type, with and without <|constrain|> (#31520) + tst.test("<|channel|>final code<|message|>Hello, world!\nWhat's up?").expect(message_assist).run(); + tst.test("<|channel|>final <|constrain|>json<|message|>Hello, world!\nWhat's up?").expect(message_assist).run(); + // Basic content only - commentary channel tst.test("<|channel|>commentary<|message|>Hello, world!\nWhat's up?").expect(message_assist).run(); diff --git a/tests/test-fp8.cpp b/tests/test-fp8.cpp index a9278783696f..dfd8ac5419c8 100644 --- a/tests/test-fp8.cpp +++ b/tests/test-fp8.cpp @@ -2,13 +2,16 @@ #include "log.h" #include "ggml.h" #include "ggml-backend.h" -#include "ggml-cpu.h" #include "ggml-cpp.h" #include "gguf.h" #include "llama.h" #include "llama-cpp.h" +#include "../src/llama-context.h" +#include "../src/llama-kv-cache.h" +#include "../src/llama-model.h" #include +#include #include #include #include @@ -286,6 +289,306 @@ static void check_fp8_requantize( } } +static void check_fp8_attention(llama_model * model, bool offload_kqv = true) { + // TODO: enable CPU KV cache tests with tensor split after the meta backend handles the attention transfers. + if (model->split_mode() == LLAMA_SPLIT_MODE_TENSOR && !offload_kqv) { + LOG("FP8 attention: tensor split with CPU KV cache is not supported, skipping\n"); + return; + } + + ggml_context_ptr scale_ctx(ggml_init({ 4096, nullptr, true })); + ggml_tensor * k_scale = ggml_new_tensor_1d(scale_ctx.get(), GGML_TYPE_F32, 1); + ggml_tensor * v_scale = ggml_new_tensor_1d(scale_ctx.get(), GGML_TYPE_F32, 1); + ggml_set_name(k_scale, "blk.0.attn_k.k_scale"); + ggml_set_name(v_scale, "blk.0.attn_v.v_scale"); + ggml_backend_buffer_ptr scale_buf(ggml_backend_alloc_ctx_tensors_from_buft(scale_ctx.get(), ggml_backend_dev_buffer_type(model->dev_layer(0)))); + ggml_tensor * old_k_scale = model->layers[0].k_cache_s; + ggml_tensor * old_v_scale = model->layers[0].v_cache_s; + + const int64_t head_size = model->hparams.n_embd_head_k(); + const int64_t n_head = model->hparams.n_head(); + const int64_t n_tokens = 4; + GGML_ASSERT(n_head == model->hparams.n_head_kv()); + GGML_ASSERT(head_size == model->hparams.n_embd_head_v()); + + for (llama_flash_attn_type flash : { LLAMA_FLASH_ATTN_TYPE_DISABLED, LLAMA_FLASH_ATTN_TYPE_ENABLED }) { + if (model->split_mode() == LLAMA_SPLIT_MODE_TENSOR && flash == LLAMA_FLASH_ATTN_TYPE_DISABLED) { + LOG("FP8 attention: tensor split requires flash attention, skipping disabled cases\n"); + continue; + } + for (int scaled : { 1, 2, 3 }) { + const float sk = scaled & 1 ? 0.03125f : 1.0f; + const float sv = scaled & 2 ? 0.125f : 1.0f; + ggml_backend_tensor_set(k_scale, &sk, 0, sizeof(sk)); + ggml_backend_tensor_set(v_scale, &sv, 0, sizeof(sv)); + model->layers[0].k_cache_s = scaled & 1 ? k_scale : nullptr; + model->layers[0].v_cache_s = scaled & 2 ? v_scale : nullptr; + + std::vector tensors[4]; + auto params = llama_context_default_params(); + params.n_ctx = 256; + params.n_batch = n_tokens; + params.n_ubatch = n_tokens; + params.n_threads = 1; + params.n_threads_batch = 1; + params.type_k = scaled & 1 ? GGML_TYPE_F8_E4M3 : GGML_TYPE_F16; + params.type_v = scaled & 2 ? GGML_TYPE_F8_E4M3 : GGML_TYPE_F16; + params.flash_attn_type = flash; + params.offload_kqv = offload_kqv; + if (model->split_mode() == LLAMA_SPLIT_MODE_TENSOR) { + ggml_context_ptr ctx(ggml_init({ 8*ggml_tensor_overhead(), nullptr, true })); + ggml_tensor * q = ggml_new_tensor_4d(ctx.get(), GGML_TYPE_F32, head_size, n_tokens, n_head, 1); + ggml_tensor * k = ggml_new_tensor_4d(ctx.get(), params.type_k, head_size, params.n_ctx, n_head, 1); + ggml_tensor * v = ggml_new_tensor_4d(ctx.get(), params.type_v, head_size, params.n_ctx, n_head, 1); + ggml_tensor * mask = ggml_new_tensor_4d(ctx.get(), GGML_TYPE_F16, params.n_ctx, params.n_ctx, 1, 1); + ggml_tensor * attn = ggml_flash_attn_ext(ctx.get(), q, k, v, mask, 1.0f/sqrtf(head_size), 0.0f, 0.0f); + auto * dev = model->dev_layer(0); + if (!ggml_backend_dev_supports_op(dev, attn)) { + LOG("FP8 attention: scaled=%d offload=%d: tensor-split flash attention not supported on %s, skipping\n", + scaled, offload_kqv, ggml_backend_dev_name(dev)); + continue; + } + } + if (offload_kqv) { + // Cache writes cannot fall back from a preallocated device buffer. + llama_kv_cache probe(*model, false, model->hparams, params.type_k, params.type_v, + flash == LLAMA_FLASH_ATTN_TYPE_DISABLED, false, true, params.n_ctx, 1, + 0, LLAMA_SWA_TYPE_NONE, nullptr, nullptr, nullptr, nullptr); + ggml_context_ptr ctx(ggml_init({ 64*ggml_tensor_overhead(), nullptr, true })); + llama_kv_cache::slot_info slots; + bool supported = true; + for (uint32_t il = 0; il < model->hparams.n_layer(); ++il) { + ggml_tensor * k = ggml_new_tensor_3d(ctx.get(), GGML_TYPE_F32, head_size, n_head, n_tokens); + ggml_tensor * v = ggml_new_tensor_3d(ctx.get(), GGML_TYPE_F32, head_size, n_head, n_tokens); + ggml_tensor * k_idxs = ggml_new_tensor_1d(ctx.get(), GGML_TYPE_I64, n_tokens); + ggml_tensor * v_idxs = ggml_new_tensor_1d(ctx.get(), GGML_TYPE_I64, + flash == LLAMA_FLASH_ATTN_TYPE_DISABLED ? n_tokens*head_size*n_head : n_tokens); + auto * dev = model->dev_layer(il); + if (!ggml_backend_dev_supports_op(dev, probe.cpy_k(ctx.get(), k, k_idxs, il, slots)) || + !ggml_backend_dev_supports_op(dev, probe.cpy_v(ctx.get(), v, v_idxs, il, slots))) { + LOG("FP8 attention: flash=%s scaled=%d offload=1: cache writes not supported on %s, skipping\n", + llama_flash_attn_type_name(flash), scaled, ggml_backend_dev_name(dev)); + supported = false; + break; + } + } + if (!supported) { + continue; + } + } + params.cb_eval_user_data = tensors; + params.cb_eval = [](ggml_tensor * t, bool ask, void * user_data) { + const char * names[] = { "Qcur-0", "Kcur-0", "Vcur-0", "kqv_out-0" }; + for (int i = 0; i < 4; ++i) { + if (strcmp(t->name, names[i]) != 0) { + continue; + } + if (!ask) { + GGML_ASSERT(t->type == GGML_TYPE_F32 && ggml_is_contiguous(t)); + auto & data = static_cast *>(user_data)[i]; + data.resize(ggml_nelements(t)); + ggml_backend_tensor_get(t, data.data(), 0, data.size()*sizeof(float)); + } + return true; + } + return false; + }; + llama_context_ptr lctx(llama_init_from_model(model, params)); + GGML_ASSERT(lctx); + GGML_ASSERT(lctx->get_cparams().flash_attn == (flash == LLAMA_FLASH_ATTN_TYPE_ENABLED)); + get_logits(model, lctx.get(), { 1, 2, 3, 4 }); + llama_synchronize(lctx.get()); + for (const auto & data : tensors) { + GGML_ASSERT(data.size() == (size_t) (head_size*n_head*n_tokens)); + } + + // Round the original K/V independently of the cache write graph. + for (int i : { 1, 2 }) { + const float scale = i == 1 ? sk : sv; + const ggml_type type = i == 1 ? params.type_k : params.type_v; + auto & data = tensors[i]; + if (type == GGML_TYPE_F8_E4M3) { + for (float & value : data) { + value /= scale; + } + std::vector quantized(data.size()); + ggml_get_type_traits(type)->from_float_ref(data.data(), quantized.data(), data.size()); + ggml_get_type_traits(type)->to_float(quantized.data(), data.data(), data.size()); + for (float & value : data) { + value *= scale; + } + } else { + for (float & value : data) { + value = ggml_fp16_to_fp32(ggml_fp32_to_fp16(value)); + } + } + } + + std::vector expected(tensors[3].size(), 0.0f); + for (int64_t token = 0; token < n_tokens; ++token) { + for (int64_t head = 0; head < n_head; ++head) { + const int64_t q_offset = (token*n_head + head)*head_size; + std::vector scores(token + 1); + double max_score = -INFINITY; + for (int64_t past = 0; past <= token; ++past) { + const int64_t kv_offset = (past*n_head + head)*head_size; + double dot = 0.0; + for (int64_t d = 0; d < head_size; ++d) { + dot += (double) tensors[0][q_offset + d]*tensors[1][kv_offset + d]; + } + scores[past] = dot/sqrt((double) head_size); + max_score = std::max(max_score, scores[past]); + } + double sum = 0.0; + for (double & score : scores) { + score = exp(score - max_score); + sum += score; + } + for (int64_t past = 0; past <= token; ++past) { + const int64_t kv_offset = (past*n_head + head)*head_size; + for (int64_t d = 0; d < head_size; ++d) { + expected[q_offset + d] += scores[past]/sum*tensors[2][kv_offset + d]; + } + } + } + } + const double error = nmse(expected, tensors[3]); + if (!std::isfinite(error) || error >= 1e-5) { + LOG_ERR("FP8 attention: flash=%s scaled=%d offload=%d nmse=%.2e\n", llama_flash_attn_type_name(flash), scaled, offload_kqv, error); + } + GGML_ASSERT(std::isfinite(error) && error < 1e-5); + } + } + model->layers[0].k_cache_s = old_k_scale; + model->layers[0].v_cache_s = old_v_scale; +} + +static void check_fp8_context_shift(llama_model * model) { + ggml_backend_ptr backend(ggml_backend_init_by_type(GGML_BACKEND_DEVICE_TYPE_CPU, nullptr)); + GGML_ASSERT(backend); + auto * reg = ggml_backend_dev_backend_reg(ggml_backend_get_device(backend.get())); + auto set_n_threads = (ggml_backend_set_n_threads_t) ggml_backend_reg_get_proc_address(reg, "ggml_backend_set_n_threads"); + GGML_ASSERT(set_n_threads); + set_n_threads(backend.get(), 1); + + ggml_context_ptr scale_ctx(ggml_init({ 4096, nullptr, true })); + ggml_tensor * scale = ggml_new_tensor_1d(scale_ctx.get(), GGML_TYPE_F32, 1); + ggml_backend_buffer_ptr scale_buf(ggml_backend_alloc_ctx_tensors_from_buft(scale_ctx.get(), ggml_backend_cpu_buffer_type())); + ggml_tensor * old_scale = model->layers[0].k_cache_s; + + for (float scale_value : { 1.0f, 0.5f }) { + ggml_backend_tensor_set(scale, &scale_value, 0, sizeof(scale_value)); + model->layers[0].k_cache_s = scale; + + auto params = llama_context_default_params(); + params.n_ctx = 256; + params.n_batch = 4; + params.n_ubatch = 4; + params.n_threads = 1; + params.n_threads_batch = 1; + params.type_k = GGML_TYPE_F8_E4M3; + params.type_v = GGML_TYPE_F8_E4M3; + params.flash_attn_type = LLAMA_FLASH_ATTN_TYPE_DISABLED; + params.offload_kqv = false; + llama_context_ptr lctx(llama_init_from_model(model, params)); + GGML_ASSERT(lctx); + get_logits(model, lctx.get(), { 1, 2, 3, 4 }); + llama_synchronize(lctx.get()); + + auto * kv = dynamic_cast(llama_get_memory(lctx.get())); + GGML_ASSERT(kv && kv->get_can_shift()); + ggml_tensor * k = kv->get_k_storage(0); + std::vector before(ggml_nbytes(k)); + ggml_backend_tensor_get(k, before.data(), 0, before.size()); + + ggml_context_ptr ctx(ggml_init({ 2*1024*1024, nullptr, false })); + const auto & hp = model->hparams; + const auto & cp = lctx->get_cparams(); + ggml_tensor * keys = ggml_new_tensor_3d(ctx.get(), GGML_TYPE_F32, hp.n_embd_head_k(), hp.n_head_kv(), kv->get_size()); + ggml_get_type_traits(GGML_TYPE_F8_E4M3)->to_float(before.data(), (float *) keys->data, ggml_nelements(keys)); + ggml_tensor * shifts = ggml_new_tensor_1d(ctx.get(), GGML_TYPE_I32, kv->get_size()); + std::fill_n(static_cast(shifts->data), ggml_nelements(shifts), 0); + for (int i = 1; i < 4; ++i) { + ((int32_t *) shifts->data)[i] = -1; + } + ggml_tensor * expected_f32 = ggml_rope_ext(ctx.get(), keys, shifts, model->get_rope_factors(cp, 0), hp.n_rot(), hp.rope_type, + cp.n_ctx_orig_yarn, model->get_rope_freq_base(cp, 0), model->get_rope_freq_scale(cp, 0), + cp.yarn_ext_factor, cp.yarn_attn_factor, cp.yarn_beta_fast, cp.yarn_beta_slow); + ggml_cgraph * gf = ggml_new_graph(ctx.get()); + ggml_build_forward_expand(gf, expected_f32); + GGML_ASSERT(ggml_backend_graph_compute(backend.get(), gf) == GGML_STATUS_SUCCESS); + std::vector expected(before.size()); + ggml_quantize_chunk(GGML_TYPE_F8_E4M3, (float *) expected_f32->data, expected.data(), 0, + ggml_nelements(expected_f32)/keys->ne[0], keys->ne[0], nullptr); + + llama_memory_seq_add(kv, 0, 1, 4, -1); + GGML_ASSERT(kv->get_has_shift()); + GGML_ASSERT(lctx->memory_update(false)); + llama_synchronize(lctx.get()); + GGML_ASSERT(!kv->get_has_shift()); + std::vector after(before.size()); + ggml_backend_tensor_get(k, after.data(), 0, after.size()); + GGML_ASSERT(after != before); + GGML_ASSERT(after == expected); + lctx.reset(); + } + + model->layers[0].k_cache_s = old_scale; +} + +enum class test_status { + PASS, + FAIL, + SKIP, +}; + +static test_status test_fp8_kv(const std::string & path, const char * target_backend) { + ggml_backend_dev_t cpu_devices[] = {nullptr}; + auto params = llama_model_default_params(); + params.devices = cpu_devices; + params.progress_callback = silent_model_load_progress; + { + llama_model_ptr model = load_model(path, params); + if (model->arch != LLM_ARCH_LLAMA || model->hparams.n_expert > 0) { + LOG("FP8 KV: only dense Llama fixtures are supported, skipping %s\n", path.c_str()); + return test_status::SKIP; + } + LOG("FP8 KV: CPU attention and context shift\n"); + check_fp8_attention(model.get()); + check_fp8_context_shift(model.get()); + } + + std::vector devices_meta; + for (size_t i = 0; i < ggml_backend_dev_count(); ++i) { + auto * dev = ggml_backend_dev_get(i); + if (target_backend && strcmp(target_backend, ggml_backend_dev_name(dev)) != 0) { + continue; + } + if (ggml_backend_dev_buffer_type(dev) != ggml_backend_cpu_buffer_type()) { + devices_meta.push_back(dev); + } + if (ggml_backend_dev_type(dev) == GGML_BACKEND_DEVICE_TYPE_CPU) { + continue; + } + ggml_backend_dev_t devices[] = {dev, nullptr}; + params.devices = devices; + LOG("FP8 KV: device=%s\n", ggml_backend_dev_name(dev)); + llama_model_ptr model = load_model(path, params); + check_fp8_attention(model.get()); + check_fp8_attention(model.get(), false); + } + if (!target_backend && !devices_meta.empty()) { + devices_meta.push_back(nullptr); + params.devices = devices_meta.data(); + params.split_mode = LLAMA_SPLIT_MODE_TENSOR; + LOG("FP8 KV: tensor split\n"); + llama_model_ptr model = load_model(path, params); + check_fp8_attention(model.get()); + check_fp8_attention(model.get(), false); + } + return test_status::PASS; +} + static void test_fp8(const std::string & model_path) { const auto dir = std::filesystem::temp_directory_path() / ("llama-fp8-" + std::to_string(std::random_device{}())); if (!std::filesystem::create_directory(dir)) { @@ -324,51 +627,93 @@ static void test_fp8(const std::string & model_path) { check_fp8_requantize(input, reference, output, reference_output, n_embd, model_params); } +static const char * test_status_str(test_status status) { + switch (status) { + case test_status::PASS: return "PASS"; + case test_status::FAIL: return "FAIL"; + case test_status::SKIP: return "SKIP"; + } + GGML_ABORT("invalid test status"); +} + int main(int argc, char ** argv) { if (argc == 2 && (strcmp(argv[1], "-h") == 0 || strcmp(argv[1], "--help") == 0)) { - printf("Usage: %s -m model.gguf | --models DIR\n", argv[0]); + printf("Usage: %s (-m model.gguf | --models DIR) [--kv [-b backend]]\n", argv[0]); return 0; } - if (argc != 3 || (strcmp(argv[1], "-m") != 0 && strcmp(argv[1], "--model") != 0 && strcmp(argv[1], "--models") != 0)) { - fprintf(stderr, "Usage: %s -m model.gguf | --models DIR\n", argv[0]); + std::string model_file; + std::string models_dir; + const char * target_backend = nullptr; + bool kv = false; + bool valid = true; + for (int i = 1; i < argc; ++i) { + if ((strcmp(argv[i], "-m") == 0 || strcmp(argv[i], "--model") == 0 || strcmp(argv[i], "--models") == 0) && i + 1 < argc) { + if (!model_file.empty() || !models_dir.empty()) { + valid = false; + break; + } + const bool directory = strcmp(argv[i], "--models") == 0; + if (directory) { + models_dir = argv[++i]; + } else { + model_file = argv[++i]; + } + } else if (strcmp(argv[i], "--kv") == 0) { + kv = true; + } else if ((strcmp(argv[i], "-b") == 0 || strcmp(argv[i], "--backend") == 0) && i + 1 < argc) { + target_backend = argv[++i]; + } else { + valid = false; + break; + } + } + if (!valid || (model_file.empty() && models_dir.empty()) || (target_backend && !kv)) { + fprintf(stderr, "Usage: %s (-m model.gguf | --models DIR) [--kv [-b backend]]\n", argv[0]); return 1; } common_init(); llama_backend_init(); int result = 0; try { + if (target_backend && !ggml_backend_dev_by_name(target_backend)) { + throw std::runtime_error("unknown backend device: " + std::string(target_backend)); + } std::vector models; - if (strcmp(argv[1], "--models") == 0) { - if (!std::filesystem::is_directory(argv[2])) { - throw std::runtime_error("models directory does not exist: " + std::string(argv[2])); + if (!models_dir.empty()) { + if (!std::filesystem::is_directory(models_dir)) { + throw std::runtime_error("models directory does not exist: " + models_dir); } - for (const auto & entry : std::filesystem::directory_iterator(argv[2])) { + for (const auto & entry : std::filesystem::directory_iterator(models_dir)) { if (entry.is_regular_file() && entry.path().extension() == ".gguf") { models.push_back(entry.path().string()); } } std::sort(models.begin(), models.end()); if (models.empty()) { - throw std::runtime_error("no .gguf models found in: " + std::string(argv[2])); + throw std::runtime_error("no .gguf models found in: " + models_dir); } } else { - models.push_back(argv[2]); + models.push_back(model_file); } - std::vector passed; + std::vector results; for (const auto & model : models) { try { - test_fp8(model); - passed.push_back(true); + if (kv) { + results.push_back(test_fp8_kv(model, target_backend)); + } else { + test_fp8(model); + results.push_back(test_status::PASS); + } } catch (const std::exception & err) { fprintf(stderr, "FP8 test failed for '%s': %s\n", model.c_str(), err.what()); - passed.push_back(false); + results.push_back(test_status::FAIL); result = 1; } } printf("\n| Model | Result |\n|-------|--------|\n"); for (size_t i = 0; i < models.size(); ++i) { - printf("| %s | %s |\n", std::filesystem::path(models[i]).filename().string().c_str(), passed[i] ? "PASS" : "FAIL"); + printf("| %s | %s |\n", std::filesystem::path(models[i]).filename().string().c_str(), test_status_str(results[i])); } } catch (const std::exception & err) { fprintf(stderr, "FP8 test failed: %s\n", err.what()); diff --git a/tests/test-jinja.cpp b/tests/test-jinja.cpp index ce5a9749ba94..c79b0d24b873 100644 --- a/tests/test-jinja.cpp +++ b/tests/test-jinja.cpp @@ -3,6 +3,7 @@ #include #include +#include "chat-tts-norm.h" #include "json.h" #include "subproc.h" @@ -16,8 +17,8 @@ using json = common_json; -static void test_template(testing & t, const std::string & name, const std::string & tmpl, const json & vars, const std::string & expect); -static void test_template_error(testing & t, const std::string & name, const std::string & tmpl, const std::string & expect); +static void test_template(testing & t, const std::string & name, const std::string & tmpl, const json & vars, const std::string & expect, bool tts_norm = false); +static void test_template_error(testing & t, const std::string & name, const std::string & tmpl, const std::string & expect, bool tts_norm = false); static void test_whitespace_control(testing & t); static void test_conditionals(testing & t); @@ -39,6 +40,7 @@ static void test_stats(testing & t); static void test_caps(testing & t); static void test_string_parts(testing & t); static void test_fuzzing(testing & t); +static void test_tts_norm_patterns(testing & t); static bool g_python_mode = false; @@ -75,6 +77,7 @@ int main(int argc, char *argv[]) { t.test("string methods", test_string_methods); t.test("array methods", test_array_methods); t.test("object methods", test_object_methods); + t.test("tts norm patterns", test_tts_norm_patterns); if (!g_python_mode) { t.test("hasher", test_hasher); t.test("stats", test_stats); @@ -169,6 +172,18 @@ static void test_conditionals(testing & t) { "big" ); + test_template(t, "chained comparison", + "{{ 1 <= x <= 100 }},{{ 1 <= y <= 100 }},{{ 1 < 2 < 3 < 2 }},{{ 1 == 1 != 2 }},{{ 'a' in 'apple' == 'apple' }}", + {{"x", 42}, {"y", 500}}, + "True,False,False,True,True" + ); + + test_template(t, "chained comparison short circuit", + "{{ 2 < 1 < y }}", + json::object(), + "False" + ); + test_template(t, "object comparison", "{% if {0: 1, none: 2, 1.0: 3, '0': 4, true: 5} == {false: 1, none: 2, 1: 5, '0': 4} %}equal{% endif %}", json::object(), @@ -1318,6 +1333,18 @@ static void test_macros(testing & t) { "Hello, John Smith,Hi, Jane Doe" ); + test_template(t, "macro with unpacked args", + "{% macro add(a, b, c=0) %}{{ a + b + c }}{% endmacro %}{{ add(*[1, 2]) }},{{ add(1, **{'b': 2, 'c': 3}) }},{{ add(*xs, **kw) }}", + {{"xs", json::array({1, 2})}, {"kw", {{"c", 5}}}}, + "3,6,8" + ); + + test_template(t, "filter with unpacked args", + "{{ 'a-b' | replace(*['-', '+']) }}", + json::object(), + "a+b" + ); + test_template(t, "macro with caller", "\ {%- macro nest_dict(o, i, ff='') %}\n\ @@ -2426,6 +2453,12 @@ static void test_object_methods(testing & t) { {{"obj", {{"a", 3}, {"b", 1}}}}, "{\"a\": 3, \"b\": 1, \"c\": 2}" ); + + test_template(t, "dict from dict and unpacked kwargs", + "{% set o = dict(obj, **{k: 2}) %}{{ o|tojson }}", + {{"obj", {{"a", 3}, {"b", 1}}}, {"k", "c"}}, + "{\"a\": 3, \"b\": 1, \"c\": 2}" + ); } static void test_hasher(testing & t) { @@ -2713,13 +2746,134 @@ static void test_string_parts(testing & t) { } -static void test_template_error(testing & t, const std::string & name, const std::string & tmpl, const std::string & expect) { - t.test(name, [&tmpl, &expect](testing & t) { +// TTS normalization patterns: tts_norm_scan() and tts_norm_sub(), see docs/tts-norm-patterns.md +// these functions are only available to task templates (e.g. "tts"), so every test here passes tts_norm = true +static void test_tts_norm_patterns(testing & t) { + test_template(t, "not available without tts norm builtins", + "{{ tts_norm_scan is defined }}", + json::object(), + "False"); + + test_template(t, "tts_norm_scan splits text into matched and unmatched segments", + "{%- for s in tts_norm_scan(text, [['num', '%d+'], ['word', '%a+']]) -%}[{{ s.rule or '-' }}:{{ s.text }}]{%- endfor -%}", + {{"text", "abc 123, x9"}}, + "[word:abc][-: ][num:123][-:, ][word:x][num:9]", true); + + test_template(t, "tts_norm_scan groups, optional group that did not match is empty", + "{%- for s in tts_norm_scan(text, [['money', '%$(%d+)(?:%.(%d%d))?']]) -%}" + "{%- if s.rule -%}{%- set whole, cents = s.groups -%}{{ whole }}|{{ cents }}|{{ s.groups | length }};{%- endif -%}" + "{%- endfor -%}", + {{"text", "$3.50 and $7"}}, + "3|50|2;7||2;", true); + + test_template(t, "tts_norm_scan rules as dict, the first rule that matches wins", + "{%- for s in tts_norm_scan(text, {'ordinal': '%d+(?:st|nd|rd|th)%b', 'number': '%d+'}) -%}" + "{%- if s.rule -%}<{{ s.rule }}:{{ s.text }}>{%- else -%}{{ s.text }}{%- endif -%}" + "{%- endfor -%}", + {{"text", "the 3rd of 10 or 4three"}}, + "the of or three", true); + + // unlike regex, a failed sequence does not retry the other branches: (?:a|ab)c never matches "abc" + test_template(t, "ordered alternation commits to the first branch that matches", + "{%- for s in tts_norm_scan(text, '(?:a|ab)c') -%}[{{ s.rule or '-' }}:{{ s.text }}]{%- endfor -%}", + {{"text", "abc ac"}}, + "[-:abc ][match:ac]", true); + + // unlike regex, quantifiers never give back what they consumed: %d+0 never matches + test_template(t, "quantifiers are possessive", + "{{ tts_norm_sub(text, '%d+0', 'X') }}|{{ tts_norm_sub(text, '%d+', 'X') }}", + {{"text", "100 20"}}, + "100 20|X X", true); + + test_template(t, "quantifier ranges", + "{{ tts_norm_sub(text, '%d{3}-%d{2,}', '#') }}", + {{"text", "123-4 123-45 123-456789"}}, + "123-4 # #", true); + + test_template(t, "lookbehind: a minus sign is not part of a word", + "{{ tts_norm_sub(text, '(?') }}|{{ tts_norm_sub(text, '[^%w%s]', '') }}", + {{"text", "€ 5 & $6!"}}, + "<5> & <6>!| 5 6", true); + + test_template(t, "remove spaces between Han characters", + "{{ tts_norm_sub(text, '(%h) +(%h)', '%1%2') }}", + {{"text", "你 好 world 世界"}}, + "你好 world 世界", true); + + test_template(t, "mini text normalizer", + "{%- set ones = ['zero', 'one', 'two', 'three', 'four', 'five', 'six', 'seven', 'eight', 'nine', 'ten'," + " 'eleven', 'twelve', 'thirteen', 'fourteen', 'fifteen', 'sixteen', 'seventeen', 'eighteen', 'nineteen'] -%}\n" + "{%- set tens = ['', '', 'twenty', 'thirty', 'forty', 'fifty', 'sixty', 'seventy', 'eighty', 'ninety'] -%}\n" + "{%- macro num(n) -%}{{ ones[n] if n < 20 else tens[(n / 10) | int] ~ ('-' ~ ones[n % 10] if n % 10 else '') }}{%- endmacro -%}\n" + "{%- for s in tts_norm_scan(text, [\n" + " ['money', '%$(%d+)(?:%.(%d%d))?'],\n" + " ['percent', '(%d+)%%'],\n" + " ['time', '(%d%d?):(%d%d)(?!%d)'],\n" + " ['neg', '(?a)') }}", "unknown group type", true); + test_template_error(t, "error: wide lookbehind", "{{ tts_norm_scan('x', '(?<=ab)c') }}", "lookbehind must contain exactly one codepoint matcher", true); + test_template_error(t, "error: missing group in replacement", "{{ tts_norm_sub('x', '(a)', '%2') }}", "group %2 does not exist", true); + test_template_error(t, "error: bad rules", "{{ tts_norm_scan('x', 1) }}", "rules must be a string, a dict or a list", true); + } +} + +static void test_template_error(testing & t, const std::string & name, const std::string & tmpl, const std::string & expect, bool tts_norm) { + t.test(name, [&tmpl, &expect, tts_norm](testing & t) { std::string error; try { jinja::lexer lexer; jinja::program ast = jinja::parse_from_tokens(lexer.tokenize(tmpl)); jinja::context ctx(tmpl); + if (tts_norm) { + common_chat_tts_norm_add_builtins(ctx); + } jinja::runtime runtime(ctx); runtime.gather_string_parts(runtime.execute(ast)); } catch (const std::exception & e) { @@ -2729,14 +2883,17 @@ static void test_template_error(testing & t, const std::string & name, const std }); } -static void test_template_cpp(testing & t, const std::string & name, const std::string & tmpl, const json & vars, const std::string & expect) { - t.test(name, [&tmpl, &vars, &expect](testing & t) { +static void test_template_cpp(testing & t, const std::string & name, const std::string & tmpl, const json & vars, const std::string & expect, bool tts_norm) { + t.test(name, [&tmpl, &vars, &expect, tts_norm](testing & t) { jinja::lexer lexer; auto lexer_res = lexer.tokenize(tmpl); jinja::program ast = jinja::parse_from_tokens(lexer_res); jinja::context ctx(tmpl); + if (tts_norm) { + common_chat_tts_norm_add_builtins(ctx); + } jinja::global_from_json(ctx, vars, true); jinja::runtime runtime(ctx); @@ -2788,17 +2945,36 @@ env.filters["tojson"] = lambda x, ensure_ascii=False, indent=None, separators=No env.globals["strftime_now"] = lambda format: datetime.now().strftime(format) env.globals["raise_exception"] = raise_exception +if merged_input.get("tts_norm_ref"): + import importlib.util + spec = importlib.util.spec_from_file_location("chat_tts_norm", merged_input["tts_norm_ref"]) + chat_tts_norm = importlib.util.module_from_spec(spec) + sys.modules["chat_tts_norm"] = chat_tts_norm # required by dataclasses + spec.loader.exec_module(chat_tts_norm) + env.globals["tts_norm_scan"] = chat_tts_norm.tts_norm_scan + env.globals["tts_norm_sub"] = chat_tts_norm.tts_norm_sub + template = env.from_string(tmpl) result = template.render(**vars_json) sys.stdout.buffer.write(result.encode()) )"; -static void test_template_py(testing & t, const std::string & name, const std::string & tmpl, const json & vars, const std::string & expect) { - t.test(name, [&tmpl, &vars, &expect](testing & t) { +// python reference of the TTS normalization patterns, relative to this file +static std::string tts_norm_ref_path() { + std::string path = __FILE__; + const size_t pos = path.find_last_of("/\\"); + return (pos == std::string::npos ? std::string(".") : path.substr(0, pos)) + "/../scripts/chat_tts_norm.py"; +} + +static void test_template_py(testing & t, const std::string & name, const std::string & tmpl, const json & vars, const std::string & expect, bool tts_norm) { + t.test(name, [&tmpl, &vars, &expect, tts_norm](testing & t) { // Prepare arguments json merged; merged["tmpl"] = json(tmpl); merged["vars"] = vars; + if (tts_norm) { + merged["tts_norm_ref"] = tts_norm_ref_path(); + } #ifdef _WIN32 const char * python_executable = "python.exe"; @@ -2859,11 +3035,11 @@ static void test_template_py(testing & t, const std::string & name, const std::s }); } -static void test_template(testing & t, const std::string & name, const std::string & tmpl, const json & vars, const std::string & expect) { +static void test_template(testing & t, const std::string & name, const std::string & tmpl, const json & vars, const std::string & expect, bool tts_norm) { if (g_python_mode) { - test_template_py(t, name, tmpl, vars, expect); + test_template_py(t, name, tmpl, vars, expect, tts_norm); } else { - test_template_cpp(t, name, tmpl, vars, expect); + test_template_cpp(t, name, tmpl, vars, expect, tts_norm); } } diff --git a/tests/test-json-schema-to-grammar.cpp b/tests/test-json-schema-to-grammar.cpp index c7d971b29dd0..ea10ad7e7eb8 100755 --- a/tests/test-json-schema-to-grammar.cpp +++ b/tests/test-json-schema-to-grammar.cpp @@ -1325,6 +1325,66 @@ static void test_all(const std::string & title, std::function +#include + +static void init_model(llama_model & model, uint32_t n_layer) { + model.hparams = {}; + model.hparams.n_layer_all = n_layer; + model.hparams.n_embd_head_k_full = 4; + model.hparams.n_embd_head_v_full = 4; + model.hparams.n_head_arr.fill(1); + model.hparams.n_head_kv_arr.fill(1); + std::iota(model.hparams.kv_source_impl.begin(), model.hparams.kv_source_impl.end(), 0); + model.layers.resize(n_layer); +} + +int main() { + llama_backend_init(); + + { + std::string errors; + llama_log_set([](ggml_log_level level, const char * text, void * user_data) { + if (level == GGML_LOG_LEVEL_ERROR) { + *static_cast(user_data) += text; + } + }, &errors); + + for (llm_arch arch : { LLM_ARCH_DEEPSEEK2, LLM_ARCH_DEEPSEEK32, LLM_ARCH_DEEPSEEK4, + LLM_ARCH_DEEPSEEK41, LLM_ARCH_GLM_DSA, LLM_ARCH_GLM5_NEXT, LLM_ARCH_DOTS3NOTE, + LLM_ARCH_MISTRAL_LARGE4, LLM_ARCH_HY_V4, LLM_ARCH_MINIMAX_M3, LLM_ARCH_QWEN4EXP, + LLM_ARCH_DFLASH }) { + std::unique_ptr model(llama_model_create(arch, llama_model_default_params())); + init_model(*model, 1); + if (arch == LLM_ARCH_DEEPSEEK2) { + model->hparams.n_embd_head_k_mla_impl = 4; + model->hparams.n_embd_head_v_mla_impl = 4; + } + + for (ggml_type type_k : { GGML_TYPE_F16, GGML_TYPE_F8_E4M3 }) { + for (ggml_type type_v : { GGML_TYPE_F16, GGML_TYPE_F8_E4M3 }) { + if (type_k == GGML_TYPE_F16 && type_v == GGML_TYPE_F16) { + continue; + } + auto params = llama_context_default_params(); + params.type_k = type_k; + params.type_v = type_v; + errors.clear(); + llama_context * ctx = llama_init_from_model(model.get(), params); + GGML_ASSERT(ctx == nullptr); + if (type_k == type_v || (!model->hparams.is_mla() && arch != LLM_ARCH_DEEPSEEK4 && arch != LLM_ARCH_DEEPSEEK41)) { + GGML_ASSERT(errors.find("FP8 cache is not supported for architecture") != std::string::npos); + } else { + GGML_ASSERT(errors.find("model does not support different K") != std::string::npos); + } + } + } + } + + llama_log_set(nullptr, nullptr); + } + + { + std::unique_ptr target(llama_model_create(LLM_ARCH_GEMMA4, llama_model_default_params())); + std::unique_ptr assistant(llama_model_create(LLM_ARCH_GEMMA4_ASSISTANT, llama_model_default_params())); + init_model(*target, 3); + init_model(*assistant, 2); + + ggml_context_ptr ctx(ggml_init({ 4096, nullptr, false })); + target->layers[2].k_cache_s = ggml_new_tensor_1d(ctx.get(), GGML_TYPE_F32, 1); + target->layers[2].v_cache_s = ggml_new_tensor_1d(ctx.get(), GGML_TYPE_F32, 1); + assistant->layers[0].k_cache_s = ggml_new_tensor_1d(ctx.get(), GGML_TYPE_F32, 1); + assistant->layers[0].v_cache_s = ggml_new_tensor_1d(ctx.get(), GGML_TYPE_F32, 1); + assistant->layers[1].k_cache_s = assistant->layers[0].k_cache_s; + assistant->layers[1].v_cache_s = assistant->layers[0].v_cache_s; + *static_cast(target->layers[2].k_cache_s->data) = 0.5f; + *static_cast(target->layers[2].v_cache_s->data) = 0.25f; + *static_cast(assistant->layers[0].k_cache_s->data) = 2.0f; + *static_cast(assistant->layers[0].v_cache_s->data) = 4.0f; + + for (ggml_type type_k : { GGML_TYPE_F16, GGML_TYPE_F8_E4M3 }) { + for (ggml_type type_v : { GGML_TYPE_F16, GGML_TYPE_F8_E4M3 }) { + llama_kv_cache source(*target, false, target->hparams, type_k, type_v, + false, false, true, 32, 1, 0, LLAMA_SWA_TYPE_NONE, nullptr, nullptr, + [](uint32_t il) { return il == 0 ? 2 : -1; }, nullptr); + llama_kv_cache shared(*assistant, false, assistant->hparams, type_k, type_v, + false, false, true, 32, 1, 0, LLAMA_SWA_TYPE_NONE, &source, nullptr, nullptr, + [](int32_t il) { return il == 0 ? 2 : 1; }); + + GGML_ASSERT(source.get_k_scale(2) == (type_k == GGML_TYPE_F8_E4M3 ? target->layers[2].k_cache_s : nullptr)); + GGML_ASSERT(source.get_v_scale(2) == (type_v == GGML_TYPE_F8_E4M3 ? target->layers[2].v_cache_s : nullptr)); + GGML_ASSERT(source.get_k_scale(0) == source.get_k_scale(2)); + GGML_ASSERT(source.get_v_scale(0) == source.get_v_scale(2)); + GGML_ASSERT(shared.get_k_storage(0) == source.get_k_storage(2)); + GGML_ASSERT(shared.get_k_scale(0) == source.get_k_scale(2)); + GGML_ASSERT(shared.get_v_scale(0) == source.get_v_scale(2)); + GGML_ASSERT(shared.get_k_scale(1) == nullptr); + GGML_ASSERT(shared.get_v_scale(1) == nullptr); + } + } + } + + llama_backend_free(); + return 0; +} diff --git a/tests/test-llama-archs.cpp b/tests/test-llama-archs.cpp index 0e61ebab5f4f..1f4c8a46b2bb 100644 --- a/tests/test-llama-archs.cpp +++ b/tests/test-llama-archs.cpp @@ -95,6 +95,22 @@ static void fill_ssm_a(std::vector & dst, const char * name, const tensor } } +// RoPE factors divide the frequencies, a value close to 0 gives angles too large for F32 sin/cos to stay accurate +static void fill_rope_factors(std::vector & dst, const char * name, const tensor_data_params & params) { + fill_normal(dst, name, params); + for (float & v : dst) { + v = 1.0f + fabsf(v); + } +} + +// FFN act scales divide the activation, a value close to 0 overflows the F16 staging of the Metal mul-mat kernel +static void fill_ffn_act(std::vector & dst, const char * name, const tensor_data_params & params) { + fill_normal(dst, name, params); + for (float & v : dst) { + v = copysignf(1.0f + fabsf(v), v); + } +} + // a norm preserving residual gate holds alpha^2 + beta^2 == 1, and the conversion resolves both from one raw weight // the alpha and the beta of a block share that weight, so the key drops the suffix and keeps the block static void fill_resid_gate(std::vector & dst, const char * name, const tensor_data_params & params) { @@ -116,9 +132,12 @@ static void fill_exp_probs_b(std::vector & dst, const char *, const tenso // a tensor whose values hold a property of their own, matched on a substring of its name, first match wins static const struct { const char * match; tensor_filler fill; } TENSOR_FILLERS[] = { - { "ssm_a", fill_ssm_a }, - { "_resid_", fill_resid_gate }, - { "exp_probs_b", fill_exp_probs_b }, + { "ssm_a", fill_ssm_a }, + { "_resid_", fill_resid_gate }, + { "exp_probs_b", fill_exp_probs_b }, + { "ffn.act", fill_ffn_act }, + { "rope_freqs", fill_rope_factors }, + { "rope_factors", fill_rope_factors }, }; static void set_tensor_data(struct ggml_tensor * tensor, void * userdata) { @@ -157,6 +176,7 @@ static void usage(char ** argv) { LOG(" -o, --out Save generated test models to instead of running backend tests\n"); LOG(" -v Set log verbosity level\n"); LOG(" -b, --backend Run only on the given backend device\n"); + LOG(" --kv-shift Check K-shift against a decode at the shifted positions (CPU only)\n"); LOG(" -h, --help Show this help message\n\n"); LOG("Examples:\n"); LOG(" %s\n", argv[0]); @@ -227,7 +247,7 @@ struct kv_writer { } }; -static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) { +static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe, const uint32_t n_layer_min = 0) { gguf_context_ptr ret(gguf_init_empty()); kv_writer ms(arch, ret.get()); const uint32_t n_ctx = 256; @@ -267,7 +287,7 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) { n_layer = 6; } else if (arch == LLM_ARCH_STEP35 || arch == LLM_ARCH_LAGUNA) { n_embd = 160; // exercise per-head tensor split granularity with head size 80 - } else if (arch == LLM_ARCH_QWEN3 || arch == LLM_ARCH_MUSE_GLIMMER || arch == LLM_ARCH_AFMOE) { + } else if (arch == LLM_ARCH_QWEN3 || arch == LLM_ARCH_MUSE_GLIMMER || arch == LLM_ARCH_AFMOE || arch == LLM_ARCH_RICURSIVE) { n_head = 4; } else if (arch == LLM_ARCH_LIMITE) { n_head = 4; @@ -279,18 +299,26 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) { || arch == LLM_ARCH_KIMI_LINEAR || arch == LLM_ARCH_BAILINGMOE3 || arch == LLM_ARCH_KIMI_K3 - || arch == LLM_ARCH_GLM5_NEXT + || arch == LLM_ARCH_KIMI_K31 || arch == LLM_ARCH_MISTRAL4 || arch == LLM_ARCH_MISTRAL_LARGE4 - || arch == LLM_ARCH_HY_V4) { + || arch == LLM_ARCH_HY_V4 + || arch == LLM_ARCH_XING4_0) { n_embd = 128; n_head = 1; n_ff = 192; if (arch == LLM_ARCH_MISTRAL_LARGE4) { n_layer = 4; // cover NoPE, shared indexer and KV sharing layers } + } else if (arch == LLM_ARCH_GLM5_NEXT) { + // several heads so that -sm tensor can split them and the MLA flash attention has GQA > 1 + n_embd = 128; + n_head = 16; + n_ff = 192; } else if (arch == LLM_ARCH_NEMOTRON_H || arch == LLM_ARCH_NEMOTRON_H_MOE) { n_layer = 3; + } else if (arch == LLM_ARCH_METIS16) { + n_layer = 4; // 2 passes over 2 physical layers, one attention and one Mamba-2 } else if (arch == LLM_ARCH_CHAMELEON) { n_vocab = 10240; } else if (arch == LLM_ARCH_QWEN3TTS) { @@ -304,7 +332,7 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) { if (arch == LLM_ARCH_QWEN3) { n_head_kv = 1; // MQA coverage } else if (arch == LLM_ARCH_MUSE_GLIMMER || arch == LLM_ARCH_MUSE_SPARK || arch == LLM_ARCH_AFMOE || arch == LLM_ARCH_MAION_CODER || - arch == LLM_ARCH_KOLIBRI1 || arch == LLM_ARCH_LIMITE || arch == LLM_ARCH_GEMMA4) { + arch == LLM_ARCH_KOLIBRI1 || arch == LLM_ARCH_LIMITE || arch == LLM_ARCH_GEMMA4 || arch == LLM_ARCH_RICURSIVE) { n_head_kv = 2; // GQA coverage } if (arch == LLM_ARCH_BEAM) { @@ -312,6 +340,7 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) { n_head_kv = 2; n_layer = 4; } + n_layer = std::max(n_layer, n_layer_min); const uint32_t n_embd_head = n_embd / n_head; ms.add_kv(LLM_KV_GENERAL_ARCHITECTURE, llama_arch_name(arch)); @@ -346,7 +375,8 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) { if (arch == LLM_ARCH_PLAMO2 || arch == LLM_ARCH_JAMBA || arch == LLM_ARCH_NEMOTRON_H || arch == LLM_ARCH_NEMOTRON_H_MOE || arch == LLM_ARCH_GRANITE_HYBRID || arch == LLM_ARCH_LFM2 || arch == LLM_ARCH_LFM2MOE || arch == LLM_ARCH_LFM3 || - arch == LLM_ARCH_KIMI_LINEAR || arch == LLM_ARCH_BAILINGMOE3 || arch == LLM_ARCH_KIMI_K3 || arch == LLM_ARCH_GLM5_NEXT + arch == LLM_ARCH_KIMI_LINEAR || arch == LLM_ARCH_BAILINGMOE3 || arch == LLM_ARCH_KIMI_K3 || arch == LLM_ARCH_KIMI_K31 || + arch == LLM_ARCH_GLM5_NEXT ) { GGML_ASSERT(n_layer >= 2); std::vector n_head_per_layer; @@ -357,10 +387,23 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) { // GLM5 next KDA heads come from the uniform head count, only head_count_kv is per layer. if (arch == LLM_ARCH_GLM5_NEXT) { ms.add_kv(LLM_KV_ATTENTION_HEAD_COUNT, n_head); + // the MLA cache holds one latent head + for (uint32_t & n : n_head_per_layer) { + n = std::min(n, uint32_t(1)); + } } else { ms.add_kv(LLM_KV_ATTENTION_HEAD_COUNT, n_head_per_layer); } ms.add_kv(LLM_KV_ATTENTION_HEAD_COUNT_KV, n_head_per_layer); + } else if (arch == LLM_ARCH_METIS16) { + // the layer types repeat in every pass + std::vector n_head_kv_per_layer; + n_head_kv_per_layer.reserve(n_layer); + for (uint32_t il = 0; il < n_layer; il++) { + n_head_kv_per_layer.push_back(il % 2 == 0 ? n_head_kv : 0); + } + ms.add_kv(LLM_KV_ATTENTION_HEAD_COUNT, n_head); + ms.add_kv(LLM_KV_ATTENTION_HEAD_COUNT_KV, n_head_kv_per_layer); } else { ms.add_kv(LLM_KV_ATTENTION_HEAD_COUNT, n_head); ms.add_kv(LLM_KV_ATTENTION_HEAD_COUNT_KV, (arch == LLM_ARCH_DEEPSEEK4 || arch == LLM_ARCH_DEEPSEEK41) ? uint32_t(1) : n_head_kv); @@ -378,10 +421,12 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) { || arch == LLM_ARCH_KIMI_LINEAR || arch == LLM_ARCH_BAILINGMOE3 || arch == LLM_ARCH_KIMI_K3 + || arch == LLM_ARCH_KIMI_K31 || arch == LLM_ARCH_GLM5_NEXT || arch == LLM_ARCH_HY_V4 || arch == LLM_ARCH_MISTRAL4 - || arch == LLM_ARCH_MISTRAL_LARGE4) { + || arch == LLM_ARCH_MISTRAL_LARGE4 + || arch == LLM_ARCH_XING4_0) { // GLM5 next MLA is nope only, the cache row is the compressed latent alone. ms.add_kv(LLM_KV_ATTENTION_KEY_LENGTH, arch == LLM_ARCH_GLM5_NEXT ? uint32_t(512) : uint32_t(576)); ms.add_kv(LLM_KV_ATTENTION_VALUE_LENGTH, uint32_t(512)); @@ -411,6 +456,13 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) { if (arch == LLM_ARCH_SENSENOVA_U1) { ms.add_kv(LLM_KV_ROPE_FREQ_BASE_SPATIAL, 10000.0f); } + + if (arch == LLM_ARCH_XING4_0) { + // Manifold-Constrained Hyper-Connections (MHC) multi-residual-stream blocks + ms.add_kv(LLM_KV_HYPER_CONNECTION_COUNT, uint32_t(4)); + ms.add_kv(LLM_KV_HYPER_CONNECTION_SINKHORN_ITERATIONS, uint32_t(20)); + ms.add_kv(LLM_KV_HYPER_CONNECTION_EPSILON, 1e-6f); + } ms.add_kv(LLM_KV_ATTENTION_CLAMP_KQV, 1.0f); ms.add_kv(LLM_KV_ATTENTION_LAYERNORM_EPS, 1e-5f); ms.add_kv(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, 1e-5f); @@ -438,7 +490,7 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) { ms.add_kv(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, pattern); } else if (arch == LLM_ARCH_COHERE2MOE || arch == LLM_ARCH_MIMO2 || arch == LLM_ARCH_STEP35 || arch == LLM_ARCH_SPARK2_5 || arch == LLM_ARCH_MUSE_GLIMMER || arch == LLM_ARCH_MUSE_SPARK || arch == LLM_ARCH_GRANITE_SWA || arch == LLM_ARCH_DOTS3NOTE || - arch == LLM_ARCH_KOLIBRI1 || arch == LLM_ARCH_MAPLE || arch == LLM_ARCH_FLINT || arch == LLM_ARCH_BEAM) { + arch == LLM_ARCH_KOLIBRI1 || arch == LLM_ARCH_MAPLE || arch == LLM_ARCH_FLINT || arch == LLM_ARCH_BEAM || arch == LLM_ARCH_RICURSIVE) { std::vector pattern; pattern.reserve(n_layer); for (uint32_t il = 0; il < n_layer; il++) { @@ -608,6 +660,28 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) { ms.add_kv(LLM_KV_SWIGLU_CLAMP_EXP, 7.0f); } + if (arch == LLM_ARCH_RICURSIVE) { + std::vector swiglu_clamp; + swiglu_clamp.reserve(n_layer); + for (uint32_t il = 0; il < n_layer; il++) { + // the random weight scales keep the FFN activations small, a small limit still clamps so the roundtrip checks the per-layer values + swiglu_clamp.push_back(il % 3 == 1 ? 0.0f : 1e-3f); + } + ms.add_kv(LLM_KV_EMBEDDING_SCALE, 16.0f); + ms.add_kv(LLM_KV_SWIGLU_CLAMP, swiglu_clamp); + } + + if (arch == LLM_ARCH_METIS16) { + ms.add_kv(LLM_KV_METIS_PASS_COUNT, uint32_t(2)); + ms.add_kv(LLM_KV_HYPER_CONNECTION_COUNT, uint32_t(4)); + ms.add_kv(LLM_KV_METIS_MEMORY_LENGTH, uint32_t(64)); + ms.add_kv(LLM_KV_METIS_ROUTE_LENGTH, uint32_t(32)); + ms.add_kv(LLM_KV_ATTENTION_PASS_LORA_RANK, uint32_t(8)); + ms.add_kv(LLM_KV_METIS_CONTINUATION_LOGIT_LIMIT, 4.0f); + ms.add_kv(LLM_KV_SSM_NORM_EPS, 1e-5f); + ms.add_kv(LLM_KV_MOE_LATENT_SIZE, n_embd/2); + } + if (arch == LLM_ARCH_LIMITE) { ms.add_kv(LLM_KV_ATTENTION_SCALE, 0.1f); ms.add_kv(LLM_KV_ATTENTION_XSA_EPS, 1e-4f); @@ -698,8 +772,14 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) { ms.add_kv(LLM_KV_KDA_SAFE_GATE, true); ms.add_kv(LLM_KV_KDA_GATE_LOWER_BOUND, -5.0f); if (arch == LLM_ARCH_BAILINGMOE3) { - ms.add_kv(LLM_KV_SWIGLU_CLAMP_EXP, std::vector({0.0f, 4.0f})); - ms.add_kv(LLM_KV_SWIGLU_CLAMP_SHEXP, std::vector({0.0f, 5.0f})); + std::vector clamp_exp; + std::vector clamp_shexp; + for (uint32_t il = 0; il < n_layer; il++) { + clamp_exp .push_back(il % 2 ? 4.0f : 0.0f); + clamp_shexp.push_back(il % 2 ? 5.0f : 0.0f); + } + ms.add_kv(LLM_KV_SWIGLU_CLAMP_EXP, clamp_exp); + ms.add_kv(LLM_KV_SWIGLU_CLAMP_SHEXP, clamp_shexp); } ms.add_kv(LLM_KV_WKV_HEAD_SIZE, n_embd/n_head); ms.add_kv(LLM_KV_SHORTCONV_L_CACHE, uint32_t(3)); @@ -928,6 +1008,7 @@ static bool moe_mandatory(const llm_arch arch) { case LLM_ARCH_DOTS3NOTE: case LLM_ARCH_DEEPSEEK4: case LLM_ARCH_DEEPSEEK41: + case LLM_ARCH_XING4_0: case LLM_ARCH_GLM4_MOE: case LLM_ARCH_GLM_DSA: case LLM_ARCH_EXAONE_MOE: @@ -955,6 +1036,7 @@ static bool moe_mandatory(const llm_arch arch) { case LLM_ARCH_MIMO2: case LLM_ARCH_KIMI_LINEAR: case LLM_ARCH_KIMI_K3: + case LLM_ARCH_KIMI_K31: case LLM_ARCH_GLM5_NEXT: case LLM_ARCH_STEP35: case LLM_ARCH_MISTRAL4: @@ -964,6 +1046,7 @@ static bool moe_mandatory(const llm_arch arch) { case LLM_ARCH_MAPLE: case LLM_ARCH_MUSE_SPARK: case LLM_ARCH_KOLIBRI1: + case LLM_ARCH_METIS16: return true; default: return false; @@ -1384,6 +1467,319 @@ static int test_backends(const std::string & arch_filter, const size_t seed, con return all_ok ? 0 : 1; } +struct kv_shift_move { + llama_pos p0; + llama_pos p1; + llama_pos delta; +}; + +static void decode_range(llama_context * lctx, const llama_token * tokens, uint32_t n_tokens, llama_pos pos0, llama_seq_id seq_id, bool output) { + common_batch batch(lctx); + for (uint32_t i = 0; i < n_tokens; i++) { + batch.add(tokens[i], pos0 + i, seq_id, output || i == n_tokens - 1); + } + if (llama_process(lctx, LLAMA_PROCESS_TYPE_DECODE, batch.get())) { + throw std::runtime_error("failed to decode batch"); + } +} + +// the raw attention scores of each layer are linear in K, unlike the logits they do not depend on how sharp the attention is +struct kq_capture { + bool active = false; + std::vector>> data; +}; + +// the scores are compared cell by cell, so the cells of the probe must match: prefix and probe fit in the SWA window, +// otherwise the probe can reuse a masked SWA cell in one run and not in the other +static bool kq_capture_cb(struct ggml_tensor * t, bool ask, void * user_data) { + kq_capture * cap = (kq_capture *) user_data; + const char * name = ggml_get_name(t); + const bool want = cap->active && t->type == GGML_TYPE_F32 && ggml_is_contiguous(t) && + (strncmp(name, "kq-", 3) == 0 || strncmp(name, "indexer_kq-", 11) == 0 || strncmp(name, "indexer_score-", 14) == 0); + if (ask) { + return want; + } + if (want) { + std::vector buf(ggml_nelements(t)); + ggml_backend_tensor_get(t, buf.data(), 0, ggml_nbytes(t)); + auto it = std::find_if(cap->data.begin(), cap->data.end(), [&](const auto & e) { return e.first == name; }); + if (it == cap->data.end()) { + cap->data.emplace_back(name, std::move(buf)); + } else { + it->second = std::move(buf); + } + } + return true; +} + +struct kq_diff { + double worst = 0.0; + std::string first_bad; // first tensor above the threshold, later layers can inherit its error through Q + std::string bad; // all tensors above the threshold, with their nmse +}; + +static kq_diff compare_kq(const std::vector>> & ref, + const std::vector>> & cur, double thold) { + if (ref.size() != cur.size()) { + throw std::runtime_error("kq tensor count mismatch"); + } + kq_diff res; + for (size_t i = 0; i < ref.size(); i++) { + if (ref[i].first != cur[i].first || ref[i].second.size() != cur[i].second.size()) { + throw std::runtime_error("kq tensor mismatch: " + ref[i].first); + } + double mse_a_b = 0.0; + double mse_a_0 = 0.0; + for (size_t j = 0; j < ref[i].second.size(); j++) { + const double a = ref[i].second[j]; + const double b = cur[i].second[j]; + // masked rows may hold -inf in both + if (!std::isfinite(a) && a == b) { + continue; + } + mse_a_b += (a - b)*(a - b); + mse_a_0 += a*a; + } + const double val = mse_a_b == 0.0 ? 0.0 : mse_a_0 == 0.0 ? INFINITY : mse_a_b/mse_a_0; + res.worst = std::max(res.worst, std::isnan(val) ? INFINITY : val); + if (!(val <= thold)) { + if (res.first_bad.empty()) { + res.first_bad = ref[i].first; + } + char buf[64]; + snprintf(buf, sizeof(buf), " %s=%.1e", ref[i].first.c_str(), val); + res.bad += buf; + } + } + return res; +} + +// decodes the prefix at pos_prefix, applies each step of moves with its own K-shift and returns the logits of the probe at pos_probe +static std::vector get_logits_kv_shift( + llama_model * model, llama_context * lctx, kq_capture & cap, const std::vector & prefix, const std::vector & probe, + llama_pos pos_prefix, const std::vector> & steps, llama_pos pos_probe) { + const uint32_t n_vocab = llama_vocab_n_tokens(llama_model_get_vocab(model)); + llama_memory_t mem = llama_get_memory(lctx); + + cap.data.clear(); + llama_memory_clear(mem, true); + decode_range(lctx, prefix.data(), prefix.size(), pos_prefix, 0, false); + + for (size_t i = 0; i < steps.size(); i++) { + for (const kv_shift_move & mv : steps[i]) { + llama_memory_seq_add(mem, 0, mv.p0, mv.p1, mv.delta); + } + // there is no public API to apply a pending shift, a decode of a throwaway token in seq 1 does it + if (i + 1 < steps.size()) { + decode_range(lctx, probe.data(), 1, 0, 1, false); + llama_memory_seq_rm(mem, 1, -1, -1); + } + } + + cap.active = true; + decode_range(lctx, probe.data(), probe.size(), pos_probe, 0, true); + cap.active = false; + + std::vector ret; + ret.reserve(probe.size()*n_vocab); + for (uint32_t i = 0; i < probe.size(); i++) { + const float * logits_ith = llama_get_logits_ith(lctx, i); + ret.insert(ret.end(), logits_ith, logits_ith + n_vocab); + } + return ret; +} + +// a K-shift is correct if decoding at [d, d+n) and shifting by -d gives the same probe logits as decoding at [0, n) +static int test_kv_shift(const std::string & arch_filter, const size_t seed, const float stdev, const int verbosity) { + struct user_data_t { + struct { + ggml_log_callback callback; + void * user_data; + } log_old; + + int verbosity; + + user_data_t(int verbosity) : verbosity(verbosity) { + llama_log_get(&log_old.callback, &log_old.user_data); + } + }; + user_data_t ud(verbosity); + + llama_log_set([](ggml_log_level level, const char * text, void * user_data) { + const user_data_t * ud = (const user_data_t *) user_data; + int verbosity = common_log_get_verbosity(level); + if (verbosity <= ud->verbosity) { + ud->log_old.callback(level, text, ud->log_old.user_data); + } + }, &ud); + + // n_prefix + n_probe stays below the fixture SWA window (n_ctx/8 = 32) + const uint32_t n_prefix = 20; + const uint32_t n_probe = 8; + const llama_pos d = 64; + const llama_pos gap = 4; + + const std::vector tokens = get_tokens(n_prefix + n_probe, 128, seed); + const std::vector prefix(tokens.begin(), tokens.begin() + n_prefix); + const std::vector probe (tokens.begin() + n_prefix, tokens.end()); + + const std::vector> steps_shift = { { { d, d + (llama_pos) n_prefix, -d } } }; + const std::vector> steps_shift2 = { { { d, d + (llama_pos) n_prefix, -d/2 } }, { { d/2, d/2 + (llama_pos) n_prefix, -d/2 } } }; + // control: the second half of the prefix ends up gap positions too far, which tells if the probe is sensitive to K positions at all + const std::vector> steps_ctrl = { { + { d, d + (llama_pos) n_prefix/2, -d }, + { d + (llama_pos) n_prefix/2, d + (llama_pos) n_prefix, -d + gap }, + } }; + + size_t max_arch_name_length = 0; + for (const llm_arch & arch : llama_arch_all()) { + max_arch_name_length = std::max(max_arch_name_length, strlen(llama_arch_name(arch))); + } + const std::string template_row = "|%" + std::to_string(max_arch_name_length) + "s|%6s|%10s|%10s|%10s|%10s|%s\n"; + + size_t n_tests = 0; + size_t n_failed = 0; + common_log_flush(common_log_main()); + LOG(template_row.c_str(), "Model arch.", "Config", "Shift kq", "Shift2 kq", "Logits", "Control kq", "Status"); + + for (const llm_arch & arch : llama_arch_all()) { + if (arch == LLM_ARCH_UNKNOWN || !arch_matches(arch_filter, arch) || !arch_supported(arch)) { + continue; + } + if (arch == LLM_ARCH_EAGLE3 || arch == LLM_ARCH_DFLASH || arch == LLM_ARCH_GEMMA4_ASSISTANT) { + continue; + } + const bool encode = arch == LLM_ARCH_T5 || arch == LLM_ARCH_DREAM || arch == LLM_ARCH_LLADA || arch == LLM_ARCH_LLADA_MOE || arch == LLM_ARCH_RND1; + if (encode) { + continue; + } + for (bool moe : {false, true}) { + if (moe && !moe_implemented(arch)) { + continue; + } + if (!moe && moe_mandatory(arch)) { + continue; + } + const char * config_name = moe ? "MoE" : "Dense"; + + // the attention scale depends on the absolute position, so the hidden states of the prefix do too + const double thold = arch == LLM_ARCH_MISTRAL_LARGE4 ? 1e-4 : 1e-6; + + // at least 4 layers, so that "every 4th layer" patterns (NoPE, full attention) are covered + // exaone4 is SWA only with 64 layers, smallthinker always ropes with 4 + const uint32_t n_layer_min = arch == LLM_ARCH_EXAONE4 ? 64 : arch == LLM_ARCH_SMALLTHINKER ? 8 : 4; + gguf_context_ptr gguf_ctx = get_gguf_ctx(arch, moe, n_layer_min); + // different bases, so that a shift using the wrong one is caught + gguf_set_val_f32(gguf_ctx.get(), llama_arch_kv_name(arch, LLM_KV_ROPE_FREQ_BASE).c_str(), 20000.0f); + gguf_set_val_f32(gguf_ctx.get(), llama_arch_kv_name(arch, LLM_KV_ROPE_FREQ_BASE_SWA).c_str(), 10000.0f); + if (arch == LLM_ARCH_BAILINGMOE3) { + GGML_ASSERT(gguf_remove_key(gguf_ctx.get(), "bailingmoe3.kda.safe_gate") >= 0); + } + + llama_model_ptr model; + try { + model = get_model(gguf_ctx.get(), nullptr, seed, stdev, {}); + } catch (const std::exception & err) { + LOG(template_row.c_str(), llama_arch_name(arch), config_name, "", "", "", "", "\033[1;31mERROR\033[0m (model load)"); + n_tests++; + n_failed++; + continue; + } + // absolute position embeddings are not translation invariant, the comparison does not hold + if (llama_model_rope_type(model.get()) == LLAMA_ROPE_TYPE_NONE) { + LOG(template_row.c_str(), llama_arch_name(arch), config_name, "", "", "", "", "\033[1;33mSKIP\033[0m (no rope)"); + continue; + } + + LOG("|%*s|%6s|", (int) max_arch_name_length, llama_arch_name(arch), config_name); + fflush(stdout); + + llama_context_params ctx_params = llama_context_default_params(); + ctx_params.n_ctx = 0; + ctx_params.n_threads = 4; + ctx_params.n_threads_batch = 4; + ctx_params.n_seq_max = 2; + ctx_params.kv_unified = true; + ctx_params.type_k = GGML_TYPE_F32; + ctx_params.type_v = GGML_TYPE_F32; + ctx_params.flash_attn_type = LLAMA_FLASH_ATTN_TYPE_DISABLED; + // differs from the SWA default, so that a shift using the wrong scale is caught + ctx_params.rope_scaling_type = LLAMA_ROPE_SCALING_TYPE_LINEAR; + ctx_params.rope_freq_scale = 0.5f; + kq_capture cap; + ctx_params.cb_eval = kq_capture_cb; + ctx_params.cb_eval_user_data = ∩ + + ud.verbosity = LOG_LEVEL_OUTPUT; + llama_context_ptr lctx(llama_init_from_model(model.get(), ctx_params)); + ud.verbosity = verbosity; + if (!lctx) { + LOG("%10s|%10s|%10s|%10s|\033[1;33mSKIP\033[0m (context)\n", "", "", "", ""); + continue; + } + llama_memory_t mem = llama_get_memory(lctx.get()); + if (mem == nullptr || !llama_memory_can_shift(mem)) { + LOG("%10s|%10s|%10s|%10s|\033[1;33mSKIP\033[0m (no shift)\n", "", "", "", ""); + continue; + } + + std::string status; + char shift_str[16] = ""; + char shift2_str[16] = ""; + char logits_str[16] = ""; + char ctrl_str[16] = ""; + n_tests++; + try { + const std::vector logits_ref = get_logits_kv_shift(model.get(), lctx.get(), cap, prefix, probe, 0, {}, n_prefix); + const auto kq_ref = cap.data; + if (verbosity >= LOG_LEVEL_INFO) { + for (const auto & e : kq_ref) { + LOG("[%s: %zu] ", e.first.c_str(), e.second.size()); + } + } + const std::vector logits_shift = get_logits_kv_shift(model.get(), lctx.get(), cap, prefix, probe, d, steps_shift, n_prefix); + const kq_diff diff_shift = compare_kq(kq_ref, cap.data, thold); + get_logits_kv_shift(model.get(), lctx.get(), cap, prefix, probe, d, steps_shift2, n_prefix); + const kq_diff diff_shift2 = compare_kq(kq_ref, cap.data, thold); + const std::vector logits_ctrl = get_logits_kv_shift(model.get(), lctx.get(), cap, prefix, probe, d, steps_ctrl, n_prefix + gap); + const kq_diff diff_ctrl = compare_kq(kq_ref, cap.data, thold); + + const double nmse_logits = nmse(logits_ref, logits_shift); + snprintf(shift_str, sizeof(shift_str), "%.2e", diff_shift.worst); + snprintf(shift2_str, sizeof(shift2_str), "%.2e", diff_shift2.worst); + snprintf(logits_str, sizeof(logits_str), "%.2e", nmse_logits); + snprintf(ctrl_str, sizeof(ctrl_str), "%.2e", diff_ctrl.worst); + + if (kq_ref.empty()) { + status = "\033[1;33mSKIP\033[0m (no kq captured)"; + } else if (!diff_shift.first_bad.empty() || !diff_shift2.first_bad.empty()) { + n_failed++; + status = "\033[1;31mFAIL\033[0m" + (diff_shift.bad.empty() ? " x2:" + diff_shift2.bad : diff_shift.bad); + } else if (!(diff_ctrl.worst > 10*thold)) { + status = "\033[1;33mOK\033[0m (low sensitivity)"; + } else { + status = "\033[1;32mOK\033[0m"; + } + } catch (const std::exception & err) { + n_failed++; + status = std::string("\033[1;31mERROR\033[0m (") + err.what() + ")"; + } + LOG("%10s|%10s|%10s|%10s|%s\n", shift_str, shift2_str, logits_str, ctrl_str, status.c_str()); + } + } + + if (n_tests == 0) { + LOG("Summary: no tests executed\n"); + } else if (n_failed == 0) { + LOG("Summary: all %zu test(s) passed\n", n_tests); + } else { + LOG("Summary: %zu test(s) executed, %zu failed\n", n_tests, n_failed); + } + + llama_log_set(ud.log_old.callback, ud.log_old.user_data); + return n_failed == 0 ? 0 : 1; +} + int main(int argc, char ** argv) { // init the logger at max verbosity. filter with a custom callback respecting the user-configure verbosity common_log_set_verbosity_thold(LOG_LEVEL_DEBUG); @@ -1397,6 +1793,7 @@ int main(int argc, char ** argv) { float stdev = 0.1f; std::string out; const char * target_backend = nullptr; + bool kv_shift = false; int verbosity = LOG_LEVEL_ERROR; @@ -1464,6 +1861,8 @@ int main(int argc, char ** argv) { usage(argv); return 1; } + } else if (strcmp(argv[i], "--kv-shift") == 0) { + kv_shift = true; } else { LOG_ERR("%s: unknown argument: %s\n", __func__, argv[i]); usage(argv); @@ -1482,6 +1881,9 @@ int main(int argc, char ** argv) { if (!out.empty()) { return save_models(arch_filter, seed, stdev, verbosity, out); } + if (kv_shift) { + return test_kv_shift(arch_filter, seed, stdev, verbosity); + } return test_backends(arch_filter, seed, stdev, verbosity, target_backend); } catch (const std::exception & err) { fprintf(stderr, "encountered runtime error: %s\n", err.what()); diff --git a/tests/test-llama-context.cpp b/tests/test-llama-context.cpp index 0e905a7c7e5f..62fedea50296 100644 --- a/tests/test-llama-context.cpp +++ b/tests/test-llama-context.cpp @@ -16,6 +16,7 @@ #include #include #include +#include #include #include #include @@ -202,6 +203,30 @@ static void print_cases() { printf("\nthe [rs] cases are also run against a cache pre-filled with a known pattern\n\n"); } +static unsigned test_hardware_concurrency() { + const char * env = std::getenv("TEST_ARG_HARDWARE_CONCURRENCY"); + if (env != nullptr) { + const long n = std::strtol(env, nullptr, 10); + if (n > 0) { + return (unsigned) n; + } + } + + return std::thread::hardware_concurrency(); +} + +static int test_parallel_workers(int default_workers) { + const char * env = std::getenv("TEST_ARG_PARALLEL_WORKERS"); + if (env != nullptr) { + const long n = std::strtol(env, nullptr, 10); + if (n > 0) { + return (int) n; + } + } + + return default_workers; +} + int main(int argc, char ** argv) { std::setlocale(LC_NUMERIC, "C"); @@ -220,7 +245,7 @@ int main(int argc, char ** argv) { bool list_cases = false; bool dump_logs_on_fail = true; - int parallel_workers = 0; // 0 = auto, same flag as in test-backend-ops + int parallel_workers = test_parallel_workers(0); auto parse_parallel_workers = [&](const char * flag, const char * value) { try { @@ -401,7 +426,7 @@ int main(int argc, char ** argv) { return 1; } - const unsigned n_hw_2 = std::thread::hardware_concurrency() / 2; + const unsigned n_hw_2 = test_hardware_concurrency() / 2; const size_t n_auto = std::max(1, std::min(n_hw_2, 4)); const size_t n_pool = parallel_workers > 0 ? std::min((size_t) parallel_workers, models.size()) diff --git a/tests/test-llama-mmap.cpp b/tests/test-llama-mmap.cpp new file mode 100644 index 000000000000..06f0be188135 --- /dev/null +++ b/tests/test-llama-mmap.cpp @@ -0,0 +1,232 @@ +#include "testing.h" + +#include "../src/llama-mmap.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#if defined(_WIN32) +#include +#endif + +static constexpr size_t test_file_size = LLAMA_DIRECT_IO_BUFFER_SIZE + 3*1024*1024 + 1; +static constexpr size_t test_io_size = 1024*1024; + +static uint8_t test_byte(size_t offset) { + // Mix high offset bits into each byte to detect repeated chunks. + uint64_t value = offset; + value = (value ^ (value >> 30)) * 0xbf58476d1ce4e5b9ULL; + value = (value ^ (value >> 27)) * 0x94d049bb133111ebULL; + return static_cast(value ^ (value >> 31)); +} + +static bool matches(const void * data, size_t offset, size_t size) { + const auto * bytes = static_cast(data); + for (size_t i = 0; i < size; ++i) { + if (bytes[i] != test_byte(offset + i)) { + return false; + } + } + return true; +} + +static void check_test_pattern(testing & t) { + uint8_t buffer[256]; + for (size_t i = 0; i < sizeof(buffer); ++i) { + buffer[i] = test_byte(i); + } + t.assert_true("reject data repeated after 512 KiB", !matches(buffer, 512 * 1024, sizeof(buffer))); + t.assert_true("reject data repeated after a direct I/O chunk", !matches(buffer, LLAMA_DIRECT_IO_BUFFER_SIZE, sizeof(buffer))); +} + +struct temporary_file { + std::filesystem::path path; + + temporary_file() { + const auto now = std::chrono::steady_clock::now().time_since_epoch().count(); + path = std::filesystem::temp_directory_path() / ("test-llama-mmap-" + std::to_string(now) + ".bin"); + } + + ~temporary_file() { + std::error_code error; + std::filesystem::remove(path, error); + } +}; + +struct aligned_buffer { + void * data = nullptr; + + aligned_buffer(size_t alignment, size_t size) { + alignment = std::max(alignment, alignof(std::max_align_t)); +#if defined(_WIN32) + data = _aligned_malloc(size, alignment); +#else + if (posix_memalign(&data, alignment, size) != 0) { + data = nullptr; + } +#endif + } + + ~aligned_buffer() { +#if defined(_WIN32) + _aligned_free(data); +#else + free(data); +#endif + } + + aligned_buffer(const aligned_buffer &) = delete; + aligned_buffer & operator=(const aligned_buffer &) = delete; +}; + +static void write_test_file(const std::filesystem::path & path) { + std::ofstream file(path, std::ios::binary); + if (!file) { + throw std::runtime_error("failed to create test file"); + } + + std::vector buffer(test_io_size); + for (size_t offset = 0; offset < test_file_size; offset += buffer.size()) { + const size_t size = std::min(buffer.size(), test_file_size - offset); + for (size_t i = 0; i < size; ++i) { + buffer[i] = test_byte(offset + i); + } + file.write(reinterpret_cast(buffer.data()), size); + if (!file) { + throw std::runtime_error("failed to write test file"); + } + } +} + +static void check_file(testing & t, const std::filesystem::path & path, bool use_direct_io) { + llama_file file(path.string().c_str(), "rb", use_direct_io); + if (use_direct_io && !file.has_direct_io()) { + t.skip("direct I/O is unavailable for the test file"); + return; + } + t.assert_equal("requested I/O mode", use_direct_io, file.has_direct_io()); + t.assert_equal(test_file_size, file.size()); + t.assert_true("read alignment is nonzero", file.read_alignment() != 0); + + const size_t alignment = file.read_alignment(); + const std::vector> ranges = { + { 0, 1 }, + { 1, 31 }, + { alignment - 1, alignment + 3 }, + { LLAMA_DIRECT_IO_BUFFER_SIZE - 11, 27 }, + { test_file_size - 31, 31 }, + { 1, LLAMA_DIRECT_IO_BUFFER_SIZE + 31 }, + { 1, test_file_size - 1 }, + {0, test_file_size} + }; + for (const auto & range : ranges) { + std::vector buffer(range.second); + file.seek(range.first, SEEK_SET); + file.read_raw(buffer.data(), buffer.size()); + t.assert_equal("I/O mode after read", use_direct_io, file.has_direct_io()); + t.assert_true("read expected bytes", matches(buffer.data(), range.first, buffer.size())); + t.assert_equal(range.first + range.second, file.tell()); + } + + std::vector buffer(test_io_size - 1); + file.seek(0, SEEK_SET); + for (size_t offset = 0; offset < test_file_size; offset += buffer.size()) { + const size_t size = std::min(buffer.size(), test_file_size - offset); + file.read_raw(buffer.data(), size); + t.assert_equal("I/O mode after sequential read", use_direct_io, file.has_direct_io()); + t.assert_true("sequential read expected bytes", matches(buffer.data(), offset, size)); + t.assert_equal(offset + size, file.tell()); + } + + const size_t unsafe_size = ((LLAMA_DIRECT_IO_BUFFER_SIZE + 3 * alignment - 1) / alignment) * alignment; + aligned_buffer buffer_aligned(alignment, unsafe_size); + t.assert_true("allocate aligned buffer", buffer_aligned.data != nullptr); + if (buffer_aligned.data != nullptr) { + const size_t offset = 7 * alignment; + file.seek(offset, SEEK_SET); + file.read_raw_unsafe(buffer_aligned.data, unsafe_size); + t.assert_equal("I/O mode after unsafe read", use_direct_io, file.has_direct_io()); + t.assert_true("unsafe read expected bytes", matches(buffer_aligned.data, offset, unsafe_size)); + t.assert_equal(offset + unsafe_size, file.tell()); + + const size_t final_offset = (test_file_size - 1) & ~(alignment - 1); + file.seek(final_offset, SEEK_SET); + file.read_raw_unsafe(buffer_aligned.data, alignment); + t.assert_equal("I/O mode after unsafe EOF read", use_direct_io, file.has_direct_io()); + t.assert_true("unsafe EOF read expected bytes", matches(buffer_aligned.data, final_offset, test_file_size - final_offset)); + t.assert_equal(test_file_size, file.tell()); + } +} + +static void check_direct_io_fallback(testing & t, const std::filesystem::path & path) { + llama_file file(path.string().c_str(), "rb", true); + if (!file.has_direct_io()) { + t.skip("direct I/O is unavailable for the test file"); + return; + } + t.assert_true("direct I/O is enabled before fallback", file.has_direct_io()); + + const size_t alignment = file.read_alignment(); + aligned_buffer buffer(alignment, alignment + 1); + if (!t.assert_true("allocate aligned buffer", buffer.data != nullptr)) { + return; + } + auto * unaligned = static_cast(buffer.data) + 1; + const size_t offset = 7 * alignment; + file.seek(offset, SEEK_SET); + file.read_raw_unsafe(unaligned, alignment); + +#ifdef _WIN32 + t.assert_true("unaligned read disables direct I/O", !file.has_direct_io()); + t.assert_equal(size_t(1), file.read_alignment()); +#endif + t.assert_true("fallback read expected bytes", matches(unaligned, offset, alignment)); + t.assert_equal(offset + alignment, file.tell()); + + file.read_raw(unaligned, alignment); +#ifdef _WIN32 + t.assert_true("subsequent read stays buffered", !file.has_direct_io()); +#endif + t.assert_true("subsequent read expected bytes", matches(unaligned, offset + alignment, alignment)); + t.assert_equal(offset + 2 * alignment, file.tell()); +} + +static void check_mmap(testing & t, const std::filesystem::path & path) { + if (!llama_mmap::SUPPORTED) { + return; + } + + llama_file file(path.string().c_str(), "rb"); + llama_mmap mapping(&file, 0, false); + t.assert_equal(test_file_size, mapping.size()); + const auto * data = static_cast(mapping.addr()); + t.assert_true("mapping has an address", data != nullptr); + if (data != nullptr) { + t.assert_true("first byte", matches(data, 0, 1)); + t.assert_true("chunk boundary", matches(data + LLAMA_DIRECT_IO_BUFFER_SIZE - 11, LLAMA_DIRECT_IO_BUFFER_SIZE - 11, 27)); + t.assert_true("last bytes", matches(data + test_file_size - 31, test_file_size - 31, 31)); + } +} + +int main() { + testing t; + temporary_file test_file; + t.test("test_pattern", [&] (testing & t) { check_test_pattern(t); }); + t.test("write_test_file", [&] (testing &) { write_test_file(test_file.path); }); + + t.test("llama_file.buffered", [&] (testing & t) { check_file(t, test_file.path, false); }); + t.test("llama_file.direct", [&] (testing & t) { check_file(t, test_file.path, true); }); + t.test("llama_file.direct_fallback", [&] (testing & t) { check_direct_io_fallback(t, test_file.path); }); + t.test("llama_mmap", [&] (testing & t) { check_mmap(t, test_file.path); }); + return t.summary(); +} diff --git a/tests/test-model-resolution.cpp b/tests/test-model-resolution.cpp index 705fd65fa950..e2576a1c9514 100644 --- a/tests/test-model-resolution.cpp +++ b/tests/test-model-resolution.cpp @@ -153,6 +153,14 @@ static const std::vector vendors = { "BlackSheep-24B.Q8_0.gguf", }; +// a tag must match the full token: F16 ends inside BF16, Q4_XS inside IQ4_XS +static const std::vector twins = { + "model-BF16.gguf", + "model-F16.gguf", + "model-IQ4_XS.gguf", + "model-Q4_XS.gguf", +}; + // every speculative sidecar type at the same quant static const std::vector quad = { "model-Q8_0.gguf", @@ -270,6 +278,23 @@ static const plan_case plan_cases[] = { "TheDrummer_Model-24B-v4.1-Q8_0.gguf", {"TheDrummer_Model-24B-v4.1-Q8_0.gguf"}, "", "", "", "", ""}, + // the tag picks the file carrying it as a full token, never a longer one + {"twins f16 tag", twins, "test/repo:F16", "", false, false, + "model-F16.gguf", {"model-F16.gguf"}, + "", "", "", "", ""}, + + {"twins bf16 tag", twins, "test/repo:BF16", "", false, false, + "model-BF16.gguf", {"model-BF16.gguf"}, + "", "", "", "", ""}, + + {"twins q4_xs tag", twins, "test/repo:Q4_XS", "", false, false, + "model-Q4_XS.gguf", {"model-Q4_XS.gguf"}, + "", "", "", "", ""}, + + {"twins iq4_xs tag", twins, "test/repo:IQ4_XS", "", false, false, + "model-IQ4_XS.gguf", {"model-IQ4_XS.gguf"}, + "", "", "", "", ""}, + // every sidecar type resolves at the tag {"quad exact tag", quad, "test/repo:Q8_0", "", true, false, "model-Q8_0.gguf", {"model-Q8_0.gguf"}, @@ -868,7 +893,7 @@ static const real_repo real_repos[] = { {"Q8_0", "MTP/Qwen3.8-27B-Uncensored-OrcaRouter-MTP-Q8_0.gguf", 1, "", "", "", "", ""}, {"BF16", "BF16/Qwen3.8-27B-Uncensored-OrcaRouter-FP8-derived-BF16-00001-of-00002.gguf", 2, "", "", "", "", ""}, {"Q5_K_M", "Qwen3.8-27B-Uncensored-OrcaRouter-Q5_K_M.gguf", 1, "", "", "", "", ""}, - {"F16", "BF16/Qwen3.8-27B-Uncensored-OrcaRouter-FP8-derived-BF16-00001-of-00002.gguf", 2, "", "", "", "", ""}}, + {"F16", "", 0, "", "", "", "", ""}}, {"Q8_0", "Q5_K_M", "Q4_K_M", "BF16"}}, {"ChrisColeTech/qwen-image-edit-uncensored-GGUF", @@ -980,7 +1005,7 @@ static const real_repo real_repos[] = { {"Q8_0", "Qwen3.5-9B-The-Defiant-Fable-Uncnr-Heretic-NEO-MAX-MTP-Q8_0.gguf", 1, "mmproj-BF16.gguf", "", "", "", ""}, {"IQ2_M", "Qwen3.5-9B-The-Defiant-Fable-Uncnr-Heretic-NEO-MAX-IQ2_M.gguf", 1, "mmproj-BF16.gguf", "", "", "", ""}, {"BF16", "Qwen3.5-9B-The-Defiant-Fable-Uncnr-Heretic-plusIQ-NEO-MAX-MTP-bf16.gguf", 1, "mmproj-BF16.gguf", "", "", "", ""}, - {"F16", "Qwen3.5-9B-The-Defiant-Fable-Uncnr-Heretic-plusIQ-NEO-MAX-MTP-bf16.gguf", 1, "mmproj-BF16.gguf", "", "", "", ""}}, + {"F16", "", 0, "", "", "", "", ""}}, {"Q8_0", "BF16", "Q4_K_S", "Q4_K_M", "IQ2_M"}}, {"DavidAU/Qwen3.6-27B-Heretic-Uncensored-FINETUNE-NEO-CODE-Di-IMatrix-MAX-GGUF", @@ -1044,7 +1069,7 @@ static const real_repo real_repos[] = { {"Q4_K_M", "Qwythos-27B-MTP-Q4_K_M.gguf", 1, "mmproj-Qwythos-27B-F16.gguf", "", "", "", ""}, {"Q8_0", "Qwythos-27B-MTP-Q8_0.gguf", 1, "mmproj-Qwythos-27B-F16.gguf", "", "", "", ""}, {"BF16", "Qwythos-27B-BF16.gguf", 1, "mmproj-Qwythos-27B-F16.gguf", "", "", "", ""}, - {"F16", "Qwythos-27B-BF16.gguf", 1, "mmproj-Qwythos-27B-F16.gguf", "", "", "", ""}}, + {"F16", "", 0, "", "", "", "", ""}}, {"Q8_0", "Q5_K_M", "Q4_K_M", "BF16"}}, {"empero-ai/Qwythos-9B-v2-GGUF", @@ -1199,6 +1224,20 @@ static const real_repo real_repos[] = { {"BF16", "", 0, "", "", "", "eagle3-gpt-oss-120b-BF16.gguf", ""}}, {"MXFP4"}}, + // F16 and BF16 in one repo, the F16 tag must not pick the BF16 file + {"ggml-org/Llama-2-7B-GGUF", + {{"Llama-2-7B-BF16.gguf"}, + {"Llama-2-7B-F16.gguf"}, + {"Llama-2-7B-Q4_0.gguf"}, + {"Llama-2-7B-Q8_0.gguf"}}, + { + {"", "Llama-2-7B-Q8_0.gguf", 1, "", "", "", "", ""}, + {"Q8_0", "Llama-2-7B-Q8_0.gguf", 1, "", "", "", "", ""}, + {"Q4_0", "Llama-2-7B-Q4_0.gguf", 1, "", "", "", "", ""}, + {"F16", "Llama-2-7B-F16.gguf", 1, "", "", "", "", ""}, + {"BF16", "Llama-2-7B-BF16.gguf", 1, "", "", "", "", ""}}, + {"F16", "Q4_0", "Q8_0", "BF16"}}, + {"ggml-org/MiMo-V2.6-Distill-Qwen-9B-GGUF", {{"MiMo-V2.6-Distill-Qwen-9B-Q8_0.gguf"}, {"mmproj-MiMo-V2.6-Distill-Qwen-9B-Q8_0.gguf"}}, @@ -1412,7 +1451,7 @@ static const real_repo real_repos[] = { {"Q4_K_M", "Qwen3.5-27B-Uncensored-HauhauCS-Aggressive-Q4_K_M.gguf", 1, "mmproj-Qwen3.5-27B-Uncensored-HauhauCS-Aggressive-f16.gguf", "", "", "", ""}, {"Q8_0", "Qwen3.5-27B-Uncensored-HauhauCS-Aggressive-Q8_0.gguf", 1, "mmproj-Qwen3.5-27B-Uncensored-HauhauCS-Aggressive-f16.gguf", "", "", "", ""}, {"BF16", "Qwen3.5-27B-Uncensored-HauhauCS-Aggressive-BF16.gguf", 1, "mmproj-Qwen3.5-27B-Uncensored-HauhauCS-Aggressive-f16.gguf", "", "", "", ""}, - {"F16", "Qwen3.5-27B-Uncensored-HauhauCS-Aggressive-BF16.gguf", 1, "mmproj-Qwen3.5-27B-Uncensored-HauhauCS-Aggressive-f16.gguf", "", "", "", ""}}, + {"F16", "", 0, "", "", "", "", ""}}, {"Q8_0", "Q4_K_M", "Q3_K_M", "BF16"}}, {"HauhauCS/Qwen3.5-2B-Uncensored-HauhauCS-Aggressive", @@ -1426,7 +1465,7 @@ static const real_repo real_repos[] = { {"Q4_K_M", "Qwen3.5-2B-Uncensored-HauhauCS-Aggressive-Q4_K_M.gguf", 1, "mmproj-Qwen3.5-2B-Uncensored-HauhauCS-Aggressive-f16.gguf", "", "", "", ""}, {"Q8_0", "Qwen3.5-2B-Uncensored-HauhauCS-Aggressive-Q8_0.gguf", 1, "mmproj-Qwen3.5-2B-Uncensored-HauhauCS-Aggressive-f16.gguf", "", "", "", ""}, {"BF16", "Qwen3.5-2B-Uncensored-HauhauCS-Aggressive-BF16.gguf", 1, "mmproj-Qwen3.5-2B-Uncensored-HauhauCS-Aggressive-f16.gguf", "", "", "", ""}, - {"F16", "Qwen3.5-2B-Uncensored-HauhauCS-Aggressive-BF16.gguf", 1, "mmproj-Qwen3.5-2B-Uncensored-HauhauCS-Aggressive-f16.gguf", "", "", "", ""}}, + {"F16", "", 0, "", "", "", "", ""}}, {"Q8_0", "Q6_K", "Q4_K_M", "BF16"}}, {"HauhauCS/Qwen3.5-4B-Uncensored-HauhauCS-Aggressive", @@ -1622,7 +1661,7 @@ static const real_repo real_repos[] = { {"IQ2_M", "Qwen3.8-27B-Uncensored-IQ2_M.gguf", 1, "mmproj-Qwen3.8-27B-Uncensored-F16.gguf", "", "", "", ""}, {"Q4_0", "Qwen3.8-27B-Uncensored-draft-Q4_0.gguf", 1, "mmproj-Qwen3.8-27B-Uncensored-F16.gguf", "", "", "", ""}, {"F16", "", 0, "", "", "", "", ""}, - {"mtp", "Qwen3.8-27B-Uncensored-noMTP-IQ2_M.gguf", 1, "mmproj-Qwen3.8-27B-Uncensored-F16.gguf", "", "", "", ""}}, + {"mtp", "", 0, "", "", "", "", ""}}, {"Q8_0", "Q5_K_M", "Q4_K_M", "IQ2_M", "Q4_0"}}, {"kai-os/Carnice-V2-27b-GGUF", @@ -1903,7 +1942,7 @@ static const real_repo real_repos[] = { {"Q4_K_M", "Huihui-Qwen3-VL-4B-Instruct-abliterated-Q4_K_M.gguf", 1, "mmproj-BF16.gguf", "", "", "", ""}, {"Q8_0", "Huihui-Qwen3-VL-4B-Instruct-abliterated-Q8_0.gguf", 1, "mmproj-BF16.gguf", "", "", "", ""}, {"BF16", "Huihui-Qwen3-VL-4B-Instruct-abliterated-BF16.gguf", 1, "mmproj-BF16.gguf", "", "", "", ""}, - {"F16", "Huihui-Qwen3-VL-4B-Instruct-abliterated-BF16.gguf", 1, "mmproj-BF16.gguf", "", "", "", ""}}, + {"F16", "", 0, "", "", "", "", ""}}, {"Q8_0", "Q4_K_M", "IQ4_XS", "BF16"}}, {"OBLITERATUS/Gemma-4-12B-OBLITERATED", @@ -2291,7 +2330,7 @@ static const real_repo real_repos[] = { {"Q8_0", "embeddinggemma-2-Q8_0.gguf", 1, "mmproj-Q8_0.gguf", "", "", "", ""}, {"BF16", "embeddinggemma-2-BF16.gguf", 1, "mmproj-BF16.gguf", "", "", "", ""}, {"UD-Q6_K_XL", "embeddinggemma-2-UD-Q6_K_XL.gguf", 1, "mmproj-Q8_0.gguf", "", "", "", ""}, - {"F16", "embeddinggemma-2-BF16.gguf", 1, "mmproj-BF16.gguf", "", "", "", ""}}, + {"F16", "", 0, "", "", "", "", ""}}, {"Q6_K_XL", "Q4_K_XL", "Q8_0", "BF16"}}, {"unsloth/GLM-4.7-Flash-GGUF", @@ -2320,7 +2359,7 @@ static const real_repo real_repos[] = { {"Q8_0", "Q8_0/GLM-5.3-Flash-Q8_0-00001-of-00008.gguf", 8, "mmproj-BF16.gguf", "", "", "", ""}, {"BF16", "BF16/GLM-5.3-Flash-BF16-00001-of-00014.gguf", 14, "mmproj-BF16.gguf", "", "", "", ""}, {"UD-Q6_K_XL", "UD-Q6_K_XL/GLM-5.3-Flash-UD-Q6_K_XL-00001-of-00007.gguf", 7, "mmproj-BF16.gguf", "", "", "", ""}, - {"F16", "BF16/GLM-5.3-Flash-BF16-00001-of-00014.gguf", 14, "mmproj-BF16.gguf", "", "", "", ""}}, + {"F16", "", 0, "", "", "", "", ""}}, {"Q6_K_XL", "IQ4_XS", "Q8_0", "BF16"}}, {"unsloth/GLM-5.3-GGUF", @@ -2359,7 +2398,7 @@ static const real_repo real_repos[] = { {"", "BF16/Kimi-K2.6-BF16-00001-of-00046.gguf", 46, "mmproj-BF16.gguf", "", "", "", ""}, {"BF16", "BF16/Kimi-K2.6-BF16-00001-of-00046.gguf", 46, "mmproj-BF16.gguf", "", "", "", ""}, {"UD-Q8_K_XL", "UD-Q8_K_XL/Kimi-K2.6-UD-Q8_K_XL-00001-of-00014.gguf", 14, "mmproj-BF16.gguf", "", "", "", ""}, - {"F16", "BF16/Kimi-K2.6-BF16-00001-of-00046.gguf", 46, "mmproj-BF16.gguf", "", "", "", ""}}, + {"F16", "", 0, "", "", "", "", ""}}, {"Q8_K_XL", "Q4_K_XL", "BF16"}}, {"unsloth/Kimi-K2.7-Code-GGUF", @@ -2405,7 +2444,7 @@ static const real_repo real_repos[] = { {"Q8_0", "LFM2.5-VL-3B-Q8_0.gguf", 1, "mmproj-BF16.gguf", "", "", "", ""}, {"BF16", "LFM2.5-VL-3B-BF16.gguf", 1, "mmproj-BF16.gguf", "", "", "", ""}, {"UD-Q8_K_XL", "LFM2.5-VL-3B-UD-Q8_K_XL.gguf", 1, "mmproj-BF16.gguf", "", "", "", ""}, - {"F16", "LFM2.5-VL-3B-BF16.gguf", 1, "mmproj-BF16.gguf", "", "", "", ""}}, + {"F16", "", 0, "", "", "", "", ""}}, {"Q8_K_XL", "Q8_0", "Q5_K_S", "Q4_K_M", "BF16"}}, {"unsloth/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-GGUF", @@ -2449,7 +2488,7 @@ static const real_repo real_repos[] = { {"Q8_0", "Qwen3-VL-30B-A3B-Instruct-Q8_0.gguf", 1, "mmproj-BF16.gguf", "", "", "", ""}, {"BF16", "BF16/Qwen3-VL-30B-A3B-Instruct-BF16-00001-of-00002.gguf", 2, "mmproj-BF16.gguf", "", "", "", ""}, {"UD-TQ1_0", "Qwen3-VL-30B-A3B-Instruct-UD-TQ1_0.gguf", 1, "mmproj-BF16.gguf", "", "", "", ""}, - {"F16", "BF16/Qwen3-VL-30B-A3B-Instruct-BF16-00001-of-00002.gguf", 2, "mmproj-BF16.gguf", "", "", "", ""}}, + {"F16", "", 0, "", "", "", "", ""}}, {"TQ1_0", "Q8_0", "Q6_K", "Q4_K_M", "BF16"}}, {"unsloth/Qwen3.5-27B-GGUF", @@ -2467,7 +2506,7 @@ static const real_repo real_repos[] = { {"Q8_0", "Qwen3.5-27B-Q8_0.gguf", 1, "mmproj-BF16.gguf", "", "", "", ""}, {"BF16", "BF16/Qwen3.5-27B-BF16-00001-of-00002.gguf", 2, "mmproj-BF16.gguf", "", "", "", ""}, {"UD-Q8_K_XL", "Qwen3.5-27B-UD-Q8_K_XL.gguf", 1, "mmproj-BF16.gguf", "", "", "", ""}, - {"F16", "BF16/Qwen3.5-27B-BF16-00001-of-00002.gguf", 2, "mmproj-BF16.gguf", "", "", "", ""}}, + {"F16", "", 0, "", "", "", "", ""}}, {"Q8_K_XL", "Q8_0", "Q6_K", "Q4_K_M", "BF16"}}, {"unsloth/Qwen3.5-35B-A3B-GGUF", @@ -2485,7 +2524,7 @@ static const real_repo real_repos[] = { {"Q8_0", "Qwen3.5-35B-A3B-Q8_0.gguf", 1, "mmproj-BF16.gguf", "", "", "", ""}, {"BF16", "BF16/Qwen3.5-35B-A3B-BF16-00001-of-00002.gguf", 2, "mmproj-BF16.gguf", "", "", "", ""}, {"UD-Q8_K_XL", "Qwen3.5-35B-A3B-UD-Q8_K_XL.gguf", 1, "mmproj-BF16.gguf", "", "", "", ""}, - {"F16", "BF16/Qwen3.5-35B-A3B-BF16-00001-of-00002.gguf", 2, "mmproj-BF16.gguf", "", "", "", ""}}, + {"F16", "", 0, "", "", "", "", ""}}, {"Q8_K_XL", "IQ3_S", "Q8_0", "Q4_K_M", "BF16"}}, {"unsloth/Qwen3.5-397B-A17B-GGUF", @@ -2503,7 +2542,7 @@ static const real_repo real_repos[] = { {"Q8_0", "Q8_0/Qwen3.5-397B-A17B-Q8_0-00001-of-00010.gguf", 10, "mmproj-BF16.gguf", "", "", "", ""}, {"BF16", "BF16/Qwen3.5-397B-A17B-BF16-00001-of-00017.gguf", 17, "mmproj-BF16.gguf", "", "", "", ""}, {"UD-Q8_K_XL", "UD-Q8_K_XL/Qwen3.5-397B-A17B-UD-Q8_K_XL-00001-of-00010.gguf", 10, "mmproj-BF16.gguf", "", "", "", ""}, - {"F16", "BF16/Qwen3.5-397B-A17B-BF16-00001-of-00017.gguf", 17, "mmproj-BF16.gguf", "", "", "", ""}}, + {"F16", "", 0, "", "", "", "", ""}}, {"Q8_K_XL", "IQ3_XXS", "Q8_0", "Q4_K_M", "BF16"}}, {"unsloth/Qwen3.6-35B-A3B-MTP-GGUF", @@ -2519,7 +2558,7 @@ static const real_repo real_repos[] = { {"Q8_0", "Qwen3.6-35B-A3B-Q8_0.gguf", 1, "mmproj-BF16.gguf", "", "", "", ""}, {"BF16", "BF16/Qwen3.6-35B-A3B-BF16-00001-of-00002.gguf", 2, "mmproj-BF16.gguf", "", "", "", ""}, {"UD-Q8_K_XL", "Qwen3.6-35B-A3B-UD-Q8_K_XL.gguf", 1, "mmproj-BF16.gguf", "", "", "", ""}, - {"F16", "BF16/Qwen3.6-35B-A3B-BF16-00001-of-00002.gguf", 2, "mmproj-BF16.gguf", "", "", "", ""}}, + {"F16", "", 0, "", "", "", "", ""}}, {"Q8_K_XL", "Q3_K_M", "Q8_0", "BF16"}}, {"unsloth/Qwen3.8-Flash-Next-GGUF", @@ -2557,7 +2596,7 @@ static const real_repo real_repos[] = { {"Q8_0", "Q8_0/Step-3.7-Flash-Q8_0-00001-of-00006.gguf", 6, "mmproj-BF16.gguf", "", "", "", ""}, {"BF16", "BF16/Step-3.7-Flash-BF16-00001-of-00008.gguf", 8, "mmproj-BF16.gguf", "", "", "", ""}, {"UD-Q8_K_XL", "UD-Q8_K_XL/Step-3.7-Flash-UD-Q8_K_XL-00001-of-00006.gguf", 6, "mmproj-BF16.gguf", "", "", "", ""}, - {"F16", "BF16/Step-3.7-Flash-BF16-00001-of-00008.gguf", 8, "mmproj-BF16.gguf", "", "", "", ""}}, + {"F16", "", 0, "", "", "", "", ""}}, {"Q8_K_XL", "Q3_K_M", "Q8_0", "BF16"}}, {"vmlinux/Qwen3.8-Flash-Next-Uncensored-Gufo-Q4Mix-GGUF", diff --git a/tests/test-mtmd-impl.cpp b/tests/test-mtmd-impl.cpp index 536dfaa8087f..3fd72008865e 100644 --- a/tests/test-mtmd-impl.cpp +++ b/tests/test-mtmd-impl.cpp @@ -1,11 +1,17 @@ #include "testing.h" +#include "ggml-backend.h" +#include "ggml-cpp.h" #include "mtmd-audio.h" +#include "mtmd-subcontext.h" +#include "mtmd-helper.h" #include "mtmd-image.h" #include "mtmd-internal.h" +#include #include #include +#include #include #include #include @@ -47,12 +53,12 @@ struct test_registry { // MAKE_TEST(test_image_preprocessor_longest_edge) { - clip_hparams hparams; + mtmd_hparams hparams; hparams.image_longest_edge = 1540; hparams.patch_size = 14; hparams.n_merge = 2; - const std::vector> cases = { + const std::vector> cases = { { { 463, 463 }, { 476, 476 } }, { { 463, 300 }, { 476, 308 } }, { { 448, 280 }, { 448, 280 } }, @@ -60,7 +66,7 @@ MAKE_TEST(test_image_preprocessor_longest_edge) { }; for (const auto & [input, expected] : cases) { - const clip_image_size actual = mtmd_image_preprocessor_longest_edge::get_target_size(hparams, input); + const mtmd_image_size actual = mtmd_image_preprocessor_longest_edge::get_target_size(hparams, input); const std::string name = std::to_string(input.width) + "x" + std::to_string(input.height); t.assert_equal(name + " width", expected.width, actual.width); t.assert_equal(name + " height", expected.height, actual.height); @@ -70,7 +76,7 @@ MAKE_TEST(test_image_preprocessor_longest_edge) { MAKE_TEST(test_image_preprocessor_dyn_size) { // { patch_size, image size, expected size } // a side that is an even number plus .5 merged patches rounds down, like Python round() in smart_resize - const std::vector> cases = { + const std::vector> cases = { { 16, { 768, 400 }, { 768, 384 } }, { 16, { 800, 336 }, { 800, 320 } }, { 16, { 400, 400 }, { 384, 384 } }, @@ -79,20 +85,47 @@ MAKE_TEST(test_image_preprocessor_dyn_size) { }; for (const auto & [patch_size, input, expected] : cases) { - clip_hparams hparams; + mtmd_hparams hparams; hparams.patch_size = patch_size; hparams.n_merge = 2; hparams.image_round_half_even = true; hparams.set_limit_image_tokens(8, 4096); - const clip_image_size actual = mtmd_image_preprocessor_dyn_size::get_target_size(hparams, input); + const mtmd_image_size actual = mtmd_image_preprocessor_dyn_size::get_target_size(hparams, input); const std::string name = std::to_string(input.width) + "x" + std::to_string(input.height); t.assert_equal(name + " width", expected.width, actual.width); t.assert_equal(name + " height", expected.height, actual.height); } } -static std::string image_size_to_str(const clip_image_size & size) { +MAKE_TEST(test_image_preprocessor_qwen3vl_video) { + mtmd_hparams hparams; + hparams.patch_size = 16; + hparams.n_merge = 2; + const mtmd_image_preprocessor_qwen3vl preproc(hparams); + + // { n_frames, frame size, expected size from Qwen3VLVideoProcessor smart_resize } + const std::vector> cases = { + { 20, { 720, 358 }, { 704, 352 } }, // 720 / 32 = 22.5 rounds half to even + { 768, { 1920, 1080 }, { 224, 128 } }, // over the longest_edge budget + { 4, { 16, 40 }, { 32, 64 } }, // edge below the factor is upscaled first + { 2, { 32, 32 }, { 64, 64 } }, // under the shortest_edge budget + { 5, { 2760, 1552 }, { 2752, 1536 } }, // 5 / 2 = 2.5 rounds half to even + { 4, { 250, 6000 }, { 256, 6016 } }, + }; + + for (const auto & [n_frames, input, expected] : cases) { + mtmd_image_size actual; + const std::string name = std::to_string(n_frames) + " frames of " + std::to_string(input.width) + "x" + std::to_string(input.height); + if (!t.assert_true(name + " has a frame size", preproc.get_video_frame_size(n_frames, input.width, input.height, actual))) { + continue; + } + t.assert_equal(name + " width", expected.width, actual.width); + t.assert_equal(name + " height", expected.height, actual.height); + } +} + +static std::string image_size_to_str(const mtmd_image_size & size) { return std::to_string(size.width) + "x" + std::to_string(size.height); } @@ -111,8 +144,8 @@ static std::string resize_algo_to_str(resize_algo algo) { } // pseudo-random pixels: s = s * 1103515245 + 12345 from s = 1, one byte (s >> 16) & 255 per channel -static clip_image_u8 make_test_image(const clip_image_size & size) { - clip_image_u8 img; +static mtmd_image_u8 make_test_image(const mtmd_image_size & size) { + mtmd_image_u8 img; img.set_size(size, false); uint32_t s = 1; auto next = [&s]() { @@ -128,7 +161,7 @@ static clip_image_u8 make_test_image(const clip_image_size & size) { } // FNV-1a of the pixels -static uint32_t hash_pixels(const clip_image_u8 & img, uint32_t hash = 2166136261u) { +static uint32_t hash_pixels(const mtmd_image_u8 & img, uint32_t hash = 2166136261u) { for (uint8_t v : img.get_ro_buf()) { hash = (hash ^ v) * 16777619u; } @@ -136,7 +169,7 @@ static uint32_t hash_pixels(const clip_image_u8 & img, uint32_t hash = 216613626 } MAKE_TEST(test_image_preprocessor_lfm2) { - clip_hparams hparams; + mtmd_hparams hparams; hparams.patch_size = 16; hparams.n_merge = 2; hparams.set_limit_image_tokens(64, 256); @@ -146,7 +179,7 @@ MAKE_TEST(test_image_preprocessor_lfm2) { hparams.preproc_max_pixels_tolerance = 2.0f; // { image size, expected tiling } - const std::vector> cases = { + const std::vector> cases = { { { 704, 704 }, false }, // 720 / (patch_size * n_merge) is exactly 22.5, so this only matches HF // if round_by_factor rounds half to even (22) instead of away from zero (23) @@ -166,7 +199,7 @@ MAKE_TEST(test_image_preprocessor_lfm2) { } // { image size, expected size from HF smart_resize } - const std::vector> resize_cases = { + const std::vector> resize_cases = { { { 400, 600 }, { 384, 608 } }, // 400 / 32 = 12.5 rounds half to even { { 480, 336 }, { 480, 320 } }, { { 64, 169 }, { 160, 416 } }, // 169 * beta / 32 is exactly 13 in double @@ -179,7 +212,7 @@ MAKE_TEST(test_image_preprocessor_lfm2) { } // { image size, expected grid from HF } - const std::vector> grid_cases = { + const std::vector> grid_cases = { { { 720, 1600 }, { 2, 4 } }, // 0.45 is an exact tie between 2/4 and 2/5 in double, not in float { { 420, 1440 }, { 1, 3 } }, { { 255, 2160 }, { 1, 8 } }, @@ -199,7 +232,7 @@ MAKE_TEST(test_image_preprocessor_lfm2) { } MAKE_TEST(test_image_preprocessor_lfm2_torch_resize) { - clip_hparams hparams; + mtmd_hparams hparams; hparams.patch_size = 16; hparams.n_merge = 2; hparams.set_limit_image_tokens(64, 256); @@ -211,7 +244,7 @@ MAKE_TEST(test_image_preprocessor_lfm2_torch_resize) { hparams.image_pad_rf = PAD_NONE; // { resize algo, image size, expected hash of the tiles and the thumbnail from Lfm2VlImageProcessor (torchvision 0.29, native uint8 resize: arm64, or x86 with AVX2/AVX512) } - const std::vector> cases = { + const std::vector> cases = { { RESIZE_ALGO_BILINEAR, { 620, 410 }, 0xf5e9c941 }, // 608x416 { RESIZE_ALGO_BILINEAR, { 2000, 1500 }, 0x3331f56c }, // 3x2 tiles, 576x416 thumbnail { RESIZE_ALGO_BICUBIC, { 620, 410 }, 0x2b34eac9 }, @@ -301,12 +334,136 @@ MAKE_TEST(test_temporal_merge_grouping) { } } +// +// mtmd_helper video +// + +MAKE_TEST(test_video_config) { + mtmd_hparams hparams; + hparams.patch_size = 16; + hparams.n_merge = 2; + const mtmd_image_preprocessor_qwen3vl preproc(hparams); + + // { template, n_frames and fps of the source, expected fps, n_frames and frame size } + // the 1920x1080 sizes are from Qwen3VLVideoProcessor smart_resize + const std::vector> cases = { + { MTMD_VIDEO_TEMPLATE_QWEN3VL, 36000, 30.0f, 2.0f, 768, { 224, 128 } }, + { MTMD_VIDEO_TEMPLATE_LFM3, 36000, 30.0f, 2.0f, 768, { 224, 128 } }, + { MTMD_VIDEO_TEMPLATE_LFM3_OMNI, 36000, 30.0f, 2.0f, 384, { 320, 192 } }, + { MTMD_VIDEO_TEMPLATE_QWEN3VL, -1, 30.0f, 2.0f, 0, { 0, 0 } }, // frame count unknown + { MTMD_VIDEO_TEMPLATE_DEFAULT, 36000, 30.0f, 0.0f, 0, { 0, 0 } }, + }; + + for (const auto & [tmpl, n_frames, fps, expected_fps, expected_n_frames, expected_size] : cases) { + const mtmd_video_config cfg = mtmd_video_config_init(tmpl, &preproc, 1920, 1080, n_frames, fps); + const std::string name = "template " + std::to_string(tmpl) + " with " + std::to_string(n_frames) + " frames"; + t.assert_equal(name + " template", (int) tmpl, (int) cfg.tmpl); + t.assert_equal(name + " fps", expected_fps, cfg.fps); + t.assert_equal(name + " n_frames", expected_n_frames, cfg.n_frames); + t.assert_equal(name + " frame size", image_size_to_str(expected_size), image_size_to_str({ cfg.frame_width, cfg.frame_height })); + } +} + +// n_frames source frames, each one starts with its index +struct test_video_source : mtmd_helper_video_seq { + int32_t n_frames; + int32_t n_read = 0; + + test_video_source(int32_t n_frames) : n_frames(n_frames) {} + + bool read_raw_frame() override { + if (n_read >= n_frames) { + return false; + } + frame_buf[0] = static_cast(n_read & 255); + frame_buf[1] = static_cast(n_read >> 8); + n_read++; + return true; + } +}; + +// text as is, a frame as [source index], an audio bitmap as {n_samples} +static std::string video_sequence_to_str(testing & t, mtmd_helper_video_seq & video) { + std::string out; + mtmd_bitmap * bitmap; + char * text; + int32_t ret; + while ((ret = video.read_next(&bitmap, &text)) == 0) { + if (text) { + out += text; + free(text); + } else if (mtmd_bitmap_is_audio(bitmap)) { + out += "{" + std::to_string(mtmd_bitmap_get_n_bytes(bitmap) / sizeof(float)) + "}"; + mtmd_bitmap_free(bitmap); + } else { + const unsigned char * data = mtmd_bitmap_get_data(bitmap); + out += "[" + std::to_string(data[0] | data[1] << 8) + "]"; + mtmd_bitmap_free(bitmap); + } + } + t.assert_equal("end of stream", -1, ret); + return out; +} + +MAKE_TEST(test_video_sequence_qwen3vl) { + mtmd_hparams hparams; + hparams.patch_size = 16; + hparams.n_merge = 2; + const mtmd_image_preprocessor_qwen3vl preproc(hparams); + + // { source video, expected frame size and sequence from Qwen3VLProcessor (transformers 4.57.1) } + const std::vector> cases = { + { { 720, 358, 24.0f, 240 }, { 704, 352 }, + "<0.3 seconds>[0][13]<1.3 seconds>[25][38]<2.4 seconds>[50][63]<3.4 seconds>[75][88]<4.5 seconds>[101][113]" + "<5.5 seconds>[126][138]<6.6 seconds>[151][164]<7.6 seconds>[176][189]<8.6 seconds>[201][214]<9.7 seconds>[226][239]" }, + // odd frame count: the last group repeats its last frame + { { 640, 480, 10.0f, 35 }, { 640, 480 }, + "<0.3 seconds>[0][6]<1.4 seconds>[11][17]<2.5 seconds>[23][28]<3.4 seconds>[34]" }, + // fewer frames than the minimum of 4 + { { 320, 240, 30.0f, 3 }, { 320, 256 }, + "<0.0 seconds>[0][1]<0.1 seconds>[2]" }, + { { 1280, 720, 30000.0f / 1001.0f, 300 }, { 1280, 704 }, + "<0.3 seconds>[0][16]<1.3 seconds>[31][47]<2.4 seconds>[63][79]<3.4 seconds>[94][110]<4.5 seconds>[126][142]" + "<5.5 seconds>[157][173]<6.6 seconds>[189][205]<7.6 seconds>[220][236]<8.7 seconds>[252][268]<9.7 seconds>[283][299]" }, + }; + + for (const auto & [src, expected_size, expected] : cases) { + const std::string name = std::to_string(src.n_frames) + " frames of " + std::to_string(src.width) + "x" + std::to_string(src.height); + const mtmd_video_config cfg = mtmd_video_config_init(MTMD_VIDEO_TEMPLATE_QWEN3VL, &preproc, src.width, src.height, src.n_frames, src.fps); + test_video_source video(src.n_frames); + video.configure(cfg, src, 0.0f); + video.src_alive = true; + t.assert_equal(name + " frame size", image_size_to_str(expected_size), image_size_to_str({ (int) video.info.width, (int) video.info.height })); + t.assert_equal(name + " sequence", expected, video_sequence_to_str(t, video)); + } +} + +// the soundtrack of each group comes after its frames +MAKE_TEST(test_video_sequence_lfm3_omni) { + mtmd_hparams hparams; + hparams.patch_size = 16; + hparams.n_merge = 2; + const mtmd_image_preprocessor_lfm3vl preproc(hparams); + + const mtmd_helper_video_src src = { 64, 64, 2.0f, 9 }; + const mtmd_video_config cfg = mtmd_video_config_init(MTMD_VIDEO_TEMPLATE_LFM3_OMNI, &preproc, src.width, src.height, src.n_frames, src.fps); + test_video_source video(src.n_frames); + video.configure(cfg, src, 0.0f); + video.src_alive = true; + video.audio_rate = 100; + video.soundtrack.assign(450, 0.0f); + + t.assert_equal("sequence", + std::string("<0.2 seconds>[0][1]{100}<1.2 seconds>[2][3]{100}<2.2 seconds>[4][5]{100}<3.2 seconds>[6][7]{100}<4.0 seconds>[8]{50}"), + video_sequence_to_str(t, video)); +} + // // mtmd_audio // MAKE_TEST(test_audio_preprocessor_conformer) { - clip_hparams hparams; + mtmd_hparams hparams; hparams.n_mel_bins = 128; hparams.audio_sample_rate = 16000; hparams.audio_n_fft = 512; @@ -326,15 +483,6 @@ MAKE_TEST(test_audio_preprocessor_conformer) { } } - mtmd_audio_preprocessor_conformer preproc(hparams); - preproc.initialize(); - std::vector mels; - if (!t.assert_true("preprocess", preproc.preprocess(samples.data(), samples.size(), mels))) { - return; - } - const mtmd_audio_mel & mel = mels[0]; - t.assert_equal("n_len", (int64_t) 101, mel.n_len); - // reference values from NeMo AudioToMelSpectrogramPreprocessor (liquid-audio 1.3.0) // { mel bin, frame, value } const std::vector> cases = { @@ -343,10 +491,257 @@ MAKE_TEST(test_audio_preprocessor_conformer) { { 24, 60, -0.0476f }, // silence -> quiet tone { 100, 20, -0.0913f }, // tones, high bin }; - for (const auto & [bin, frame, expected] : cases) { - const float actual = mel.data[bin * mel.n_len + frame]; - t.assert_true("mel[" + std::to_string(bin) + "][" + std::to_string(frame) + "] = " + std::to_string(actual) + ", expected " + std::to_string(expected), std::fabs(actual - expected) < 1e-3f); + + ggml_backend_ptr backend(ggml_backend_init_by_type(GGML_BACKEND_DEVICE_TYPE_CPU, nullptr)); + ggml_backend_t backends[] = { backend.get() }; + ggml_backend_sched_ptr sched(ggml_backend_sched_new(backends, nullptr, 1, 256, false, true)); + + // the result must not depend on the number of threads + for (int n_threads : { 1, 4, 7 }) { + mtmd_audio_preprocessor_conformer preproc(hparams, sched.get()); + preproc.mel_config.n_threads = n_threads; + preproc.initialize(); + mtmd_audio_preproc_out out = preproc.preprocess(samples.data(), samples.size()); + if (!t.assert_true("preprocess", !out.entries.empty() && preproc.compute(out))) { + return; + } + const mtmd_image_f32 & mel = out.entries[0]; + t.assert_equal("n_len", 101, mel.nx()); + + for (const auto & [bin, frame, expected] : cases) { + const float actual = mel.get_ro_buf()[bin * mel.nx() + frame]; + t.assert_true("mel[" + std::to_string(bin) + "][" + std::to_string(frame) + "] = " + std::to_string(actual) + ", expected " + std::to_string(expected), std::fabs(actual - expected) < 1e-3f); + } + } +} + +MAKE_TEST(test_audio_preprocessor_threads) { + mtmd_hparams hparams; + hparams.n_mel_bins = 128; + hparams.audio_sample_rate = 16000; + hparams.audio_n_fft = 512; + hparams.audio_window_len = 400; + hparams.audio_hop_len = 160; + + // chirp: every frame has a different content + std::vector samples(hparams.audio_sample_rate * 5); + for (size_t i = 0; i < samples.size(); i++) { + samples[i] = (float) (0.3 * std::sin(0.05 * i + 2e-6 * i * i)); + } + + ggml_backend_ptr backend(ggml_backend_init_by_type(GGML_BACKEND_DEVICE_TYPE_CPU, nullptr)); + ggml_backend_t backends[] = { backend.get() }; + ggml_backend_sched_ptr sched(ggml_backend_sched_new(backends, nullptr, 1, 256, false, true)); + + mtmd_audio_preprocessor_conformer preproc(hparams, sched.get()); + preproc.initialize(); + // tokenize only gets the shapes, encode must get the same ones + mtmd_audio_preproc_out ref = preproc.preprocess(samples.data(), samples.size()); + if (!t.assert_true("preprocess", ref.entries.size() == 1 && ref.needs_compute())) { + return; + } + const mtmd_image_size shape = ref.entries[0].get_size(); + if (!t.assert_true("compute", preproc.compute(ref))) { + return; + } + t.assert_true("shape", ref.entries.size() == 1 && ref.entries[0].get_size() == shape); + t.assert_true("computed", !ref.needs_compute()); + + // frames are independent, so the whole output must be bit-exact with the single-threaded one + for (int n_threads : { 4, 12 }) { + preproc.mel_config.n_threads = n_threads; + mtmd_audio_preproc_out out = preproc.preprocess(samples.data(), samples.size()); + const bool ok = preproc.compute(out); + t.assert_true("n_threads = " + std::to_string(n_threads), ok && out.entries[0].get_ro_buf() == ref.entries[0].get_ro_buf()); + } +} + +MAKE_TEST(test_audio_log_mel_precision) { + const int n_mel = 80; + const int n_fft = 400; + const int hop = 160; + const int n_frames = 200; + + mtmd_audio_cache cache; + cache.fill_sin_cos_table(n_fft); + cache.fill_hann_window(n_fft, true); + cache.fill_mel_filterbank_matrix(n_mel, n_fft, 16000); + + ggml_backend_ptr backend(ggml_backend_init_by_type(GGML_BACKEND_DEVICE_TYPE_CPU, nullptr)); + ggml_backend_t backends[] = { backend.get() }; + ggml_backend_sched_ptr sched(ggml_backend_sched_new(backends, nullptr, 1, 256, false, true)); + + mtmd_audio_mel_config config; + config.sched = sched.get(); + config.n_threads = 2; + config.n_mel = n_mel; + config.n_fft_bins = 1 + n_fft / 2; + config.hop_length = hop; + config.cache = &cache; + + // a loud tone next to a 74 dB weaker one: rounding the DFT inputs to F16 or TF32 loses the weak one + const double pi = 3.14159265358979323846; + std::vector samples((size_t) (n_frames - 1) * hop + n_fft); + for (size_t i = 0; i < samples.size(); i++) { + samples[i] = (float) (0.5 * std::sin(2 * pi * 200 * i / 16000) + 1e-4 * std::sin(2 * pi * 6000 * i / 16000)); + } + + std::vector out((size_t) n_frames * n_mel); + if (!t.assert_true("log mel", mtmd_audio_log_mel(config, samples.data(), cache.hann_window.data(), n_frames, out.data()))) { + return; + } + + const int n_bins = n_fft / 2 + 1; + double max_diff = 0.0; + std::vector power(n_bins); + for (int i = 0; i < n_frames; i++) { + for (int k = 0; k < n_bins; k++) { + double re = 0.0, im = 0.0; + for (int j = 0; j < n_fft; j++) { + const double x = (double) cache.hann_window[j] * samples[(size_t) i * hop + j]; + re += x * std::cos(2 * pi * j * k / n_fft); + im -= x * std::sin(2 * pi * j * k / n_fft); + } + power[k] = re * re + im * im; + } + for (int m = 0; m < n_mel; m++) { + double sum = 0.0; + for (int k = 0; k < n_bins; k++) { + sum += power[k] * cache.filters.data[(size_t) m * n_bins + k]; + } + const double expected = std::log10(std::max(sum, (double) config.mel_floor)); + max_diff = std::max(max_diff, std::fabs(expected - out[(size_t) m * n_frames + i])); + } } + t.assert_true("max_diff = " + std::to_string(max_diff), max_diff < 0.05); +} + +MAKE_TEST(test_audio_log_mel_blocks) { + const int n_mel = 80; + const int n_fft = 400; + const int hop = 160; + const int n_frames = 8192 + 100; // long audio is split into blocks of 8192 frames + + mtmd_audio_cache cache; + cache.fill_sin_cos_table(n_fft); + cache.fill_hann_window(n_fft, true); + cache.fill_mel_filterbank_matrix(n_mel, n_fft, 16000); + + ggml_backend_ptr backend(ggml_backend_init_by_type(GGML_BACKEND_DEVICE_TYPE_CPU, nullptr)); + ggml_backend_t backends[] = { backend.get() }; + ggml_backend_sched_ptr sched(ggml_backend_sched_new(backends, nullptr, 1, 256, false, true)); + + mtmd_audio_mel_config config; + config.sched = sched.get(); + config.n_threads = 2; + config.n_mel = n_mel; + config.n_fft_bins = 1 + n_fft / 2; + config.hop_length = hop; + config.cache = &cache; + + std::vector samples((size_t) (n_frames - 1) * hop + n_fft); + for (size_t i = 0; i < samples.size(); i++) { + samples[i] = (float) (0.3 * std::sin(0.05 * i + 1e-6 * i * i)); + } + + // frames are independent: the frames around the seam must not change when computed on their own + // not bit-exact, matmul kernels may round differently depending on the shape + const int off = 8192 - 50; + const int n_sub = n_frames - off; + std::vector all((size_t) n_frames * n_mel); + std::vector sub((size_t) n_sub * n_mel); + const float * window = cache.hann_window.data(); + if (!t.assert_true("all", mtmd_audio_log_mel(config, samples.data(), window, n_frames, all.data())) || + !t.assert_true("sub", mtmd_audio_log_mel(config, samples.data() + (size_t) off * hop, window, n_sub, sub.data()))) { + return; + } + + float max_diff = 0.0f; + for (int j = 0; j < n_mel; j++) { + for (int i = 0; i < n_sub; i++) { + max_diff = std::max(max_diff, std::fabs(all[(size_t) j * n_frames + off + i] - sub[(size_t) j * n_sub + i])); + } + } + t.assert_true("max_diff = " + std::to_string(max_diff), max_diff < 1e-5f); +} + +// +// mtmd_helper_resampler +// + +static std::vector make_tone(double freq, int rate, size_t n) { + const double pi = 3.14159265358979323846; + std::vector x(n); + for (size_t i = 0; i < n; i++) { + x[i] = (float) (0.5 * std::sin(2 * pi * freq * i / rate)); + } + return x; +} + +// amplitude of the freq component, skipping the edges +static double tone_amplitude(const std::vector & x, double freq, int rate) { + const double pi = 3.14159265358979323846; + const size_t skip = rate / 10; + double re = 0.0, im = 0.0; + for (size_t i = skip; i < x.size() - skip; i++) { + re += x[i] * std::cos(2 * pi * freq * i / rate); + im += x[i] * std::sin(2 * pi * freq * i / rate); + } + return 2.0 * std::sqrt(re * re + im * im) / (x.size() - 2 * skip); +} + +static std::vector resample_chunked(mtmd_helper_resampler * rs, const std::vector & in, size_t chunk) { + std::vector res; + for (size_t off = 0; off < in.size(); off += chunk) { + const float * out = nullptr; + const int64_t n = mtmd_helper_resampler_process(rs, in.data() + off, std::min(chunk, in.size() - off), &out); + if (n < 0) { + return {}; + } + res.insert(res.end(), out, out + n); + } + return res; +} + +MAKE_TEST(test_helper_resampler) { + const int rate_out = 16000; + for (int rate_in : { 8000, 24000, 44100, 48000 }) { + const std::string name = std::to_string(rate_in) + " -> " + std::to_string(rate_out); + const std::vector in = make_tone(440, rate_in, (size_t) rate_in * 2); + + mtmd_helper::resampler_ptr rs_all(mtmd_helper_resampler_init(rate_in, rate_out)); + mtmd_helper::resampler_ptr rs_chk(mtmd_helper_resampler_init(rate_in, rate_out)); + if (!t.assert_true(name + " init", rs_all && rs_chk)) { + continue; + } + const std::vector all = resample_chunked(rs_all.get(), in, in.size()); + const std::vector chk = resample_chunked(rs_chk.get(), in, 1234); // odd size, not a multiple of the ratio + + t.assert_equal(name + " n_samples", (size_t) rate_out * 2, all.size()); + t.assert_true(name + " chunked == one-shot", all == chk); + + const double amp = tone_amplitude(all, 440, rate_out); + t.assert_true(name + " amplitude = " + std::to_string(amp), std::fabs(amp - 0.5) < 0.02); + } + + // above the output Nyquist: 10 kHz at 48 kHz aliases to 6 kHz at 16 kHz, the low-pass filter must cut it + { + mtmd_helper::resampler_ptr rs(mtmd_helper_resampler_init(48000, rate_out)); + const std::vector out = resample_chunked(rs.get(), make_tone(10000, 48000, 48000 * 2), 4800); + const double db = 20 * std::log10(tone_amplitude(out, 6000, rate_out) / 0.5); + t.assert_true("aliasing = " + std::to_string(db) + " dB", db < -15); // about -20 dB with lpfOrder = 8, -10 dB with the default + } + + // same rate: no copy + { + mtmd_helper::resampler_ptr rs(mtmd_helper_resampler_init(rate_out, rate_out)); + const std::vector in = make_tone(440, rate_out, 1000); + const float * out = nullptr; + const int64_t n = rs ? mtmd_helper_resampler_process(rs.get(), in.data(), in.size(), &out) : -1; + t.assert_true("passthrough", n == (int64_t) in.size() && out == in.data()); + } + + t.assert_true("invalid rate", mtmd_helper_resampler_init(0, rate_out) == nullptr); } // @@ -357,6 +752,16 @@ int main(int argc, char ** argv) { testing t(std::cout); t.verbose = true; + // the audio preprocessors run on the CPU backend + ggml_backend_load_all(); + + // the video helper logs every frame at debug level + mtmd_helper_log_set([](ggml_log_level level, const char * text, void *) { + if (level != GGML_LOG_LEVEL_DEBUG) { + fputs(text, stderr); + } + }, nullptr); + // usage: test-mtmd-impl [filter_regex] for (int i = 1; i < argc; i++) { t.set_filter(argv[i]); diff --git a/tests/test-quantize-be.sh b/tests/test-quantize-be.sh index ba2d923d2286..2840b79d47f0 100755 --- a/tests/test-quantize-be.sh +++ b/tests/test-quantize-be.sh @@ -34,7 +34,7 @@ if [ "$(uname -m)" != s390x ]; then exit 1 fi -MODEL_URL=https://huggingface.co/ggml-org/models/resolve/499bc8821c6b12b4e53c5bffcb21ec206f212d81/tinyllamas/stories15M.gguf +MODEL_URL=https://huggingface.co/ggml-org-dev/stories15M/resolve/main/stories15M.gguf MODEL_SHA256=61b50d457809a5194818fd22e6724b456cd7bb9a6264c52c8110684c53f3704a F32_SHA256=55174d7a7d46f29fc6be163ccb040330c357aa727b22ec13e2c5d713e5ba5b76 diff --git a/tests/test-quantize-fns.cpp b/tests/test-quantize-fns.cpp index ab88a237f1d6..1542ab656878 100644 --- a/tests/test-quantize-fns.cpp +++ b/tests/test-quantize-fns.cpp @@ -37,7 +37,7 @@ static const char* RESULT_STR[] = {"ok", "FAILED"}; // Generate synthetic data static void generate_data(float offset, size_t n, float * dst, float amplitude = 2.0f) { for (size_t i = 0; i < n; i++) { - dst[i] = 0.1 + amplitude*cosf(i + offset); + dst[i] = 0.1 + amplitude*std::cos(i + offset); } } @@ -48,7 +48,7 @@ static float array_rmse(const float * a1, const float * a2, size_t n) { double diff = a1[i] - a2[i]; sum += diff * diff; } - return sqrtf(sum) / n; + return std::sqrt(sum) / n; } // Total quantization error on test data @@ -106,7 +106,7 @@ static float dot_product_error(const ggml_type_traits_cpu * qfns_cpu, ggml_type qfns_cpu->vec_dot(test_size, &result, 0, tmp_q1.data(), 0, tmp_q2.data(), 0, 1); const float dot_ref = dot_product(test_data1, test_data2, test_size); - return fabsf(result - dot_ref) / test_size; + return std::fabs(result - dot_ref) / test_size; } // nrc == 2: kernel computes a 2x2 dot product matrix @@ -125,7 +125,7 @@ static float dot_product_error(const ggml_type_traits_cpu * qfns_cpu, ggml_type const float ref11 = dot_product(test_data3, test_data4, test_size); const auto err = [test_size](float val, float ref) { - const float e = fabsf(val - ref) / test_size; + const float e = std::fabs(val - ref) / test_size; return std::isfinite(e) ? e : INFINITY; }; @@ -144,7 +144,7 @@ static int test_vec_dot_f32(bool verbose) { float result = 0.0f; f32->vec_dot(n, &result, 0, a.data(), 0, b.data(), 0, 1); const float ref = dot_product(a.data(), b.data(), n); - const float error = fabsf(result - ref) / n; + const float error = std::fabs(result - ref) / n; const bool failed = !(error < MAX_QUANTIZATION_REFERENCE_ERROR); num_failed += failed; @@ -177,18 +177,18 @@ static const float F8_E4M3_VALUES[] = { }; static uint8_t fp32_to_f8_e4m3_ref(float value) { - if (isnan(value)) { + if (std::isnan(value)) { return 0x7F; } - const uint8_t sign = signbit(value) ? 0x80 : 0; - const double magnitude = fabs((double) value); + const uint8_t sign = std::signbit(value) ? 0x80 : 0; + const double magnitude = std::fabs((double) value); if (magnitude >= F8_E4M3_VALUES[126]) { return sign | 126; } int nearest = 0; double distance = magnitude; for (int i = 1; i < 127; ++i) { - const double candidate = fabs(magnitude - F8_E4M3_VALUES[i]); + const double candidate = std::fabs(magnitude - F8_E4M3_VALUES[i]); if (candidate < distance || (candidate == distance && (i & 1) == 0)) { nearest = i; distance = candidate; @@ -221,9 +221,9 @@ static int test_f8_e4m3(bool verbose) { for (int i = 0; i < 256; ++i) { const bool is_nan = (i & 0x7F) == 0x7F; - const float expected = is_nan ? NAN : copysignf(F8_E4M3_VALUES[i & 0x7F], i & 0x80 ? -1.0f : 1.0f); - const bool decode_failed = is_nan ? !isnan(decoded[i]) - : decoded[i] != expected || (expected == 0.0f && signbit(decoded[i]) != signbit(expected)); + const float expected = is_nan ? NAN : std::copysign(F8_E4M3_VALUES[i & 0x7F], i & 0x80 ? -1.0f : 1.0f); + const bool decode_failed = is_nan ? !std::isnan(decoded[i]) + : decoded[i] != expected || (expected == 0.0f && std::signbit(decoded[i]) != std::signbit(expected)); const bool encode_failed = !is_nan && roundtrip[i] != encoded[i]; if (decode_failed || encode_failed) { num_failed++; @@ -241,14 +241,14 @@ static int test_f8_e4m3(bool verbose) { } for (int i = 0; i < 126; ++i) { const float midpoint = (F8_E4M3_VALUES[i] + F8_E4M3_VALUES[i + 1]) / 2.0f; - for (float value : {nextafterf(midpoint, -INFINITY), midpoint, nextafterf(midpoint, INFINITY)}) { + for (float value : {std::nextafter(midpoint, -INFINITY), midpoint, std::nextafter(midpoint, INFINITY)}) { inputs.push_back(value); inputs.push_back(-value); } } for (float value : {std::numeric_limits::denorm_min(), std::numeric_limits::max(), - nextafterf(448.0f, INFINITY), nextafterf(464.0f, -INFINITY), 464.0f, - nextafterf(464.0f, INFINITY), INFINITY, NAN}) { + std::nextafter(448.0f, INFINITY), std::nextafter(464.0f, -INFINITY), 464.0f, + std::nextafter(464.0f, INFINITY), INFINITY, NAN}) { inputs.push_back(value); inputs.push_back(-value); } @@ -276,11 +276,11 @@ static int test_f8_e4m3(bool verbose) { const uint8_t code = (uint8_t) (i % 254 + (i % 254 >= 127)); x[i + 1] = code; y[i + 1] = (i % 7 - 3) * 0.125f; - expected += copysignf(F8_E4M3_VALUES[code & 0x7F], code & 0x80 ? -1.0f : 1.0f) * y[i + 1]; + expected += std::copysign(F8_E4M3_VALUES[code & 0x7F], code & 0x80 ? -1.0f : 1.0f) * y[i + 1]; } float actual; traits_cpu->vec_dot(n, &actual, 0, x.data() + 1, 0, y.data() + 1, 0, 1); - const bool failed = fabs(actual - expected) > 1e-5 * std::max(1.0, fabs(expected)); + const bool failed = std::fabs(actual - expected) > 1e-5 * std::max(1.0, std::fabs(expected)); num_failed += failed; if (verbose || failed) { printf("f8_e4m3 dot n=%d: %s (ref=%f got=%f)\n", n, RESULT_STR[failed], expected, actual); @@ -293,7 +293,7 @@ static int test_f8_e4m3(bool verbose) { x[0] = x.back() = code; float actual; traits_cpu->vec_dot(x.size(), &actual, 0, x.data(), 0, y.data(), 0, 1); - num_failed += !isnan(actual); + num_failed += !std::isnan(actual); } if (verbose || num_failed) { diff --git a/tests/test-rope.cpp b/tests/test-rope.cpp index 801e4cd82709..9b0ef69a4897 100644 --- a/tests/test-rope.cpp +++ b/tests/test-rope.cpp @@ -124,6 +124,83 @@ static void ggml_graph_compute_helper(std::vector & buf, ggml_cgraph * ggml_graph_compute(graph, &plan); } +static void test_yarn(ggml_type type, int mode, bool truncate, bool forward, bool inplace) { + const int n_dims = 128; + const int n_offs = 8; + const int ne0 = n_dims + 2*n_offs; + const int32_t positions[] = {0, 1, 4096, 8192, -67}; + const int n_pos = sizeof(positions)/sizeof(positions[0]); + const double base = 1000000.0; + const double factor = 256.0; + const double pi = std::acos(-1.0); + double low = n_dims/2 * std::log(4096.0/(32.0*2*pi))/std::log(base); + double high = n_dims/2 * std::log(4096.0/( 1.0*2*pi))/std::log(base); + if (truncate) { + low = std::floor(low); + high = std::ceil(high); + } + const double mscale = 1.0 + 0.1*std::log(factor); + + ggml_init_params params = {1024*1024, nullptr, false}; + ggml_context * ctx = ggml_init(params); + ggml_tensor * x = ggml_new_tensor_3d(ctx, type, ne0, 2, n_pos); + ggml_tensor * pos = ggml_new_tensor_1d(ctx, GGML_TYPE_I32, n_pos); + std::vector input(ggml_nelements(x)); + for (size_t i = 0; i < input.size(); ++i) { + input[i] = (int(i % 31) - 15)/16.0f; + if (type == GGML_TYPE_F32) { + ((float *) x->data)[i] = input[i]; + } else { + ((ggml_fp16_t *) x->data)[i] = ggml_fp32_to_fp16(input[i]); + } + } + for (int i = 0; i < n_pos; ++i) { + ((int32_t *) pos->data)[i] = positions[i]; + } + ggml_tensor * out; + if (!forward) { + out = ggml_rope_ext_back(ctx, x, pos, nullptr, n_dims, mode, 4096, base, 1.0/factor, 1.0f, 1.0f, 32.0f, 1.0f); + } else if (inplace) { + out = ggml_rope_ext_inplace(ctx, x, pos, nullptr, n_dims, mode, 4096, base, 1.0/factor, 1.0f, 1.0f, 32.0f, 1.0f); + } else { + out = ggml_rope_ext(ctx, x, pos, nullptr, n_dims, mode, 4096, base, 1.0/factor, 1.0f, 1.0f, 32.0f, 1.0f); + } + ggml_rope_set_truncate(out, truncate); + ggml_rope_set_offset(out, n_offs); + ggml_cgraph * graph = ggml_new_graph(ctx); + ggml_build_forward_expand(graph, out); + std::vector work; + ggml_graph_compute_helper(work, graph, 2); + + std::vector expected = input; + for (int p = 0; p < n_pos; ++p) { + for (int h = 0; h < 2; ++h) { + const int row = (p*2 + h)*ne0 + n_offs; + for (int i = 0; i < n_dims/2; ++i) { + const double ramp = 1.0 - MIN(1.0, MAX(0.0, (i - low)/(high - low))); + const double freq = std::pow(base, -2.0*i/n_dims); + const double theta = positions[p]*freq*((1.0 - ramp)/factor + ramp); + const double c = std::cos(theta)*mscale; + const double s = std::sin(theta)*mscale*(forward ? 1.0 : -1.0); + const int i0 = row + (mode == GGML_ROPE_TYPE_NORMAL ? 2*i : i); + const int i1 = i0 + (mode == GGML_ROPE_TYPE_NORMAL ? 1 : n_dims/2); + expected[i0] = input[i0]*c - input[i1]*s; + expected[i1] = input[i0]*s + input[i1]*c; + } + } + } + float max_error = 0.0f; + for (size_t i = 0; i < expected.size(); ++i) { + const float actual = type == GGML_TYPE_F32 ? ((float *) out->data)[i] : ggml_fp16_to_fp32(((ggml_fp16_t *) out->data)[i]); + GGML_ASSERT(std::isfinite(actual)); + max_error = MAX(max_error, std::fabs(actual - expected[i])); + } + printf("yarn type=%s mode=%d truncate=%d forward=%d inplace=%d max_error=%g\n", + ggml_type_name(type), mode, truncate, forward, inplace, max_error); + GGML_ASSERT(max_error < (type == GGML_TYPE_F32 ? 0.003f : 0.005f)); + ggml_free(ctx); +} + int main(int /*argc*/, const char ** /*argv*/) { struct ggml_init_params params = { /* .mem_size = */ 128*1024*1024, @@ -259,5 +336,15 @@ int main(int /*argc*/, const char ** /*argv*/) { ggml_free(ctx0); + for (ggml_type type : {GGML_TYPE_F32, GGML_TYPE_F16}) { + for (int mode : {GGML_ROPE_TYPE_NORMAL, GGML_ROPE_TYPE_NEOX}) { + for (bool truncate : {false, true}) { + test_yarn(type, mode, truncate, true, false); + test_yarn(type, mode, truncate, true, true); + test_yarn(type, mode, truncate, false, false); + } + } + } + return 0; } diff --git a/tests/test-rpc-unreachable.cpp b/tests/test-rpc-unreachable.cpp new file mode 100644 index 000000000000..50e1f2d44c49 --- /dev/null +++ b/tests/test-rpc-unreachable.cpp @@ -0,0 +1,33 @@ +#include "ggml-backend.h" +#include "ggml-rpc.h" +#include "ggml.h" + +#include + +// Regression test: an unreachable or malformed RPC endpoint must be reported as an +// error instead of aborting the calling process. +// +// Before the fix, ggml_backend_rpc_add_server() called GGML_ABORT (SIGABRT) when the +// endpoint could not be parsed, the transport failed to initialize, the socket could +// not connect, or the HELLO handshake failed - so a typo in an endpoint or a node +// that was still booting killed the whole application, and this test would die +// instead of failing cleanly. +// +// After the fix it returns nullptr. +// +// ref: https://github.com/ggml-org/llama.cpp/pull/30415 + +int main(void) { + ggml_backend_load_all(); + + // 1) nothing listens on this port: the connect must fail, not abort + ggml_backend_reg_t reg = ggml_backend_rpc_add_server("127.0.0.1:1"); + GGML_ASSERT(reg == nullptr); + + // 2) a malformed endpoint must take the parse-failure path and also return nullptr + reg = ggml_backend_rpc_add_server("this-is-not-an-endpoint"); + GGML_ASSERT(reg == nullptr); + + printf("ok\n"); + return 0; +} diff --git a/tests/test-server-component.cpp b/tests/test-server-component.cpp index 2c76e8904db5..c4ed1cacce6e 100644 --- a/tests/test-server-component.cpp +++ b/tests/test-server-component.cpp @@ -1,5 +1,6 @@ #include "testing.h" +#include "../tools/server/server-chat.h" #include "../tools/server/server-common.h" #include "../tools/server/server-decision.h" #include "../tools/server/server-schema.h" @@ -7,6 +8,7 @@ #include #include #include +#include #include #include #include @@ -665,6 +667,79 @@ static void test_glob_match(testing & t) { }); } +// +// OpenAI speech API +// + +static void test_speech_convert(testing & t) { + const json base = {{"model", "tts"}, {"input", "Hello"}, {"voice", "alba"}}; + + auto with = [&](const char * key, const json & value) { + json body = base; + body[key] = value; + return body; + }; + auto rejected = [&](const char * key, const json & value) { + try { + convert_speech_to_tts(with(key, value)); + } catch (const std::invalid_argument &) { + return true; + } + return false; + }; + + // defaults + json out = convert_speech_to_tts(base); + t.assert_equal("input kept", std::string("Hello"), out.at("input").get()); + t.assert_equal("voice kept", std::string("alba"), out.at("voice").get()); + t.assert_equal("default format", std::string("wav"), out.at("response_format").get()); + t.assert_true ("no stream", !out.contains("stream")); + + // response_format + out = convert_speech_to_tts(with("response_format", "wav")); + t.assert_equal("wav", std::string("wav"), out.at("response_format").get()); + out = convert_speech_to_tts(with("response_format", "pcm")); + t.assert_equal("pcm is 16-bit", std::string("pcm16"), out.at("response_format").get()); + for (const char * format : {"mp3", "opus", "aac", "flac", "pcm16"}) { + t.assert_true(std::string("rejects ") + format, rejected("response_format", format)); + } + + // custom voice + out = convert_speech_to_tts(with("voice", json{{"id", "voice_1234"}})); + t.assert_equal("voice object", std::string("voice_1234"), out.at("voice").get()); + + // stream_format + out = convert_speech_to_tts(with("stream_format", "audio")); + t.assert_true ("audio streams", out.at("stream").get()); + t.assert_true ("rejects sse", rejected("stream_format", "sse")); + + // speed + t.assert_true ("speed 1.0", !rejected("speed", 1.0)); + t.assert_true ("rejects speed 1.5", rejected("speed", 1.5)); + + // instructions are ignored + t.assert_true ("instructions", !rejected("instructions", "whisper")); +} + +static void test_pcm_f32_to_s16(testing & t) { + auto sample = [](const std::string & bytes, size_t i) { + int16_t v; + memcpy(&v, bytes.data() + i * sizeof(v), sizeof(v)); + return (int) v; + }; + + const std::vector samples = {0.0f, 1.0f, -1.0f, 0.5f, 2.0f, -2.0f}; + const std::string bytes = pcm_f32_to_s16(samples.data(), samples.size()); + t.assert_equal("size", (size_t) 12, bytes.size()); + t.assert_equal("zero", 0, sample(bytes, 0)); + t.assert_equal("max", 32767, sample(bytes, 1)); + t.assert_equal("min", -32767, sample(bytes, 2)); + t.assert_equal("rounds", 16384, sample(bytes, 3)); + t.assert_equal("clamps above", 32767, sample(bytes, 4)); + t.assert_equal("clamps below", -32767, sample(bytes, 5)); + t.assert_true ("empty", pcm_f32_to_s16(nullptr, 0).empty()); +} + int main(int argc, char ** argv) { testing t; @@ -690,6 +765,8 @@ int main(int argc, char ** argv) { t.test("decision_depth", test_decision_depth); t.test("glob_match", test_glob_match); t.test("dry_sequence_breakers", test_dry_sequence_breakers); + t.test("speech_convert", test_speech_convert); + t.test("pcm_f32_to_s16", test_pcm_f32_to_s16); return t.summary(); } diff --git a/tests/test-tokenizer-0.py b/tests/test-tokenizer-0.py index 4f3f1c8a6770..334b2df968b7 100644 --- a/tests/test-tokenizer-0.py +++ b/tests/test-tokenizer-0.py @@ -12,6 +12,7 @@ fname_tok = args.fname_tok tokenizer = AutoTokenizer.from_pretrained(dir_tokenizer) +assert tokenizer is not None print('tokenizing file: ', fname_tok) # noqa: NP100 fname_out = fname_tok + '.tok' @@ -19,7 +20,7 @@ lines = f.readlines() s = ''.join(lines) t_start = time.time() - res = tokenizer.encode(s, add_special_tokens=False) # ty: ignore[unresolved-attribute] + res = tokenizer.encode(s, add_special_tokens=False) t_end = time.time() print('\nmain : tokenized in', "{:.3f}".format(1000.0 * (t_end - t_start)), 'ms (py)') # noqa: NP100 with open(fname_out, 'w', encoding='utf-8') as f: diff --git a/tests/test-tokenizer-random.py b/tests/test-tokenizer-random.py index 8fc476b63c3f..7f5ea3f45684 100644 --- a/tests/test-tokenizer-random.py +++ b/tests/test-tokenizer-random.py @@ -80,7 +80,7 @@ def __init__(self, libllama: LibLlama, path_model: str, mparams={}, cparams={}): raise RuntimeError("error: failed to load model '%s'" % path_model) if isinstance(cparams, dict): cparams = libllama.context_default_params(**cparams) - self.ctx = self.lib.llama_new_context_with_model(self.model, cparams) + self.ctx = self.lib.llama_init_from_model(self.model, cparams) if not self.ctx: raise RuntimeError("error: failed to create context for model '%s'" % path_model) n_tokens_max = self.lib.llama_n_ctx(self.ctx) diff --git a/tests/test_kv_cache_scales.py b/tests/test_kv_cache_scales.py new file mode 100644 index 000000000000..7dedf4b31735 --- /dev/null +++ b/tests/test_kv_cache_scales.py @@ -0,0 +1,57 @@ +"""Test KV cache scales with synthetic tensors; no GPU or model downloads required. + +Install dependencies and run from the repository root: + python -m pip install -r requirements/requirements-convert_hf_to_gguf.txt + python -m pip install -e ./gguf-py + python -m unittest discover -s tests -p 'test_kv_cache_scales.py' -v + +Use unittest discovery so the repository root is available for imports. +""" + +import tempfile +import unittest +from pathlib import Path + +import torch + +from conversion.base import ModelBase +import gguf + + +class TestKVCacheScales(unittest.TestCase): + def test_projection_names(self): + cases = ( + (gguf.MODEL_ARCH.PHI3, "model.layers.2.self_attn.qkv_proj", "model.layers.2.self_attn.qkv_proj"), + (gguf.MODEL_ARCH.GPT2, "transformer.h.2.attn.c_attn", "transformer.h.2.attn.c_attn"), + (gguf.MODEL_ARCH.LLAMA, "model.layers.2.self_attn.k_proj", "model.layers.2.self_attn.v_proj"), + ) + for arch, k_name, v_name in cases: + for dtype in (torch.float32, torch.bfloat16): + with self.subTest(arch=arch, dtype=dtype), tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "scales.gguf" + model = object.__new__(ModelBase) + model.tensor_map = gguf.get_tensor_name_map(arch, 3) + model.gguf_writer = gguf.GGUFWriter(path, gguf.MODEL_ARCH_NAMES[arch]) + model.model_tensors = { + k_name + ".k_scale": lambda: torch.tensor(0.5, dtype=dtype), + v_name + ".v_scale": lambda: torch.tensor(1.0, dtype=dtype), + } + model._prepare_kv_cache_scales() + self.assertFalse(model.model_tensors) + model.gguf_writer.write_header_to_file() + model.gguf_writer.write_kv_data_to_file() + model.gguf_writer.write_tensors_to_file() + model.gguf_writer.close() + + # Release the reader's memory map before Windows removes the temporary directory. + self._check_scales(path) + + def _check_scales(self, path): + reader = gguf.GGUFReader(path) + self.assertEqual([t.name for t in reader.tensors], ["blk.2.attn_k.k_scale", "blk.2.attn_v.v_scale"]) + self.assertEqual([t.tensor_type for t in reader.tensors], [gguf.GGMLQuantizationType.F32] * 2) + self.assertEqual([float(t.data[0]) for t in reader.tensors], [0.5, 1.0]) + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/cli/README.md b/tools/cli/README.md index f1f6e6ed422e..475becd2edfc 100644 --- a/tools/cli/README.md +++ b/tools/cli/README.md @@ -50,9 +50,8 @@ | `-kvo, --kv-offload, -nkvo, --no-kv-offload` | whether to enable KV cache offloading (default: enabled)
(env: LLAMA_ARG_KV_OFFLOAD) | | `--repack, -nr, --no-repack` | whether to enable weight repacking (default: enabled)
(env: LLAMA_ARG_REPACK) | | `--no-host` | bypass host buffer allowing extra buffers to be used
(env: LLAMA_ARG_NO_HOST) | -| `-ctk, --cache-type-k TYPE` | KV cache data type for K
allowed values: f32, f16, bf16, q8_0, q4_0, q4_1, iq4_nl, q5_0, q5_1
(default: f16)
(env: LLAMA_ARG_CACHE_TYPE_K) | -| `-ctv, --cache-type-v TYPE` | KV cache data type for V
allowed values: f32, f16, bf16, q8_0, q4_0, q4_1, iq4_nl, q5_0, q5_1
(default: f16)
(env: LLAMA_ARG_CACHE_TYPE_V) | -| `-dt, --defrag-thold N` | KV cache defragmentation threshold (DEPRECATED)
(env: LLAMA_ARG_DEFRAG_THOLD) | +| `-ctk, --cache-type-k TYPE` | KV cache data type for K
allowed values: f32, f16, bf16, f8_e4m3, q8_0, q4_0, q4_1, iq4_nl, q5_0, q5_1
(default: f16)
(env: LLAMA_ARG_CACHE_TYPE_K) | +| `-ctv, --cache-type-v TYPE` | KV cache data type for V
allowed values: f32, f16, bf16, f8_e4m3, q8_0, q4_0, q4_1, iq4_nl, q5_0, q5_1
(default: f16)
(env: LLAMA_ARG_CACHE_TYPE_V) | | `-np, --parallel N` | number of parallel sequences to decode (default: 1)
(env: LLAMA_ARG_N_PARALLEL) | | `--rpc SERVERS` | comma-separated list of RPC servers (host:port)
(env: LLAMA_ARG_RPC) | | `-lm, --load-mode MODE` | model loading mode (default: auto)
- auto: mmap, unless a device does not support it
- none: no special loading mode
- mmap: memory-map model (if mmap disabled, slower load but may reduce pageouts if not using mlock)
- mlock: force system to keep model in RAM rather than swapping or compressing
- mmap+mlock: mmap + force system to keep model in RAM rather than swapping or compressing
- dio: use DirectIO if available

(env: LLAMA_ARG_LOAD_MODE) | @@ -64,6 +63,7 @@ | `-cmoe, --cpu-moe` | keep all Mixture of Experts (MoE) weights in the CPU
(env: LLAMA_ARG_CPU_MOE) | | `-ncmoe, --n-cpu-moe N` | keep the Mixture of Experts (MoE) weights of the first N layers in the CPU
(env: LLAMA_ARG_N_CPU_MOE) | | `--moe-cache-mib N` | GPU cache size in MiB for the MoE experts kept in the CPU. with multiple GPUs, it is split among them like the layers (--tensor-split) (default: 0, disabled)
(env: LLAMA_ARG_MOE_CACHE_MIB) | +| `--moe-cache-cpu-share F` | share of the MoE cache misses that the CPU computes from the host experts instead of uploading them to the GPU once the cache is full, from 0 to 1 (default: 0.00)
(env: LLAMA_ARG_MOE_CACHE_CPU_SHARE) | | `-ncffn, --n-cpu-ffn N` | keep the dense FFN weights of the first N layers in the CPU
(dense models; for MoE expert weights use --n-cpu-moe)
(env: LLAMA_ARG_N_CPU_FFN) | | `-ngl, --gpu-layers, --n-gpu-layers N` | max. number of layers to store in VRAM, either an exact number, 'auto', or 'all' (default: auto)
(env: LLAMA_ARG_N_GPU_LAYERS) | | `-sm, --split-mode {none,layer,row,tensor}` | how to split the model across multiple GPUs, one of:
- none: use one GPU only
- layer (default): split layers and KV across GPUs (pipelined)
- row: split weight across GPUs by rows (parallelized)
- tensor: split weights and KV across GPUs (parallelized, EXPERIMENTAL)
(env: LLAMA_ARG_SPLIT_MODE) | @@ -81,6 +81,7 @@ | `--control-vector-scaled FNAME:SCALE,...` | add a control vector with user defined scaling SCALE
note: use comma-separated values (format: FNAME:SCALE,...) | | `--control-vector-layer-range START END` | layer range to apply the control vector(s) to, start and end inclusive | | `-m, --model FNAME` | model path to load
(env: LLAMA_ARG_MODEL) | +| `--ple, --shard FNAME,...` | extra GGUF shards for the model, comma separated; may be given more than once
paths may live anywhere, including on different drives; --ple is an alias for --shard
(env: LLAMA_ARG_SHARD) | | `-mu, --model-url MODEL_URL` | model download url (default: unused)
(env: LLAMA_ARG_MODEL_URL) | | `-dr, --docker-repo [/][:quant]` | Docker Hub model repository. repo is optional, default to ai/. quant is optional, default to :latest.
example: gemma3
(default: unused)
(env: LLAMA_ARG_DOCKER_REPO) | | `-hf, -hfr, --hf-repo /[:quant]` | Hugging Face model repository; quant is optional, case-insensitive, default to Q4_K_M, or falls back to the first file in the repo if Q4_K_M doesn't exist.
mmproj is also downloaded automatically if available. to disable, add --no-mmproj
example: ggml-org/GLM-4.7-Flash-GGUF:Q4_K_M
(default: unused)
(env: LLAMA_ARG_HF_REPO) | @@ -95,8 +96,8 @@ | `-lv, --verbosity, --log-verbosity N` | Set the verbosity threshold. Messages with a higher verbosity will be ignored. Values:
- 0: generic output
- 1: error
- 2: warning
- 3: info
- 4: trace (more info)
- 5: debug
(default: 3)

(env: LLAMA_ARG_LOG_VERBOSITY) | | `--log-prefix, --no-log-prefix` | Enable prefix in log messages
(env: LLAMA_ARG_LOG_PREFIX) | | `--log-timestamps, --no-log-timestamps` | Enable timestamps in log messages
(env: LLAMA_ARG_LOG_TIMESTAMPS) | -| `--spec-draft-type-k, -ctkd, --cache-type-k-draft TYPE` | KV cache data type for K for the draft model
allowed values: f32, f16, bf16, q8_0, q4_0, q4_1, iq4_nl, q5_0, q5_1
(default: f16)
(env: LLAMA_ARG_SPEC_DRAFT_CACHE_TYPE_K) | -| `--spec-draft-type-v, -ctvd, --cache-type-v-draft TYPE` | KV cache data type for V for the draft model
allowed values: f32, f16, bf16, q8_0, q4_0, q4_1, iq4_nl, q5_0, q5_1
(default: f16)
(env: LLAMA_ARG_SPEC_DRAFT_CACHE_TYPE_V) | +| `--spec-draft-type-k, -ctkd, --cache-type-k-draft TYPE` | KV cache data type for K for the draft model
allowed values: f32, f16, bf16, f8_e4m3, q8_0, q4_0, q4_1, iq4_nl, q5_0, q5_1
(default: f16)
(env: LLAMA_ARG_SPEC_DRAFT_CACHE_TYPE_K) | +| `--spec-draft-type-v, -ctvd, --cache-type-v-draft TYPE` | KV cache data type for V for the draft model
allowed values: f32, f16, bf16, f8_e4m3, q8_0, q4_0, q4_1, iq4_nl, q5_0, q5_1
(default: f16)
(env: LLAMA_ARG_SPEC_DRAFT_CACHE_TYPE_V) | ### Sampling params @@ -177,7 +178,7 @@ | `--reasoning-effort LEVEL` | reasoning effort level given to the chat template: 'default' to keep the template default,
or a level such as 'minimal', 'low', 'medium', 'high', 'xhigh' or 'max' (default: default)
(env: LLAMA_ARG_REASONING_EFFORT) | | `--reasoning-budget N` | token budget for thinking: -1 for unrestricted, 0 for immediate end, N>0 for token budget (default: -1)
(env: LLAMA_ARG_THINK_BUDGET) | | `--reasoning-budget-message MESSAGE` | message injected before the end-of-thinking tag when reasoning budget is exhausted (default: none)
(env: LLAMA_ARG_THINK_BUDGET_MESSAGE) | -| `--reasoning-preserve, --no-reasoning-preserve` | preserve reasoning trace in the full history, not just the last assistant message (default: template default)
compatible with certain templates having 'supports_preserve_reasoning' capability
example: https://docs.z.ai/guides/capabilities/thinking-mode#preserved-thinking
(env: LLAMA_ARG_REASONING_PRESERVE) | +| `--reasoning-preserve, --no-reasoning-preserve` | preserve reasoning trace in the full history, not just the last assistant message (default: enabled)
compatible with certain templates having 'supports_preserve_reasoning' capability
example: https://docs.z.ai/guides/capabilities/thinking-mode#preserved-thinking
(env: LLAMA_ARG_REASONING_PRESERVE) | | `--chat-template JINJA_TEMPLATE` | set custom jinja chat template (default: template taken from model's metadata)
if suffix/prefix are specified, template will be disabled
only commonly used templates are accepted (unless --jinja is set before this flag):
list of built-in templates:
bailing, bailing-think, bailing2, chatglm3, chatglm4, chatml, command-r, deepseek, deepseek-ocr, deepseek2, deepseek3, exaone-moe, exaone3, exaone4, falcon3, gemma, gigachat, glmedge, gpt-oss, granite, granite-4.0, granite-4.1, grok-2, hunyuan-dense, hunyuan-moe, hunyuan-vl, kimi-k2, llama2, llama2-sys, llama2-sys-bos, llama2-sys-strip, llama3, llama4, megrez, minicpm, mistral-v1, mistral-v3, mistral-v3-tekken, mistral-v7, mistral-v7-tekken, monarch, openchat, orion, pangu-embedded, phi3, phi4, rwkv-world, seed_oss, smolvlm, solar-open, vicuna, vicuna-orca, yandex, zephyr
(env: LLAMA_ARG_CHAT_TEMPLATE) | | `--chat-template-file JINJA_TEMPLATE_FILE` | set custom jinja chat template file (default: template taken from model's metadata)
if suffix/prefix are specified, template will be disabled
only commonly used templates are accepted (unless --jinja is set before this flag):
list of built-in templates:
bailing, bailing-think, bailing2, chatglm3, chatglm4, chatml, command-r, deepseek, deepseek-ocr, deepseek2, deepseek3, exaone-moe, exaone3, exaone4, falcon3, gemma, gigachat, glmedge, gpt-oss, granite, granite-4.0, granite-4.1, grok-2, hunyuan-dense, hunyuan-moe, hunyuan-vl, kimi-k2, llama2, llama2-sys, llama2-sys-bos, llama2-sys-strip, llama3, llama4, megrez, minicpm, mistral-v1, mistral-v3, mistral-v3-tekken, mistral-v7, mistral-v7-tekken, monarch, openchat, orion, pangu-embedded, phi3, phi4, rwkv-world, seed_oss, smolvlm, solar-open, vicuna, vicuna-orca, yandex, zephyr
(env: LLAMA_ARG_CHAT_TEMPLATE_FILE) | | `--skip-chat-parsing, --no-skip-chat-parsing` | force a pure content parser, even if a Jinja template is specified; model will output everything in the content section, including any reasoning and/or tool calls (default: disabled)
(env: LLAMA_ARG_SKIP_CHAT_PARSING) | diff --git a/tools/cli/cli-context.cpp b/tools/cli/cli-context.cpp index d8787c13d21e..2049008bc9dd 100644 --- a/tools/cli/cli-context.cpp +++ b/tools/cli/cli-context.cpp @@ -362,7 +362,7 @@ bool cli_context::generate_completion(generated_content & content_out, cli_timin ui::assistant_turn a; - std::string err = client.post_sse("/v1/chat/completions", body.dump(), should_stop, [&](const std::string & payload) { + std::string err = client.post_sse("/v1/chat/completions", body.dump_safe(), should_stop, [&](const std::string & payload) { json chunk = json::parse_no_throw(payload); if (chunk.is_discarded()) { return; diff --git a/tools/completion/README.md b/tools/completion/README.md index 7174a04ff984..cc6aae3d04d9 100644 --- a/tools/completion/README.md +++ b/tools/completion/README.md @@ -133,9 +133,8 @@ llama-completion.exe -m models\gemma-1.1-7b-it.Q4_K_M.gguf --ignore-eos -n -1 | `-kvo, --kv-offload, -nkvo, --no-kv-offload` | whether to enable KV cache offloading (default: enabled)
(env: LLAMA_ARG_KV_OFFLOAD) | | `--repack, -nr, --no-repack` | whether to enable weight repacking (default: enabled)
(env: LLAMA_ARG_REPACK) | | `--no-host` | bypass host buffer allowing extra buffers to be used
(env: LLAMA_ARG_NO_HOST) | -| `-ctk, --cache-type-k TYPE` | KV cache data type for K
allowed values: f32, f16, bf16, q8_0, q4_0, q4_1, iq4_nl, q5_0, q5_1
(default: f16)
(env: LLAMA_ARG_CACHE_TYPE_K) | -| `-ctv, --cache-type-v TYPE` | KV cache data type for V
allowed values: f32, f16, bf16, q8_0, q4_0, q4_1, iq4_nl, q5_0, q5_1
(default: f16)
(env: LLAMA_ARG_CACHE_TYPE_V) | -| `-dt, --defrag-thold N` | KV cache defragmentation threshold (DEPRECATED)
(env: LLAMA_ARG_DEFRAG_THOLD) | +| `-ctk, --cache-type-k TYPE` | KV cache data type for K
allowed values: f32, f16, bf16, f8_e4m3, q8_0, q4_0, q4_1, iq4_nl, q5_0, q5_1
(default: f16)
(env: LLAMA_ARG_CACHE_TYPE_K) | +| `-ctv, --cache-type-v TYPE` | KV cache data type for V
allowed values: f32, f16, bf16, f8_e4m3, q8_0, q4_0, q4_1, iq4_nl, q5_0, q5_1
(default: f16)
(env: LLAMA_ARG_CACHE_TYPE_V) | | `-np, --parallel N` | number of parallel sequences to decode (default: 1)
(env: LLAMA_ARG_N_PARALLEL) | | `--rpc SERVERS` | comma-separated list of RPC servers (host:port)
(env: LLAMA_ARG_RPC) | | `-lm, --load-mode MODE` | model loading mode (default: auto)
- auto: mmap, unless a device does not support it
- none: no special loading mode
- mmap: memory-map model (if mmap disabled, slower load but may reduce pageouts if not using mlock)
- mlock: force system to keep model in RAM rather than swapping or compressing
- mmap+mlock: mmap + force system to keep model in RAM rather than swapping or compressing
- dio: use DirectIO if available

(env: LLAMA_ARG_LOAD_MODE) | @@ -147,6 +146,7 @@ llama-completion.exe -m models\gemma-1.1-7b-it.Q4_K_M.gguf --ignore-eos -n -1 | `-cmoe, --cpu-moe` | keep all Mixture of Experts (MoE) weights in the CPU
(env: LLAMA_ARG_CPU_MOE) | | `-ncmoe, --n-cpu-moe N` | keep the Mixture of Experts (MoE) weights of the first N layers in the CPU
(env: LLAMA_ARG_N_CPU_MOE) | | `--moe-cache-mib N` | GPU cache size in MiB for the MoE experts kept in the CPU. with multiple GPUs, it is split among them like the layers (--tensor-split) (default: 0, disabled)
(env: LLAMA_ARG_MOE_CACHE_MIB) | +| `--moe-cache-cpu-share F` | share of the MoE cache misses that the CPU computes from the host experts instead of uploading them to the GPU once the cache is full, from 0 to 1 (default: 0.00)
(env: LLAMA_ARG_MOE_CACHE_CPU_SHARE) | | `-ncffn, --n-cpu-ffn N` | keep the dense FFN weights of the first N layers in the CPU
(dense models; for MoE expert weights use --n-cpu-moe)
(env: LLAMA_ARG_N_CPU_FFN) | | `-ngl, --gpu-layers, --n-gpu-layers N` | max. number of layers to store in VRAM, either an exact number, 'auto', or 'all' (default: auto)
(env: LLAMA_ARG_N_GPU_LAYERS) | | `-sm, --split-mode {none,layer,row,tensor}` | how to split the model across multiple GPUs, one of:
- none: use one GPU only
- layer (default): split layers and KV across GPUs (pipelined)
- row: split weight across GPUs by rows (parallelized)
- tensor: split weights and KV across GPUs (parallelized, EXPERIMENTAL)
(env: LLAMA_ARG_SPLIT_MODE) | @@ -164,6 +164,7 @@ llama-completion.exe -m models\gemma-1.1-7b-it.Q4_K_M.gguf --ignore-eos -n -1 | `--control-vector-scaled FNAME:SCALE,...` | add a control vector with user defined scaling SCALE
note: use comma-separated values (format: FNAME:SCALE,...) | | `--control-vector-layer-range START END` | layer range to apply the control vector(s) to, start and end inclusive | | `-m, --model FNAME` | model path to load
(env: LLAMA_ARG_MODEL) | +| `--ple, --shard FNAME,...` | extra GGUF shards for the model, comma separated; may be given more than once
paths may live anywhere, including on different drives; --ple is an alias for --shard
(env: LLAMA_ARG_SHARD) | | `-mu, --model-url MODEL_URL` | model download url (default: unused)
(env: LLAMA_ARG_MODEL_URL) | | `-dr, --docker-repo [/][:quant]` | Docker Hub model repository. repo is optional, default to ai/. quant is optional, default to :latest.
example: gemma3
(default: unused)
(env: LLAMA_ARG_DOCKER_REPO) | | `-hf, -hfr, --hf-repo /[:quant]` | Hugging Face model repository; quant is optional, case-insensitive, default to Q4_K_M, or falls back to the first file in the repo if Q4_K_M doesn't exist.
mmproj is also downloaded automatically if available. to disable, add --no-mmproj
example: ggml-org/GLM-4.7-Flash-GGUF:Q4_K_M
(default: unused)
(env: LLAMA_ARG_HF_REPO) | @@ -178,8 +179,8 @@ llama-completion.exe -m models\gemma-1.1-7b-it.Q4_K_M.gguf --ignore-eos -n -1 | `-lv, --verbosity, --log-verbosity N` | Set the verbosity threshold. Messages with a higher verbosity will be ignored. Values:
- 0: generic output
- 1: error
- 2: warning
- 3: info
- 4: trace (more info)
- 5: debug
(default: 3)

(env: LLAMA_ARG_LOG_VERBOSITY) | | `--log-prefix, --no-log-prefix` | Enable prefix in log messages
(env: LLAMA_ARG_LOG_PREFIX) | | `--log-timestamps, --no-log-timestamps` | Enable timestamps in log messages
(env: LLAMA_ARG_LOG_TIMESTAMPS) | -| `--spec-draft-type-k, -ctkd, --cache-type-k-draft TYPE` | KV cache data type for K for the draft model
allowed values: f32, f16, bf16, q8_0, q4_0, q4_1, iq4_nl, q5_0, q5_1
(default: f16)
(env: LLAMA_ARG_SPEC_DRAFT_CACHE_TYPE_K) | -| `--spec-draft-type-v, -ctvd, --cache-type-v-draft TYPE` | KV cache data type for V for the draft model
allowed values: f32, f16, bf16, q8_0, q4_0, q4_1, iq4_nl, q5_0, q5_1
(default: f16)
(env: LLAMA_ARG_SPEC_DRAFT_CACHE_TYPE_V) | +| `--spec-draft-type-k, -ctkd, --cache-type-k-draft TYPE` | KV cache data type for K for the draft model
allowed values: f32, f16, bf16, f8_e4m3, q8_0, q4_0, q4_1, iq4_nl, q5_0, q5_1
(default: f16)
(env: LLAMA_ARG_SPEC_DRAFT_CACHE_TYPE_K) | +| `--spec-draft-type-v, -ctvd, --cache-type-v-draft TYPE` | KV cache data type for V for the draft model
allowed values: f32, f16, bf16, f8_e4m3, q8_0, q4_0, q4_1, iq4_nl, q5_0, q5_1
(default: f16)
(env: LLAMA_ARG_SPEC_DRAFT_CACHE_TYPE_V) | ### Sampling params @@ -255,7 +256,7 @@ llama-completion.exe -m models\gemma-1.1-7b-it.Q4_K_M.gguf --ignore-eos -n -1 | `--reasoning-effort LEVEL` | reasoning effort level given to the chat template: 'default' to keep the template default,
or a level such as 'minimal', 'low', 'medium', 'high', 'xhigh' or 'max' (default: default)
(env: LLAMA_ARG_REASONING_EFFORT) | | `--reasoning-budget N` | token budget for thinking: -1 for unrestricted, 0 for immediate end, N>0 for token budget (default: -1)
(env: LLAMA_ARG_THINK_BUDGET) | | `--reasoning-budget-message MESSAGE` | message injected before the end-of-thinking tag when reasoning budget is exhausted (default: none)
(env: LLAMA_ARG_THINK_BUDGET_MESSAGE) | -| `--reasoning-preserve, --no-reasoning-preserve` | preserve reasoning trace in the full history, not just the last assistant message (default: template default)
compatible with certain templates having 'supports_preserve_reasoning' capability
example: https://docs.z.ai/guides/capabilities/thinking-mode#preserved-thinking
(env: LLAMA_ARG_REASONING_PRESERVE) | +| `--reasoning-preserve, --no-reasoning-preserve` | preserve reasoning trace in the full history, not just the last assistant message (default: enabled)
compatible with certain templates having 'supports_preserve_reasoning' capability
example: https://docs.z.ai/guides/capabilities/thinking-mode#preserved-thinking
(env: LLAMA_ARG_REASONING_PRESERVE) | | `--chat-template JINJA_TEMPLATE` | set custom jinja chat template (default: template taken from model's metadata)
if suffix/prefix are specified, template will be disabled
only commonly used templates are accepted (unless --jinja is set before this flag):
list of built-in templates:
bailing, bailing-think, bailing2, chatglm3, chatglm4, chatml, command-r, deepseek, deepseek-ocr, deepseek2, deepseek3, exaone-moe, exaone3, exaone4, falcon3, gemma, gigachat, glmedge, gpt-oss, granite, granite-4.0, granite-4.1, grok-2, hunyuan-dense, hunyuan-moe, hunyuan-vl, kimi-k2, llama2, llama2-sys, llama2-sys-bos, llama2-sys-strip, llama3, llama4, megrez, minicpm, mistral-v1, mistral-v3, mistral-v3-tekken, mistral-v7, mistral-v7-tekken, monarch, openchat, orion, pangu-embedded, phi3, phi4, rwkv-world, seed_oss, smolvlm, solar-open, vicuna, vicuna-orca, yandex, zephyr
(env: LLAMA_ARG_CHAT_TEMPLATE) | | `--chat-template-file JINJA_TEMPLATE_FILE` | set custom jinja chat template file (default: template taken from model's metadata)
if suffix/prefix are specified, template will be disabled
only commonly used templates are accepted (unless --jinja is set before this flag):
list of built-in templates:
bailing, bailing-think, bailing2, chatglm3, chatglm4, chatml, command-r, deepseek, deepseek-ocr, deepseek2, deepseek3, exaone-moe, exaone3, exaone4, falcon3, gemma, gigachat, glmedge, gpt-oss, granite, granite-4.0, granite-4.1, grok-2, hunyuan-dense, hunyuan-moe, hunyuan-vl, kimi-k2, llama2, llama2-sys, llama2-sys-bos, llama2-sys-strip, llama3, llama4, megrez, minicpm, mistral-v1, mistral-v3, mistral-v3-tekken, mistral-v7, mistral-v7-tekken, monarch, openchat, orion, pangu-embedded, phi3, phi4, rwkv-world, seed_oss, smolvlm, solar-open, vicuna, vicuna-orca, yandex, zephyr
(env: LLAMA_ARG_CHAT_TEMPLATE_FILE) | | `--skip-chat-parsing, --no-skip-chat-parsing` | force a pure content parser, even if a Jinja template is specified; model will output everything in the content section, including any reasoning and/or tool calls (default: disabled)
(env: LLAMA_ARG_SKIP_CHAT_PARSING) | diff --git a/tools/export-lora/export-lora.cpp b/tools/export-lora/export-lora.cpp index cd7ac39d257f..5ffd97814903 100644 --- a/tools/export-lora/export-lora.cpp +++ b/tools/export-lora/export-lora.cpp @@ -182,6 +182,8 @@ struct lora_merge_ctx { ggml_type get_out_tensor_type(struct ggml_tensor * t) { if (t->type == GGML_TYPE_F32) { return GGML_TYPE_F32; + } else if (t->type == GGML_TYPE_BF16) { + return GGML_TYPE_BF16; } else { return GGML_TYPE_F16; } @@ -409,8 +411,7 @@ struct lora_merge_ctx { static void print_usage(int, char ** argv) { printf("\nexample usage:\n"); - printf("\n %s -m base-model.gguf --lora lora-file.gguf -o merged-model-f16.gguf\n", argv[0]); - printf("\nNOTE: output model is F16\n"); + printf("\n %s -m base-model.gguf --lora lora-file.gguf -o merged-model.gguf\n", argv[0]); printf("\n"); } diff --git a/tools/fit-params/fit-params.cpp b/tools/fit-params/fit-params.cpp index 3e78c89290a7..99082e51bc1a 100644 --- a/tools/fit-params/fit-params.cpp +++ b/tools/fit-params/fit-params.cpp @@ -1,5 +1,4 @@ #include "llama.h" -#include "../src/llama-ext.h" #include "arg.h" #include "common.h" @@ -31,7 +30,7 @@ int llama_fit_params(int argc, char ** argv) { auto cparams = common_context_params_to_llama(params); if (!params.fit_params_print) { - const common_params_fit_status status = common_fit_params(params.model.path.c_str(), &mparams, &cparams, + const common_params_fit_status status = common_fit_params(params.model.get_paths(), &mparams, &cparams, params.tensor_split, params.tensor_buft_overrides.data(), params.fit_params_target.data(), params.fit_params_min_ctx, nullptr, params.verbosity >= LOG_LEVEL_DEBUG ? GGML_LOG_LEVEL_DEBUG : GGML_LOG_LEVEL_ERROR); @@ -71,7 +70,7 @@ int llama_fit_params(int argc, char ** argv) { LOG_INF("%s: printing estimated memory in MiB to stdout (device, model, context, compute) ...\n", __func__); common_log_flush(common_log_main()); - common_fit_print(params.model.path.c_str(), &mparams, &cparams); + common_fit_print(params.model.get_paths(), &mparams, &cparams); } return 0; diff --git a/tools/gguf-split/gguf-split.cpp b/tools/gguf-split/gguf-split.cpp index 73587b9fcbdf..962a59ba03c1 100644 --- a/tools/gguf-split/gguf-split.cpp +++ b/tools/gguf-split/gguf-split.cpp @@ -454,6 +454,17 @@ static void gguf_merge(const split_params & split_params) { ctx_ggufs.push_back(ctx_gguf); ctx_metas.push_back(ctx_meta); + // Verify that this split belongs to the same set as the first one + if (i_split > 0) { + auto key_split_no = gguf_find_key(ctx_gguf, LLM_KV_SPLIT_NO); + auto key_split_count = gguf_find_key(ctx_gguf, LLM_KV_SPLIT_COUNT); + if (key_split_no < 0 || gguf_get_val_u16(ctx_gguf, key_split_no) != i_split || + key_split_count < 0 || gguf_get_val_u16(ctx_gguf, key_split_count) != n_split) { + fprintf(stderr, "\n%s: %s is not split %d of %d\n", __func__, split_path, i_split + 1, n_split); + exit(EXIT_FAILURE); + } + } + if (i_split == 0) { auto key_n_split = gguf_find_key(ctx_gguf, LLM_KV_SPLIT_COUNT); if (key_n_split < 0) { @@ -508,6 +519,14 @@ static void gguf_merge(const split_params & split_params) { fprintf(stderr, "\033[3Ddone\n"); } + + // Verify that no tensor is missing + auto key_n_tensors = gguf_find_key(ctx_ggufs[0], LLM_KV_SPLIT_TENSORS_COUNT); + if (key_n_tensors < 0 || gguf_get_val_i32(ctx_ggufs[0], key_n_tensors) != total_tensors) { + fprintf(stderr, "%s: splits contain %d tensors, which does not match %s\n", __func__, total_tensors, LLM_KV_SPLIT_TENSORS_COUNT); + exit(EXIT_FAILURE); + } + std::ofstream fout; if (!split_params.dry_run) { fout.open(split_params.output.c_str(), std::ios::binary); diff --git a/tools/imatrix/imatrix.cpp b/tools/imatrix/imatrix.cpp index 2122bf59467f..7abd83eadcb1 100644 --- a/tools/imatrix/imatrix.cpp +++ b/tools/imatrix/imatrix.cpp @@ -1,5 +1,4 @@ #include "llama.h" -#include "../src/llama-ext.h" #include "arg.h" #include "common.h" diff --git a/tools/llama-bench/llama-bench.cpp b/tools/llama-bench/llama-bench.cpp index 96ae4870ff3c..5cbdda14c736 100644 --- a/tools/llama-bench/llama-bench.cpp +++ b/tools/llama-bench/llama-bench.cpp @@ -500,6 +500,9 @@ static ggml_type ggml_type_from_name(const std::string & s) { if (s == "bf16") { return GGML_TYPE_BF16; } + if (s == "f8_e4m3") { + return GGML_TYPE_F8_E4M3; + } if (s == "q8_0") { return GGML_TYPE_Q8_0; } @@ -2352,7 +2355,7 @@ int llama_bench(int argc, char ** argv) { uint32_t n_ctx_needed = std::max(inst.n_prompt + inst.n_gen + inst.n_depth, inst.fit_min_ctx); cparams.n_ctx = std::max(cparams.n_ctx, n_ctx_needed); - common_fit_params(inst.model.c_str(), &mparams, &cparams, + common_fit_params({ inst.model }, &mparams, &cparams, fit_tensor_split.data(), fit_overrides.data(), margins.data(), diff --git a/tools/mtmd/CMakeLists.txt b/tools/mtmd/CMakeLists.txt index f1ec3c456645..c19815d67114 100644 --- a/tools/mtmd/CMakeLists.txt +++ b/tools/mtmd/CMakeLists.txt @@ -12,22 +12,30 @@ endif() find_package(Threads REQUIRED) +set(MTMD_CORE_SOURCES + mtmd.cpp + mtmd-audio.cpp + mtmd-image.cpp + mtmd.h + mtmd-internal.h + mtmd-helper.cpp + mtmd-helper-gen.cpp + mtmd-helper-common.h + mtmd-helper.h + mtmd-model.cpp + mtmd-subcontext.cpp + mtmd-graph.cpp + mtmd-subcontext.h + mtmd-impl.h + mtmd-model.h + mtmd-graph.h +) + add_library(mtmd - mtmd.cpp - mtmd-audio.cpp - mtmd-image.cpp - mtmd.h - mtmd-internal.h - mtmd-helper.cpp - mtmd-helper-gen.cpp - mtmd-helper-common.h - mtmd-helper.h - clip.cpp - clip.h - clip-impl.h - clip-model.h - clip-graph.h + ${MTMD_CORE_SOURCES} models/models.h + models/chatterbox-gen.cpp + models/chatterbox-spkenc.cpp models/cogvlm.cpp models/conformer.cpp models/deepseek4v.cpp @@ -47,6 +55,7 @@ add_library(mtmd models/kimik25.cpp models/nemotron-v2-vl.cpp models/nemotron-omni.cpp + models/nemotron-transcribe.cpp models/muse-glimmer.cpp models/muse-spark-audio.cpp models/llama4.cpp @@ -58,6 +67,7 @@ add_library(mtmd models/sensenova-u1.cpp models/minimax-m3.cpp models/qwen3vl.cpp + models/magevl.cpp models/lfm3vl.cpp models/lfm3a.cpp models/lfm3a-gen.cpp @@ -67,6 +77,7 @@ add_library(mtmd models/mimo-audio.cpp models/qwen3tts-spkenc.cpp models/qwen3tts-gen.cpp + models/drama3-gen.cpp models/pockettts-seanet.cpp models/pockettts-spkenc.cpp models/pockettts-gen.cpp @@ -87,13 +98,22 @@ set_target_properties(mtmd PROPERTIES VERSION ${LLAMA_VERSION_BASE} SOVERSION ${LLAMA_VERSION_MAJOR} MACHO_CURRENT_VERSION 0 # keep macOS linker from seeing oversized version number + UNITY_BUILD ON + UNITY_BUILD_BATCH_SIZE 16 ) +# exclude non-model sources from unity build +set_source_files_properties(${MTMD_CORE_SOURCES} + PROPERTIES SKIP_UNITY_BUILD_INCLUSION ON) + target_link_libraries (mtmd PUBLIC ggml llama) target_link_libraries (mtmd PRIVATE Threads::Threads vendor::hash vendor::miniaudio vendor::stb vendor::sheredom) target_include_directories(mtmd PUBLIC .) target_compile_features (mtmd PRIVATE cxx_std_17) +# for M_PI, set here because a unity batch can include before the source that needs it +target_compile_definitions(mtmd PRIVATE _USE_MATH_DEFINES) + if (MTMD_VIDEO) target_compile_definitions(mtmd PRIVATE MTMD_VIDEO) endif() diff --git a/tools/mtmd/README-dev.md b/tools/mtmd/README-dev.md index 5dad86100f44..b24c393a2e6a 100644 --- a/tools/mtmd/README-dev.md +++ b/tools/mtmd/README-dev.md @@ -7,6 +7,7 @@ Please refer to [multimodal.md](../../docs/multimodal.md) for a broader context. In short: - `libmtmd` started as a wrapper around `libllava` / `clip.cpp` - Various components that used to be in `clip.cpp` are moved progressively to mtmd. For example, preprocessor is now part of mtmd +- `clip.cpp` has since been split into `mtmd-model` (hparams and weights, one `mtmd_model` per modality), `mtmd-subcontext` (backends and compute, one `mtmd_subcontext` per modality) and `mtmd-graph` (graph builders, see `models/`). Only the GGUF format keeps the `clip.` name (metadata keys and the `clip` architecture), for compatibility with existing mmproj files ## Terminologies @@ -15,6 +16,29 @@ In short: - tiles / slices: for llava-uhd-style models, the preprocessor breaks a large input into smaller square images called tiles or slices - chunk: a mtmd_input_chunk represents a preprocessed input that can then be passed through `mtmd_encode()` +## Structure + +Overview of the `libmtmd` structure: + +- `mtmd_context` (`mtmd.cpp`): the public handle + - `mtmd_models`: one mmproj file, loaded by `mtmd_model_load()` + - `mtmd_model` `v` / `a` / `gen_a` (`mtmd-model.h`): one per modality, holds hparams, weights (`ctx_data` + `buf`) and requirements + - `mtmd_backends` (`mtmd-subcontext.h`): GPU/CPU backend instances, CPU threadpool and one scheduler, shared by the subcontexts + - `compute_mutex`: held by every public function that runs a compute pass, so only one pass at a time runs on the backends + - `mtmd_subcontext` `ctx_v` / `ctx_a` / `ctx_gen_a` (`mtmd-subcontext.h`): one per model, holds a const ref to it and a ref to the backends + - `buf_compute_meta` and the resolved flash attn of its graphs + - `mtmd_graph` (`mtmd-graph.h`, `models/`): built for each `encode()` call, temporary + - `mtmd_image_preprocessor` / `mtmd_audio_preprocessor` (`mtmd-image.h`, `mtmd-audio.h`): read the hparams of their model +- `mtmd_gen_state` / `mtmd_audio_stream`: public, one per sequence / stream + - `mtmd_subctx_gen_state` / `mtmd_subctx_enc_state`: allocated on the backend of a subcontext, see `gen_state_init()` / `enc_state_init()` + +Rules of thumb: +- `mtmd_model` is read-only once loaded. Anything that depends on the backend (flash attn resolution, warmup) belongs to `mtmd_subcontext` +- Like the pp/tg graphs of `llama_context`, the subcontexts share one scheduler: each warmup reserves its worst-case graph on it, so the compute buffers fit the largest one, and switching modality re-plans inside them +- A model declares its runtime needs (e.g. `max_nodes`, `flash_attn_auto`) in `load_hparams`, the subcontext applies them +- Models and backends must outlive the subcontexts, `mtmd_context` declares them first for that reason +- Locking happens only at the public API boundary (`mtmd.cpp`): a new public function that runs a graph must take `compute_mutex`. The internals (subcontexts, preprocessors, `mtmd_audio_log_mel()`) assume it is held and never lock + ## Pipeline A typical pipeline of the core libmtmd is as follows: @@ -67,21 +91,21 @@ Due to wide variety of audio generation pipelines, the `mtmd_gen_audio` system i - Backbone model should be converted to a normal text model (loadable via `libllama`) - If model used hard-coded embedding row ID, append them to token embeddings and assign token name for them (see `qwen3tts.py`) - If model have a specific output logits head for audio codes (usually semantic code), keep the head as-is and pad the logits at inference time (see `src/models/qwen3vl.cpp`) - - Sidecar models (code2wav, bigvgan, etc) must live inside the mmproj GGUF (but can be in different `clip_context` if necessary) + - Sidecar models (code2wav, bigvgan, etc) must live inside the mmproj GGUF (but can be in different `mtmd_subcontext` if necessary) - Note: it should use `ggml_build_forward_select` to select graphs if multiple graphs living in the same context - - Reuse existing GGUF metadata key name and tensor name whenever possible; think twice before adding extensive changes to GGUF writer. For example, Qwen3-TTS hard-code part of the hparams to `clip.cpp` as they won't likely to change. + - Reuse existing GGUF metadata key name and tensor name whenever possible; think twice before adding extensive changes to GGUF writer. For example, Qwen3-TTS hard-code part of the hparams in `mtmd-model.cpp` as they won't likely to change. - For tensor naming: - Prefixed with `a.*` for tensors used by speaker encoder pipeline - Prefixed with `a.gen.*` for generation stages (code / mel-spectrogram / PCM generation) - For GGUF metadata: - Reuse as many existing keys as possible - - In most cases, you can hard-code model configs in the model graph class, or in `clip_hparams` + - In most cases, you can hard-code model configs in the model graph class, or in `mtmd_hparams` - If some values need to be exposed to the `mtmd_helper` layer, hard-code them in `mtmd_helper` and distinguish by pipeline and `mtmd_gen_audio_info::model_variant` if necessary - Do NOT add new GGUF metadata or new fields to `mtmd_gen_audio_info` unless you can prove that you absolutely need them 4. Make sure most of the changes happen inside `mtmd-helper-gen.cpp`. A good PR looks like this: - 10-20% changes is to add new backbone (text) model and conversion - 60% changes inside `mtmd-helper-gen.cpp` - - 10% changes inside `libmtmd` and `clip.cpp` systems + - 10% changes inside `libmtmd` core (`mtmd-model`, `mtmd-subcontext`, `models/`) - The rest downstream code (CLI, server) should have no changes at all 5. Update usage documentation in `tools/tts/README.md` diff --git a/tools/mtmd/README.md b/tools/mtmd/README.md index 70194194718b..f0290aae5476 100644 --- a/tools/mtmd/README.md +++ b/tools/mtmd/README.md @@ -32,7 +32,7 @@ Consequently, running a multimodal model typically requires two GGUF files: As outlined in the history, `libmtmd` is the modern library designed to replace the original `llava.cpp` implementation for handling multimodal inputs. -Built upon `clip.cpp` (similar to `llava.cpp`), `libmtmd` offers several advantages: +Originally built upon `clip.cpp` (similar to `llava.cpp`), `libmtmd` offers several advantages: - **Unified Interface:** Aims to consolidate interaction for various multimodal models. - **Improved UX/DX:** Features a more intuitive API, inspired by the `Processor` class in the Hugging Face `transformers` library. - **Flexibility:** Designed to support multiple input types (text, audio, images) while respecting the wide variety of chat templates used by different models. diff --git a/tools/mtmd/clip.cpp b/tools/mtmd/clip.cpp deleted file mode 100644 index 9a043d7f9ca3..000000000000 --- a/tools/mtmd/clip.cpp +++ /dev/null @@ -1,7172 +0,0 @@ -#include "clip.h" -#include "clip-impl.h" -#include "clip-model.h" -#include "clip-graph.h" -#include "models/models.h" - -#include "ggml.h" -#include "ggml-cpp.h" -#include "ggml-alloc.h" -#include "ggml-backend.h" -#include "gguf.h" - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -struct clip_logger_state g_logger_state = {clip_log_callback_default, NULL}; - -//#define CLIP_DEBUG_FUNCTIONS - -#ifdef CLIP_DEBUG_FUNCTIONS -static void clip_image_write_image_to_ppm(const clip_image_u8& img, const std::string& filename) { - std::ofstream file(filename, std::ios::binary); - if (!file.is_open()) { - LOG_ERR("Failed to open file for writing: %s\n", filename.c_str()); - return; - } - - // PPM header: P6 format, width, height, and max color value - const auto ppm_size = img.get_size(); - file << "P6\n" << ppm_size.width << " " << ppm_size.height << "\n255\n"; - - // Write pixel data - const auto & ppm_buf = img.get_ro_buf(); - for (size_t i = 0; i < ppm_buf.size(); i += 3) { - // PPM expects binary data in RGB format, which matches our image buffer - file.write(reinterpret_cast(&ppm_buf[i]), 3); - } - - file.close(); -} - -static void clip_image_save_to_bmp(const clip_image_u8& img, const std::string& filename) { - std::ofstream file(filename, std::ios::binary); - if (!file.is_open()) { - LOG_ERR("Failed to open file for writing: %s\n", filename.c_str()); - return; - } - - const auto bmp_size = img.get_size(); - int fileSize = 54 + 3 * bmp_size.width * bmp_size.height; // File header + info header + pixel data - int bytesPerPixel = 3; - int widthInBytes = bmp_size.width * bytesPerPixel; - int paddingAmount = (4 - (widthInBytes % 4)) % 4; - int stride = widthInBytes + paddingAmount; - - // Bitmap file header - unsigned char fileHeader[14] = { - 'B','M', // Signature - 0,0,0,0, // Image file size in bytes - 0,0,0,0, // Reserved - 54,0,0,0 // Start of pixel array - }; - - // Total file size - fileSize = 54 + (stride * bmp_size.height); - fileHeader[2] = (unsigned char)(fileSize); - fileHeader[3] = (unsigned char)(fileSize >> 8); - fileHeader[4] = (unsigned char)(fileSize >> 16); - fileHeader[5] = (unsigned char)(fileSize >> 24); - - // Bitmap information header (BITMAPINFOHEADER) - unsigned char infoHeader[40] = { - 40,0,0,0, // Size of this header (40 bytes) - 0,0,0,0, // Image width - 0,0,0,0, // Image height - 1,0, // Number of color planes - 24,0, // Bits per pixel - 0,0,0,0, // No compression - 0,0,0,0, // Image size (can be 0 for no compression) - 0,0,0,0, // X pixels per meter (not specified) - 0,0,0,0, // Y pixels per meter (not specified) - 0,0,0,0, // Total colors (color table not used) - 0,0,0,0 // Important colors (all are important) - }; - - // Width and height in the information header - infoHeader[4] = (unsigned char)(bmp_size.width); - infoHeader[5] = (unsigned char)(bmp_size.width >> 8); - infoHeader[6] = (unsigned char)(bmp_size.width >> 16); - infoHeader[7] = (unsigned char)(bmp_size.width >> 24); - infoHeader[8] = (unsigned char)(bmp_size.height); - infoHeader[9] = (unsigned char)(bmp_size.height >> 8); - infoHeader[10] = (unsigned char)(bmp_size.height >> 16); - infoHeader[11] = (unsigned char)(bmp_size.height >> 24); - - // Write file headers - file.write(reinterpret_cast(fileHeader), sizeof(fileHeader)); - file.write(reinterpret_cast(infoHeader), sizeof(infoHeader)); - - // Pixel data - std::vector padding(3, 0); // Max padding size to be added to each row - for (int y = bmp_size.height - 1; y >= 0; --y) { // BMP files are stored bottom-to-top - for (int x = 0; x < bmp_size.width; ++x) { - // Each pixel - const auto px = img.get_pixel(x, y); - unsigned char pixel[3] = { - px[2], // BMP stores pixels in BGR format - px[1], - px[0] - }; - file.write(reinterpret_cast(pixel), 3); - } - // Write padding for the row - file.write(reinterpret_cast(padding.data()), paddingAmount); - } - - file.close(); -} - -// debug function to convert f32 to u8 -static void clip_image_convert_f32_to_u8(const clip_image_f32& src, clip_image_u8& dst) { - dst.set_size(src.get_size(), false); - const auto & src_buf = src.get_ro_buf(); - std::vector dst_buf(src.n_elements()); - for (size_t i = 0; i < src.n_elements(); ++i) { - dst_buf[i] = static_cast(std::min(std::max(int(src_buf[i] * 255.0f), 0), 255)); - } - dst.cpy_buf(dst_buf); -} -#endif - - -struct clip_ctx { - clip_model model; - - gguf_context_ptr ctx_gguf; - ggml_context_ptr ctx_data; - - std::vector buf_compute_meta; - - std::vector backend_ptrs; - std::vector backend_buft; - - ggml_backend_t backend = nullptr; - ggml_backend_t backend_cpu = nullptr; - ggml_backend_buffer_ptr buf; - - - int max_nodes = 8192; - ggml_backend_sched_ptr sched; - clip_flash_attn_type flash_attn_type = CLIP_FLASH_ATTN_TYPE_AUTO; - bool is_allocated = false; - - bool debug_output_embeddings = false; - - // for measuring memory usage - bool no_alloc = false; - std::map mem_usage; - std::map mem_compute; - - bool support_batch = false; - - clip_ctx(clip_context_params & ctx_params) { - flash_attn_type = ctx_params.flash_attn_type; - no_alloc = ctx_params.no_alloc; - backend_cpu = ggml_backend_init_by_type(GGML_BACKEND_DEVICE_TYPE_CPU, nullptr); - if (!backend_cpu) { - throw std::runtime_error("failed to initialize CPU backend"); - } - if (ctx_params.use_gpu) { - if (ctx_params.device != nullptr) { - backend = ggml_backend_dev_init(ctx_params.device, nullptr); - if (!backend) { - throw std::runtime_error(string_format("%s: failed to initialize \"%s\" backend\n", - __func__, ggml_backend_dev_name(ctx_params.device))); - } - } else { - backend = ggml_backend_init_by_type(GGML_BACKEND_DEVICE_TYPE_GPU, nullptr); - backend = backend ? backend : ggml_backend_init_by_type(GGML_BACKEND_DEVICE_TYPE_IGPU, nullptr); - } - } - - if (backend) { - LOG_INF("%s: CLIP using %s backend\n", __func__, ggml_backend_name(backend)); - backend_ptrs.push_back(backend); - backend_buft.push_back(ggml_backend_get_default_buffer_type(backend)); - } else { - backend = backend_cpu; - LOG_INF("%s: CLIP using CPU backend\n", __func__); - } - - if (ctx_params.image_min_tokens > 0) { - model.hparams.custom_image_min_tokens = ctx_params.image_min_tokens; - } - if (ctx_params.image_max_tokens > 0) { - model.hparams.custom_image_max_tokens = ctx_params.image_max_tokens; - } - - backend_ptrs.push_back(backend_cpu); - backend_buft.push_back(ggml_backend_get_default_buffer_type(backend_cpu)); - - sched.reset( - ggml_backend_sched_new(backend_ptrs.data(), backend_buft.data(), backend_ptrs.size(), 8192, false, true) - ); - - if (ctx_params.cb_eval != nullptr) { - ggml_backend_sched_set_eval_callback(sched.get(), ctx_params.cb_eval, ctx_params.cb_eval_user_data); - } - - debug_output_embeddings = std::getenv("MTMD_DEBUG_EMBEDDINGS") != nullptr; - } - - ~clip_ctx() { - ggml_backend_free(backend); - if (backend != backend_cpu) { - ggml_backend_free(backend_cpu); - } - } - - // this function is added so that we don't change too much of the existing code - projector_type proj_type() const { - return model.proj_type; - } -}; - -// -// clip_graph -// - -clip_graph::clip_graph(clip_ctx * ctx, const clip_image_f32 & img) : - model(ctx->model), - hparams(model.hparams), - proj_type(ctx->proj_type()), - img(img), - patch_size(hparams.patch_size), - n_patches_x(img.nx() / patch_size), - n_patches_y(img.ny() / patch_size), - n_patches(n_patches_x * n_patches_y), - n_embd(hparams.n_embd), - n_head(hparams.n_head), - n_head_kv(hparams.n_head_kv), - d_head(hparams.n_embd_head > 0 ? hparams.n_embd_head : (n_head > 0 ? n_embd / n_head : 0)), - n_layer(hparams.n_layer), - n_mmproj_embd(clip_n_mmproj_embd(ctx)), - eps(hparams.eps), - kq_scale(d_head > 0 ? 1.0f / sqrtf((float)d_head) : 0.0f), - flash_attn_type(ctx->flash_attn_type) { - struct ggml_init_params params = { - /*.mem_size =*/ ctx->buf_compute_meta.size(), - /*.mem_buffer =*/ ctx->buf_compute_meta.data(), - /*.no_alloc =*/ true, - }; - ctx0_ptr.reset(ggml_init(params)); - ctx0 = ctx0_ptr.get(); - gf = ggml_new_graph_custom(ctx0, ctx->max_nodes, false); -} - -clip_graph::clip_graph(const clip_graph & parent) : - model(parent.model), - hparams(parent.hparams), - proj_type(parent.proj_type), - img(parent.img), - patch_size(parent.patch_size), - n_patches_x(parent.n_patches_x), - n_patches_y(parent.n_patches_y), - n_patches(parent.n_patches), - n_embd(parent.n_embd), - n_head(parent.n_head), - n_head_kv(parent.n_head_kv), - d_head(parent.d_head), - n_layer(parent.n_layer), - n_mmproj_embd(parent.n_mmproj_embd), - eps(parent.eps), - kq_scale(parent.kq_scale), - flash_attn_type(parent.flash_attn_type) { - // reuse from parent - ctx0 = parent.ctx0; - gf = parent.gf; -} - -ggml_tensor * clip_graph::build_mm(ggml_tensor * w, ggml_tensor * x) const { - return ggml_mul_mat(ctx0, w, x); -} - -void clip_graph::cb(ggml_tensor * cur, const char * name, int il) const { - if (il >= 0) { - ggml_format_name(cur, "%s-%d", name, il); - } else { - ggml_set_name(cur, name); - } -} - -// siglip2 naflex -ggml_tensor * clip_graph::resize_position_embeddings(uint32_t interpolation_mode) { - ggml_tensor * pos_embd = model.position_embeddings; - const int height = img.ny() / patch_size; - const int width = img.nx() / patch_size; - const uint32_t mode = interpolation_mode; - const int n_per_side = (int)std::sqrt(pos_embd->ne[1]); - - GGML_ASSERT(pos_embd); - - if (height == n_per_side && width == n_per_side) { - return pos_embd; - } - - pos_embd = ggml_reshape_3d(ctx0, pos_embd, n_embd, n_per_side, n_per_side); // -> (n_embd, n_per_side, n_per_side) - pos_embd = ggml_permute(ctx0, pos_embd, 2, 0, 1, 3); // -> (n_per_side, n_per_side, n_embd) - pos_embd = ggml_interpolate(ctx0, pos_embd, width, height, n_embd, 1, mode); // -> (width, height, n_embd) - pos_embd = ggml_permute(ctx0, pos_embd, 1, 2, 0, 3); // -> (n_embd, width, height) - pos_embd = ggml_cont_2d(ctx0, pos_embd, n_embd, width * height); // -> (n_embd, width * height) - - return pos_embd; -} - -// build vision transformer (ViT) cgraph -// this function should cover most of the models -// if your model has specific features, you should probably duplicate this function -ggml_tensor * clip_graph::build_vit( - ggml_tensor * inp, - int64_t n_pos, - norm_type norm_t, - ffn_op_type ffn_t, - ggml_tensor * learned_pos_embd, - std::function add_pos, - const build_vit_opts & opts - ) { - // batch dim: inp is [n_embd, n_pos, B] - const int64_t B = inp->ne[2]; - - if (learned_pos_embd) { - inp = ggml_add(ctx0, inp, learned_pos_embd); - cb(inp, "pos_embed", -1); - } - - // flatten batch; unflatten again in attention - inp = ggml_reshape_2d(ctx0, inp, n_embd, n_pos * B); - - ggml_tensor * inpL = inp; - - // pre-layernorm - if (model.pre_ln_w) { - inpL = build_norm(inpL, model.pre_ln_w, model.pre_ln_b, norm_t, eps, -1); - cb(inpL, "pre_ln", -1); - } - - // loop over layers - for (int il = 0; il < n_layer; il++) { - auto & layer = model.layers[il]; - ggml_tensor * cur = inpL; // inpL = residual, cur = hidden_states - - ggml_tensor * attn_mask = opts.attn_mask; - if (opts.attn_mask_layers.size() > (size_t) il) { - attn_mask = opts.attn_mask_layers[il]; - } - - // layernorm1 - cur = build_norm(cur, layer.ln_1_w, layer.ln_1_b, norm_t, eps, il); - cb(cur, "layer_inp_normed", il); - - // self-attention - { - ggml_tensor * Qcur = nullptr; - ggml_tensor * Kcur = nullptr; - ggml_tensor * Vcur = nullptr; - if (layer.qkv_w != nullptr) { - // fused qkv - cur = build_mm(layer.qkv_w, cur); - if (layer.qkv_b != nullptr) { - cur = ggml_add(ctx0, cur, layer.qkv_b); - } - - // Q/K/V as [d_head, n_head, n_pos, B], the batch stride is cur->nb[1]*n_pos. - Qcur = ggml_view_4d(ctx0, cur, d_head, n_head, n_pos, B, - /* nb1 */ ggml_row_size(cur->type, d_head), - /* nb2 */ cur->nb[1], - /* nb3 */ cur->nb[1] * n_pos, - /* offset */ 0); - - Kcur = ggml_view_4d(ctx0, cur, d_head, n_head, n_pos, B, - /* nb1 */ ggml_row_size(cur->type, d_head), - /* nb2 */ cur->nb[1], - /* nb3 */ cur->nb[1] * n_pos, - /* offset */ ggml_row_size(cur->type, n_head * d_head)); - - Vcur = ggml_view_4d(ctx0, cur, d_head, n_head, n_pos, B, - /* nb1 */ ggml_row_size(cur->type, d_head), - /* nb2 */ cur->nb[1], - /* nb3 */ cur->nb[1] * n_pos, - /* offset */ ggml_row_size(cur->type, 2 * n_head * d_head)); - - if (layer.q_norm) { - GGML_ASSERT(layer.q_norm->ne[0] == Qcur->ne[0]); - Qcur = build_norm(Qcur, layer.q_norm, NULL, norm_t, eps, il); - cb(Qcur, "Qcur_norm", il); - } - - if (layer.k_norm) { - GGML_ASSERT(layer.k_norm->ne[0] == Kcur->ne[0]); - Kcur = build_norm(Kcur, layer.k_norm, NULL, norm_t, eps, il); - cb(Kcur, "Kcur_norm", il); - } - - } else { - // separate q, k, v - Qcur = build_mm(layer.q_w, cur); - if (layer.q_b) { - Qcur = ggml_add(ctx0, Qcur, layer.q_b); - } - - Kcur = build_mm(layer.k_w, cur); - if (layer.k_b) { - Kcur = ggml_add(ctx0, Kcur, layer.k_b); - } - - Vcur = build_mm(layer.v_w, cur); - if (layer.v_b) { - Vcur = ggml_add(ctx0, Vcur, layer.v_b); - } - - // if true, norm must be applied after reshaping to (d_head, n_head, n_pos) - bool norm_per_head = layer.q_norm && layer.q_norm->ne[0] == d_head; - - if (!norm_per_head) { - if (layer.q_norm) { - Qcur = build_norm(Qcur, layer.q_norm, NULL, norm_t, eps, il); - cb(Qcur, "Qcur_norm", il); - } - if (layer.k_norm) { - Kcur = build_norm(Kcur, layer.k_norm, NULL, norm_t, eps, il); - cb(Kcur, "Kcur_norm", il); - } - } - - Qcur = ggml_reshape_4d(ctx0, Qcur, d_head, n_head, n_pos, B); - Kcur = ggml_reshape_4d(ctx0, Kcur, d_head, n_head_kv, n_pos, B); - Vcur = ggml_reshape_4d(ctx0, Vcur, d_head, n_head_kv, n_pos, B); - - if (norm_per_head) { - if (layer.q_norm) { - Qcur = build_norm(Qcur, layer.q_norm, NULL, norm_t, eps, il); - cb(Qcur, "Qcur_norm_per_head", il); - } - if (layer.k_norm) { - Kcur = build_norm(Kcur, layer.k_norm, NULL, norm_t, eps, il); - cb(Kcur, "Kcur_norm_per_head", il); - } - } - } - - cb(Qcur, "Qcur", il); - cb(Kcur, "Kcur", il); - cb(Vcur, "Vcur", il); - - if (add_pos) { - Qcur = add_pos(Qcur, layer); - Kcur = add_pos(Kcur, layer); - cb(Qcur, "Qcur_pos", il); - cb(Kcur, "Kcur_pos", il); - } - - if (proj_type == PROJECTOR_TYPE_GEMMA4V) { - Vcur = ggml_rms_norm(ctx0, Vcur, eps); - cb(Vcur, "Vcur_normed", il); - } - - // build_attn returns a flat 2D [n_embd, n_pos*B] - cur = build_attn(layer.o_w, layer.o_b, - Qcur, Kcur, Vcur, attn_mask, kq_scale, il); - cb(cur, "attn_out", il); - } - - if (layer.ls_1_w) { - cur = ggml_mul(ctx0, cur, layer.ls_1_w); - cb(cur, "attn_out_scaled", il); - } - - if (layer.attn_post_norm_w) { - cur = build_norm(cur, layer.attn_post_norm_w, nullptr, norm_t, eps, il); - cb(cur, "attn_post_normed", il); - } - - // re-add the layer input, e.g., residual - cur = ggml_add(ctx0, cur, inpL); - - inpL = cur; // inpL = residual, cur = hidden_states - - if (opts.callback_layer_out) { - opts.callback_layer_out(cur, il); - } - - cb(cur, "ffn_inp", il); - - // layernorm2 (pre-ffn norm) - cur = build_norm(cur, layer.ln_2_w, layer.ln_2_b, norm_t, eps, il); - cb(cur, "ffn_inp_normed", il); - - // ffn - cur = layer.ff_gate_exps_w - ? build_moe_ffn(cur, layer, ffn_t, il) - : build_ffn(cur, - layer.ff_up_w, layer.ff_up_b, - layer.ff_gate_w, layer.ff_gate_b, - layer.ff_down_w, layer.ff_down_b, - ffn_t, il); - - cb(cur, "ffn_out", il); - - if (layer.ff_post_norm_w) { - cur = build_norm(cur, layer.ff_post_norm_w, nullptr, norm_t, eps, il); - cb(cur, "ffn_post_normed", il); - } - - if (layer.ls_2_w) { - cur = ggml_mul(ctx0, cur, layer.ls_2_w); - cb(cur, "ffn_out_scaled", il); - } - - // residual 2 - cur = ggml_add(ctx0, inpL, cur); - cb(cur, "layer_out", il); - - if (layer.ls_out_w) { - cur = ggml_mul(ctx0, cur, layer.ls_out_w); - cb(cur, "layer_out_scaled", il); - } - - inpL = cur; - } - - if (model.audio_has_avgpool()) { - ggml_tensor * cur = inpL; - cur = ggml_transpose(ctx0, cur); - cur = ggml_cont(ctx0, cur); - cur = ggml_pool_1d(ctx0, cur, GGML_OP_POOL_AVG, 2, 2, 0); - cur = ggml_transpose(ctx0, cur); - cur = ggml_cont(ctx0, cur); - inpL = cur; - } - - // post-layernorm - if (model.post_ln_w && !opts.skip_post_ln) { - inpL = build_norm(inpL, model.post_ln_w, model.post_ln_b, norm_t, eps, -1); - } - - // restore the batch dim - GGML_ASSERT(inpL->ne[1] % B == 0); - inpL = ggml_reshape_3d(ctx0, inpL, n_embd, inpL->ne[1] / B, B); - return inpL; -} - -// build the input after conv2d (inp_raw --> patches) -// returns tensor with shape [n_embd, n_patches] -ggml_tensor * clip_graph::build_inp() { - ggml_tensor * inp_raw = build_inp_raw(); - ggml_tensor * inp = ggml_conv_2d(ctx0, model.patch_embeddings_0, inp_raw, patch_size, patch_size, 0, 0, 1, 1); - inp = ggml_reshape_3d(ctx0, inp, n_patches, n_embd, n_batch); - inp = ggml_cont(ctx0, ggml_transpose(ctx0, inp)); - if (model.patch_bias) { - inp = ggml_add(ctx0, inp, model.patch_bias); - cb(inp, "patch_bias", -1); - } - return inp; -} - -ggml_tensor * clip_graph::build_inp_raw(int channels) { - ggml_tensor * inp_raw = ggml_new_tensor_4d(ctx0, GGML_TYPE_F32, img.nx(), img.ny(), channels, n_batch); - ggml_set_name(inp_raw, "inp_raw"); - ggml_set_input(inp_raw); - return inp_raw; -} - -// qwen-vl style temporal merge: Conv3D(kt=2) split into 2 Conv2D -ggml_tensor * clip_graph::build_inp_temporal_merge() { - ggml_tensor * inp_raw = build_inp_raw(); - - GGML_ASSERT(img.nx() % (patch_size * 2) == 0); - GGML_ASSERT(img.ny() % (patch_size * 2) == 0); - - const size_t nb1 = ggml_row_size(inp_raw->type, img.nx()); - const size_t nb2 = ggml_row_size(inp_raw->type, img.nx() * img.ny()); - - if (n_batch == 1) { - // still image input - return ggml_add(ctx0, - ggml_conv_2d(ctx0, model.patch_embeddings_0, inp_raw, patch_size, patch_size, 0, 0, 1, 1), - ggml_conv_2d(ctx0, model.patch_embeddings_1, inp_raw, patch_size, patch_size, 0, 0, 1, 1)); - } else if (n_batch == 2) { - // 2 frames input (video input) - ggml_tensor * inp_0 = ggml_view_3d(ctx0, inp_raw, - img.nx(), img.ny(), 3, nb1, nb2, 0); - ggml_tensor * inp_1 = ggml_view_3d(ctx0, inp_raw, - img.nx(), img.ny(), 3, nb1, nb2, - nb2 * 3); // move to the second frame - return ggml_add(ctx0, - ggml_conv_2d(ctx0, model.patch_embeddings_0, inp_0, patch_size, patch_size, 0, 0, 1, 1), - ggml_conv_2d(ctx0, model.patch_embeddings_1, inp_1, patch_size, patch_size, 0, 0, 1, 1)); - } else { - GGML_ASSERT(false && "n_batch > 2 is not supported"); - } -} - -static std::string get_attn_mask_name(int idx) { - return idx == 0 ? "attn_mask" : "attn_mask_" + std::to_string(idx); -} - -ggml_tensor * clip_graph::build_inp_attn_mask(int64_t n_kv, int64_t n_q, int idx) { - const ggml_type type = flash_attn_type == CLIP_FLASH_ATTN_TYPE_ENABLED ? GGML_TYPE_F16 : GGML_TYPE_F32; - ggml_tensor * mask = ggml_new_tensor_2d(ctx0, type, n_kv, n_q); - ggml_set_name(mask, get_attn_mask_name(idx).c_str()); - ggml_set_input(mask); - return mask; -} - -ggml_tensor * clip_graph::build_norm( - ggml_tensor * cur, - ggml_tensor * mw, - ggml_tensor * mb, - norm_type type, - float norm_eps, - int il) const { - - cur = type == NORM_TYPE_RMS - ? ggml_rms_norm(ctx0, cur, norm_eps) - : ggml_norm(ctx0, cur, norm_eps); - - if (mw) { - cur = ggml_mul(ctx0, cur, mw); - cb(cur, "norm_w", il); - } - - if (mb) { - cur = ggml_add(ctx0, cur, mb); - cb(cur, "norm_b", il); - } - - return cur; -} - -ggml_tensor * clip_graph::build_ffn( - ggml_tensor * cur, - ggml_tensor * up, - ggml_tensor * up_b, - ggml_tensor * gate, - ggml_tensor * gate_b, - ggml_tensor * down, - ggml_tensor * down_b, - ffn_op_type type_op, - int il) const { - - ggml_tensor * tmp = up ? build_mm(up, cur) : cur; - cb(tmp, "ffn_up", il); - - if (up_b) { - tmp = ggml_add(ctx0, tmp, up_b); - cb(tmp, "ffn_up_b", il); - } - - if (gate) { - cur = build_mm(gate, cur); - cb(cur, "ffn_gate", il); - - if (gate_b) { - cur = ggml_add(ctx0, cur, gate_b); - cb(cur, "ffn_gate_b", il); - } - } else { - cur = tmp; - } - - // we only support parallel ffn for now - switch (type_op) { - case FFN_SILU: - if (gate) { - cur = hparams.swiglu_clamp > 0.0f - ? ggml_swiglu_clamp(ctx0, cur, tmp, hparams.swiglu_clamp) - : ggml_swiglu_split(ctx0, cur, tmp); - cb(cur, "ffn_swiglu", il); - } else { - cur = ggml_silu(ctx0, cur); - cb(cur, "ffn_silu", il); - } break; - case FFN_GELU: - if (gate) { - cur = ggml_geglu_split(ctx0, cur, tmp); - cb(cur, "ffn_geglu", il); - } else { - cur = ggml_gelu(ctx0, cur); - cb(cur, "ffn_gelu", il); - } break; - case FFN_GELU_ERF: - if (gate) { - cur = ggml_geglu_erf_split(ctx0, cur, tmp); - cb(cur, "ffn_geglu_erf", il); - } else { - cur = ggml_gelu_erf(ctx0, cur); - cb(cur, "ffn_gelu_erf", il); - } break; - case FFN_GELU_QUICK: - if (gate) { - cur = ggml_geglu_quick_split(ctx0, cur, tmp); - cb(cur, "ffn_geglu_quick", il); - } else { - cur = ggml_gelu_quick(ctx0, cur); - cb(cur, "ffn_gelu_quick", il); - } break; - case FFN_RELU_SQR: - { - cur = ggml_relu(ctx0, cur); - cur = ggml_sqr(ctx0, cur); - cb(cur, "ffn_relu_sqr", il); - } break; - } - - if (down) { - cur = build_mm(down, cur); - } - - if (down_b) { - cb(cur, "ffn_down", il); - } - - if (down_b) { - cur = ggml_add(ctx0, cur, down_b); - } - - return cur; -} - -// MoE FFN with sigmoid router and normalized top-k weights (dots3note vision) -// the router runs in fp32; exp_probs_b only affects expert selection, not the weights -ggml_tensor * clip_graph::build_moe_ffn(ggml_tensor * cur, const clip_layer & layer, ffn_op_type type_op, int il) const { - const int64_t n_tokens = cur->ne[1]; - const int64_t n_expert = layer.ff_gate_exps_w->ne[2]; - const int64_t n_expert_used = std::min((int64_t) hparams.n_expert_used, n_expert); - GGML_ASSERT(n_expert_used > 0); - GGML_ASSERT(type_op == FFN_SILU); - - ggml_tensor * probs = ggml_sigmoid(ctx0, build_mm(layer.ff_gate_inp_w, cur)); // [n_expert, n_tokens] - cb(probs, "ffn_moe_probs", il); - - ggml_tensor * sel = layer.ff_exp_probs_b - ? ggml_add(ctx0, probs, layer.ff_exp_probs_b) - : probs; - ggml_tensor * selected = ggml_top_k(ctx0, sel, n_expert_used); // [n_expert_used, n_tokens] - - ggml_tensor * weights = ggml_get_rows(ctx0, - ggml_reshape_3d(ctx0, probs, 1, n_expert, n_tokens), selected); - weights = ggml_reshape_2d(ctx0, weights, n_expert_used, n_tokens); - weights = ggml_div(ctx0, weights, ggml_sum_rows(ctx0, weights)); - weights = ggml_reshape_3d(ctx0, weights, 1, n_expert_used, n_tokens); - cb(weights, "ffn_moe_weights", il); - - cur = ggml_reshape_3d(ctx0, cur, cur->ne[0], 1, n_tokens); - ggml_tensor * gate = ggml_mul_mat_id(ctx0, layer.ff_gate_exps_w, cur, selected); // [n_ff, n_expert_used, n_tokens] - ggml_tensor * up = ggml_mul_mat_id(ctx0, layer.ff_up_exps_w, cur, selected); - cur = ggml_mul(ctx0, ggml_silu(ctx0, gate), up); - cur = ggml_mul_mat_id(ctx0, layer.ff_down_exps_w, cur, selected); // [n_embd, n_expert_used, n_tokens] - cur = ggml_mul(ctx0, cur, weights); - - // sum over the selected experts - ggml_tensor * out = nullptr; - for (int64_t i = 0; i < n_expert_used; i++) { - ggml_tensor * v = ggml_view_2d(ctx0, cur, cur->ne[0], n_tokens, cur->nb[2], i * cur->nb[1]); - out = out ? ggml_add(ctx0, out, v) : v; - } - if (n_expert_used == 1) { - out = ggml_cont(ctx0, out); - } - cb(out, "ffn_moe_out", il); - return out; -} - -ggml_tensor * clip_graph::build_attn( - ggml_tensor * wo, - ggml_tensor * wo_b, - ggml_tensor * q_cur, - ggml_tensor * k_cur, - ggml_tensor * v_cur, - ggml_tensor * kq_mask, - float kq_scale, - int il, - ggml_tensor * sinks) const { - // these nodes are added to the graph together so that they are not reordered - // by doing so, the number of splits in the graph is reduced - // the order is fixed without the compute flag, so an unselected branch stays out of the compute set - ggml_build_forward_order(gf, q_cur); - ggml_build_forward_order(gf, k_cur); - ggml_build_forward_order(gf, v_cur); - - ggml_tensor * q = ggml_permute(ctx0, q_cur, 0, 2, 1, 3); - //cb(q, "q", il); - - ggml_tensor * k = ggml_permute(ctx0, k_cur, 0, 2, 1, 3); - //cb(k, "k", il); - - ggml_tensor * cur; - - if (flash_attn_type == CLIP_FLASH_ATTN_TYPE_ENABLED) { - ggml_tensor * v = ggml_permute(ctx0, v_cur, 0, 2, 1, 3); - - k = ggml_cast(ctx0, k, GGML_TYPE_F16); - v = ggml_cast(ctx0, v, GGML_TYPE_F16); - // mask must be f16 here, use build_inp_attn_mask() - GGML_ASSERT(!kq_mask || kq_mask->type == GGML_TYPE_F16); - - cur = ggml_flash_attn_ext(ctx0, q, k, v, kq_mask, kq_scale, 0.0f, 0.0f); - ggml_prec_set_acc(cur, GGML_PREC_F32); - if (sinks != nullptr) { - ggml_flash_attn_ext_add_sinks(cur, sinks); - } - - cur = ggml_reshape_2d(ctx0, cur, cur->ne[0]*cur->ne[1], cur->ne[2]*cur->ne[3]); - - } else { - ggml_tensor * v = ggml_permute(ctx0, v_cur, 1, 2, 0, 3); - v = ggml_cont(ctx0, v); - - ggml_tensor * kq = ggml_mul_mat(ctx0, k, q); - // F32 may not needed for vision encoders? - // ggml_prec_set_acc(kq, GGML_PREC_F32); - - kq = ggml_soft_max_ext(ctx0, kq, kq_mask, kq_scale, 0.0f); - if (sinks != nullptr) { - ggml_soft_max_add_sinks(kq, sinks); - } - - ggml_tensor * kqv = ggml_mul_mat(ctx0, v, kq); - cur = ggml_permute(ctx0, kqv, 0, 2, 1, 3); - cur = ggml_cont_2d(ctx0, cur, cur->ne[0] * cur->ne[1], cur->ne[2] * cur->ne[3]); - } - - cb(cur, "kqv_out", il); - - if (wo) { - cur = build_mm(wo, cur); - } - - if (wo_b) { - cur = ggml_add(ctx0, cur, wo_b); - } - - return cur; -} - -// implementation of the 2D RoPE without adding a new op in ggml -ggml_tensor * clip_graph::build_rope_2d( - ggml_context * ctx0, - ggml_tensor * cur, - ggml_tensor * pos_a, // first half - ggml_tensor * pos_b, // second half - const float freq_base, - const bool interleave_freq -) { - const int64_t n_dim = cur->ne[0]; - - // for example, if we have cur tensor of shape (n_dim=8, n_head, n_pos) - // we will have a list of 4 inv_freq: 1e-0, 1e-1, 1e-2, 1e-3 - // first half of cur will use 1e-0, 1e-2 (even) - // second half of cur will use 1e-1, 1e-3 (odd) - // the trick here is to rotate just half of n_dim, so inv_freq will automatically be even - // ^ don't ask me why, it's math! -2(2i) / n_dim == -2i / (n_dim/2) - // then for the second half, we use freq_scale to shift the inv_freq - // ^ why? replace (2i) with (2i+1) in the above equation - const float freq_scale_odd = interleave_freq - ? std::pow(freq_base, (float)-2/n_dim) - : 1.0; - - // first half, dims [0, n_dim/2) - cur = ggml_rope_ext( - ctx0, - cur, - pos_a, // positions - nullptr, // freq factors - n_dim/2, // n_dims - 0, 0, freq_base, - 1.0f, 0.0f, 1.0f, 0.0f, 0.0f - ); - - // second half, dims [n_dim/2, n_dim) - cur = ggml_rope_ext( - ctx0, - cur, - pos_b, // positions - nullptr, // freq factors - n_dim/2, // n_dims - 0, 0, freq_base, - freq_scale_odd, - 0.0f, 1.0f, 0.0f, 0.0f - ); - cur = ggml_rope_set_offset(cur, n_dim/2); - - return cur; -} - -// Generic function to stack frames for audio processing -// Abstracts out the StackAudioFrames logic used by ultravox -ggml_tensor * clip_graph::build_stack(ggml_tensor * cur, int32_t stack_factor, int32_t n_embed) { - if (stack_factor <= 1) { - return cur; - } - - int64_t total_elements = ggml_nelements(cur); - int64_t stride = n_embed * stack_factor; - - // Calculate padded length - int64_t padded_len = GGML_PAD(total_elements, stride); - int64_t pad = padded_len - total_elements; - - if (pad > 0) { - // Pad the tensor to make it divisible by stride - cur = ggml_view_1d(ctx0, cur, total_elements, 0); - cur = ggml_pad(ctx0, cur, pad, 0, 0, 0); - } - - // Reshape to [stride, padded_len / stride] - cur = ggml_view_2d(ctx0, cur, stride, padded_len / stride, - ggml_row_size(cur->type, stride), 0); - return cur; -} - -// aka pixel_shuffle / pixel_unshuffle / patch_merger (Kimi-VL) -// support dynamic resolution -ggml_tensor * clip_graph::build_suffix(ggml_tensor * cur) { - for (int idx : clip_suffix_rows(img.suffix_type)) { - GGML_ASSERT(model.tok_embd_sep && idx < model.tok_embd_sep->ne[1]); - ggml_tensor * row = ggml_view_2d(ctx0, model.tok_embd_sep, model.tok_embd_sep->ne[0], 1, - model.tok_embd_sep->nb[1], idx * model.tok_embd_sep->nb[1]); - cur = ggml_concat(ctx0, cur, ggml_cast(ctx0, row, cur->type), 1); - } - return cur; -} - -ggml_tensor * clip_graph::build_patch_merge_permute(ggml_tensor * cur, int scale_factor) { - GGML_ASSERT(scale_factor > 1); - - const int n_embd = cur->ne[0]; - int width = img.nx() / patch_size; - int height = img.ny() / patch_size; - - // pad width and height to factor - const int64_t pad_width = CLIP_ALIGN(width, scale_factor) - width; - const int64_t pad_height = CLIP_ALIGN(height, scale_factor) - height; - cur = ggml_reshape_3d(ctx0, cur, n_embd, width, height); - if (pad_width || pad_height) { - cur = ggml_pad(ctx0, cur, 0, pad_width, pad_height, 0); - width += pad_width; - height += pad_height; - } - - // unshuffle h - cur = ggml_reshape_3d(ctx0, cur, n_embd * scale_factor, width / scale_factor, height); - cur = ggml_permute(ctx0, cur, 0, 2, 1, 3); - - // unshuffle w - cur = ggml_cont_3d(ctx0, cur, n_embd * scale_factor * scale_factor, height / scale_factor, width / scale_factor); - cur = ggml_permute(ctx0, cur, 0, 2, 1, 3); - - cur = ggml_cont_2d(ctx0, cur, cur->ne[0], cur->ne[1] * cur->ne[2]); - cb(cur, "pixel_shuffle", -1); - - return cur; -} - -static std::unique_ptr clip_get_graph_builder(clip_ctx * ctx, const clip_image_f32_batch & imgs, - const clip_encode_params * params = nullptr) { - const clip_image_f32 & img = imgs.entries[0]; - std::unique_ptr builder; - - switch (ctx->proj_type()) { - case PROJECTOR_TYPE_GEMMA3: - case PROJECTOR_TYPE_IDEFICS3: - case PROJECTOR_TYPE_COHERE2V: - case PROJECTOR_TYPE_LFM2: - case PROJECTOR_TYPE_D1OMNI_V: - case PROJECTOR_TYPE_JANUS_PRO: - case PROJECTOR_TYPE_PHI4: - { - builder = std::make_unique(ctx, img); - } break; - case PROJECTOR_TYPE_GEMMA3NV: - { - builder = std::make_unique(ctx, img); - } break; - case PROJECTOR_TYPE_GEMMA4V: - { - builder = std::make_unique(ctx, img); - } break; - case PROJECTOR_TYPE_GEMMA4UV: - { - builder = std::make_unique(ctx, img); - } break; - case PROJECTOR_TYPE_PIXTRAL: - case PROJECTOR_TYPE_LIGHTONOCR: - { - builder = std::make_unique(ctx, img); - } break; - case PROJECTOR_TYPE_DOTS_OCR: - case PROJECTOR_TYPE_DOTS3NOTE_V: // same ViT + merger; pyramid MoE is handled by build_vit - { - builder = std::make_unique(ctx, img); - } break; - case PROJECTOR_TYPE_DOTS3NOTE_A: - { - builder = std::make_unique(ctx, img); - } break; - case PROJECTOR_TYPE_QWEN2VL: - case PROJECTOR_TYPE_QWEN25VL: - { - builder = std::make_unique(ctx, img); - } break; - case PROJECTOR_TYPE_QWEN3VL: - { - builder = std::make_unique(ctx, img); - } break; - case PROJECTOR_TYPE_SENSENOVA_U1: - { - builder = std::make_unique(ctx, img); - } break; - case PROJECTOR_TYPE_LFM3VL: - { - builder = std::make_unique(ctx, img); - } break; - case PROJECTOR_TYPE_LING3VL: - { - builder = std::make_unique(ctx, img); - } break; - case PROJECTOR_TYPE_EXAONE4_5: - { - builder = std::make_unique(ctx, img); - } break; - case PROJECTOR_TYPE_MIMOVL: - { - builder = std::make_unique(ctx, img); - } break; - case PROJECTOR_TYPE_MINIMAX_M3: - { - builder = std::make_unique(ctx, img); - } break; - case PROJECTOR_TYPE_MUSE_GLIMMER: - { - builder = std::make_unique(ctx, img); - } break; - case PROJECTOR_TYPE_STEP3VL: - { - builder = std::make_unique(ctx, img); - } break; - case PROJECTOR_TYPE_MINICPMV: - { - builder = std::make_unique(ctx, img); - } break; - case PROJECTOR_TYPE_MINICPMV4_6: - case PROJECTOR_TYPE_MINICPMV4_7: - case PROJECTOR_TYPE_MINICPM_OCR_1_0: - { - builder = std::make_unique(ctx, img); - } break; - case PROJECTOR_TYPE_INTERNVL: - { - builder = std::make_unique(ctx, img); - } break; - case PROJECTOR_TYPE_NEMOTRON_V2_VL: - { - builder = std::make_unique(ctx, img); - } break; - case PROJECTOR_TYPE_NEMOTRON_OMNI: - { - builder = std::make_unique(ctx, img); - } break; - case PROJECTOR_TYPE_LLAMA4: - { - builder = std::make_unique(ctx, img); - } break; - case PROJECTOR_TYPE_VOXTRAL_RT: - { - builder = std::make_unique(ctx, img, params ? params->enc_state : nullptr); - } break; - case PROJECTOR_TYPE_ULTRAVOX: - case PROJECTOR_TYPE_VOXTRAL: - case PROJECTOR_TYPE_QWEN2A: - case PROJECTOR_TYPE_GLMA: - case PROJECTOR_TYPE_MERALION: - case PROJECTOR_TYPE_MUSIC_FLAMINGO: - { - builder = std::make_unique(ctx, img); - } break; - case PROJECTOR_TYPE_MUSE_SPARK_AUDIO: - { - builder = std::make_unique(ctx, img); - } break; - case PROJECTOR_TYPE_KIMIVL: - { - builder = std::make_unique(ctx, img); - } break; - case PROJECTOR_TYPE_PADDLEOCR: - { - builder = std::make_unique(ctx, img); - } break; - case PROJECTOR_TYPE_KIMIK25: - { - builder = std::make_unique(ctx, img); - } break; - case PROJECTOR_TYPE_DEEPSEEK4V: - case PROJECTOR_TYPE_DEEPSEEK41V: - { - builder = std::make_unique(ctx, img); - } break; - case PROJECTOR_TYPE_COGVLM: - { - builder = std::make_unique(ctx, img); - } break; - case PROJECTOR_TYPE_HUNYUANVL: - { - builder = std::make_unique(ctx, img); - } break; - case PROJECTOR_TYPE_MLP: - case PROJECTOR_TYPE_MLP_NORM: - case PROJECTOR_TYPE_LDP: - case PROJECTOR_TYPE_LDPV2: - case PROJECTOR_TYPE_GLM_EDGE: - { - builder = std::make_unique(ctx, img); - } break; - case PROJECTOR_TYPE_DEEPSEEKOCR: - { - builder = std::make_unique(ctx, img); - } break; - case PROJECTOR_TYPE_DEEPSEEKOCR2: - { - builder = std::make_unique(ctx, img); - } break; - case PROJECTOR_TYPE_LFM2A: - case PROJECTOR_TYPE_D1OMNI_A: - { - builder = std::make_unique(ctx, img); - } break; - case PROJECTOR_TYPE_LFM3A: - { - builder = std::make_unique(ctx, img); - } break; - case PROJECTOR_TYPE_GEMMA4A: - { - builder = std::make_unique(ctx, img); - } break; - case PROJECTOR_TYPE_GEMMA4UA: - { - builder = std::make_unique(ctx, img); - } break; - case PROJECTOR_TYPE_GRANITE_SPEECH: - { - builder = std::make_unique(ctx, img); - } break; - case PROJECTOR_TYPE_GLM4V: - case PROJECTOR_TYPE_GLM5V: - { - builder = std::make_unique(ctx, img); - } break; - case PROJECTOR_TYPE_QWEN3A: - { - builder = std::make_unique(ctx, img); - } break; - case PROJECTOR_TYPE_MIMO_AUDIO: - { - builder = std::make_unique(ctx, img); - } break; - case PROJECTOR_TYPE_QWEN3TTS_SPKENC: - { - builder = std::make_unique(ctx, img); - } break; - case PROJECTOR_TYPE_POCKETTTS_SPKENC: - { - builder = std::make_unique(ctx, img); - } break; - case PROJECTOR_TYPE_POCKETTTS_GEN: - { - const auto gen_process = params ? params->gen_process : CLIP_GEN_PROCESS_GEN_CODE; - const int n_step = ctx->model.hparams.flow_n_step; - const int64_t n_latent = ctx->model.gen_input_lin_w->ne[0]; - GGML_ASSERT(n_step > 0); - GGML_ASSERT(n_latent > 0); - // "inp_feats" takes the caller's buffer as-is, the graph must consume all of it - if (params && params->feats) { - GGML_ASSERT(params->feats->size() % (size_t) n_latent == 0); - GGML_ASSERT(params->feats->size() >= (size_t) n_latent); - } - const int n_frames = params && params->feats ? (int) (params->feats->size() / n_latent) : 1; - builder = std::make_unique(ctx, img, gen_process, n_step, n_frames); - } break; - case PROJECTOR_TYPE_LFM3A_GEN: - { - const auto gen_process = params ? params->gen_process : CLIP_GEN_PROCESS_GEN_CODE; - const int64_t n_cb = ctx->model.gen_code_embd_w->ne[2]; - bool all_known = false; - int n_frames = 1; - if (params && params->codes) { - GGML_ASSERT(params->codes->size() % (size_t) n_cb == 0); - if (gen_process == CLIP_GEN_PROCESS_GEN_WAV) { - n_frames = (int) (params->codes->size() / n_cb); - GGML_ASSERT(n_frames > 0); - } else { - // -1 marks a masked codebook, none masked means only the feedback embedding is needed - all_known = std::all_of(params->codes->begin(), params->codes->end(), [](int32_t c) { return c >= 0; }); - } - } - builder = std::make_unique(ctx, img, gen_process, all_known, n_frames); - } break; - case PROJECTOR_TYPE_QWEN3TTS_GEN: - { - const auto gen_process = params ? params->gen_process : CLIP_GEN_PROCESS_GEN_CODE; - const int top_k = params ? params->top_k : 50; - const float top_p = params ? params->top_p : 1.0f; - builder = std::make_unique(ctx, img, gen_process, top_k, top_p); - } break; - case PROJECTOR_TYPE_SOPRANO: - { - const int n_frames = params && params->feats ? (int) (params->feats->size() / ctx->model.hparams.projection_dim) : 2; - builder = std::make_unique(ctx, img, n_frames); - } break; - case PROJECTOR_TYPE_YOUTUVL: - { - builder = std::make_unique(ctx, img); - } break; - case PROJECTOR_TYPE_YASA2: - { - builder = std::make_unique(ctx, img); - } break; - case PROJECTOR_TYPE_PARAKEET: - { - builder = std::make_unique(ctx, img); - } break; - case PROJECTOR_TYPE_GRANITE4_VISION: - { - builder = std::make_unique(ctx, img); - } break; - default: - GGML_ABORT("missing cgraph builder"); - } - - builder->img_batch = &imgs; - - // TODO [QWEN_VIDEO]: improve this in the future - builder->n_batch = imgs.entries.size(); - - return builder; -} - -// -// clip_model_loader -// - -struct clip_model_loader { - ggml_context_ptr ctx_meta; - gguf_context_ptr ctx_gguf; - - std::string fname; - - size_t model_size = 0; // in bytes - - bool has_vision = false; - bool has_audio = false; - bool has_gen_audio = false; - - mtmd_progress_callback progress_callback = nullptr; - void * progress_callback_user_data = nullptr; - - // TODO @ngxson : we should not pass clip_ctx here, it should be clip_model - clip_model_loader(const char * fname, - bool skip_tensors = false, - mtmd_progress_callback progress_cb = nullptr, - void * progress_user_data = nullptr) - : fname(fname), - progress_callback(progress_cb), - progress_callback_user_data(progress_user_data) { - struct ggml_context * meta = nullptr; - - struct gguf_init_params params = { - /*.no_alloc = */ true, - /*.ctx = */ &meta, - }; - - ctx_gguf = gguf_context_ptr(gguf_init_from_file(fname, params)); - if (!ctx_gguf.get()) { - throw std::runtime_error(string_format("%s: failed to load CLIP model from %s. Does this file exist?\n", __func__, fname)); - } - - ctx_meta.reset(meta); - - const int n_tensors = gguf_get_n_tensors(ctx_gguf.get()); - - // print gguf info - { - std::string name; - get_string(KEY_NAME, name, false); - std::string description; - get_string(KEY_DESCRIPTION, description, false); - LOG_INF("%s: model name: %s\n", __func__, name.c_str()); - LOG_INF("%s: description: %s\n", __func__, description.c_str()); - LOG_INF("%s: GGUF version: %d\n", __func__, gguf_get_version(ctx_gguf.get())); - LOG_INF("%s: alignment: %zu\n", __func__, gguf_get_alignment(ctx_gguf.get())); - LOG_INF("%s: n_tensors: %d\n", __func__, n_tensors); - LOG_INF("%s: n_kv: %d\n", __func__, (int)gguf_get_n_kv(ctx_gguf.get())); - LOG_INF("\n"); - } - - // modalities - { - get_bool(KEY_HAS_VISION_ENC, has_vision, false); - get_bool(KEY_HAS_AUDIO_ENC, has_audio, false); - get_bool(KEY_HAS_GEN_AUDIO_ENC, has_gen_audio, false); - - if (has_vision) { - LOG_INF("%s: has vision encoder\n", __func__); - } - if (has_audio) { - LOG_INF("%s: has audio encoder\n", __func__); - } - if (has_gen_audio) { - LOG_INF("%s: has audio generation (gen) encoder\n", __func__); - } - } - - // tensors - if (!skip_tensors) { - for (int i = 0; i < n_tensors; ++i) { - const char * name = gguf_get_tensor_name(ctx_gguf.get(), i); - const size_t offset = gguf_get_tensor_offset(ctx_gguf.get(), i); - enum ggml_type type = gguf_get_tensor_type(ctx_gguf.get(), i); - ggml_tensor * cur = ggml_get_tensor(meta, name); - size_t tensor_size = ggml_nbytes(cur); - model_size += tensor_size; - LOG_DBG("%s: tensor[%d]: n_dims = %d, name = %s, tensor_size=%zu, offset=%zu, shape:[%" PRIu64 ", %" PRIu64 ", %" PRIu64 ", %" PRIu64 "], type = %s\n", - __func__, i, ggml_n_dims(cur), cur->name, tensor_size, offset, cur->ne[0], cur->ne[1], cur->ne[2], cur->ne[3], ggml_type_name(type)); - } - } - } - - void load_hparams(clip_model & model, clip_modality modality) { - auto & hparams = model.hparams; - std::string log_ffn_op; // for logging - - // sanity check - if (modality == CLIP_MODALITY_VISION) { - GGML_ASSERT(has_vision); - } else if (modality == CLIP_MODALITY_AUDIO) { - GGML_ASSERT(has_audio); - } else if (modality == CLIP_MODALITY_GEN_AUDIO) { - GGML_ASSERT(has_gen_audio); - } - model.modality = modality; - - - // projector type - std::string proj_type; - { - // default key - get_string(KEY_PROJ_TYPE, proj_type, false); - - // for models with mixed modalities - if (proj_type.empty()) { - if (modality == CLIP_MODALITY_VISION) { - get_string(KEY_VISION_PROJ_TYPE, proj_type, false); - } else if (modality == CLIP_MODALITY_AUDIO) { - get_string(KEY_AUDIO_PROJ_TYPE, proj_type, false); - } else if (modality == CLIP_MODALITY_GEN_AUDIO) { - get_string(KEY_GEN_AUDIO_PROJ_TYPE, proj_type, false); - } else { - GGML_ABORT("unknown modality"); - } - } - - model.proj_type = clip_projector_type_from_string(proj_type); - - if (model.proj_type == PROJECTOR_TYPE_UNKNOWN) { - throw std::runtime_error(string_format("%s: unknown projector type: %s\n", __func__, proj_type.c_str())); - } - - // correct arch for multimodal models (legacy method) - if (model.proj_type == PROJECTOR_TYPE_QWEN25O) { - model.proj_type = modality == CLIP_MODALITY_VISION - ? PROJECTOR_TYPE_QWEN25VL - : PROJECTOR_TYPE_QWEN2A; - } - } - - const bool is_vision = model.modality == CLIP_MODALITY_VISION; - const bool is_audio = model.modality == CLIP_MODALITY_AUDIO; - const bool is_gen_audio = model.modality == CLIP_MODALITY_GEN_AUDIO; - - // other hparams - { - const char * prefix = is_vision ? "vision" : (is_audio ? "audio" : "gen.audio"); - get_u32(string_format(KEY_N_EMBD, prefix), hparams.n_embd); - get_u32(string_format(KEY_N_HEAD, prefix), hparams.n_head); - get_u32(string_format(KEY_N_EMBD_HEAD, prefix), hparams.n_embd_head, false); - get_u32(string_format(KEY_N_FF, prefix), hparams.n_ff); - get_u32(string_format(KEY_N_BLOCK, prefix), hparams.n_layer); - get_u32(string_format(KEY_PROJ_DIM, prefix), hparams.projection_dim); - get_f32(string_format(KEY_LAYER_NORM_EPS, prefix), hparams.eps); - - // n_head_kv is optional (for GQA), default to n_head - hparams.n_head_kv = hparams.n_head; - get_u32(string_format(KEY_N_HEAD_KV, prefix), hparams.n_head_kv, false); - - if (is_vision) { - get_u32(KEY_IMAGE_SIZE, hparams.image_size); - get_u32(KEY_PATCH_SIZE, hparams.patch_size); - get_u32(KEY_MAX_SLICE_NUMS, hparams.max_slice_nums, false); - get_i32(KEY_MINICPMV_VERSION, hparams.minicpmv_version, false); // legacy - get_u32(KEY_MINICPMV_QUERY_NUM, hparams.minicpmv_query_num, false); - if (hparams.minicpmv_query_num == 0) { - // Fallback to hardcoded values for legacy models - if (hparams.minicpmv_version == 3) { - hparams.minicpmv_query_num = 64; - } else if (hparams.minicpmv_version == 4) { - hparams.minicpmv_query_num = 64; - } else if (hparams.minicpmv_version == 5) { - hparams.minicpmv_query_num = 64; - } else if (hparams.minicpmv_version == 6) { - hparams.minicpmv_query_num = 64; - } else if (hparams.minicpmv_version == 100045) { - hparams.minicpmv_query_num = 64; - } else { - hparams.minicpmv_query_num = 96; - } - } - } else if (is_audio) { - get_u32(KEY_A_NUM_MEL_BINS, hparams.n_mel_bins); - // some hparams are unused, but still need to set to avoid issues - hparams.image_size = 0; - hparams.patch_size = 1; - - } else if (is_gen_audio) { - // these are unused, but still need to be set to avoid issues - hparams.image_size = 0; - hparams.patch_size = 1; - get_string(KEY_GEN_AUDIO_VARIANT, hparams.gen_model_variant, false); - - } else { - GGML_ASSERT(false && "unknown modality"); - } - - // for pinpoints, we need to convert it into a list of resolution candidates - { - std::vector pinpoints; - get_arr_int(KEY_IMAGE_GRID_PINPOINTS, pinpoints, false); - if (pinpoints.size() % 2 != 0) { - throw std::runtime_error(string_format("%s: image_grid_pinpoints must have an even number of elements, got %zu\n", __func__, pinpoints.size())); - } - if (!pinpoints.empty()) { - for (size_t i = 0; i < pinpoints.size(); i += 2) { - hparams.image_res_candidates.push_back({ - pinpoints[i], - pinpoints[i+1], - }); - } - } - } - - // default warmup value - hparams.warmup_image_size = hparams.image_size; - - { - bool use_gelu = false; - bool use_silu = false; - get_bool(KEY_USE_GELU, use_gelu, false); - get_bool(KEY_USE_SILU, use_silu, false); - if (use_gelu && use_silu) { - throw std::runtime_error(string_format("%s: both use_gelu and use_silu are set to true\n", __func__)); - } - if (use_gelu) { - hparams.ffn_op = FFN_GELU; - log_ffn_op = "gelu"; - } else if (use_silu) { - hparams.ffn_op = FFN_SILU; - log_ffn_op = "silu"; - } else { - hparams.ffn_op = FFN_GELU_QUICK; - log_ffn_op = "gelu_quick"; - } - } - - { - std::string mm_patch_merge_type; - get_string(KEY_MM_PATCH_MERGE_TYPE, mm_patch_merge_type, false); - if (mm_patch_merge_type == "spatial_unpad") { - hparams.mm_patch_merge_type = PATCH_MERGE_SPATIAL_UNPAD; - } - } - - if (is_vision) { - std::vector image_mean; - std::vector image_std; - get_arr_f32(KEY_IMAGE_MEAN, image_mean, false); - get_arr_f32(KEY_IMAGE_STD , image_std, false); - if (image_mean.size() < 3 || image_std.size() < 3) { - throw std::runtime_error(string_format("%s: image_mean/image_std arrays must have at least 3 elements, got %zu and %zu\n", __func__, image_mean.size(), image_std.size())); - } - for (int i = 0; i < 3; ++i) { - hparams.image_mean[i] = image_mean[i]; - hparams.image_std[i] = image_std[i]; - } - } - - // Load the vision/audio feature layer indices if they are explicitly provided - // NOTE: gguf conversions should standardize the values of the vision feature layer to be non-negative, since we use -1 to mark values as unset here. - get_arr_int(string_format(KEY_FEATURE_LAYERS, prefix), hparams.feature_layers, false); - for (const auto & v : hparams.feature_layers) { - if (v > (int) hparams.n_layer) { - throw std::runtime_error(string_format("%s: feature layer index %d is out of range (n_layer: %d)", - __func__, v, hparams.n_layer)); - } - } - - // model-specific params - switch (model.proj_type) { - case PROJECTOR_TYPE_MLP: - case PROJECTOR_TYPE_MLP_NORM: - case PROJECTOR_TYPE_LDP: - case PROJECTOR_TYPE_LDPV2: - case PROJECTOR_TYPE_COGVLM: - { - hparams.has_llava_projector = model.proj_type != PROJECTOR_TYPE_COGVLM; - hparams.image_pad_color = {122, 116, 104}; - if (!hparams.image_res_candidates.empty()) { - hparams.image_resize_pad = PAD_CEIL; - hparams.image_resize_algo = RESIZE_ALGO_BICUBIC; - } else { - // llava-1.6 default params - hparams.image_pad_ov = PAD_NONE; - hparams.image_pad_rf = PAD_CEIL; - hparams.image_pad_color_rf = {122, 116, 104}; - } - } break; - case PROJECTOR_TYPE_GLM_EDGE: - { - hparams.image_resize_pad = PAD_CEIL; - hparams.image_resize_algo = RESIZE_ALGO_BICUBIC; - } break; - case PROJECTOR_TYPE_MINICPMV: - { - // use default llava-uhd preprocessing params - if (hparams.minicpmv_version == 0) { - hparams.minicpmv_version = 2; // default to 2 if not set - } - } break; - case PROJECTOR_TYPE_MINICPMV4_6: - case PROJECTOR_TYPE_MINICPMV4_7: - case PROJECTOR_TYPE_MINICPM_OCR_1_0: - { - // MiniCPM-V 4.6/4.7 unified merger projector - // ViT merger 2x2 + final merger 2x2 = 4x spatial merge per dimension - hparams.n_merge = 4; - get_u32(KEY_PROJ_SCALE_FACTOR, hparams.n_merge, false); - GGML_ASSERT(hparams.n_merge == 2 || hparams.n_merge == 4); - - // no padding: the reference stretches the refined image to the target size - hparams.image_pad_ov = PAD_NONE; - hparams.image_pad_rf = PAD_NONE; - - // borrow wa_layer_indexes for vit_merger insertion point - std::vector wa_layer_indexes_vec; - get_arr_int(KEY_WIN_ATTN_LAYER_INDEXES, wa_layer_indexes_vec, false); - if (!wa_layer_indexes_vec.empty()) { - const int insert_lid = wa_layer_indexes_vec[0]; - if (insert_lid < 0 || insert_lid >= (int) hparams.n_layer) { - throw std::runtime_error(string_format("%s: layer index %d is out of range (n_layer: %d)", - __func__, insert_lid, hparams.n_layer)); - } - hparams.insert_layer_id = insert_lid; - } - } break; - case PROJECTOR_TYPE_INTERNVL: - { - // use default llava-uhd preprocessing params - // older version of internvl doesn't have min/max tiles, we need to provide default values for them to avoid issues - hparams.preproc_min_tiles = 1; - hparams.preproc_max_tiles = 12; - get_u32(KEY_PROJ_SCALE_FACTOR, hparams.n_merge, false); - get_u32(KEY_PREPROC_MIN_TILES, hparams.preproc_min_tiles, false); - get_u32(KEY_PREPROC_MAX_TILES, hparams.preproc_max_tiles, false); - GGML_ASSERT(hparams.preproc_min_tiles <= hparams.preproc_max_tiles && hparams.preproc_max_tiles < INT32_MAX); - set_internvl_dhr_res_candidates(model); - } break; - case PROJECTOR_TYPE_NEMOTRON_V2_VL: - { - get_u32(KEY_PROJ_SCALE_FACTOR, hparams.n_merge, false); - } break; - case PROJECTOR_TYPE_NEMOTRON_OMNI: - { - hparams.ffn_op = FFN_GELU_ERF; - log_ffn_op = "gelu_erf"; - hparams.image_resize_pad = PAD_NONE; - get_u32(KEY_PROJ_SCALE_FACTOR, hparams.n_merge); - get_u32(KEY_IMAGE_MIN_PIXELS, hparams.image_min_pixels); - get_u32(KEY_IMAGE_MAX_PIXELS, hparams.image_max_pixels); - hparams.warmup_image_size = hparams.image_size; - } break; - case PROJECTOR_TYPE_PARAKEET: - { - get_u32(KEY_AUDIO_SUBSMPL_FACTOR, hparams.subsampling_factor); - GGML_ASSERT(hparams.subsampling_factor == 8 && - "subsampling_factor must match the conv strides in clip_graph_parakeet::build()"); - get_u32(KEY_A_CONV_KERNEL_SIZE, hparams.audio_conv_kernel_size); - GGML_ASSERT(hparams.audio_conv_kernel_size > 0 && hparams.audio_conv_kernel_size % 2 == 1 && - "audio_conv_kernel_size must be a positive odd integer"); - hparams.audio_chunk_len = 0; - hparams.audio_sample_rate = 16000; - hparams.audio_n_fft = 512; - hparams.audio_window_len = 400; - hparams.audio_hop_len = 160; - } break; - case PROJECTOR_TYPE_IDEFICS3: - { - // use default llava-uhd preprocessing params - hparams.image_resize_algo = RESIZE_ALGO_LANCZOS; - get_u32(KEY_PROJ_SCALE_FACTOR, hparams.n_merge, false); - get_u32(KEY_PREPROC_IMAGE_SIZE, hparams.image_longest_edge, false); - hparams.set_limit_image_tokens(); - } break; - case PROJECTOR_TYPE_COHERE2V: - { - hparams.image_pad_rf = PAD_NONE; - get_u32(KEY_PROJ_SCALE_FACTOR, hparams.n_merge); - get_u32(KEY_PREPROC_MAX_TILES, hparams.preproc_max_tiles); - if (hparams.preproc_max_tiles <= 0 || hparams.preproc_max_tiles > 256) { - throw std::runtime_error(string_format("%s: preproc_max_tiles (%d) must be in range [1, 256]\n", __func__, hparams.preproc_max_tiles)); - } - } break; - case PROJECTOR_TYPE_LFM2: - case PROJECTOR_TYPE_D1OMNI_V: - { - // default for older GGUFs - std::string resize_algo = "bilinear"; - get_string(KEY_IMAGE_RESIZE_ALGO, resize_algo, false); - // Lfm2VlImageProcessor and d1-omni vision.py resize bilinear and bicubic with torchvision - if (resize_algo == "bilinear") { - hparams.set_image_resize_algo(RESIZE_ALGO_BILINEAR, true); - } else if (resize_algo == "bicubic") { - hparams.set_image_resize_algo(RESIZE_ALGO_BICUBIC, true); - } else if (resize_algo == "lanczos") { - hparams.set_image_resize_algo(RESIZE_ALGO_LANCZOS); - } else { - throw std::runtime_error(string_format("%s: unsupported image resize algo: %s\n", __func__, resize_algo.c_str())); - } - hparams.image_resize_algo_rf = hparams.image_resize_algo; - hparams.image_resize_algo_ov = hparams.image_resize_algo; - // the tiles stretch the image to the grid (Lfm2VlImageProcessor, d1-omni vision.py) - hparams.image_pad_rf = PAD_NONE; - get_u32(KEY_PROJ_SCALE_FACTOR, hparams.n_merge, false); - // defaults for GGUFs converted before tiling params were written - // ref: https://huggingface.co/LiquidAI/LFM2.5-VL-1.6B/blob/main/processor_config.json - hparams.preproc_min_tiles = 2; - hparams.preproc_max_tiles = 10; - hparams.preproc_tile_size = 512; - hparams.preproc_max_pixels_tolerance = 2.0f; - get_u32(KEY_PREPROC_MIN_TILES, hparams.preproc_min_tiles, false); - get_u32(KEY_PREPROC_MAX_TILES, hparams.preproc_max_tiles, false); - get_u32(KEY_PREPROC_TILE_SIZE, hparams.preproc_tile_size, false); - get_f32(KEY_PREPROC_MAX_PIXELS_TOLERANCE, hparams.preproc_max_pixels_tolerance, false); - GGML_ASSERT(hparams.preproc_min_tiles >= 1 && hparams.preproc_min_tiles <= hparams.preproc_max_tiles); - hparams.set_limit_image_tokens(64, 256); - } break; - case PROJECTOR_TYPE_PHI4: - { - hparams.n_merge = 1; - hparams.image_resize_algo = RESIZE_ALGO_BILINEAR; - get_u32(KEY_IMAGE_MIN_PIXELS, hparams.image_min_pixels); - get_u32(KEY_IMAGE_MAX_PIXELS, hparams.image_max_pixels); - hparams.set_warmup_n_tokens(16*16); - } break; - case PROJECTOR_TYPE_PIXTRAL: - { - // ref: https://huggingface.co/mistral-community/pixtral-12b/blob/main/preprocessor_config.json - // TODO: verify the image_min_tokens - hparams.n_merge = 1; // the original pixtral does not use patch merging - hparams.image_resize_algo = RESIZE_ALGO_BICUBIC; - hparams.rope_theta = 10000.0f; - get_u32(KEY_SPATIAL_MERGE_SIZE, hparams.n_merge, false); - get_u32(KEY_PREPROC_IMAGE_SIZE, hparams.image_longest_edge, false); - if (hparams.image_longest_edge > 0) { - const int64_t patch_size = int64_t(hparams.patch_size) * hparams.n_merge; - if (patch_size <= 0) { - throw std::runtime_error("Pixtral merged patch size must be positive"); - } - const int64_t n_tokens_per_side = - (hparams.image_longest_edge + patch_size - 1) / patch_size; - if (n_tokens_per_side * patch_size > std::sqrt(std::numeric_limits::max())) { - throw std::runtime_error("Pixtral preprocessing image size is too large"); - } - hparams.image_resize_pad = PAD_NONE; - hparams.image_floor_before_align = true; - hparams.set_limit_image_tokens(0, n_tokens_per_side * n_tokens_per_side); - } else { - hparams.set_limit_image_tokens(8, 1024); - } - hparams.set_warmup_n_tokens(256); // avoid OOM on warmup - } break; - case PROJECTOR_TYPE_LIGHTONOCR: - { - hparams.n_merge = 1; - hparams.image_resize_algo = RESIZE_ALGO_BICUBIC; - hparams.rope_theta = 10000.0f; - get_u32(KEY_SPATIAL_MERGE_SIZE, hparams.n_merge, false); - hparams.image_longest_edge = hparams.image_size; - get_u32(KEY_PREPROC_IMAGE_SIZE, hparams.image_longest_edge, false); - hparams.set_limit_image_tokens(); - hparams.set_warmup_n_tokens(256); // avoid OOM on warmup - } break; - case PROJECTOR_TYPE_DOTS_OCR: - { - hparams.rope_theta = 10000.0f; - hparams.image_resize_pad = PAD_NONE; // the HF processor resizes to the smart_resize size, no padding - hparams.image_round_half_even = true; // smart_resize uses Python round() - get_u32(KEY_PROJ_SCALE_FACTOR, hparams.n_merge); - get_u32(KEY_IMAGE_MIN_PIXELS, hparams.image_min_pixels); - get_u32(KEY_IMAGE_MAX_PIXELS, hparams.image_max_pixels); - hparams.set_warmup_n_tokens(46*46); // avoid OOM on warmup - } break; - case PROJECTOR_TYPE_DOTS3NOTE_V: - { - hparams.rope_theta = 10000.0f; - hparams.image_resize_algo = RESIZE_ALGO_BICUBIC; - get_u32(KEY_SPATIAL_MERGE_SIZE, hparams.n_merge); - get_u32(KEY_IMAGE_MIN_PIXELS, hparams.image_min_pixels); - get_u32(KEY_IMAGE_MAX_PIXELS, hparams.image_max_pixels); - get_u32(KEY_VISION_N_EXPERT_USED, hparams.n_expert_used); - hparams.set_warmup_n_tokens(46*46); // avoid OOM on warmup - } break; - case PROJECTOR_TYPE_DOTS3NOTE_A: - { - hparams.rope_theta = 10000.0f; - hparams.audio_chunk_len = 60; // in seconds - hparams.audio_sample_rate = 16000; - hparams.audio_n_fft = 400; - hparams.audio_window_len = 400; - hparams.audio_hop_len = 160; - } break; - case PROJECTOR_TYPE_KIMIVL: - { - hparams.image_resize_algo = RESIZE_ALGO_BICUBIC; - hparams.rope_theta = 10000.0f; - get_u32(KEY_PROJ_SCALE_FACTOR, hparams.n_merge, false); - // TODO: check kimivl preprocessor for exact values - hparams.set_limit_image_tokens(8, 1024); - hparams.set_warmup_n_tokens(256); // avoid OOM on warmup - } break; - case PROJECTOR_TYPE_KIMIK25: - { - hparams.image_resize_algo = RESIZE_ALGO_BICUBIC; - hparams.rope_theta = 10000.0f; - get_u32(KEY_PROJ_SCALE_FACTOR, hparams.n_merge, false); - - int min_pixels = 0, max_pixels = 0; - get_u32(KEY_IMAGE_MIN_PIXELS, min_pixels, false); - get_u32(KEY_IMAGE_MAX_PIXELS, max_pixels, false); - if (min_pixels > 0 && max_pixels > 0) { - hparams.image_min_pixels = min_pixels; - hparams.image_max_pixels = max_pixels; - hparams.warmup_image_size = static_cast(std::sqrt(max_pixels)); - } else { - hparams.set_limit_image_tokens(2, 4096); - } - } break; - case PROJECTOR_TYPE_DEEPSEEK4V: - { - hparams.image_resize_algo = RESIZE_ALGO_BICUBIC; - hparams.image_pad_color = {127, 127, 127}; - hparams.rope_theta = 10000.0f; - get_u32(KEY_PROJ_SCALE_FACTOR, hparams.n_merge); - get_u32(KEY_IMAGE_MIN_PIXELS, hparams.image_min_pixels); - hparams.dsv4_max_n_token = 384; - hparams.dsv4_max_wh_ratio = 8; - const int patch_area = hparams.patch_size * hparams.patch_size * hparams.n_merge * hparams.n_merge; - // handle min/max token counts from CLI - if (hparams.custom_image_min_tokens > 0) { - hparams.image_min_pixels = hparams.custom_image_min_tokens * patch_area; - } - if (hparams.custom_image_max_tokens > 0) { - // the cap is on the whole token block, keep some room for the resize solver - hparams.dsv4_max_n_token = std::max(hparams.custom_image_max_tokens, 16); - } - hparams.image_max_pixels = hparams.dsv4_max_n_token * patch_area; - // a small custom max token count also lowers the min-pixel upscale threshold - hparams.image_min_pixels = std::min(hparams.image_min_pixels, hparams.image_max_pixels); - // avoid OOM on warmup - const int warmup_side = (int) std::sqrt((double) std::min(256, hparams.dsv4_max_n_token)); - hparams.set_warmup_n_tokens(warmup_side * warmup_side); - } break; - case PROJECTOR_TYPE_DEEPSEEK41V: - { - // ref: load_image in inference/image_processor.py (V4.1), PIL ImageOps.pad on gray 127 - hparams.image_resize_algo = RESIZE_ALGO_BICUBIC; - hparams.image_pad_color = {127, 127, 127}; - hparams.rope_theta = 10000.0f; - get_u32(KEY_PROJ_SCALE_FACTOR, hparams.n_merge); - get_u32(KEY_IMAGE_MIN_PIXELS, hparams.image_min_pixels); - // vision_max_n_token = 1024, vision_max_wh_ratio = null (converter asserts both) - hparams.dsv4_max_n_token = 1024; - hparams.dsv4_max_wh_ratio = 0; - const int patch_area = hparams.patch_size * hparams.patch_size * hparams.n_merge * hparams.n_merge; - if (hparams.custom_image_min_tokens > 0) { - hparams.image_min_pixels = hparams.custom_image_min_tokens * patch_area; - } - if (hparams.custom_image_max_tokens > 0) { - // the cap is on the whole token block (newlines + start/end included) - hparams.dsv4_max_n_token = std::max(hparams.custom_image_max_tokens, 16); - } - hparams.image_max_pixels = hparams.dsv4_max_n_token * patch_area; - hparams.image_min_pixels = std::min(hparams.image_min_pixels, hparams.image_max_pixels); - const int warmup_side = (int) std::sqrt((double) std::min(256, hparams.dsv4_max_n_token)); - hparams.set_warmup_n_tokens(warmup_side * warmup_side); - } break; - case PROJECTOR_TYPE_GEMMA3: - { - // default value (used by all model sizes in gemma 3 family) - // number of patches for each **side** is reduced by a factor of 4 - hparams.n_merge = 4; - hparams.image_resize_algo = RESIZE_ALGO_BILINEAR; - // test model (tinygemma3) has a different value, we optionally read it - get_u32(KEY_PROJ_SCALE_FACTOR, hparams.n_merge, false); - } break; - - case PROJECTOR_TYPE_GEMMA4V: - case PROJECTOR_TYPE_GEMMA4UV: - { - hparams.rope_theta = 100.0f; - hparams.ffn_op = FFN_GELU; - log_ffn_op = "gelu"; - hparams.n_merge = 3; // pooling_kernel_size - hparams.image_resize_algo = RESIZE_ALGO_BICUBIC; - hparams.image_resize_pad = PAD_NONE; - get_u32(KEY_PROJ_SCALE_FACTOR, hparams.n_merge, false); - if (model.proj_type == PROJECTOR_TYPE_GEMMA4UV) { - // for "unified" variant, we directly use a bigger patch size, because the "token merging" is done directly on conv layer - hparams.patch_size = hparams.patch_size * hparams.n_merge; - hparams.n_merge = 1; - } - hparams.set_limit_image_tokens(70, 1120); - hparams.set_warmup_n_tokens(256); // avoid OOM on warmup - } break; - - case PROJECTOR_TYPE_GEMMA3NV: - { - // Gemma3n uses MobileNetV5 which produces 256 tokens (16x16) - // Similar configuration to Gemma3 - hparams.n_merge = 1; // MobileNetV5 handles resizing internally - hparams.image_resize_algo = RESIZE_ALGO_BILINEAR; - get_u32(KEY_PROJ_SCALE_FACTOR, hparams.n_merge, false); - } break; - case PROJECTOR_TYPE_QWEN2VL: - case PROJECTOR_TYPE_QWEN25VL: - case PROJECTOR_TYPE_QWEN3VL: - case PROJECTOR_TYPE_LING3VL: - { - hparams.n_merge = 2; // default value for Qwen 2 and 2.5 - hparams.image_resize_algo = RESIZE_ALGO_BICUBIC; - hparams.image_resize_pad = PAD_NONE; // the HF processor resizes to the smart_resize size, no padding - hparams.image_round_half_even = true; // smart_resize uses Python round() - get_u32(KEY_SPATIAL_MERGE_SIZE, hparams.n_merge, false); - get_u32(KEY_WIN_ATTN_PATTERN, hparams.n_wa_pattern, model.proj_type == PROJECTOR_TYPE_QWEN25VL); // only 2.5 requires it - // ref: https://huggingface.co/Qwen/Qwen2.5-VL-7B-Instruct/blob/main/preprocessor_config.json - hparams.set_limit_image_tokens(8, 4096); - // optional limits of the model, the custom values take precedence - if (hparams.custom_image_min_tokens <= 0) { - get_u32(KEY_IMAGE_MIN_PIXELS, hparams.image_min_pixels, false); - } - if (hparams.custom_image_max_tokens <= 0) { - get_u32(KEY_IMAGE_MAX_PIXELS, hparams.image_max_pixels, false); - } - hparams.set_warmup_n_tokens(46*46); // avoid OOM on warmup - const int warn_min_pixels = 1024 * hparams.n_merge * hparams.n_merge * hparams.patch_size * hparams.patch_size; - if (hparams.image_min_pixels < warn_min_pixels) { - LOG_WRN("%s: Qwen-VL models require at minimum 1024 image tokens to function correctly on grounding tasks\n", __func__); - LOG_WRN("%s: if you encounter problems with accuracy, try adding --image-min-tokens 1024\n", __func__); - LOG_WRN("%s: more info: https://github.com/ggml-org/llama.cpp/issues/16842\n\n", __func__); - } - } break; - case PROJECTOR_TYPE_SENSENOVA_U1: - { - hparams.n_merge = 2; - hparams.image_resize_algo = RESIZE_ALGO_BICUBIC; - get_u32(KEY_SPATIAL_MERGE_SIZE, hparams.n_merge, false); - get_u32(KEY_IMAGE_MIN_PIXELS, hparams.image_min_pixels); - get_u32(KEY_IMAGE_MAX_PIXELS, hparams.image_max_pixels); - hparams.rope_theta = 10000.0f; - hparams.set_limit_image_tokens(16, 4096); - if (hparams.n_merge != 2) { - throw std::runtime_error(string_format("%s: SenseNova U1 requires spatial_merge_size=2, got %u", __func__, hparams.n_merge)); - } - hparams.set_warmup_n_tokens(16*16); - } break; - case PROJECTOR_TYPE_LFM3VL: - { - std::string resize_algo; - get_string(KEY_IMAGE_RESIZE_ALGO, resize_algo); - // Lfm3VlImageProcessor resizes with torchvision - if (resize_algo == "bilinear") { - hparams.set_image_resize_algo(RESIZE_ALGO_BILINEAR, true); - } else if (resize_algo == "bicubic") { - hparams.set_image_resize_algo(RESIZE_ALGO_BICUBIC, true); - } else { - throw std::runtime_error(string_format("%s: unsupported image resize algo: %s\n", __func__, resize_algo.c_str())); - } - // smart_resize stretches to the aligned size, no letterbox - hparams.image_resize_pad = PAD_NONE; - get_u32(KEY_SPATIAL_MERGE_SIZE, hparams.n_merge); - // token counts and the projector graph assume a 2x2 merge - if (hparams.n_merge != 2) { - throw std::runtime_error(string_format("%s: unsupported spatial_merge_size %d for lfm3vl, only 2 is supported\n", __func__, hparams.n_merge)); - } - get_u32(string_format(KEY_N_HEAD_KV, "vision"), hparams.n_head_kv); - hparams.rope_theta = 1000000.0f; // vision_rope_theta - // HF processor size; the max is also capped at 4096 tokens as an OOM guard - int min_pixels = 0; - int max_pixels = 0; - get_u32(KEY_IMAGE_MIN_PIXELS, min_pixels); - get_u32(KEY_IMAGE_MAX_PIXELS, max_pixels); - if (min_pixels <= 0 || max_pixels < min_pixels) { - throw std::runtime_error(string_format("%s: invalid image min/max pixels %d/%d\n", __func__, min_pixels, max_pixels)); - } - const int patch_area = hparams.patch_size * hparams.patch_size * hparams.n_merge * hparams.n_merge; - hparams.set_limit_image_tokens(min_pixels / patch_area, std::min(4096, max_pixels / patch_area)); - hparams.set_warmup_n_tokens(46*46); // avoid OOM on warmup - } break; - case PROJECTOR_TYPE_MINIMAX_M3: - { - hparams.n_merge = 2; // spatial_merge_size - hparams.image_resize_algo = RESIZE_ALGO_BICUBIC; - hparams.image_resize_pad = PAD_NONE; - get_u32(KEY_SPATIAL_MERGE_SIZE, hparams.n_merge, false); - // n_merge is used as a divisor in clip_image_batch_encode - // (gh / n_merge); reject 0 to avoid int div-by-zero (DoS). - GGML_ASSERT(hparams.n_merge > 0); - hparams.rope_theta = 10000.0f; // vision_config.rope_theta - // MiniMax-M3: max_pixels 451584 (=672^2) -> 576 merged tokens (image_seq_length) - hparams.set_limit_image_tokens(8, 576); - hparams.set_warmup_n_tokens(16*16); - } break; - case PROJECTOR_TYPE_MUSE_SPARK_AUDIO: - { - hparams.audio_sample_rate = 24000; // mono waveform, no frontend - get_u32(KEY_A_CONV_KERNEL_SIZE, hparams.audio_conv_kernel_size, true); - get_arr_int(KEY_A_HIDDEN_SIZES, hparams.audio_hidden_sizes, true); - get_arr_int(KEY_A_PATCH_SIZES, hparams.audio_patch_sizes, true); - get_arr_int(KEY_A_DEPTHS, hparams.audio_depths, true); - - // the graph indexes the three arrays by stage and model.layers by the running depth sum - const size_t n_stage = hparams.audio_patch_sizes.size(); - GGML_ASSERT(n_stage > 0); - GGML_ASSERT(hparams.audio_hidden_sizes.size() == n_stage); - GGML_ASSERT(hparams.audio_depths.size() == n_stage); - GGML_ASSERT(hparams.n_mel_bins == 1); - GGML_ASSERT(hparams.audio_conv_kernel_size >= 1); - - int64_t n_blocks = 0; - for (size_t i = 0; i < n_stage; ++i) { - GGML_ASSERT(hparams.audio_hidden_sizes[i] >= 1); - GGML_ASSERT(hparams.audio_depths[i] >= 0); - n_blocks += hparams.audio_depths[i]; - } - if (n_blocks != hparams.n_layer) { - throw std::runtime_error(string_format("%s: sum of audio depths (%" PRId64 ") does not match n_layer (%d)\n", __func__, n_blocks, hparams.n_layer)); - } - hparams.warmup_audio_size = hparams.audio_samples_per_token(); - } break; - case PROJECTOR_TYPE_MUSE_GLIMMER: - { - hparams.n_merge = 2; // pixel-shuffle downsample after the ViT - hparams.image_resize_algo = RESIZE_ALGO_LANCZOS; - hparams.rope_theta = 10000.0f; - hparams.muse_glimmer_patch_temporal = 2; - hparams.muse_glimmer_sparse_factor = 4; // 3 sparse layers + 1 global, repeating - get_u32(KEY_SPATIAL_MERGE_SIZE, hparams.n_merge, false); - hparams.set_limit_image_tokens(1, 4096); - hparams.set_warmup_n_tokens(32*32); - } break; - case PROJECTOR_TYPE_MIMOVL: - { - hparams.n_merge = 2; // spatial_merge_size - hparams.image_resize_algo = RESIZE_ALGO_BICUBIC; - hparams.image_resize_pad = PAD_NONE; // the HF processor resizes to the smart_resize size, no padding - hparams.image_round_half_even = true; // smart_resize uses Python round() - get_u32(KEY_SPATIAL_MERGE_SIZE, hparams.n_merge, false); - get_u32(string_format(KEY_N_HEAD_KV, "vision"), hparams.n_head_kv); - // 1D banded sliding-window radius (visual_token_window_size); required - get_u32(KEY_ATTN_WINDOW_SIZE, hparams.attn_window_size); - std::vector pat; - get_arr_int(KEY_WA_PATTERN_MODE, pat, true); - GGML_ASSERT((int) pat.size() == hparams.n_layer && "mimovl wa_pattern_mode length must equal n_layer"); - hparams.wa_pattern_mode.assign(pat.begin(), pat.end()); - get_u32(KEY_IMAGE_MIN_PIXELS, hparams.image_min_pixels); - get_u32(KEY_IMAGE_MAX_PIXELS, hparams.image_max_pixels); - hparams.set_warmup_n_tokens(46*46); // avoid OOM on warmup - } break; - case PROJECTOR_TYPE_STEP3VL: - { - hparams.n_merge = 4; // two stride-2 downsamplers after patching - get_u32(KEY_PROJ_SCALE_FACTOR, hparams.n_merge, false); - hparams.rope_theta = 10000.0f; - get_u32(KEY_PREPROC_IMAGE_SIZE, hparams.image_longest_edge, false); - if (hparams.image_longest_edge == 0) { - hparams.image_longest_edge = 3024; - } - // note: the step3vl preprocessor slices based on a fixed window grid, so it does not support custom min/max image tokens - hparams.warmup_image_size = hparams.image_size; - } break; - case PROJECTOR_TYPE_YOUTUVL: - { - hparams.n_merge = 2; - hparams.image_resize_algo = RESIZE_ALGO_BILINEAR; - hparams.image_resize_pad = PAD_NONE; - get_u32(KEY_SPATIAL_MERGE_SIZE, hparams.n_merge, false); - get_u32(KEY_ATTN_WINDOW_SIZE, hparams.attn_window_size, true); - std::vector wa_layer_indexes_vec; - get_arr_int(KEY_WIN_ATTN_LAYER_INDEXES, wa_layer_indexes_vec, true); - for (auto & layer : wa_layer_indexes_vec) { - hparams.wa_layer_indexes.insert(layer); - } - // support max_height * max_width = 8000 * 8000. 8000/16/2 = 250 image tokens - hparams.set_limit_image_tokens(1, 62500); - hparams.set_warmup_n_tokens(16*16); // avoid OOM on warmup - } break; - case PROJECTOR_TYPE_YASA2: - { - hparams.ffn_op = FFN_GELU_ERF; - log_ffn_op = "gelu_erf"; - hparams.image_resize_algo = RESIZE_ALGO_BICUBIC; - - // reka model performs better when the image is stretched to fit - // fixed square size (no padding) - hparams.image_resize_pad = PAD_NONE; - } break; - case PROJECTOR_TYPE_GLM4V: - { - hparams.rope_theta = 10000.0f; - hparams.n_merge = 2; // default value for GLM4-V - hparams.image_resize_algo = RESIZE_ALGO_BICUBIC; - get_u32(KEY_SPATIAL_MERGE_SIZE, hparams.n_merge, false); - hparams.set_limit_image_tokens(8, 4096); - hparams.set_warmup_n_tokens(46*46); // avoid OOM on warmup - } break; - case PROJECTOR_TYPE_GLM5V: - { - // glm4v tower with clamped SwiGLU, ceil-aligned resize and its own token budget - hparams.rope_theta = 10000.0f; - hparams.n_merge = 2; - hparams.image_resize_algo = RESIZE_ALGO_BICUBIC; - get_u32(KEY_SPATIAL_MERGE_SIZE, hparams.n_merge, false); - get_f32(KEY_SWIGLU_CLAMP, hparams.swiglu_clamp, true); - get_u32(KEY_IMAGE_MIN_PIXELS, hparams.image_min_pixels); - get_u32(KEY_IMAGE_MAX_PIXELS, hparams.image_max_pixels); - hparams.set_limit_image_tokens(); - hparams.set_warmup_n_tokens(46*46); // avoid OOM on warmup - } break; - case PROJECTOR_TYPE_LLAMA4: - { - hparams.rope_theta = 10000.0f; - hparams.image_resize_algo = RESIZE_ALGO_BILINEAR; - get_u32(KEY_PROJ_SCALE_FACTOR, hparams.n_merge, false); - set_llava_uhd_res_candidates(model, 3); - } break; - case PROJECTOR_TYPE_ULTRAVOX: - case PROJECTOR_TYPE_QWEN2A: - case PROJECTOR_TYPE_QWEN3A: - case PROJECTOR_TYPE_GLMA: - case PROJECTOR_TYPE_VOXTRAL: - case PROJECTOR_TYPE_MERALION: - case PROJECTOR_TYPE_MUSIC_FLAMINGO: - { - bool require_stack = model.proj_type == PROJECTOR_TYPE_ULTRAVOX || - model.proj_type == PROJECTOR_TYPE_VOXTRAL || - model.proj_type == PROJECTOR_TYPE_MERALION || - model.proj_type == PROJECTOR_TYPE_GLMA; - get_u32(KEY_A_PROJ_STACK_FACTOR, hparams.proj_stack_factor, require_stack); - hparams.ffn_op = FFN_GELU_ERF; - log_ffn_op = "gelu_erf"; // temporary solution for logging - - // audio preprocessing params - hparams.audio_chunk_len = 30; // in seconds - hparams.audio_sample_rate = 16000; - hparams.audio_n_fft = 400; - hparams.audio_window_len = 400; - hparams.audio_hop_len = 160; - } break; - case PROJECTOR_TYPE_VOXTRAL_RT: - { - get_u32(KEY_A_PROJ_STACK_FACTOR, hparams.proj_stack_factor); - get_u32(KEY_A_ATTN_WINDOW_SIZE, hparams.attn_window_size); - hparams.rope_theta = 1000000.0f; // params.json - hparams.ffn_op = FFN_SILU; - log_ffn_op = "silu"; - - // audio preprocessing params - hparams.audio_sample_rate = 16000; - hparams.audio_n_fft = 400; - hparams.audio_window_len = 400; - hparams.audio_hop_len = 160; - // 2 context frames + 8 s of new frames, see clip_enc_state_init() - hparams.warmup_audio_size = 2 + 2*400; - } break; - case PROJECTOR_TYPE_MIMO_AUDIO: - { - get_u32(KEY_A_RVQ_NUM_QUANTIZERS, hparams.rvq_num_quantizers, false); - get_arr_int(KEY_A_RVQ_CODEBOOK_SIZE, hparams.rvq_codebook_size, false); - if (hparams.rvq_num_quantizers <= 0) { - throw std::runtime_error(string_format("%s: mimo_audio: missing %s\n", __func__, KEY_A_RVQ_NUM_QUANTIZERS)); - } - if ((int) hparams.rvq_codebook_size.size() != hparams.rvq_num_quantizers) { - throw std::runtime_error(string_format( - "%s: mimo_audio: %s length (%zu) must equal %s (%d)\n", __func__, - KEY_A_RVQ_CODEBOOK_SIZE, hparams.rvq_codebook_size.size(), - KEY_A_RVQ_NUM_QUANTIZERS, hparams.rvq_num_quantizers)); - } - hparams.ffn_op = FFN_GELU_ERF; // PyTorch F.gelu default (approximate="none") - hparams.rope_theta = 10000.0f; - - // audio preprocessing params (mel spectrogram) - hparams.audio_sample_rate = 24000; - hparams.audio_n_fft = 960; - hparams.audio_window_len = 960; - hparams.audio_hop_len = 240; - - get_u32(KEY_A_ATTN_WINDOW_SIZE, hparams.attn_window_size); - std::vector wa_pattern; - get_arr_int(KEY_A_WA_PATTERN_MODE, wa_pattern, true); - if ((int) wa_pattern.size() != hparams.n_layer) { - throw std::runtime_error(string_format( - "%s: mimo_audio: %s length (%zu) must equal n_layer (%d)\n", __func__, - KEY_A_WA_PATTERN_MODE, wa_pattern.size(), hparams.n_layer)); - } - hparams.wa_pattern_mode.assign(wa_pattern.begin(), wa_pattern.end()); - - get_u32(KEY_A_LOCAL_BLOCK_COUNT, hparams.audio_local_n_layer); - get_u32(KEY_A_LOCAL_GROUP_SIZE, hparams.audio_local_group_size); - if (hparams.audio_local_group_size <= 0) { - throw std::runtime_error(string_format( - "%s: mimo_audio: %s must be > 0\n", __func__, KEY_A_LOCAL_GROUP_SIZE)); - } - } break; - case PROJECTOR_TYPE_QWEN3TTS_SPKENC: - { - // ECAPA-TDNN speaker encoder, mel front-end uses the Slaney default (fmin=0, fmax=sr/2) - hparams.audio_sample_rate = 24000; - hparams.audio_n_fft = 1024; - hparams.audio_window_len = 1024; - hparams.audio_hop_len = 256; - } break; - case PROJECTOR_TYPE_LFM3A_GEN: - { - // fixed by LiquidCodecConfig, head counts come from the tensors - hparams.rope_theta = 1000000.0f; - - hparams.wav_tfm_n_layer = 8; - hparams.wav_tfm_n_embd = 512; - hparams.wav_tfm_eps = 1e-5f; - hparams.wav_tfm_rope_theta = 1000000.0f; - hparams.wav_tfm_swa = 30; // decoder_window - hparams.wav_frame_stack = 6; // stft_frame_stack, STFT frames per codec frame - // the ISTFT runs on the host, see mtmd_gen_audio_process() - hparams.audio_sample_rate = 24000; - hparams.audio_n_fft = 1280; - hparams.audio_window_len = 1280; - hparams.audio_hop_len = 320; - - // speech-code sampling, the converter writes the model card values; else the LFM3-Omni card - hparams.gen_temp = 0.4f; - hparams.gen_top_k = 0; - hparams.gen_penalty_repeat = 1.2f; - hparams.gen_penalty_last_n = 64; - get_f32(KEY_GEN_AUDIO_TEMP, hparams.gen_temp, false); - get_i32(KEY_GEN_AUDIO_TOP_K, hparams.gen_top_k, false); - get_f32(KEY_GEN_AUDIO_PENALTY_REPEAT, hparams.gen_penalty_repeat, false); - get_i32(KEY_GEN_AUDIO_PENALTY_LAST_N, hparams.gen_penalty_last_n, false); - } break; - case PROJECTOR_TYPE_QWEN3TTS_GEN: - { - // TODO: hardcoded for now, read from code_predictor_config instead - hparams.rope_theta = 1000000.0f; - - // code2wav params - hparams.wav_tfm_n_layer = 8; - hparams.wav_tfm_n_embd = 512; - hparams.wav_tfm_n_ff = 1024; - hparams.wav_tfm_n_head = 16; - hparams.wav_tfm_n_head_kv = 16; - hparams.wav_tfm_eps = 1e-5f; - hparams.wav_tfm_rope_theta = 10000.0f; - hparams.wav_upsample_n_block = 2; - hparams.wav_dac_n_block = 4; - hparams.wav_dac_n_res = 3; - // matches the reference decoder's sliding_window (speech_tokenizer/config.json) - hparams.wav_tfm_swa = 72; - } break; - case PROJECTOR_TYPE_POCKETTTS_SPKENC: - case PROJECTOR_TYPE_POCKETTTS_GEN: - { - // mimi front-end takes the raw waveform, no mel - hparams.audio_sample_rate = 24000; - // seanet ratios are [6,5,4] in the config, the encoder reverses them - hparams.seanet_ratios = { 4, 5, 6 }; - hparams.seanet_n_stage = (int32_t) hparams.seanet_ratios.size(); - hparams.mimi_downsample = 16; - // matches the reference transformer's "context" - hparams.mimi_tfm_context = 250; - hparams.rope_theta = 10000.0f; - // flow_lm defaults, see pocket_tts/default_parameters.py - hparams.flow_n_step = 1; - hparams.gen_eos_threshold = -4.0f; - } break; - case PROJECTOR_TYPE_SOPRANO: - { - // SopranoDecoder defaults, n_fft comes from the head tensor - hparams.audio_sample_rate = 32000; - hparams.audio_hop_len = 512; - } break; - case PROJECTOR_TYPE_PADDLEOCR: - { - hparams.n_merge = 2; - hparams.image_resize_algo = RESIZE_ALGO_BICUBIC; - get_u32(KEY_IMAGE_MIN_PIXELS, hparams.image_min_pixels); - get_u32(KEY_IMAGE_MAX_PIXELS, hparams.image_max_pixels); - - hparams.set_warmup_n_tokens(28*28); // avoid OOM on warmup - } break; - case PROJECTOR_TYPE_DEEPSEEKOCR: - case PROJECTOR_TYPE_DEEPSEEKOCR2: - { - hparams.patch_size = 16; - hparams.image_size = 1024; - hparams.warmup_image_size = 1024; - hparams.image_resize_algo = RESIZE_ALGO_BICUBIC; - hparams.image_pad_color = {127, 127, 127}; - - get_u32(KEY_SAM_N_BLOCK, hparams.sam_n_layer, true); - get_u32(KEY_SAM_N_HEAD, hparams.sam_n_head, true); - get_u32(KEY_SAM_N_EMBD, hparams.sam_n_embd, true); - get_u32(KEY_ATTN_WINDOW_SIZE, hparams.attn_window_size, true); - hparams.preproc_min_tiles = 2; - if (model.proj_type == PROJECTOR_TYPE_DEEPSEEKOCR) { - hparams.preproc_max_tiles = 9; - hparams.preproc_tile_size = 640; - // the CLIP/ViT body runs its layernorms at 1e-5 (the SAM stage uses 1e-6) - hparams.eps = 1e-5f; - } - if (model.proj_type == PROJECTOR_TYPE_DEEPSEEKOCR2) { - hparams.preproc_max_tiles = 6; - hparams.preproc_tile_size = 768; - // qwen2 encoder is GQA, requires KEY_N_HEAD_KV - get_u32(string_format(KEY_N_HEAD_KV, "vision"), hparams.n_head_kv); - } - // unlimited-ocr shares the v1 projector but tiles up to 32 - get_u32(KEY_PREPROC_MIN_TILES, hparams.preproc_min_tiles, false); - get_u32(KEY_PREPROC_MAX_TILES, hparams.preproc_max_tiles, false); - GGML_ASSERT(hparams.preproc_min_tiles >= 0 - && hparams.preproc_min_tiles <= hparams.preproc_max_tiles - && hparams.preproc_max_tiles <= 256); - } break; - case PROJECTOR_TYPE_HUNYUANVL: - { - hparams.n_merge = 2; - hparams.image_resize_algo = RESIZE_ALGO_LANCZOS; - hparams.image_resize_pad = PAD_NONE; - hparams.ffn_op = FFN_GELU; - hparams.set_limit_image_tokens(256, 16384); - get_u32(KEY_SPATIAL_MERGE_SIZE, hparams.n_merge, false); - get_u32(KEY_IMAGE_MIN_PIXELS, hparams.image_min_pixels, false); - get_u32(KEY_IMAGE_MAX_PIXELS, hparams.image_max_pixels, false); - hparams.set_warmup_n_tokens(32*32); - } break; - case PROJECTOR_TYPE_LFM2A: - case PROJECTOR_TYPE_D1OMNI_A: - { - // audio preprocessing params - hparams.audio_chunk_len = 1; // in seconds - hparams.audio_sample_rate = 16000; - hparams.audio_n_fft = 512; - hparams.audio_window_len = 400; - hparams.audio_hop_len = 160; - } break; - case PROJECTOR_TYPE_LFM3A: - { - get_u32(KEY_A_ATTN_WINDOW_SIZE, hparams.attn_window_size); - if (hparams.attn_window_size <= 0) { - throw std::runtime_error(string_format("%s: invalid audio attention window %d\n", __func__, hparams.attn_window_size)); - } - hparams.rope_theta = 1000000.0f; // checked by the converter - // audio preprocessing params, see mtmd_audio_preprocessor_lfm3a - hparams.audio_chunk_len = 0; - hparams.audio_sample_rate = 16000; - hparams.audio_n_fft = 400; - hparams.audio_window_len = 400; - hparams.audio_hop_len = 160; - } break; - case PROJECTOR_TYPE_EXAONE4_5: - { - hparams.n_merge = 2; - hparams.image_resize_pad = PAD_NONE; // the HF processor resizes to the smart_resize size, no padding - hparams.image_round_half_even = true; // smart_resize uses Python round() - get_u32(KEY_SPATIAL_MERGE_SIZE, hparams.n_merge, false); - get_u32(KEY_WIN_ATTN_PATTERN, hparams.n_wa_pattern, false); - get_u32(KEY_IMAGE_MIN_PIXELS, hparams.image_min_pixels); - get_u32(KEY_IMAGE_MAX_PIXELS, hparams.image_max_pixels); - hparams.set_warmup_n_tokens(46 * 46); - if (hparams.rope_theta <= 0.0f) { - hparams.rope_theta = 10000.0f; - } - get_u32(string_format(KEY_N_HEAD_KV, "vision"), hparams.n_head_kv); - } break; - case PROJECTOR_TYPE_GEMMA4A: - { - // Gemma4 feature_extraction_gemma4.py: - // frame_length_ms=20 -> 320 samples, n_fft=512, hop=10ms -> 160 - hparams.audio_chunk_len = 0; // no fixed-length padding - hparams.audio_sample_rate = 16000; - hparams.audio_n_fft = 512; - hparams.audio_window_len = 320; // 20ms frame (NOT 25ms/400) - hparams.audio_hop_len = 160; - // due to a mistake in the original conversion code, rms_norm_eps is set to a wrong value - // since all gemma4a models use 1e-6, we just hardcode it here to avoid re-conversion - hparams.eps = 1e-6f; - } break; - case PROJECTOR_TYPE_GEMMA4UA: - { - // Encoder-free: raw 16 kHz waveform chunked into 640-sample frames. - hparams.audio_chunk_len = 0; - hparams.audio_sample_rate = 16000; - hparams.eps = 1e-6f; - hparams.n_mel_bins = 640; - } break; - case PROJECTOR_TYPE_GRANITE_SPEECH: - { - hparams.audio_chunk_len = 0; - hparams.audio_sample_rate = 16000; - hparams.audio_n_fft = 512; - hparams.audio_window_len = 400; - hparams.audio_hop_len = 160; - get_u32(KEY_A_CHUNK_SIZE, hparams.audio_chunk_size); - // context_size is squared for the attn_dists/mask buffers; cap to prevent int32 overflow - // (legitimate values are small, e.g. 12-200; 8192^2 = 67M still fits int32) - GGML_ASSERT(hparams.audio_chunk_size > 0 && hparams.audio_chunk_size <= 8192); - get_u32(KEY_A_CONV_KERNEL_SIZE, hparams.audio_conv_kernel_size); - get_u32(KEY_A_MAX_POS_EMB, hparams.audio_max_pos_emb); - get_u32(KEY_A_PROJ_WINDOW_SIZE, hparams.audio_proj_window_size); - get_u32(KEY_A_PROJ_DOWNSAMPLE_RATE, hparams.audio_proj_downsample_rate); - get_u32(KEY_A_PROJ_HEAD_COUNT, hparams.audio_proj_head_count); - // NOTE: feature layers loaded above in common path - } break; - case PROJECTOR_TYPE_JANUS_PRO: - { - hparams.image_pad_color = {127, 127, 127}; - hparams.image_resize_algo = RESIZE_ALGO_BICUBIC; - } break; - case PROJECTOR_TYPE_GRANITE4_VISION: - { - // SigLIP tower. - hparams.image_resize_algo = RESIZE_ALGO_BICUBIC; - hparams.image_resize_pad = PAD_CEIL; - - // NOTE: feature_layers loaded in common path as optional - get_arr_int(KEY_PROJ_SPATIAL_OFFSETS, hparams.proj_spatial_offsets); - if (hparams.feature_layers.size() != hparams.proj_spatial_offsets.size()) { - throw std::runtime_error(string_format("%s: feature_layers.size() %d != proj_spatial_offsets.size() %d", - hparams.feature_layers.size(), hparams.proj_spatial_offsets.size())); - } - - get_u32(KEY_PROJ_SAMPLE_QUERY_SIDE, hparams.downsample_query_side); - get_u32(KEY_PROJ_SAMPLE_WINDOW_SIDE, hparams.downsample_window_side); - hparams.warmup_image_size = hparams.image_size; - } break; - default: - throw std::runtime_error(string_format("%s: unknown vision projector type %s\n", __func__, proj_type.c_str())); - } - - // sanity check - { - if (hparams.image_size < 0) { - // note: some models having hparams.image_size == 0, which means the image size is dynamic - throw std::runtime_error(string_format("%s: image_size (%d) cannot be negative\n", __func__, hparams.image_size)); - } - if (hparams.image_size > 8192) { - // cap prevents int32 overflow in n_patches = (image_size/patch_size)^2 - throw std::runtime_error(string_format("%s: image_size (%d) is too large (max 8192)\n", __func__, hparams.image_size)); - } - if (hparams.patch_size <= 0 || hparams.patch_size >= 65536) { - throw std::runtime_error(string_format("%s: patch_size (%d) must be positive and less than 65536\n", __func__, hparams.patch_size)); - } - if (hparams.n_embd <= 0) { - throw std::runtime_error(string_format("%s: n_embd (%d) must be greater than 0\n", __func__, hparams.n_embd)); - } - if (hparams.image_max_pixels < hparams.image_min_pixels) { - throw std::runtime_error(string_format("%s: image_max_pixels (%d) is less than image_min_pixels (%d)\n", __func__, hparams.image_max_pixels, hparams.image_min_pixels)); - } - if (hparams.n_merge <= 0 || hparams.n_merge >= 65536) { - throw std::runtime_error(string_format("%s: n_merge (%d) must be greater than 0 and less than 65536\n", __func__, hparams.n_merge)); - } - if (hparams.attn_window_size > 4096) { - throw std::runtime_error(string_format("%s: attn_window_size (%d) is too large (max 4096)\n", __func__, hparams.attn_window_size)); - } - } - - LOG_INF("%s: projector: %s\n", __func__, proj_type.c_str()); - LOG_INF("%s: n_embd: %d\n", __func__, hparams.n_embd); - LOG_INF("%s: n_head: %d\n", __func__, hparams.n_head); - LOG_INF("%s: n_ff: %d\n", __func__, hparams.n_ff); - LOG_INF("%s: n_layer: %d\n", __func__, hparams.n_layer); - LOG_INF("%s: ffn_op: %s\n", __func__, log_ffn_op.c_str()); - LOG_INF("%s: projection_dim: %d\n", __func__, hparams.projection_dim); - if (is_vision) { - LOG_INF("\n--- vision hparams ---\n"); - LOG_INF("%s: image_size: %d\n", __func__, hparams.image_size); - LOG_INF("%s: patch_size: %d\n", __func__, hparams.patch_size); - LOG_INF("%s: has_llava_proj: %d\n", __func__, hparams.has_llava_projector); - LOG_INF("%s: minicpmv_version: %d\n", __func__, hparams.minicpmv_version); - LOG_INF("%s: n_merge: %d\n", __func__, hparams.n_merge); - LOG_INF("%s: n_wa_pattern: %d\n", __func__, hparams.n_wa_pattern); - if (!hparams.wa_layer_indexes.empty()) { - LOG_INF("%s: wa_layer_indexes: ", __func__); - for (auto & layer : hparams.wa_layer_indexes) { - LOG_INF("%d ", layer); - } - LOG_INF("\n"); - } - if (hparams.image_min_pixels > 0) { - LOG_INF("%s: image_min_pixels: %d%s\n", __func__, hparams.image_min_pixels, hparams.custom_image_min_tokens > 0 ? " (custom value)" : ""); - } - if (hparams.image_max_pixels > 0) { - LOG_INF("%s: image_max_pixels: %d%s\n", __func__, hparams.image_max_pixels, hparams.custom_image_max_tokens > 0 ? " (custom value)" : ""); - } - if (hparams.preproc_max_tiles > 0) { - LOG_INF("%s: preproc_tiles: %d - %d\n", __func__, hparams.preproc_min_tiles, hparams.preproc_max_tiles); - } - } else if (is_audio) { - LOG_INF("\n--- audio hparams ---\n"); - LOG_INF("%s: n_mel_bins: %d\n", __func__, hparams.n_mel_bins); - LOG_INF("%s: proj_stack_factor: %d\n", __func__, hparams.proj_stack_factor); - LOG_INF("%s: audio_chunk_len: %d\n", __func__, hparams.audio_chunk_len); - LOG_INF("%s: audio_sample_rate: %d\n", __func__, hparams.audio_sample_rate); - LOG_INF("%s: audio_n_fft: %d\n", __func__, hparams.audio_n_fft); - LOG_INF("%s: audio_window_len: %d\n", __func__, hparams.audio_window_len); - LOG_INF("%s: audio_hop_len: %d\n", __func__, hparams.audio_hop_len); - - // GEMMA4UA is encoder-free: it uses n_mel_bins as a raw-waveform frame size (640) and has no FFT/filterbank, so the mel-range and FFT - // checks below do not apply to it. - // pocket-tts is encoder-free in the same sense: mimi convolves the raw waveform - // Muse Spark is encoder-free in the same sense: its conv pyramid reads the raw waveform, one channel takes the place of the mel bins - const bool fft_based = model.proj_type != PROJECTOR_TYPE_GEMMA4UA && - model.proj_type != PROJECTOR_TYPE_MUSE_SPARK_AUDIO && - model.proj_type != PROJECTOR_TYPE_POCKETTTS_SPKENC; - - // Validate audio hparams loaded from GGUF metadata - if (hparams.n_mel_bins <= 0 || (fft_based && hparams.n_mel_bins > 256)) { - throw std::runtime_error(string_format("%s: n_mel_bins (%d) must be in range [1, 256]\n", __func__, hparams.n_mel_bins)); - } - if (fft_based && (hparams.audio_sample_rate <= 0 || hparams.audio_n_fft <= 0 || hparams.audio_hop_len <= 0 || hparams.audio_window_len <= 0)) { - throw std::runtime_error(string_format("%s: audio hparams invalid: sample_rate=%d n_fft=%d window_len=%d hop_len=%d\n", - __func__, hparams.audio_sample_rate, hparams.audio_n_fft, hparams.audio_window_len, hparams.audio_hop_len)); - } - } - LOG_INF("\n"); - LOG_INF("%s: model size: %.2f MiB\n", __func__, model_size / 1024.0 / 1024.0); - LOG_INF("%s: metadata size: %.2f MiB\n", __func__, ggml_get_mem_size(ctx_meta.get()) / 1024.0 / 1024.0); - } - } - - void load_tensors(clip_ctx & ctx_clip) { - auto & model = ctx_clip.model; - auto & hparams = model.hparams; - std::map tensor_offset; - std::vector tensors_to_load; - - auto fin = open_ifstream_binary(fname); - if (!fin) { - throw std::runtime_error(string_format("%s: failed to open %s\n", __func__, fname.c_str())); - } - - // TODO @ngxson : support both audio and video in the future - const char * prefix = model.modality == CLIP_MODALITY_AUDIO ? "a" - : model.modality == CLIP_MODALITY_GEN_AUDIO ? "a.gen.code" - : "v"; - - // get offsets - for (int64_t i = 0; i < gguf_get_n_tensors(ctx_gguf.get()); ++i) { - const char * name = gguf_get_tensor_name(ctx_gguf.get(), i); - tensor_offset[name] = gguf_get_data_offset(ctx_gguf.get()) + gguf_get_tensor_offset(ctx_gguf.get(), i); - } - - // create data context - struct ggml_init_params params = { - /*.mem_size =*/ static_cast(gguf_get_n_tensors(ctx_gguf.get()) + 1) * ggml_tensor_overhead(), - /*.mem_buffer =*/ NULL, - /*.no_alloc =*/ true, - }; - ctx_clip.ctx_data.reset(ggml_init(params)); - if (!ctx_clip.ctx_data) { - throw std::runtime_error(string_format("%s: failed to init ggml context\n", __func__)); - } - - // helper function - std::unordered_set loaded_tensor_names; - auto get_tensor = [&](const std::string & name, bool required = true) { - // Each tensor should only be loaded once; duplicates indicate a bug - if (loaded_tensor_names.count(name)) { - throw std::runtime_error(string_format("%s: tensor already loaded: %s\n", __func__, name.c_str())); - } - ggml_tensor * cur = ggml_get_tensor(ctx_meta.get(), name.c_str()); - if (!cur && required) { - throw std::runtime_error(string_format("%s: unable to find tensor %s\n", __func__, name.c_str())); - } - if (cur) { - tensors_to_load.push_back(cur); - ggml_tensor * data_tensor = ggml_dup_tensor(ctx_clip.ctx_data.get(), cur); - ggml_set_name(data_tensor, cur->name); - loaded_tensor_names.insert(name); - cur = data_tensor; - // add to weight memory counter - ctx_clip.mem_usage[ggml_backend_get_device(ctx_clip.backend)] += ggml_nbytes(cur); - } - return cur; - }; - - // pocket-tts: the encoder and the decoder share the same layout, only the prefix differs - auto load_seanet = [&](clip_seanet & seanet, bool is_decoder) { - const char * conv_in = is_decoder ? TN_A_GEN_WAV_SEANET_CONV_IN : TN_A_SEANET_CONV_IN; - const char * conv_out = is_decoder ? TN_A_GEN_WAV_SEANET_CONV_OUT : TN_A_SEANET_CONV_OUT; - const char * res1 = is_decoder ? TN_A_GEN_WAV_SEANET_RES_CONV1 : TN_A_SEANET_RES_CONV1; - const char * res2 = is_decoder ? TN_A_GEN_WAV_SEANET_RES_CONV2 : TN_A_SEANET_RES_CONV2; - const char * scale = is_decoder ? TN_A_GEN_WAV_SEANET_SCALE_CONV : TN_A_SEANET_SCALE_CONV; - - seanet.conv_in_w = get_tensor(string_format(conv_in, "weight")); - seanet.conv_in_b = get_tensor(string_format(conv_in, "bias")); - seanet.conv_out_w = get_tensor(string_format(conv_out, "weight")); - seanet.conv_out_b = get_tensor(string_format(conv_out, "bias")); - - seanet.stages.resize(hparams.seanet_n_stage); - for (int i = 0; i < hparams.seanet_n_stage; i++) { - auto & stage = seanet.stages[i]; - stage.res_conv1_w = get_tensor(string_format(res1, i, "weight")); - stage.res_conv1_b = get_tensor(string_format(res1, i, "bias")); - stage.res_conv2_w = get_tensor(string_format(res2, i, "weight")); - stage.res_conv2_b = get_tensor(string_format(res2, i, "bias")); - stage.scale_conv_w = get_tensor(string_format(scale, i, "weight")); - stage.scale_conv_b = get_tensor(string_format(scale, i, "bias")); - } - }; - - auto get_vector = [&](const std::string & name) { - std::vector result; - auto it = tensor_offset.find(name); - if (it == tensor_offset.end()) { - return result; - } - - const int64_t idx = gguf_find_tensor(ctx_gguf.get(), name.c_str()); - if (idx < 0) { - throw std::runtime_error(string_format("%s: failed to find tensor %s\n", __func__, name.c_str())); - } - - if (const auto type = gguf_get_tensor_type(ctx_gguf.get(), idx); type != GGML_TYPE_F32) { - throw std::runtime_error(string_format("%s: %s must be %s, was %s\n", __func__, - name.c_str(), ggml_type_name(GGML_TYPE_F32), ggml_type_name(type))); - } - - const size_t n_bytes = gguf_get_tensor_size(ctx_gguf.get(), idx); - if (n_bytes == 0) { - throw std::runtime_error(string_format("%s: tensor %s is empty\n", __func__, name.c_str())); - } - - const size_t n_elems = n_bytes / sizeof(float); - result.resize(n_elems); - fin.seekg(it->second, std::ios::beg); - fin.read(reinterpret_cast(result.data()), n_bytes); - return result; - }; - - auto get_scalar = [&](const std::string & name, float default_val) { - auto v = get_vector(name); - if (v.empty()) { - return default_val; - } - if (v.size() != 1) { - throw std::runtime_error(string_format("%s: expected scalar tensor '%s' but got %d elements\n", - __func__, name.c_str(), (int) v.size())); - } - - return v[0]; - }; - - model.class_embedding = get_tensor(TN_CLASS_EMBD, false); - - model.pre_ln_w = get_tensor(string_format(TN_LN_PRE, prefix, "weight"), false); - model.pre_ln_b = get_tensor(string_format(TN_LN_PRE, prefix, "bias"), false); - - model.post_ln_w = get_tensor(string_format(TN_LN_POST, prefix, "weight"), false); - model.post_ln_b = get_tensor(string_format(TN_LN_POST, prefix, "bias"), false); - - model.patch_bias = get_tensor(TN_PATCH_BIAS, false); - model.patch_embeddings_0 = get_tensor(TN_PATCH_EMBD, false); - model.patch_embeddings_1 = get_tensor(TN_PATCH_EMBD_1, false); - - model.norm_embd_w = get_tensor(string_format(TN_NORM_EMBD, "weight"), false); - model.norm_embd_b = get_tensor(string_format(TN_NORM_EMBD, "bias"), false); - - model.position_embeddings = get_tensor(string_format(TN_POS_EMBD, prefix), false); - - // lfm3vl - model.lfm3vl_embedding_norm_w = get_tensor(string_format(TN_LFM3VL_EMBEDDING_NORM, prefix, "weight"), false); - - const bool has_standard_layers = ( - model.proj_type != PROJECTOR_TYPE_GEMMA3NV && - model.proj_type != PROJECTOR_TYPE_QWEN3TTS_SPKENC && - model.proj_type != PROJECTOR_TYPE_POCKETTTS_GEN && - model.proj_type != PROJECTOR_TYPE_SOPRANO && - model.proj_type != PROJECTOR_TYPE_MUSE_SPARK_AUDIO); - - // layers - const int n_layers_to_load = has_standard_layers ? hparams.n_layer : 0; - model.layers.resize(n_layers_to_load); - for (int il = 0; il < n_layers_to_load; ++il) { - auto & layer = model.layers[il]; - // lfm3vl / lfm3a conv layers have no attention tensors - const bool o_w_required = model.proj_type != PROJECTOR_TYPE_LFM3VL - && model.proj_type != PROJECTOR_TYPE_LFM3A; - layer.k_w = get_tensor(string_format(TN_ATTN_K, prefix, il, "weight"), false); - layer.q_w = get_tensor(string_format(TN_ATTN_Q, prefix, il, "weight"), false); - layer.v_w = get_tensor(string_format(TN_ATTN_V, prefix, il, "weight"), false); - layer.o_w = get_tensor(string_format(TN_ATTN_OUTPUT, prefix, il, "weight"), o_w_required); - layer.qkv_w = get_tensor(string_format(TN_ATTN_QKV, prefix, il, "weight"), false); - layer.k_norm = get_tensor(string_format(TN_ATTN_K_NORM, prefix, il, "weight"), false); - layer.q_norm = get_tensor(string_format(TN_ATTN_Q_NORM, prefix, il, "weight"), false); - layer.ln_1_w = get_tensor(string_format(TN_LN_1, prefix, il, "weight"), false); - layer.ln_2_w = get_tensor(string_format(TN_LN_2, prefix, il, "weight"), false); - layer.ls_1_w = get_tensor(string_format(TN_LS_1, prefix, il, "weight"), false); // no bias - layer.ls_2_w = get_tensor(string_format(TN_LS_2, prefix, il, "weight"), false); // no bias - layer.ls_out_w = get_tensor(string_format(TN_LS_OUT, prefix, il, "weight"), false); // no bias - layer.attn_post_norm_w = get_tensor(string_format(TN_ATTN_POST_NORM, prefix, il, "weight"), false); // no bias - layer.ff_post_norm_w = get_tensor(string_format(TN_FFN_POST_NORM, prefix, il, "weight"), false); // no bias - - layer.k_b = get_tensor(string_format(TN_ATTN_K, prefix, il, "bias"), false); - layer.q_b = get_tensor(string_format(TN_ATTN_Q, prefix, il, "bias"), false); - layer.v_b = get_tensor(string_format(TN_ATTN_V, prefix, il, "bias"), false); - layer.o_b = get_tensor(string_format(TN_ATTN_OUTPUT, prefix, il, "bias"), false); - layer.qkv_b = get_tensor(string_format(TN_ATTN_QKV, prefix, il, "bias"), false); - layer.ln_1_b = get_tensor(string_format(TN_LN_1, prefix, il, "bias"), false); - layer.ln_2_b = get_tensor(string_format(TN_LN_2, prefix, il, "bias"), false); - - // MoE ffn (dots3note vision pyramid blocks); replaces the dense ffn when present - layer.ff_gate_inp_w = get_tensor(string_format(TN_FFN_GATE_INP, prefix, il, "weight"), false); - layer.ff_gate_exps_w = get_tensor(string_format(TN_FFN_GATE_EXPS, prefix, il, "weight"), false); - layer.ff_up_exps_w = get_tensor(string_format(TN_FFN_UP_EXPS, prefix, il, "weight"), false); - layer.ff_down_exps_w = get_tensor(string_format(TN_FFN_DOWN_EXPS, prefix, il, "weight"), false); - layer.ff_exp_probs_b = get_tensor(string_format(TN_FFN_EXP_PROBS_B, prefix, il, "weight"), false); - const bool is_moe = layer.ff_gate_exps_w != nullptr; - - // ffn - layer.ff_up_w = get_tensor(string_format(TN_FFN_UP, prefix, il, "weight"), !is_moe); - layer.ff_up_b = get_tensor(string_format(TN_FFN_UP, prefix, il, "bias"), false); - layer.ff_gate_w = get_tensor(string_format(TN_FFN_GATE, prefix, il, "weight"), false); - layer.ff_gate_b = get_tensor(string_format(TN_FFN_GATE, prefix, il, "bias"), false); - layer.ff_down_w = get_tensor(string_format(TN_FFN_DOWN, prefix, il, "weight"), !is_moe); - layer.ff_down_b = get_tensor(string_format(TN_FFN_DOWN, prefix, il, "bias"), false); - - // mimovl per-head attention sink bias - layer.attn_sinks = get_tensor(string_format(TN_ATTN_SINKS, prefix, il), false); - - // qwen3vl deepstack layer - layer.deepstack_norm_w = get_tensor(string_format(TN_DEEPSTACK_NORM, il, "weight"), false); - layer.deepstack_norm_b = get_tensor(string_format(TN_DEEPSTACK_NORM, il, "bias"), false); - layer.deepstack_fc1_w = get_tensor(string_format(TN_DEEPSTACK_FC1, il, "weight"), false); - layer.deepstack_fc1_b = get_tensor(string_format(TN_DEEPSTACK_FC1, il, "bias"), false); - layer.deepstack_fc2_w = get_tensor(string_format(TN_DEEPSTACK_FC2, il, "weight"), false); - layer.deepstack_fc2_b = get_tensor(string_format(TN_DEEPSTACK_FC2, il, "bias"), false); - if (layer.has_deepstack()) { - model.n_deepstack_layers++; - } - - // lfm3vl conv layers - layer.conv2d_in_proj_w = get_tensor(string_format(TN_LFM3VL_CONV2D_IN_PROJ, prefix, il, "weight"), false); - layer.conv2d_spatial_w = get_tensor(string_format(TN_LFM3VL_CONV2D_SPATIAL, prefix, il, "weight"), false); - layer.conv2d_out_proj_w = get_tensor(string_format(TN_LFM3VL_CONV2D_OUT_PROJ, prefix, il, "weight"), false); - - // some models already exported with legacy (incorrect) naming which is quite messy, let's fix it here - // note: Qwen model converted from the old surgery script has n_ff = 0, so we cannot use n_ff to check! - bool is_ffn_swapped = ( - // only old models need this fix - model.proj_type == PROJECTOR_TYPE_MLP - || model.proj_type == PROJECTOR_TYPE_MLP_NORM - || model.proj_type == PROJECTOR_TYPE_LDP - || model.proj_type == PROJECTOR_TYPE_LDPV2 - || model.proj_type == PROJECTOR_TYPE_QWEN2VL - || model.proj_type == PROJECTOR_TYPE_QWEN25VL - || model.proj_type == PROJECTOR_TYPE_EXAONE4_5 - || model.proj_type == PROJECTOR_TYPE_GLM_EDGE - || model.proj_type == PROJECTOR_TYPE_GEMMA3 - || model.proj_type == PROJECTOR_TYPE_IDEFICS3 - || model.proj_type == PROJECTOR_TYPE_MINICPMV - || model.proj_type == PROJECTOR_TYPE_MINICPMV4_6 - || model.proj_type == PROJECTOR_TYPE_MINICPMV4_7 - || model.proj_type == PROJECTOR_TYPE_MINICPM_OCR_1_0 - ) && layer.ff_up_w && layer.ff_down_w && layer.ff_down_w->ne[0] == hparams.n_embd; - if (is_ffn_swapped) { - // swap up and down weights - ggml_tensor * tmp = layer.ff_up_w; - layer.ff_up_w = layer.ff_down_w; - layer.ff_down_w = tmp; - // swap up and down biases - tmp = layer.ff_up_b; - layer.ff_up_b = layer.ff_down_b; - layer.ff_down_b = tmp; - if (il == 0) { - LOG_WRN("%s: ffn up/down are swapped\n", __func__); - } - } - } - - - switch (model.proj_type) { - case PROJECTOR_TYPE_MLP: - case PROJECTOR_TYPE_MLP_NORM: - { - // LLaVA projection - model.mm_0_w = get_tensor(string_format(TN_LLAVA_PROJ, 0, "weight"), false); - model.mm_0_b = get_tensor(string_format(TN_LLAVA_PROJ, 0, "bias"), false); - // Yi-type llava - model.mm_1_w = get_tensor(string_format(TN_LLAVA_PROJ, 1, "weight"), false); - model.mm_1_b = get_tensor(string_format(TN_LLAVA_PROJ, 1, "bias"), false); - // missing in Yi-type llava - model.mm_2_w = get_tensor(string_format(TN_LLAVA_PROJ, 2, "weight"), false); - model.mm_2_b = get_tensor(string_format(TN_LLAVA_PROJ, 2, "bias"), false); - // Yi-type llava - model.mm_3_w = get_tensor(string_format(TN_LLAVA_PROJ, 3, "weight"), false); - model.mm_3_b = get_tensor(string_format(TN_LLAVA_PROJ, 3, "bias"), false); - model.mm_4_w = get_tensor(string_format(TN_LLAVA_PROJ, 4, "weight"), false); - model.mm_4_b = get_tensor(string_format(TN_LLAVA_PROJ, 4, "bias"), false); - if (model.mm_3_w) { - // TODO: this is a hack to support Yi-type llava - model.proj_type = PROJECTOR_TYPE_MLP_NORM; - } - model.image_newline = get_tensor(TN_IMAGE_NEWLINE, false); - } break; - case PROJECTOR_TYPE_LDP: - { - // MobileVLM projection - model.mm_model_mlp_1_w = get_tensor(string_format(TN_MVLM_PROJ_MLP, 1, "weight")); - model.mm_model_mlp_1_b = get_tensor(string_format(TN_MVLM_PROJ_MLP, 1, "bias")); - model.mm_model_mlp_3_w = get_tensor(string_format(TN_MVLM_PROJ_MLP, 3, "weight")); - model.mm_model_mlp_3_b = get_tensor(string_format(TN_MVLM_PROJ_MLP, 3, "bias")); - model.mm_model_block_1_block_0_0_w = get_tensor(string_format(TN_MVLM_PROJ_BLOCK, 1, 0, "0.weight")); - model.mm_model_block_1_block_0_1_w = get_tensor(string_format(TN_MVLM_PROJ_BLOCK, 1, 0, "1.weight")); - model.mm_model_block_1_block_0_1_b = get_tensor(string_format(TN_MVLM_PROJ_BLOCK, 1, 0, "1.bias")); - model.mm_model_block_1_block_1_fc1_w = get_tensor(string_format(TN_MVLM_PROJ_BLOCK, 1, 1, "fc1.weight")); - model.mm_model_block_1_block_1_fc1_b = get_tensor(string_format(TN_MVLM_PROJ_BLOCK, 1, 1, "fc1.bias")); - model.mm_model_block_1_block_1_fc2_w = get_tensor(string_format(TN_MVLM_PROJ_BLOCK, 1, 1, "fc2.weight")); - model.mm_model_block_1_block_1_fc2_b = get_tensor(string_format(TN_MVLM_PROJ_BLOCK, 1, 1, "fc2.bias")); - model.mm_model_block_1_block_2_0_w = get_tensor(string_format(TN_MVLM_PROJ_BLOCK, 1, 2, "0.weight")); - model.mm_model_block_1_block_2_1_w = get_tensor(string_format(TN_MVLM_PROJ_BLOCK, 1, 2, "1.weight")); - model.mm_model_block_1_block_2_1_b = get_tensor(string_format(TN_MVLM_PROJ_BLOCK, 1, 2, "1.bias")); - model.mm_model_block_2_block_0_0_w = get_tensor(string_format(TN_MVLM_PROJ_BLOCK, 2, 0, "0.weight")); - model.mm_model_block_2_block_0_1_w = get_tensor(string_format(TN_MVLM_PROJ_BLOCK, 2, 0, "1.weight")); - model.mm_model_block_2_block_0_1_b = get_tensor(string_format(TN_MVLM_PROJ_BLOCK, 2, 0, "1.bias")); - model.mm_model_block_2_block_1_fc1_w = get_tensor(string_format(TN_MVLM_PROJ_BLOCK, 2, 1, "fc1.weight")); - model.mm_model_block_2_block_1_fc1_b = get_tensor(string_format(TN_MVLM_PROJ_BLOCK, 2, 1, "fc1.bias")); - model.mm_model_block_2_block_1_fc2_w = get_tensor(string_format(TN_MVLM_PROJ_BLOCK, 2, 1, "fc2.weight")); - model.mm_model_block_2_block_1_fc2_b = get_tensor(string_format(TN_MVLM_PROJ_BLOCK, 2, 1, "fc2.bias")); - model.mm_model_block_2_block_2_0_w = get_tensor(string_format(TN_MVLM_PROJ_BLOCK, 2, 2, "0.weight")); - model.mm_model_block_2_block_2_1_w = get_tensor(string_format(TN_MVLM_PROJ_BLOCK, 2, 2, "1.weight")); - model.mm_model_block_2_block_2_1_b = get_tensor(string_format(TN_MVLM_PROJ_BLOCK, 2, 2, "1.bias")); - } break; - case PROJECTOR_TYPE_LDPV2: - { - // MobilVLM_V2 projection - model.mm_model_mlp_0_w = get_tensor(string_format(TN_MVLM_PROJ_MLP, 0, "weight")); - model.mm_model_mlp_0_b = get_tensor(string_format(TN_MVLM_PROJ_MLP, 0, "bias")); - model.mm_model_mlp_2_w = get_tensor(string_format(TN_MVLM_PROJ_MLP, 2, "weight")); - model.mm_model_mlp_2_b = get_tensor(string_format(TN_MVLM_PROJ_MLP, 2, "bias")); - model.mm_model_peg_0_w = get_tensor(string_format(TN_MVLM_PROJ_PEG, 0, "weight")); - model.mm_model_peg_0_b = get_tensor(string_format(TN_MVLM_PROJ_PEG, 0, "bias")); - } break; - case PROJECTOR_TYPE_MINICPMV: - { - // model.mm_model_pos_embed = get_tensor(new_clip->ctx_data, TN_MINICPMV_POS_EMBD); - model.mm_model_pos_embed_k = get_tensor(TN_MINICPMV_POS_EMBD_K); - model.mm_model_query = get_tensor(TN_MINICPMV_QUERY); - model.mm_model_proj = get_tensor(TN_MINICPMV_PROJ); - model.mm_model_kv_proj = get_tensor(TN_MINICPMV_KV_PROJ); - model.mm_model_attn_q_w = get_tensor(string_format(TN_MINICPMV_ATTN, "q", "weight")); - model.mm_model_attn_k_w = get_tensor(string_format(TN_MINICPMV_ATTN, "k", "weight")); - model.mm_model_attn_v_w = get_tensor(string_format(TN_MINICPMV_ATTN, "v", "weight")); - model.mm_model_attn_q_b = get_tensor(string_format(TN_MINICPMV_ATTN, "q", "bias")); - model.mm_model_attn_k_b = get_tensor(string_format(TN_MINICPMV_ATTN, "k", "bias")); - model.mm_model_attn_v_b = get_tensor(string_format(TN_MINICPMV_ATTN, "v", "bias")); - model.mm_model_attn_o_w = get_tensor(string_format(TN_MINICPMV_ATTN, "out", "weight")); - model.mm_model_attn_o_b = get_tensor(string_format(TN_MINICPMV_ATTN, "out", "bias")); - model.mm_model_ln_q_w = get_tensor(string_format(TN_MINICPMV_LN, "q", "weight")); - model.mm_model_ln_q_b = get_tensor(string_format(TN_MINICPMV_LN, "q", "bias")); - model.mm_model_ln_kv_w = get_tensor(string_format(TN_MINICPMV_LN, "kv", "weight")); - model.mm_model_ln_kv_b = get_tensor(string_format(TN_MINICPMV_LN, "kv", "bias")); - model.mm_model_ln_post_w = get_tensor(string_format(TN_MINICPMV_LN, "post", "weight")); - model.mm_model_ln_post_b = get_tensor(string_format(TN_MINICPMV_LN, "post", "bias")); - } break; - case PROJECTOR_TYPE_MINICPMV4_6: - case PROJECTOR_TYPE_MINICPMV4_7: - case PROJECTOR_TYPE_MINICPM_OCR_1_0: - { - const bool merger_required = hparams.n_merge == 4; - auto get_merger_tensor = [&](const std::string & name, bool required = true) { - return get_tensor(name, merger_required && required); - }; - - // ViT merger: window self-attention - model.vit_merger_ln1_w = get_merger_tensor(string_format(TN_VIT_MERGER_LN1, "weight")); - model.vit_merger_ln1_b = get_merger_tensor(string_format(TN_VIT_MERGER_LN1, "bias")); - model.vit_merger_attn_q_w = get_merger_tensor(string_format(TN_VIT_MERGER_ATTN_Q, "weight")); - model.vit_merger_attn_q_b = get_merger_tensor(string_format(TN_VIT_MERGER_ATTN_Q, "bias"), false); - model.vit_merger_attn_k_w = get_merger_tensor(string_format(TN_VIT_MERGER_ATTN_K, "weight")); - model.vit_merger_attn_k_b = get_merger_tensor(string_format(TN_VIT_MERGER_ATTN_K, "bias"), false); - model.vit_merger_attn_v_w = get_merger_tensor(string_format(TN_VIT_MERGER_ATTN_V, "weight")); - model.vit_merger_attn_v_b = get_merger_tensor(string_format(TN_VIT_MERGER_ATTN_V, "bias"), false); - model.vit_merger_attn_o_w = get_merger_tensor(string_format(TN_VIT_MERGER_ATTN_O, "weight")); - model.vit_merger_attn_o_b = get_merger_tensor(string_format(TN_VIT_MERGER_ATTN_O, "bias"), false); - // ViT merger: MLP downsample - model.vit_merger_ds_ln_w = get_merger_tensor(string_format(TN_VIT_MERGER_DS_LN, "weight")); - model.vit_merger_ds_ln_b = get_merger_tensor(string_format(TN_VIT_MERGER_DS_LN, "bias")); - model.vit_merger_ds_up_w = get_merger_tensor(string_format(TN_VIT_MERGER_DS_UP, "weight")); - model.vit_merger_ds_up_b = get_merger_tensor(string_format(TN_VIT_MERGER_DS_UP, "bias"), false); - model.vit_merger_ds_down_w = get_merger_tensor(string_format(TN_VIT_MERGER_DS_DOWN, "weight")); - model.vit_merger_ds_down_b = get_merger_tensor(string_format(TN_VIT_MERGER_DS_DOWN, "bias"), false); - // Final Merger (DownsampleMLP) - model.mm_input_norm_w = get_tensor(TN_MM_INP_NORM); - model.mm_input_norm_b = get_tensor(TN_MM_INP_NORM_B, false); - model.mm_ffn_up_w = get_tensor(string_format(TN_MM_UP, "weight")); - model.mm_ffn_up_b = get_tensor(string_format(TN_MM_UP, "bias"), false); - model.mm_ffn_down_w = get_tensor(string_format(TN_MM_DOWN, "weight")); - model.mm_ffn_down_b = get_tensor(string_format(TN_MM_DOWN, "bias"), false); - model.tok_embd_sep = get_tensor(TN_TOK_EMBD_SEP, model.proj_type == PROJECTOR_TYPE_MINICPMV4_7); - } break; - case PROJECTOR_TYPE_GLM_EDGE: - { - model.mm_model_adapter_conv_w = get_tensor(string_format(TN_GLM_ADAPER_CONV, "weight")); - model.mm_model_adapter_conv_b = get_tensor(string_format(TN_GLM_ADAPER_CONV, "bias")); - model.mm_model_mlp_0_w = get_tensor(string_format(TN_GLM_ADAPTER_LINEAR, "weight")); - model.mm_model_ln_q_w = get_tensor(string_format(TN_GLM_ADAPTER_NORM_1, "weight")); - model.mm_model_ln_q_b = get_tensor(string_format(TN_GLM_ADAPTER_NORM_1, "bias")); - model.mm_model_mlp_1_w = get_tensor(string_format(TN_GLM_ADAPTER_D_H_2_4H, "weight")); - model.mm_model_mlp_2_w = get_tensor(string_format(TN_GLM_ADAPTER_GATE, "weight")); - model.mm_model_mlp_3_w = get_tensor(string_format(TN_GLM_ADAPTER_D_4H_2_H, "weight")); - model.mm_boi = get_tensor(string_format(TN_TOK_GLM_BOI)); - model.mm_eoi = get_tensor(string_format(TN_TOK_GLM_EOI)); - } break; - case PROJECTOR_TYPE_QWEN2VL: - case PROJECTOR_TYPE_QWEN25VL: - case PROJECTOR_TYPE_EXAONE4_5: - { - model.mm_0_w = get_tensor(string_format(TN_LLAVA_PROJ, 0, "weight")); - model.mm_0_b = get_tensor(string_format(TN_LLAVA_PROJ, 0, "bias")); - model.mm_1_w = get_tensor(string_format(TN_LLAVA_PROJ, 2, "weight")); - model.mm_1_b = get_tensor(string_format(TN_LLAVA_PROJ, 2, "bias")); - } break; - case PROJECTOR_TYPE_QWEN3VL: - { - model.mm_0_w = get_tensor(string_format(TN_LLAVA_PROJ, 0, "weight")); - model.mm_0_b = get_tensor(string_format(TN_LLAVA_PROJ, 0, "bias")); - model.mm_1_w = get_tensor(string_format(TN_LLAVA_PROJ, 2, "weight")); - model.mm_1_b = get_tensor(string_format(TN_LLAVA_PROJ, 2, "bias")); - } break; - case PROJECTOR_TYPE_LFM3VL: - { - model.mm_input_norm_w = get_tensor(TN_MM_INP_NORM); - model.mm_input_norm_b = get_tensor(TN_MM_INP_NORM_B); - model.mm_0_w = get_tensor(string_format(TN_LLAVA_PROJ, 0, "weight")); - model.mm_0_b = get_tensor(string_format(TN_LLAVA_PROJ, 0, "bias")); - model.mm_1_w = get_tensor(string_format(TN_LLAVA_PROJ, 2, "weight")); - model.mm_1_b = get_tensor(string_format(TN_LLAVA_PROJ, 2, "bias")); - // each layer must be either a conv block or a complete attention block - for (size_t il = 0; il < model.layers.size(); ++il) { - const auto & layer = model.layers[il]; - const bool is_conv = layer.conv2d_in_proj_w != nullptr; - const bool is_attn = layer.q_w && layer.k_w && layer.v_w && layer.o_w; - if (is_conv == is_attn) { - throw std::runtime_error(string_format("%s: lfm3vl layer %zu is neither a complete conv nor a complete attention block\n", __func__, il)); - } - } - } break; - case PROJECTOR_TYPE_LING3VL: - { - model.mm_input_norm_w = get_tensor(TN_MM_INP_NORM); // merger.norm - model.mm_input_norm_b = get_tensor(TN_MM_INP_NORM_B); // merger.norm - model.mm_0_w = get_tensor(string_format(TN_LLAVA_PROJ, 0, "weight")); // linear_proj.0 - model.mm_0_b = get_tensor(string_format(TN_LLAVA_PROJ, 0, "bias")); - model.mm_1_w = get_tensor(string_format(TN_LLAVA_PROJ, 2, "weight")); // linear_proj.2 - model.mm_1_b = get_tensor(string_format(TN_LLAVA_PROJ, 2, "bias")); - } break; - case PROJECTOR_TYPE_MIMOVL: - { - model.mm_0_w = get_tensor(string_format(TN_LLAVA_PROJ, 0, "weight")); - model.mm_0_b = get_tensor(string_format(TN_LLAVA_PROJ, 0, "bias"), false); - model.mm_1_w = get_tensor(string_format(TN_LLAVA_PROJ, 2, "weight")); - model.mm_1_b = get_tensor(string_format(TN_LLAVA_PROJ, 2, "bias"), false); - } break; - case PROJECTOR_TYPE_MINIMAX_M3: - { - // per-patch MLP: mm.1 -> gelu -> mm.2 - model.mm_1_w = get_tensor(string_format(TN_LLAVA_PROJ, 1, "weight")); - model.mm_1_b = get_tensor(string_format(TN_LLAVA_PROJ, 1, "bias")); - model.mm_2_w = get_tensor(string_format(TN_LLAVA_PROJ, 2, "weight")); - model.mm_2_b = get_tensor(string_format(TN_LLAVA_PROJ, 2, "bias")); - // 2x2 merge MLP: mm.merge.fc1 -> gelu -> mm.merge.fc2 - model.mm_merger_fc1_w = get_tensor(string_format(TN_MM_MERGER_FC1, "weight")); - model.mm_merger_fc1_b = get_tensor(string_format(TN_MM_MERGER_FC1, "bias")); - model.mm_merger_fc2_w = get_tensor(string_format(TN_MM_MERGER_FC2, "weight")); - model.mm_merger_fc2_b = get_tensor(string_format(TN_MM_MERGER_FC2, "bias")); - } break; - case PROJECTOR_TYPE_MUSE_SPARK_AUDIO: - { - model.layers.resize(hparams.n_layer); - model.a_patches.resize(hparams.audio_patch_sizes.size()); - for (size_t i = 0; i < model.a_patches.size(); ++i) { - auto & pe = model.a_patches[i]; - pe.proj_w = get_tensor(string_format(TN_A_PATCH, (int) i, "weight")); - pe.proj_b = get_tensor(string_format(TN_A_PATCH, (int) i, "bias")); - pe.norm_w = get_tensor(string_format(TN_A_PATCH_NORM, (int) i, "weight")); - if (pe.proj_w->ne[1] != hparams.audio_hidden_sizes[i]) { - throw std::runtime_error(string_format("%s: audio stage %zu projection has width %" PRId64 ", expected %d\n", __func__, i, pe.proj_w->ne[1], hparams.audio_hidden_sizes[i])); - } - } - - for (int il = 0; il < hparams.n_layer; ++il) { - auto & layer = model.layers[il]; - layer.conv_w = get_tensor(string_format(TN_CONV1D, il, "weight")); - layer.conv_b = get_tensor(string_format(TN_CONV1D, il, "bias")); - layer.conv_norm_w = get_tensor(string_format(TN_A_CONV1D_NORM, il, "weight")); - layer.ff_up_w = get_tensor(string_format(TN_FFN_UP, "a", il, "weight")); - layer.ff_up_b = get_tensor(string_format(TN_FFN_UP, "a", il, "bias")); - layer.ff_down_w = get_tensor(string_format(TN_FFN_DOWN, "a", il, "weight")); - layer.ff_down_b = get_tensor(string_format(TN_FFN_DOWN, "a", il, "bias")); - } - - // pre_ln and post_ln come from the generic path above, where they are optional - if (!model.pre_ln_w || !model.post_ln_w) { - throw std::runtime_error(string_format("%s: missing pre_ln or post_ln weight\n", __func__)); - } - model.mm_fc_w = get_tensor(string_format(TN_MM_AUDIO_FC, "weight")); - model.mm_fc_b = get_tensor(string_format(TN_MM_AUDIO_FC, "bias")); - } break; - case PROJECTOR_TYPE_MUSE_GLIMMER: - { - // 3-linear MLP: fc -> erf-GELU -> proj -> erf-GELU -> vision_proj (into LLM residual dim) - model.mm_0_w = get_tensor(string_format(TN_LLAVA_PROJ, 0, "weight")); - model.mm_1_w = get_tensor(string_format(TN_LLAVA_PROJ, 1, "weight")); - model.mm_2_w = get_tensor(string_format(TN_LLAVA_PROJ, 2, "weight")); - } break; - case PROJECTOR_TYPE_STEP3VL: - { - model.mm_0_w = get_tensor(string_format(TN_LLAVA_PROJ, 0, "weight")); - model.mm_0_b = get_tensor(string_format(TN_LLAVA_PROJ, 0, "bias"), false); - model.mm_1_w = get_tensor(string_format(TN_LLAVA_PROJ, 1, "weight")); - model.mm_1_b = get_tensor(string_format(TN_LLAVA_PROJ, 1, "bias"), false); - model.mm_model_proj = get_tensor(string_format(TN_MM_PROJECTOR, "weight")); - } break; - case PROJECTOR_TYPE_YOUTUVL: - { - model.mm_input_norm_w = get_tensor(TN_MM_INP_NORM); // merger.ln_q (RMS norm) - model.mm_0_w = get_tensor(string_format(TN_LLAVA_PROJ, 0, "weight")); // merger.mlp.0 - model.mm_0_b = get_tensor(string_format(TN_LLAVA_PROJ, 0, "bias")); - model.mm_1_w = get_tensor(string_format(TN_LLAVA_PROJ, 2, "weight")); // merger.mlp.2 - model.mm_1_b = get_tensor(string_format(TN_LLAVA_PROJ, 2, "bias")); - } break; - case PROJECTOR_TYPE_YASA2: - { - // reuse tensors already loaded by the common section - // (TN_PATCH_EMBD and TN_PATCH_BIAS have the same tensor names) - GGML_ASSERT(model.patch_embeddings_0 && "yasa2 requires v.patch_embd.weight"); - model.yasa_patch_w = model.patch_embeddings_0; - model.yasa_patch_b = model.patch_bias; - model.yasa_patch_ln_w = get_tensor(TN_YASA_PATCH_LN_W, false); - model.yasa_patch_ln_b = get_tensor(TN_YASA_PATCH_LN_B, false); - model.yasa_backbone_ln_w = get_tensor(TN_YASA_BACKBONE_LN_W, false); - model.yasa_backbone_ln_b = get_tensor(TN_YASA_BACKBONE_LN_B, false); - model.yasa_vision_pos_embed = get_tensor(TN_YASA_POS_EMBD, false); - model.mm_0_w = get_tensor(string_format(TN_LLAVA_PROJ, 0, "weight")); - model.mm_0_b = get_tensor(string_format(TN_LLAVA_PROJ, 0, "bias"), false); - model.mm_2_w = get_tensor(string_format(TN_LLAVA_PROJ, 2, "weight")); - model.mm_2_b = get_tensor(string_format(TN_LLAVA_PROJ, 2, "bias"), false); - - model.yasa_stages.clear(); - for (int s = 0; ; ++s) { - yasa2_stage stage; - stage.down_ln_w = get_tensor(string_format(TN_YASA_STAGE_DOWN_LN, s, "weight"), false); - stage.down_ln_b = get_tensor(string_format(TN_YASA_STAGE_DOWN_LN, s, "bias"), false); - stage.down_conv_w = get_tensor(string_format(TN_YASA_STAGE_DOWN_CONV, s, "weight"), false); - stage.down_conv_b = get_tensor(string_format(TN_YASA_STAGE_DOWN_CONV, s, "bias"), false); - - for (int bi = 0; ; ++bi) { - yasa2_block blk; - blk.dw_w = get_tensor(string_format(TN_YASA_STAGE_BLK, s, bi, "dw", "weight"), false); - if (!blk.dw_w) { - break; - } - blk.dw_b = get_tensor(string_format(TN_YASA_STAGE_BLK, s, bi, "dw", "bias"), false); - blk.ln_w = get_tensor(string_format(TN_YASA_STAGE_BLK, s, bi, "ln", "weight"), false); - blk.ln_b = get_tensor(string_format(TN_YASA_STAGE_BLK, s, bi, "ln", "bias"), false); - blk.pw1_w = get_tensor(string_format(TN_YASA_STAGE_BLK, s, bi, "pw1", "weight"), false); - blk.pw1_b = get_tensor(string_format(TN_YASA_STAGE_BLK, s, bi, "pw1", "bias"), false); - blk.grn_w = get_tensor(string_format(TN_YASA_STAGE_BLK, s, bi, "grn", "weight"), false); - blk.grn_b = get_tensor(string_format(TN_YASA_STAGE_BLK, s, bi, "grn", "bias"), false); - blk.pw2_w = get_tensor(string_format(TN_YASA_STAGE_BLK, s, bi, "pw2", "weight"), false); - blk.pw2_b = get_tensor(string_format(TN_YASA_STAGE_BLK, s, bi, "pw2", "bias"), false); - stage.blocks.push_back(blk); - } - - if (!stage.down_conv_w && stage.blocks.empty()) { - break; - } - model.yasa_stages.push_back(std::move(stage)); - } - } break; - case PROJECTOR_TYPE_GLM4V: - case PROJECTOR_TYPE_GLM5V: - { - model.mm_fc_w = get_tensor(string_format(TN_MM_PROJECTOR, "weight")); - model.mm_ffn_up_w = get_tensor(string_format(TN_MM_UP, "weight")); - model.mm_ffn_up_b = get_tensor(string_format(TN_MM_UP, "bias"), false); - model.mm_ffn_gate_w = get_tensor(string_format(TN_MM_GATE, "weight")); - model.mm_ffn_gate_b = get_tensor(string_format(TN_MM_GATE, "bias"), false); - model.mm_ffn_down_w = get_tensor(string_format(TN_MM_DOWN, "weight")); - model.mm_ffn_down_b = get_tensor(string_format(TN_MM_DOWN, "bias"), false); - model.mm_post_norm_w = get_tensor(string_format(TN_MM_POST_NORM, "weight")); - model.mm_post_norm_b = get_tensor(string_format(TN_MM_POST_NORM, "bias"), false); - model.mm_patch_merger_w = get_tensor(string_format(TN_MM_PATCH_MERGER, "weight")); - model.mm_patch_merger_b = get_tensor(string_format(TN_MM_PATCH_MERGER, "bias")); - } break; - case PROJECTOR_TYPE_GEMMA3: - { - model.mm_input_proj_w = get_tensor(TN_MM_INP_PROJ); - model.mm_soft_emb_norm_w = get_tensor(TN_MM_SOFT_EMB_N); - } break; - case PROJECTOR_TYPE_GEMMA4V: - { - model.mm_input_proj_w = get_tensor(TN_MM_INP_PROJ); - model.std_bias = get_tensor(TN_STD_BIAS, false); - model.std_scale = get_tensor(TN_STD_SCALE, false); - // load scalar for Gemma4ClippableLinear - for (auto * tensor : tensors_to_load) { - std::string name = tensor->name; - if (string_ends_with(name, ".weight")) { - std::string name_inp_max = name; - std::string name_inp_min = name; - std::string name_out_max = name; - std::string name_out_min = name; - string_replace_all(name_inp_max, ".weight", ".input_max"); - string_replace_all(name_inp_min, ".weight", ".input_min"); - string_replace_all(name_out_max, ".weight", ".output_max"); - string_replace_all(name_out_min, ".weight", ".output_min"); - model.clamp_info_map[name] = { - get_scalar(name_inp_max, FLT_MAX), - get_scalar(name_inp_min, -FLT_MAX), - get_scalar(name_out_max, FLT_MAX), - get_scalar(name_out_min, -FLT_MAX) - }; - } - } - } break; - case PROJECTOR_TYPE_GEMMA4UV: - { - model.mm_input_proj_w = get_tensor(TN_MM_INP_PROJ); - model.patch_norm_1_w = get_tensor(string_format(TN_PATCH_NORM, 1, "weight")); - model.patch_norm_1_b = get_tensor(string_format(TN_PATCH_NORM, 1, "bias")); - model.patch_norm_2_w = get_tensor(string_format(TN_PATCH_NORM, 2, "weight")); - model.patch_norm_2_b = get_tensor(string_format(TN_PATCH_NORM, 2, "bias")); - model.patch_norm_3_w = get_tensor(string_format(TN_PATCH_NORM, 3, "weight")); // pos_norm - model.patch_norm_3_b = get_tensor(string_format(TN_PATCH_NORM, 3, "bias")); // pos_norm - } break; - case PROJECTOR_TYPE_GEMMA3NV: - { - model.mobilenet_stem_conv_w = get_tensor(TN_MNV5_STEM_CONV, false); - model.mobilenet_stem_conv_b = get_tensor(TN_MNV5_STEM_BIAS, false); - model.mobilenet_stem_norm_w = get_tensor(TN_MNV5_STEM_BN, false); - - model.msfa_ffn_expand_w = get_tensor(TN_MNV5_MSFA_FFN_EXP_W, false); - model.msfa_ffn_expand_bn = get_tensor(TN_MNV5_MSFA_FFN_EXP_BN, false); // Consume BN if present but likely folded - model.msfa_ffn_project_w = get_tensor(TN_MNV5_MSFA_FFN_PROJ_W, false); - model.msfa_ffn_project_bn = get_tensor(TN_MNV5_MSFA_FFN_PROJ_BN, false); - - model.msfa_concat_norm_w = get_tensor(TN_MNV5_MSFA_NORM, false); - - // Dynamically load blocks stage by stage - for (int stage = 0; stage < 4; ++stage) { - int blocks_found_in_stage = 0; - - for (int blk_idx = 0; ; ++blk_idx) { - bool found_block = false; - mobilenetv5_block block; - - // 1. Check for Edge Residual (S0) - block.s0_conv_exp_w = get_tensor(string_format(TN_MNV5_BLK_S0_EXP_W, stage, blk_idx), false); - if (block.s0_conv_exp_w) { - found_block = true; - block.s0_bn1_w = get_tensor(string_format(TN_MNV5_BLK_S0_BN1_W, stage, blk_idx), false); - block.s0_conv_pwl_w = get_tensor(string_format(TN_MNV5_BLK_S0_PWL_W, stage, blk_idx), false); - block.s0_bn2_w = get_tensor(string_format(TN_MNV5_BLK_S0_BN2_W, stage, blk_idx), false); - } - // 2. Check for UIR (Universal Inverted Residual) - else { - // Check for dw_start OR pw_exp (some UIR blocks skip dw_start) - block.dw_start_w = get_tensor(string_format(TN_MNV5_BLK_DW_START_W, stage, blk_idx), false); - block.pw_exp_w = get_tensor(string_format(TN_MNV5_BLK_PW_EXP_W, stage, blk_idx), false); - - if (block.dw_start_w || block.pw_exp_w) { - found_block = true; - if (block.dw_start_w) { - block.dw_start_bn_w = get_tensor(string_format(TN_MNV5_BLK_DW_START_BN, stage, blk_idx), false); - } - if (block.pw_exp_w) { - block.pw_exp_bn_w = get_tensor(string_format(TN_MNV5_BLK_PW_EXP_BN, stage, blk_idx), false); - } - block.dw_mid_w = get_tensor(string_format(TN_MNV5_BLK_DW_MID_W, stage, blk_idx), false); - if (block.dw_mid_w) { - block.dw_mid_bn_w = get_tensor(string_format(TN_MNV5_BLK_DW_MID_BN, stage, blk_idx), false); - } - block.pw_proj_w = get_tensor(string_format(TN_MNV5_BLK_PW_PROJ_W, stage, blk_idx), false); - if (block.pw_proj_w) { - block.pw_proj_bn_w = get_tensor(string_format(TN_MNV5_BLK_PW_PROJ_BN, stage, blk_idx), false); - } - block.layer_scale_w = get_tensor(string_format(TN_MNV5_BLK_LAYER_SCALE, stage, blk_idx), false); - } - } - - // 3. Check for Attention (MQA) - // Even if UIR/Edge check failed, this might be a pure attention block - ggml_tensor* attn_q_check = get_tensor(string_format(TN_MNV5_ATTN_Q_W, stage, blk_idx), false); - if (attn_q_check) { - found_block = true; - block.attn_q_w = attn_q_check; - block.attn_k_w = get_tensor(string_format(TN_MNV5_ATTN_K_W, stage, blk_idx), false); - block.attn_v_w = get_tensor(string_format(TN_MNV5_ATTN_V_W, stage, blk_idx), false); - block.attn_o_w = get_tensor(string_format(TN_MNV5_ATTN_O_W, stage, blk_idx), false); - block.attn_k_dw_w = get_tensor(string_format(TN_MNV5_ATTN_K_DW, stage, blk_idx), false); - block.attn_k_norm_w = get_tensor(string_format(TN_MNV5_ATTN_K_NORM, stage, blk_idx), false); - block.attn_v_dw_w = get_tensor(string_format(TN_MNV5_ATTN_V_DW, stage, blk_idx), false); - block.attn_v_norm_w = get_tensor(string_format(TN_MNV5_ATTN_V_NORM, stage, blk_idx), false); - block.attn_norm_w = get_tensor(string_format(TN_MNV5_ATTN_NORM, stage, blk_idx), false); - // Note: Attention blocks also have layer_scale, load it if not already loaded by UIR check - if (!block.layer_scale_w) { - block.layer_scale_w = get_tensor(string_format(TN_MNV5_BLK_LAYER_SCALE, stage, blk_idx), false); - } - } - - if (found_block) { - model.mobilenet_blocks.push_back(block); - blocks_found_in_stage++; - } else { - // End of blocks for this stage - break; - } - } - - // Track where this stage ends in the flat vector - if (blocks_found_in_stage > 0) { - model.mobilenet_stage_ends.push_back(model.mobilenet_blocks.size() - 1); - LOG_INF("%s: Stage %d ended at global block index %zu\n", __func__, stage, model.mobilenet_blocks.size() - 1); - } - } - model.mm_input_proj_w = get_tensor(TN_MM_INP_PROJ); - model.mm_soft_emb_norm_w = get_tensor(TN_MM_SOFT_EMB_N); - } break; - case PROJECTOR_TYPE_IDEFICS3: - { - model.mm_fc_w = get_tensor(string_format(TN_MM_PROJECTOR, "weight")); - } break; - case PROJECTOR_TYPE_COHERE2V: - { - model.mm_1_w = get_tensor(string_format(TN_LLAVA_PROJ, 1, "weight")); - model.mm_1_b = get_tensor(string_format(TN_LLAVA_PROJ, 1, "bias")); - model.mm_2_w = get_tensor(string_format(TN_LLAVA_PROJ, 2, "weight")); - model.mm_2_b = get_tensor(string_format(TN_LLAVA_PROJ, 2, "bias")); - } break; - case PROJECTOR_TYPE_LFM2: - case PROJECTOR_TYPE_D1OMNI_V: - { - model.mm_input_norm_w = get_tensor(TN_MM_INP_NORM, false); - model.mm_input_norm_b = get_tensor(TN_MM_INP_NORM_B, false); - model.mm_1_w = get_tensor(string_format(TN_LLAVA_PROJ, 1, "weight")); - model.mm_1_b = get_tensor(string_format(TN_LLAVA_PROJ, 1, "bias")); - model.mm_2_w = get_tensor(string_format(TN_LLAVA_PROJ, 2, "weight")); - model.mm_2_b = get_tensor(string_format(TN_LLAVA_PROJ, 2, "bias")); - } break; - case PROJECTOR_TYPE_KIMIVL: - case PROJECTOR_TYPE_PADDLEOCR: - case PROJECTOR_TYPE_KIMIK25: - { - model.mm_input_norm_w = get_tensor(TN_MM_INP_NORM); - model.mm_input_norm_b = get_tensor(TN_MM_INP_NORM_B); - model.mm_1_w = get_tensor(string_format(TN_LLAVA_PROJ, 1, "weight")); - model.mm_1_b = get_tensor(string_format(TN_LLAVA_PROJ, 1, "bias")); - model.mm_2_w = get_tensor(string_format(TN_LLAVA_PROJ, 2, "weight")); - model.mm_2_b = get_tensor(string_format(TN_LLAVA_PROJ, 2, "bias")); - } break; - case PROJECTOR_TYPE_DEEPSEEK4V: - { - model.mm_1_w = get_tensor(string_format(TN_LLAVA_PROJ, 1, "weight")); - model.mm_1_b = get_tensor(string_format(TN_LLAVA_PROJ, 1, "bias")); - model.mm_2_w = get_tensor(string_format(TN_LLAVA_PROJ, 2, "weight")); - model.mm_2_b = get_tensor(string_format(TN_LLAVA_PROJ, 2, "bias")); - // sentinel token embeddings written into the output block - model.image_newline = get_tensor(TN_IMAGE_NEWLINE); - model.token_embd_img_start = get_tensor(TN_TOK_IMG_START); - model.token_embd_img_end = get_tensor(TN_TOK_IMG_END); - model.token_embd_img_pad = get_tensor(TN_TOK_IMG_PAD); - } break; - case PROJECTOR_TYPE_DEEPSEEK41V: - { - model.mm_1_w = get_tensor(string_format(TN_LLAVA_PROJ, 1, "weight")); - model.mm_1_b = get_tensor(string_format(TN_LLAVA_PROJ, 1, "bias")); - model.mm_2_w = get_tensor(string_format(TN_LLAVA_PROJ, 2, "weight")); - model.mm_2_b = get_tensor(string_format(TN_LLAVA_PROJ, 2, "bias")); - // image_start / image_end / image_newline, written into the output block - model.image_newline = get_tensor(TN_IMAGE_NEWLINE); - model.token_embd_img_start = get_tensor(TN_TOK_IMG_START); - model.token_embd_img_end = get_tensor(TN_TOK_IMG_END); - } break; - case PROJECTOR_TYPE_PIXTRAL: - { - model.mm_1_w = get_tensor(string_format(TN_LLAVA_PROJ, 1, "weight")); - model.mm_1_b = get_tensor(string_format(TN_LLAVA_PROJ, 1, "bias"), false); - model.mm_2_w = get_tensor(string_format(TN_LLAVA_PROJ, 2, "weight")); - model.mm_2_b = get_tensor(string_format(TN_LLAVA_PROJ, 2, "bias"), false); - // [IMG_BREAK] token embedding - model.token_embd_img_break = get_tensor(TN_TOK_IMG_BREAK); - // for mistral small 3.1 - model.mm_input_norm_w = get_tensor(TN_MM_INP_NORM, false); - model.mm_patch_merger_w = get_tensor(string_format(TN_MM_PATCH_MERGER, "weight"), false); - } break; - case PROJECTOR_TYPE_LIGHTONOCR: - { - model.mm_1_w = get_tensor(string_format(TN_LLAVA_PROJ, 1, "weight")); - model.mm_1_b = get_tensor(string_format(TN_LLAVA_PROJ, 1, "bias"), false); - model.mm_2_w = get_tensor(string_format(TN_LLAVA_PROJ, 2, "weight")); - model.mm_2_b = get_tensor(string_format(TN_LLAVA_PROJ, 2, "bias"), false); - model.mm_input_norm_w = get_tensor(TN_MM_INP_NORM, false); - model.mm_patch_merger_w = get_tensor(string_format(TN_MM_PATCH_MERGER, "weight"), false); - } break; - case PROJECTOR_TYPE_DOTS_OCR: - case PROJECTOR_TYPE_DOTS3NOTE_V: - { - model.mm_0_w = get_tensor(string_format(TN_LLAVA_PROJ, 0, "weight")); - model.mm_0_b = get_tensor(string_format(TN_LLAVA_PROJ, 0, "bias")); - model.mm_2_w = get_tensor(string_format(TN_LLAVA_PROJ, 2, "weight")); - model.mm_2_b = get_tensor(string_format(TN_LLAVA_PROJ, 2, "bias")); - model.mm_input_norm_w = get_tensor(TN_MM_INP_NORM); - model.mm_input_norm_b = get_tensor(TN_MM_INP_NORM_B); - // post_trunk_norm: applied after all ViT blocks, before the merger - model.post_ln_w = get_tensor(string_format(TN_MM_POST_NORM, "weight")); - } break; - case PROJECTOR_TYPE_DOTS3NOTE_A: - { - model.conv2d_1_w = get_tensor(string_format(TN_CONV2D, 1, "weight")); - model.conv2d_1_b = get_tensor(string_format(TN_CONV2D, 1, "bias")); - model.conv2d_2_w = get_tensor(string_format(TN_CONV2D, 2, "weight")); - model.conv2d_2_b = get_tensor(string_format(TN_CONV2D, 2, "bias")); - model.conv2d_3_w = get_tensor(string_format(TN_CONV2D, 3, "weight")); - model.conv2d_3_b = get_tensor(string_format(TN_CONV2D, 3, "bias")); - model.conv_out_w = get_tensor(string_format(TN_CONV_OUT, "weight")); // no bias - // adapter: LayerNorm -> Linear -> GELU -> Linear - model.mm_norm_pre_w = get_tensor(string_format(TN_MM_NORM_PRE, "weight")); - model.mm_norm_pre_b = get_tensor(string_format(TN_MM_NORM_PRE, "bias")); - model.mm_1_w = get_tensor(string_format(TN_MM_AUDIO_MLP, 1, "weight")); - model.mm_1_b = get_tensor(string_format(TN_MM_AUDIO_MLP, 1, "bias")); - model.mm_2_w = get_tensor(string_format(TN_MM_AUDIO_MLP, 3, "weight")); - model.mm_2_b = get_tensor(string_format(TN_MM_AUDIO_MLP, 3, "bias")); - } break; - case PROJECTOR_TYPE_ULTRAVOX: - { - model.conv1d_1_w = get_tensor(string_format(TN_CONV1D, 1, "weight")); - model.conv1d_1_b = get_tensor(string_format(TN_CONV1D, 1, "bias")); - model.conv1d_2_w = get_tensor(string_format(TN_CONV1D, 2, "weight")); - model.conv1d_2_b = get_tensor(string_format(TN_CONV1D, 2, "bias")); - model.mm_1_w = get_tensor(string_format(TN_MM_AUDIO_MLP, 1, "weight")); - model.mm_2_w = get_tensor(string_format(TN_MM_AUDIO_MLP, 2, "weight")); - model.mm_norm_pre_w = get_tensor(string_format(TN_MM_NORM_PRE, "weight")); - model.mm_norm_mid_w = get_tensor(string_format(TN_MM_NORM_MID, "weight")); - } break; - case PROJECTOR_TYPE_MERALION: - { - // Whisper encoder conv layers - model.conv1d_1_w = get_tensor(string_format(TN_CONV1D, 1, "weight")); - model.conv1d_1_b = get_tensor(string_format(TN_CONV1D, 1, "bias")); - model.conv1d_2_w = get_tensor(string_format(TN_CONV1D, 2, "weight")); - model.conv1d_2_b = get_tensor(string_format(TN_CONV1D, 2, "bias")); - // MERaLiON adaptor: 4 linear layers + ln_pre - // linear_0 = frame compression (19200->6400) + SiLU - // linear_1 = gate_proj (6400->6400) for GLU - // linear_2 = pool_proj (6400->6400) for GLU - // linear_3 = out_proj (6400->3584) - model.mm_0_w = get_tensor(string_format(TN_MM_AUDIO_MLP, 0, "weight")); - model.mm_0_b = get_tensor(string_format(TN_MM_AUDIO_MLP, 0, "bias")); - model.mm_1_w = get_tensor(string_format(TN_MM_AUDIO_MLP, 1, "weight")); - model.mm_1_b = get_tensor(string_format(TN_MM_AUDIO_MLP, 1, "bias")); - model.mm_2_w = get_tensor(string_format(TN_MM_AUDIO_MLP, 2, "weight")); - model.mm_2_b = get_tensor(string_format(TN_MM_AUDIO_MLP, 2, "bias")); - model.mm_3_w = get_tensor(string_format(TN_MM_AUDIO_MLP, 3, "weight")); - model.mm_3_b = get_tensor(string_format(TN_MM_AUDIO_MLP, 3, "bias")); - // ln_speech (LayerNorm before adaptor) - model.mm_norm_pre_w = get_tensor(string_format(TN_MM_NORM_PRE, "weight")); - model.mm_norm_pre_b = get_tensor(string_format(TN_MM_NORM_PRE, "bias")); - } break; - case PROJECTOR_TYPE_QWEN2A: - { - model.conv1d_1_w = get_tensor(string_format(TN_CONV1D, 1, "weight")); - model.conv1d_1_b = get_tensor(string_format(TN_CONV1D, 1, "bias")); - model.conv1d_2_w = get_tensor(string_format(TN_CONV1D, 2, "weight")); - model.conv1d_2_b = get_tensor(string_format(TN_CONV1D, 2, "bias")); - model.mm_fc_w = get_tensor(string_format(TN_MM_AUDIO_FC, "weight")); - model.mm_fc_b = get_tensor(string_format(TN_MM_AUDIO_FC, "bias")); - } break; - case PROJECTOR_TYPE_QWEN3A: - { - model.conv2d_1_w = get_tensor(string_format(TN_CONV2D, 1, "weight")); - model.conv2d_1_b = get_tensor(string_format(TN_CONV2D, 1, "bias")); - model.conv2d_2_w = get_tensor(string_format(TN_CONV2D, 2, "weight")); - model.conv2d_2_b = get_tensor(string_format(TN_CONV2D, 2, "bias")); - model.conv2d_3_w = get_tensor(string_format(TN_CONV2D, 3, "weight")); - model.conv2d_3_b = get_tensor(string_format(TN_CONV2D, 3, "bias")); - model.conv_out_w = get_tensor(string_format(TN_CONV_OUT, "weight")); // no bias - model.mm_1_w = get_tensor(string_format(TN_MM_AUDIO_MLP, 1, "weight")); - model.mm_1_b = get_tensor(string_format(TN_MM_AUDIO_MLP, 1, "bias")); - model.mm_2_w = get_tensor(string_format(TN_MM_AUDIO_MLP, 2, "weight")); - model.mm_2_b = get_tensor(string_format(TN_MM_AUDIO_MLP, 2, "bias")); - } break; - case PROJECTOR_TYPE_MIMO_AUDIO: - { - model.conv1d_1_w = get_tensor(string_format(TN_CONV1D, 1, "weight")); - model.conv1d_1_b = get_tensor(string_format(TN_CONV1D, 1, "bias")); - model.conv1d_2_w = get_tensor(string_format(TN_CONV1D, 2, "weight")); - model.conv1d_2_b = get_tensor(string_format(TN_CONV1D, 2, "bias")); - model.downsample_conv_w = get_tensor(string_format(TN_A_DOWNSAMPLE_CONV, "weight")); - model.downsample_norm_w = get_tensor(string_format(TN_A_DOWNSAMPLE_NORM, "weight")); - model.downsample_norm_b = get_tensor(string_format(TN_A_DOWNSAMPLE_NORM, "bias")); - model.rvq_codebook = get_tensor(string_format(TN_A_RVQ_CODEBOOK, "weight"), false); - model.mm_a_code_embd = get_tensor(string_format(TN_MM_A_CODE_EMBD, "weight"), false); - if (!model.rvq_codebook || !model.mm_a_code_embd) { - throw std::runtime_error(string_format("%s: mimo_audio: missing %s or %s\n", __func__, - TN_A_RVQ_CODEBOOK, TN_MM_A_CODE_EMBD)); - } - // hparams.rvq_codebook_size comes from GGUF metadata and is independent of the - // tensors' actual shapes - bound it so codebook/code_embd views built from it - // (mimo-audio.cpp) can never read past either tensor's allocated bins/vocab. - for (int32_t bins : hparams.rvq_codebook_size) { - if (bins <= 0 || bins > model.rvq_codebook->ne[1] || bins > model.mm_a_code_embd->ne[1]) { - throw std::runtime_error(string_format( - "%s: mimo_audio: %s entry (%d) out of range for codebook/code_embd tensors\n", - __func__, KEY_A_RVQ_CODEBOOK_SIZE, bins)); - } - } - - // LLM-side connector: input_local_transformer + projection - model.mm_a_local_layers.resize(hparams.audio_local_n_layer); - for (int il = 0; il < hparams.audio_local_n_layer; il++) { - auto & layer = model.mm_a_local_layers[il]; - layer.q_w = get_tensor(string_format(TN_MM_A_LOCAL_ATTN_Q, il, "weight")); - layer.q_b = get_tensor(string_format(TN_MM_A_LOCAL_ATTN_Q, il, "bias")); - layer.k_w = get_tensor(string_format(TN_MM_A_LOCAL_ATTN_K, il, "weight")); - layer.k_b = get_tensor(string_format(TN_MM_A_LOCAL_ATTN_K, il, "bias")); - layer.v_w = get_tensor(string_format(TN_MM_A_LOCAL_ATTN_V, il, "weight")); - layer.v_b = get_tensor(string_format(TN_MM_A_LOCAL_ATTN_V, il, "bias")); - layer.o_w = get_tensor(string_format(TN_MM_A_LOCAL_ATTN_OUT, il, "weight")); - layer.ff_gate_w = get_tensor(string_format(TN_MM_A_LOCAL_FFN_GATE, il, "weight")); - layer.ff_up_w = get_tensor(string_format(TN_MM_A_LOCAL_FFN_UP, il, "weight")); - layer.ff_down_w = get_tensor(string_format(TN_MM_A_LOCAL_FFN_DOWN, il, "weight")); - layer.ln_1_w = get_tensor(string_format(TN_MM_A_LOCAL_LN1, il, "weight")); - layer.ln_2_w = get_tensor(string_format(TN_MM_A_LOCAL_LN2, il, "weight")); - } - model.mm_a_local_norm_w = get_tensor(string_format(TN_MM_A_LOCAL_NORM, "weight")); - - model.mm_1_w = get_tensor(string_format(TN_MM_AUDIO_MLP, 1, "weight")); - model.mm_2_w = get_tensor(string_format(TN_MM_AUDIO_MLP, 2, "weight")); - } break; - case PROJECTOR_TYPE_QWEN3TTS_SPKENC: - { - // stem TDNN (block 0) - model.conv1d_1_w = get_tensor(string_format(TN_CONV1D, 0, "weight")); - model.conv1d_1_b = get_tensor(string_format(TN_CONV1D, 0, "bias")); - - // SE-Res2Net blocks (GGUF bid 1..3, one per hparams.n_layer) - model.layers.resize(hparams.n_layer); - for (int il = 0; il < hparams.n_layer; il++) { - auto & layer = model.layers[il]; - int bid = il + 1; - layer.conv_pw1_w = get_tensor(string_format(TN_CONV_PW1, prefix, bid, "weight")); - layer.conv_pw1_b = get_tensor(string_format(TN_CONV_PW1, prefix, bid, "bias")); - layer.conv_pw2_w = get_tensor(string_format(TN_CONV_PW2, prefix, bid, "weight")); - layer.conv_pw2_b = get_tensor(string_format(TN_CONV_PW2, prefix, bid, "bias")); - layer.se_conv1_w = get_tensor(string_format(TN_A_SE_CONV1, bid, "weight")); - layer.se_conv1_b = get_tensor(string_format(TN_A_SE_CONV1, bid, "bias")); - layer.se_conv2_w = get_tensor(string_format(TN_A_SE_CONV2, bid, "weight")); - layer.se_conv2_b = get_tensor(string_format(TN_A_SE_CONV2, bid, "bias")); - layer.res2_conv_w.resize(7); - layer.res2_conv_b.resize(7); - for (int xid = 0; xid < 7; xid++) { - layer.res2_conv_w[xid] = get_tensor(string_format(TN_A_CONV_RES2, bid, xid, "weight")); - layer.res2_conv_b[xid] = get_tensor(string_format(TN_A_CONV_RES2, bid, xid, "bias")); - } - } - - // multi-layer feature aggregation - model.conv_out_w = get_tensor(string_format(TN_CONV_OUT, "weight")); - model.conv_out_b = get_tensor(string_format(TN_CONV_OUT, "bias")); - - // attentive statistics pooling - model.spk_asp_attn_w = get_tensor(string_format(TN_A_ASP_ATTN, "weight")); - model.spk_asp_attn_b = get_tensor(string_format(TN_A_ASP_ATTN, "bias")); - model.spk_asp_tdnn_w = get_tensor(string_format(TN_A_ASP_TDNN, "weight")); - model.spk_asp_tdnn_b = get_tensor(string_format(TN_A_ASP_TDNN, "bias")); - - // final speaker embedding projection - model.mm_fc_w = get_tensor(string_format(TN_MM_AUDIO_FC, "weight")); - model.mm_fc_b = get_tensor(string_format(TN_MM_AUDIO_FC, "bias")); - } break; - case PROJECTOR_TYPE_POCKETTTS_SPKENC: - { - load_seanet(model.seanet, false); - model.downsample_w = get_tensor(string_format(TN_A_DOWNSAMPLE_CONV, "weight")); - model.spk_proj_w = get_tensor(string_format(TN_A_SPEAKER_PROJ, "weight")); - } break; - case PROJECTOR_TYPE_POCKETTTS_GEN: - { - auto & flow = model.flow; - flow.input_proj_w = get_tensor(string_format(TN_A_GEN_FLOW_INPUT_PROJ, "weight")); - flow.input_proj_b = get_tensor(string_format(TN_A_GEN_FLOW_INPUT_PROJ, "bias")); - flow.cond_embd_w = get_tensor(string_format(TN_A_GEN_FLOW_COND_EMBD, "weight")); - flow.cond_embd_b = get_tensor(string_format(TN_A_GEN_FLOW_COND_EMBD, "bias")); - flow.final_ada_w = get_tensor(string_format(TN_A_GEN_FLOW_FINAL_ADA, "weight")); - flow.final_ada_b = get_tensor(string_format(TN_A_GEN_FLOW_FINAL_ADA, "bias")); - flow.final_proj_w = get_tensor(string_format(TN_A_GEN_FLOW_FINAL_PROJ, "weight")); - flow.final_proj_b = get_tensor(string_format(TN_A_GEN_FLOW_FINAL_PROJ, "bias")); - - flow.time.resize(2); - for (size_t i = 0; i < flow.time.size(); i++) { - auto & t = flow.time[i]; - t.freqs = get_tensor(string_format(TN_A_GEN_FLOW_TIME_FREQS, (int) i)); - t.up_w = get_tensor(string_format(TN_A_GEN_FLOW_TIME_UP, (int) i, "weight")); - t.up_b = get_tensor(string_format(TN_A_GEN_FLOW_TIME_UP, (int) i, "bias")); - t.down_w = get_tensor(string_format(TN_A_GEN_FLOW_TIME_DOWN, (int) i, "weight")); - t.down_b = get_tensor(string_format(TN_A_GEN_FLOW_TIME_DOWN, (int) i, "bias")); - t.norm = get_tensor(string_format(TN_A_GEN_FLOW_TIME_NORM, (int) i)); - } - - // one AdaLN block per flow depth, the count is only known from the tensors - for (int il = 0; ; il++) { - ggml_tensor * probe = get_tensor(string_format(TN_A_GEN_FLOW_BLK_NORM, il, "weight"), false); - if (probe == nullptr) { - break; - } - clip_flow_net::block blk; - blk.norm_w = probe; - blk.norm_b = get_tensor(string_format(TN_A_GEN_FLOW_BLK_NORM, il, "bias")); - blk.up_w = get_tensor(string_format(TN_A_GEN_FLOW_BLK_UP, il, "weight")); - blk.up_b = get_tensor(string_format(TN_A_GEN_FLOW_BLK_UP, il, "bias")); - blk.down_w = get_tensor(string_format(TN_A_GEN_FLOW_BLK_DOWN, il, "weight")); - blk.down_b = get_tensor(string_format(TN_A_GEN_FLOW_BLK_DOWN, il, "bias")); - blk.ada_w = get_tensor(string_format(TN_A_GEN_FLOW_BLK_ADA, il, "weight")); - blk.ada_b = get_tensor(string_format(TN_A_GEN_FLOW_BLK_ADA, il, "bias")); - flow.blocks.push_back(blk); - } - - model.gen_out_eos_w = get_tensor(string_format(TN_A_GEN_OUT_EOS, "weight")); - model.gen_out_eos_b = get_tensor(string_format(TN_A_GEN_OUT_EOS, "bias")); - model.gen_input_lin_w = get_tensor(string_format(TN_A_GEN_INPUT_LINEAR, "weight")); - model.gen_emb_mean = get_tensor(TN_A_GEN_EMB_MEAN); - model.gen_emb_std = get_tensor(TN_A_GEN_EMB_STD); - - // mimi decoder - model.gen_quant_out_w = get_tensor(string_format(TN_A_GEN_WAV_QUANT_OUT, "weight")); - model.gen_upsample_w = get_tensor(string_format(TN_A_GEN_WAV_UPSAMPLE, "weight")); - load_seanet(model.seanet, true); - model.gen_tfm_layers.resize(hparams.n_layer); - for (int il = 0; il < hparams.n_layer; il++) { - auto & layer = model.gen_tfm_layers[il]; - const char * p = "a.gen.wav.tfm"; - layer.ln_1_w = get_tensor(string_format(TN_LN_1, p, il, "weight")); - layer.ln_1_b = get_tensor(string_format(TN_LN_1, p, il, "bias")); - layer.q_w = get_tensor(string_format(TN_ATTN_Q, p, il, "weight")); - layer.k_w = get_tensor(string_format(TN_ATTN_K, p, il, "weight")); - layer.v_w = get_tensor(string_format(TN_ATTN_V, p, il, "weight")); - layer.o_w = get_tensor(string_format(TN_ATTN_OUTPUT, p, il, "weight")); - layer.ls_1_w = get_tensor(string_format(TN_LS_1, p, il, "weight")); - layer.ln_2_w = get_tensor(string_format(TN_LN_2, p, il, "weight")); - layer.ln_2_b = get_tensor(string_format(TN_LN_2, p, il, "bias")); - layer.ff_up_w = get_tensor(string_format(TN_FFN_UP, p, il, "weight")); - layer.ff_down_w = get_tensor(string_format(TN_FFN_DOWN, p, il, "weight")); - layer.ls_2_w = get_tensor(string_format(TN_LS_2, p, il, "weight")); - } - - for (int64_t i = 0; i < gguf_get_n_tensors(ctx_gguf.get()); ++i) { - const char * name = gguf_get_tensor_name(ctx_gguf.get(), i); - if (strncmp(name, "voice.", 6) != 0) { - continue; - } - const ggml_tensor * t = ggml_get_tensor(ctx_meta.get(), name); - if (t->type != GGML_TYPE_F32 || ggml_n_dims(t) > 2 || t->ne[0] != model.gen_input_lin_w->ne[1]) { - throw std::runtime_error(string_format("%s: invalid voice tensor %s\n", __func__, name)); - } - auto & embd = model.gen_voices[name + 6]; - embd.resize(ggml_nelements(t)); - fin.seekg(tensor_offset.at(name), std::ios::beg); - if (!fin.read(reinterpret_cast(embd.data()), ggml_nbytes(t))) { - throw std::runtime_error(string_format("%s: failed to read %s\n", __func__, name)); - } - } - if (!model.gen_voices.empty()) { - LOG_INF("%s: %zu predefined voices\n", __func__, model.gen_voices.size()); - } - } break; - case PROJECTOR_TYPE_LFM3A_GEN: - { - // must match the "tts" chat template, see conversion/lfm3.py - // note for contributor: this is one-off fix, do not repeat this for future models - model.gen_voices = { {"female", {}}, {"male", {}} }; - - // depthformer, the generic loop above loaded the a.gen.code.blk.* tensors - for (int il = 0; il < hparams.n_layer; ++il) { - model.layers[il].attn_gate_w = get_tensor(string_format(TN_ATTN_GATE, prefix, il, "weight")); - } - model.gen_code_proj_in_w = get_tensor(string_format(TN_A_GEN_CODE_PROJ_IN, "weight")); - model.gen_code_proj_in_b = get_tensor(string_format(TN_A_GEN_CODE_PROJ_IN, "bias")); - model.gen_code_embd_w = get_tensor(string_format(TN_A_GEN_CODE_EMBD, "weight")); // tied head - model.gen_code_out_embd_w = get_tensor(string_format(TN_A_GEN_CODE_OUT_EMBD, "weight")); - model.gen_code_norm_w = get_tensor(string_format(TN_A_GEN_CODE_NORM, "weight")); - model.gen_code_head_norm_w = get_tensor(string_format(TN_A_GEN_CODE_HEAD_NORM, "weight")); - model.gen_code_mask_embd = get_tensor(TN_A_GEN_CODE_MASK_EMBD); - - // codec decoder: FSQ level table, one projection over all codebooks, LFM3 decoder -> polar STFT frames - auto & c2w = model.c2w; - c2w.quant_first_cb_w = get_tensor(string_format(TN_A_GEN_WAV_QUANT_FIRST_CB, "weight")); - c2w.quant_first_out_w = get_tensor(string_format(TN_A_GEN_WAV_QUANT_FIRST_OUT, "weight")); - c2w.quant_first_out_b = get_tensor(string_format(TN_A_GEN_WAV_QUANT_FIRST_OUT, "bias")); - c2w.tfm_in_proj_w = get_tensor(string_format(TN_A_GEN_WAV_TFM_IN_PROJ, "weight")); - c2w.tfm_in_proj_b = get_tensor(string_format(TN_A_GEN_WAV_TFM_IN_PROJ, "bias")); - c2w.tfm_out_proj_w = get_tensor(string_format(TN_A_GEN_WAV_TFM_OUT_PROJ, "weight")); - c2w.tfm_out_proj_b = get_tensor(string_format(TN_A_GEN_WAV_TFM_OUT_PROJ, "bias")); - c2w.tfm_output_norm_w = get_tensor(string_format(TN_A_GEN_WAV_TFM_OUT_NORM, "weight")); - - // each layer is a short conv block or a gated attention block, both with sandwich norms - c2w.tfm_layers.resize(hparams.wav_tfm_n_layer); - for (int il = 0; il < hparams.wav_tfm_n_layer; il++) { - auto & layer = c2w.tfm_layers[il]; - const char * p = "a.gen.wav.tfm"; - layer.ln_1_w = get_tensor(string_format(TN_LN_1, p, il, "weight")); - layer.attn_post_norm_w = get_tensor(string_format(TN_ATTN_POST_NORM, p, il, "weight")); - layer.ln_2_w = get_tensor(string_format(TN_LN_2, p, il, "weight")); - layer.ff_post_norm_w = get_tensor(string_format(TN_FFN_POST_NORM, p, il, "weight")); - layer.ff_gate_w = get_tensor(string_format(TN_FFN_GATE, p, il, "weight")); - layer.ff_up_w = get_tensor(string_format(TN_FFN_UP, p, il, "weight")); - layer.ff_down_w = get_tensor(string_format(TN_FFN_DOWN, p, il, "weight")); - layer.conv_pw1_w = get_tensor(string_format(TN_CONV_PW1, p, il, "weight"), false); - layer.conv_dw_w = get_tensor(string_format(TN_CONV_DW, p, il, "weight"), false); - layer.conv_pw2_w = get_tensor(string_format(TN_CONV_PW2, p, il, "weight"), false); - layer.q_w = get_tensor(string_format(TN_ATTN_Q, p, il, "weight"), false); - layer.k_w = get_tensor(string_format(TN_ATTN_K, p, il, "weight"), false); - layer.v_w = get_tensor(string_format(TN_ATTN_V, p, il, "weight"), false); - layer.o_w = get_tensor(string_format(TN_ATTN_OUTPUT, p, il, "weight"), false); - layer.q_norm = get_tensor(string_format(TN_ATTN_Q_NORM, p, il, "weight"), false); - layer.k_norm = get_tensor(string_format(TN_ATTN_K_NORM, p, il, "weight"), false); - layer.attn_gate_w = get_tensor(string_format(TN_ATTN_GATE, p, il, "weight"), false); - const bool is_conv = layer.conv_pw1_w && layer.conv_dw_w && layer.conv_pw2_w; - const bool is_attn = layer.q_w && layer.k_w && layer.v_w && layer.o_w && layer.q_norm && layer.k_norm && layer.attn_gate_w; - if (is_conv == is_attn) { - throw std::runtime_error(string_format("%s: codec layer %d is neither a conv nor an attention block\n", __func__, il)); - } - if (is_attn) { - const int d_head = layer.q_norm->ne[0]; - hparams.wav_tfm_n_head = layer.q_w->ne[1] / d_head; - hparams.wav_tfm_n_head_kv = layer.k_w->ne[1] / d_head; - } - } - } break; - case PROJECTOR_TYPE_QWEN3TTS_GEN: - { - // code_predictor, proj_in is absent when the talker and the predictor share the hidden size - model.gen_code_proj_in_w = get_tensor(string_format(TN_A_GEN_CODE_PROJ_IN, "weight"), false); - model.gen_code_proj_in_b = get_tensor(string_format(TN_A_GEN_CODE_PROJ_IN, "bias"), false); - model.gen_code_embd_w = get_tensor(string_format(TN_A_GEN_CODE_EMBD, "weight")); - model.gen_code_head_w = get_tensor(string_format(TN_A_GEN_CODE_HEAD, "weight")); - model.gen_code_out_embd_w = get_tensor(string_format(TN_A_GEN_CODE_OUT_EMBD, "weight")); - model.gen_code_norm_w = get_tensor(string_format(TN_A_GEN_CODE_NORM, "weight")); - - // code2wav: RVQ codes -> raw PCM, lives in the same ctx as code_predictor - { - auto & c2w = model.c2w; - - c2w.quant_first_in_w = get_tensor(string_format(TN_A_GEN_WAV_QUANT_FIRST_IN, "weight")); - c2w.quant_first_out_w = get_tensor(string_format(TN_A_GEN_WAV_QUANT_FIRST_OUT, "weight")); - c2w.quant_first_cb_w = get_tensor(string_format(TN_A_GEN_WAV_QUANT_FIRST_CB, "weight")); - c2w.quant_rest_in_w = get_tensor(string_format(TN_A_GEN_WAV_QUANT_REST_IN, "weight")); - c2w.quant_rest_out_w = get_tensor(string_format(TN_A_GEN_WAV_QUANT_REST_OUT, "weight")); - c2w.quant_rest_cb_w = get_tensor(string_format(TN_A_GEN_WAV_QUANT_REST_CB, "weight")); - - c2w.pre_conv_w = get_tensor(string_format(TN_A_GEN_WAV_PRE_CONV, "weight")); - c2w.pre_conv_b = get_tensor(string_format(TN_A_GEN_WAV_PRE_CONV, "bias")); - - c2w.tfm_in_proj_w = get_tensor(string_format(TN_A_GEN_WAV_TFM_IN_PROJ, "weight")); - c2w.tfm_in_proj_b = get_tensor(string_format(TN_A_GEN_WAV_TFM_IN_PROJ, "bias")); - c2w.tfm_out_proj_w = get_tensor(string_format(TN_A_GEN_WAV_TFM_OUT_PROJ, "weight")); - c2w.tfm_out_proj_b = get_tensor(string_format(TN_A_GEN_WAV_TFM_OUT_PROJ, "bias")); - c2w.tfm_output_norm_w = get_tensor(string_format(TN_A_GEN_WAV_TFM_OUT_NORM, "weight")); - - // loaded manually, the generic model.layers loop is taken by code_predictor - c2w.tfm_layers.resize(hparams.wav_tfm_n_layer); - for (int il = 0; il < hparams.wav_tfm_n_layer; il++) { - auto & layer = c2w.tfm_layers[il]; - const char * p = "a.gen.wav.tfm"; - layer.q_w = get_tensor(string_format(TN_ATTN_Q, p, il, "weight")); - layer.k_w = get_tensor(string_format(TN_ATTN_K, p, il, "weight")); - layer.v_w = get_tensor(string_format(TN_ATTN_V, p, il, "weight")); - layer.o_w = get_tensor(string_format(TN_ATTN_OUTPUT, p, il, "weight")); - layer.ln_1_w = get_tensor(string_format(TN_LN_1, p, il, "weight")); - layer.ln_2_w = get_tensor(string_format(TN_LN_2, p, il, "weight")); - layer.ls_1_w = get_tensor(string_format(TN_LS_1, p, il, "weight")); - layer.ls_2_w = get_tensor(string_format(TN_LS_2, p, il, "weight")); - layer.ff_gate_w = get_tensor(string_format(TN_FFN_GATE, p, il, "weight")); - layer.ff_up_w = get_tensor(string_format(TN_FFN_UP, p, il, "weight")); - layer.ff_down_w = get_tensor(string_format(TN_FFN_DOWN, p, il, "weight")); - } - - // upsample: 2x (causal ConvTranspose1d + ConvNeXt block) - c2w.upsample.resize(hparams.wav_upsample_n_block); - for (int il = 0; il < hparams.wav_upsample_n_block; il++) { - auto & up = c2w.upsample[il]; - up.conv_w = get_tensor(string_format(TN_A_GEN_WAV_UP_CONV, il, "weight")); - up.conv_b = get_tensor(string_format(TN_A_GEN_WAV_UP_CONV, il, "bias")); - up.dwconv_w = get_tensor(string_format(TN_A_GEN_WAV_UP_DWCONV, il, "weight")); - up.dwconv_b = get_tensor(string_format(TN_A_GEN_WAV_UP_DWCONV, il, "bias")); - up.norm_w = get_tensor(string_format(TN_A_GEN_WAV_UP_NORM, il, "weight")); - up.norm_b = get_tensor(string_format(TN_A_GEN_WAV_UP_NORM, il, "bias")); - up.pw1_w = get_tensor(string_format(TN_A_GEN_WAV_UP_PW1, il, "weight")); - up.pw1_b = get_tensor(string_format(TN_A_GEN_WAV_UP_PW1, il, "bias")); - up.pw2_w = get_tensor(string_format(TN_A_GEN_WAV_UP_PW2, il, "weight")); - up.pw2_b = get_tensor(string_format(TN_A_GEN_WAV_UP_PW2, il, "bias")); - up.gamma = get_tensor(string_format(TN_A_GEN_WAV_UP_GAMMA, il)); - } - - // DAC decoder: conv_pre + n upsample blocks (each with n_res residual units) + conv_post - c2w.dac_entry_w = get_tensor(string_format(TN_A_GEN_WAV_DAC_ENTRY, "weight")); - c2w.dac_entry_b = get_tensor(string_format(TN_A_GEN_WAV_DAC_ENTRY, "bias")); - - c2w.dac.resize(hparams.wav_dac_n_block); - for (int il = 0; il < hparams.wav_dac_n_block; il++) { - auto & blk = c2w.dac[il]; - blk.snake_alpha = get_tensor(string_format(TN_A_GEN_WAV_DAC_SNAKE, il, "alpha")); - blk.snake_beta = get_tensor(string_format(TN_A_GEN_WAV_DAC_SNAKE, il, "beta")); - blk.conv_w = get_tensor(string_format(TN_A_GEN_WAV_DAC_CONV, il, "weight")); - blk.conv_b = get_tensor(string_format(TN_A_GEN_WAV_DAC_CONV, il, "bias")); - - blk.res.resize(hparams.wav_dac_n_res); - for (int ir = 0; ir < hparams.wav_dac_n_res; ir++) { - auto & res = blk.res[ir]; - res.act1_alpha = get_tensor(string_format(TN_A_GEN_WAV_DAC_RES_ACT1, il, ir, "alpha")); - res.act1_beta = get_tensor(string_format(TN_A_GEN_WAV_DAC_RES_ACT1, il, ir, "beta")); - res.conv1_w = get_tensor(string_format(TN_A_GEN_WAV_DAC_RES_CONV1, il, ir, "weight")); - res.conv1_b = get_tensor(string_format(TN_A_GEN_WAV_DAC_RES_CONV1, il, ir, "bias")); - res.act2_alpha = get_tensor(string_format(TN_A_GEN_WAV_DAC_RES_ACT2, il, ir, "alpha")); - res.act2_beta = get_tensor(string_format(TN_A_GEN_WAV_DAC_RES_ACT2, il, ir, "beta")); - res.conv2_w = get_tensor(string_format(TN_A_GEN_WAV_DAC_RES_CONV2, il, ir, "weight")); - res.conv2_b = get_tensor(string_format(TN_A_GEN_WAV_DAC_RES_CONV2, il, ir, "bias")); - } - } - - c2w.dac_post_snake_alpha = get_tensor(string_format(TN_A_GEN_WAV_DAC_POST_SNAKE, "alpha")); - c2w.dac_post_snake_beta = get_tensor(string_format(TN_A_GEN_WAV_DAC_POST_SNAKE, "beta")); - c2w.dac_post_conv_w = get_tensor(string_format(TN_A_GEN_WAV_DAC_POST_CONV, "weight")); - c2w.dac_post_conv_b = get_tensor(string_format(TN_A_GEN_WAV_DAC_POST_CONV, "bias")); - } - } break; - case PROJECTOR_TYPE_SOPRANO: - { - auto & v = model.vocos; - v.input_w = get_tensor(string_format(TN_A_GEN_WAV_INPUT, "weight")); - v.input_b = get_tensor(string_format(TN_A_GEN_WAV_INPUT, "bias")); - v.norm_w = get_tensor(string_format(TN_A_GEN_WAV_NORM, "weight")); - v.norm_b = get_tensor(string_format(TN_A_GEN_WAV_NORM, "bias")); - v.output_norm_w = get_tensor(string_format(TN_A_GEN_WAV_OUTPUT_NORM, "weight")); - v.output_norm_b = get_tensor(string_format(TN_A_GEN_WAV_OUTPUT_NORM, "bias")); - v.output_w = get_tensor(string_format(TN_A_GEN_WAV_OUTPUT, "weight")); - v.output_b = get_tensor(string_format(TN_A_GEN_WAV_OUTPUT, "bias")); - v.blocks.resize(hparams.n_layer); - for (int il = 0; il < hparams.n_layer; ++il) { - auto & b = v.blocks[il]; - b.dwconv_w = get_tensor(string_format(TN_A_GEN_WAV_UP_DWCONV, il, "weight")); - b.dwconv_b = get_tensor(string_format(TN_A_GEN_WAV_UP_DWCONV, il, "bias")); - b.norm_w = get_tensor(string_format(TN_A_GEN_WAV_UP_NORM, il, "weight")); - b.norm_b = get_tensor(string_format(TN_A_GEN_WAV_UP_NORM, il, "bias")); - b.pw1_w = get_tensor(string_format(TN_A_GEN_WAV_UP_PW1, il, "weight")); - b.pw1_b = get_tensor(string_format(TN_A_GEN_WAV_UP_PW1, il, "bias")); - b.pw2_w = get_tensor(string_format(TN_A_GEN_WAV_UP_PW2, il, "weight")); - b.pw2_b = get_tensor(string_format(TN_A_GEN_WAV_UP_PW2, il, "bias")); - b.gamma = get_tensor(string_format(TN_A_GEN_WAV_UP_GAMMA, il)); - } - } break; - case PROJECTOR_TYPE_VOXTRAL: - case PROJECTOR_TYPE_VOXTRAL_RT: - { - model.conv1d_1_w = get_tensor(string_format(TN_CONV1D, 1, "weight")); - model.conv1d_1_b = get_tensor(string_format(TN_CONV1D, 1, "bias")); - model.conv1d_2_w = get_tensor(string_format(TN_CONV1D, 2, "weight")); - model.conv1d_2_b = get_tensor(string_format(TN_CONV1D, 2, "bias")); - model.mm_1_w = get_tensor(string_format(TN_MM_AUDIO_MLP, 1, "weight")); - model.mm_2_w = get_tensor(string_format(TN_MM_AUDIO_MLP, 2, "weight")); - } break; - case PROJECTOR_TYPE_MUSIC_FLAMINGO: - { - model.conv1d_1_w = get_tensor(string_format(TN_CONV1D, 1, "weight")); - model.conv1d_1_b = get_tensor(string_format(TN_CONV1D, 1, "bias")); - model.conv1d_2_w = get_tensor(string_format(TN_CONV1D, 2, "weight")); - model.conv1d_2_b = get_tensor(string_format(TN_CONV1D, 2, "bias")); - model.mm_1_w = get_tensor(string_format(TN_MM_AUDIO_MLP, 1, "weight")); - model.mm_1_b = get_tensor(string_format(TN_MM_AUDIO_MLP, 1, "bias")); - model.mm_2_w = get_tensor(string_format(TN_MM_AUDIO_MLP, 2, "weight")); - model.mm_2_b = get_tensor(string_format(TN_MM_AUDIO_MLP, 2, "bias")); - } break; - case PROJECTOR_TYPE_INTERNVL: - { - model.mm_0_w = get_tensor(string_format(TN_MVLM_PROJ_MLP, 0, "weight")); - model.mm_0_b = get_tensor(string_format(TN_MVLM_PROJ_MLP, 0, "bias")); - model.mm_1_w = get_tensor(string_format(TN_MVLM_PROJ_MLP, 1, "weight")); - model.mm_1_b = get_tensor(string_format(TN_MVLM_PROJ_MLP, 1, "bias")); - model.mm_3_w = get_tensor(string_format(TN_MVLM_PROJ_MLP, 3, "weight")); - model.mm_3_b = get_tensor(string_format(TN_MVLM_PROJ_MLP, 3, "bias")); - } break; - case PROJECTOR_TYPE_NEMOTRON_V2_VL: - case PROJECTOR_TYPE_NEMOTRON_OMNI: - { - model.mm_0_w = get_tensor(string_format(TN_MVLM_PROJ_MLP, 0, "weight")); - model.mm_1_w = get_tensor(string_format(TN_MVLM_PROJ_MLP, 1, "weight")); - model.mm_3_w = get_tensor(string_format(TN_MVLM_PROJ_MLP, 3, "weight")); - } break; - case PROJECTOR_TYPE_GLMA: - { - model.conv1d_1_w = get_tensor(string_format(TN_CONV1D, 1, "weight")); - model.conv1d_1_b = get_tensor(string_format(TN_CONV1D, 1, "bias")); - model.conv1d_2_w = get_tensor(string_format(TN_CONV1D, 2, "weight")); - model.conv1d_2_b = get_tensor(string_format(TN_CONV1D, 2, "bias")); - model.mm_1_w = get_tensor(string_format(TN_MM_AUDIO_MLP, 1, "weight")); - model.mm_1_b = get_tensor(string_format(TN_MM_AUDIO_MLP, 1, "bias")); - model.mm_2_w = get_tensor(string_format(TN_MM_AUDIO_MLP, 2, "weight")); - model.mm_2_b = get_tensor(string_format(TN_MM_AUDIO_MLP, 2, "bias")); - model.mm_norm_pre_w = get_tensor(string_format(TN_MM_NORM_PRE, "weight")); - model.mm_norm_pre_b = get_tensor(string_format(TN_MM_NORM_PRE, "bias")); - model.mm_boi = get_tensor(string_format(TN_TOK_BOI)); - model.mm_eoi = get_tensor(string_format(TN_TOK_EOI)); - } break; - case PROJECTOR_TYPE_LLAMA4: - { - model.mm_model_proj = get_tensor(string_format(TN_MM_PROJECTOR, "weight")); - model.mm_model_mlp_1_w = get_tensor(string_format(TN_MVLM_PROJ_MLP, 1, "weight")); - model.mm_model_mlp_2_w = get_tensor(string_format(TN_MVLM_PROJ_MLP, 2, "weight")); - } break; - case PROJECTOR_TYPE_COGVLM: - { - model.mm_model_proj = get_tensor(string_format(TN_MM_PROJECTOR, "weight")); - model.mm_post_fc_norm_w = get_tensor(string_format(TN_MM_POST_FC_NORM, "weight")); - model.mm_post_fc_norm_b = get_tensor(string_format(TN_MM_POST_FC_NORM, "bias")); - model.mm_h_to_4h_w = get_tensor(string_format(TN_MM_H_TO_4H, "weight")); - model.mm_gate_w = get_tensor(string_format(TN_MM_GATE, "weight")); - model.mm_4h_to_h_w = get_tensor(string_format(TN_MM_4H_TO_H, "weight")); - model.mm_boi = get_tensor(TN_TOK_BOI); - model.mm_eoi = get_tensor(TN_TOK_EOI); - } break; - case PROJECTOR_TYPE_HUNYUANVL: - { - // proj.0 -> mm.0 (conv1), proj.2 -> mm.2 (conv2), mlp -> mm.model.fc (linear) - model.mm_0_w = get_tensor(string_format(TN_LLAVA_PROJ, 0, "weight")); - model.mm_0_b = get_tensor(string_format(TN_LLAVA_PROJ, 0, "bias")); - model.mm_1_w = get_tensor(string_format(TN_LLAVA_PROJ, 2, "weight")); - model.mm_1_b = get_tensor(string_format(TN_LLAVA_PROJ, 2, "bias")); - model.mm_model_proj = get_tensor(string_format(TN_MM_PROJECTOR, "weight")); - model.mm_model_proj_b = get_tensor(string_format(TN_MM_PROJECTOR, "bias")); - model.mm_pre_norm_w = get_tensor(string_format(TN_MM_PRE_NORM, "weight")); - model.mm_post_norm_w = get_tensor(string_format(TN_MM_POST_NORM, "weight")); - model.mm_img_begin = get_tensor(TN_MM_IMG_BEGIN); - model.mm_img_end = get_tensor(TN_MM_IMG_END); - model.image_newline = get_tensor(TN_IMAGE_NEWLINE); - model.view_seperator = get_tensor(TN_IMAGE_SEPERATOR, false); - } break; - case PROJECTOR_TYPE_JANUS_PRO: - { - model.mm_0_w = get_tensor(string_format(TN_LLAVA_PROJ, 0, "weight")); - model.mm_0_b = get_tensor(string_format(TN_LLAVA_PROJ, 0, "bias")); - model.mm_1_w = get_tensor(string_format(TN_LLAVA_PROJ, 1, "weight")); - model.mm_1_b = get_tensor(string_format(TN_LLAVA_PROJ, 1, "bias")); - } break; - case PROJECTOR_TYPE_PHI4: - { - model.mm_0_w = get_tensor(string_format(TN_LLAVA_PROJ, 0, "weight")); - model.mm_0_b = get_tensor(string_format(TN_LLAVA_PROJ, 0, "bias")); - model.mm_2_w = get_tensor(string_format(TN_LLAVA_PROJ, 2, "weight")); - model.mm_2_b = get_tensor(string_format(TN_LLAVA_PROJ, 2, "bias")); - } break; - case PROJECTOR_TYPE_DEEPSEEKOCR: - case PROJECTOR_TYPE_DEEPSEEKOCR2: - { - model.pos_embed = get_tensor(string_format(TN_SAM_POS_EMBD, "weight")); - model.patch_embed_proj_w = get_tensor(string_format(TN_SAM_PATCH_EMBD, "weight")); - model.patch_embed_proj_b = get_tensor(string_format(TN_SAM_PATCH_EMBD, "bias")); - model.n_sam_layers = hparams.sam_n_layer; - model.sam_layers.resize(model.n_sam_layers); - for (int il = 0; il < model.n_sam_layers; ++il) { - auto & layer = model.sam_layers[il]; - layer.qkv_w = get_tensor(string_format(TN_SAM_ATTN_QKV, il, "weight")); - layer.qkv_b = get_tensor(string_format(TN_SAM_ATTN_QKV, il, "bias")); - layer.o_w = get_tensor(string_format(TN_SAM_ATTN_OUT, il, "weight")); - layer.o_b = get_tensor(string_format(TN_SAM_ATTN_OUT, il, "bias")); - layer.ln_1_w = get_tensor(string_format(TN_SAM_PRE_NORM, il, "weight")); - layer.ln_1_b = get_tensor(string_format(TN_SAM_PRE_NORM, il, "bias")); - layer.ln_2_w = get_tensor(string_format(TN_SAM_POST_NORM, il, "weight")); - layer.ln_2_b = get_tensor(string_format(TN_SAM_POST_NORM, il, "bias")); - layer.rel_pos_h = get_tensor(string_format(TN_SAM_ATTN_POS_H, il, "weight")); - layer.rel_pos_w = get_tensor(string_format(TN_SAM_ATTN_POS_W, il, "weight")); - layer.ff_up_w = get_tensor(string_format(TN_SAM_FFN_UP, il, "weight")); - layer.ff_up_b = get_tensor(string_format(TN_SAM_FFN_UP, il, "bias")); - layer.ff_down_w = get_tensor(string_format(TN_SAM_FFN_DOWN, il, "weight")); - layer.ff_down_b = get_tensor(string_format(TN_SAM_FFN_DOWN, il, "bias")); - } - model.neck_0_w = get_tensor(string_format(TN_SAM_NECK, 0, "weight")); - model.neck_1_b = get_tensor(string_format(TN_SAM_NECK, 1, "bias")); - model.neck_1_w = get_tensor(string_format(TN_SAM_NECK, 1, "weight")); - model.neck_2_w = get_tensor(string_format(TN_SAM_NECK, 2, "weight")); - model.neck_3_b = get_tensor(string_format(TN_SAM_NECK, 3, "bias")); - model.neck_3_w = get_tensor(string_format(TN_SAM_NECK, 3, "weight")); - model.net_2 = get_tensor(string_format(TN_SAM_NET, 2, "weight")); - model.net_3 = get_tensor(string_format(TN_SAM_NET, 3, "weight")); - model.image_newline = get_tensor(TN_IMAGE_NEWLINE, false); - model.view_seperator = get_tensor(TN_IMAGE_SEPERATOR); - model.mm_fc_w = get_tensor(string_format(TN_MM_PROJECTOR, "weight")); - model.mm_fc_b = get_tensor(string_format(TN_MM_PROJECTOR, "bias")); - model.resample_query_768 = get_tensor(string_format(TN_RESMPL_QUERY, 768, "weight"), false); - model.resample_query_1024 = get_tensor(string_format(TN_RESMPL_QUERY, 1024, "weight"), false); - } break; - case PROJECTOR_TYPE_GEMMA4A: - { - for (int i = 0; i < 2; i++) { - model.sscp_conv_w[i] = get_tensor(string_format(TN_A_CONV1D, i, "weight")); - model.sscp_conv_b[i] = get_tensor(string_format(TN_A_CONV1D, i, "bias"), false); - model.sscp_norm_w[i] = get_tensor(string_format(TN_A_CONV1D_NORM, i, "weight"), false); - } - model.sscp_inp_proj_w = get_tensor(string_format(TN_A_INP_PROJ, "weight")); - model.sscp_inp_proj_b = get_tensor(string_format(TN_A_INP_PROJ, "bias"), false); - model.audio_out_proj_w = get_tensor(string_format(TN_A_OUT_PROJ, "weight"), false); - model.audio_out_proj_b = get_tensor(string_format(TN_A_OUT_PROJ, "bias"), false); - // audio multimodal embedder (mm.a.* namespace, not mm.*) - model.mm_soft_emb_norm_w = get_tensor(string_format(TN_A_MM_SOFT_EMB_N, "weight"), false); - model.mm_input_proj_w = get_tensor(string_format(TN_A_MM_INP_PROJ, "weight"), false); - - // Per-layer tensors NOT loaded by the generic loop above - for (int il = 0; il < hparams.n_layer; ++il) { - auto & layer = model.layers[il]; - - // Gemma4 audio conformer-specific tensors - layer.ff_norm_w = get_tensor(string_format(TN_FFN_NORM, prefix, il, "weight")); - layer.attn_pre_norm_w = get_tensor(string_format(TN_A_ATTN_PRE_NORM, prefix, il, "weight"), false); - layer.per_dim_scale_w = get_tensor(string_format(TN_A_PER_DIM_SCALE, prefix, il, "weight"), false); - layer.per_dim_k_scale_w = get_tensor(string_format(TN_A_PER_DIM_K_SCALE, prefix, il, "weight"), false); - layer.attn_k_rel_w = get_tensor(string_format(TN_A_ATTN_K_REL, prefix, il, "weight"), false); - - // Convolution module - // Note: conv_norm / norm_conv are swapped in GGUF due to - // upstream tensor_mapping.py, so we load them in reverse order - layer.norm_conv_w = get_tensor(string_format(TN_CONV_NORM, prefix, il, "weight"), false); - layer.norm_conv_b = get_tensor(string_format(TN_CONV_NORM, prefix, il, "bias"), false); - layer.conv_pw1_w = get_tensor(string_format(TN_CONV_PW1, prefix, il, "weight")); - layer.conv_pw1_b = get_tensor(string_format(TN_CONV_PW1, prefix, il, "bias"), false); - layer.conv_dw_w = get_tensor(string_format(TN_CONV_DW, prefix, il, "weight")); - layer.conv_dw_b = get_tensor(string_format(TN_CONV_DW, prefix, il, "bias"), false); - layer.conv_norm_w = get_tensor(string_format(TN_NORM_CONV, prefix, il, "weight"), false); - layer.conv_norm_b = get_tensor(string_format(TN_NORM_CONV, prefix, il, "bias"), false); - layer.conv_pw2_w = get_tensor(string_format(TN_CONV_PW2, prefix, il, "weight")); - layer.conv_pw2_b = get_tensor(string_format(TN_CONV_PW2, prefix, il, "bias"), false); - - // FFN2 (second half-step) - layer.ff_norm_1_w = get_tensor(string_format(TN_FFN_NORM_1, prefix, il, "weight")); - layer.ff_up_1_w = get_tensor(string_format(TN_FFN_UP_1, prefix, il, "weight")); - layer.ff_up_1_b = get_tensor(string_format(TN_FFN_UP_1, prefix, il, "bias"), false); - layer.ff_down_1_w = get_tensor(string_format(TN_FFN_DOWN_1, prefix, il, "weight")); - layer.ff_down_1_b = get_tensor(string_format(TN_FFN_DOWN_1, prefix, il, "bias"), false); - layer.ff_post_norm_1_w = get_tensor(string_format(TN_A_FFN_POST_NORM_1, prefix, il, "weight"), false); - } - - // Load clamp info for ClippableLinear AFTER all tensors are loaded - for (auto * tensor : tensors_to_load) { - std::string name = tensor->name; - if (string_ends_with(name, ".weight")) { - std::string name_inp_max = name; - std::string name_inp_min = name; - std::string name_out_max = name; - std::string name_out_min = name; - string_replace_all(name_inp_max, ".weight", ".input_max"); - string_replace_all(name_inp_min, ".weight", ".input_min"); - string_replace_all(name_out_max, ".weight", ".output_max"); - string_replace_all(name_out_min, ".weight", ".output_min"); - model.clamp_info_map[name] = { - get_scalar(name_inp_max, FLT_MAX), - get_scalar(name_inp_min, -FLT_MAX), - get_scalar(name_out_max, FLT_MAX), - get_scalar(name_out_min, -FLT_MAX) - }; - } - } - } break; - case PROJECTOR_TYPE_GEMMA4UA: - { - model.mm_input_proj_w = get_tensor(string_format(TN_A_MM_INP_PROJ, "weight")); - } break; - case PROJECTOR_TYPE_LFM2A: - case PROJECTOR_TYPE_D1OMNI_A: - { - for (int i : {0, 2, 3, 5, 6}) { - model.pre_encode_conv_X_w[i] = get_tensor(string_format(TN_CONV1D, i, "weight")); - model.pre_encode_conv_X_b[i] = get_tensor(string_format(TN_CONV1D, i, "bias")); - } - model.pre_encode_out_w = get_tensor(string_format(TN_PRE_ENCODE_OUT, "weight")); - model.pre_encode_out_b = get_tensor(string_format(TN_PRE_ENCODE_OUT, "bias")); - - model.mm_0_w = get_tensor(string_format(TN_MM_AUDIO_MLP, 0, "weight")); - model.mm_0_b = get_tensor(string_format(TN_MM_AUDIO_MLP, 0, "bias")); - model.mm_1_w = get_tensor(string_format(TN_MM_AUDIO_MLP, 1, "weight")); - model.mm_1_b = get_tensor(string_format(TN_MM_AUDIO_MLP, 1, "bias")); - model.mm_3_w = get_tensor(string_format(TN_MM_AUDIO_MLP, 3, "weight")); - model.mm_3_b = get_tensor(string_format(TN_MM_AUDIO_MLP, 3, "bias")); - - // residual block after the projector: norm, down, up - if (model.proj_type == PROJECTOR_TYPE_D1OMNI_A) { - model.mm_4_w = get_tensor(string_format(TN_MM_AUDIO_MLP, 4, "weight")); - model.mm_4_b = get_tensor(string_format(TN_MM_AUDIO_MLP, 4, "bias")); - model.mm_5_w = get_tensor(string_format(TN_MM_AUDIO_MLP, 5, "weight")); - model.mm_5_b = get_tensor(string_format(TN_MM_AUDIO_MLP, 5, "bias")); - model.mm_6_w = get_tensor(string_format(TN_MM_AUDIO_MLP, 6, "weight")); - model.mm_6_b = get_tensor(string_format(TN_MM_AUDIO_MLP, 6, "bias")); - } - - for (int il = 0; il < hparams.n_layer; ++il) { - auto & layer = model.layers[il]; - - layer.ff_norm_w = get_tensor(string_format(TN_FFN_NORM, prefix, il, "weight")); - layer.ff_norm_b = get_tensor(string_format(TN_FFN_NORM, prefix, il, "bias")); - layer.ff_norm_1_w = get_tensor(string_format(TN_FFN_NORM_1, prefix, il, "weight")); - layer.ff_norm_1_b = get_tensor(string_format(TN_FFN_NORM_1, prefix, il, "bias")); - layer.ff_up_1_w = get_tensor(string_format(TN_FFN_UP_1, prefix, il, "weight")); - layer.ff_up_1_b = get_tensor(string_format(TN_FFN_UP_1, prefix, il, "bias")); - layer.ff_down_1_w = get_tensor(string_format(TN_FFN_DOWN_1, prefix, il, "weight")); - layer.ff_down_1_b = get_tensor(string_format(TN_FFN_DOWN_1, prefix, il, "bias")); - - layer.pos_bias_u = get_tensor(string_format(TN_POS_BIAS_U, prefix, il)); - layer.pos_bias_v = get_tensor(string_format(TN_POS_BIAS_V, prefix, il)); - - layer.norm_conv_w = get_tensor(string_format(TN_NORM_CONV, prefix, il, "weight")); - layer.norm_conv_b = get_tensor(string_format(TN_NORM_CONV, prefix, il, "bias")); - - layer.linear_pos_w = get_tensor(string_format(TN_LINEAR_POS, prefix, il, "weight")); - - layer.conv_norm_w = get_tensor(string_format(TN_CONV_NORM, prefix, il, "weight")); - layer.conv_norm_b = get_tensor(string_format(TN_CONV_NORM, prefix, il, "bias")); - layer.conv_dw_w = get_tensor(string_format(TN_CONV_DW, prefix, il, "weight")); - layer.conv_dw_b = get_tensor(string_format(TN_CONV_DW, prefix, il, "bias")); - layer.conv_pw1_w = get_tensor(string_format(TN_CONV_PW1, prefix, il, "weight")); - layer.conv_pw1_b = get_tensor(string_format(TN_CONV_PW1, prefix, il, "bias")); - layer.conv_pw2_w = get_tensor(string_format(TN_CONV_PW2, prefix, il, "weight")); - layer.conv_pw2_b = get_tensor(string_format(TN_CONV_PW2, prefix, il, "bias")); - } - } break; - case PROJECTOR_TYPE_LFM3A: - { - for (int i : {0, 2, 3, 5, 6}) { - model.pre_encode_conv_X_w[i] = get_tensor(string_format(TN_CONV1D, i, "weight")); - model.pre_encode_conv_X_b[i] = get_tensor(string_format(TN_CONV1D, i, "bias")); - } - model.pre_encode_out_w = get_tensor(string_format(TN_PRE_ENCODE_OUT, "weight")); - model.pre_encode_out_b = get_tensor(string_format(TN_PRE_ENCODE_OUT, "bias")); - - model.mm_0_w = get_tensor(string_format(TN_MM_AUDIO_MLP, 0, "weight")); - model.mm_0_b = get_tensor(string_format(TN_MM_AUDIO_MLP, 0, "bias")); - model.mm_1_w = get_tensor(string_format(TN_MM_AUDIO_MLP, 1, "weight")); - model.mm_1_b = get_tensor(string_format(TN_MM_AUDIO_MLP, 1, "bias")); - model.mm_3_w = get_tensor(string_format(TN_MM_AUDIO_MLP, 3, "weight")); - model.mm_3_b = get_tensor(string_format(TN_MM_AUDIO_MLP, 3, "bias")); - - if (!model.post_ln_w) { - throw std::runtime_error(string_format("%s: missing tensor %s\n", __func__, string_format(TN_LN_POST, prefix, "weight").c_str())); - } - - // each layer is either a short conv block or a complete attention block - for (int il = 0; il < hparams.n_layer; ++il) { - auto & layer = model.layers[il]; - layer.conv_pw1_w = get_tensor(string_format(TN_CONV_PW1, prefix, il, "weight"), false); - layer.conv_dw_w = get_tensor(string_format(TN_CONV_DW, prefix, il, "weight"), false); - layer.conv_pw2_w = get_tensor(string_format(TN_CONV_PW2, prefix, il, "weight"), false); - const bool is_conv = layer.conv_pw1_w && layer.conv_dw_w && layer.conv_pw2_w; - const bool is_attn = layer.q_w && layer.k_w && layer.v_w && layer.o_w && layer.q_norm && layer.k_norm; - if (is_conv == is_attn) { - throw std::runtime_error(string_format("%s: lfm3a layer %d is neither a complete conv nor a complete attention block\n", __func__, il)); - } - } - } break; - case PROJECTOR_TYPE_PARAKEET: - { - - hparams.mel_filters = get_vector(TN_MEL_FILTERS); - hparams.window = get_vector(TN_WINDOW); - - // Subsampling layers (conv1d) - for (int i : {0, 2, 3, 5, 6}) { - model.pre_encode_conv_X_w[i] = get_tensor(string_format(TN_CONV1D, i, "weight")); - model.pre_encode_conv_X_b[i] = get_tensor(string_format(TN_CONV1D, i, "bias")); - } - model.pre_encode_out_w = get_tensor(string_format(TN_PRE_ENCODE_OUT, "weight")); - model.pre_encode_out_b = get_tensor(string_format(TN_PRE_ENCODE_OUT, "bias")); - - // Projection layers - model.mm_norm_pre_w = get_tensor(string_format(TN_MM_NORM_PRE, "weight"), false); - model.mm_0_w = get_tensor(string_format(TN_MM_AUDIO_MLP, 1, "weight"), false); - model.mm_1_w = get_tensor(string_format(TN_MM_AUDIO_MLP, 2, "weight"), false); - - // Encoder layers - for (int il = 0; il < hparams.n_layer; ++il) { - auto & layer = model.layers[il]; - - // Attention (from shared above) - - // Relative position encoding - layer.linear_pos_w = get_tensor(string_format(TN_LINEAR_POS, prefix, il, "weight")); - layer.pos_bias_u = get_tensor(string_format(TN_POS_BIAS_U, prefix, il)); - layer.pos_bias_v = get_tensor(string_format(TN_POS_BIAS_V, prefix, il)); - - // Convolution module - layer.conv_pw1_w = get_tensor(string_format(TN_CONV_PW1, prefix, il, "weight")); - layer.conv_pw1_b = get_tensor(string_format(TN_CONV_PW1, prefix, il, "bias"), false); - layer.conv_dw_w = get_tensor(string_format(TN_CONV_DW, prefix, il, "weight")); - layer.conv_dw_b = get_tensor(string_format(TN_CONV_DW, prefix, il, "bias"), false); - layer.conv_norm_w = get_tensor(string_format(TN_CONV_NORM, prefix, il, "weight")); - layer.conv_norm_b = get_tensor(string_format(TN_CONV_NORM, prefix, il, "bias")); - layer.conv_norm_mean = get_tensor(string_format(TN_CONV_NORM_MEAN, prefix, il)); - layer.conv_norm_var = get_tensor(string_format(TN_CONV_NORM_VAR, prefix, il)); - layer.conv_pw2_w = get_tensor(string_format(TN_CONV_PW2, prefix, il, "weight")); - layer.conv_pw2_b = get_tensor(string_format(TN_CONV_PW2, prefix, il, "bias"), false); - - // Feed-forward networks - layer.ff_norm_w = get_tensor(string_format(TN_FFN_NORM, prefix, il, "weight")); - layer.ff_norm_b = get_tensor(string_format(TN_FFN_NORM, prefix, il, "bias")); - - layer.ff_norm_1_w = get_tensor(string_format(TN_FFN_NORM_1, prefix, il, "weight")); - layer.ff_norm_1_b = get_tensor(string_format(TN_FFN_NORM_1, prefix, il, "bias")); - layer.ff_up_1_w = get_tensor(string_format(TN_FFN_UP_1, prefix, il, "weight")); - layer.ff_up_1_b = get_tensor(string_format(TN_FFN_UP_1, prefix, il, "bias"), false); - layer.ff_down_1_w = get_tensor(string_format(TN_FFN_DOWN_1, prefix, il, "weight")); - layer.ff_down_1_b = get_tensor(string_format(TN_FFN_DOWN_1, prefix, il, "bias"), false); - - // Layer norms - layer.norm_conv_w = get_tensor(string_format(TN_NORM_CONV, prefix, il, "weight")); - layer.norm_conv_b = get_tensor(string_format(TN_NORM_CONV, prefix, il, "bias")); - } - - model.mm_model_mlp_1_w = get_tensor(string_format(TN_MVLM_PROJ_MLP, 0, "weight")); - model.mm_model_mlp_2_w = get_tensor(string_format(TN_MVLM_PROJ_MLP, 1, "weight")); - model.mm_model_mlp_3_w = get_tensor(string_format(TN_MVLM_PROJ_MLP, 3, "weight")); - } break; - case PROJECTOR_TYPE_GRANITE_SPEECH: - { - model.inp_proj_w = get_tensor(string_format(TN_INP_PROJ, "weight")); - model.inp_proj_b = get_tensor(string_format(TN_INP_PROJ, "bias")); - model.ctc_out_w = get_tensor(string_format(TN_CTC_OUT, "weight")); - model.ctc_out_b = get_tensor(string_format(TN_CTC_OUT, "bias")); - model.ctc_out_mid_w = get_tensor(string_format(TN_CTC_OUT_MID, "weight")); - model.ctc_out_mid_b = get_tensor(string_format(TN_CTC_OUT_MID, "bias")); - - // per-layer tensors not loaded by the generic loop above - for (int il = 0; il < hparams.n_layer; ++il) { - auto & layer = model.layers[il]; - - layer.attn_rel_pos_emb = get_tensor(string_format(TN_ATTN_REL_POS_EMB, prefix, il)); - - layer.ff_norm_w = get_tensor(string_format(TN_FFN_NORM, prefix, il, "weight")); - layer.ff_norm_b = get_tensor(string_format(TN_FFN_NORM, prefix, il, "bias")); - - layer.ff_norm_1_w = get_tensor(string_format(TN_FFN_NORM_1, prefix, il, "weight")); - layer.ff_norm_1_b = get_tensor(string_format(TN_FFN_NORM_1, prefix, il, "bias")); - layer.ff_up_1_w = get_tensor(string_format(TN_FFN_UP_1, prefix, il, "weight")); - layer.ff_up_1_b = get_tensor(string_format(TN_FFN_UP_1, prefix, il, "bias")); - layer.ff_down_1_w = get_tensor(string_format(TN_FFN_DOWN_1, prefix, il, "weight")); - layer.ff_down_1_b = get_tensor(string_format(TN_FFN_DOWN_1, prefix, il, "bias")); - - layer.norm_conv_w = get_tensor(string_format(TN_NORM_CONV, prefix, il, "weight")); - layer.norm_conv_b = get_tensor(string_format(TN_NORM_CONV, prefix, il, "bias")); - layer.conv_norm_w = get_tensor(string_format(TN_CONV_NORM, prefix, il, "weight")); - layer.conv_norm_b = get_tensor(string_format(TN_CONV_NORM, prefix, il, "bias")); - layer.conv_dw_w = get_tensor(string_format(TN_CONV_DW, prefix, il, "weight")); - layer.conv_pw1_w = get_tensor(string_format(TN_CONV_PW1, prefix, il, "weight")); - layer.conv_pw1_b = get_tensor(string_format(TN_CONV_PW1, prefix, il, "bias")); - layer.conv_pw2_w = get_tensor(string_format(TN_CONV_PW2, prefix, il, "weight")); - layer.conv_pw2_b = get_tensor(string_format(TN_CONV_PW2, prefix, il, "bias")); - } - - model.qf_proj_blocks.resize(1); - auto & qf = model.qf_proj_blocks[0]; - qf.qf_proj_query = get_tensor(string_format(TN_QF_PROJ_QUERY, prefix)); - qf.qf_proj_norm_w = get_tensor(string_format(TN_QF_PROJ_NORM, prefix, "weight")); - qf.qf_proj_norm_b = get_tensor(string_format(TN_QF_PROJ_NORM, prefix, "bias")); - qf.qf_proj_linear_w = get_tensor(string_format(TN_QF_PROJ_LINEAR, prefix, "weight")); - qf.qf_proj_linear_b = get_tensor(string_format(TN_QF_PROJ_LINEAR, prefix, "bias")); - - const int n_proj_layers = 2; - qf.qf_proj_layers.resize(n_proj_layers); - for (int il = 0; il < n_proj_layers; ++il) { - auto & pl = qf.qf_proj_layers[il]; - - pl.q_w = get_tensor(string_format(TN_QF_SELF_ATTN_Q, prefix, il, "weight")); - pl.q_b = get_tensor(string_format(TN_QF_SELF_ATTN_Q, prefix, il, "bias")); - pl.k_w = get_tensor(string_format(TN_QF_SELF_ATTN_K, prefix, il, "weight")); - pl.k_b = get_tensor(string_format(TN_QF_SELF_ATTN_K, prefix, il, "bias")); - pl.v_w = get_tensor(string_format(TN_QF_SELF_ATTN_V, prefix, il, "weight")); - pl.v_b = get_tensor(string_format(TN_QF_SELF_ATTN_V, prefix, il, "bias")); - pl.o_w = get_tensor(string_format(TN_QF_SELF_ATTN_O, prefix, il, "weight")); - pl.o_b = get_tensor(string_format(TN_QF_SELF_ATTN_O, prefix, il, "bias")); - pl.ln_1_w = get_tensor(string_format(TN_QF_SELF_ATTN_N, prefix, il, "weight")); - pl.ln_1_b = get_tensor(string_format(TN_QF_SELF_ATTN_N, prefix, il, "bias")); - - pl.cross_attn_q_w = get_tensor(string_format(TN_QF_CROSS_ATTN_Q, prefix, il, "weight")); - pl.cross_attn_q_b = get_tensor(string_format(TN_QF_CROSS_ATTN_Q, prefix, il, "bias")); - pl.cross_attn_k_w = get_tensor(string_format(TN_QF_CROSS_ATTN_K, prefix, il, "weight")); - pl.cross_attn_k_b = get_tensor(string_format(TN_QF_CROSS_ATTN_K, prefix, il, "bias")); - pl.cross_attn_v_w = get_tensor(string_format(TN_QF_CROSS_ATTN_V, prefix, il, "weight")); - pl.cross_attn_v_b = get_tensor(string_format(TN_QF_CROSS_ATTN_V, prefix, il, "bias")); - pl.cross_attn_o_w = get_tensor(string_format(TN_QF_CROSS_ATTN_O, prefix, il, "weight")); - pl.cross_attn_o_b = get_tensor(string_format(TN_QF_CROSS_ATTN_O, prefix, il, "bias")); - pl.cross_attn_norm_w = get_tensor(string_format(TN_QF_CROSS_ATTN_N, prefix, il, "weight")); - pl.cross_attn_norm_b = get_tensor(string_format(TN_QF_CROSS_ATTN_N, prefix, il, "bias")); - - pl.ff_up_w = get_tensor(string_format(TN_QF_FFN_UP, prefix, il, "weight")); - pl.ff_up_b = get_tensor(string_format(TN_QF_FFN_UP, prefix, il, "bias")); - pl.ff_down_w = get_tensor(string_format(TN_QF_FFN_DOWN, prefix, il, "weight")); - pl.ff_down_b = get_tensor(string_format(TN_QF_FFN_DOWN, prefix, il, "bias")); - pl.ln_2_w = get_tensor(string_format(TN_QF_FFN_NORM, prefix, il, "weight")); - pl.ln_2_b = get_tensor(string_format(TN_QF_FFN_NORM, prefix, il, "bias")); - } - } break; - case PROJECTOR_TYPE_GRANITE4_VISION: - { - // image_newline lives at the top-level. - model.image_newline = get_tensor(TN_IMAGE_NEWLINE); - - // Load separate layerwise and spatial projector tensors - const auto projector_count = hparams.feature_layers.size(); - model.qf_proj_blocks.resize(projector_count); - for (size_t bid = 0; bid < projector_count; ++bid) { - auto & b = model.qf_proj_blocks[bid]; - - // non-layerwise tensors - b.qf_proj_img_pos = get_tensor(string_format(TN_MULTI_PROJ_IMG_POS, bid)); - b.qf_proj_query = get_tensor(string_format(TN_MULTI_PROJ_QUERY, prefix, bid)); - b.qf_proj_linear_w = get_tensor(string_format(TN_MULTI_PROJ_LINEAR, prefix, bid, "weight")); - b.qf_proj_linear_b = get_tensor(string_format(TN_MULTI_PROJ_LINEAR, prefix, bid, "bias")); - b.qf_proj_norm_w = get_tensor(string_format(TN_MULTI_PROJ_NORM, prefix, bid, "weight")); - b.qf_proj_norm_b = get_tensor(string_format(TN_MULTI_PROJ_NORM, prefix, bid, "bias")); - b.qf_proj_post_norm_w = get_tensor(string_format(TN_MULTI_PROJ_POST_NORM, prefix, bid, "weight")); - b.qf_proj_post_norm_b = get_tensor(string_format(TN_MULTI_PROJ_POST_NORM, prefix, bid, "bias")); - - // laywerwise tensors - // NOTE: If any model uses multi-layer qformers, this will need to change - b.qf_proj_layers.resize(1); - auto & pl = b.qf_proj_layers[0]; - - pl.q_w = get_tensor(string_format(TN_QF_SELF_ATTN_Q, prefix, bid, "weight")); - pl.q_b = get_tensor(string_format(TN_QF_SELF_ATTN_Q, prefix, bid, "bias")); - pl.k_w = get_tensor(string_format(TN_QF_SELF_ATTN_K, prefix, bid, "weight")); - pl.k_b = get_tensor(string_format(TN_QF_SELF_ATTN_K, prefix, bid, "bias")); - pl.v_w = get_tensor(string_format(TN_QF_SELF_ATTN_V, prefix, bid, "weight")); - pl.v_b = get_tensor(string_format(TN_QF_SELF_ATTN_V, prefix, bid, "bias")); - pl.o_w = get_tensor(string_format(TN_QF_SELF_ATTN_O, prefix, bid, "weight")); - pl.o_b = get_tensor(string_format(TN_QF_SELF_ATTN_O, prefix, bid, "bias")); - pl.ln_1_w = get_tensor(string_format(TN_QF_SELF_ATTN_N, prefix, bid, "weight")); - pl.ln_1_b = get_tensor(string_format(TN_QF_SELF_ATTN_N, prefix, bid, "bias")); - - pl.cross_attn_q_w = get_tensor(string_format(TN_QF_CROSS_ATTN_Q, prefix, bid, "weight")); - pl.cross_attn_q_b = get_tensor(string_format(TN_QF_CROSS_ATTN_Q, prefix, bid, "bias")); - pl.cross_attn_k_w = get_tensor(string_format(TN_QF_CROSS_ATTN_K, prefix, bid, "weight")); - pl.cross_attn_k_b = get_tensor(string_format(TN_QF_CROSS_ATTN_K, prefix, bid, "bias")); - pl.cross_attn_v_w = get_tensor(string_format(TN_QF_CROSS_ATTN_V, prefix, bid, "weight")); - pl.cross_attn_v_b = get_tensor(string_format(TN_QF_CROSS_ATTN_V, prefix, bid, "bias")); - pl.cross_attn_o_w = get_tensor(string_format(TN_QF_CROSS_ATTN_O, prefix, bid, "weight")); - pl.cross_attn_o_b = get_tensor(string_format(TN_QF_CROSS_ATTN_O, prefix, bid, "bias")); - pl.cross_attn_norm_w = get_tensor(string_format(TN_QF_CROSS_ATTN_N, prefix, bid, "weight")); - pl.cross_attn_norm_b = get_tensor(string_format(TN_QF_CROSS_ATTN_N, prefix, bid, "bias")); - - pl.ff_up_w = get_tensor(string_format(TN_QF_FFN_UP, prefix, bid, "weight")); - pl.ff_up_b = get_tensor(string_format(TN_QF_FFN_UP, prefix, bid, "bias")); - pl.ff_down_w = get_tensor(string_format(TN_QF_FFN_DOWN, prefix, bid, "weight")); - pl.ff_down_b = get_tensor(string_format(TN_QF_FFN_DOWN, prefix, bid, "bias")); - pl.ln_2_w = get_tensor(string_format(TN_QF_FFN_NORM, prefix, bid, "weight")); - pl.ln_2_b = get_tensor(string_format(TN_QF_FFN_NORM, prefix, bid, "bias")); - } - - } break; - case PROJECTOR_TYPE_SENSENOVA_U1: - { - model.mm_0_w = get_tensor(string_format(TN_LLAVA_PROJ, 0, "weight")); - model.mm_0_b = get_tensor(string_format(TN_LLAVA_PROJ, 0, "bias")); - } break; - default: - GGML_ASSERT(false && "unknown projector type"); - } - - // load data - { - std::vector read_buf; - - // start loading event - if (progress_callback){ - progress_callback(0.0, progress_callback_user_data); - } - - // compute total tensor data size for progress reporting - size_t total_data_size = 0; - for (auto & t : tensors_to_load) { - total_data_size += ggml_nbytes(t); - } - - // alloc memory and offload data - ggml_backend_buffer_type_t buft = ggml_backend_get_default_buffer_type(ctx_clip.backend); - ctx_clip.buf.reset(ggml_backend_alloc_ctx_tensors_from_buft(ctx_clip.ctx_data.get(), buft)); - ggml_backend_buffer_set_usage(ctx_clip.buf.get(), GGML_BACKEND_BUFFER_USAGE_WEIGHTS); - // read the weight from file - if (!ctx_clip.no_alloc) { - size_t data_loaded = 0; - for (auto & t : tensors_to_load) { - ggml_tensor * cur = ggml_get_tensor(ctx_clip.ctx_data.get(), t->name); - GGML_ASSERT(cur && "tensor not found in ctx_data"); - auto it_off = tensor_offset.find(t->name); - GGML_ASSERT(it_off != tensor_offset.end() && "no offset for tensor"); - const size_t offset = it_off->second; - fin.seekg(offset, std::ios::beg); - if (!fin) { - throw std::runtime_error(string_format("%s: failed to seek for tensor %s\n", __func__, t->name)); - } - size_t num_bytes = ggml_nbytes(cur); - if (ggml_backend_buft_is_host(buft)) { - // for the CPU and Metal backend, we can read directly into the tensor - fin.read(reinterpret_cast(cur->data), num_bytes); - } else { - // read into a temporary buffer first, then copy to device memory - read_buf.resize(num_bytes); - fin.read(reinterpret_cast(read_buf.data()), num_bytes); - ggml_backend_tensor_set(cur, read_buf.data(), 0, num_bytes); - } - data_loaded += num_bytes; - if (progress_callback && total_data_size > 0) { - const float progress = (float)data_loaded / (float)total_data_size; - if (!progress_callback(progress, progress_callback_user_data)) { - throw std::runtime_error(string_format("%s: model loading cancelled by progress_callback\n", __func__)); - } - } - } - LOG_DBG("%s: loaded %zu tensors from %s\n", __func__, tensors_to_load.size(), fname.c_str()); - } else { - LOG_DBG("%s: no_alloc is set, skipping tensor data loading (%zu tensors)\n", __func__, tensors_to_load.size()); - } - fin.close(); - } - - } - - struct support_info_op { - ggml_tensor * op; - - // true if the op runs on the accelerated ctx_clip.backend - bool is_accel = true; - }; - - struct support_info_graph { - // whether the clip_ctx.backend supports flash attention - bool fattn = true; - ggml_tensor * fattn_op = nullptr; // for debugging - - std::vector ops; - }; - - static clip_image_f32_batch get_dummy_batch(clip_ctx & ctx_clip) { - // create a fake batch - const auto & hparams = ctx_clip.model.hparams; - clip_image_f32_batch batch; - clip_image_f32 img; - if (ctx_clip.model.modality == CLIP_MODALITY_VISION) { - const int sz = hparams.warmup_image_size; - img.set_size({sz, sz}, false, false); - LOG_INF("%s: warmup with image size = %d x %d\n", __func__, sz, sz); - } else { - // GEMMA4UA uses n_mel_bins as a raw-waveform frame size (640), not a mel-bin count, - // so the [1, 256] bound only applies to FFT-based models. - const bool fft_based = ctx_clip.model.proj_type != PROJECTOR_TYPE_GEMMA4UA; - if (hparams.n_mel_bins <= 0 || (fft_based && hparams.n_mel_bins > 256)) { - throw std::runtime_error(string_format("%s: invalid n_mel_bins (%d), must be in [1, 256]\n", __func__, hparams.n_mel_bins)); - } - img.set_size({hparams.warmup_audio_size, hparams.n_mel_bins}, false, false); - LOG_INF("%s: warmup with audio size = %d\n", __func__, hparams.warmup_audio_size); - } - batch.entries.push_back(img); - return batch; - } - - static void init_ctx(clip_ctx & ctx_clip) { - ctx_clip.buf_compute_meta.resize(ctx_clip.max_nodes * ggml_tensor_overhead() + ggml_graph_overhead()); - - // check batching support - auto batch = get_dummy_batch(ctx_clip); - auto builder = clip_get_graph_builder(&ctx_clip, batch); - ctx_clip.support_batch = builder->support_batch(); - } - - static void warmup(clip_ctx & ctx_clip) { - auto batch = get_dummy_batch(ctx_clip); - warmup(ctx_clip, batch); - } - - static void warmup(clip_ctx & ctx_clip, const clip_image_f32_batch & batch) { - support_info_graph info; - - if (ctx_clip.flash_attn_type == CLIP_FLASH_ATTN_TYPE_AUTO) { - // try to enable flash attention to see if it's supported - ctx_clip.flash_attn_type = CLIP_FLASH_ATTN_TYPE_ENABLED; - info = reserve_compute_meta(ctx_clip, batch); - if (!info.fattn && info.fattn_op) { - auto op = info.fattn_op; - LOG_WRN("%s: *****************************************************************\n", __func__); - LOG_WRN("%s: WARNING: flash attention not supported by %s, memory usage will increase\n", __func__, ggml_backend_name(ctx_clip.backend)); - LOG_WRN("%s: op params: \n", __func__); - static auto print_shape = [](const char * fn, const char * name, ggml_tensor * t) { - LOG_WRN("%s: %s: type = %s, ne = [%d %d %d %d], nb = [%d %d %d %d]\n", fn, - name, ggml_type_name(t->type), - t->ne[0], t->ne[1], t->ne[2], t->ne[3], - t->nb[0], t->nb[1], t->nb[2], t->nb[3]); - }; - print_shape(__func__, " dst", op); - print_shape(__func__, "src0", op->src[0]); - print_shape(__func__, "src1", op->src[1]); - print_shape(__func__, "src2", op->src[2]); - LOG_WRN("%s: please report this on github as an issue\n", __func__); - LOG_WRN("%s: *****************************************************************\n", __func__); - ctx_clip.flash_attn_type = CLIP_FLASH_ATTN_TYPE_DISABLED; - reserve_compute_meta(ctx_clip, batch); - } - } else { - info = reserve_compute_meta(ctx_clip, batch); - if (!info.fattn && ctx_clip.flash_attn_type == CLIP_FLASH_ATTN_TYPE_ENABLED) { - LOG_WRN("%s: flash attention is not supported by the current backend; falling back to CPU (performance will be degraded)\n", __func__); - } - } - - ctx_clip.is_allocated = true; // mark buffers as allocated - - LOG_INF("%s: flash attention is %s\n", __func__, - (ctx_clip.flash_attn_type == CLIP_FLASH_ATTN_TYPE_ENABLED) ? "enabled" : "disabled"); - - // print ops that are not supported by the GPU backend (if there is one) - if (ctx_clip.backend && ctx_clip.backend != ctx_clip.backend_cpu) { - std::vector unsupported_ops; - for (const auto & op : info.ops) { - if (!op.is_accel) { - unsupported_ops.push_back(op); - } - } - if (!unsupported_ops.empty()) { - LOG_WRN("%s: *****************************************************************\n", __func__); - LOG_WRN("%s: WARNING: the CLIP graph uses unsupported operators by the backend\n", __func__); - LOG_WRN("%s: the performance will be suboptimal \n", __func__); - LOG_WRN("%s: list of unsupported ops (backend=%s):\n", __func__, ggml_backend_name(ctx_clip.backend)); - for (const auto & op : unsupported_ops) { - LOG_WRN("%s: %16s: type = %s, ne = [%d %d %d %d]\n", __func__, - ggml_op_name(op.op->op), - ggml_type_name(op.op->type), - op.op->ne[0], op.op->ne[1], op.op->ne[2], op.op->ne[3]); - } - LOG_WRN("%s: flash attention is %s\n", __func__, - (ctx_clip.flash_attn_type == CLIP_FLASH_ATTN_TYPE_ENABLED) ? "enabled" : "disabled"); - LOG_WRN("%s: please report this on github as an issue\n", __func__); - LOG_WRN("%s: ref: https://github.com/ggml-org/llama.cpp/pull/16837#issuecomment-3461676118\n", __func__); - LOG_WRN("%s: *****************************************************************\n", __func__); - } - } - } - - // only initialize backend buffers, but do not allocate them yet - static support_info_graph reserve_compute_meta(clip_ctx & ctx_clip, const clip_image_f32_batch & batch) { - ggml_cgraph * gf = clip_get_graph_builder(&ctx_clip, batch)->build(); - ggml_backend_sched_reserve(ctx_clip.sched.get(), gf); - - ctx_clip.mem_compute.clear(); - for (size_t i = 0; i < ctx_clip.backend_ptrs.size(); ++i) { - ggml_backend_t backend = ctx_clip.backend_ptrs[i]; - ggml_backend_buffer_type_t buft = ctx_clip.backend_buft[i]; - size_t size = ggml_backend_sched_get_buffer_size(ctx_clip.sched.get(), backend); - if (size > 1) { - LOG_INF("%s: %10s compute buffer size = %8.2f MiB\n", __func__, - ggml_backend_buft_name(buft), - size / 1024.0 / 1024.0); - } - ctx_clip.mem_compute[ggml_backend_get_device(backend)] += size; - } - - const int n_splits = ggml_backend_sched_get_n_splits(ctx_clip.sched.get()); - const int n_nodes = ggml_graph_n_nodes(gf); - - LOG_INF("%s: graph splits = %d, nodes = %d\n", __func__, n_splits, n_nodes); - - support_info_graph res { - /*.fattn = */ true, - /*.fattn_op = */ nullptr, - /*.ops = */ {}, - }; - - // check op support - for (int i = 0; i < ggml_graph_n_nodes(gf); i++) { - ggml_tensor * node = ggml_graph_node(gf, i); - res.ops.push_back({node, true}); - if (!ggml_backend_supports_op(ctx_clip.backend, node)) { - res.ops.back().is_accel = false; - if (node->op == GGML_OP_FLASH_ATTN_EXT) { - res.fattn = false; - res.fattn_op = node; - } - } - } - - return res; - } - - void get_bool(const std::string & key, bool & output, bool required = true) const { - const int i = gguf_find_key(ctx_gguf.get(), key.c_str()); - if (i < 0) { - if (required) { - throw std::runtime_error("Key not found: " + key); - } - return; - } - output = gguf_get_val_bool(ctx_gguf.get(), i); - } - - void get_i32(const std::string & key, int & output, bool required = true) const { - const int i = gguf_find_key(ctx_gguf.get(), key.c_str()); - if (i < 0) { - if (required) { - throw std::runtime_error("Key not found: " + key); - } - return; - } - output = gguf_get_val_i32(ctx_gguf.get(), i); - } - - void get_u32(const std::string & key, int & output, bool required = true) const { - const int i = gguf_find_key(ctx_gguf.get(), key.c_str()); - if (i < 0) { - if (required) { - throw std::runtime_error("Key not found: " + key); - } - return; - } - const uint32_t val = gguf_get_val_u32(ctx_gguf.get(), i); - // sanity check - if (val > (uint32_t) INT32_MAX) { - throw std::runtime_error(string_format("%s: value %u for key '%s' exceeds INT32_MAX\n", - __func__, val, key.c_str())); - } - output = (int) val; - } - - void get_f32(const std::string & key, float & output, bool required = true) const { - const int i = gguf_find_key(ctx_gguf.get(), key.c_str()); - if (i < 0) { - if (required) { - throw std::runtime_error("Key not found: " + key); - } - return; - } - output = gguf_get_val_f32(ctx_gguf.get(), i); - } - - void get_arr_f32(const std::string & key, std::vector & output, bool required = true) const { - const int i = gguf_find_key(ctx_gguf.get(), key.c_str()); - if (i < 0) { - if (required) { - throw std::runtime_error("Key not found: " + key); - } - return; - } - if (gguf_get_kv_type(ctx_gguf.get(), i) != GGUF_TYPE_ARRAY) { - throw std::runtime_error(string_format("%s: key '%s' is not an array\n", __func__, key.c_str())); - } - const auto type = gguf_get_arr_type(ctx_gguf.get(), i); - if (type != GGUF_TYPE_FLOAT32) { - throw std::runtime_error(string_format("%s: array '%s' has type %d, expected %d (GGUF_TYPE_FLOAT32)\n", __func__, key.c_str(), type, GGUF_TYPE_FLOAT32)); - } - const size_t n = gguf_get_arr_n(ctx_gguf.get(), i); - if (n > (size_t) std::numeric_limits::max()) { - throw std::runtime_error(string_format("%s: array '%s' is too large (%zu elements)\n", __func__, key.c_str(), n)); - } - output.resize(n); - const float * values = (const float *)gguf_get_arr_data(ctx_gguf.get(), i); - for (size_t j = 0; j < n; ++j) { - output[j] = values[j]; - } - } - - void get_string(const std::string & key, std::string & output, bool required = true) const { - const int i = gguf_find_key(ctx_gguf.get(), key.c_str()); - if (i < 0) { - if (required) { - throw std::runtime_error("Key not found: " + key); - } - return; - } - output = std::string(gguf_get_val_str(ctx_gguf.get(), i)); - } - - void get_arr_int(const std::string & key, std::vector & output, bool required = true) const { - const int i = gguf_find_key(ctx_gguf.get(), key.c_str()); - if (i < 0) { - if (required) { - throw std::runtime_error("Key not found: " + key); - } - return; - } - if (gguf_get_kv_type(ctx_gguf.get(), i) != GGUF_TYPE_ARRAY) { - throw std::runtime_error(string_format("%s: key '%s' is not an array\n", __func__, key.c_str())); - } - const auto type = gguf_get_arr_type(ctx_gguf.get(), i); - if (type != GGUF_TYPE_INT32) { - throw std::runtime_error(string_format("%s: array '%s' has type %d, expected %d (GGUF_TYPE_INT32)\n", __func__, key.c_str(), type, GGUF_TYPE_INT32)); - } - const size_t n = gguf_get_arr_n(ctx_gguf.get(), i); - if (n > (size_t) std::numeric_limits::max()) { - throw std::runtime_error(string_format("%s: array '%s' is too large (%zu elements)\n", __func__, key.c_str(), n)); - } - output.resize(n); - const int32_t * values = (const int32_t *)gguf_get_arr_data(ctx_gguf.get(), i); - for (size_t j = 0; j < n; ++j) { - output[j] = values[j]; - } - } - - static void set_llava_uhd_res_candidates(clip_model & model, const int max_patches_per_side) { - auto & hparams = model.hparams; - for (int x = 1; x <= max_patches_per_side; x++) { - for (int y = 1; y <= max_patches_per_side; y++) { - if (x == 1 && y == 1) { - continue; // skip the first point - } - hparams.image_res_candidates.push_back(clip_image_size{ - x*hparams.image_size, - y*hparams.image_size, - }); - } - } - } - - static void set_internvl_dhr_res_candidates(clip_model & model) { - auto & hparams = model.hparams; - int min_num = hparams.preproc_min_tiles; - int max_num = hparams.preproc_max_tiles; - if (min_num < 1) { - return; // avoid divide by 0 - } - for (int a = min_num; a <= max_num; ++a) { - int b_lo = (min_num + a - 1) / a; - int b_hi = max_num / a; - b_lo = std::max(b_lo, min_num); - b_hi = std::min(b_hi, max_num); - for (int b = b_lo; b <= b_hi; ++b) { - hparams.image_res_candidates.push_back(clip_image_size { - a*hparams.image_size, - b*hparams.image_size, - }); - } - } - } -}; - -struct clip_init_result clip_init(const char * fname, struct clip_context_params ctx_params) { - clip_ctx * ctx_vision = nullptr; - clip_ctx * ctx_audio = nullptr; - clip_ctx * ctx_gen_audio = nullptr; - - try { - clip_model_loader loader(fname, - /* skip_tensors */ false, - ctx_params.progress_callback, - ctx_params.progress_callback_user_data); - bool skip_audio = false; - - if (loader.has_vision) { - ctx_vision = new clip_ctx(ctx_params); - loader.load_hparams(ctx_vision->model, CLIP_MODALITY_VISION); - loader.load_tensors(*ctx_vision); - loader.init_ctx(*ctx_vision); - if (ctx_params.warmup) { - loader.warmup(*ctx_vision); - } - - // TODO: we don't support audio for Gemma 3N, but GGUF contains audio tensors - // we can remove this check when we implement audio support for Gemma 3N - skip_audio = ctx_vision->model.proj_type == PROJECTOR_TYPE_GEMMA3NV; - } - - if (loader.has_audio && !skip_audio) { - ctx_audio = new clip_ctx(ctx_params); - loader.load_hparams(ctx_audio->model, CLIP_MODALITY_AUDIO); - loader.load_tensors(*ctx_audio); - loader.init_ctx(*ctx_audio); - if (ctx_params.warmup) { - loader.warmup(*ctx_audio); - } - } - - if (loader.has_gen_audio) { - ctx_gen_audio = new clip_ctx(ctx_params); - loader.load_hparams(ctx_gen_audio->model, CLIP_MODALITY_GEN_AUDIO); - // TODO: CUDA has no flash attention kernel for the depthformer head dim 32, enable it once it does - if (ctx_gen_audio->model.proj_type == PROJECTOR_TYPE_LFM3A_GEN && ctx_gen_audio->flash_attn_type == CLIP_FLASH_ATTN_TYPE_AUTO) { - ctx_gen_audio->flash_attn_type = CLIP_FLASH_ATTN_TYPE_DISABLED; - } - loader.load_tensors(*ctx_gen_audio); - // TODO: fix warmup - ctx_gen_audio->buf_compute_meta.resize(ctx_gen_audio->max_nodes * ggml_tensor_overhead() + ggml_graph_overhead()); - } - - } catch (const std::exception & e) { - LOG_ERR("%s: failed to load model '%s': %s\n", __func__, fname, e.what()); - - delete ctx_vision; - delete ctx_audio; - delete ctx_gen_audio; - - return {nullptr, nullptr, nullptr}; - } - - return {ctx_vision, ctx_audio, ctx_gen_audio}; -} - -struct clip_cap clip_get_cap(const char * fname) { - clip_cap res; - clip_model_loader loader(fname, /* skip_tensors= */ true); - res.has_vision = loader.has_vision; - res.has_audio = loader.has_audio; - res.has_gen_audio = loader.has_gen_audio; - return res; -} - -int clip_get_image_max_tokens(const clip_ctx * ctx) { - const auto & hparams = ctx->model.hparams; - if (ctx->proj_type() == PROJECTOR_TYPE_DEEPSEEK4V || ctx->proj_type() == PROJECTOR_TYPE_DEEPSEEK41V) { - return hparams.dsv4_max_n_token; - } - if (hparams.image_max_pixels <= 0) { - return -1; - } - const int patch_area = hparams.patch_size * hparams.patch_size * hparams.n_merge * hparams.n_merge; - return hparams.image_max_pixels / patch_area; -} - -void clip_free(clip_ctx * ctx) { - if (ctx == nullptr) { - return; - } - delete ctx; -} - -const char * clip_patch_merge_type(const struct clip_ctx * ctx) { - return ctx->model.hparams.mm_patch_merge_type == PATCH_MERGE_SPATIAL_UNPAD ? "spatial_unpad" : "flat"; -} - -int clip_n_output_tokens_x(const clip_ctx * ctx, const clip_image_f32 * img) { - const auto & params = ctx->model.hparams; - const int n_total = clip_n_output_tokens(ctx, img); - const auto & proj = ctx->proj_type(); - switch (proj) { - case PROJECTOR_TYPE_QWEN2VL: - case PROJECTOR_TYPE_QWEN25VL: - case PROJECTOR_TYPE_QWEN3VL: - case PROJECTOR_TYPE_SENSENOVA_U1: - case PROJECTOR_TYPE_LING3VL: - case PROJECTOR_TYPE_EXAONE4_5: - case PROJECTOR_TYPE_MIMOVL: - case PROJECTOR_TYPE_GLM4V: - case PROJECTOR_TYPE_GLM5V: - case PROJECTOR_TYPE_PADDLEOCR: - case PROJECTOR_TYPE_HUNYUANVL: - case PROJECTOR_TYPE_YOUTUVL: - case PROJECTOR_TYPE_MUSE_GLIMMER: - case PROJECTOR_TYPE_LFM3VL: - return (img->nx() / params.patch_size) / 2; - case PROJECTOR_TYPE_STEP3VL: - case PROJECTOR_TYPE_MINICPMV4_6: - case PROJECTOR_TYPE_MINICPMV4_7: - case PROJECTOR_TYPE_MINICPM_OCR_1_0: - return img->nx() / (params.patch_size * params.n_merge); - case PROJECTOR_TYPE_DEEPSEEKOCR: - case PROJECTOR_TYPE_DEEPSEEKOCR2: - return (img->nx() / params.patch_size) / 4; - default: - break; - } - return n_total; -} - -int clip_n_output_tokens_y(const clip_ctx * ctx, const clip_image_f32 * img) { - const auto & params = ctx->model.hparams; - const auto & proj = ctx->proj_type(); - switch (proj) { - case PROJECTOR_TYPE_QWEN2VL: - case PROJECTOR_TYPE_QWEN25VL: - case PROJECTOR_TYPE_QWEN3VL: - case PROJECTOR_TYPE_SENSENOVA_U1: - case PROJECTOR_TYPE_LING3VL: - case PROJECTOR_TYPE_EXAONE4_5: - case PROJECTOR_TYPE_MIMOVL: - case PROJECTOR_TYPE_GLM4V: - case PROJECTOR_TYPE_GLM5V: - case PROJECTOR_TYPE_PADDLEOCR: - case PROJECTOR_TYPE_HUNYUANVL: - case PROJECTOR_TYPE_YOUTUVL: - case PROJECTOR_TYPE_MUSE_GLIMMER: - case PROJECTOR_TYPE_LFM3VL: - return (img->ny() / params.patch_size) / 2; - case PROJECTOR_TYPE_STEP3VL: - case PROJECTOR_TYPE_MINICPMV4_6: - case PROJECTOR_TYPE_MINICPMV4_7: - case PROJECTOR_TYPE_MINICPM_OCR_1_0: - return img->ny() / (params.patch_size * params.n_merge); - default: - break; - } - return 1; -} - -int clip_n_output_tokens(const clip_ctx * ctx, const clip_image_f32 * img) { - const auto & params = ctx->model.hparams; - - // for models with fixed size image, the input image is already pre-processed and resized to square - int patch_size = params.patch_size; - int n_patches = (img->nx() / patch_size) * (img->ny() / patch_size); - - projector_type proj = ctx->proj_type(); - - switch (proj) { - case PROJECTOR_TYPE_MLP: - case PROJECTOR_TYPE_MLP_NORM: - case PROJECTOR_TYPE_JANUS_PRO: - case PROJECTOR_TYPE_PHI4: - { - // do nothing - } break; - case PROJECTOR_TYPE_YASA2: - { - n_patches = 64; // adaptive average pooling to 8x8 tokens - } break; - case PROJECTOR_TYPE_LDP: - case PROJECTOR_TYPE_LDPV2: - case PROJECTOR_TYPE_GLM_EDGE: - { - n_patches /= 4; - if (ctx->model.mm_boi) { - n_patches += 2; // for BOI and EOI token embeddings - } - } break; - case PROJECTOR_TYPE_MINICPMV: - { - // Use actual config value if available, otherwise fall back to hardcoded values - if (params.minicpmv_query_num > 0) { - n_patches = params.minicpmv_query_num; - } else { - // Fallback to hardcoded values for legacy models - if (params.minicpmv_version == 2) { - n_patches = 96; - } else if (params.minicpmv_version == 3) { - n_patches = 64; - } else if (params.minicpmv_version == 4) { - n_patches = 64; - } else if (params.minicpmv_version == 5) { - // MiniCPM-V 4.0 - n_patches = 64; - } else if (params.minicpmv_version == 6) { - // MiniCPM-V 4.5 - n_patches = 64; - } else if (params.minicpmv_version == 100045) { - // MiniCPM-o 4.5 - n_patches = 64; - } else { - GGML_ABORT("Unknown minicpmv version"); - } - } - } break; - case PROJECTOR_TYPE_MINICPMV4_6: - case PROJECTOR_TYPE_MINICPMV4_7: - case PROJECTOR_TYPE_MINICPM_OCR_1_0: - { - n_patches /= params.n_merge * params.n_merge; - } break; - case PROJECTOR_TYPE_QWEN2VL: - case PROJECTOR_TYPE_QWEN25VL: - case PROJECTOR_TYPE_QWEN3VL: - case PROJECTOR_TYPE_SENSENOVA_U1: - case PROJECTOR_TYPE_LING3VL: - case PROJECTOR_TYPE_EXAONE4_5: - case PROJECTOR_TYPE_MIMOVL: - case PROJECTOR_TYPE_MINIMAX_M3: - case PROJECTOR_TYPE_GLM4V: - case PROJECTOR_TYPE_GLM5V: - case PROJECTOR_TYPE_YOUTUVL: - case PROJECTOR_TYPE_MUSE_GLIMMER: - case PROJECTOR_TYPE_LFM3VL: - { - // dynamic size (2 conv, so double patch size) - int x_patch = img->nx() / (params.patch_size * 2); - int y_patch = img->ny() / (params.patch_size * 2); - n_patches = x_patch * y_patch; - } break; - case PROJECTOR_TYPE_STEP3VL: - { - int x_patch = img->nx() / (params.patch_size * params.n_merge); - int y_patch = img->ny() / (params.patch_size * params.n_merge); - n_patches = x_patch * y_patch; - } break; - case PROJECTOR_TYPE_GEMMA3: - case PROJECTOR_TYPE_GEMMA4V: - case PROJECTOR_TYPE_GEMMA4UV: - case PROJECTOR_TYPE_IDEFICS3: - case PROJECTOR_TYPE_COHERE2V: - case PROJECTOR_TYPE_INTERNVL: - case PROJECTOR_TYPE_NEMOTRON_V2_VL: - case PROJECTOR_TYPE_NEMOTRON_OMNI: - case PROJECTOR_TYPE_LLAMA4: - { - // both X and Y are downscaled by the scale factor - int scale_factor = ctx->model.hparams.n_merge; - n_patches /= (scale_factor * scale_factor); - } break; - case PROJECTOR_TYPE_GEMMA3NV: - { - // MobileNetV5 MSFA adapter always outputs fixed 16x16 resolution - // regardless of input size (see architecture description) - n_patches = ctx->model.hparams.image_size / ctx->model.hparams.patch_size; - } break; - case PROJECTOR_TYPE_LFM2: - case PROJECTOR_TYPE_D1OMNI_V: - case PROJECTOR_TYPE_KIMIVL: - case PROJECTOR_TYPE_KIMIK25: - { - // dynamic size - int out_patch_size = params.patch_size * ctx->model.hparams.n_merge; - int x_patch = CLIP_ALIGN(img->nx(), out_patch_size) / out_patch_size; - int y_patch = CLIP_ALIGN(img->ny(), out_patch_size) / out_patch_size; - n_patches = x_patch * y_patch; - } break; - case PROJECTOR_TYPE_DEEPSEEK4V: - { - const int out_patch_size = params.patch_size * params.n_merge; - const int n_llm_w = CLIP_ALIGN(img->nx(), out_patch_size) / out_patch_size; - const int n_llm_h = CLIP_ALIGN(img->ny(), out_patch_size) / out_patch_size; - n_patches = dsv4_get_block_layout(n_llm_w, n_llm_h, img->lead_pad).n_out; - } break; - case PROJECTOR_TYPE_DEEPSEEK41V: - { - const int out_patch_size = params.patch_size * params.n_merge; - const int n_llm_w = CLIP_ALIGN(img->nx(), out_patch_size) / out_patch_size; - const int n_llm_h = CLIP_ALIGN(img->ny(), out_patch_size) / out_patch_size; - n_patches = dsv41_n_block_tokens(n_llm_w, n_llm_h); - } break; - case PROJECTOR_TYPE_PADDLEOCR: - case PROJECTOR_TYPE_DOTS_OCR: - case PROJECTOR_TYPE_DOTS3NOTE_V: - { - // dynamic size - int n_merge = ctx->model.hparams.n_merge; - int stride = n_merge * n_merge; - n_patches = CLIP_ALIGN(n_patches, stride) / stride; - } break; - case PROJECTOR_TYPE_DOTS3NOTE_A: - { - // 3x stride-2 conv2d over mel frames - n_patches = (img->nx() + 7) / 8; - } break; - case PROJECTOR_TYPE_PIXTRAL: - case PROJECTOR_TYPE_LIGHTONOCR: - { - // dynamic size - int n_merge = ctx->model.hparams.n_merge; - int n_patches_x = img->nx() / patch_size / n_merge; - int n_patches_y = img->ny() / patch_size / n_merge; - if (ctx->model.token_embd_img_break) { - n_patches = n_patches_y * n_patches_x + n_patches_y - 1; // + one [IMG_BREAK] per row, except the last row - } else { - n_patches = n_patches_y * n_patches_x; - } - } break; - case PROJECTOR_TYPE_VOXTRAL_RT: - { - // 2 context frames, conv stride 2, then stacked frames - n_patches = (img->nx() - 2) / 2 / ctx->model.hparams.proj_stack_factor; - } break; - case PROJECTOR_TYPE_MUSE_SPARK_AUDIO: - { - const int64_t fold = ctx->model.hparams.audio_samples_per_token(); - n_patches = (int) ((img->nx() + fold - 1) / fold); - } break; - case PROJECTOR_TYPE_VOXTRAL: - case PROJECTOR_TYPE_ULTRAVOX: - case PROJECTOR_TYPE_QWEN2A: - case PROJECTOR_TYPE_MERALION: - case PROJECTOR_TYPE_MUSIC_FLAMINGO: - { - n_patches = img->nx(); - - const int proj_stack_factor = ctx->model.hparams.proj_stack_factor; - if (ctx->model.audio_has_stack_frames()) { - GGML_ASSERT(proj_stack_factor > 0); - const int n_len = CLIP_ALIGN(n_patches, proj_stack_factor); - n_patches = n_len / proj_stack_factor; - } - - // whisper downscales input token by half after conv1d - n_patches /= 2; - - if (ctx->model.audio_has_avgpool()) { - // divide by 2 because of nn.AvgPool1d(2, stride=2) - n_patches /= 2; - } - } break; - case PROJECTOR_TYPE_QWEN3A: - { - // chunk_size=100 frames --> 3x stride-2 conv2d --> 13 tokens per chunk - const int chunk_size = 100; - const int tokens_per_chunk = 13; - n_patches = (img->nx() / chunk_size) * tokens_per_chunk; - } break; - case PROJECTOR_TYPE_GLMA: - { - n_patches = img->nx(); - // whisper downscales input token by half after conv1d - n_patches /= 2; - // reshape by merge_factor - n_patches /= ctx->model.hparams.proj_stack_factor; - // for BOI and EOI token embeddings - n_patches += 2; - } break; - case PROJECTOR_TYPE_COGVLM: - { - n_patches += 2; // for BOI and EOI token embeddings - } break; - case PROJECTOR_TYPE_DEEPSEEKOCR: - { - // SAM encoder applies two stride-2 convolutions (net_2 and net_3) - // that reduce spatial dimensions by 4x in each direction (16x total) - // E.g., 64x64 -> 16x16 patches - n_patches /= 16; - - if (img->add_viewsep) { - // global view: one image-newline per token-row + trailing view separator - const int h = static_cast(std::sqrt(static_cast(n_patches))); - n_patches = h * (h + 1) + 1; - } else if (img->ny() >= img->nx() && img->ny() % img->nx() == 0) { - // tile row: one image-newline per token-row - const int grid_w = img->ny() / img->nx(); - const int tile_patches = img->nx() / (patch_size * 4); // patches per tile side (SAM divides by 4) - const int h = tile_patches; - n_patches = (tile_patches * grid_w + 1) * h; - } - } break; - case PROJECTOR_TYPE_HUNYUANVL: - { - int merge = ctx->model.hparams.n_merge; - int ow = (img->nx() / patch_size) / merge; - int oh = (img->ny() / patch_size) / merge; - n_patches = (ow + 1) * oh + 2; - } break; - case PROJECTOR_TYPE_DEEPSEEKOCR2: - { - // 1024 global view -> 256 query tokens + 1 view separator = 257; - // 768 local tile -> 144 query tokens, no separator. - n_patches /= 16; - if (img->add_viewsep) { - n_patches += 1; // view separator, appended only after the global view - } - } break; - case PROJECTOR_TYPE_LFM2A: - case PROJECTOR_TYPE_D1OMNI_A: - case PROJECTOR_TYPE_LFM3A: - { - n_patches = ((((img->nx() + 1) / 2) + 1) / 2 + 1) / 2; - } break; - case PROJECTOR_TYPE_GEMMA4A: - { - // Two Conv2D stride-2: O = floor((I + 2p - k) / s) + 1, p=1, k=3, s=2 - // O = floor((I - 1) / 2) + 1 - int n = img->nx(); - for (int i = 0; i < 2; i++) { - n = (n - 1) / 2 + 1; - } - n_patches = n; - } break; - case PROJECTOR_TYPE_PARAKEET: - { - n_patches = (img->nx() + (params.subsampling_factor - 1)) / params.subsampling_factor; - } break; - case PROJECTOR_TYPE_GEMMA4UA: - { - n_patches = img->nx(); // no downsampling: one token per raw waveform frame - } break; - case PROJECTOR_TYPE_MIMO_AUDIO: - { - // conv1(s=1) + conv2(s=2) -> RVQ-encoder downsample conv(k=2,s=2) - int n = img->nx(); - n = (n - 1) / 2 + 1; // conv1 + conv2 - n = (n - 2) / 2 + 1; // downsample conv - const int group_size = params.audio_local_group_size; - n_patches = (n + group_size - 1) / group_size; - } break; - case PROJECTOR_TYPE_GRANITE_SPEECH: - { - const int ws = ctx->model.hparams.audio_proj_window_size; - const int ds = ctx->model.hparams.audio_proj_downsample_rate; - n_patches = ((img->nx() + ws - 1) / ws) * (ws / ds); - } break; - case PROJECTOR_TYPE_QWEN3TTS_SPKENC: - { - // pooling gives one speaker embedding, whatever the clip length is - n_patches = 1; - } break; - case PROJECTOR_TYPE_QWEN3TTS_GEN: - case PROJECTOR_TYPE_LFM3A_GEN: - { - // one hidden-state vector fed back to the talker per call - n_patches = 1; - } break; - case PROJECTOR_TYPE_POCKETTTS_SPKENC: - { - // one conditioning row per 12.5Hz frame - const int hop = ctx->model.hparams.mimi_downsample * 120; - n_patches = img->nx() / hop; - } break; - case PROJECTOR_TYPE_POCKETTTS_GEN: - { - // one latent per call for GEN_CODE, GEN_WAV sizes its input from the caller - n_patches = 1; - } break; - case PROJECTOR_TYPE_GRANITE4_VISION: - { - // Per-tile output token count: each projector block outputs - // query_side^2 tokens per window x n^2 windows. - // For 384x384 input: n = 24/8 = 3, query_side = 4 -> 144. - const int window_side = ctx->model.hparams.downsample_window_side; - const int query_side = ctx->model.hparams.downsample_query_side; - const int side = img->nx() / params.patch_size; - const int n = side / window_side; - const int out_side = query_side * n; - n_patches = out_side * out_side; - if (img->anyres.is_tiled()) { - // overview tile, then the unpadded tile grid with one newline per row - int off_x, off_y, w, h; - clip_anyres_unpad(img->anyres.grid_x * out_side, img->anyres.grid_y * out_side, - img->anyres.orig_nx, img->anyres.orig_ny, off_x, off_y, w, h); - n_patches += h * (w + 1); - } - } break; - default: - GGML_ABORT("unsupported projector type"); - } - - n_patches += (int) clip_suffix_rows(img->suffix_type).size(); - - return n_patches; -} - -bool clip_image_encode(struct clip_ctx * ctx, int n_threads, const clip_image_f32 * img, std::vector & out_vec) { - clip_image_f32_batch imgs; - clip_image_f32 img_copy = *img; - imgs.entries.push_back(std::move(img_copy)); - - return clip_image_batch_encode(ctx, n_threads, &imgs, out_vec); -} - -bool clip_image_batch_encode(clip_ctx * ctx, int n_threads, const clip_image_f32_batch * imgs_c_ptr, std::vector & out_batch_embd) { - clip_encode_params params; - params.imgs = imgs_c_ptr; - params.n_threads = n_threads; - params.out_embd = &out_batch_embd; - - return clip_encode(ctx, ¶ms); -} - -// persisted state slots of the gen-audio decoders and the streaming audio encoders -static std::vector list_gen_state_slots(const clip_hparams & hparams, const clip_model & model) { - switch (model.proj_type) { - case PROJECTOR_TYPE_QWEN3TTS_GEN: return list_c2w_state_slots(hparams, model); - case PROJECTOR_TYPE_POCKETTTS_GEN: return list_pockettts_state_slots(hparams, model); - case PROJECTOR_TYPE_LFM3A_GEN: return list_lfm3a_gen_state_slots(hparams, model); - default: return {}; - } -} - -// state_out can have another shape than its slot, only the byte size must match -static void copy_gen_state_out(ggml_tensor * src, ggml_tensor * dst) { - GGML_ASSERT(ggml_nbytes(src) == ggml_nbytes(dst)); - if (src->type == dst->type && ggml_are_same_shape(src, dst) && ggml_is_contiguous(src)) { - ggml_backend_tensor_copy(src, dst); - return; - } - std::vector tmp(ggml_nbytes(src)); - ggml_backend_tensor_get(src, tmp.data(), 0, tmp.size()); - ggml_backend_tensor_set(dst, tmp.data(), 0, tmp.size()); -} - -clip_gen_state * clip_gen_state_init(clip_ctx * ctx, uint32_t seed) { - auto st = std::make_unique(); - st->rng.seed(seed == UINT32_MAX ? std::random_device{}() : seed); - - const auto slots = list_gen_state_slots(ctx->model.hparams, ctx->model); - if (slots.empty()) { - return st.release(); - } - - ggml_init_params params = { - /*.mem_size =*/ ggml_tensor_overhead() * slots.size(), - /*.mem_buffer =*/ nullptr, - /*.no_alloc =*/ true, - }; - st->ctx_data.reset(ggml_init(params)); - for (const auto & slot : slots) { - ggml_tensor * t = ggml_new_tensor_2d(st->ctx_data.get(), GGML_TYPE_F32, slot.ne0, slot.ne1); - ggml_format_name(t, "gen_state_%s", slot.name.c_str()); - st->slots.push_back(t); - } - st->buf.reset(ggml_backend_alloc_ctx_tensors(st->ctx_data.get(), ctx->backend)); - if (!st->buf) { - LOG_ERR("%s: failed to allocate the gen state\n", __func__); - return nullptr; - } - // a zero state is the cold start - ggml_backend_buffer_clear(st->buf.get(), 0); - - return st.release(); -} - -void clip_gen_state_free(clip_gen_state * state) { - delete state; -} - -void clip_gen_state_reset(clip_gen_state * state) { - if (state->buf) { - ggml_backend_buffer_clear(state->buf.get(), 0); - } -} - -const std::map> & clip_gen_get_voices(const clip_ctx * ctx) { - return ctx->model.gen_voices; -} - -clip_enc_state * clip_enc_state_init(clip_ctx * ctx, int32_t n_frames_max) { - const auto & hparams = ctx->model.hparams; - if (ctx->proj_type() == PROJECTOR_TYPE_LFM3A) { - // downsampler context, short-conv taps and attention window, see list_lfm3a_state_slots() - const auto slots = list_lfm3a_state_slots(hparams, ctx->model); - auto st = std::make_unique(); - ggml_init_params params = { - /*.mem_size =*/ ggml_tensor_overhead() * slots.size(), - /*.mem_buffer =*/ nullptr, - /*.no_alloc =*/ true, - }; - st->ctx_data.reset(ggml_init(params)); - for (const auto & slot : slots) { - ggml_tensor * t = ggml_new_tensor_2d(st->ctx_data.get(), GGML_TYPE_F32, slot.ne0, slot.ne1); - ggml_format_name(t, "enc_state_%s", slot.name.c_str()); - st->slots.push_back(t); - } - st->buf.reset(ggml_backend_alloc_ctx_tensors(st->ctx_data.get(), ctx->backend)); - if (!st->buf) { - LOG_ERR("%s: failed to allocate the encoder state\n", __func__); - return nullptr; - } - // a zero state is the cold start - ggml_backend_buffer_clear(st->buf.get(), 0); - return st.release(); - } - if (ctx->proj_type() != PROJECTOR_TYPE_VOXTRAL_RT || hparams.attn_window_size <= 0) { - LOG_ERR("%s: the encoder does not support streaming input\n", __func__); - return nullptr; - } - if (n_frames_max <= 0) { - LOG_ERR("%s: invalid n_frames_max %d\n", __func__, n_frames_max); - return nullptr; - } - - const int n_layer = hparams.n_layer; - const int d_head = hparams.n_embd_head > 0 ? hparams.n_embd_head : hparams.n_embd / hparams.n_head; - const int n_embd_kv = hparams.n_head_kv * d_head; - - auto st = std::make_unique(); - st->n_ctx = hparams.attn_window_size + n_frames_max; - st->slot_pos.assign(st->n_ctx, -1); - - ggml_init_params params = { - /*.mem_size =*/ ggml_tensor_overhead() * 2 * n_layer, - /*.mem_buffer =*/ nullptr, - /*.no_alloc =*/ true, - }; - st->ctx_data.reset(ggml_init(params)); - for (int il = 0; il < n_layer; il++) { - ggml_tensor * k = ggml_new_tensor_2d(st->ctx_data.get(), GGML_TYPE_F16, n_embd_kv, st->n_ctx); - ggml_tensor * v = ggml_new_tensor_2d(st->ctx_data.get(), GGML_TYPE_F16, n_embd_kv, st->n_ctx); - ggml_format_name(k, "enc_state_k_%d", il); - ggml_format_name(v, "enc_state_v_%d", il); - st->k.push_back(k); - st->v.push_back(v); - } - st->buf.reset(ggml_backend_alloc_ctx_tensors(st->ctx_data.get(), ctx->backend)); - if (!st->buf) { - LOG_ERR("%s: failed to allocate the encoder state\n", __func__); - return nullptr; - } - // empty slots are masked out, but they must not hold NaN - ggml_backend_buffer_clear(st->buf.get(), 0); - - LOG_INF("%s: encoder state: %d slots, %.2f MiB\n", __func__, st->n_ctx, ggml_backend_buffer_get_size(st->buf.get()) / 1024.0 / 1024.0); - - return st.release(); -} - -void clip_enc_state_free(clip_enc_state * state) { - delete state; -} - -bool clip_encode(struct clip_ctx * ctx, struct clip_encode_params * params) { - const clip_image_f32_batch & imgs = *params->imgs; - int n_batch_cur = imgs.entries.size(); - - // [QWEN_VIDEO] for video models, the batch dimension is used as temporal dimension for merged frames - if (!ctx->support_batch && n_batch_cur > clip_model_n_temporal_merge(ctx)) { - LOG_ERR("%s: batch size %d exceeds maximum supported batch/temporal-merge size %d\n", __func__, n_batch_cur, clip_model_n_temporal_merge(ctx)); - return false; - } - - // if buffers are not allocated, we need to do a warmup run to allocate them - if (!ctx->is_allocated) { - clip_model_loader::warmup(*ctx, *params->imgs); - } - - if (params->gen_process != CLIP_GEN_PROCESS_GEN_UNKNOWN && params->gen_state == nullptr) { - LOG_ERR("%s: audio gen requires a gen state\n", __func__); - return false; - } - - // build the inference graph - ggml_backend_sched_reset(ctx->sched.get()); - ggml_cgraph * gf = clip_get_graph_builder(ctx, imgs, params)->build(); - - // the last node is the embedding tensor, code2wav has no out_embd - // read it before scheduling: backends may reorder the nodes of graphs with several outputs (Metal) - ggml_tensor * embeddings = params->out_embd ? ggml_graph_node(gf, -1) : nullptr; - - if (!ggml_backend_sched_alloc_graph(ctx->sched.get(), gf)) { - LOG_ERR("%s: failed to allocate compute graph\n", __func__); - return false; - } - - // set inputs - const auto & model = ctx->model; - const auto & hparams = model.hparams; - - const int image_size_width = imgs.entries[0].nx(); - const int image_size_height = imgs.entries[0].ny(); - - const int patch_size = hparams.patch_size; - const int num_patches = ((image_size_width / patch_size) * (image_size_height / patch_size)); - const int n_pos = num_patches + (model.class_embedding ? 1 : 0); - const int pos_w = image_size_width / patch_size; - const int pos_h = image_size_height / patch_size; - - - auto get_inp_tensor = [&gf](const char * name) { - ggml_tensor * inp = ggml_graph_get_tensor(gf, name); - if (inp == nullptr) { - GGML_ABORT("Failed to get tensor %s", name); - } - if (!(inp->flags & GGML_TENSOR_FLAG_INPUT)) { - GGML_ABORT("Tensor %s is not an input tensor", name); - } - return inp; - }; - - auto set_input_f32 = [&get_inp_tensor](const char * name, const std::vector & values) { - ggml_tensor * cur = get_inp_tensor(name); - GGML_ASSERT(cur->type == GGML_TYPE_F32); - GGML_ASSERT(ggml_nelements(cur) == (int64_t)values.size()); - ggml_backend_tensor_set(cur, values.data(), 0, ggml_nbytes(cur)); - }; - - // mask from build_inp_attn_mask(), f16 if flash attn is enabled - auto set_input_attn_mask = [&get_inp_tensor](const std::vector & values, int idx = 0) { - ggml_tensor * cur = get_inp_tensor(get_attn_mask_name(idx).c_str()); - GGML_ASSERT(ggml_nelements(cur) == (int64_t)values.size()); - if (cur->type == GGML_TYPE_F16) { - std::vector values_f16(values.size()); - ggml_fp32_to_fp16_row(values.data(), values_f16.data(), values.size()); - ggml_backend_tensor_set(cur, values_f16.data(), 0, ggml_nbytes(cur)); - } else { - GGML_ASSERT(cur->type == GGML_TYPE_F32); - ggml_backend_tensor_set(cur, values.data(), 0, ggml_nbytes(cur)); - } - }; - - auto set_input_i32 = [&get_inp_tensor](const char * name, std::vector & values) { - ggml_tensor * cur = get_inp_tensor(name); - GGML_ASSERT(cur->type == GGML_TYPE_I32); - GGML_ASSERT(ggml_nelements(cur) == (int64_t)values.size()); - ggml_backend_tensor_set(cur, values.data(), 0, ggml_nbytes(cur)); - }; - - // state of the previous call, zero on a cold start - auto set_gen_state_in = [&]() { - const auto & slots = params->gen_state->slots; - size_t i = 0; - for (const auto & slot : list_gen_state_slots(hparams, model)) { - GGML_ASSERT(i < slots.size()); - ggml_backend_tensor_copy(slots[i++], get_inp_tensor(("state_in_" + slot.name).c_str())); - } - }; - - // lfm3a encoder and codec: causal sliding window over [cached window | this call] - auto set_lfm3a_window_inputs = [&](int base, int n_pos, int window) { - std::vector positions(n_pos); - for (int i = 0; i < n_pos; i++) { - positions[i] = base + i; - } - set_input_i32("positions", positions); - - // key j holds position base - prefix + j - const int prefix = window - 1; - const int n_kv = prefix + n_pos; - std::vector mask((size_t) n_kv * n_pos, -INFINITY); - for (int q = 0; q < n_pos; q++) { - for (int j = std::max(0, q + prefix - window + 1); j <= q + prefix; j++) { - if (base - prefix + j >= 0) { - mask[(size_t) q * n_kv + j] = 0.0f; - } - } - } - set_input_attn_mask(mask); - }; - - // rope positions and attention mask of the mimi transformers (pocket-tts). - // the mask is causal with a sliding window, see _build_attention_mask() in the reference - auto set_pockettts_tfm_inputs = [&]() { - const int64_t n_pos = ggml_nelements(get_inp_tensor("inp_pos")); - GGML_ASSERT(n_pos > 0); - std::vector positions((size_t) n_pos); - for (int64_t i = 0; i < n_pos; i++) { - positions[(size_t) i] = (int32_t) i; - } - set_input_i32("inp_pos", positions); - - // the preprocessor truncates the waveform to keep this mask bounded - const int64_t max_pos = (int64_t) clip_hparams::pockettts_max_spk_seconds * hparams.audio_sample_rate / 120; - GGML_ASSERT(n_pos <= max_pos && "pocket-tts speaker reference too long for a dense mask"); - - const int64_t context = hparams.mimi_tfm_context; - std::vector mask((size_t) n_pos * n_pos, -INFINITY); - for (int64_t q = 0; q < n_pos; q++) { - for (int64_t k = 0; k < n_pos; k++) { - const int64_t delta = q - k; - if (delta >= 0 && delta < context) { - mask[(size_t) q * n_pos + k] = 0.0f; - } - } - } - set_input_attn_mask(mask); - }; - - // set input pixel values - if (!imgs.is_audio) { - size_t nelem = 0; - for (const auto & img : imgs.entries) { - nelem += img.nx() * img.ny() * 3; - } - std::vector inp_raw(nelem); - - // layout of data (note: the channel dim is unrolled to better visualize the layout): - // - // ┌──W──┐ - // │ H │ channel = R - // ├─────┤ │ - // │ H │ channel = G - // ├─────┤ │ - // │ H │ channel = B - // └─────┘ │ - // ──────┘ x B - - // IMPORTANT: [QWEN_VIDEO] the batch dim is currently used for temporal dim in Qwen-VL models - // All entries must have the same spatial size (enforced by can_batch_with() during merging) - { - const int nx = imgs.entries[0].nx(); - const int ny = imgs.entries[0].ny(); - const int n = nx * ny; - - for (int b = 0; b < n_batch_cur; b++) { - LOG_DBG("%s: copying image %d/%d to input buffer (nx=%d, ny=%d)\n", __func__, b+1, n_batch_cur, nx, ny); - const auto & buf = imgs.entries[b].get_ro_buf(); - float * batch_entry = inp_raw.data() + b * (3*n); - for (int y = 0; y < ny; y++) { - for (int x = 0; x < nx; x++) { - size_t base_src = 3*(y * nx + x); - size_t base_dst = y * nx + x; - batch_entry[ base_dst] = buf[base_src ]; - batch_entry[1*n + base_dst] = buf[base_src + 1]; - batch_entry[2*n + base_dst] = buf[base_src + 2]; - } - } - } - } - set_input_f32("inp_raw", inp_raw); - - } else if (params->gen_process != CLIP_GEN_PROCESS_GEN_WAV && ggml_graph_get_tensor(gf, "inp_raw") != nullptr) { - // audio input. GEN_WAV is not here: it takes codes or feats, set in the switch below. - // a GEN_CODE graph may have no hidden-state input (lfm3a_gen feedback embedding) - GGML_ASSERT(imgs.entries.size() == 1); - - const auto & mel_inp = imgs.entries[0]; - const auto & buf = mel_inp.get_ro_buf(); - const int n_step = mel_inp.nx(); - const int n_mel = mel_inp.ny(); - GGML_ASSERT((size_t)n_step * n_mel == buf.size()); - - set_input_f32("inp_raw", buf); - } - - // set input per projector - switch (ctx->model.proj_type) { - case PROJECTOR_TYPE_VOXTRAL_RT: - { - clip_enc_state * st = params->enc_state; - if (!st) { - LOG_ERR("%s: this encoder needs a clip_enc_state\n", __func__); - return false; - } - const int n_window = hparams.attn_window_size; - const int n_new = (imgs.entries[0].nx() - 2) / 2; - GGML_ASSERT(n_new <= st->n_ctx - n_window); - - std::vector pos(n_new); - std::vector idx(n_new); - std::vector slot_pos = st->slot_pos; // including the frames written by this call - for (int i = 0; i < n_new; i++) { - pos[i] = st->n_past + i; - idx[i] = pos[i] % st->n_ctx; - slot_pos[idx[i]] = pos[i]; - } - set_input_i32("inp_pos", pos); - ggml_backend_tensor_set(get_inp_tensor("inp_kv_idx"), idx.data(), 0, idx.size() * sizeof(int64_t)); - - // each frame sees itself and the n_window - 1 frames before it - std::vector mask((size_t) st->n_ctx * n_new, -INFINITY); - for (int i = 0; i < n_new; i++) { - for (int s = 0; s < st->n_ctx; s++) { - const int32_t p = slot_pos[s]; - if (p >= 0 && p <= pos[i] && p > pos[i] - n_window) { - mask[(size_t) i * st->n_ctx + s] = 0.0f; - } - } - } - set_input_attn_mask(mask); - - std::vector tail = st->stem_tail; - tail.resize(hparams.n_embd, 0.0f); - set_input_f32("inp_stem_tail", tail); - } break; - case PROJECTOR_TYPE_MUSE_GLIMMER: - { - const int grid_w = pos_w; // image_size_width / patch_size - const int grid_h = pos_h; // image_size_height / patch_size - const int n_tok = grid_w * grid_h; - const int pgrid = (int) std::sqrt((double) ctx->model.position_embeddings->ne[1]); // 32 - const int f = hparams.n_merge; // downsample 2 - - // pixel patchify runs inside the graph via build_inp() (ggml_conv_2d); - // pos-emb bilinear interp via resize_position_embeddings(). - - // --- sparse window grouping (pgrid x pgrid windows) --- - const int win = pgrid; - const int nwin_h = (grid_h + win - 1) / win; - const int nwin_w = (grid_w + win - 1) / win; - std::vector sp_perm; sp_perm.reserve(n_tok); - std::vector sp_slens; - for (int wy = 0; wy < nwin_h; wy++) { - for (int wx = 0; wx < nwin_w; wx++) { - int cnt = 0; - for (int hh = 0; hh < win; hh++) { - for (int ww = 0; ww < win; ww++) { - const int gy = wy * win + hh; - const int gx = wx * win + ww; - if (gy < grid_h && gx < grid_w) { sp_perm.push_back(gy * grid_w + gx); cnt++; } - } - } - if (cnt > 0) sp_slens.push_back(cnt); - } - } - std::vector rpos_w(n_tok), rpos_h(n_tok), inv_perm(n_tok); - for (int i = 0; i < n_tok; i++) { - const int orig = sp_perm[i]; - rpos_w[i] = (orig % grid_w) + 1; // 1-indexed - rpos_h[i] = (orig / grid_w) + 1; - inv_perm[orig] = i; - } - set_input_i32("muse_glimmer_sp_perm", sp_perm); - set_input_i32("muse_glimmer_inv_perm", inv_perm); - set_input_i32("muse_glimmer_pos_w", rpos_w); - set_input_i32("muse_glimmer_pos_h", rpos_h); - - // block-diagonal window mask (permuted order) - std::vector sp_mask((size_t) n_tok * n_tok, -INFINITY); - { - int off = 0; - for (int s : sp_slens) { - for (int a = 0; a < s; a++) - for (int b = 0; b < s; b++) - sp_mask[(size_t) (off + a) * n_tok + (off + b)] = 0.0f; - off += s; - } - } - set_input_attn_mask(sp_mask); - - // pixel-shuffle gather (original order): f*f spatial neighbours grouped - std::vector dsp; dsp.reserve(n_tok); - for (int oy = 0; oy < grid_h / f; oy++) - for (int ox = 0; ox < grid_w / f; ox++) - for (int ry = 0; ry < f; ry++) - for (int rx = 0; rx < f; rx++) - dsp.push_back((oy * f + ry) * grid_w + (ox * f + rx)); - set_input_i32("muse_glimmer_ds_perm", dsp); - } break; - case PROJECTOR_TYPE_MINICPMV: - { - // inspired from siglip: - // -> https://huggingface.co/HuggingFaceM4/siglip-so400m-14-980-flash-attn2-navit - // -> https://huggingface.co/HuggingFaceM4/siglip-so400m-14-980-flash-attn2-navit/blob/d66538faeba44480d0bfaa42145eef26f9423199/modeling_siglip.py#L316 - std::vector positions(pos_h * pos_w); - // note: sized by the actual patch counts; a tall/wide image produces more - // than 1024 patches per side and a fixed [1024] array would be overrun - std::vector bucket_coords_h(pos_h); - std::vector bucket_coords_w(pos_w); - for (int i = 0; i < pos_h; i++){ - bucket_coords_h[i] = std::floor(70.0*i/pos_h); - } - for (int i = 0; i < pos_w; i++){ - bucket_coords_w[i] = std::floor(70.0*i/pos_w); - } - for (int i = 0, id = 0; i < pos_h; i++){ - for (int j = 0; j < pos_w; j++){ - positions[id++] = bucket_coords_h[i]*70 + bucket_coords_w[j]; - } - } - set_input_i32("positions", positions); - - // inputs for resampler projector - // set the 2D positions (using float for sinusoidal embedding) - int n_patches_per_col = image_size_width / patch_size; - std::vector pos_data(n_pos); - // dimension H - for (int i = 0; i < n_pos; i++) { - pos_data[i] = static_cast(i / n_patches_per_col); - } - set_input_f32("pos_h", pos_data); - // dimension W - for (int i = 0; i < n_pos; i++) { - pos_data[i] = static_cast(i % n_patches_per_col); - } - set_input_f32("pos_w", pos_data); - // base frequency omega - const float base_freq = 10000.0f; - const int n_embd_proj = clip_n_mmproj_embd(ctx); - std::vector omega(n_embd_proj / 4); - for (int i = 0; i < n_embd_proj / 4; ++i) { - omega[i] = 1.0f / std::pow(base_freq, static_cast(i) / (n_embd_proj / 4)); - } - set_input_f32("omega", omega); - } break; - case PROJECTOR_TYPE_MINICPMV4_6: - case PROJECTOR_TYPE_MINICPMV4_7: - case PROJECTOR_TYPE_MINICPM_OCR_1_0: - { - const bool is_4x = hparams.n_merge == 2; - - // SigLIP position buckets (same as resampler path) - std::vector positions(pos_h * pos_w); - std::vector bucket_coords_h(pos_h); - std::vector bucket_coords_w(pos_w); - for (int i = 0; i < pos_h; i++){ - bucket_coords_h[i] = std::floor(70.0*i/pos_h); - } - for (int i = 0; i < pos_w; i++){ - bucket_coords_w[i] = std::floor(70.0*i/pos_w); - } - for (int i = 0, id = 0; i < pos_h; i++){ - for (int j = 0; j < pos_w; j++){ - positions[id++] = bucket_coords_h[i]*70 + bucket_coords_w[j]; - } - } - set_input_i32("positions", positions); - - const int half_h = pos_h / 2; - const int half_w = pos_w / 2; - - auto make_ds_idx = [](int off_r, int off_c, int ds_h, int ds_w, int stride_w) { - std::vector idx(ds_h * ds_w); - for (int i = 0; i < ds_h; i++) { - for (int j = 0; j < ds_w; j++) { - idx[i * ds_w + j] = (2*i + off_r) * stride_w + (2*j + off_c); - } - } - return idx; - }; - - if (!is_4x) { - // window reorder indices for 2x2 windows - std::vector window_idx(n_pos); - std::vector inv_window_idx(n_pos); - { - int k = 0; - for (int wi = 0; wi < half_h; wi++) { - for (int wj = 0; wj < half_w; wj++) { - window_idx[k++] = (2*wi ) * pos_w + (2*wj ); - window_idx[k++] = (2*wi ) * pos_w + (2*wj + 1); - window_idx[k++] = (2*wi + 1) * pos_w + (2*wj ); - window_idx[k++] = (2*wi + 1) * pos_w + (2*wj + 1); - } - } - for (int i = 0; i < n_pos; i++) { - inv_window_idx[window_idx[i]] = i; - } - } - set_input_i32("vit_merger_window_idx", window_idx); - set_input_i32("vit_merger_inv_window_idx", inv_window_idx); - - // block-diagonal attention mask: tokens in the same 4-token - // window attend to each other (mask = 0), all other positions - // are masked out (-inf). matches the window-major reorder above. - std::vector window_mask_data(n_pos * n_pos, std::numeric_limits::lowest()); - for (int wi = 0; wi < n_pos / 4; wi++) { - for (int i = 0; i < 4; i++) { - for (int j = 0; j < 4; j++) { - window_mask_data[(wi*4 + i) * n_pos + (wi*4 + j)] = 0.0f; - } - } - } - set_input_attn_mask(window_mask_data); - - // ViT merger 2x2 downsample indices - auto vit_merger_ds_0 = make_ds_idx(0, 0, half_h, half_w, pos_w); - auto vit_merger_ds_1 = make_ds_idx(0, 1, half_h, half_w, pos_w); - auto vit_merger_ds_2 = make_ds_idx(1, 0, half_h, half_w, pos_w); - auto vit_merger_ds_3 = make_ds_idx(1, 1, half_h, half_w, pos_w); - set_input_i32("vit_merger_ds_idx_0", vit_merger_ds_0); - set_input_i32("vit_merger_ds_idx_1", vit_merger_ds_1); - set_input_i32("vit_merger_ds_idx_2", vit_merger_ds_2); - set_input_i32("vit_merger_ds_idx_3", vit_merger_ds_3); - } - - const int merger_h = is_4x ? pos_h : half_h; - const int merger_w = is_4x ? pos_w : half_w; - auto m_ds_0 = make_ds_idx(0, 0, merger_h / 2, merger_w / 2, merger_w); - auto m_ds_1 = make_ds_idx(0, 1, merger_h / 2, merger_w / 2, merger_w); - auto m_ds_2 = make_ds_idx(1, 0, merger_h / 2, merger_w / 2, merger_w); - auto m_ds_3 = make_ds_idx(1, 1, merger_h / 2, merger_w / 2, merger_w); - set_input_i32("merger_ds_idx_0", m_ds_0); - set_input_i32("merger_ds_idx_1", m_ds_1); - set_input_i32("merger_ds_idx_2", m_ds_2); - set_input_i32("merger_ds_idx_3", m_ds_3); - } break; - case PROJECTOR_TYPE_LFM3VL: - { - // patches in raster-scan order (no interleave before encoder layers) - const int pw = image_size_width / patch_size; - const int ph = image_size_height / patch_size; - std::vector positions(n_pos * 4); - int ptr = 0; - for (int y = 0; y < ph; y++) { - for (int x = 0; x < pw; x++) { - positions[ ptr] = y; - positions[ num_patches + ptr] = x; - positions[2 * num_patches + ptr] = y; - positions[3 * num_patches + ptr] = x; - ptr++; - } - } - set_input_i32("positions", positions); - } break; - case PROJECTOR_TYPE_QWEN2VL: - case PROJECTOR_TYPE_QWEN3VL: - case PROJECTOR_TYPE_LING3VL: - case PROJECTOR_TYPE_GLM4V: - case PROJECTOR_TYPE_GLM5V: - { - const int merge_ratio = hparams.n_merge; - const int pw = image_size_width / patch_size / merge_ratio; - const int ph = image_size_height / patch_size / merge_ratio; - std::vector positions(n_pos * 4); - int ptr = 0; - for (int ty = 0; ty < ph; ty++) { - for (int tx = 0; tx < pw; tx++) { - for (int dy = 0; dy < merge_ratio; dy++) { - for (int dx = 0; dx < merge_ratio; dx++) { - const int y = ty * merge_ratio + dy; - const int x = tx * merge_ratio + dx; - positions[ ptr] = y; - positions[ num_patches + ptr] = x; - positions[2 * num_patches + ptr] = y; - positions[3 * num_patches + ptr] = x; - ptr++; - } - } - } - } - - set_input_i32("positions", positions); - } break; - case PROJECTOR_TYPE_STEP3VL: - { - std::vector pos_data(n_pos); - for (int i = 0; i < n_pos; i++) { - pos_data[i] = i / pos_w; - } - set_input_i32("pos_h", pos_data); - for (int i = 0; i < n_pos; i++) { - pos_data[i] = i % pos_w; - } - set_input_i32("pos_w", pos_data); - } break; - case PROJECTOR_TYPE_PADDLEOCR: - { - const int merge_ratio = hparams.n_merge; - const int pw = image_size_width / patch_size / merge_ratio; - const int ph = image_size_height / patch_size / merge_ratio; - std::vector positions(n_pos * 4); - int ptr = 0; - // NOTE: same as Qwen-VL, but x and y are swapped - for (int ty = 0; ty < ph; ty++) { - for (int dy = 0; dy < merge_ratio; dy++) { - for (int tx = 0; tx < pw; tx++) { - for (int dx = 0; dx < merge_ratio; dx++) { - const int y = ty * merge_ratio + dy; - const int x = tx * merge_ratio + dx; - positions[ ptr] = y; - positions[ num_patches + ptr] = x; - positions[2 * num_patches + ptr] = y; - positions[3 * num_patches + ptr] = x; - ptr++; - } - } - } - } - - set_input_i32("positions", positions); - } break; - case PROJECTOR_TYPE_MINIMAX_M3: - { - const int n_merge = hparams.n_merge; - const int gh = image_size_height / patch_size; - const int gw = image_size_width / patch_size; - std::vector pos_h, pos_w; - pos_h.reserve(gh * gw); - pos_w.reserve(gh * gw); - for (int bh = 0; bh < gh / n_merge; bh++) - for (int bw = 0; bw < gw / n_merge; bw++) - for (int mh = 0; mh < n_merge; mh++) - for (int mw = 0; mw < n_merge; mw++) { - pos_h.push_back(bh * n_merge + mh); - pos_w.push_back(bw * n_merge + mw); - } - set_input_i32("minimax_pos_h", pos_h); - set_input_i32("minimax_pos_w", pos_w); - } break; - case PROJECTOR_TYPE_DOTS_OCR: - case PROJECTOR_TYPE_DOTS3NOTE_V: - { - const int pw = image_size_width / patch_size; - const int ph = image_size_height / patch_size; - const int n_pos = ph * pw; - std::vector positions(n_pos * 4); - int ptr = 0; - - // flat layout: [h, w, h, w] for each patch - // patches are in raster order (matching conv2d output) - for (int y = 0; y < ph; y++) { - for (int x = 0; x < pw; x++) { - positions[ ptr] = y; - positions[ n_pos + ptr] = x; - positions[2*n_pos + ptr] = y; - positions[3*n_pos + ptr] = x; - ptr++; - } - } - - set_input_i32("positions", positions); - } break; - case PROJECTOR_TYPE_QWEN25VL: - case PROJECTOR_TYPE_EXAONE4_5: - case PROJECTOR_TYPE_YOUTUVL: - { - // pw * ph = number of tokens output by ViT after apply patch merger - // ipw * ipw = number of vision token been processed inside ViT - const bool use_window_attn = - (ctx->model.proj_type == PROJECTOR_TYPE_QWEN25VL || ctx->model.proj_type == PROJECTOR_TYPE_EXAONE4_5) - ? hparams.n_wa_pattern > 0 - : !hparams.wa_layer_indexes.empty(); - const int merge_ratio = 2; - const int pw = image_size_width / patch_size / merge_ratio; - const int ph = image_size_height / patch_size / merge_ratio; - const int ipw = image_size_width / patch_size; - const int iph = image_size_height / patch_size; - - std::vector idx (ph * pw); - std::vector inv_idx(ph * pw); - - if (use_window_attn) { - const int attn_window_size = hparams.attn_window_size > 0 ? hparams.attn_window_size : 112; - const int grid_window = attn_window_size / patch_size / merge_ratio; - int dst = 0; - // [num_vision_tokens, num_vision_tokens] attention mask tensor - std::vector mask(pow(ipw * iph, 2), std::numeric_limits::lowest()); - int mask_row = 0; - - for (int y = 0; y < ph; y += grid_window) { - for (int x = 0; x < pw; x += grid_window) { - const int win_h = std::min(grid_window, ph - y); - const int win_w = std::min(grid_window, pw - x); - const int dst_0 = dst; - // group all tokens belong to the same window togather (to a continue range) - for (int dy = 0; dy < win_h; dy++) { - for (int dx = 0; dx < win_w; dx++) { - const int src = (y + dy) * pw + (x + dx); - GGML_ASSERT(src < (int)idx.size()); - GGML_ASSERT(dst < (int)inv_idx.size()); - idx [src] = dst; - inv_idx[dst] = src; - dst++; - } - } - - for (int r=0; r < win_h * win_w * merge_ratio * merge_ratio; r++) { - int row_offset = mask_row * (ipw * iph); - std::fill( - mask.begin() + row_offset + (dst_0 * merge_ratio * merge_ratio), - mask.begin() + row_offset + (dst * merge_ratio * merge_ratio), - 0.0); - mask_row++; - } - } - } - - set_input_i32("window_idx", idx); - set_input_i32("inv_window_idx", inv_idx); - set_input_attn_mask(mask); - } else { - for (int i = 0; i < ph * pw; i++) { - idx[i] = i; - } - } - - const int mpow = merge_ratio * merge_ratio; - std::vector positions(n_pos * 4); - - int ptr = 0; - for (int ty = 0; ty < ph; ty++) { - for (int tx = 0; tx < pw; tx++) { - for (int dy = 0; dy < merge_ratio; dy++) { - for (int dx = 0; dx < merge_ratio; dx++) { - auto remap = idx[ptr / mpow]; - remap = (remap * mpow) + (ptr % mpow); - - const int y = ty * merge_ratio + dy; - const int x = tx * merge_ratio + dx; - positions[ remap] = y; - positions[ num_patches + remap] = x; - positions[2 * num_patches + remap] = y; - positions[3 * num_patches + remap] = x; - ptr++; - } - } - } - } - - set_input_i32("positions", positions); - } break; - case PROJECTOR_TYPE_MIMOVL: - { - const int merge = hparams.n_merge; // 2 - const int merge_unit = merge * merge; // 4 - const int patch = hparams.patch_size; // 16 - const int H = image_size_height / patch; - const int W = image_size_width / patch; - const int n_pos_full = H * W; - const int llm_h = H / merge; - const int llm_w = W / merge; - const int n_units = llm_h * llm_w; // n_pos / merge_unit - - // Row-major merge-tile-ordered (h, w) positions - std::vector pos_h_row(n_pos_full); - std::vector pos_w_row(n_pos_full); - { - int idx = 0; - for (int ty = 0; ty < llm_h; ty++) { - for (int tx = 0; tx < llm_w; tx++) { - for (int dy = 0; dy < merge; dy++) { - for (int dx = 0; dx < merge; dx++) { - pos_h_row[idx] = ty * merge + dy; - pos_w_row[idx] = tx * merge + dx; - idx++; - } - } - } - } - } - - // Col-major merge-unit permutation - std::vector idx_col(n_units); - for (int r = 0; r < llm_h; r++) { - for (int c = 0; c < llm_w; c++) { - int u_row = r * llm_w + c; - int u_col = c * llm_h + r; - idx_col[u_col] = (float) u_row; - } - } - - // Col-mode positions: permute pos_*_row by idx_col - std::vector pos_h_col(n_pos_full); - std::vector pos_w_col(n_pos_full); - for (int u = 0; u < n_units; u++) { - int src = (int) idx_col[u]; - for (int k = 0; k < merge_unit; k++) { - pos_h_col[u * merge_unit + k] = pos_h_row[src * merge_unit + k]; - pos_w_col[u * merge_unit + k] = pos_w_row[src * merge_unit + k]; - } - } - - // Pack into ggml_rope_multi VISION-mode layout. The non-CPU kernels - // only read slots 0 and 1, so pack h in slot 0, w in slot 1: - // positions[0..n_pos) = h - // positions[n_pos..2*n_pos) = w - // positions[2*n_pos..3*n_pos) = 0 - // positions[3*n_pos..4*n_pos) = 0 - std::vector positions_row(static_cast(n_pos_full) * 4, 0); - std::vector positions_col(static_cast(n_pos_full) * 4, 0); - for (int i = 0; i < n_pos_full; i++) { - positions_row[0 * n_pos_full + i] = pos_h_row[i]; - positions_row[1 * n_pos_full + i] = pos_w_row[i]; - positions_col[0 * n_pos_full + i] = pos_h_col[i]; - positions_col[1 * n_pos_full + i] = pos_w_col[i]; - } - - // Banded 1D sliding-window mask - const int window = hparams.attn_window_size; - GGML_ASSERT(window > 0); - std::vector mask(static_cast(n_pos_full) * n_pos_full, std::numeric_limits::lowest()); - for (int q = 0; q < n_pos_full; q++) { - int lo = std::max(0, q - window); - int hi = std::min(n_pos_full - 1, q + window); - for (int k = lo; k <= hi; k++) { - mask[static_cast(q) * n_pos_full + k] = 0.0f; - } - } - - set_input_i32("mimovl_positions_row", positions_row); - set_input_i32("mimovl_positions_col", positions_col); - set_input_f32("mimovl_idx_col", idx_col); - set_input_attn_mask(mask); - } break; - case PROJECTOR_TYPE_PIXTRAL: - case PROJECTOR_TYPE_KIMIVL: - case PROJECTOR_TYPE_KIMIK25: - case PROJECTOR_TYPE_LIGHTONOCR: - { - // set the 2D positions - int n_patches_per_col = image_size_width / patch_size; - std::vector pos_data(n_pos); - // dimension H - for (int i = 0; i < n_pos; i++) { - pos_data[i] = i / n_patches_per_col; - } - set_input_i32("pos_h", pos_data); - // dimension W - for (int i = 0; i < n_pos; i++) { - pos_data[i] = i % n_patches_per_col; - } - set_input_i32("pos_w", pos_data); - } break; - case PROJECTOR_TYPE_SENSENOVA_U1: - { - const int n_patches_per_row = image_size_width / patch_size; - std::vector pos_data(n_pos); - for (int i = 0; i < n_pos; i++) { - pos_data[i] = i / n_patches_per_row; - } - set_input_i32("pos_y", pos_data); - for (int i = 0; i < n_pos; i++) { - pos_data[i] = i % n_patches_per_row; - } - set_input_i32("pos_x", pos_data); - } break; - case PROJECTOR_TYPE_DEEPSEEK4V: - { - // set the 2D positions (mrope layout, only the first 2 channels are used) - int n_patches_per_row = image_size_width / patch_size; - std::vector positions(n_pos * 4, 0); - for (int i = 0; i < n_pos; i++) { - positions[i] = i / n_patches_per_row; // row - positions[n_pos + i] = i % n_patches_per_row; // col - } - set_input_i32("positions", positions); - - // token block layout index (see clip_graph_deepseek4v::build) - // rows [0, n_grid) are the aligner output, the sentinels follow - const int n_merge = hparams.n_merge; - const int n_llm_w = CLIP_ALIGN(pos_w, n_merge) / n_merge; - const int n_llm_h = CLIP_ALIGN(pos_h, n_merge) / n_merge; - const int n_grid = n_llm_w * n_llm_h; - const int idx_start = n_grid; - const int idx_end = n_grid + 1; - const int idx_newline = n_grid + 2; - const int idx_pad = n_grid + 3; - - const int lead_pad = imgs.entries[0].lead_pad; - const auto bl = dsv4_get_block_layout(n_llm_w, n_llm_h, lead_pad); - - std::vector idx; - idx.reserve(bl.n_out); - for (int i = 0; i < lead_pad; i++) { - idx.push_back(idx_pad); - } - idx.push_back(idx_start); - // pairs of adjacent rows are interleaved column-wise ("N-layout") - // ref: build_image_block in inference/image_processor.py - for (int t = 0; t < bl.rows * bl.row_len; t++) { - const int g = t / (2 * bl.row_len); - const int rem = t % (2 * bl.row_len); - const int c = rem / 2; // column - const int r = 2 * g + rem % 2; // row - if (r >= n_llm_h) { - idx.push_back(idx_pad); - } else if (c == n_llm_w) { - idx.push_back(idx_newline); - } else { - idx.push_back(r * n_llm_w + c); - } - } - for (int i = 0; i < bl.pad_last; i++) { - idx.push_back(idx_pad); - } - idx.push_back(idx_end); - set_input_i32("layout_idx", idx); - } break; - case PROJECTOR_TYPE_DEEPSEEK41V: - { - // mrope layout, only the first 2 channels are used - const int n_patches_per_row = image_size_width / patch_size; - std::vector positions(n_pos * 4, 0); - for (int i = 0; i < n_pos; i++) { - positions[i] = i / n_patches_per_row; - positions[n_pos + i] = i % n_patches_per_row; - } - set_input_i32("positions", positions); - - // reading-order block: rows [0, n_grid) are the aligner output, then start / end / newline - const int n_merge = hparams.n_merge; - const int n_llm_w = CLIP_ALIGN(pos_w, n_merge) / n_merge; - const int n_llm_h = CLIP_ALIGN(pos_h, n_merge) / n_merge; - const int n_grid = n_llm_w * n_llm_h; - std::vector idx; - idx.reserve(dsv41_n_block_tokens(n_llm_w, n_llm_h)); - idx.push_back(n_grid); // IMAGE_START - for (int r = 0; r < n_llm_h; r++) { - for (int c = 0; c < n_llm_w; c++) { - idx.push_back(r * n_llm_w + c); - } - idx.push_back(n_grid + 2); // IMAGE_NEW_LINE - } - idx.push_back(n_grid + 1); // IMAGE_END - set_input_i32("layout_idx", idx); - } break; - case PROJECTOR_TYPE_GLM_EDGE: - { - // llava and other models - std::vector positions(n_pos); - for (int i = 0; i < n_pos; i++) { - positions[i] = i; - } - set_input_i32("positions", positions); - } break; - case PROJECTOR_TYPE_MLP: - case PROJECTOR_TYPE_MLP_NORM: - case PROJECTOR_TYPE_LDP: - case PROJECTOR_TYPE_LDPV2: - { - // llava and other models - std::vector positions(n_pos); - for (int i = 0; i < n_pos; i++) { - positions[i] = i; - } - set_input_i32("positions", positions); - - // The patches vector is used to get rows to index into the embeds with; - // we should skip dim 0 only if we have CLS to avoid going out of bounds - // when retrieving the rows. - int patch_offset = model.class_embedding ? 1 : 0; - std::vector patches(num_patches); - for (int i = 0; i < num_patches; i++) { - patches[i] = i + patch_offset; - } - set_input_i32("patches", patches); - } break; - case PROJECTOR_TYPE_POCKETTTS_SPKENC: - { - set_pockettts_tfm_inputs(); - } break; - case PROJECTOR_TYPE_POCKETTTS_GEN: - { - if (params->gen_process == CLIP_GEN_PROCESS_GEN_WAV) { - GGML_ASSERT(params->feats != nullptr); - set_input_f32("inp_feats", *params->feats); - // positions and mask are derived in-graph from the persisted counter - set_gen_state_in(); - } else { - // flow matching starts from gaussian noise, std = sqrt(temp) - ggml_tensor * t = get_inp_tensor("inp_noise"); - // Config.default_temperature, for a caller that does not set one - const float temp = params->temp > 0.0f ? params->temp : 0.7f; - std::normal_distribution dist(0.0f, std::sqrt(temp)); - std::vector noise(ggml_nelements(t)); - for (auto & v : noise) { - v = dist(params->gen_state->rng); - } - set_input_f32("inp_noise", noise); - } - } break; - case PROJECTOR_TYPE_SOPRANO: - { - set_input_f32("inp_feats", *params->feats); - } break; - case PROJECTOR_TYPE_GEMMA4V: - case PROJECTOR_TYPE_GEMMA4UV: - { - // set (col, row) patch positions for learned positional embedding - const int n_cols = image_size_width / patch_size; - std::vector pos_x(num_patches), pos_y(num_patches); - for (int i = 0; i < num_patches; i++) { - pos_x[i] = i % n_cols; - pos_y[i] = i / n_cols; - } - set_input_i32("pos_x", pos_x); - set_input_i32("pos_y", pos_y); - } break; - case PROJECTOR_TYPE_DEEPSEEKOCR: - case PROJECTOR_TYPE_DEEPSEEKOCR2: - { - GGML_ASSERT( - (pos_w == pos_h) // overview image - || (pos_h >= pos_w && pos_h % pos_w == 0) // tile images - ); - - const int window = hparams.attn_window_size; - const int pos = pos_w; - std::vector rel_pos_indices_local(window * window); - std::vector rel_pos_indices_global(pos * pos); - - for (int q = 0; q < window; q++) { - for (int k = 0; k < window; k++) { - rel_pos_indices_local[q * window + k] = q - k + window - 1; - } - } - - for (int q = 0; q < pos; q++) { - for (int k = 0; k < pos; k++) { - rel_pos_indices_global[q * pos + k] = q - k + pos - 1; - } - } - - set_input_i32("rel_pos_indices_local", rel_pos_indices_local); - set_input_i32("rel_pos_indices_global", rel_pos_indices_global); - - if (ctx->proj_type() == PROJECTOR_TYPE_DEEPSEEKOCR2) { - - // qwen2 encoder attention mask - - // num_image_tokens = num_patches / 16 - // 256 for 1024 global view - // 144 for 768 tile views - const int num_image_tokens = num_patches / 16; - const int seq_len = num_image_tokens * 2; - std::vector qwen2_mask(static_cast(seq_len) * seq_len, 0.0f); - - // attention mask layout - // +--------------+---------------+ - // | all 0 | all -inf | - // +--------------+---------------+ - // | all 0 | lower tri 0 | - // +--------------+---------------+ - for (int i = 0; i < seq_len; i++) { - for (int j = 0; j < seq_len; j++) { - const bool zero = i < num_image_tokens ? - j < num_image_tokens : - j < num_image_tokens || j <= i; - qwen2_mask[static_cast(i) * seq_len + j] = zero ? 0.0f : -1e9f; - } - } - set_input_attn_mask(qwen2_mask); - } - } break; - case PROJECTOR_TYPE_GEMMA3: - case PROJECTOR_TYPE_GEMMA3NV: - case PROJECTOR_TYPE_IDEFICS3: - case PROJECTOR_TYPE_COHERE2V: - case PROJECTOR_TYPE_INTERNVL: - case PROJECTOR_TYPE_NEMOTRON_V2_VL: - case PROJECTOR_TYPE_NEMOTRON_OMNI: - case PROJECTOR_TYPE_QWEN2A: - case PROJECTOR_TYPE_QWEN3A: - case PROJECTOR_TYPE_GLMA: - case PROJECTOR_TYPE_ULTRAVOX: - case PROJECTOR_TYPE_LFM2: - case PROJECTOR_TYPE_D1OMNI_V: - case PROJECTOR_TYPE_VOXTRAL: - case PROJECTOR_TYPE_MERALION: - case PROJECTOR_TYPE_MUSIC_FLAMINGO: - case PROJECTOR_TYPE_JANUS_PRO: - case PROJECTOR_TYPE_PHI4: - case PROJECTOR_TYPE_COGVLM: - case PROJECTOR_TYPE_YASA2: - case PROJECTOR_TYPE_GEMMA4UA: - case PROJECTOR_TYPE_MUSE_SPARK_AUDIO: - case PROJECTOR_TYPE_QWEN3TTS_SPKENC: - { - // do nothing - } break; - case PROJECTOR_TYPE_LFM3A_GEN: - { - const int64_t n_cb = model.gen_code_embd_w->ne[2]; - const int64_t n_vocab = model.gen_code_embd_w->ne[1]; - if (params->gen_process == CLIP_GEN_PROCESS_GEN_WAV) { - GGML_ASSERT(params->codes != nullptr); - // codes index the FSQ table via ggml_get_rows, the end-of-audio code has no waveform - const int64_t n_codes = model.c2w.quant_first_cb_w->ne[1]; - for (int32_t c : *params->codes) { - if (c < 0 || c >= n_codes) { - LOG_ERR("%s: code out of range (%d, codebook size %lld)\n", __func__, c, (long long) n_codes); - return false; - } - } - std::vector codes = *params->codes; - set_input_i32("inp_codes", codes); - set_gen_state_in(); - // the "pos" slot leads the state - float base = 0.0f; - ggml_backend_tensor_get(params->gen_state->slots[0], &base, 0, sizeof(float)); - set_lfm3a_window_inputs((int) base, (int) ggml_nelements(get_inp_tensor("positions")), hparams.wav_tfm_swa); - } else { - // rows of the merged per-codebook tables; a masked codebook (-1) takes row 0, the mask drops it - std::vector codes(n_cb, -1); - if (params->codes) { - GGML_ASSERT(params->codes->size() == (size_t) n_cb); - codes = *params->codes; - } - std::vector mask(n_cb, 0.0f); - for (int64_t i = 0; i < n_cb; i++) { - if (codes[i] >= n_vocab) { - LOG_ERR("%s: code out of range (%d, vocab %lld)\n", __func__, codes[i], (long long) n_vocab); - return false; - } - if (codes[i] < 0) { - codes[i] = 0; - } else { - mask[i] = 1.0f; - } - codes[i] += (int32_t) (i * n_vocab); - } - set_input_i32("inp_codes", codes); - if (ggml_graph_get_tensor(gf, "inp_mask") != nullptr) { - set_input_f32("inp_mask", mask); - } - } - } break; - case PROJECTOR_TYPE_QWEN3TTS_GEN: - { - if (params->gen_process == CLIP_GEN_PROCESS_GEN_WAV) { - GGML_ASSERT(params->codes != nullptr); - - // frame-major input to group-major, rear-padded with code 0 up to one window - const int64_t n_codes = model.gen_code_head_w->ne[2] + 1; - const int64_t n_frames_w = hparams.wav_tfm_swa; - const int64_t n_frames = (int64_t) params->codes->size() / n_codes; - GGML_ASSERT(n_frames > 0 && n_frames <= n_frames_w); - - // codes are used as ggml_get_rows indices, so check them against the codebook vocab - const int64_t vocab_first = model.c2w.quant_first_cb_w->ne[1]; - const int64_t vocab_rest = model.c2w.quant_rest_cb_w->ne[1]; - for (int64_t f = 0; f < n_frames; f++) { - for (int64_t g = 0; g < n_codes; g++) { - const int32_t c = (*params->codes)[f * n_codes + g]; - const int64_t vocab = (g == 0) ? vocab_first : vocab_rest; - if (c < 0 || (int64_t) c >= vocab) { - LOG_ERR("%s: code out of range (frame %lld, group %lld, code %d, vocab %lld)\n", - __func__, (long long) f, (long long) g, c, (long long) vocab); - return false; - } - } - } - - std::vector codes(n_frames_w * n_codes, 0); - for (int64_t f = 0; f < n_frames; f++) { - for (int64_t g = 0; g < n_codes; g++) { - codes[g * n_frames_w + f] = (*params->codes)[f * n_codes + g]; - } - } - set_input_i32("inp_codes", codes); - set_gen_state_in(); - } else { - // code0 indexes gen_code_out_embd_w via ggml_get_rows; bound it - const int64_t vocab0 = model.gen_code_out_embd_w->ne[1]; - if (params->code0 < 0 || (int64_t) params->code0 >= vocab0) { - LOG_ERR("%s: code0 out of range (%d, vocab %lld)\n", __func__, params->code0, (long long) vocab0); - return false; - } - std::vector code0 = { params->code0 }; - set_input_i32("inp_code0", code0); - - // one uniform(0,1) draw per codebook, used by do_sampling() - std::uniform_real_distribution dist(0.0f, 1.0f); - const int64_t n_acoustic = model.gen_code_head_w->ne[2]; - for (int64_t g = 0; g < n_acoustic; g++) { - std::vector r = { dist(params->gen_state->rng) }; - set_input_f32(("inp_rand_" + std::to_string(g)).c_str(), r); - } - } - } break; - case PROJECTOR_TYPE_HUNYUANVL: - { - // Compute the HunyuanVL 2D position embedding on CPU (with the - // custom sf=(target+0.1)/n_grid bilinear sampling that the - // reference implementation uses) and upload it to the graph - // input declared in clip_graph_hunyuanvl::build(). - GGML_ASSERT(model.position_embeddings != nullptr); - ggml_tensor * src_t = model.position_embeddings; - const int64_t n_embd = src_t->ne[0]; - const int64_t n_pos = src_t->ne[1]; // = n_grid * n_grid - const int n_grid = (int)std::lround(std::sqrt((double)n_pos)); - GGML_ASSERT((int64_t)n_grid * n_grid == n_pos); - const int out_w = pos_w; // pw - const int out_h = pos_h; // ph - - // Pull weight to host. - std::vector src(n_embd * n_pos); - ggml_backend_tensor_get(src_t, src.data(), 0, ggml_nbytes(src_t)); - - // Output layout matches ggml_new_tensor_2d(F32, n_embd, out_h*out_w): - // ne[0] = n_embd (fastest), ne[1] = out_h*out_w - // dst[(y*out_w + x) * n_embd + c] - std::vector dst((size_t)n_embd * out_h * out_w); - - const float sx = (float)(out_w + 0.1f) / (float)n_grid; - const float sy = (float)(out_h + 0.1f) / (float)n_grid; - - for (int y = 0; y < out_h; ++y) { - // Match ggml_compute_forward_upscale_f32 pixel-center - // convention (align_corners=False): src_y = (y+0.5)/sy - 0.5. - const float fy = ((float)y + 0.5f) / sy - 0.5f; - int y0 = (int)std::floor(fy); - int y1 = y0 + 1; - y0 = std::clamp(y0, 0, n_grid - 1); - y1 = std::clamp(y1, 0, n_grid - 1); - float wy1 = std::clamp(fy - (float)y0, 0.0f, 1.0f); - const float wy0 = 1.0f - wy1; - for (int x = 0; x < out_w; ++x) { - const float fx = ((float)x + 0.5f) / sx - 0.5f; - int x0 = (int)std::floor(fx); - int x1 = x0 + 1; - x0 = std::clamp(x0, 0, n_grid - 1); - x1 = std::clamp(x1, 0, n_grid - 1); - float wx1 = std::clamp(fx - (float)x0, 0.0f, 1.0f); - const float wx0 = 1.0f - wx1; - - const float w00 = wy0 * wx0; - const float w01 = wy0 * wx1; - const float w10 = wy1 * wx0; - const float w11 = wy1 * wx1; - - const float * s00 = &src[((size_t)y0 * n_grid + x0) * n_embd]; - const float * s01 = &src[((size_t)y0 * n_grid + x1) * n_embd]; - const float * s10 = &src[((size_t)y1 * n_grid + x0) * n_embd]; - const float * s11 = &src[((size_t)y1 * n_grid + x1) * n_embd]; - float * d = &dst[((size_t)y * out_w + x) * n_embd]; - for (int c = 0; c < n_embd; ++c) { - d[c] = w00 * s00[c] + w01 * s01[c] + w10 * s10[c] + w11 * s11[c]; - } - } - } - - set_input_f32("hunyuanvl_pos_embd", dst); - } break; - case PROJECTOR_TYPE_LLAMA4: - { - // set the 2D positions - int n_patches_per_col = image_size_width / patch_size; - std::vector pos_data(num_patches + 1, 0); // +1 for the [CLS] token - // last pos is always kept 0, it's for CLS - // dimension H - for (int i = 0; i < num_patches; i++) { - pos_data[i] = (i / n_patches_per_col) + 1; - } - set_input_i32("pos_h", pos_data); - // dimension W - for (int i = 0; i < num_patches; i++) { - pos_data[i] = (i % n_patches_per_col) + 1; - } - set_input_i32("pos_w", pos_data); - } break; - case PROJECTOR_TYPE_DOTS3NOTE_A: - { - GGML_ASSERT(imgs.entries.size() == 1); - const int n_pos = (imgs.entries.front().nx() + 7) / 8; // 3x stride-2 conv2d - std::vector positions(n_pos); - for (int i = 0; i < n_pos; i++) { - positions[i] = i; - } - set_input_i32("positions", positions); - } break; - case PROJECTOR_TYPE_GEMMA4A: - { - GGML_ASSERT(imgs.entries.size() == 1); - const auto & img0 = imgs.entries.front(); - // Compute n_pos matching SSCP output: two stride-2 convs - int n_pos = img0.nx(); - for (int i = 0; i < 2; i++) { n_pos = (n_pos - 1) / 2 + 1; } - - // Chunked local attention: blocked causal mask and RPE - const int chunk_size = 12; - const int max_past = 12; - const int context_size = chunk_size + max_past; - const int num_blocks = (n_pos + chunk_size - 1) / chunk_size; - - // Blocked causal attention mask: [context_size, chunk_size, num_blocks] - { - std::vector mask(context_size * chunk_size * num_blocks, -1e9f); - for (int b = 0; b < num_blocks; b++) { - for (int q = 0; q < chunk_size; q++) { - int gq = b * chunk_size + q; - for (int k = 0; k < context_size; k++) { - int gk = b * chunk_size - max_past + k; - if (gq < n_pos && gk >= 0 && gk < n_pos && gk <= gq && (gq - gk) < max_past) { - mask[k + q * context_size + b * context_size * chunk_size] = 0.0f; - } - } - } - } - set_input_f32("kq_mask", mask); - } - - // Sinusoidal RPE: 13 positions [12, 11, ..., 0] - { - const int n_embd = ctx->model.hparams.n_embd; - const int num_timescales = n_embd / 2; - const float log_timescale_increment = logf(10000.0f) / std::max(num_timescales - 1, 1); - const int rpe_len = max_past + 1; - std::vector pos_emb(n_embd * rpe_len, 0.0f); - for (int p = 0; p < rpe_len; p++) { - float position = (float)(max_past - p); - for (int i = 0; i < num_timescales; i++) { - float inv_ts = expf(-(float)i * log_timescale_increment); - float scaled = position * inv_ts; - pos_emb[p * n_embd + i] = sinf(scaled); - pos_emb[p * n_embd + i + num_timescales] = cosf(scaled); - } - } - set_input_f32("pos_emb", pos_emb); - } - } break; - case PROJECTOR_TYPE_MIMO_AUDIO: - { - GGML_ASSERT(imgs.entries.size() == 1); - const int n_frames = imgs.entries.front().nx(); - const int n_pos = (n_frames - 1) / 2 + 1; // matches conv1(s=1)+conv2(s=2) output length - - std::vector positions(n_pos); - for (int i = 0; i < n_pos; i++) { - positions[i] = i; - } - set_input_i32("mimo_audio_positions", positions); - - const int window = hparams.attn_window_size; - GGML_ASSERT(window > 0); - - const float neg_inf = std::numeric_limits::lowest(); - std::vector full_mask((size_t) n_pos * n_pos); - std::vector window_mask((size_t) n_pos * n_pos); - for (int q = 0; q < n_pos; q++) { - for (int k = 0; k < n_pos; k++) { - const bool causal_ok = k <= q; - full_mask[(size_t) q * n_pos + k] = causal_ok ? 0.0f : neg_inf; - window_mask[(size_t) q * n_pos + k] = (causal_ok && (q - k) <= window) ? 0.0f : neg_inf; - } - } - set_input_attn_mask(full_mask, 0); - set_input_attn_mask(window_mask, 1); - - // input_local_transformer: block-diagonal mask + in-group positions - { - const int n_pos_ds = (n_pos - 2) / 2 + 1; // matches downsample conv (k=2,s=2,p=0) - const int group_size = hparams.audio_local_group_size; - GGML_ASSERT(group_size > 0); - const int n_groups = (n_pos_ds + group_size - 1) / group_size; - const int n_padded = n_groups * group_size; - - std::vector local_positions(n_padded); - for (int i = 0; i < n_padded; i++) { - local_positions[i] = i % group_size; - } - set_input_i32("mimo_audio_local_positions", local_positions); - - std::vector local_mask((size_t) n_padded * n_padded); - for (int q = 0; q < n_padded; q++) { - for (int k = 0; k < n_padded; k++) { - const bool same_group = (q / group_size) == (k / group_size); - local_mask[(size_t) q * n_padded + k] = same_group ? 0.0f : neg_inf; - } - } - set_input_attn_mask(local_mask, 2); - } - } break; - case PROJECTOR_TYPE_LFM2A: - case PROJECTOR_TYPE_D1OMNI_A: - { - GGML_ASSERT(imgs.entries.size() == 1); - const auto n_frames = clip_n_output_tokens(ctx, &imgs.entries.front()); - - auto d_model = 512; - auto seq_len = n_frames * 2 - 1; - std::vector pos_emb(d_model*seq_len); - std::vector inv_freq(d_model / 2); - for (size_t i = 0; i < inv_freq.size(); ++i) { - inv_freq[i] = std::exp(-(std::log(10000.0) / (float)d_model) * (2.0f * (float)(i))); - } - for (int64_t pos = 0; pos < seq_len; ++pos) { - for (size_t i = 0; i < inv_freq.size(); ++i) { - const float ang = (n_frames - pos - 1) * inv_freq[i]; - pos_emb[pos*d_model + 2*i + 0] = sinf(ang); // even - pos_emb[pos*d_model + 2*i + 1] = cosf(ang); // odd - } - } - set_input_f32("pos_emb", pos_emb); - } break; - case PROJECTOR_TYPE_LFM3A: - { - GGML_ASSERT(imgs.entries.size() == 1); - const int n_pos = clip_n_output_tokens(ctx, &imgs.entries.front()); - - // the "pos" slot leads the state; without a stream the state is zero, the clip is encoded from its start - int base = 0; - if (params->enc_state != nullptr) { - const auto & slots = params->enc_state->slots; - float base_f = 0.0f; - ggml_backend_tensor_get(slots[0], &base_f, 0, sizeof(float)); - base = (int) base_f; - size_t i = 0; - for (const auto & slot : list_lfm3a_state_slots(hparams, model)) { - ggml_backend_tensor_copy(slots[i++], get_inp_tensor(("state_in_" + slot.name).c_str())); - } - } else { - for (const auto & slot : list_lfm3a_state_slots(hparams, model)) { - ggml_tensor * t = get_inp_tensor(("state_in_" + slot.name).c_str()); - ggml_backend_tensor_memset(t, 0, 0, ggml_nbytes(t)); - } - } - - set_lfm3a_window_inputs(base, n_pos, hparams.attn_window_size); - } break; - case PROJECTOR_TYPE_PARAKEET: - { - GGML_ASSERT(imgs.entries.size() == 1); - struct ggml_tensor * attn_mask = ggml_graph_get_tensor(gf, "attn_mask"); - const int n_q = attn_mask->ne[1]; - const int n_k = attn_mask->ne[0]; - const int n_frames = imgs.entries.front().nx(); - const int n_tokens_real = (n_frames + hparams.subsampling_factor-1) / hparams.subsampling_factor; - const float mask_value = -1e30f; - - std::vector mask_data(n_q * n_k); - if (n_k == n_q) { - // full attention: mask keys that are padding - for (int q = 0; q < n_q; ++q) { - for (int k = 0; k < n_k; ++k) { - mask_data[q * n_k + k] = (k >= n_tokens_real) ? mask_value : 0.0f; - } - } - } else { - // local attention: mask keys outside the valid window - const int att_left = n_k / 2; - for (int q = 0; q < n_q; ++q) { - for (int k = 0; k < n_k; ++k) { - const int key = q - att_left + k; - mask_data[q * n_k + k] = (key >= 0 && key < n_tokens_real) ? 0.0f : mask_value; - } - } - } - set_input_f32(attn_mask->name, mask_data); - - // local attention skew mask: zeroes out the probs that were - // computed for keys outside the valid sliding window. - if (struct ggml_tensor * local_mask = ggml_graph_get_tensor(gf, "local_mask")) { - const int lm_k = local_mask->ne[0]; - const int lm_q = local_mask->ne[1]; - const int window_size = lm_k - lm_q + 1; - std::vector lm_data(lm_q * lm_k); - for (int q = 0; q < lm_q; ++q) { - for (int k = 0; k < lm_k; ++k) { - const int rel = k - q; - lm_data[q * lm_k + k] = (rel >= 0 && rel < window_size) ? 1.0f : 0.0f; - } - } - set_input_f32(local_mask->name, lm_data); - } - - // Generate rotation frequencies for relative positional encoding. - { - const int n_state = hparams.n_embd; - const int d_half = n_state / 2; - const float log_10000 = logf(10000.0f); - std::vector freqs(d_half); - for (int k = 0; k < d_half; ++k) { - freqs[k] = expf(-(float(k * 2) * log_10000 / float(n_state))); - } - set_input_f32("pos_freqs", freqs); - } - - // Generate relative positional distance values which scaled by - // the frequency to produce the angles for sin/cos. - { - // window_size is only known after graph construction since it depends on - // n_time from the conv output, so we read it back from the graph tensor. - struct ggml_tensor * rel_pos = ggml_graph_get_tensor(gf, "rel_positions"); - const int window_size = rel_pos->ne[1]; - std::vector pos(window_size); - // local attention: window is fixed at [att_left, att_right] - // full attention: window covers the full sequence, centered - if (ggml_graph_get_tensor(gf, "local_mask")) { - const int att_left = window_size / 2; - for (int t = 0; t < window_size; ++t) { - pos[t] = float(att_left - t); - } - } else { - const int n_time = (window_size + 1) / 2; - for (int t = 0; t < window_size; ++t) { - pos[t] = float(n_time - 1 - t); - } - } - set_input_f32(rel_pos->name, pos); - } - } break; - case PROJECTOR_TYPE_GRANITE_SPEECH: - { - const int context_size = ctx->model.hparams.audio_chunk_size; - const int max_pos_emb = ctx->model.hparams.audio_max_pos_emb; - - std::vector dists((size_t) context_size * (size_t) context_size); - for (int i = 0; i < context_size; i++) { - for (int j = 0; j < context_size; j++) { - int d = i - j; - if (d < -context_size) d = -context_size; - if (d > context_size) d = context_size; - dists[(size_t) i * (size_t) context_size + (size_t) j] = d + max_pos_emb; - } - } - set_input_i32("attn_dists", dists); - - const int n_frames = image_size_width; - const int remainder = n_frames % context_size; - if (remainder > 0) { - const int num_blocks = (n_frames + context_size - 1) / context_size; - std::vector mask((size_t) context_size * (size_t) context_size * (size_t) num_blocks, 0.0f); - const float neg_inf = -INFINITY; - const size_t last_block_offset = (size_t) (num_blocks - 1) * (size_t) context_size * (size_t) context_size; - for (int q = 0; q < context_size; q++) { - for (int k = 0; k < context_size; k++) { - if (q >= remainder || k >= remainder) { - mask[last_block_offset + (size_t) q * (size_t) context_size + (size_t) k] = neg_inf; - } - } - } - set_input_f32("attn_mask", mask); - } - } break; - case PROJECTOR_TYPE_GRANITE4_VISION: - { - // Granite Vision 4.1 uses precomputed permutation index - // tensors to express the _win / _unwin / spatial sampling - // reshapes as ggml_get_rows gathers. The names are set - // by g4v_gather() in models/granite4-vision.cpp. - const int patch_size = model.hparams.patch_size; - const int image_side = imgs.entries.front().nx() / patch_size; - const int window_side = hparams.downsample_window_side; - const int query_side = hparams.downsample_query_side; - const int n = image_side / window_side; - const int new_side = n * query_side; - - // Builds the raster→window permutation indices for a - // (side, side) grid split into (n × n) windows of (win × win) - // tokens each. dst[w * win*win + p] = source raster index. - auto make_win_idx = [](int side, int win) { - const int nn = side / win; - std::vector idx(static_cast(side) * side); - for (int wy = 0; wy < nn; ++wy) { - for (int wx = 0; wx < nn; ++wx) { - for (int iy = 0; iy < win; ++iy) { - for (int ix = 0; ix < win; ++ix) { - const int w = wy * nn + wx; - const int p = iy * win + ix; - const int y = wy * win + iy; - const int x = wx * win + ix; - idx[static_cast(w) * (win*win) + p] = y * side + x; - } - } - } - } - return idx; - }; - - auto make_unwin_idx = [&](int side, int win) { - const std::vector fwd = make_win_idx(side, win); - std::vector inv(fwd.size()); - for (size_t i = 0; i < fwd.size(); ++i) { - inv[fwd[i]] = static_cast(i); - } - return inv; - }; - - auto make_spatial_idx = [](int side, int offset) { - const int off_y = (offset >> 1) & 1; - const int off_x = offset & 1; - const int new_s = side / 2; - std::vector idx(static_cast(new_s) * new_s); - for (int y = 0; y < new_s; ++y) { - for (int x = 0; x < new_s; ++x) { - idx[y * new_s + x] = (y * 2 + off_y) * side + (x * 2 + off_x); - } - } - return idx; - }; - - // the same permutation is applied to every tile of the stacked image - auto upload = [&](const std::string & name, const std::vector & idx) { - ggml_tensor * t = ggml_graph_get_tensor(gf, name.c_str()); - GGML_ASSERT(t); - GGML_ASSERT(ggml_nelements(t) % (int64_t) idx.size() == 0); - const int n_rep = ggml_nelements(t) / idx.size(); - std::vector buf; - buf.reserve(idx.size() * n_rep); - for (int i = 0; i < n_rep; ++i) { - buf.insert(buf.end(), idx.begin(), idx.end()); - } - ggml_backend_tensor_set(t, buf.data(), 0, ggml_nbytes(t)); - }; - - // Stage 1b only uses block 0's permutations; future stages - // will upload all blocks. - for (size_t bid = 0; bid < hparams.feature_layers.size(); ++bid) { - const std::string prefix = "g4v_blk" + std::to_string(bid) + "_"; - upload(prefix + "win_idx", make_win_idx(image_side, window_side)); - upload(prefix + "qwin_idx", make_win_idx(new_side, query_side)); - upload(prefix + "unwin_idx", make_unwin_idx(new_side, query_side)); - const auto spatial_offset = hparams.proj_spatial_offsets[bid]; - if (spatial_offset >= 0) { - upload(prefix + "spatial_idx", make_spatial_idx(image_side,spatial_offset)); - } - } - } break; - default: - GGML_ABORT("Unknown projector type"); - } - - // ggml_backend_cpu_set_n_threads(ctx->backend_cpu, n_threads); - ggml_backend_dev_t dev = ggml_backend_get_device(ctx->backend_cpu); - ggml_backend_reg_t reg = dev ? ggml_backend_dev_backend_reg(dev) : nullptr; - if (reg) { - auto ggml_backend_set_n_threads_fn = (ggml_backend_set_n_threads_t) ggml_backend_reg_get_proc_address(reg, "ggml_backend_set_n_threads"); - if (ggml_backend_set_n_threads_fn) { - ggml_backend_set_n_threads_fn(ctx->backend_cpu, params->n_threads); - } - } - - auto status = ggml_backend_sched_graph_compute(ctx->sched.get(), gf); - if (status != GGML_STATUS_SUCCESS) { - LOG_ERR("%s: ggml_backend_sched_graph_compute failed with error %d\n", __func__, status); - return false; - } - - if (embeddings != nullptr) { - // sanity check (assuming that all images in batch have the same number of tokens, so we only check the first one) - const int n_tokens_out = embeddings->ne[1]; - const int expected_n_tokens_out = clip_n_output_tokens(ctx, &imgs.entries[0]); - if (n_tokens_out != expected_n_tokens_out) { - LOG_ERR("%s: expected output %d tokens, got %d\n", __func__, expected_n_tokens_out, n_tokens_out); - GGML_ABORT("Invalid number of output tokens"); - } - - LOG_DBG("%s: output embedding shape [%d, %d, %d]\n", __func__, - (int)embeddings->ne[0], (int)embeddings->ne[1], (int)embeddings->ne[2]); - - // copy output to user buffer if provided - // if output is empty, skip the copy - auto & out_batch_embd = *params->out_embd; - if (!out_batch_embd.empty()) { - if (out_batch_embd.size() != (size_t)ggml_nelements(embeddings)) { - LOG_ERR("%s: output buffer has %zu elements but expected %zu\n", __func__, out_batch_embd.size(), (size_t)ggml_nelements(embeddings)); - GGML_ABORT("Output buffer size mismatch"); - } - ggml_backend_tensor_get(embeddings, out_batch_embd.data(), 0, ggml_nbytes(embeddings)); - } else { - LOG_WRN("%s: output buffer is empty, skipping copy\n", __func__); - } - } - - // - // for audio gen models - // - - // optional outputs: a pipeline yields codes or feats, and not all have an eos head - if (params->out_codes != nullptr) { - ggml_tensor * codes = ggml_graph_get_tensor(gf, "out_codes"); - if (codes != nullptr) { - auto & out_codes = *params->out_codes; - out_codes.resize(ggml_nelements(codes)); - ggml_backend_tensor_get(codes, out_codes.data(), 0, ggml_nbytes(codes)); - } - } - if (params->out_feats != nullptr) { - ggml_tensor * feats = ggml_graph_get_tensor(gf, "out_feats"); - if (feats != nullptr) { - auto & out_feats = *params->out_feats; - out_feats.resize(ggml_nelements(feats)); - ggml_backend_tensor_get(feats, out_feats.data(), 0, ggml_nbytes(feats)); - } - } - if (params->out_is_eos != nullptr) { - ggml_tensor * eos = ggml_graph_get_tensor(gf, "out_eos_score"); - if (eos != nullptr) { - GGML_ASSERT(ggml_nelements(eos) == 1); - float score = 0.0f; - ggml_backend_tensor_get(eos, &score, 0, sizeof(float)); - *params->out_is_eos = score > hparams.gen_eos_threshold; - } - } - if (params->out_audio != nullptr) { - ggml_tensor * audio = ggml_graph_get_tensor(gf, "out_audio"); - if (audio == nullptr) { - GGML_ABORT("out_audio requested but graph has no \"out_audio\" tensor"); - } - auto & out_audio = *params->out_audio; - out_audio.resize(ggml_nelements(audio)); - ggml_backend_tensor_get(audio, out_audio.data(), 0, ggml_nbytes(audio)); - - if (model.proj_type == PROJECTOR_TYPE_SOPRANO) { - std::vector spectrum = std::move(out_audio); - clip_graph_soprano::decode_spectrum(model, spectrum, out_audio); - } - - // drop the tail audio that comes from the code-0 rear padding - const int64_t n_codes = params->codes ? model.gen_code_head_w->ne[2] + 1 : 0; - const int64_t n_frames_w = hparams.wav_tfm_swa; - const int64_t n_frames = params->codes ? (int64_t) params->codes->size() / n_codes : n_frames_w; - if (n_frames < n_frames_w) { - const size_t hop = out_audio.size() / n_frames_w; - out_audio.resize((size_t) n_frames * hop); - } - } - if (params->enc_state != nullptr && model.proj_type == PROJECTOR_TYPE_LFM3A) { - const auto & slots = params->enc_state->slots; - size_t i = 0; - for (const auto & slot : list_lfm3a_state_slots(hparams, model)) { - ggml_tensor * t = ggml_graph_get_tensor(gf, ("state_out_" + slot.name).c_str()); - GGML_ASSERT(t != nullptr); - copy_gen_state_out(t, slots[i++]); - } - } else if (params->enc_state != nullptr) { - clip_enc_state * st = params->enc_state; - ggml_tensor * tail = ggml_graph_get_tensor(gf, "out_stem_tail"); - GGML_ASSERT(tail != nullptr); - st->stem_tail.resize(ggml_nelements(tail)); - ggml_backend_tensor_get(tail, st->stem_tail.data(), 0, ggml_nbytes(tail)); - - const int n_new = (imgs.entries[0].nx() - 2) / 2; - for (int i = 0; i < n_new; i++) { - st->slot_pos[(st->n_past + i) % st->n_ctx] = st->n_past + i; - } - st->n_past += n_new; - } - - if (params->gen_process == CLIP_GEN_PROCESS_GEN_WAV) { - const auto & slots = params->gen_state->slots; - size_t i = 0; - for (const auto & slot : list_gen_state_slots(hparams, model)) { - ggml_tensor * t = ggml_graph_get_tensor(gf, ("state_out_" + slot.name).c_str()); - if (t == nullptr) { - GGML_ABORT("graph has no \"state_out_%s\" tensor", slot.name.c_str()); - } - GGML_ASSERT(i < slots.size()); - copy_gen_state_out(t, slots[i++]); - } - } - - // - // Debug: dump final embeddings if MTMD_DEBUG_EMBEDDINGS is set - // - - if (ctx->debug_output_embeddings && embeddings != nullptr) { - const int64_t n_embd = embeddings->ne[0]; - const int64_t n_tokens = embeddings->ne[1]; - std::vector emb_data(ggml_nelements(embeddings)); - ggml_backend_tensor_get(embeddings, emb_data.data(), 0, ggml_nbytes(embeddings)); - - LOG_INF("\n=== MTMD_DEBUG_EMBEDDINGS ===\n"); - LOG_INF("Shape: [%lld, %lld]\n", (long long)n_embd, (long long)n_tokens); - - // TEMP debugging (parity validation), will be removed before merge - // when the env var holds a path, dump the raw data: [int32 n_tokens][int32 n_embd][f32 data] - const char * dump_path = std::getenv("MTMD_DEBUG_EMBEDDINGS"); - if (dump_path && strcmp(dump_path, "1") != 0) { - FILE * f = fopen(dump_path, "wb"); - if (f) { - const int32_t hdr[2] = { (int32_t)n_tokens, (int32_t)n_embd }; - fwrite(hdr, sizeof(hdr), 1, f); - fwrite(emb_data.data(), sizeof(float), emb_data.size(), f); - fclose(f); - } - } - - // Print first few values of first token - LOG_INF("Token 0 (first 16 values): "); - for (int i = 0; i < std::min((int64_t)16, n_embd); i++) { - LOG_INF("%.6f ", emb_data[i]); - } - LOG_INF("\n"); - - // Print last few values of first token - if (n_embd > 16) { - LOG_INF("Token 0 (last 16 values): "); - for (int64_t i = n_embd - 16; i < n_embd; i++) { - LOG_INF("%.6f ", emb_data[i]); - } - LOG_INF("\n"); - } - - // Compute and print statistics - float sum = 0.0f, sum_sq = 0.0f, min_val = emb_data[0], max_val = emb_data[0]; - for (size_t i = 0; i < emb_data.size(); i++) { - sum += emb_data[i]; - sum_sq += emb_data[i] * emb_data[i]; - min_val = std::min(min_val, emb_data[i]); - max_val = std::max(max_val, emb_data[i]); - } - float mean = sum / emb_data.size(); - float variance = (sum_sq / emb_data.size()) - (mean * mean); - LOG_INF("Stats: mean=%.6f, std=%.6f, min=%.6f, max=%.6f, sum=%.6f\n", - mean, sqrtf(variance), min_val, max_val, sum); - LOG_INF("=== END MTMD_DEBUG_EMBEDDINGS ===\n\n"); - } - - return true; -} - -int clip_n_mmproj_embd(const struct clip_ctx * ctx) { - switch (ctx->model.proj_type) { - case PROJECTOR_TYPE_LDP: - return ctx->model.mm_model_block_1_block_2_1_b->ne[0]; - case PROJECTOR_TYPE_LDPV2: - return ctx->model.mm_model_peg_0_b->ne[0]; - case PROJECTOR_TYPE_MLP: - case PROJECTOR_TYPE_PHI4: - case PROJECTOR_TYPE_PIXTRAL: - case PROJECTOR_TYPE_LIGHTONOCR: - case PROJECTOR_TYPE_DOTS_OCR: - case PROJECTOR_TYPE_DOTS3NOTE_V: - case PROJECTOR_TYPE_DOTS3NOTE_A: - return ctx->model.mm_2_w->ne[1]; - case PROJECTOR_TYPE_MLP_NORM: - return ctx->model.mm_3_b->ne[0]; - case PROJECTOR_TYPE_MINICPMV: - return ctx->model.mm_model_proj->ne[0]; - case PROJECTOR_TYPE_MINICPMV4_6: - case PROJECTOR_TYPE_MINICPMV4_7: - case PROJECTOR_TYPE_MINICPM_OCR_1_0: - return ctx->model.mm_ffn_down_w->ne[1]; - case PROJECTOR_TYPE_GLM_EDGE: - return ctx->model.mm_model_mlp_3_w->ne[1]; - case PROJECTOR_TYPE_MINIMAX_M3: - return ctx->model.mm_merger_fc2_b->ne[0]; - case PROJECTOR_TYPE_MUSE_GLIMMER: - return ctx->model.mm_2_w->ne[1]; - case PROJECTOR_TYPE_QWEN2VL: - case PROJECTOR_TYPE_QWEN25VL: - case PROJECTOR_TYPE_LFM3VL: - case PROJECTOR_TYPE_EXAONE4_5: - case PROJECTOR_TYPE_JANUS_PRO: - case PROJECTOR_TYPE_YOUTUVL: - return ctx->model.mm_1_b->ne[0]; - case PROJECTOR_TYPE_SENSENOVA_U1: - return ctx->model.hparams.projection_dim; - case PROJECTOR_TYPE_QWEN3VL: - // main path + deepstack paths - return ctx->model.mm_1_b->ne[0] * (1 + ctx->model.n_deepstack_layers); - case PROJECTOR_TYPE_LING3VL: - return ctx->model.mm_1_b->ne[0]; - case PROJECTOR_TYPE_MIMOVL: - return ctx->model.mm_1_w->ne[1]; - case PROJECTOR_TYPE_STEP3VL: - return ctx->model.mm_model_proj->ne[1]; - case PROJECTOR_TYPE_GEMMA3: - case PROJECTOR_TYPE_GEMMA3NV: - return ctx->model.mm_input_proj_w->ne[0]; - case PROJECTOR_TYPE_GEMMA4V: - case PROJECTOR_TYPE_GEMMA4UV: - case PROJECTOR_TYPE_GEMMA4A: - case PROJECTOR_TYPE_GEMMA4UA: - return ctx->model.mm_input_proj_w->ne[1]; - case PROJECTOR_TYPE_IDEFICS3: - case PROJECTOR_TYPE_MUSE_SPARK_AUDIO: - return ctx->model.mm_fc_w->ne[1]; - case PROJECTOR_TYPE_ULTRAVOX: - case PROJECTOR_TYPE_VOXTRAL: - case PROJECTOR_TYPE_VOXTRAL_RT: - case PROJECTOR_TYPE_MUSIC_FLAMINGO: - return ctx->model.mm_2_w->ne[1]; - case PROJECTOR_TYPE_MERALION: - return ctx->model.mm_3_w->ne[1]; // out_proj output dim - case PROJECTOR_TYPE_INTERNVL: - case PROJECTOR_TYPE_NEMOTRON_V2_VL: - case PROJECTOR_TYPE_NEMOTRON_OMNI: - return ctx->model.mm_3_w->ne[1]; - case PROJECTOR_TYPE_LLAMA4: - return ctx->model.mm_model_proj->ne[1]; - case PROJECTOR_TYPE_QWEN2A: - return ctx->model.mm_fc_w->ne[1]; - case PROJECTOR_TYPE_QWEN3A: - return ctx->model.mm_2_w->ne[1]; - case PROJECTOR_TYPE_GLMA: - case PROJECTOR_TYPE_LFM2: - case PROJECTOR_TYPE_D1OMNI_V: - case PROJECTOR_TYPE_KIMIVL: - case PROJECTOR_TYPE_PADDLEOCR: - case PROJECTOR_TYPE_KIMIK25: - case PROJECTOR_TYPE_YASA2: - case PROJECTOR_TYPE_DEEPSEEK4V: - case PROJECTOR_TYPE_DEEPSEEK41V: - case PROJECTOR_TYPE_COHERE2V: - return ctx->model.mm_2_w->ne[1]; - case PROJECTOR_TYPE_HUNYUANVL: - return ctx->model.mm_model_proj->ne[1]; - case PROJECTOR_TYPE_COGVLM: - return ctx->model.mm_4h_to_h_w->ne[1]; - case PROJECTOR_TYPE_DEEPSEEKOCR: - case PROJECTOR_TYPE_DEEPSEEKOCR2: - return ctx->model.mm_fc_w->ne[1]; - case PROJECTOR_TYPE_LFM2A: - return ctx->model.position_embeddings->ne[0]; - case PROJECTOR_TYPE_D1OMNI_A: - case PROJECTOR_TYPE_LFM3A: - return ctx->model.mm_3_w->ne[1]; - case PROJECTOR_TYPE_GRANITE_SPEECH: - return ctx->model.qf_proj_blocks[0].qf_proj_linear_w->ne[1]; - case PROJECTOR_TYPE_GRANITE4_VISION: - return ctx->model.qf_proj_blocks.size() * ctx->model.hparams.projection_dim; - case PROJECTOR_TYPE_GLM4V: - case PROJECTOR_TYPE_GLM5V: - return ctx->model.mm_ffn_down_w->ne[1]; - case PROJECTOR_TYPE_MIMO_AUDIO: - return ctx->model.mm_2_w->ne[1]; - case PROJECTOR_TYPE_QWEN3TTS_SPKENC: - return ctx->model.mm_fc_w->ne[2]; - case PROJECTOR_TYPE_QWEN3TTS_GEN: - case PROJECTOR_TYPE_LFM3A_GEN: - return ctx->model.gen_code_out_embd_w->ne[0]; - case PROJECTOR_TYPE_POCKETTTS_SPKENC: - return ctx->model.spk_proj_w->ne[1]; - case PROJECTOR_TYPE_POCKETTTS_GEN: - return ctx->model.gen_input_lin_w->ne[1]; - case PROJECTOR_TYPE_SOPRANO: - return ctx->model.hparams.projection_dim; - case PROJECTOR_TYPE_PARAKEET: - return ctx->model.mm_1_w->ne[1]; - default: - GGML_ABORT("Unknown projector type"); - } -} - -bool clip_is_llava(const struct clip_ctx * ctx) { - return ctx->model.hparams.has_llava_projector; -} - -bool clip_has_vision_encoder(const struct clip_ctx * ctx) { - return ctx->model.modality == CLIP_MODALITY_VISION; -} - -bool clip_has_audio_encoder(const struct clip_ctx * ctx) { - return ctx->model.modality == CLIP_MODALITY_AUDIO; -} - -bool clip_support_batch(const struct clip_ctx * ctx) { - return ctx->support_batch; -} - -// TODO @ngxson : this is no longer correct with mtmd_batch API -// this was only meant to be used by qwen-vl-based models, to fuse 2 input images into one (qwen-vl video support) -// this logic should be refactored in near future to distinctly handle "merge frames" and "batching" -int clip_model_n_temporal_merge(const struct clip_ctx * ctx) { - switch (ctx->proj_type()) { - case PROJECTOR_TYPE_QWEN2VL: - case PROJECTOR_TYPE_QWEN25VL: - case PROJECTOR_TYPE_QWEN3VL: - case PROJECTOR_TYPE_LFM3VL: - case PROJECTOR_TYPE_NEMOTRON_OMNI: - case PROJECTOR_TYPE_LING3VL: - return 2; - default: - return 1; - } -} - -// -// API used internally with mtmd -// - -projector_type clip_get_projector_type(const struct clip_ctx * ctx) { - return ctx->proj_type(); -} - -const clip_hparams * clip_get_hparams(const struct clip_ctx * ctx) { - return &ctx->model.hparams; -} - -std::map clip_get_mem_usage(const struct clip_ctx * ctx) { - std::map result = ctx->mem_usage; - for (auto & [dev, size] : ctx->mem_compute) { - result[dev] += size; - } - return result; -} - -// -// API for debugging -// - -void clip_set_debug_output_embeddings(clip_ctx * ctx, bool enable) { - ctx->debug_output_embeddings = enable; -} diff --git a/tools/mtmd/clip.h b/tools/mtmd/clip.h deleted file mode 100644 index b2c9e07597c7..000000000000 --- a/tools/mtmd/clip.h +++ /dev/null @@ -1,162 +0,0 @@ -#pragma once - -#include "ggml.h" -#include "mtmd.h" - -#include -#include - -#include - -// !!! Internal header, to be used by mtmd only !!! - -#define MTMD_INTERNAL_HEADER - -struct clip_ctx; -struct clip_enc_state; -struct clip_gen_state; - -struct clip_image_size { - int width; - int height; - bool operator==(const clip_image_size & other) const { - return width == other.width && height == other.height; - } - bool operator!=(const clip_image_size & other) const { - return !(*this == other); - } - int area() const { - // avoid overflow when computing area - GGML_ASSERT(width >= 0 && width <= 46000); - GGML_ASSERT(height >= 0 && height <= 46000); - return width * height; - } -}; - -struct clip_image_f32; -struct clip_image_f32_batch; - -enum clip_modality { - CLIP_MODALITY_VISION, - CLIP_MODALITY_AUDIO, - CLIP_MODALITY_GEN_AUDIO, -}; - -enum clip_flash_attn_type { - CLIP_FLASH_ATTN_TYPE_AUTO = -1, - CLIP_FLASH_ATTN_TYPE_DISABLED = 0, - CLIP_FLASH_ATTN_TYPE_ENABLED = 1, -}; - -struct clip_context_params { - bool use_gpu; - ggml_backend_dev_t device; - enum clip_flash_attn_type flash_attn_type; - int image_min_tokens; - int image_max_tokens; - bool warmup; - ggml_backend_sched_eval_callback cb_eval; - void * cb_eval_user_data; - bool no_alloc; - mtmd_progress_callback progress_callback; - void * progress_callback_user_data; -}; - -struct clip_init_result { - struct clip_ctx * ctx_v; // vision context - struct clip_ctx * ctx_a; // audio context - struct clip_ctx * ctx_gen_a; // audio generation context -}; - -struct clip_init_result clip_init(const char * fname, struct clip_context_params ctx_params); - -// max number of output tokens per image, -1 if not dynamic size -int clip_get_image_max_tokens(const struct clip_ctx * ctx); - -void clip_free(struct clip_ctx * ctx); - -// TODO: should be enum, not string -const char * clip_patch_merge_type(const struct clip_ctx * ctx); - -int clip_n_output_tokens(const clip_ctx * ctx, const clip_image_f32 * img); - -// for M-RoPE, this will be the number of token positions in X and Y directions -// for other models, X will be the total number of tokens and Y will be 1 -int clip_n_output_tokens_x(const clip_ctx * ctx, const clip_image_f32 * img); -int clip_n_output_tokens_y(const clip_ctx * ctx, const clip_image_f32 * img); - -// this should be equal to the embedding dimension of the text model -int clip_n_mmproj_embd(const struct clip_ctx * ctx); - -// TODO: remove clip_image_encode() and always use batched version -bool clip_image_encode (struct clip_ctx * ctx, int n_threads, const clip_image_f32 * img, std::vector & out_vec); -bool clip_image_batch_encode(struct clip_ctx * ctx, int n_threads, const struct clip_image_f32_batch * imgs, std::vector & out_batch_embd); - -enum clip_gen_process_type { - CLIP_GEN_PROCESS_GEN_UNKNOWN, - CLIP_GEN_PROCESS_GEN_CODE, // h_state to codes - CLIP_GEN_PROCESS_GEN_WAV, // codes to raw PCM audio -}; -struct clip_encode_params { - int n_threads = 1; - const clip_image_f32_batch * imgs = nullptr; - std::vector * out_embd = nullptr; - - // for audio gen, imgs has exactly one entry: hidden state from backbone (GEN_CODE) or unused (GEN_WAV) - clip_gen_process_type gen_process = CLIP_GEN_PROCESS_GEN_UNKNOWN; - - // GEN_CODE: out_embd receives the embd to feed back to the backbone - int32_t code0 = 0; // semantic code sampled by the backbone - int32_t top_k = 50; - float top_p = 1.0f; - std::vector * out_codes = nullptr; // this frame's 16 sampled codes - std::vector * out_feats = nullptr; // continuous counterpart of out_codes - float temp = 0.0f; // sampling temperature, noise scale for flow-matching decoders - bool * out_is_eos = nullptr; - - // GEN_WAV - const std::vector * codes = nullptr; // this frame's 16 RVQ codes - const std::vector * feats = nullptr; // continuous counterpart of codes - std::vector * out_audio = nullptr; // decoded PCM samples, F32 - - // audio gen: state of one sequence, clip_encode() updates it in place - struct clip_gen_state * gen_state = nullptr; - - // streaming audio input, updated in place by clip_encode() - struct clip_enc_state * enc_state = nullptr; -}; -bool clip_encode(struct clip_ctx * ctx, struct clip_encode_params * params); - -// persistent state of a causal audio encoder, for streaming input (one per stream) -// it holds the sliding window of past keys/values, so each clip_encode() call only takes the new frames -// n_frames_max is the maximum number of encoder frames per call -struct clip_enc_state * clip_enc_state_init(struct clip_ctx * ctx, int32_t n_frames_max); -void clip_enc_state_free(struct clip_enc_state * state); - -// per-sequence state of audio generation: RNG and the slots kept between GEN_WAV calls -// seed UINT32_MAX means random; reset() clears the slots and keeps the RNG stream -struct clip_gen_state * clip_gen_state_init(struct clip_ctx * ctx, uint32_t seed); -void clip_gen_state_free(struct clip_gen_state * state); -void clip_gen_state_reset(struct clip_gen_state * state); - -const std::map> & clip_gen_get_voices(const struct clip_ctx * ctx); - -bool clip_is_llava(const struct clip_ctx * ctx); -// note for contributor: this clip_is_(model) pattern is deprecated -// do NOT add new functions like this - -bool clip_has_vision_encoder(const struct clip_ctx * ctx); -bool clip_has_audio_encoder(const struct clip_ctx * ctx); - -bool clip_support_batch(const struct clip_ctx * ctx); - -int clip_model_n_temporal_merge(const struct clip_ctx * ctx); // TODO @ngxson : remove, refactor this - -std::map clip_get_mem_usage(const struct clip_ctx * ctx); - -struct clip_cap { - bool has_vision; - bool has_audio; - bool has_gen_audio; -}; -struct clip_cap clip_get_cap(const char * fname); diff --git a/tools/mtmd/debug/mtmd-debug.cpp b/tools/mtmd/debug/mtmd-debug.cpp index 2719dae9b255..894ad7d6f434 100644 --- a/tools/mtmd/debug/mtmd-debug.cpp +++ b/tools/mtmd/debug/mtmd-debug.cpp @@ -81,7 +81,7 @@ int main(int argc, char ** argv) { llama_init = common_init_from_params(params); { auto * model = llama_init->model(); - const char * clip_path = params.mmproj.path.c_str(); + const char * mmproj_path = params.mmproj.path.c_str(); mtmd_context_params mparams = mtmd_context_params_default(); mparams.use_gpu = params.mmproj_use_gpu; mparams.device = params.mmproj_device; @@ -96,9 +96,9 @@ int main(int argc, char ** argv) { mparams.cb_eval_user_data = &cb_data; mparams.cb_eval = common_debug_cb_eval; } - ctx_mtmd.reset(mtmd_init_from_file(clip_path, model, mparams)); + ctx_mtmd.reset(mtmd_init_from_file(mmproj_path, model, mparams)); if (!ctx_mtmd.get()) { - LOG_ERR("Failed to load vision model from %s\n", clip_path); + LOG_ERR("Failed to load vision model from %s\n", mmproj_path); exit(1); } } diff --git a/tools/mtmd/models/chatterbox-gen.cpp b/tools/mtmd/models/chatterbox-gen.cpp new file mode 100644 index 000000000000..017329bf4f13 --- /dev/null +++ b/tools/mtmd/models/chatterbox-gen.cpp @@ -0,0 +1,541 @@ +#include "models.h" + +// Chatterbox generation graphs: flow encoder, cfm estimator, s3 tokenizer and hift vocoder + +// x [C, T]: y = W x + b with torch Linear weights stored as [in, out] +ggml_tensor * mtmd_graph_chatterbox_base::cbx_linear(ggml_tensor * w, ggml_tensor * b, ggml_tensor * x) const { + ggml_tensor * y = ggml_mul_mat(ctx0, w, x); + if (b) { + y = ggml_add(ctx0, y, b); + } + return y; +} + +// conv1d over time with asymmetric zero padding: x [C, T] -> [OC, T_out], kernel [K, IC, OC] +ggml_tensor * mtmd_graph_chatterbox_base::cbx_conv1d(ggml_tensor * k, ggml_tensor * b, ggml_tensor * x, + int stride, int pad_l, int pad_r) const { + ggml_tensor * xt = ggml_cont(ctx0, ggml_transpose(ctx0, x)); // [T, C] + if (pad_l > 0) { + ggml_tensor * z = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, pad_l, xt->ne[1]); + z = ggml_scale(ctx0, z, 0.0f); + xt = ggml_concat(ctx0, z, xt, 0); + } + if (pad_r > 0) { + ggml_tensor * z = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, pad_r, xt->ne[1]); + z = ggml_scale(ctx0, z, 0.0f); + xt = ggml_concat(ctx0, xt, z, 0); + } + ggml_tensor * y = ggml_conv_1d(ctx0, k, xt, stride, 0, 1); // [T_out, OC] + y = ggml_cont(ctx0, ggml_transpose(ctx0, y)); // [OC, T_out] + if (b) { + y = ggml_add(ctx0, y, b); + } + return y; +} + +// x [C, T] -> symmetric-padded dilated conv -> [OC, T] +ggml_tensor * mtmd_graph_chatterbox_base::cbx_conv1d_dil(ggml_tensor * k, ggml_tensor * b, ggml_tensor * x, + int pad, int dil) const { + ggml_tensor * xt = ggml_cont(ctx0, ggml_transpose(ctx0, x)); + ggml_tensor * y = ggml_conv_1d(ctx0, k, xt, 1, pad, dil); + y = ggml_cont(ctx0, ggml_transpose(ctx0, y)); + if (b) { + y = ggml_add(ctx0, y, b); + } + return y; +} + +// Transformer-XL relative shift: bd [2T-1, T, H] -> [T, T, H] with out[j, i, h] = bd[(T-1) - i + j, i, h] +// same buffer walk as the espnet rel_shift: pad one column, drop the first row, keep the first T columns +static ggml_tensor * cbx_rel_shift(ggml_context * ctx0, ggml_tensor * bd, int T) { + const int H = (int) bd->ne[2]; + ggml_tensor * z = ggml_new_tensor_3d(ctx0, GGML_TYPE_F32, 1, T, H); + z = ggml_scale(ctx0, z, 0.0f); + ggml_tensor * p = ggml_cont(ctx0, ggml_concat(ctx0, z, bd, 0)); // [2T, T, H] + p = ggml_reshape_3d(ctx0, p, T, 2 * T, H); // [T, 2T, H] + p = ggml_view_3d(ctx0, p, T, 2 * T - 1, H, p->nb[1], p->nb[2], p->nb[1]); // drop first row + p = ggml_cont(ctx0, p); + p = ggml_reshape_3d(ctx0, p, 2 * T - 1, T, H); // [2T-1, T, H] + p = ggml_view_3d(ctx0, p, T, T, H, p->nb[1], p->nb[2], 0); // first T columns + return ggml_cont(ctx0, p); +} + +// espnet rel-pos self attention block, pre-norm, x [512, T], pos [512, 2T-1] +ggml_tensor * mtmd_graph_chatterbox::enc_layer(const mtmd_chatterbox::enc_layer & l, ggml_tensor * x, + ggml_tensor * pos, int T) { + const int n_head = 8; + const int d_head = 64; + const float scale = 1.0f / sqrtf((float) d_head); + + ggml_tensor * res = x; + ggml_tensor * cur = build_norm(x, l.norm_mha_w, l.norm_mha_b, NORM_TYPE_NORMAL, 1e-5f, -1); + + ggml_tensor * q = cbx_linear(l.attn_q_w, l.attn_q_b, cur); + ggml_tensor * k = cbx_linear(l.attn_k_w, l.attn_k_b, cur); + ggml_tensor * v = cbx_linear(l.attn_v_w, l.attn_v_b, cur); + ggml_tensor * pe = cbx_linear(l.attn_pos_w, nullptr, pos); // [512, 2T-1] + + q = ggml_reshape_3d(ctx0, q, d_head, n_head, T); + k = ggml_reshape_3d(ctx0, k, d_head, n_head, T); + v = ggml_reshape_3d(ctx0, v, d_head, n_head, T); + pe = ggml_reshape_3d(ctx0, pe, d_head, n_head, 2 * T - 1); + + ggml_tensor * u = l.attn_pos_bias_u; // [64, 8] + ggml_tensor * w = l.attn_pos_bias_v; + + ggml_tensor * qu = ggml_add(ctx0, q, ggml_reshape_3d(ctx0, u, d_head, n_head, 1)); + ggml_tensor * qv = ggml_add(ctx0, q, ggml_reshape_3d(ctx0, w, d_head, n_head, 1)); + + // per head: [64, T] tensors, scores [T(k), T(q)] + qu = ggml_cont(ctx0, ggml_permute(ctx0, qu, 0, 2, 1, 3)); // [64, T, 8] + qv = ggml_cont(ctx0, ggml_permute(ctx0, qv, 0, 2, 1, 3)); + k = ggml_cont(ctx0, ggml_permute(ctx0, k, 0, 2, 1, 3)); + v = ggml_cont(ctx0, ggml_permute(ctx0, v, 0, 2, 1, 3)); + pe = ggml_cont(ctx0, ggml_permute(ctx0, pe, 0, 2, 1, 3)); // [64, 2T-1, 8] + + ggml_tensor * ac = ggml_mul_mat(ctx0, k, qu); // [T(k), T(q), 8] + ggml_tensor * bd = ggml_mul_mat(ctx0, pe, qv); // [2T-1, T(q), 8] + bd = cbx_rel_shift(ctx0, bd, T); // [T(k), T(q), 8] + + ggml_tensor * scores = ggml_scale(ctx0, ggml_add(ctx0, ac, bd), scale); + ggml_tensor * probs = ggml_soft_max(ctx0, scores); + + ggml_tensor * o = ggml_mul_mat(ctx0, ggml_cont(ctx0, ggml_transpose(ctx0, v)), probs); // [64, T, 8] + o = ggml_cont(ctx0, ggml_permute(ctx0, o, 0, 2, 1, 3)); // [64, 8, T] + o = ggml_reshape_2d(ctx0, o, n_head * d_head, T); + o = cbx_linear(l.attn_out_w, l.attn_out_b, o); + x = ggml_add(ctx0, res, o); + + res = x; + cur = build_norm(x, l.norm_ff_w, l.norm_ff_b, NORM_TYPE_NORMAL, 1e-5f, -1); + cur = cbx_linear(l.ffn_1_w, l.ffn_1_b, cur); + cur = ggml_silu(ctx0, cur); // swish + cur = cbx_linear(l.ffn_2_w, l.ffn_2_b, cur); + x = ggml_add(ctx0, res, cur); + return x; +} + +// mish = x * tanh(softplus(x)) +static ggml_tensor * cbx_mish(ggml_context * ctx0, ggml_tensor * x) { + return ggml_mul(ctx0, x, ggml_tanh(ctx0, ggml_softplus(ctx0, x))); +} + +// causal block: conv k3 left-padded, layer norm over channels, mish; x [C, T] +ggml_tensor * mtmd_graph_chatterbox::causal_block(const mtmd_chatterbox::causal_block & b, ggml_tensor * x) { + x = cbx_conv1d(b.conv_w, b.conv_b, x, 1, (int) b.conv_w->ne[0] - 1, 0); + x = build_norm(x, b.norm_w, b.norm_b, NORM_TYPE_NORMAL, 1e-5f, -1); + return cbx_mish(ctx0, x); +} + +// resnet block with time conditioning; x [C, T], temb [1024] +ggml_tensor * mtmd_graph_chatterbox::resnet(const mtmd_chatterbox::resnet & r, ggml_tensor * x, ggml_tensor * temb) { + ggml_tensor * h = causal_block(r.block1, x); + ggml_tensor * tproj = cbx_linear(r.mlp_w, r.mlp_b, cbx_mish(ctx0, temb)); + h = ggml_add(ctx0, h, tproj); // broadcast [256, 1] over T + h = causal_block(r.block2, h); + ggml_tensor * res = cbx_conv1d(r.res_conv_w, r.res_conv_b, x, 1, 0, 0); + return ggml_add(ctx0, h, res); +} + +// diffusers-style transformer block, full attention; x [256, T] +ggml_tensor * mtmd_graph_chatterbox::tfm_block(const mtmd_chatterbox::tfm_block & b, ggml_tensor * x) { + const int n_head = 8; + const int d_head = 64; + const int T = (int) x->ne[1]; + + ggml_tensor * cur = build_norm(x, b.norm1_w, b.norm1_b, NORM_TYPE_NORMAL, 1e-5f, -1); + ggml_tensor * q = ggml_reshape_3d(ctx0, ggml_mul_mat(ctx0, b.attn_q_w, cur), d_head, n_head, T); + ggml_tensor * k = ggml_reshape_3d(ctx0, ggml_mul_mat(ctx0, b.attn_k_w, cur), d_head, n_head, T); + ggml_tensor * v = ggml_reshape_3d(ctx0, ggml_mul_mat(ctx0, b.attn_v_w, cur), d_head, n_head, T); + x = ggml_add(ctx0, x, build_attn(b.attn_out_w, b.attn_out_b, q, k, v, nullptr, 1.0f / sqrtf((float) d_head), -1)); + + cur = build_norm(x, b.norm3_w, b.norm3_b, NORM_TYPE_NORMAL, 1e-5f, -1); + cur = cbx_linear(b.ff_in_w, b.ff_in_b, cur); + cur = ggml_gelu_erf(ctx0, cur); + cur = cbx_linear(b.ff_out_w, b.ff_out_b, cur); + return ggml_add(ctx0, x, cur); +} + +// one estimator evaluation; x_noise, mu, cond [80, T], spks [80], temb [1024] +ggml_tensor * mtmd_graph_chatterbox::estimator(ggml_tensor * x_noise, ggml_tensor * mu, ggml_tensor * spks, + ggml_tensor * cond, ggml_tensor * temb, int T) { + const auto & c = model.cbx; + + // channels live on ne0, time on ne1: pack along ne0 + ggml_tensor * x = ggml_concat(ctx0, x_noise, mu, 0); // [160, T] + ggml_tensor * spks_b = ggml_repeat(ctx0, ggml_reshape_2d(ctx0, spks, 80, 1), ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, 80, T)); + x = ggml_concat(ctx0, x, spks_b, 0); // [240, T] + x = ggml_concat(ctx0, x, cond, 0); // [320, T] + x = ggml_cont(ctx0, x); + + // down + ggml_tensor * skip; + x = resnet(c.est_down.res, x, temb); + for (const auto & b : c.est_down.tfm) { + x = tfm_block(b, x); + } + skip = x; + x = cbx_conv1d(c.est_down.conv_w, c.est_down.conv_b, x, 1, (int) c.est_down.conv_w->ne[0] - 1, 0); + + // mid + for (const auto & m : c.est_mid) { + x = resnet(m.res, x, temb); + for (const auto & b : m.tfm) { + x = tfm_block(b, x); + } + } + + // up with skip + x = ggml_concat(ctx0, x, skip, 0); // [512, T] + x = resnet(c.est_up.res, x, temb); + for (const auto & b : c.est_up.tfm) { + x = tfm_block(b, x); + } + x = cbx_conv1d(c.est_up.conv_w, c.est_up.conv_b, x, 1, (int) c.est_up.conv_w->ne[0] - 1, 0); + + x = causal_block(c.est_final_block, x); + x = cbx_conv1d(c.est_final_proj_w, c.est_final_proj_b, x, 1, 0, 0); // [80, T] + return x; +} + +// s3 tokenizer encoder: log-mel [T, 128] -> fsq code [8, T / 4], rounded to base 3 tokens on the host +ggml_cgraph * mtmd_graph_chatterbox::build_s3tok(int T) { + const auto & c = model.cbx; + const int n_head = 20; + const int d_head = 64; + const int T1 = (T - 1) / 2 + 1; + const int T2 = (T1 - 1) / 2 + 1; + + ggml_tensor * inp = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, T, 128); + ggml_set_name(inp, "inp_raw"); + ggml_set_input(inp); + + ggml_tensor * pos = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, T2); + ggml_set_name(pos, "inp_pos"); + ggml_set_input(pos); + + ggml_tensor * x = ggml_cont(ctx0, ggml_transpose(ctx0, inp)); // [128, T] + x = cbx_conv1d(c.s3tok_conv1_w, c.s3tok_conv1_b, x, 2, 1, 1); + x = ggml_gelu_erf(ctx0, x); + x = cbx_conv1d(c.s3tok_conv2_w, c.s3tok_conv2_b, x, 2, 1, 1); + x = ggml_gelu_erf(ctx0, x); // [1280, T2] + + for (const auto & blk : c.s3tok_blocks) { + ggml_tensor * res = x; + ggml_tensor * cur = build_norm(x, blk.attn_ln_w, blk.attn_ln_b, NORM_TYPE_NORMAL, 1e-5f, -1); + ggml_tensor * q = cbx_linear(blk.attn_q_w, blk.attn_q_b, cur); + ggml_tensor * k = ggml_mul_mat(ctx0, blk.attn_k_w, cur); + ggml_tensor * v = cbx_linear(blk.attn_v_w, blk.attn_v_b, cur); + + // fsmn memory: depthwise conv over time on v plus residual, added to the attention output + ggml_tensor * fsm = ggml_cont(ctx0, ggml_transpose(ctx0, v)); // [T2, 1280] + { + ggml_tensor * w = blk.fsmn_w; + ggml_tensor * m = ggml_conv_1d_dw(ctx0, w, fsm, 1, ((int) w->ne[0] - 1) / 2, 1); + fsm = ggml_add(ctx0, ggml_reshape_2d(ctx0, m, fsm->ne[0], fsm->ne[1]), fsm); + } + fsm = ggml_cont(ctx0, ggml_transpose(ctx0, fsm)); // [1280, T2] + + q = ggml_reshape_3d(ctx0, q, d_head, n_head, T2); + k = ggml_reshape_3d(ctx0, k, d_head, n_head, T2); + v = ggml_reshape_3d(ctx0, v, d_head, n_head, T2); + q = ggml_rope_ext(ctx0, q, pos, nullptr, d_head, GGML_ROPE_TYPE_NEOX, 0, 10000.0f, 1.0f, 0.0f, 1.0f, 0.0f, 0.0f); + k = ggml_rope_ext(ctx0, k, pos, nullptr, d_head, GGML_ROPE_TYPE_NEOX, 0, 10000.0f, 1.0f, 0.0f, 1.0f, 0.0f, 0.0f); + ggml_tensor * o = build_attn(blk.attn_out_w, blk.attn_out_b, q, k, v, nullptr, 1.0f / sqrtf((float) d_head), -1); + x = ggml_add(ctx0, res, ggml_add(ctx0, o, fsm)); + + res = x; + cur = build_norm(x, blk.mlp_ln_w, blk.mlp_ln_b, NORM_TYPE_NORMAL, 1e-5f, -1); + cur = cbx_linear(blk.mlp_in_w, blk.mlp_in_b, cur); + cur = ggml_gelu_erf(ctx0, cur); + cur = cbx_linear(blk.mlp_out_w, blk.mlp_out_b, cur); + x = ggml_add(ctx0, res, cur); + } + + x = cbx_linear(c.s3tok_down_w, c.s3tok_down_b, x); // [8, T2] + x = ggml_tanh(ctx0, x); + + ggml_set_name(x, "out_fsq"); + ggml_set_output(x); + ggml_build_forward_expand(gf, x); + return gf; +} + +ggml_cgraph * mtmd_graph_chatterbox::build() { + if (gen_process == MTMD_SUBCTX_GEN_PROCESS_TTS_VOCODE) { + return build_vocoder(vocode_n_mel, vocode_n_stft); + } + if (gen_process == MTMD_SUBCTX_GEN_PROCESS_TOKENIZE) { + return build_s3tok(img.nx()); + } + if (gen_process != MTMD_SUBCTX_GEN_PROCESS_TTS) { + // load-time buffer sizing path + ggml_tensor * inp = ggml_new_tensor_1d(ctx0, GGML_TYPE_F32, 1); + ggml_set_name(inp, "inp_stub"); + ggml_set_input(inp); + ggml_tensor * cur = ggml_dup(ctx0, inp); + ggml_set_name(cur, "out_stub"); + ggml_set_output(cur); + ggml_build_forward_expand(gf, cur); + return gf; + } + + const auto & c = model.cbx; + + const int T1 = n_tokens; // token-rate length (prompt + generated) + const int T2 = 2 * n_tokens; // mel-rate length after the x2 upsample + + ggml_tensor * inp_tokens = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, T1); + ggml_set_name(inp_tokens, "inp_tokens"); + ggml_set_input(inp_tokens); + + ggml_tensor * pos1 = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, 512, 2 * T1 - 1); + ggml_set_name(pos1, "inp_pos1"); + ggml_set_input(pos1); + + ggml_tensor * pos2 = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, 512, 2 * T2 - 1); + ggml_set_name(pos2, "inp_pos2"); + ggml_set_input(pos2); + + // token embedding + ggml_tensor * x = ggml_get_rows(ctx0, c.input_embedding_w, inp_tokens); // [512, T1] + + // embed: linear + layer norm, then the espnet xscale + x = cbx_linear(c.embed_linear_w, c.embed_linear_b, x); + x = build_norm(x, c.embed_norm_w, c.embed_norm_b, NORM_TYPE_NORMAL, 1e-5f, -1); + x = ggml_scale(ctx0, x, sqrtf(512.0f)); + cb(x, "fenc_embd", -1); + + // pre-lookahead: conv k=4 right-padded 3, leaky 0.01, conv k=3 left-padded 2, residual + { + ggml_tensor * res = x; + ggml_tensor * cur = cbx_conv1d(c.pre_conv1_w, c.pre_conv1_b, x, 1, 0, 3); + cur = ggml_leaky_relu(ctx0, cur, 0.01f, false); + cur = cbx_conv1d(c.pre_conv2_w, c.pre_conv2_b, cur, 1, 2, 0); + x = ggml_add(ctx0, res, cur); + cb(x, "fenc_pre_lookahead", -1); + } + + for (size_t i = 0; i < c.enc.size(); i++) { + x = enc_layer(c.enc[i], x, pos1, T1); + cb(x, "fenc_enc", (int) i); + } + + // upsample x2: nearest repeat, left pad 4, conv k=5 + { + ggml_tensor * xt = ggml_cont(ctx0, ggml_transpose(ctx0, x)); // [T1, 512] + xt = ggml_interpolate(ctx0, xt, 2 * T1, 512, 1, 1, GGML_SCALE_MODE_NEAREST); + ggml_tensor * z = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, 4, 512); + z = ggml_scale(ctx0, z, 0.0f); + xt = ggml_concat(ctx0, z, xt, 0); + ggml_tensor * y = ggml_conv_1d(ctx0, c.up_conv_w, xt, 1, 0, 1); + x = ggml_cont(ctx0, ggml_transpose(ctx0, y)); // [512, T2] + x = ggml_add(ctx0, x, c.up_conv_b); + cb(x, "fenc_upsample", -1); + } + + // up embed: linear + layer norm + xscale + x = cbx_linear(c.up_embed_linear_w, c.up_embed_linear_b, x); + x = build_norm(x, c.up_embed_norm_w, c.up_embed_norm_b, NORM_TYPE_NORMAL, 1e-5f, -1); + x = ggml_scale(ctx0, x, sqrtf(512.0f)); + + for (size_t i = 0; i < c.up_enc.size(); i++) { + x = enc_layer(c.up_enc[i], x, pos2, T2); + cb(x, "fenc_up_enc", (int) i); + } + + x = build_norm(x, c.after_norm_w, c.after_norm_b, NORM_TYPE_NORMAL, 1e-5f, -1); + + // encoder projection to the mel channel count + ggml_tensor * mu = cbx_linear(c.encoder_proj_w, c.encoder_proj_b, x); // [80, T2] + cb(mu, "flow_mu", -1); + + // cfm solver unrolled in the graph: meanflow takes 2 euler steps without cfg, its time embeds mix t and r + // the classic solver takes 10 euler steps on the cosine schedule with cfg 0.7 + const bool meanflow = c.time_embed_mixer_w != nullptr; + const int n_steps = meanflow ? 2 : 10; + + // span points, same schedule as the host side sinusoid fill in mtmd-subcontext.cpp + float span[11]; + for (int i = 0; i <= n_steps; i++) { + const float u = (float) i / n_steps; + span[i] = meanflow ? u : 1.0f - cosf(u * (float) M_PI / 2.0f); + } + + ggml_tensor * noise = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, 80, T2); + ggml_set_name(noise, "inp_noise"); + ggml_set_input(noise); + // sinusoidal time embeddings, one row per span point + ggml_tensor * temb_sin = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, 320, n_steps + 1); + ggml_set_name(temb_sin, "inp_temb"); + ggml_set_input(temb_sin); + + auto time_mlp = [&](ggml_tensor * e) { + e = cbx_linear(c.time_mlp_1_w, c.time_mlp_1_b, e); + e = ggml_silu(ctx0, e); + e = cbx_linear(c.time_mlp_2_w, c.time_mlp_2_b, e); + return e; + }; + auto span_emb = [&](int i) { + return ggml_view_2d(ctx0, temb_sin, 320, 1, temb_sin->nb[1], (size_t) i * temb_sin->nb[1]); + }; + auto step_temb = [&](int i) { + if (!meanflow) { + return time_mlp(span_emb(i)); + } + ggml_tensor * e = ggml_concat(ctx0, time_mlp(span_emb(i)), time_mlp(span_emb(i + 1)), 0); // [2048, 1] + return ggml_mul_mat(ctx0, c.time_embed_mixer_w, e); // [1024, 1] + }; + + // mel-rate conditions of the voice: prompt features then zeros, and the speaker vector + ggml_tensor * pf = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, 80, n_prompt_mel); + ggml_set_name(pf, "inp_prompt_feat"); + ggml_set_input(pf); + ggml_tensor * zc = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, 80, T2 - n_prompt_mel); + zc = ggml_scale(ctx0, zc, 0.0f); + ggml_tensor * cond = ggml_concat(ctx0, pf, zc, 1); // [80, T2] + // raw 192-dim campplus x-vector, normalized then through the flow speaker affine + ggml_tensor * xvec = ggml_new_tensor_1d(ctx0, GGML_TYPE_F32, 192); + ggml_set_name(xvec, "inp_xvec"); + ggml_set_input(xvec); + ggml_tensor * n2 = ggml_sqrt(ctx0, ggml_sum(ctx0, ggml_mul(ctx0, xvec, xvec))); + ggml_tensor * unit = ggml_div(ctx0, xvec, n2); + ggml_tensor * spks = cbx_linear(c.spk_affine_w, c.spk_affine_b, + ggml_reshape_2d(ctx0, unit, 192, 1)); + spks = ggml_cont(ctx0, ggml_reshape_1d(ctx0, spks, 80)); + + const float cfg = 0.7f; + ggml_tensor * mu_zero = meanflow ? nullptr : ggml_scale(ctx0, mu, 0.0f); + ggml_tensor * cond_zero = meanflow ? nullptr : ggml_scale(ctx0, cond, 0.0f); + ggml_tensor * spks_zero = meanflow ? nullptr : ggml_scale(ctx0, spks, 0.0f); + + ggml_tensor * mx = noise; + for (int i = 0; i < n_steps; i++) { + ggml_tensor * temb = step_temb(i); + ggml_tensor * d = estimator(mx, mu, spks, cond, temb, T2); + if (!meanflow) { + ggml_tensor * du = estimator(mx, mu_zero, spks_zero, cond_zero, temb, T2); + d = ggml_add(ctx0, ggml_scale(ctx0, d, 1.0f + cfg), ggml_scale(ctx0, du, -cfg)); + } + mx = ggml_add(ctx0, mx, ggml_scale(ctx0, d, span[i + 1] - span[i])); + cb(mx, "cfm_step", i); + } + ggml_tensor * mel = mx; + + // trim the prompt frames at mel rate + mel = ggml_view_2d(ctx0, mel, 80, T2 - n_prompt_mel, mel->nb[1], (size_t) n_prompt_mel * mel->nb[1]); + mel = ggml_cont(ctx0, mel); + ggml_set_name(mel, "out_mel"); + ggml_set_output(mel); + ggml_build_forward_expand(gf, mel); + + // f0 predictor on the trimmed mel: 5x (conv k3 same-pad + elu), abs(linear) + { + ggml_tensor * fx = mel; + for (const auto & fc : c.f0_condnet) { + fx = cbx_conv1d(fc.w, fc.b, fx, 1, 1, 1); + fx = ggml_elu(ctx0, fx); + } + fx = cbx_linear(c.f0_classifier_w, c.f0_classifier_b, fx); + fx = ggml_abs(ctx0, fx); // [1, T] + ggml_set_name(fx, "out_f0"); + ggml_set_output(fx); + ggml_build_forward_expand(gf, fx); + } + return gf; +} + +// snake activation with per-channel alpha: x + sin^2(alpha x) / alpha +static ggml_tensor * cbx_snake(ggml_context * ctx0, ggml_tensor * x, ggml_tensor * alpha) { + ggml_tensor * sx = ggml_sin(ctx0, ggml_mul(ctx0, x, alpha)); + sx = ggml_mul(ctx0, sx, sx); + sx = ggml_div(ctx0, sx, alpha); + return ggml_add(ctx0, x, sx); +} + +// hifigan-snake resblock; kernels with dilations 1/3/5 on convs1, 1 on convs2 +ggml_tensor * mtmd_graph_chatterbox::hift_resblock(const mtmd_chatterbox::hift_res & r, ggml_tensor * x) { + static const int dil[3] = {1, 3, 5}; + for (size_t j = 0; j < r.units.size(); j++) { + const auto & u = r.units[j]; + const int d = dil[j % 3]; + const int p1 = (int) (u.conv1_w->ne[0] - 1) / 2 * d; + const int p2 = (int) (u.conv2_w->ne[0] - 1) / 2; + ggml_tensor * xt = cbx_snake(ctx0, x, u.act1_alpha); + xt = cbx_conv1d_dil(u.conv1_w, u.conv1_b, xt, p1, d); + xt = cbx_snake(ctx0, xt, u.act2_alpha); + xt = cbx_conv1d_dil(u.conv2_w, u.conv2_b, xt, p2, 1); + x = ggml_add(ctx0, x, xt); + } + return x; +} + +// mel [80, T] + source stft [18, T_stft] -> conv_post output [18, T_stft2] +ggml_cgraph * mtmd_graph_chatterbox::build_vocoder(int n_mel, int n_stft) { + const auto & c = model.cbx; + + ggml_tensor * mel = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, 80, n_mel); + ggml_set_name(mel, "inp_mel"); + ggml_set_input(mel); + ggml_tensor * sstft = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, 18, n_stft); + ggml_set_name(sstft, "inp_sstft"); + ggml_set_input(sstft); + + ggml_tensor * x = cbx_conv1d_dil(c.hift_pre_w, c.hift_pre_b, mel, 3, 1); + + for (size_t i = 0; i < c.hift_ups.size(); i++) { + const auto & up = c.hift_ups[i]; + ggml_tensor * uk = up.up_w; + const int K = (int) uk->ne[0]; + const int S = K / 2; + const int P = (K - S) / 2; + + x = ggml_leaky_relu(ctx0, x, 0.1f, false); + // conv transpose then trim the torch padding P on both sides + ggml_tensor * xt = ggml_cont(ctx0, ggml_transpose(ctx0, x)); + xt = ggml_conv_transpose_1d(ctx0, uk, xt, S, 0, 1); + xt = ggml_cont(ctx0, ggml_view_2d(ctx0, xt, xt->ne[0] - 2 * P, xt->ne[1], xt->nb[1], (size_t) P * ggml_element_size(xt))); + x = ggml_cont(ctx0, ggml_transpose(ctx0, xt)); + x = ggml_add(ctx0, x, up.up_b); + + const bool is_last = i + 1 == c.hift_ups.size(); + if (is_last) { + ggml_tensor * xr = ggml_cont(ctx0, ggml_transpose(ctx0, x)); + xr = ggml_pad_reflect_1d(ctx0, xr, 1, 0); + x = ggml_cont(ctx0, ggml_transpose(ctx0, xr)); + } + + // source injection: strided conv on the source stft, one resblock + ggml_tensor * sk = up.source_down_w; + const int SK = (int) sk->ne[0]; + const int SS = SK > 1 ? SK / 2 : 1; + const int SP = SK > 1 ? SS / 2 : 0; + ggml_tensor * si; + { + ggml_tensor * st = ggml_cont(ctx0, ggml_transpose(ctx0, sstft)); + st = ggml_conv_1d(ctx0, sk, st, SS, SP, 1); + si = ggml_cont(ctx0, ggml_transpose(ctx0, st)); + si = ggml_add(ctx0, si, up.source_down_b); + } + si = hift_resblock(up.source_res, si); + // align lengths: the reflection pad on the last stage adds one step + if ((int) si->ne[1] != (int) x->ne[1]) { + const int n = (int) std::min(si->ne[1], x->ne[1]); + si = ggml_cont(ctx0, ggml_view_2d(ctx0, si, si->ne[0], n, si->nb[1], 0)); + x = ggml_cont(ctx0, ggml_view_2d(ctx0, x, x->ne[0], n, x->nb[1], 0)); + } + x = ggml_add(ctx0, x, si); + + ggml_tensor * acc = hift_resblock(up.res_0, x); + acc = ggml_add(ctx0, acc, hift_resblock(up.res_1, x)); + acc = ggml_add(ctx0, acc, hift_resblock(up.res_2, x)); + x = ggml_scale(ctx0, acc, 1.0f / 3.0f); + } + + x = ggml_leaky_relu(ctx0, x, 0.01f, false); + x = cbx_conv1d_dil(c.hift_post_w, c.hift_post_b, x, 3, 1); + ggml_set_name(x, "out_spec"); + ggml_set_output(x); + ggml_build_forward_expand(gf, x); + return gf; +} diff --git a/tools/mtmd/models/chatterbox-spkenc.cpp b/tools/mtmd/models/chatterbox-spkenc.cpp new file mode 100644 index 000000000000..fc73aa8056d4 --- /dev/null +++ b/tools/mtmd/models/chatterbox-spkenc.cpp @@ -0,0 +1,156 @@ +#include "models.h" + +// Chatterbox speaker encoder: CAMPPlus x-vector on kaldi fbank features, see s3gen/xvector.py + +static constexpr float bn_eps = 1e-5f; + +// per-channel batchnorm on x [C, T], w/b are null on the affine=False variant +ggml_tensor * mtmd_graph_chatterbox_spkenc::bn1d(const mtmd_chatterbox::bn & n, ggml_tensor * x) { + ggml_tensor * sd = ggml_sqrt(ctx0, ggml_scale_bias(ctx0, n.var, 1.0f, bn_eps)); + if (!n.w) { + return ggml_div(ctx0, ggml_sub(ctx0, x, n.mean), sd); + } + ggml_tensor * a = ggml_div(ctx0, n.w, sd); + ggml_tensor * shift = ggml_sub(ctx0, n.b, ggml_mul(ctx0, n.mean, a)); + return ggml_add(ctx0, ggml_mul(ctx0, x, a), shift); +} + +// batchnorm on a conv2d activation [W=T, H=F, C, 1], stats on ne2 +static ggml_tensor * cbx_bn2d(ggml_context * ctx0, const mtmd_chatterbox::bn & n, ggml_tensor * x) { + const int C = (int) x->ne[2]; + ggml_tensor * mean = ggml_reshape_4d(ctx0, n.mean, 1, 1, C, 1); + ggml_tensor * var = ggml_reshape_4d(ctx0, n.var, 1, 1, C, 1); + ggml_tensor * w = ggml_reshape_4d(ctx0, n.w, 1, 1, C, 1); + ggml_tensor * b = ggml_reshape_4d(ctx0, n.b, 1, 1, C, 1); + ggml_tensor * a = ggml_div(ctx0, w, ggml_sqrt(ctx0, ggml_scale_bias(ctx0, var, 1.0f, bn_eps))); + ggml_tensor * shift = ggml_sub(ctx0, b, ggml_mul(ctx0, mean, a)); + return ggml_add(ctx0, ggml_mul(ctx0, x, a), shift); +} + +ggml_tensor * mtmd_graph_chatterbox_spkenc::bn2d_relu(const mtmd_chatterbox::bn & n, ggml_tensor * x) { + return ggml_relu(ctx0, cbx_bn2d(ctx0, n, x)); +} + +// fcm residual 2d block, stride on the frequency axis only +ggml_tensor * mtmd_graph_chatterbox_spkenc::res2d(const mtmd_chatterbox::spk_res2d & r, ggml_tensor * x, + int stride) { + ggml_tensor * cur = ggml_conv_2d(ctx0, r.conv1_w, x, 1, stride, 1, 1, 1, 1); + cur = bn2d_relu(r.bn1, cur); + cur = ggml_conv_2d(ctx0, r.conv2_w, cur, 1, 1, 1, 1, 1, 1); + // bn2 without the relu, applied before the residual add + cur = cbx_bn2d(ctx0, r.bn2, cur); + ggml_tensor * res = x; + if (r.shortcut_w) { + res = ggml_conv_2d(ctx0, r.shortcut_w, x, 1, stride, 0, 0, 1, 1); + res = cbx_bn2d(ctx0, r.shortcut_bn, res); + } + return ggml_relu(ctx0, ggml_add(ctx0, cur, res)); +} + +// cam dense tdnn layer: bottleneck then context-gated conv; x [C_in, T] -> [growth, T] +ggml_tensor * mtmd_graph_chatterbox_spkenc::cam_layer(const mtmd_chatterbox::spk_cam_layer & l, ggml_tensor * x, + int dil, ggml_tensor * segfix) { + ggml_tensor * h = ggml_relu(ctx0, bn1d(l.nl1_bn, x)); + h = cbx_conv1d(l.linear1_w, nullptr, h, 1, 0, 0); + h = ggml_relu(ctx0, bn1d(l.nl2_bn, h)); + + ggml_tensor * k = l.local_w; + const int pad = ((int) k->ne[0] - 1) / 2 * dil; + ggml_tensor * y = cbx_conv1d_dil(k, nullptr, h, pad, dil); + + // context: global mean plus ceil-mode segment means of length 100 + const int T = (int) h->ne[1]; + const int C = (int) h->ne[0]; + const int S = (T + 99) / 100; + ggml_tensor * ht = ggml_cont(ctx0, ggml_transpose(ctx0, h)); // [T, C] + ggml_tensor * gmean = ggml_cont(ctx0, ggml_transpose(ctx0, ggml_mean(ctx0, ht))); // [C, 1] + ggml_tensor * seg; + { + ggml_tensor * padded = ht; + if (S * 100 != T) { + ggml_tensor * z = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, S * 100 - T, C); + z = ggml_scale(ctx0, z, 0.0f); + padded = ggml_concat(ctx0, ht, z, 0); + } + ggml_tensor * pooled = ggml_pool_1d(ctx0, padded, GGML_OP_POOL_AVG, 100, 100, 0); // [S, C] + pooled = ggml_mul(ctx0, pooled, segfix); + ggml_tensor * exp = ggml_interpolate(ctx0, pooled, S * 100, C, 1, 1, GGML_SCALE_MODE_NEAREST); + exp = ggml_cont(ctx0, ggml_view_2d(ctx0, exp, T, C, exp->nb[1], 0)); + seg = ggml_cont(ctx0, ggml_transpose(ctx0, exp)); // [C, T] + } + ggml_tensor * context = ggml_add(ctx0, seg, gmean); + context = cbx_conv1d(l.ctx1_w, l.ctx1_b, context, 1, 0, 0); + context = ggml_relu(ctx0, context); + context = cbx_conv1d(l.ctx2_w, l.ctx2_b, context, 1, 0, 0); + ggml_tensor * m = ggml_sigmoid(ctx0, context); + return ggml_mul(ctx0, y, m); +} + +ggml_cgraph * mtmd_graph_chatterbox_spkenc::build() { + const auto & c = model.cbx; + + const int T = img.nx(); + const int T1 = (T - 1) / 2 + 1; + const int S = (T1 + 99) / 100; + ggml_tensor * segfix = ggml_new_tensor_1d(ctx0, GGML_TYPE_F32, S); + ggml_set_name(segfix, "inp_segfix"); + ggml_set_input(segfix); + + // fbank features [T, 80] from the preprocessor + ggml_tensor * inp = build_inp_raw(1); + + // fcm 2d front: [W=T, H=F=80, C=1] -> [T, 10, 32] -> [320, T] + ggml_tensor * x = ggml_reshape_4d(ctx0, inp, T, 80, 1, 1); + x = ggml_conv_2d(ctx0, c.spk_conv1_w, x, 1, 1, 1, 1, 1, 1); + x = bn2d_relu(c.spk_bn1, x); + x = res2d(c.spk_layer1_0, x, 2); + x = res2d(c.spk_layer1_1, x, 1); + x = res2d(c.spk_layer2_0, x, 2); + x = res2d(c.spk_layer2_1, x, 1); + x = ggml_conv_2d(ctx0, c.spk_conv2_w, x, 1, 2, 1, 1, 1, 1); + x = bn2d_relu(c.spk_bn2, x); + x = ggml_reshape_2d(ctx0, x, T, 320); + x = ggml_cont(ctx0, ggml_transpose(ctx0, x)); // [320, T] + cb(x, "spk_fcm", -1); + + // tdnn k5 stride 2 over time, then the three cam dense blocks + x = cbx_conv1d(c.spk_tdnn_w, nullptr, x, 2, 2, 2); // [128, T1] + x = ggml_relu(ctx0, bn1d(c.spk_tdnn_bn, x)); + cb(x, "spk_tdnn", -1); + + static const int block_dil[3] = {1, 2, 2}; + const mtmd_chatterbox::spk_cam_block * blocks[3] = { &c.spk_block1, &c.spk_block2, &c.spk_block3 }; + for (int bi = 1; bi <= 3; bi++) { + const auto & blk = *blocks[bi - 1]; + for (const auto & l : blk.layers) { + ggml_tensor * out = cam_layer(l, x, block_dil[bi - 1], segfix); + x = ggml_concat(ctx0, x, out, 0); + } + x = ggml_relu(ctx0, bn1d(blk.transit_bn, x)); + x = cbx_conv1d(blk.transit_w, nullptr, x, 1, 0, 0); + cb(x, "spk_block", bi); + } + x = ggml_relu(ctx0, bn1d(c.spk_out_bn, x)); // [512, T1] + + // statistics pooling: mean and unbiased std over time -> [1024, 1] + ggml_tensor * xt = ggml_cont(ctx0, ggml_transpose(ctx0, x)); // [T1, 512] + ggml_tensor * mean = ggml_mean(ctx0, xt); // [1, 512] + ggml_tensor * m2 = ggml_mean(ctx0, ggml_mul(ctx0, xt, xt)); + ggml_tensor * var = ggml_sub(ctx0, m2, ggml_mul(ctx0, mean, mean)); + var = ggml_scale(ctx0, var, (float) T1 / (float) (T1 - 1)); + ggml_tensor * sd = ggml_sqrt(ctx0, ggml_relu(ctx0, var)); + ggml_tensor * stats = ggml_concat(ctx0, ggml_cont(ctx0, ggml_transpose(ctx0, mean)), + ggml_cont(ctx0, ggml_transpose(ctx0, sd)), 0); // [1024, 1] + cb(stats, "spk_stats_pool", -1); + + // the x-vector stays raw, the gen graph normalizes it + ggml_tensor * dw = ggml_reshape_2d(ctx0, c.spk_dense_w, 1024, 192); + ggml_tensor * emb = ggml_mul_mat(ctx0, dw, stats); // [192, 1] + emb = bn1d(c.spk_dense_bn, emb); + emb = ggml_cont(ctx0, ggml_reshape_1d(ctx0, emb, 192)); + cb(emb, "spk_embd", -1); + ggml_set_name(emb, "out_xvec"); + ggml_set_output(emb); + ggml_build_forward_expand(gf, emb); + return gf; +} diff --git a/tools/mtmd/models/cogvlm.cpp b/tools/mtmd/models/cogvlm.cpp index 44bc884421df..aeaa33e4fa55 100644 --- a/tools/mtmd/models/cogvlm.cpp +++ b/tools/mtmd/models/cogvlm.cpp @@ -1,6 +1,6 @@ #include "models.h" -ggml_cgraph * clip_graph_cogvlm::build() { +ggml_cgraph * mtmd_graph_cogvlm::build() { GGML_ASSERT(model.class_embedding != nullptr); GGML_ASSERT(model.position_embeddings != nullptr); diff --git a/tools/mtmd/models/conformer.cpp b/tools/mtmd/models/conformer.cpp index 9463f9f1ceb5..4cd2bee533b1 100644 --- a/tools/mtmd/models/conformer.cpp +++ b/tools/mtmd/models/conformer.cpp @@ -1,6 +1,6 @@ #include "models.h" -ggml_cgraph * clip_graph_conformer::build() { +ggml_cgraph * mtmd_graph_conformer::build() { const int n_frames = img.nx(); const int n_pos = n_frames / 2; const int n_pos_embd = (((((n_frames + 1) / 2) + 1) / 2 + 1) / 2) * 2 - 1; diff --git a/tools/mtmd/models/deepseek4v.cpp b/tools/mtmd/models/deepseek4v.cpp index 06005d52c8b8..e3c0d95ed17a 100644 --- a/tools/mtmd/models/deepseek4v.cpp +++ b/tools/mtmd/models/deepseek4v.cpp @@ -11,14 +11,14 @@ // // each aligner row ends with a NEWLINE, an odd row count is padded with a full row of PADs // pairs of adjacent rows are interleaved column-wise ("N-layout") -// the mapping is precomputed on CPU as the "layout_idx" input (see set_input in clip.cpp) +// the mapping is precomputed on CPU as the "layout_idx" input (see set_input in mtmd-subcontext.cpp) // // ref: inference/vision.py and inference/image_processor.py in the HF repo // // DeepSeek-V4.1-Flash (deepseek41v) shares the encoder and aligner; its block is plain reading order // [START] ([IMAGE]*n_llm_w [NEWLINE])*n_llm_h [END], no lead pads, no pad row, no IMAGE_PAD tensor -ggml_cgraph * clip_graph_deepseek4v::build() { +ggml_cgraph * mtmd_graph_deepseek4v::build() { const int n_merge = hparams.n_merge; // 2D input positions @@ -27,7 +27,7 @@ ggml_cgraph * clip_graph_deepseek4v::build() { ggml_set_input(positions); int sections[4] = {d_head/4, d_head/4, 0, 0}; - auto add_pos = [&](ggml_tensor * cur, const clip_layer &) { + auto add_pos = [&](ggml_tensor * cur, const mtmd_layer &) { return ggml_rope_multi(ctx0, cur, positions, nullptr, d_head/2, sections, GGML_ROPE_TYPE_VISION, 0, hparams.rope_theta, 1.0f, 0.0f, 1.0f, 0.0f, 0.0f); @@ -92,8 +92,8 @@ ggml_cgraph * clip_graph_deepseek4v::build() { } } - const int n_llm_w = CLIP_ALIGN(n_patches_x, n_merge) / n_merge; - const int n_llm_h = CLIP_ALIGN(n_patches_y, n_merge) / n_merge; + const int n_llm_w = MTMD_ALIGN(n_patches_x, n_merge) / n_merge; + const int n_llm_h = MTMD_ALIGN(n_patches_y, n_merge) / n_merge; const int n_out = is_v41 ? dsv41_n_block_tokens(n_llm_w, n_llm_h) : dsv4_get_block_layout(n_llm_w, n_llm_h, img.lead_pad).n_out; GGML_ASSERT(n_grid == n_llm_w * n_llm_h); diff --git a/tools/mtmd/models/deepseekocr.cpp b/tools/mtmd/models/deepseekocr.cpp index b784cdad6011..af69c0332731 100644 --- a/tools/mtmd/models/deepseekocr.cpp +++ b/tools/mtmd/models/deepseekocr.cpp @@ -105,7 +105,7 @@ static ggml_tensor * conv_2d_f32(ggml_context * ctx0, ggml_tensor * a, ggml_tens } -ggml_tensor * clip_graph_deepseekocr::build_sam(ggml_tensor * inp_raw) { +ggml_tensor * mtmd_graph_deepseekocr::build_sam(ggml_tensor * inp_raw) { // Building SAM const int n_embd = hparams.sam_n_embd; const int n_layer = hparams.sam_n_layer; @@ -211,7 +211,7 @@ ggml_tensor * clip_graph_deepseekocr::build_sam(ggml_tensor * inp_raw) { mask = ggml_add(ctx0, rw, rh); // [B*n_heads, H*W, H, W] mask = ggml_reshape_4d(ctx0, mask, W * H, W * H, n_heads, B); // casting mask to F16 only required when flash-attn is enabled - if (flash_attn_type == CLIP_FLASH_ATTN_TYPE_ENABLED) { + if (flash_attn_type == MTMD_FLASH_ATTN_TYPE_ENABLED) { mask = ggml_cast(ctx0, mask, GGML_TYPE_F16); } @@ -264,7 +264,7 @@ ggml_tensor * clip_graph_deepseekocr::build_sam(ggml_tensor * inp_raw) { return cur; } -ggml_cgraph * clip_graph_deepseekocr::build() { +ggml_cgraph * mtmd_graph_deepseekocr::build() { // patch embedding ggml_tensor * inp_raw = build_inp_raw(); diff --git a/tools/mtmd/models/deepseekocr2.cpp b/tools/mtmd/models/deepseekocr2.cpp index 7088bb97b54b..fb20e9d1d8d3 100644 --- a/tools/mtmd/models/deepseekocr2.cpp +++ b/tools/mtmd/models/deepseekocr2.cpp @@ -1,6 +1,6 @@ #include "models.h" -ggml_cgraph * clip_graph_deepseekocr2::build() { +ggml_cgraph * mtmd_graph_deepseekocr2::build() { GGML_ASSERT(hparams.n_head_kv > 0); GGML_ASSERT(n_head % hparams.n_head_kv == 0); @@ -45,7 +45,7 @@ ggml_cgraph * clip_graph_deepseekocr2::build() { ggml_tensor * inp_pos = ggml_cast(ctx0, ggml_arange(ctx0, 0, seq_len, 1), GGML_TYPE_I32); - auto add_rope = [&](ggml_tensor * x, const clip_layer &) { + auto add_rope = [&](ggml_tensor * x, const mtmd_layer &) { return ggml_rope_ext(ctx0, x, inp_pos, nullptr, d_head, GGML_ROPE_TYPE_NEOX, 131072, 1000000, 1, 0, 1, 0, 0); }; diff --git a/tools/mtmd/models/dots3note.cpp b/tools/mtmd/models/dots3note.cpp index 93c14fc7967a..c942ea7dab64 100644 --- a/tools/mtmd/models/dots3note.cpp +++ b/tools/mtmd/models/dots3note.cpp @@ -1,6 +1,6 @@ #include "models.h" -ggml_cgraph * clip_graph_dots3note_a::build() { +ggml_cgraph * mtmd_graph_dots3note_a::build() { // inp_raw: [n_frames, n_mel, 1], one 60s chunk, mel frames not padded // the reference impl zero-masks conv inputs beyond the valid length at each stage; // running on exactly the valid frames with the convs' zero padding is equivalent @@ -37,7 +37,7 @@ ggml_cgraph * clip_graph_dots3note_a::build() { ggml_set_input(positions); // partial rotary: first half of each head, NEOX style - auto add_pos = [&](ggml_tensor * cur, const clip_layer &) { + auto add_pos = [&](ggml_tensor * cur, const mtmd_layer &) { return ggml_rope_ext(ctx0, cur, positions, nullptr, d_head/2, GGML_ROPE_TYPE_NEOX, 0, hparams.rope_theta, 1.0f, 0.0f, 1.0f, 0.0f, 0.0f); }; diff --git a/tools/mtmd/models/dotsocr.cpp b/tools/mtmd/models/dotsocr.cpp index 92974bb670d8..dcede11c34a4 100644 --- a/tools/mtmd/models/dotsocr.cpp +++ b/tools/mtmd/models/dotsocr.cpp @@ -1,6 +1,6 @@ #include "models.h" -ggml_cgraph * clip_graph_dotsocr::build() { +ggml_cgraph * mtmd_graph_dotsocr::build() { const int n_pos = n_patches; const int num_position_ids = n_pos * 4; // m-rope requires 4 dim per position @@ -11,7 +11,7 @@ ggml_cgraph * clip_graph_dotsocr::build() { ggml_set_name(positions, "positions"); ggml_set_input(positions); - auto add_pos = [&](ggml_tensor * cur, const clip_layer &) { + auto add_pos = [&](ggml_tensor * cur, const mtmd_layer &) { return ggml_rope_multi( ctx0, cur, positions, nullptr, d_head/2, mrope_sections, GGML_ROPE_TYPE_VISION, diff --git a/tools/mtmd/models/drama3-gen.cpp b/tools/mtmd/models/drama3-gen.cpp new file mode 100644 index 000000000000..e77fcf540daf --- /dev/null +++ b/tools/mtmd/models/drama3-gen.cpp @@ -0,0 +1,256 @@ +#include "models.h" + +#include +#include + +// tower RoPE: interleaved, first 64 dims of each head +static constexpr int DRAMA3_N_ROT = 64; +static constexpr float DRAMA3_ROPE_THETA = 1e7f; + +std::vector list_drama3_state_slots(const mtmd_hparams & hparams, const mtmd_model & model) { + std::vector slots; + slots.push_back({"tower_pos", 1, 1, true}); + for (size_t il = 0; il < model.layers.size(); il++) { + const int64_t n_kv = model.layers[il].k_w->ne[1]; + slots.push_back({"tower_k_" + std::to_string(il), n_kv, mtmd_graph_drama3_gen::n_pos_max, true}); + slots.push_back({"tower_v_" + std::to_string(il), n_kv, mtmd_graph_drama3_gen::n_pos_max, true}); + } + for (auto & slot : list_c2w_state_slots(hparams, model)) { + slots.push_back(slot); + } + return slots; +} + +// h_out = out_norm(Block(eh_proj([enorm(mask * E[code] + extra); hnorm(h_in)]))) +ggml_tensor * mtmd_graph_drama3_gen::tower_layer(ggml_tensor * x, int il, ggml_tensor * code, ggml_tensor * mask, ggml_tensor * extra, + ggml_tensor * inp_pos, ggml_tensor * k_slot, ggml_tensor * v_slot, int pos) { + const auto & layer = model.layers[il]; + const auto & tl = model.drama3_layers[il]; + const int n_head = hparams.n_head; + const int n_head_kv = hparams.n_head_kv; + const int64_t d_head = layer.q_w->ne[1] / n_head; + const int64_t n_kv = d_head * n_head_kv; + const float eps = hparams.eps; + + ggml_tensor * e = ggml_mul(ctx0, ggml_get_rows(ctx0, tl.embd_w, code), mask); + if (extra) { + e = ggml_add(ctx0, e, extra); + } + e = ggml_mul(ctx0, ggml_rms_norm(ctx0, e, eps), tl.enorm_w); + ggml_tensor * h = ggml_mul(ctx0, ggml_rms_norm(ctx0, x, eps), tl.hnorm_w); + ggml_tensor * mixed = ggml_mul_mat(ctx0, tl.eh_proj_w, ggml_concat(ctx0, e, h, 0)); + + ggml_tensor * cur = ggml_mul(ctx0, ggml_rms_norm(ctx0, mixed, eps), layer.ln_1_w); + ggml_tensor * q = ggml_reshape_3d(ctx0, ggml_mul_mat(ctx0, layer.q_w, cur), d_head, n_head, 1); + ggml_tensor * k = ggml_reshape_3d(ctx0, ggml_mul_mat(ctx0, layer.k_w, cur), d_head, n_head_kv, 1); + ggml_tensor * v = ggml_mul_mat(ctx0, layer.v_w, cur); + ggml_tensor * gate = ggml_mul_mat(ctx0, layer.attn_gate_w, cur); + + q = ggml_mul(ctx0, ggml_rms_norm(ctx0, q, eps), layer.q_norm); + k = ggml_mul(ctx0, ggml_rms_norm(ctx0, k, eps), layer.k_norm); + q = ggml_rope_ext(ctx0, q, inp_pos, nullptr, DRAMA3_N_ROT, GGML_ROPE_TYPE_NORMAL, 0, DRAMA3_ROPE_THETA, 1.0f, 0.0f, 1.0f, 0.0f, 0.0f); + k = ggml_rope_ext(ctx0, k, inp_pos, nullptr, DRAMA3_N_ROT, GGML_ROPE_TYPE_NORMAL, 0, DRAMA3_ROPE_THETA, 1.0f, 0.0f, 1.0f, 0.0f, 0.0f); + + // write K/V of this row to the slot, attention uses the cached rows + this row + ggml_tensor * k_new = ggml_reshape_2d(ctx0, k, n_kv, 1); + ggml_build_forward_expand(gf, ggml_cpy(ctx0, k_new, ggml_view_2d(ctx0, k_slot, n_kv, 1, k_slot->nb[1], (size_t) pos * k_slot->nb[1]))); + ggml_build_forward_expand(gf, ggml_cpy(ctx0, v, ggml_view_2d(ctx0, v_slot, n_kv, 1, v_slot->nb[1], (size_t) pos * v_slot->nb[1]))); + ggml_tensor * k_all = k_new; + ggml_tensor * v_all = v; + if (pos > 0) { + k_all = ggml_concat(ctx0, ggml_view_2d(ctx0, k_slot, n_kv, pos, k_slot->nb[1], 0), k_new, 1); + v_all = ggml_concat(ctx0, ggml_view_2d(ctx0, v_slot, n_kv, pos, v_slot->nb[1], 0), v, 1); + } + + ggml_tensor * attn = build_attn(nullptr, nullptr, + ggml_reshape_4d(ctx0, q, d_head, n_head, 1, 1), + ggml_reshape_4d(ctx0, k_all, d_head, n_head_kv, pos + 1, 1), + ggml_reshape_4d(ctx0, v_all, d_head, n_head_kv, pos + 1, 1), + nullptr, 1.0f / sqrtf((float) d_head), il); + attn = ggml_mul(ctx0, attn, ggml_sigmoid(ctx0, gate)); + cur = ggml_add(ctx0, mixed, ggml_mul_mat(ctx0, layer.o_w, attn)); + + ggml_tensor * ff = ggml_mul(ctx0, ggml_rms_norm(ctx0, cur, eps), layer.ln_2_w); + ff = ggml_swiglu_split(ctx0, ggml_mul_mat(ctx0, layer.ff_gate_w, ff), ggml_mul_mat(ctx0, layer.ff_up_w, ff)); + cur = ggml_add(ctx0, cur, ggml_mul_mat(ctx0, layer.ff_down_w, ff)); + + cur = ggml_mul(ctx0, ggml_rms_norm(ctx0, cur, eps), tl.out_norm_w); + cb(cur, "tower_out", il); + return cur; +} + +// one row: backbone hidden + next main code -> other codes of that frame + next backbone input +ggml_cgraph * mtmd_graph_drama3_gen::build_tower() { + const int n_layer = (int) model.layers.size(); + const int n_heads = (int) model.gen_code_head_w->ne[2]; // one per residual codebook + const int n_shared = n_layer - n_heads; + const int64_t n_cb = model.gen_code_embd_w->ne[2]; + const int64_t n_embd_cb = model.gen_code_embd_w->ne[0]; + + int pos = 0; + std::vector slots; + const auto slot_list = list_drama3_state_slots(hparams, model); + if (gen_state) { + float pos_f = 0.0f; + ggml_backend_tensor_get(gen_state->slots[0], &pos_f, 0, sizeof(float)); + pos = (int) pos_f; + slots = gen_state->slots; + } else { + // buffer reserve: no gen state, use stand-in slots + for (const auto & slot : slot_list) { + ggml_tensor * t = nullptr; + if (slot.in_place) { + t = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, slot.ne0, slot.ne1); + ggml_set_input(t); + } + slots.push_back(t); + } + } + GGML_ASSERT(pos < n_pos_max); + + ggml_tensor * h = ggml_new_tensor_1d(ctx0, GGML_TYPE_F32, n_mmproj_embd); + ggml_set_name(h, "inp_raw"); + ggml_set_input(h); + + ggml_tensor * inp_code0 = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, 1); + ggml_set_name(inp_code0, "inp_code0"); + ggml_set_input(inp_code0); + + // codes 2.. of the input frame, as row ids of the merged codebook table + ggml_tensor * inp_extra = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_cb - 2); + ggml_set_name(inp_extra, "inp_extra"); + ggml_set_input(inp_extra); + + // [next frame is speech, input frame is speech] + ggml_tensor * inp_scale = ggml_new_tensor_1d(ctx0, GGML_TYPE_F32, 2); + ggml_set_name(inp_scale, "inp_scale"); + ggml_set_input(inp_scale); + + ggml_tensor * inp_pos = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, 1); + ggml_set_name(inp_pos, "inp_pos"); + ggml_set_input(inp_pos); + + ggml_tensor * inp_rand = ggml_new_tensor_1d(ctx0, GGML_TYPE_F32, n_heads); + ggml_set_name(inp_rand, "inp_rand"); + ggml_set_input(inp_rand); + + ggml_tensor * mask = ggml_view_1d(ctx0, inp_scale, 1, 0); + ggml_tensor * extra_on = ggml_view_1d(ctx0, inp_scale, 1, sizeof(float)); + + ggml_tensor * cb_all = ggml_reshape_2d(ctx0, model.gen_code_embd_w, n_embd_cb, model.gen_code_embd_w->ne[1] * n_cb); + ggml_tensor * extra = ggml_cont(ctx0, ggml_transpose(ctx0, ggml_get_rows(ctx0, cb_all, inp_extra))); + extra = ggml_mul(ctx0, ggml_reshape_2d(ctx0, ggml_sum_rows(ctx0, extra), n_embd_cb, 1), extra_on); + + mtmd_graph_qwen3tts_gen::code_gen cg(*this, top_k, top_p); + ggml_tensor * out_codes = cg.cache_set(ggml_new_tensor_2d(ctx0, GGML_TYPE_I32, 1, n_cb), 0, inp_code0); + + ggml_tensor * code = inp_code0; + ggml_tensor * x = ggml_reshape_2d(ctx0, h, n_mmproj_embd, 1); + for (int il = 0; il < n_layer; il++) { + x = tower_layer(x, il, code, mask, il == 0 ? extra : nullptr, inp_pos, slots[1 + 2 * il], slots[2 + 2 * il], pos); + if (il < n_shared) { + continue; + } + const int ih = il - n_shared; + ggml_tensor * head = ggml_view_2d(ctx0, model.gen_code_head_w, model.gen_code_head_w->ne[0], model.gen_code_head_w->ne[1], + model.gen_code_head_w->nb[1], (size_t) ih * model.gen_code_head_w->nb[2]); + ggml_tensor * logits = ggml_mul_mat(ctx0, head, x); + if (temp > 0.0f && temp != 1.0f) { + logits = ggml_scale(ctx0, logits, 1.0f / temp); + } + code = cg.do_sampling(logits, ggml_view_1d(ctx0, inp_rand, 1, (size_t) ih * sizeof(float))); + out_codes = cg.cache_set(out_codes, ih + 1, code); + } + + ggml_tensor * pos_slot = slots[0]; + ggml_build_forward_expand(gf, ggml_cpy(ctx0, ggml_scale_bias(ctx0, pos_slot, 1.0f, 1.0f), pos_slot)); + + out_codes = ggml_cont(ctx0, out_codes); + ggml_set_name(out_codes, "out_codes"); + ggml_set_output(out_codes); + ggml_build_forward_expand(gf, out_codes); + + // backbone input of the next row: (E_lm[main] + E_cb1[code 1]) / sqrt(2) + ggml_tensor * code1 = ggml_view_1d(ctx0, out_codes, 1, out_codes->nb[1]); + ggml_tensor * cb1 = ggml_view_2d(ctx0, model.gen_code_embd_w, n_embd_cb, model.gen_code_embd_w->ne[1], + model.gen_code_embd_w->nb[1], model.gen_code_embd_w->nb[2]); + ggml_tensor * out_embd = ggml_add(ctx0, ggml_get_rows(ctx0, model.gen_code_out_embd_w, inp_code0), ggml_get_rows(ctx0, cb1, code1)); + out_embd = ggml_scale(ctx0, out_embd, 1.0f / sqrtf(2.0f)); + cb(out_embd, "gen_audio_out", -1); + ggml_build_forward_expand(gf, out_embd); + + return gf; +} + +// frames -> PCM, qwen3tts code2wav blocks without pre_conv and tfm projections +ggml_cgraph * mtmd_graph_drama3_gen::build_codec() { + const auto & c2w = model.c2w; + const int64_t n_cb = model.gen_code_embd_w->ne[2]; + + ggml_tensor * inp_codes = ggml_new_tensor_2d(ctx0, GGML_TYPE_I32, n_frames, n_cb); + ggml_set_name(inp_codes, "inp_codes"); + ggml_set_input(inp_codes); + + mtmd_graph_qwen3tts_gen::code2wav dec(*this); + dec.gelu_erf = true; + for (const auto & slot : list_c2w_state_slots(hparams, model)) { + ggml_tensor * t = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, slot.ne0, slot.ne1); + ggml_set_name(t, ("state_in_" + slot.name).c_str()); + ggml_set_input(t); + dec.state_in[slot.name] = t; + } + + // out_proj is folded into the codebook tables at conversion + auto group_ids = [&](int g) { + return ggml_view_1d(ctx0, inp_codes, n_frames, (size_t) g * inp_codes->nb[1]); + }; + ggml_tensor * cur = ggml_get_rows(ctx0, c2w.quant_first_cb_w, group_ids(0)); // [dim, T] + for (int g = 1; g < n_cb; g++) { + ggml_tensor * table = ggml_view_2d(ctx0, c2w.quant_rest_cb_w, c2w.quant_rest_cb_w->ne[0], c2w.quant_rest_cb_w->ne[1], + c2w.quant_rest_cb_w->nb[1], (size_t) (g - 1) * c2w.quant_rest_cb_w->nb[2]); + cur = ggml_add(ctx0, cur, ggml_get_rows(ctx0, table, group_ids(g))); + } + + for (int il = 0; il < hparams.wav_tfm_n_layer; il++) { + cur = dec.tfm_layer_forward(cur, c2w.tfm_layers[il], il); + } + cur = ggml_mul(ctx0, ggml_rms_norm(ctx0, cur, hparams.wav_tfm_eps), c2w.tfm_output_norm_w); + cb(cur, "wav_tfm_out", -1); + + ggml_tensor * x = ggml_cont(ctx0, ggml_transpose(ctx0, cur)); // [T, dim] + for (size_t il = 0; il < c2w.upsample.size(); il++) { + const auto & up = c2w.upsample[il]; + x = dec.causal_conv_transpose1d(x, up.conv_w, up.conv_b, 2, ""); + x = dec.convnext_block(x, up, "up" + std::to_string(il)); + } + + static constexpr int DAC_DILATIONS[3] = { 1, 3, 9 }; + x = dec.causal_conv1d(x, c2w.dac_entry_w, c2w.dac_entry_b, 1, "dac_entry"); + for (size_t il = 0; il < c2w.dac.size(); il++) { + const auto & blk = c2w.dac[il]; + const int stride = (int) (blk.conv_w->ne[0] / 2); + const std::string blk_name = "dac" + std::to_string(il); + x = dec.snake(x, blk.snake_alpha, blk.snake_beta); + x = dec.causal_conv_transpose1d(x, blk.conv_w, blk.conv_b, stride, blk_name + "_tail"); + for (size_t ir = 0; ir < blk.res.size(); ir++) { + x = dec.dac_res_unit(x, blk.res[ir], DAC_DILATIONS[ir], blk_name + "_res" + std::to_string(ir)); + } + } + x = dec.snake(x, c2w.dac_post_snake_alpha, c2w.dac_post_snake_beta); + x = dec.causal_conv1d(x, c2w.dac_post_conv_w, c2w.dac_post_conv_b, 1, "dac_post_conv"); + x = ggml_tanh(ctx0, ggml_reshape_1d(ctx0, x, x->ne[0])); + ggml_set_name(x, "out_audio"); + ggml_set_output(x); + ggml_build_forward_expand(gf, x); + + for (auto & slot : dec.state_out) { + ggml_set_name(slot.second, ("state_out_" + slot.first).c_str()); + ggml_set_output(slot.second); + ggml_build_forward_expand(gf, slot.second); + } + return gf; +} + +ggml_cgraph * mtmd_graph_drama3_gen::build() { + return gen_process == MTMD_SUBCTX_GEN_PROCESS_GEN_WAV ? build_codec() : build_tower(); +} diff --git a/tools/mtmd/models/exaone4_5.cpp b/tools/mtmd/models/exaone4_5.cpp index d3372a1cd7bd..768620e59e2c 100644 --- a/tools/mtmd/models/exaone4_5.cpp +++ b/tools/mtmd/models/exaone4_5.cpp @@ -1,7 +1,7 @@ // similar to qwen2vl, except for GQA attention #include "models.h" -ggml_cgraph * clip_graph_exaone4_5::build() { +ggml_cgraph * mtmd_graph_exaone4_5::build() { GGML_ASSERT(model.patch_bias == nullptr); GGML_ASSERT(model.class_embedding == nullptr); diff --git a/tools/mtmd/models/gemma4a.cpp b/tools/mtmd/models/gemma4a.cpp index f98a8b6fc198..2b81f036c15b 100644 --- a/tools/mtmd/models/gemma4a.cpp +++ b/tools/mtmd/models/gemma4a.cpp @@ -1,5 +1,5 @@ /** - * Gemma 4 Audio Conformer Encoder (clip_graph_gemma4a) + * Gemma 4 Audio Conformer Encoder (mtmd_graph_gemma4a) * * Architecture: Conformer with dual half-step FFN, full self-attention * with sinusoidal RPE, depthwise light conv, and output projection. @@ -7,19 +7,60 @@ #include "models.h" #include +#include -ggml_cgraph * clip_graph_gemma4a::build() { +// state of the streaming encoder, a zero state is a cold start +std::vector list_gemma4a_state_slots(const mtmd_hparams & hparams, const mtmd_model & model) { + std::vector slots; + + const int64_t n_mel = hparams.n_mel_bins; + const int64_t n_ch = model.sscp_conv_w[0]->ne[3]; + + slots.push_back({"pos", 1, 1}); + slots.push_back({"sscp_0", n_mel, 1}); + slots.push_back({"sscp_1", (n_mel - 1) / 2 + 1, n_ch}); + for (size_t il = 0; il < model.layers.size(); il++) { + const auto & layer = model.layers[il]; + if (layer.q_w && layer.k_w && layer.v_w && layer.o_w) { + slots.push_back({"k_" + std::to_string(il), layer.k_w->ne[1], mtmd_graph_gemma4a::max_past}); + slots.push_back({"v_" + std::to_string(il), layer.v_w->ne[1], mtmd_graph_gemma4a::max_past}); + } + if (layer.norm_conv_w && layer.conv_pw1_w && layer.conv_dw_w && layer.conv_pw2_w) { + slots.push_back({"conv_" + std::to_string(il), layer.conv_dw_w->ne[0] - 1, layer.conv_dw_w->ne[1]}); + } + } + + return slots; +} + +ggml_cgraph * mtmd_graph_gemma4a::build() { const float res_weight = 0.5f; const float norm_eps = 1e-6f; + std::map state_in; + std::vector> state_out; + for (const auto & slot : list_gemma4a_state_slots(hparams, model)) { + ggml_tensor * t = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, slot.ne0, slot.ne1); + ggml_set_name(t, ("state_in_" + slot.name).c_str()); + ggml_set_input(t); + state_in[slot.name] = t; + } + // 1. Input ggml_tensor * inp = build_inp_raw(1); auto * cur = ggml_cont(ctx0, ggml_transpose(ctx0, inp)); - // 2. Subsampling Conv2D (symmetric padding=1, matching PyTorch) + // 2. Subsampling Conv2D (padding=1, matching PyTorch) + // in time, the left pad comes from the state { for (int i = 0; i < 2; i++) { - cur = ggml_conv_2d(ctx0, model.sscp_conv_w[i], cur, 2, 2, 1, 1, 1, 1); + const std::string name = "sscp_" + std::to_string(i); + ggml_tensor * left = ggml_reshape_3d(ctx0, state_in.at(name), cur->ne[0], 1, cur->ne[2]); + cur = ggml_concat(ctx0, left, cur, 1); + ggml_tensor * last = ggml_view_3d(ctx0, cur, cur->ne[0], 1, cur->ne[2], cur->nb[1], cur->nb[2], (cur->ne[1] - 1) * cur->nb[1]); + state_out.push_back({name, ggml_reshape_2d(ctx0, ggml_cont(ctx0, last), cur->ne[0], cur->ne[2])}); + cur = ggml_pad_ext(ctx0, cur, 0, 0, 0, 1, 0, 0, 0, 0); + cur = ggml_conv_2d(ctx0, model.sscp_conv_w[i], cur, 2, 2, 1, 0, 1, 1); if (model.sscp_conv_b[i]) { cur = ggml_add(ctx0, cur, model.sscp_conv_b[i]); } @@ -44,10 +85,11 @@ ggml_cgraph * clip_graph_gemma4a::build() { } const int64_t n_pos = cur->ne[1]; + state_out.push_back({"pos", ggml_scale_bias(ctx0, state_in.at("pos"), 1.0f, (float) n_pos)}); // Chunked local attention parameters - const int64_t C = 12; // chunk_size - const int64_t P = 12; // max_past_horizon (context_left - 1) + const int64_t C = chunk_size; + const int64_t P = max_past; const int64_t S = C + P; // context_size = 24 const int64_t R = P + 1; // RPE positions = 13 const int64_t B = (n_pos + C - 1) / C; // num_blocks @@ -117,12 +159,12 @@ ggml_cgraph * clip_graph_gemma4a::build() { Qcur = ggml_cont(ctx0, ggml_permute(ctx0, Qcur, 0, 3, 1, 2)); // [D, C, B, H] // K/V block context extraction via overlapping view: - // Left pad by P and right pad to S*B elements, - // then view with stride C in the block dimension (overlapping windows). - auto extract_blocks = [&](ggml_tensor * t) -> ggml_tensor * { - // [D, H, N] -> left pad by P, right pad to S*B - const int64_t pad_kv = S * B - n_pos; - t = ggml_pad_ext(ctx0, t, 0, 0, 0, 0, P, pad_kv - P, 0, 0); // [D, H, S*B] + // put the P cached positions in front, right pad to S*B, then view with stride C in the block dimension + auto extract_blocks = [&](ggml_tensor * t, const std::string & name) -> ggml_tensor * { + t = ggml_concat(ctx0, ggml_reshape_3d(ctx0, state_in.at(name), d_head, n_head, P), t, 2); + ggml_tensor * tail = ggml_view_3d(ctx0, t, d_head, n_head, P, t->nb[1], t->nb[2], n_pos * t->nb[2]); + state_out.push_back({name, ggml_reshape_2d(ctx0, ggml_cont(ctx0, tail), d_head * n_head, P)}); + t = ggml_pad_ext(ctx0, t, 0, 0, 0, 0, 0, S * B - P - n_pos, 0, 0); // [D, H, S*B] // Overlapping view: stride for B dim is C positions, not S // ne = [D, H, S, B], data_size = D*H*S*B*sizeof = source_nbytes (exact fit) // nb1=D*sizeof, nb2=D*H*sizeof, nb3=C*D*H*sizeof (overlap: C < S) @@ -132,11 +174,11 @@ ggml_cgraph * clip_graph_gemma4a::build() { return t; }; - ggml_tensor * Kblk = extract_blocks(Kcur); + ggml_tensor * Kblk = extract_blocks(Kcur, "k_" + std::to_string(il)); // [D, H, S, B] -> [D, S, B, H] via permute(0,3,1,2) Kblk = ggml_cont(ctx0, ggml_permute(ctx0, Kblk, 0, 3, 1, 2)); - ggml_tensor * Vblk = extract_blocks(Vcur); + ggml_tensor * Vblk = extract_blocks(Vcur, "v_" + std::to_string(il)); // [D, H, S, B] -> [S, D, B, H] via permute(1,3,0,2) Vblk = ggml_cont(ctx0, ggml_permute(ctx0, Vblk, 1, 3, 0, 2)); @@ -217,8 +259,13 @@ ggml_cgraph * clip_graph_gemma4a::build() { x = ggml_cont(ctx0, ggml_transpose(ctx0, x)); } - // Causal depthwise Conv1D via ggml_ssm_conv, left padded only. - x = ggml_pad_ext(ctx0, x, 4, 0, 0, 0, 0, 0, 0, 0); + // Causal depthwise Conv1D via ggml_ssm_conv, the left context comes from the state + { + const int64_t d_conv = layer.conv_dw_w->ne[0] - 1; + const std::string name = "conv_" + std::to_string(il); + x = ggml_concat(ctx0, state_in.at(name), x, 0); + state_out.push_back({name, ggml_cont(ctx0, ggml_view_2d(ctx0, x, d_conv, x->ne[1], x->nb[1], n_pos * x->nb[0]))}); + } x = ggml_ssm_conv(ctx0, x, layer.conv_dw_w); if (layer.conv_dw_b) { x = ggml_add(ctx0, x, layer.conv_dw_b); @@ -269,11 +316,18 @@ ggml_cgraph * clip_graph_gemma4a::build() { cur = build_mm(model.mm_input_proj_w, cur); } + // the embeddings must stay the last node + for (const auto & s : state_out) { + ggml_set_name(s.second, ("state_out_" + s.first).c_str()); + ggml_set_output(s.second); + ggml_build_forward_expand(gf, s.second); + } + ggml_build_forward_expand(gf, cur); return gf; } -ggml_tensor * clip_graph_gemma4a::build_mm(ggml_tensor * w, ggml_tensor * x) const { +ggml_tensor * mtmd_graph_gemma4a::build_mm(ggml_tensor * w, ggml_tensor * x) const { auto it = model.clamp_info_map.find(w->name); if (it == model.clamp_info_map.end()) { return ggml_mul_mat(ctx0, w, x); diff --git a/tools/mtmd/models/gemma4ua.cpp b/tools/mtmd/models/gemma4ua.cpp index e24bef2ed7b4..1d77b8889020 100644 --- a/tools/mtmd/models/gemma4ua.cpp +++ b/tools/mtmd/models/gemma4ua.cpp @@ -1,7 +1,7 @@ #include "models.h" #include -ggml_cgraph * clip_graph_gemma4ua::build() { +ggml_cgraph * mtmd_graph_gemma4ua::build() { ggml_tensor * inp = build_inp_raw(1); auto cur = ggml_cont(ctx0, ggml_permute(ctx0, inp, 1, 0, 2, 3)); diff --git a/tools/mtmd/models/gemma4uv.cpp b/tools/mtmd/models/gemma4uv.cpp index 96031141b175..04e139a1ba2c 100644 --- a/tools/mtmd/models/gemma4uv.cpp +++ b/tools/mtmd/models/gemma4uv.cpp @@ -1,7 +1,7 @@ #include "models.h" #include -ggml_cgraph * clip_graph_gemma4uv::build() { +ggml_cgraph * mtmd_graph_gemma4uv::build() { ggml_tensor * inp_raw = build_inp_raw(); // Gemma4UnifiedVisionEmbedder uses default pytorch LayerNorm, not RMSNorm diff --git a/tools/mtmd/models/gemma4v.cpp b/tools/mtmd/models/gemma4v.cpp index 448438947026..46733e0f30e4 100644 --- a/tools/mtmd/models/gemma4v.cpp +++ b/tools/mtmd/models/gemma4v.cpp @@ -1,7 +1,7 @@ #include "models.h" #include -ggml_cgraph * clip_graph_gemma4v::build() { +ggml_cgraph * mtmd_graph_gemma4v::build() { ggml_tensor * inp_raw = build_inp_raw(); // patches = 2 * (patches - 0.5) @@ -43,7 +43,7 @@ ggml_cgraph * clip_graph_gemma4v::build() { } // similar to build_rope_2d, but use neox ordering - auto add_pos = [&](ggml_tensor * cur, const clip_layer &) { + auto add_pos = [&](ggml_tensor * cur, const mtmd_layer &) { const int64_t n_dim = cur->ne[0]; // first half, dims [0, n_dim/2) @@ -117,7 +117,7 @@ ggml_cgraph * clip_graph_gemma4v::build() { return gf; } -ggml_tensor * clip_graph_gemma4v::build_mm(ggml_tensor * w, ggml_tensor * x) const { +ggml_tensor * mtmd_graph_gemma4v::build_mm(ggml_tensor * w, ggml_tensor * x) const { // Gemma4ClippableLinear auto it = model.clamp_info_map.find(w->name); diff --git a/tools/mtmd/models/glm4v.cpp b/tools/mtmd/models/glm4v.cpp index 1cbcea942a39..2b9a30192eba 100644 --- a/tools/mtmd/models/glm4v.cpp +++ b/tools/mtmd/models/glm4v.cpp @@ -1,6 +1,6 @@ #include "models.h" -ggml_cgraph * clip_graph_glm4v::build() { +ggml_cgraph * mtmd_graph_glm4v::build() { GGML_ASSERT(model.patch_bias != nullptr); GGML_ASSERT(model.class_embedding == nullptr); @@ -63,7 +63,7 @@ ggml_cgraph * clip_graph_glm4v::build() { cb(learned_pos_embd, "learned_pos_embd", -1); } - auto add_pos = [&](ggml_tensor * cur, const clip_layer &) { + auto add_pos = [&](ggml_tensor * cur, const mtmd_layer &) { return ggml_rope_multi( ctx0, cur, positions, nullptr, d_head/2, mrope_sections, GGML_ROPE_TYPE_VISION, diff --git a/tools/mtmd/models/granite-speech.cpp b/tools/mtmd/models/granite-speech.cpp index 9725def82733..e0180252232e 100644 --- a/tools/mtmd/models/granite-speech.cpp +++ b/tools/mtmd/models/granite-speech.cpp @@ -2,7 +2,7 @@ #include -ggml_cgraph * clip_graph_granite_speech::build() { +ggml_cgraph * mtmd_graph_granite_speech::build() { const int n_frames = img.nx(); const int context_size = hparams.audio_chunk_size; const int ctc_layer = n_layer / 2; @@ -189,7 +189,7 @@ ggml_cgraph * clip_graph_granite_speech::build() { } else { concat_result = ggml_concat(ctx0, concat_result, cur, 0); } - cb(concat_result, string_format("feature_layer_%d", il + 1).c_str(), il); + cb(concat_result, mtmd_format("feature_layer_%d", il + 1).c_str(), il); } } diff --git a/tools/mtmd/models/granite4-vision.cpp b/tools/mtmd/models/granite4-vision.cpp index a75f1cee9aa1..390d6edf6f98 100644 --- a/tools/mtmd/models/granite4-vision.cpp +++ b/tools/mtmd/models/granite4-vision.cpp @@ -1,6 +1,6 @@ #include "models.h" -#include "../clip-impl.h" -#include "../clip-model.h" +#include "../mtmd-impl.h" +#include "../mtmd-model.h" #include #include @@ -9,7 +9,7 @@ #include /* - * Granite Vision 4.1 clip graph + * Granite Vision 4.1 vision graph * * Stage 1a: SigLIP vision tower (N layers, post-norm) * Stage 1b: WindowQFormer blocks (deepstack + spatial) @@ -22,7 +22,7 @@ // --------------------------------------------------------------------------- // split the stacked tiles into the batch axis, then run the usual patch embedding -ggml_tensor * clip_graph_granite4_vision::build_tile_inp() { +ggml_tensor * mtmd_graph_granite4_vision::build_tile_inp() { ggml_tensor * inp_raw = build_inp_raw(); if (n_tiles > 1) { @@ -41,7 +41,7 @@ ggml_tensor * clip_graph_granite4_vision::build_tile_inp() { return inp; } -ggml_tensor * clip_graph_granite4_vision::gather( +ggml_tensor * mtmd_graph_granite4_vision::gather( ggml_tensor * src, const std::string & name, int idx_len) { @@ -52,7 +52,7 @@ ggml_tensor * clip_graph_granite4_vision::gather( return ggml_get_rows(ctx0, src, idx); } -ggml_tensor * clip_graph_granite4_vision::interp_down( +ggml_tensor * mtmd_graph_granite4_vision::interp_down( ggml_tensor * src, int side, int new_side) { @@ -72,7 +72,7 @@ ggml_tensor * clip_graph_granite4_vision::interp_down( // build_block - WindowQFormer block implementation // --------------------------------------------------------------------------- -ggml_tensor * clip_graph_granite4_vision::build_block( +ggml_tensor * mtmd_graph_granite4_vision::build_block( const qf_block & blk, ggml_tensor * h, int bid, @@ -244,7 +244,7 @@ ggml_tensor * clip_graph_granite4_vision::build_block( // Build the K-tiled, base-scaled newline row tensor. // Shape: (n_mmproj_embd, 1) -ggml_tensor * clip_graph_granite4_vision::build_newline_row(ggml_context * ctx0) { +ggml_tensor * mtmd_graph_granite4_vision::build_newline_row(ggml_context * ctx0) { const int K = (int) model.qf_proj_blocks.size(); GGML_ASSERT(K > 0); GGML_ASSERT(n_mmproj_embd % K == 0); @@ -272,7 +272,7 @@ ggml_tensor * clip_graph_granite4_vision::build_newline_row(ggml_context * ctx0) // Assemble [overview, tile(0,0), tile(0,1), ...] into one token sequence: // the overview tokens first, then the tile grid read in raster order with one newline per row. // ref: https://github.com/huggingface/transformers/blob/v5.0.0/src/transformers/models/llava_next/modeling_llava_next.py#L266 -ggml_tensor * clip_graph_granite4_vision::build_anyres_assembly(ggml_tensor * cur, int out_side) { +ggml_tensor * mtmd_graph_granite4_vision::build_anyres_assembly(ggml_tensor * cur, int out_side) { const int n_dim = cur->ne[0]; const int grid_x = anyres.grid_x; const int grid_y = anyres.grid_y; @@ -293,7 +293,7 @@ ggml_tensor * clip_graph_granite4_vision::build_anyres_assembly(ggml_tensor * cu // drop the tokens that only cover the padding added when resizing to the grid int off_x, off_y, w, h; - clip_anyres_unpad(cur_w, cur_h, anyres.orig_nx, anyres.orig_ny, off_x, off_y, w, h); + mtmd_anyres_unpad(cur_w, cur_h, anyres.orig_nx, anyres.orig_ny, off_x, off_y, w, h); if (w != cur_w || h != cur_h) { tiles = ggml_cont(ctx0, ggml_view_3d(ctx0, tiles, n_dim, w, h, tiles->nb[1], tiles->nb[2], @@ -307,7 +307,7 @@ ggml_tensor * clip_graph_granite4_vision::build_anyres_assembly(ggml_tensor * cu return ggml_concat(ctx0, base, tiles, 1); } -ggml_cgraph * clip_graph_granite4_vision::build() { +ggml_cgraph * mtmd_graph_granite4_vision::build() { GGML_ASSERT(model.patch_embeddings_0 != nullptr); GGML_ASSERT(model.position_embeddings != nullptr); GGML_ASSERT(model.class_embedding == nullptr); diff --git a/tools/mtmd/models/hunyuanvl.cpp b/tools/mtmd/models/hunyuanvl.cpp index 2c670979d769..b6b3fef5809b 100644 --- a/tools/mtmd/models/hunyuanvl.cpp +++ b/tools/mtmd/models/hunyuanvl.cpp @@ -1,12 +1,12 @@ #include "models.h" -ggml_cgraph * clip_graph_hunyuanvl::build() { +ggml_cgraph * mtmd_graph_hunyuanvl::build() { const int merge = hparams.n_merge; const int pw = n_patches_x; const int ph = n_patches_y; // position embedding: declared as a graph input, filled on CPU - // by clip_image_batch_encode (see PROJECTOR_TYPE_HUNYUANVL branch there). + // by mtmd_subcontext::encode() (see PROJECTOR_TYPE_HUNYUANVL branch there). ggml_tensor * pos_embd = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_embd, ph * pw); ggml_set_name(pos_embd, "hunyuanvl_pos_embd"); ggml_set_input(pos_embd); diff --git a/tools/mtmd/models/internvl.cpp b/tools/mtmd/models/internvl.cpp index af8f12cdc93d..5e0b679c453e 100644 --- a/tools/mtmd/models/internvl.cpp +++ b/tools/mtmd/models/internvl.cpp @@ -1,6 +1,6 @@ #include "models.h" -ggml_cgraph * clip_graph_internvl::build() { +ggml_cgraph * mtmd_graph_internvl::build() { GGML_ASSERT(model.class_embedding != nullptr); GGML_ASSERT(model.position_embeddings != nullptr); diff --git a/tools/mtmd/models/kimik25.cpp b/tools/mtmd/models/kimik25.cpp index cb345f0fc624..b77ea9604b3d 100644 --- a/tools/mtmd/models/kimik25.cpp +++ b/tools/mtmd/models/kimik25.cpp @@ -2,10 +2,10 @@ #include #include -// note: this is similar to clip_graph::resize_position_embeddings, major difference is having +// note: this is similar to mtmd_graph::resize_position_embeddings, major difference is having // the w/h in ne[1] and ne[2] instead of assuming with sqrt. Could try storing the tensor in 2D instead // with a w*h? Also the permute is a bit different at (2, 1, 0, 3) instead of (2, 0, 1, 3). -ggml_tensor * clip_graph_kimik25::resize_position_embeddings_3d(uint32_t interpolation_mode) { +ggml_tensor * mtmd_graph_kimik25::resize_position_embeddings_3d(uint32_t interpolation_mode) { ggml_tensor * pos_embd = model.position_embeddings; const int height = img.ny() / patch_size; const int width = img.nx() / patch_size; @@ -31,7 +31,7 @@ ggml_tensor * clip_graph_kimik25::resize_position_embeddings_3d(uint32_t interpo return pos_embd; } -ggml_cgraph * clip_graph_kimik25::build() { +ggml_cgraph * mtmd_graph_kimik25::build() { ggml_tensor * pos_h = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_patches); ggml_set_name(pos_h, "pos_h"); ggml_set_input(pos_h); @@ -40,11 +40,13 @@ ggml_cgraph * clip_graph_kimik25::build() { ggml_set_name(pos_w, "pos_w"); ggml_set_input(pos_w); - ggml_tensor * learned_pos_embd = resize_position_embeddings_3d(GGML_SCALE_MODE_BICUBIC); + const bool is_k31 = proj_type == PROJECTOR_TYPE_KIMIK31; + + ggml_tensor * learned_pos_embd = resize_position_embeddings_3d(is_k31 ? GGML_SCALE_MODE_BILINEAR : GGML_SCALE_MODE_BICUBIC); // Kimi-K2.5 uses interleaved 2D RoPE pattern natively, but // Q / K are permuted during conversion to use split format. - auto add_pos = [&](ggml_tensor * cur, const clip_layer &) { + auto add_pos = [&](ggml_tensor * cur, const mtmd_layer &) { cur = build_rope_2d(ctx0, cur, pos_w, pos_h, hparams.rope_theta, false); return cur; }; @@ -57,14 +59,25 @@ ggml_cgraph * clip_graph_kimik25::build() { ggml_tensor * cur = build_vit( inp, n_patches, - NORM_TYPE_NORMAL, + is_k31 ? NORM_TYPE_RMS : NORM_TYPE_NORMAL, hparams.ffn_op, nullptr, add_pos); cb(cur, "vit_out", -1); - { + if (is_k31) { + // patchmergerv2: no pre-norm, RMS norm after the MLP + cur = build_patch_merge_permute(cur, hparams.n_merge); + cur = build_ffn(cur, + model.mm_1_w, nullptr, + nullptr, nullptr, + model.mm_2_w, nullptr, + FFN_GELU_ERF, + -1); + cur = build_norm(cur, model.mm_post_norm_w, nullptr, NORM_TYPE_RMS, 1e-5f, -1); + cb(cur, "proj_out", -1); + } else { // patch_merger const int scale_factor = model.hparams.n_merge; cur = build_patch_merge_permute(cur, scale_factor); @@ -88,7 +101,7 @@ ggml_cgraph * clip_graph_kimik25::build() { model.mm_1_w, model.mm_1_b, nullptr, nullptr, model.mm_2_w, model.mm_2_b, - FFN_GELU, + FFN_GELU_ERF, -1); cb(cur, "proj_out", -1); diff --git a/tools/mtmd/models/kimivl.cpp b/tools/mtmd/models/kimivl.cpp index 0a06f5090e45..dd8146708df0 100644 --- a/tools/mtmd/models/kimivl.cpp +++ b/tools/mtmd/models/kimivl.cpp @@ -1,6 +1,6 @@ #include "models.h" -ggml_cgraph * clip_graph_kimivl::build() { +ggml_cgraph * mtmd_graph_kimivl::build() { // 2D input positions ggml_tensor * pos_h = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_patches); ggml_set_name(pos_h, "pos_h"); @@ -13,7 +13,7 @@ ggml_cgraph * clip_graph_kimivl::build() { ggml_tensor * learned_pos_embd = resize_position_embeddings(); // build ViT with 2D position embeddings - auto add_pos = [&](ggml_tensor * cur, const clip_layer &) { + auto add_pos = [&](ggml_tensor * cur, const mtmd_layer &) { // first half is X axis and second half is Y axis return build_rope_2d(ctx0, cur, pos_w, pos_h, hparams.rope_theta, false); }; diff --git a/tools/mtmd/models/lfm3a-gen.cpp b/tools/mtmd/models/lfm3a-gen.cpp index 998605711472..f5aedac6fd3e 100644 --- a/tools/mtmd/models/lfm3a-gen.cpp +++ b/tools/mtmd/models/lfm3a-gen.cpp @@ -9,7 +9,7 @@ // GEN_WAV : a window of frames -> polar STFT frames of the liquid-codec decoder, the ISTFT runs on the host // state carried between GEN_WAV calls: rope offset, short-conv taps and attention window of the codec decoder -std::vector list_lfm3a_gen_state_slots(const clip_hparams & hparams, const clip_model & model) { +std::vector list_lfm3a_gen_state_slots(const mtmd_hparams & hparams, const mtmd_model & model) { std::vector slots; if (model.c2w.tfm_layers.empty()) { return slots; @@ -29,7 +29,7 @@ std::vector list_lfm3a_gen_state_slots(const clip_hparams & hpar } // pre-norm LFM block, with the LFM3 sandwich norms when the layer has them -ggml_tensor * clip_graph_lfm3a_gen::build_lfm_layer(ggml_tensor * inpL, const clip_layer & layer, float eps, int il, +ggml_tensor * mtmd_graph_lfm3a_gen::build_lfm_layer(ggml_tensor * inpL, const mtmd_layer & layer, float eps, int il, const std::function & op) { ggml_tensor * cur = build_norm(inpL, layer.ln_1_w, nullptr, NORM_TYPE_RMS, eps, il); cur = op(cur); @@ -47,7 +47,7 @@ ggml_tensor * clip_graph_lfm3a_gen::build_lfm_layer(ggml_tensor * inpL, const cl } // GQA with q/k RMSNorm, NEOX rope, an optional cached K/V prefix and the LFM3 per-head sigmoid gate -ggml_tensor * clip_graph_lfm3a_gen::build_lfm_attn(ggml_tensor * cur, const clip_layer & layer, int n_head, int n_head_kv, +ggml_tensor * mtmd_graph_lfm3a_gen::build_lfm_attn(ggml_tensor * cur, const mtmd_layer & layer, int n_head, int n_head_kv, float eps, float rope_theta, int il, ggml_tensor * inp_pos, ggml_tensor * kq_mask, ggml_tensor * k_cache, ggml_tensor * v_cache, ggml_tensor ** k_all, ggml_tensor ** v_all) { @@ -89,7 +89,7 @@ ggml_tensor * clip_graph_lfm3a_gen::build_lfm_attn(ggml_tensor * cur, const clip } // short conv: in_proj -> b * x -> causal depthwise conv over [conv_state | x] -> c * conv -> out_proj -ggml_tensor * clip_graph_lfm3a_gen::build_lfm_conv(ggml_tensor * cur, const clip_layer & layer, int il, +ggml_tensor * mtmd_graph_lfm3a_gen::build_lfm_conv(ggml_tensor * cur, const mtmd_layer & layer, int il, ggml_tensor * conv_state, ggml_tensor ** conv_state_out) { const int64_t n_embd_c = cur->ne[0]; const int64_t n_pos = cur->ne[1]; @@ -110,7 +110,7 @@ ggml_tensor * clip_graph_lfm3a_gen::build_lfm_conv(ggml_tensor * cur, const clip return build_mm(layer.conv_pw2_w, cur); } -ggml_tensor * clip_graph_lfm3a_gen::build_gen_code() { +ggml_tensor * mtmd_graph_lfm3a_gen::build_gen_code() { ggml_tensor * embd_w = model.gen_code_embd_w; // [n_embd, n_vocab, n_codebooks], tied to the output head const int64_t n_cb = embd_w->ne[2]; const int64_t n_vocab = embd_w->ne[1]; @@ -169,7 +169,7 @@ ggml_tensor * clip_graph_lfm3a_gen::build_gen_code() { return fb; } -ggml_tensor * clip_graph_lfm3a_gen::build_gen_wav() { +ggml_tensor * mtmd_graph_lfm3a_gen::build_gen_wav() { const auto & c2w = model.c2w; const int64_t n_cb = model.gen_code_embd_w->ne[2]; const int64_t n_fs = hparams.wav_frame_stack; @@ -201,7 +201,7 @@ ggml_tensor * clip_graph_lfm3a_gen::build_gen_wav() { cur = ggml_repeat_4d(ctx0, ggml_reshape_3d(ctx0, cur, n_embd_dec, 1, n_frames), n_embd_dec, n_fs, n_frames, 1); cur = ggml_reshape_2d(ctx0, cur, n_embd_dec, n_pos); - // positions continue across calls from the "pos" slot, see clip_encode() + // positions continue across calls from the "pos" slot, see mtmd_subcontext::encode() ggml_tensor * inp_pos = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_pos); ggml_set_name(inp_pos, "positions"); ggml_set_input(inp_pos); @@ -250,8 +250,8 @@ ggml_tensor * clip_graph_lfm3a_gen::build_gen_wav() { return cur; } -ggml_cgraph * clip_graph_lfm3a_gen::build() { - ggml_tensor * out = gen_process == CLIP_GEN_PROCESS_GEN_WAV ? build_gen_wav() : build_gen_code(); +ggml_cgraph * mtmd_graph_lfm3a_gen::build() { + ggml_tensor * out = gen_process == MTMD_SUBCTX_GEN_PROCESS_GEN_WAV ? build_gen_wav() : build_gen_code(); ggml_build_forward_expand(gf, out); return gf; } diff --git a/tools/mtmd/models/lfm3a.cpp b/tools/mtmd/models/lfm3a.cpp index 4882fc42fe60..03128a298363 100644 --- a/tools/mtmd/models/lfm3a.cpp +++ b/tools/mtmd/models/lfm3a.cpp @@ -2,7 +2,7 @@ // streaming encoder state: base position, conv left context, short-conv taps and attention window // a zero state gives the same result as encoding the whole clip -std::vector list_lfm3a_state_slots(const clip_hparams & hparams, const clip_model & model) { +std::vector list_lfm3a_state_slots(const mtmd_hparams & hparams, const mtmd_model & model) { std::vector slots; const int64_t n_mel = hparams.n_mel_bins; @@ -27,7 +27,7 @@ std::vector list_lfm3a_state_slots(const clip_hparams & hparams, } // LFM3-Audio encoder: conv downsampler (8x in time) -> causal LFM2 stack -> MLP projector -ggml_cgraph * clip_graph_lfm3a::build() { +ggml_cgraph * mtmd_graph_lfm3a::build() { const norm_type norm_t = NORM_TYPE_RMS; const float rope_theta = hparams.rope_theta; const int64_t prefix = hparams.attn_window_size - 1; diff --git a/tools/mtmd/models/lfm3vl.cpp b/tools/mtmd/models/lfm3vl.cpp index e37390976107..6ad64ecb14e9 100644 --- a/tools/mtmd/models/lfm3vl.cpp +++ b/tools/mtmd/models/lfm3vl.cpp @@ -1,6 +1,6 @@ #include "models.h" -ggml_cgraph * clip_graph_lfm3vl::build() { +ggml_cgraph * mtmd_graph_lfm3vl::build() { GGML_ASSERT(model.patch_embeddings_0 != nullptr); GGML_ASSERT(model.lfm3vl_embedding_norm_w != nullptr); GGML_ASSERT(model.class_embedding == nullptr); diff --git a/tools/mtmd/models/ling3vl.cpp b/tools/mtmd/models/ling3vl.cpp index fb9eb7c7493c..c3b8ed96862a 100644 --- a/tools/mtmd/models/ling3vl.cpp +++ b/tools/mtmd/models/ling3vl.cpp @@ -1,6 +1,6 @@ #include "models.h" -ggml_cgraph * clip_graph_ling3vl::build() { +ggml_cgraph * mtmd_graph_ling3vl::build() { // same vision tower as qwen3vl, but the merger is norm-only (no fc1/fc2) and // the projector MLP lives at the top level (mm.0 / mm.2) GGML_ASSERT(model.class_embedding == nullptr); @@ -57,7 +57,7 @@ ggml_cgraph * clip_graph_ling3vl::build() { ggml_tensor * inpL = build_vit( inp, n_pos, norm_t, hparams.ffn_op, learned_pos_embd, - [&](ggml_tensor * c, const clip_layer &) { + [&](ggml_tensor * c, const mtmd_layer &) { return ggml_rope_multi( ctx0, c, positions, nullptr, d_head/2, mrope_sections, GGML_ROPE_TYPE_VISION, 32768, 10000, 1, 0, 1, 32, 1); diff --git a/tools/mtmd/models/llama4.cpp b/tools/mtmd/models/llama4.cpp index 01af54bbab7e..60bfd5e2ebfa 100644 --- a/tools/mtmd/models/llama4.cpp +++ b/tools/mtmd/models/llama4.cpp @@ -1,6 +1,6 @@ #include "models.h" -ggml_cgraph * clip_graph_llama4::build() { +ggml_cgraph * mtmd_graph_llama4::build() { GGML_ASSERT(model.class_embedding != nullptr); GGML_ASSERT(model.position_embeddings != nullptr); @@ -31,7 +31,7 @@ ggml_cgraph * clip_graph_llama4::build() { inp = ggml_concat(ctx0, inp, model.class_embedding, 1); // build ViT with 2D position embeddings - auto add_pos = [&](ggml_tensor * cur, const clip_layer &) { + auto add_pos = [&](ggml_tensor * cur, const mtmd_layer &) { // first half is X axis and second half is Y axis // ref: https://github.com/huggingface/transformers/blob/40a493c7ed4f19f08eadb0639cf26d49bfa5e180/src/transformers/models/llama4/modeling_llama4.py#L1312 // ref: https://github.com/Blaizzy/mlx-vlm/blob/a57156aa87b33cca6e5ee6cfc14dd4ef8f611be6/mlx_vlm/models/llama4/vision.py#L441 diff --git a/tools/mtmd/models/llava.cpp b/tools/mtmd/models/llava.cpp index 47efe68bd835..8e8c7638bd84 100644 --- a/tools/mtmd/models/llava.cpp +++ b/tools/mtmd/models/llava.cpp @@ -2,7 +2,7 @@ // this graph is used by llava, granite and glm // due to having embedding_stack (used by granite), we cannot reuse build_vit -ggml_cgraph * clip_graph_llava::build() { +ggml_cgraph * mtmd_graph_llava::build() { const int batch_size = 1; const int n_pos = n_patches + (model.class_embedding ? 1 : 0); @@ -167,10 +167,8 @@ ggml_cgraph * clip_graph_llava::build() { embeddings = ggml_add(ctx0, embeddings, model.mm_0_b); embeddings = ggml_gelu(ctx0, embeddings); - if (model.mm_2_w) { - embeddings = build_mm(model.mm_2_w, embeddings); - embeddings = ggml_add(ctx0, embeddings, model.mm_2_b); - } + embeddings = build_mm(model.mm_2_w, embeddings); + embeddings = ggml_add(ctx0, embeddings, model.mm_2_b); } else if (proj_type == PROJECTOR_TYPE_MLP_NORM) { embeddings = build_mm(model.mm_0_w, embeddings); diff --git a/tools/mtmd/models/magevl.cpp b/tools/mtmd/models/magevl.cpp new file mode 100644 index 000000000000..393f19aaf54d --- /dev/null +++ b/tools/mtmd/models/magevl.cpp @@ -0,0 +1,61 @@ +#include "models.h" + +ggml_cgraph * mtmd_graph_magevl::build() { + GGML_ASSERT(hparams.n_merge == 2); + + const int n_pair = d_head / 2; + const int unit = n_pair / 16; + + // pairs are grouped as t, h, w at conversion + int sections[4] = {4 * unit, 6 * unit, 6 * unit, 0}; + + ggml_tensor * positions = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_patches * 4); + ggml_set_name(positions, "positions"); + ggml_set_input(positions); + + // a pair uses 2 angles: even outputs come from one rope, odd outputs from the other + auto add_pos = [&](ggml_tensor * cur, const mtmd_layer &) { + ggml_tensor * freqs_a = ggml_view_1d(ctx0, model.rope_freqs, n_pair, 0); + ggml_tensor * freqs_b = ggml_view_1d(ctx0, model.rope_freqs, n_pair, ggml_row_size(model.rope_freqs->type, n_pair)); + + ggml_tensor * rope_a = ggml_rope_multi(ctx0, cur, positions, freqs_a, d_head, sections, + GGML_ROPE_TYPE_MROPE, 32768, 1.0f, 1.0f, 0.0f, 1.0f, 32.0f, 1.0f); + ggml_tensor * rope_b = ggml_rope_multi(ctx0, cur, positions, freqs_b, d_head, sections, + GGML_ROPE_TYPE_MROPE, 32768, 1.0f, 1.0f, 0.0f, 1.0f, 32.0f, 1.0f); + + ggml_tensor * even = ggml_view_3d(ctx0, rope_a, n_pair, n_head, n_patches, + rope_a->nb[1], rope_a->nb[2], 0); + ggml_tensor * odd = ggml_view_3d(ctx0, rope_b, n_pair, n_head, n_patches, + rope_b->nb[1], rope_b->nb[2], ggml_row_size(rope_b->type, n_pair)); + + return ggml_concat(ctx0, even, odd, 0); + }; + + // ggml_conv_2d rounds patches to F16 + ggml_tensor * inp_raw = build_inp_raw(); + ggml_tensor * patches = ggml_im2col(ctx0, model.patch_embeddings_0, inp_raw, + patch_size, patch_size, 0, 0, 1, 1, true, GGML_TYPE_F32); + const int64_t n_patch_dim = patches->ne[0]; + ggml_tensor * inp = ggml_mul_mat(ctx0, + ggml_reshape_2d(ctx0, model.patch_embeddings_0, n_patch_dim, n_embd), + ggml_reshape_2d(ctx0, patches, n_patch_dim, n_patches)); + + // reorder into 2x2 blocks + inp = ggml_reshape_4d(ctx0, inp, n_embd * 2, n_patches_x / 2, 2, n_patches_y / 2); + inp = ggml_permute(ctx0, inp, 0, 2, 1, 3); + inp = ggml_cont_3d(ctx0, inp, n_embd, n_patches, 1); + + ggml_tensor * cur = build_vit(inp, n_patches, NORM_TYPE_NORMAL, hparams.ffn_op, nullptr, add_pos); + + // multimodal projection + cur = ggml_reshape_3d(ctx0, cur, n_embd * 4, n_patches / 4, 1); + cur = build_ffn(cur, + model.mm_0_w, model.mm_0_b, + nullptr, nullptr, + model.mm_1_w, model.mm_1_b, + hparams.ffn_op, -1); + + ggml_build_forward_expand(gf, cur); + + return gf; +} diff --git a/tools/mtmd/models/mimo-audio.cpp b/tools/mtmd/models/mimo-audio.cpp index 0415bb60d702..92be15fc9ca9 100644 --- a/tools/mtmd/models/mimo-audio.cpp +++ b/tools/mtmd/models/mimo-audio.cpp @@ -1,6 +1,6 @@ #include "models.h" -ggml_cgraph * clip_graph_mimo_audio::build() { +ggml_cgraph * mtmd_graph_mimo_audio::build() { ggml_tensor * inp = build_inp_raw(1); // [n_frames, n_mel, 1] ggml_tensor * cur = ggml_conv_1d_ph(ctx0, model.conv1d_1_w, inp, 1, 1); @@ -43,7 +43,7 @@ ggml_cgraph * clip_graph_mimo_audio::build() { } }; - auto add_pos = [&](ggml_tensor * x, const clip_layer &) { + auto add_pos = [&](ggml_tensor * x, const mtmd_layer &) { return ggml_rope_ext(ctx0, x, inp_pos, nullptr, d_head, GGML_ROPE_TYPE_NEOX, 0, hparams.rope_theta, 1.0f, 0.0f, 1.0f, 0.0f, 0.0f); }; diff --git a/tools/mtmd/models/mimovl.cpp b/tools/mtmd/models/mimovl.cpp index 0cde450bf9c9..6f2b8934f63e 100644 --- a/tools/mtmd/models/mimovl.cpp +++ b/tools/mtmd/models/mimovl.cpp @@ -1,8 +1,8 @@ #include "models.h" -ggml_tensor * clip_graph_mimovl::build_mm(ggml_tensor * w, ggml_tensor * x) const { +ggml_tensor * mtmd_graph_mimovl::build_mm(ggml_tensor * w, ggml_tensor * x) const { ggml_tensor * cur = ggml_mul_mat(ctx0, w, x); - ggml_prec_set_acc(cur, GGML_PREC_F32); + ggml_prec_set_acc(cur, GGML_PREC_BF16); return cur; } @@ -19,7 +19,7 @@ ggml_tensor * clip_graph_mimovl::build_mm(ggml_tensor * w, ggml_tensor * x) cons // 4. 1D banded sliding window (|q-k| > window_size -> -inf) as a // single 2D mask broadcast across heads. // 5. Per-block MLP biases. -ggml_cgraph * clip_graph_mimovl::build() { +ggml_cgraph * mtmd_graph_mimovl::build() { GGML_ASSERT(model.patch_embeddings_0 != nullptr); GGML_ASSERT(model.patch_embeddings_1 != nullptr); GGML_ASSERT(model.class_embedding == nullptr); diff --git a/tools/mtmd/models/minicpmv.cpp b/tools/mtmd/models/minicpmv.cpp index 0e0fc0381148..08ede0dd13d5 100644 --- a/tools/mtmd/models/minicpmv.cpp +++ b/tools/mtmd/models/minicpmv.cpp @@ -1,6 +1,6 @@ #include "models.h" -ggml_cgraph * clip_graph_minicpmv::build() { +ggml_cgraph * mtmd_graph_minicpmv::build() { GGML_ASSERT(model.class_embedding == nullptr); const int n_pos = n_patches; const int n_embd_proj = n_mmproj_embd; @@ -113,7 +113,7 @@ ggml_cgraph * clip_graph_minicpmv::build() { return gf; } -ggml_cgraph * clip_graph_minicpmv4_6::build() { +ggml_cgraph * mtmd_graph_minicpmv4_6::build() { const bool is_4x = hparams.n_merge == 2; const int n_pos = n_patches; const int half_h = n_patches_y / 2; diff --git a/tools/mtmd/models/minimax-m3.cpp b/tools/mtmd/models/minimax-m3.cpp index 256e531057d4..2ac437bf1d87 100644 --- a/tools/mtmd/models/minimax-m3.cpp +++ b/tools/mtmd/models/minimax-m3.cpp @@ -1,6 +1,6 @@ #include "models.h" -ggml_tensor * clip_graph_minimax_m3::apply_rope( +ggml_tensor * mtmd_graph_minimax_m3::apply_rope( ggml_tensor * x, ggml_tensor * pos_h, ggml_tensor * pos_w) { const int dh = (int) x->ne[0]; const int axd = 2 * ((2 * (dh / 2) / 3) / 2); @@ -20,7 +20,7 @@ ggml_tensor * clip_graph_minimax_m3::apply_rope( return x; } -ggml_cgraph * clip_graph_minimax_m3::build() { +ggml_cgraph * mtmd_graph_minimax_m3::build() { GGML_ASSERT(model.patch_bias == nullptr); GGML_ASSERT(model.class_embedding == nullptr); GGML_ASSERT(model.patch_embeddings_0 && model.patch_embeddings_1); @@ -54,7 +54,7 @@ ggml_cgraph * clip_graph_minimax_m3::build() { ggml_tensor * inpL = build_vit( inp, n_pos, NORM_TYPE_NORMAL, FFN_GELU_ERF, nullptr, - [&](ggml_tensor * c, const clip_layer &) { + [&](ggml_tensor * c, const mtmd_layer &) { return apply_rope(c, pos_h, pos_w); }); diff --git a/tools/mtmd/models/mobilenetv5.cpp b/tools/mtmd/models/mobilenetv5.cpp index 1c42218d2aff..e127cb31e85b 100644 --- a/tools/mtmd/models/mobilenetv5.cpp +++ b/tools/mtmd/models/mobilenetv5.cpp @@ -2,7 +2,7 @@ // Helpers for MobileNetV5 Blocks // RMS Norm 2D - normalizes over channels for each spatial position -ggml_tensor * clip_graph_mobilenetv5::rms_norm_2d(ggml_tensor * inp, ggml_tensor * weight, float eps) { +ggml_tensor * mtmd_graph_mobilenetv5::rms_norm_2d(ggml_tensor * inp, ggml_tensor * weight, float eps) { // inp: [W, H, C, B] ggml_tensor * cur = ggml_permute(ctx0, inp, 2, 1, 0, 3); @@ -20,7 +20,7 @@ ggml_tensor * clip_graph_mobilenetv5::rms_norm_2d(ggml_tensor * inp, ggml_tensor } // Conv2dSame padding - asymmetric SAME padding like PyTorch/TF -ggml_tensor* clip_graph_mobilenetv5::pad_same_2d(ggml_tensor* inp, int kernel_h, int kernel_w, int stride_h, int stride_w, int dilation_h, int dilation_w) { +ggml_tensor* mtmd_graph_mobilenetv5::pad_same_2d(ggml_tensor* inp, int kernel_h, int kernel_w, int stride_h, int stride_w, int dilation_h, int dilation_w) { const int64_t ih = inp->ne[1]; // height const int64_t iw = inp->ne[0]; // width @@ -54,7 +54,7 @@ ggml_tensor* clip_graph_mobilenetv5::pad_same_2d(ggml_tensor* inp, int kernel_h, // Edge Residual Block (Stage 0) -ggml_tensor * clip_graph_mobilenetv5::build_edge_residual(ggml_tensor * inp, const mobilenetv5_block & block, int stride) { +ggml_tensor * mtmd_graph_mobilenetv5::build_edge_residual(ggml_tensor * inp, const mobilenetv5_block & block, int stride) { ggml_tensor * cur = inp; // 1. Expansion Conv (3x3) @@ -88,7 +88,7 @@ ggml_tensor * clip_graph_mobilenetv5::build_edge_residual(ggml_tensor * inp, con } // Universal Inverted Residual Block (Stage 1+) -ggml_tensor * clip_graph_mobilenetv5::build_inverted_residual(ggml_tensor * inp, const mobilenetv5_block & block, int stride) { +ggml_tensor * mtmd_graph_mobilenetv5::build_inverted_residual(ggml_tensor * inp, const mobilenetv5_block & block, int stride) { ggml_tensor * cur = inp; // 1. Depthwise Start (Optional) @@ -149,7 +149,7 @@ ggml_tensor * clip_graph_mobilenetv5::build_inverted_residual(ggml_tensor * inp, } // Attention Block (MQA) -ggml_tensor * clip_graph_mobilenetv5::build_mobilenet_attn(ggml_tensor * inp, const mobilenetv5_block & block) { +ggml_tensor * mtmd_graph_mobilenetv5::build_mobilenet_attn(ggml_tensor * inp, const mobilenetv5_block & block) { ggml_tensor * cur = inp; // Norm @@ -245,7 +245,7 @@ ggml_tensor * clip_graph_mobilenetv5::build_mobilenet_attn(ggml_tensor * inp, co return cur; } -ggml_cgraph * clip_graph_mobilenetv5::build() { +ggml_cgraph * mtmd_graph_mobilenetv5::build() { ggml_tensor * inp = build_inp_raw(); // 1. Stem - Conv2dSame(3, 64, kernel_size=(3, 3), stride=(2, 2)) diff --git a/tools/mtmd/models/models.h b/tools/mtmd/models/models.h index c75004d43725..4f68462ba9ee 100644 --- a/tools/mtmd/models/models.h +++ b/tools/mtmd/models/models.h @@ -1,6 +1,6 @@ #pragma once -#include "../clip-graph.h" +#include "../mtmd-graph.h" #include #include @@ -13,66 +13,87 @@ * We encourage human contributors to ensure the quality and reliability of the codebase. */ -struct clip_graph_siglip : clip_graph { - clip_graph_siglip(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {} +struct mtmd_graph_siglip : mtmd_graph { + mtmd_graph_siglip(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {} ggml_cgraph * build() override; }; -struct clip_graph_gemma4v : clip_graph { - clip_graph_gemma4v(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {} +struct mtmd_graph_gemma4v : mtmd_graph { + mtmd_graph_gemma4v(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {} ggml_cgraph * build() override; ggml_tensor * build_mm(ggml_tensor * w, ggml_tensor * x) const override; bool support_batch() const override { return true; } }; -struct clip_graph_gemma4uv : clip_graph { - clip_graph_gemma4uv(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {} +struct mtmd_graph_gemma4uv : mtmd_graph { + mtmd_graph_gemma4uv(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {} ggml_cgraph * build() override; }; -struct clip_graph_pixtral : clip_graph { - clip_graph_pixtral(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {} +struct mtmd_graph_nemotron_transcribe : mtmd_graph { + const mtmd_subctx_enc_state * state; // nullptr during warmup + mtmd_graph_nemotron_transcribe(mtmd_subcontext * ctx, const mtmd_image_f32 & img, const mtmd_subctx_enc_state * state) + : mtmd_graph(ctx, img), state(state) {} + ggml_cgraph * build() override; + ggml_tensor * build_spk_encoder(ggml_tensor * cur, ggml_tensor * pos); + + // one recording: encodes its windows in turn, carrying the speaker state between them + static bool encode_windows(mtmd_subcontext * ctx, const mtmd_model & model, mtmd_subctx_encode_params * params); + + // NeMo ParallelExpertEncoder online defaults, in output frames: 40 s windows with 4 s of context on each side + static constexpr int window_core = 500; + static constexpr int window_left = 50; + static constexpr int window_right = 50; +}; + +struct mtmd_graph_pixtral : mtmd_graph { + mtmd_graph_pixtral(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {} ggml_cgraph * build() override; }; -struct clip_graph_deepseek4v : clip_graph { - clip_graph_deepseek4v(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {} +struct mtmd_graph_deepseek4v : mtmd_graph { + mtmd_graph_deepseek4v(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {} ggml_cgraph * build() override; }; -struct clip_graph_qwen2vl : clip_graph { - clip_graph_qwen2vl(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {} +struct mtmd_graph_qwen2vl : mtmd_graph { + mtmd_graph_qwen2vl(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {} ggml_cgraph * build() override; }; -struct clip_graph_qwen3vl : clip_graph_qwen2vl { - clip_graph_qwen3vl(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph_qwen2vl(ctx, img) {} +struct mtmd_graph_qwen3vl : mtmd_graph_qwen2vl { + mtmd_graph_qwen3vl(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph_qwen2vl(ctx, img) {} ggml_cgraph * build() override; }; -struct clip_graph_sensenova_u1 : clip_graph { - clip_graph_sensenova_u1(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {} +struct mtmd_graph_magevl : mtmd_graph { + mtmd_graph_magevl(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {} ggml_cgraph * build() override; }; -struct clip_graph_lfm3vl : clip_graph { - clip_graph_lfm3vl(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {} +struct mtmd_graph_sensenova_u1 : mtmd_graph { + mtmd_graph_sensenova_u1(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {} ggml_cgraph * build() override; }; -struct clip_graph_ling3vl : clip_graph_qwen3vl { - clip_graph_ling3vl(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph_qwen3vl(ctx, img) {} +struct mtmd_graph_lfm3vl : mtmd_graph { + mtmd_graph_lfm3vl(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {} ggml_cgraph * build() override; }; -struct clip_graph_minimax_m3 : clip_graph { - clip_graph_minimax_m3(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {} +struct mtmd_graph_ling3vl : mtmd_graph_qwen3vl { + mtmd_graph_ling3vl(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph_qwen3vl(ctx, img) {} + ggml_cgraph * build() override; +}; + +struct mtmd_graph_minimax_m3 : mtmd_graph { + mtmd_graph_minimax_m3(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {} ggml_cgraph * build() override; ggml_tensor * apply_rope(ggml_tensor * x, ggml_tensor * pos_h, ggml_tensor * pos_w); }; -struct clip_graph_mimovl : clip_graph { - clip_graph_mimovl(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {} +struct mtmd_graph_mimovl : mtmd_graph { + mtmd_graph_mimovl(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {} ggml_cgraph * build() override; // Force F32 mat-mul accumulation to avoid F16 overflow in the FFN down-proj // when the mmproj is stored in F16 (the source weights are BF16; downcasting @@ -80,154 +101,205 @@ struct clip_graph_mimovl : clip_graph { ggml_tensor * build_mm(ggml_tensor * w, ggml_tensor * x) const override; }; -struct clip_graph_step3vl : clip_graph { - clip_graph_step3vl(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {} +struct mtmd_graph_step3vl : mtmd_graph { + mtmd_graph_step3vl(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {} ggml_cgraph * build() override; }; -struct clip_graph_youtuvl : clip_graph { - clip_graph_youtuvl(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {} +struct mtmd_graph_youtuvl : mtmd_graph { + mtmd_graph_youtuvl(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {} ggml_cgraph * build() override; }; -struct clip_graph_yasa2 : clip_graph { - clip_graph_yasa2(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {} +struct mtmd_graph_yasa2 : mtmd_graph { + mtmd_graph_yasa2(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {} ggml_cgraph * build() override; ggml_tensor * layer_norm_channels(ggml_tensor * inp, ggml_tensor * w, ggml_tensor * b, float eps = 1e-6f); ggml_tensor * convnext_grn(ggml_tensor * inp, ggml_tensor * w, ggml_tensor * b); }; -struct clip_graph_minicpmv : clip_graph { - clip_graph_minicpmv(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {} +struct mtmd_graph_minicpmv : mtmd_graph { + mtmd_graph_minicpmv(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {} ggml_cgraph * build() override; }; -struct clip_graph_minicpmv4_6 : clip_graph { - clip_graph_minicpmv4_6(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {} +struct mtmd_graph_minicpmv4_6 : mtmd_graph { + mtmd_graph_minicpmv4_6(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {} ggml_cgraph * build() override; }; -struct clip_graph_internvl : clip_graph { - clip_graph_internvl(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {} +struct mtmd_graph_internvl : mtmd_graph { + mtmd_graph_internvl(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {} ggml_cgraph * build() override; bool support_batch() const override { return true; } }; -struct clip_graph_nemotron_v2_vl : clip_graph { - clip_graph_nemotron_v2_vl(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {} +struct mtmd_graph_nemotron_v2_vl : mtmd_graph { + mtmd_graph_nemotron_v2_vl(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {} ggml_cgraph * build() override; }; -struct clip_graph_nemotron_omni : clip_graph { - clip_graph_nemotron_omni(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {} +struct mtmd_graph_nemotron_omni : mtmd_graph { + mtmd_graph_nemotron_omni(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {} ggml_cgraph * build() override; ggml_tensor * build_inp_with_temporal_merge(); }; -struct clip_graph_llama4 : clip_graph { - clip_graph_llama4(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {} +struct mtmd_graph_llama4 : mtmd_graph { + mtmd_graph_llama4(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {} ggml_cgraph * build() override; }; -struct clip_graph_kimivl : clip_graph { - clip_graph_kimivl(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {} +struct mtmd_graph_kimivl : mtmd_graph { + mtmd_graph_kimivl(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {} ggml_cgraph * build() override; }; -struct clip_graph_paddleocr : clip_graph { - clip_graph_paddleocr(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {} +struct mtmd_graph_paddleocr : mtmd_graph { + mtmd_graph_paddleocr(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {} ggml_cgraph * build() override; }; -struct clip_graph_dotsocr : clip_graph { - clip_graph_dotsocr(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {} +struct mtmd_graph_dotsocr : mtmd_graph { + mtmd_graph_dotsocr(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {} ggml_cgraph * build() override; }; -struct clip_graph_dots3note_a : clip_graph { - clip_graph_dots3note_a(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {} +struct mtmd_graph_dots3note_a : mtmd_graph { + mtmd_graph_dots3note_a(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {} ggml_cgraph * build() override; }; -struct clip_graph_cogvlm : clip_graph { - clip_graph_cogvlm(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {} +struct mtmd_graph_cogvlm : mtmd_graph { + mtmd_graph_cogvlm(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {} ggml_cgraph * build() override; }; -struct clip_graph_llava : clip_graph { - clip_graph_llava(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {} +struct mtmd_graph_llava : mtmd_graph { + mtmd_graph_llava(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {} ggml_cgraph * build() override; }; -struct clip_graph_whisper_enc : clip_graph { - clip_graph_whisper_enc(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {} +struct mtmd_graph_whisper_enc : mtmd_graph { + mtmd_graph_whisper_enc(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {} ggml_cgraph * build() override; }; // causal whisper-like encoder with a sliding window KV cache (voxtral realtime) -struct clip_graph_voxtral_rt : clip_graph { - clip_enc_state * state; // nullptr during warmup, the cache is then a graph input - clip_graph_voxtral_rt(clip_ctx * ctx, const clip_image_f32 & img, clip_enc_state * state) : clip_graph(ctx, img), state(state) {} +struct mtmd_graph_voxtral_rt : mtmd_graph { + mtmd_subctx_enc_state * state; // nullptr during warmup, the cache is then a graph input + mtmd_graph_voxtral_rt(mtmd_subcontext * ctx, const mtmd_image_f32 & img, mtmd_subctx_enc_state * state) : mtmd_graph(ctx, img), state(state) {} ggml_cgraph * build() override; }; -struct clip_graph_deepseekocr : clip_graph { - clip_graph_deepseekocr(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {} +struct mtmd_graph_deepseekocr : mtmd_graph { + mtmd_graph_deepseekocr(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {} ggml_cgraph * build() override; ggml_tensor * build_sam(ggml_tensor * inp); // build the SAM model bool support_batch() const override { return true; } }; -struct clip_graph_deepseekocr2 : clip_graph_deepseekocr { - clip_graph_deepseekocr2(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph_deepseekocr(ctx, img) {} +struct mtmd_graph_deepseekocr2 : mtmd_graph_deepseekocr { + mtmd_graph_deepseekocr2(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph_deepseekocr(ctx, img) {} ggml_cgraph * build() override; // reuses build_sam() from base bool support_batch() const override { return true; } }; -struct clip_graph_conformer : clip_graph { - clip_graph_conformer(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {} +struct mtmd_graph_conformer : mtmd_graph { + mtmd_graph_conformer(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {} ggml_cgraph * build() override; }; -struct clip_graph_lfm3a : clip_graph { - clip_graph_lfm3a(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {} +// builders shared by the chatterbox gen and spkenc graphs, defined in chatterbox-gen.cpp +struct mtmd_graph_chatterbox_base : mtmd_graph { + using mtmd_graph::mtmd_graph; + + protected: + ggml_tensor * cbx_linear(ggml_tensor * w, ggml_tensor * b, ggml_tensor * x) const; + ggml_tensor * cbx_conv1d(ggml_tensor * k, ggml_tensor * b, ggml_tensor * x, + int stride, int pad_l, int pad_r) const; + ggml_tensor * cbx_conv1d_dil(ggml_tensor * k, ggml_tensor * b, ggml_tensor * x, + int pad, int dil) const; +}; + +struct mtmd_graph_chatterbox_spkenc : mtmd_graph_chatterbox_base { + mtmd_graph_chatterbox_spkenc(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph_chatterbox_base(ctx, img) {} + ggml_cgraph * build() override; + + private: + ggml_tensor * bn1d(const mtmd_chatterbox::bn & n, ggml_tensor * x); + ggml_tensor * bn2d_relu(const mtmd_chatterbox::bn & n, ggml_tensor * x); + ggml_tensor * res2d(const mtmd_chatterbox::spk_res2d & r, ggml_tensor * x, int stride); + ggml_tensor * cam_layer(const mtmd_chatterbox::spk_cam_layer & l, ggml_tensor * x, int dil, ggml_tensor * segfix); +}; + +struct mtmd_graph_chatterbox : mtmd_graph_chatterbox_base { + mtmd_subctx_gen_process gen_process = MTMD_SUBCTX_GEN_PROCESS_GEN_CODE; + int n_tokens = 0; + int n_prompt_mel = 0; + int vocode_n_mel = 0; + int vocode_n_stft = 0; + mtmd_graph_chatterbox(mtmd_subcontext * ctx, const mtmd_image_f32 & img, mtmd_subctx_gen_process gen_process, int n_tokens, + int n_prompt_mel, int vocode_n_mel, int vocode_n_stft) + : mtmd_graph_chatterbox_base(ctx, img), gen_process(gen_process), n_tokens(n_tokens), n_prompt_mel(n_prompt_mel), + vocode_n_mel(vocode_n_mel), vocode_n_stft(vocode_n_stft) {} ggml_cgraph * build() override; + + private: + ggml_tensor * enc_layer(const mtmd_chatterbox::enc_layer & l, ggml_tensor * x, ggml_tensor * pos, int T); + ggml_tensor * causal_block(const mtmd_chatterbox::causal_block & b, ggml_tensor * x); + ggml_tensor * resnet(const mtmd_chatterbox::resnet & r, ggml_tensor * x, ggml_tensor * temb); + ggml_tensor * tfm_block(const mtmd_chatterbox::tfm_block & b, ggml_tensor * x); + ggml_tensor * estimator(ggml_tensor * x_noise, ggml_tensor * mu, ggml_tensor * spks, + ggml_tensor * cond, ggml_tensor * temb, int T); + ggml_tensor * hift_resblock(const mtmd_chatterbox::hift_res & r, ggml_tensor * x); + ggml_cgraph * build_s3tok(int T); + ggml_cgraph * build_vocoder(int n_mel, int n_stft); }; -struct clip_graph_granite_speech : clip_graph { - clip_graph_granite_speech(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {} +struct mtmd_graph_lfm3a : mtmd_graph { + mtmd_graph_lfm3a(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {} ggml_cgraph * build() override; }; -struct clip_graph_muse_spark_audio : clip_graph { - clip_graph_muse_spark_audio(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {} +struct mtmd_graph_granite_speech : mtmd_graph { + mtmd_graph_granite_speech(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {} ggml_cgraph * build() override; }; -struct clip_graph_gemma4a : clip_graph { - clip_graph_gemma4a(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {} +struct mtmd_graph_muse_spark_audio : mtmd_graph { + mtmd_graph_muse_spark_audio(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {} + ggml_cgraph * build() override; +}; + +struct mtmd_graph_gemma4a : mtmd_graph { + // attention_chunk_size, attention_context_left - 1 + static constexpr int64_t chunk_size = 12; + static constexpr int64_t max_past = 12; + + mtmd_graph_gemma4a(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {} ggml_cgraph * build() override; ggml_tensor * build_mm(ggml_tensor * w, ggml_tensor * x) const override; }; -struct clip_graph_gemma4ua : clip_graph { - clip_graph_gemma4ua(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {} +struct mtmd_graph_gemma4ua : mtmd_graph { + mtmd_graph_gemma4ua(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {} ggml_cgraph * build() override; }; -struct clip_graph_glm4v : clip_graph { - clip_graph_glm4v(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {} +struct mtmd_graph_glm4v : mtmd_graph { + mtmd_graph_glm4v(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {} ggml_cgraph * build() override; }; -struct clip_graph_hunyuanvl : clip_graph { - clip_graph_hunyuanvl(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {} +struct mtmd_graph_hunyuanvl : mtmd_graph { + mtmd_graph_hunyuanvl(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {} ggml_cgraph * build() override; }; -struct clip_graph_mobilenetv5 : clip_graph { - clip_graph_mobilenetv5(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {} +struct mtmd_graph_mobilenetv5 : mtmd_graph { + mtmd_graph_mobilenetv5(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {} ggml_cgraph * build() override; ggml_tensor * rms_norm_2d( @@ -259,34 +331,34 @@ struct clip_graph_mobilenetv5 : clip_graph { const mobilenetv5_block & block); }; -struct clip_graph_qwen3a : clip_graph { - clip_graph_qwen3a(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {} +struct mtmd_graph_qwen3a : mtmd_graph { + mtmd_graph_qwen3a(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {} ggml_cgraph * build() override; }; -struct clip_graph_mimo_audio : clip_graph { - clip_graph_mimo_audio(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {} +struct mtmd_graph_mimo_audio : mtmd_graph { + mtmd_graph_mimo_audio(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {} ggml_cgraph * build() override; }; -struct clip_graph_qwen3tts_spkenc : clip_graph { - clip_graph_qwen3tts_spkenc(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {} +struct mtmd_graph_qwen3tts_spkenc : mtmd_graph { + mtmd_graph_qwen3tts_spkenc(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {} ggml_cgraph * build() override; ggml_tensor * conv1d_same(ggml_tensor * x, ggml_tensor * w, ggml_tensor * b, int dilation) const; - ggml_tensor * res2net(ggml_tensor * x, const clip_layer & layer, int dilation, int scale) const; - ggml_tensor * se_block(ggml_tensor * x, const clip_layer & layer) const; - ggml_tensor * se_res2net_block(ggml_tensor * x, const clip_layer & layer, int dilation, int scale) const; + ggml_tensor * res2net(ggml_tensor * x, const mtmd_layer & layer, int dilation, int scale) const; + ggml_tensor * se_block(ggml_tensor * x, const mtmd_layer & layer) const; + ggml_tensor * se_res2net_block(ggml_tensor * x, const mtmd_layer & layer, int dilation, int scale) const; ggml_tensor * attentive_stats_pool(ggml_tensor * x) const; }; -struct clip_graph_qwen3tts_gen : clip_graph { - clip_graph_qwen3tts_gen(clip_ctx * ctx, const clip_image_f32 & img, clip_gen_process_type gen_process, int top_k, float top_p) - : clip_graph(ctx, img), gen_process(gen_process), top_k(top_k), top_p(top_p) {} +struct mtmd_graph_qwen3tts_gen : mtmd_graph { + mtmd_graph_qwen3tts_gen(mtmd_subcontext * ctx, const mtmd_image_f32 & img, mtmd_subctx_gen_process gen_process, int top_k, float top_p) + : mtmd_graph(ctx, img), gen_process(gen_process), top_k(top_k), top_p(top_p) {} ggml_cgraph * build() override; // which sub-graph build() constructs, fixed at graph-build time - clip_gen_process_type gen_process; + mtmd_subctx_gen_process gen_process; // sampling params, fixed at graph-build time (GEN_CODE only) int top_k; @@ -296,9 +368,9 @@ struct clip_graph_qwen3tts_gen : clip_graph { // code_gen: backbone hidden state + sampled code0 -> 16 RVQ codes // MTP-style code predictor, one token per codebook // - struct code_gen : clip_graph { - code_gen(const clip_graph & parent, int top_k, float top_p) - : clip_graph(parent), top_k(top_k), top_p(top_p) {} + struct code_gen : mtmd_graph { + code_gen(const mtmd_graph & parent, int top_k, float top_p) + : mtmd_graph(parent), top_k(top_k), top_p(top_p) {} ggml_cgraph * build() override { GGML_ABORT("call prefill()/step() instead"); } int top_k; @@ -313,7 +385,7 @@ struct clip_graph_qwen3tts_gen : clip_graph { ggml_tensor * layer_forward( ggml_tensor * cur, - const clip_layer & layer, + const mtmd_layer & layer, ggml_tensor * inp_pos, ggml_tensor * kq_mask, ggml_tensor *& k_cache_layer, @@ -341,8 +413,8 @@ struct clip_graph_qwen3tts_gen : clip_graph { // // code2wav: RVQ codes -> raw PCM (quantizer + pre_conv + pre_transformer + upsample + DAC). // - struct code2wav : clip_graph { - code2wav(const clip_graph & parent) : clip_graph(parent) {} + struct code2wav : mtmd_graph { + code2wav(const mtmd_graph & parent) : mtmd_graph(parent) {} ggml_cgraph * build() override { GGML_ABORT("call decode() instead"); } // state_in: previous call's persisted state, by slot name (see list_c2w_state_slots()) @@ -357,9 +429,12 @@ struct clip_graph_qwen3tts_gen : clip_graph { ggml_tensor * snake(ggml_tensor * x, ggml_tensor * alpha, ggml_tensor * beta) const; ggml_tensor * quant_decode(ggml_tensor * inp_codes) const; - ggml_tensor * tfm_layer_forward(ggml_tensor * cur, const clip_layer & layer, int il) const; - ggml_tensor * convnext_block(ggml_tensor * x, const clip_code2wav::upsample_block & blk, const std::string & state_prefix) const; - ggml_tensor * dac_res_unit(ggml_tensor * x, const clip_code2wav::dac_res & res, int dilation, const std::string & state_name) const; + ggml_tensor * tfm_layer_forward(ggml_tensor * cur, const mtmd_layer & layer, int il) const; + ggml_tensor * convnext_block(ggml_tensor * x, const mtmd_code2wav::upsample_block & blk, const std::string & state_prefix) const; + ggml_tensor * dac_res_unit(ggml_tensor * x, const mtmd_code2wav::dac_res & res, int dilation, const std::string & state_name) const; + + // drama3: erf GELU in ConvNeXt, not the tanh approximation + bool gelu_erf = false; // inp_codes [1, n_codes] I32 -> this frame's audio samples [n_samples] F32, clamped to [-1, 1] ggml_tensor * decode(ggml_tensor * inp_codes) const; @@ -370,8 +445,8 @@ struct clip_graph_qwen3tts_gen : clip_graph { // pocket-tts: SEANet convolution stack, shared by the voice encoder and the mimi decoder. // stateless unless state_in is populated: convs then pad instead of carrying left-context. // -struct clip_graph_pockettts_seanet : clip_graph { - clip_graph_pockettts_seanet(const clip_graph & parent) : clip_graph(parent) {} +struct mtmd_graph_pockettts_seanet : mtmd_graph { + mtmd_graph_pockettts_seanet(const mtmd_graph & parent) : mtmd_graph(parent) {} ggml_cgraph * build() override { GGML_ABORT("call encode()/decode() instead"); } // per-call streaming state, keyed by slot name (see list_pockettts_state_slots) @@ -382,7 +457,7 @@ struct clip_graph_pockettts_seanet : clip_graph { bool pad_replicate = false, const std::string & state_name = "") const; ggml_tensor * conv_transpose1d(ggml_tensor * x, ggml_tensor * w, ggml_tensor * b, int stride, const std::string & state_name = "") const; - ggml_tensor * res_unit(ggml_tensor * x, const clip_seanet::stage & stage, int dilation, + ggml_tensor * res_unit(ggml_tensor * x, const mtmd_seanet::stage & stage, int dilation, const std::string & state_prefix = "") const; // x: [T, C] -> [T / hop, dim] @@ -392,11 +467,11 @@ struct clip_graph_pockettts_seanet : clip_graph { }; // mimi encoder + speaker_proj: reference waveform -> voice conditioning rows -struct clip_graph_pockettts_spkenc : clip_graph { - clip_graph_pockettts_spkenc(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {} +struct mtmd_graph_pockettts_spkenc : mtmd_graph { + mtmd_graph_pockettts_spkenc(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {} ggml_cgraph * build() override; - ggml_tensor * tfm_layer_forward(ggml_tensor * cur, const clip_layer & layer, ggml_tensor * inp_pos, ggml_tensor * kq_mask, int il) const; + ggml_tensor * tfm_layer_forward(ggml_tensor * cur, const mtmd_layer & layer, ggml_tensor * inp_pos, ggml_tensor * kq_mask, int il) const; }; // @@ -404,18 +479,18 @@ struct clip_graph_pockettts_spkenc : clip_graph { // GEN_CODE = flow-matching decoder + end-of-speech head, one latent per call // GEN_WAV = mimi decoder, a window of latents -> PCM // -struct clip_graph_pockettts_gen : clip_graph { - clip_graph_pockettts_gen(clip_ctx * ctx, const clip_image_f32 & img, clip_gen_process_type gen_process, int n_step, int n_frames) - : clip_graph(ctx, img), gen_process(gen_process), n_step(n_step), n_frames(n_frames) {} +struct mtmd_graph_pockettts_gen : mtmd_graph { + mtmd_graph_pockettts_gen(mtmd_subcontext * ctx, const mtmd_image_f32 & img, mtmd_subctx_gen_process gen_process, int n_step, int n_frames) + : mtmd_graph(ctx, img), gen_process(gen_process), n_step(n_step), n_frames(n_frames) {} ggml_cgraph * build() override; - clip_gen_process_type gen_process; + mtmd_subctx_gen_process gen_process; int n_step; // lsd_decode steps, fixed at graph-build time int n_frames; // GEN_WAV only: number of latents to decode // AdaLN modulation: x * (1 + scale) + shift ggml_tensor * modulate(ggml_tensor * x, ggml_tensor * shift, ggml_tensor * scale) const; - ggml_tensor * time_embed(const clip_flow_net::time_embd & te, float t) const; + ggml_tensor * time_embed(const mtmd_flow_net::time_embd & te, float t) const; ggml_tensor * flow_forward(ggml_tensor * cond, ggml_tensor * x, float s, float t) const; }; @@ -424,16 +499,47 @@ struct c2w_state_slot { std::string name; int64_t ne0; int64_t ne1; + bool in_place = false; // graph writes the slot directly, no state_in/state_out copy }; -std::vector list_c2w_state_slots(const clip_hparams & hparams, const clip_model & model); +std::vector list_c2w_state_slots(const mtmd_hparams & hparams, const mtmd_model & model); + +// drama3: tower KV slots, then list_c2w_state_slots() +std::vector list_drama3_state_slots(const mtmd_hparams & hparams, const mtmd_model & model); // same, for the streaming mimi decoder (pocket-tts GEN_WAV) -std::vector list_pockettts_state_slots(const clip_hparams & hparams, const clip_model & model); +std::vector list_pockettts_state_slots(const mtmd_hparams & hparams, const mtmd_model & model); + +// +// drama3 generation: +// GEN_CODE = one codebook tower row, each layer attends over the whole sequence (KV in gen state) +// GEN_WAV = Fish DAC decoder, frames -> PCM +// +struct mtmd_graph_drama3_gen : mtmd_graph { + mtmd_graph_drama3_gen(mtmd_subcontext * ctx, const mtmd_image_f32 & img, mtmd_subctx_gen_process gen_process, mtmd_subctx_gen_state * gen_state, + int n_frames, int top_k, float top_p, float temp) + : mtmd_graph(ctx, img), gen_process(gen_process), gen_state(gen_state), n_frames(n_frames), top_k(top_k), top_p(top_p), temp(temp) {} + ggml_cgraph * build() override; + + // tower KV rows, shared by the prompt and the generated frames + static constexpr int n_pos_max = 4096; + + mtmd_subctx_gen_process gen_process; + mtmd_subctx_gen_state * gen_state; // nullptr on buffer reserve + int n_frames; // GEN_WAV only + int top_k; + float top_p; + float temp; + + ggml_cgraph * build_tower(); + ggml_cgraph * build_codec(); + ggml_tensor * tower_layer(ggml_tensor * x, int il, ggml_tensor * code, ggml_tensor * mask, ggml_tensor * extra, + ggml_tensor * inp_pos, ggml_tensor * k_slot, ggml_tensor * v_slot, int pos); +}; -struct clip_graph_soprano : clip_graph { - clip_graph_soprano(clip_ctx * ctx, const clip_image_f32 & img, int n_frames) : clip_graph(ctx, img), n_frames(n_frames) {} +struct mtmd_graph_soprano : mtmd_graph { + mtmd_graph_soprano(mtmd_subcontext * ctx, const mtmd_image_f32 & img, int n_frames) : mtmd_graph(ctx, img), n_frames(n_frames) {} ggml_cgraph * build() override; - static void decode_spectrum(const clip_model & model, const std::vector & spectrum, std::vector & pcm); + static void decode_spectrum(const mtmd_model & model, const std::vector & spectrum, std::vector & pcm); // SopranoDecoder default static constexpr int upscale = 4; int n_frames; @@ -444,54 +550,57 @@ struct clip_graph_soprano : clip_graph { // GEN_CODE = one MaskGIT pass of the depthformer, or only the feedback embedding once all codes are known // GEN_WAV = liquid-codec decoder, a window of frames -> polar STFT frames // -struct clip_graph_lfm3a_gen : clip_graph { - clip_graph_lfm3a_gen(clip_ctx * ctx, const clip_image_f32 & img, clip_gen_process_type gen_process, bool all_known, int n_frames) - : clip_graph(ctx, img), gen_process(gen_process), all_known(all_known), n_frames(n_frames) {} +struct mtmd_graph_lfm3a_gen : mtmd_graph { + mtmd_graph_lfm3a_gen(mtmd_subcontext * ctx, const mtmd_image_f32 & img, mtmd_subctx_gen_process gen_process, bool all_known, int n_frames) + : mtmd_graph(ctx, img), gen_process(gen_process), all_known(all_known), n_frames(n_frames) {} ggml_cgraph * build() override; - clip_gen_process_type gen_process; + mtmd_subctx_gen_process gen_process; bool all_known; // GEN_CODE only int n_frames; // GEN_WAV only ggml_tensor * build_gen_code(); ggml_tensor * build_gen_wav(); - ggml_tensor * build_lfm_layer(ggml_tensor * inpL, const clip_layer & layer, float eps, int il, + ggml_tensor * build_lfm_layer(ggml_tensor * inpL, const mtmd_layer & layer, float eps, int il, const std::function & op); - ggml_tensor * build_lfm_attn(ggml_tensor * cur, const clip_layer & layer, int n_head, int n_head_kv, + ggml_tensor * build_lfm_attn(ggml_tensor * cur, const mtmd_layer & layer, int n_head, int n_head_kv, float eps, float rope_theta, int il, ggml_tensor * inp_pos, ggml_tensor * kq_mask, ggml_tensor * k_cache, ggml_tensor * v_cache, ggml_tensor ** k_all, ggml_tensor ** v_all); - ggml_tensor * build_lfm_conv(ggml_tensor * cur, const clip_layer & layer, int il, + ggml_tensor * build_lfm_conv(ggml_tensor * cur, const mtmd_layer & layer, int il, ggml_tensor * conv_state, ggml_tensor ** conv_state_out); }; // state slots of the streaming codec decoder (lfm3a_gen GEN_WAV) -std::vector list_lfm3a_gen_state_slots(const clip_hparams & hparams, const clip_model & model); +std::vector list_lfm3a_gen_state_slots(const mtmd_hparams & hparams, const mtmd_model & model); // state slots of the streaming LFM3-Audio encoder (lfm3a) -std::vector list_lfm3a_state_slots(const clip_hparams & hparams, const clip_model & model); +std::vector list_lfm3a_state_slots(const mtmd_hparams & hparams, const mtmd_model & model); + +// state slots of the streaming Gemma 4 audio encoder (gemma4a) +std::vector list_gemma4a_state_slots(const mtmd_hparams & hparams, const mtmd_model & model); -struct clip_graph_kimik25 : clip_graph { - clip_graph_kimik25(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {} +struct mtmd_graph_kimik25 : mtmd_graph { + mtmd_graph_kimik25(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {} ggml_cgraph * build() override; ggml_tensor * resize_position_embeddings_3d(uint32_t interpolation_mode); }; -struct clip_graph_parakeet : clip_graph { - clip_graph_parakeet(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {} +struct mtmd_graph_parakeet : mtmd_graph { + mtmd_graph_parakeet(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {} ggml_cgraph * build() override; }; -struct clip_graph_exaone4_5 : clip_graph { - clip_graph_exaone4_5(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {} +struct mtmd_graph_exaone4_5 : mtmd_graph { + mtmd_graph_exaone4_5(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {} ggml_cgraph * build() override; }; -struct clip_graph_granite4_vision : clip_graph { - clip_graph_granite4_vision(clip_ctx * ctx, const clip_image_f32 & img) - : clip_graph(ctx, img), +struct mtmd_graph_granite4_vision : mtmd_graph { + mtmd_graph_granite4_vision(mtmd_subcontext * ctx, const mtmd_image_f32 & img) + : mtmd_graph(ctx, img), anyres(img.anyres), n_tiles(img.ny() / img.nx()), tile_side(img.nx() / patch_size) {} @@ -500,7 +609,7 @@ struct clip_graph_granite4_vision : clip_graph { private: // the input image is a stack of tiles on the Y axis: [overview, tile(0,0), tile(0,1), ...] - const clip_image_f32::anyres_info anyres; + const mtmd_image_f32::anyres_info anyres; const int n_tiles; const int tile_side; // patches per tile side @@ -515,7 +624,7 @@ struct clip_graph_granite4_vision : clip_graph { ggml_tensor * build_anyres_assembly(ggml_tensor * cur, int out_side); }; -struct clip_graph_muse_glimmer : clip_graph { - clip_graph_muse_glimmer(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {} +struct mtmd_graph_muse_glimmer : mtmd_graph { + mtmd_graph_muse_glimmer(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {} ggml_cgraph * build() override; }; diff --git a/tools/mtmd/models/muse-glimmer.cpp b/tools/mtmd/models/muse-glimmer.cpp index a397c1b223b3..869954f99cfd 100644 --- a/tools/mtmd/models/muse-glimmer.cpp +++ b/tools/mtmd/models/muse-glimmer.cpp @@ -5,13 +5,13 @@ // adapter MLP + LLM's vision_projection. // // Several quantities are precomputed on host and fed as named graph inputs (filled in -// clip.cpp set_input, PROJECTOR_TYPE_MUSE_GLIMMER branch): +// mtmd-subcontext.cpp set_input, PROJECTOR_TYPE_MUSE_GLIMMER branch): // muse_glimmer_pos_w/_h [n_tok] i32 : 1-indexed RoPE positions (sparse-permuted order) // muse_glimmer_sp_perm [n_tok] i32 : window grouping permutation (applied after ln_pre) // muse_glimmer_inv_perm [n_tok] i32 : inverse of sp_perm (applied after blocks) // muse_glimmer_ds_perm [n_tok] i32 : pixel-shuffle gather (original order) // attn_mask [n_tok, n_tok] f32 (f16 with flash attn) : block-diagonal window mask (sparse layers) -ggml_cgraph * clip_graph_muse_glimmer::build() { +ggml_cgraph * mtmd_graph_muse_glimmer::build() { const int ds = hparams.n_merge; // downsample factor (2) const int sf = hparams.muse_glimmer_sparse_factor; // 4 const int n_tok = n_patches; @@ -50,7 +50,7 @@ ggml_cgraph * clip_graph_muse_glimmer::build() { } // 2D RoPE: first half of head_dim uses width pos, second half uses height pos - auto add_pos = [&](ggml_tensor * cur, const clip_layer &) { + auto add_pos = [&](ggml_tensor * cur, const mtmd_layer &) { return build_rope_2d(ctx0, cur, pos_w, pos_h, rope_base, false); }; diff --git a/tools/mtmd/models/muse-spark-audio.cpp b/tools/mtmd/models/muse-spark-audio.cpp index d5340d5f9faa..20e770ef8511 100644 --- a/tools/mtmd/models/muse-spark-audio.cpp +++ b/tools/mtmd/models/muse-spark-audio.cpp @@ -2,7 +2,7 @@ // Muse Spark audio tower: a causal conv pyramid over the raw waveform, no attention and no spectrogram // each stage folds `patch` samples into the channel dim, projects, then runs `depth` residual blocks (depthwise causal conv, RMS norm, GELU MLP) -ggml_cgraph * clip_graph_muse_spark_audio::build() { +ggml_cgraph * mtmd_graph_muse_spark_audio::build() { const int kernel = hparams.audio_conv_kernel_size; const int n_stage = (int) hparams.audio_patch_sizes.size(); diff --git a/tools/mtmd/models/nemotron-omni.cpp b/tools/mtmd/models/nemotron-omni.cpp index c4b1bff40621..fc0f2b2ce175 100644 --- a/tools/mtmd/models/nemotron-omni.cpp +++ b/tools/mtmd/models/nemotron-omni.cpp @@ -1,6 +1,6 @@ #include "models.h" -ggml_tensor * clip_graph_nemotron_omni::build_inp_with_temporal_merge() { +ggml_tensor * mtmd_graph_nemotron_omni::build_inp_with_temporal_merge() { if (n_batch == 1 && !img_batch->is_video) { return build_inp(); } @@ -22,7 +22,7 @@ ggml_tensor * clip_graph_nemotron_omni::build_inp_with_temporal_merge() { return ggml_cont(ctx0, ggml_transpose(ctx0, inp)); } -ggml_cgraph * clip_graph_nemotron_omni::build() { +ggml_cgraph * mtmd_graph_nemotron_omni::build() { GGML_ASSERT(model.class_embedding != nullptr); GGML_ASSERT(model.position_embeddings != nullptr); diff --git a/tools/mtmd/models/nemotron-transcribe.cpp b/tools/mtmd/models/nemotron-transcribe.cpp new file mode 100644 index 000000000000..b8fe37749690 --- /dev/null +++ b/tools/mtmd/models/nemotron-transcribe.cpp @@ -0,0 +1,343 @@ +#include "models.h" + +#include +#include +#include +#include + +// NeMo ParallelExpertEncoder: ASR and speaker (Sortformer) transformers, speaker activity fused into the ASR states + +// a speaker is active in an 80 ms frame where its prediction exceeds this (speaker_activity_threshold) +static constexpr float SPK_ACTIVITY_THR = 0.5f; + +// NeMo streaming Sortformer: PEE online defaults and the checkpoint's sortformer_modules config +static constexpr int SPK_FIFO_LEN = 40; +static constexpr int SPK_CACHE_LEN = 188; +static constexpr int SPK_SIL_FRAMES = 1; +static constexpr float SPK_PRED_SCORE_THR = 0.25f; +static constexpr float SPK_BOOST_LATEST = 0.05f; +static constexpr float SPK_STRONG_BOOST_RATE = 0.75f; +static constexpr float SPK_WEAK_BOOST_RATE = 1.5f; +static constexpr float SPK_MIN_POS_RATE = 0.5f; + +// _compress_spkcache runs here on exp(score): its additive log terms become factors and -inf becomes 0 +static constexpr float SPK_SCORE_OFFSET = 2.0f; // -log(0.5) in _get_log_pred_scores +static constexpr float SPK_STRONG_BOOST = 4.0f; // _boost_topk_scores, scale_factor 2 +static constexpr float SPK_WEAK_BOOST = 2.0f; // _boost_topk_scores, scale_factor 1 +static constexpr float SPK_SPEECH_THR = 0.5f; // _disable_low_scores: a speaker's frames at or below it are not cached + +// the speaker expert has its own layers, width and head count, which build_vit does not take; full rotary +ggml_tensor * mtmd_graph_nemotron_transcribe::build_spk_encoder(ggml_tensor * cur, ggml_tensor * pos) { + const int n_head = hparams.spk_n_head; + const int64_t n_embd = cur->ne[0]; + const int64_t n_pos = cur->ne[1]; + const int64_t d_head = n_embd / n_head; + + cur = build_norm(cur, model.spk_pre_ln_w, model.spk_pre_ln_b, NORM_TYPE_NORMAL, eps, -1); + for (int il = 0; il < (int) model.spk_layers.size(); il++) { + const auto & layer = model.spk_layers[il]; + ggml_tensor * x = build_norm(cur, layer.ln_1_w, layer.ln_1_b, NORM_TYPE_NORMAL, eps, il); + + ggml_tensor * qkv = build_mm(layer.qkv_w, x); + const size_t es = ggml_element_size(qkv); + ggml_tensor * q = ggml_view_3d(ctx0, qkv, d_head, n_head, n_pos, d_head * es, qkv->nb[1], 0); + ggml_tensor * k = ggml_view_3d(ctx0, qkv, d_head, n_head, n_pos, d_head * es, qkv->nb[1], n_embd * es); + ggml_tensor * v = ggml_view_3d(ctx0, qkv, d_head, n_head, n_pos, d_head * es, qkv->nb[1], 2 * n_embd * es); + q = ggml_rope_ext(ctx0, q, pos, nullptr, d_head, GGML_ROPE_TYPE_NEOX, 0, 10000.0f, 1.0f, 0.0f, 1.0f, 0.0f, 0.0f); + k = ggml_rope_ext(ctx0, k, pos, nullptr, d_head, GGML_ROPE_TYPE_NEOX, 0, 10000.0f, 1.0f, 0.0f, 1.0f, 0.0f, 0.0f); + + x = build_attn(layer.o_w, layer.o_b, q, k, v, nullptr, 1.0f / sqrtf((float) d_head), il); + cur = ggml_add(ctx0, cur, x); + + x = build_norm(cur, layer.ln_2_w, layer.ln_2_b, NORM_TYPE_NORMAL, eps, il); + x = build_ffn(x, layer.ff_up_w, layer.ff_up_b, nullptr, nullptr, layer.ff_down_w, layer.ff_down_b, FFN_GELU_ERF, il); + cur = ggml_add(ctx0, cur, x); + } + + return build_norm(cur, model.spk_post_ln_w, model.spk_post_ln_b, NORM_TYPE_NORMAL, eps, -1); +} + +ggml_cgraph * mtmd_graph_nemotron_transcribe::build() { + const int sub = hparams.subsampling_factor; + const int64_t n_mel = hparams.n_mel_bins; + const int64_t n_mel_frames = img.nx(); + const int64_t n_pos = (n_mel_frames + sub - 1) / sub; + GGML_ASSERT(img.ny() == n_mel); + + // warmup: the largest graph, with a full speaker cache and FIFO + const int64_t n_prev = state == nullptr ? SPK_CACHE_LEN + SPK_FIFO_LEN + : (int64_t) state->spk_prev.size() / hparams.spk_n_embd; + + // [n_frames, n_mel] log-mel: raw for the speaker expert, normalized over the recording (encode_windows) for the ASR one + ggml_tensor * inp = build_inp_raw(1); + ggml_tensor * inp_norm = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_mel_frames, n_mel); + ggml_set_name(inp_norm, "inp_norm"); + ggml_set_input(inp_norm); + auto stack_mel = [&](ggml_tensor * mel) { + return build_stack(ggml_cont(ctx0, ggml_transpose(ctx0, mel)), sub, n_mel); + }; + + // positions over [cache | FIFO | window]; the ASR expert takes the first n_pos + ggml_tensor * pos = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_prev + n_pos); + ggml_set_name(pos, "positions"); + ggml_set_input(pos); + + ggml_tensor * spk_prev = nullptr; + if (n_prev > 0) { + spk_prev = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, hparams.spk_n_embd, n_prev); + ggml_set_name(spk_prev, "spk_prev"); + ggml_set_input(spk_prev); + } + + ggml_tensor * asr_pos = ggml_view_1d(ctx0, pos, n_pos, 0); + auto add_pos = [&](ggml_tensor * cur, const mtmd_layer &) { + return ggml_rope_ext(ctx0, cur, asr_pos, nullptr, hparams.audio_rope_dims, GGML_ROPE_TYPE_NEOX, 0, 10000.0f, 1.0f, + 0.0f, 1.0f, 0.0f, 0.0f); + }; + ggml_tensor * asr = build_vit(build_mm(model.inp_proj_w, stack_mel(inp_norm)), n_pos, NORM_TYPE_NORMAL, FFN_GELU_ERF, nullptr, + add_pos); + cb(asr, "asr_out", -1); + + // speaker expert over [cache | FIFO | window]; the window's rows and predictions feed the next window's state + ggml_tensor * chunk = build_mm(model.spk_inp_proj_w, stack_mel(inp)); + ggml_set_name(chunk, "spk_chunk"); + ggml_set_output(chunk); + ggml_tensor * spk = spk_prev ? ggml_concat(ctx0, spk_prev, chunk, 1) : chunk; + const int64_t n_spk_pos = n_prev + n_pos; + spk = build_spk_encoder(spk, pos); + cb(spk, "spk_out", -1); + + // Sortformer head, 10 ms frames + ggml_tensor * cur = ggml_add(ctx0, build_mm(model.spk_head_proj_w, spk), model.spk_head_proj_b); + const int64_t d_tf = cur->ne[0]; + cur = ggml_cont(ctx0, ggml_transpose(ctx0, cur)); + cur = ggml_conv_1d(ctx0, model.spk_head_up_w, cur, 1, 1, 1); // [n_spk_pos, sub*d_tf] + cur = ggml_add(ctx0, cur, ggml_reshape_2d(ctx0, model.spk_head_up_b, 1, sub * d_tf)); + cur = ggml_cont(ctx0, ggml_transpose(ctx0, cur)); + cur = ggml_reshape_2d(ctx0, cur, d_tf, sub * n_spk_pos); + cur = ggml_add(ctx0, build_mm(model.spk_head_fc1_w, ggml_relu(ctx0, cur)), model.spk_head_fc1_b); + cur = ggml_add(ctx0, build_mm(model.spk_head_fc2_w, ggml_relu(ctx0, cur)), model.spk_head_fc2_b); + ggml_tensor * probs = ggml_sigmoid(ctx0, cur); // [n_spk, sub*n_spk_pos] + cb(probs, "spk_probs", -1); + + // 80 ms predictions + ggml_tensor * preds = ggml_pool_2d(ctx0, probs, GGML_OP_POOL_AVG, 1, sub, 1, sub, 0, 0); // [n_spk, n_spk_pos] + ggml_set_name(preds, "spk_preds"); + ggml_set_output(preds); + + // fusion, on the window's rows + preds = ggml_view_2d(ctx0, preds, preds->ne[0], n_pos, preds->nb[1], n_prev * preds->nb[1]); + ggml_tensor * act = ggml_step(ctx0, ggml_scale_bias(ctx0, preds, 1.0f, -SPK_ACTIVITY_THR)); + act = build_norm(act, model.fusion_spk_norm_w, model.fusion_spk_norm_b, NORM_TYPE_NORMAL, eps, -1); + cur = build_norm(asr, model.fusion_asr_norm_w, model.fusion_asr_norm_b, NORM_TYPE_NORMAL, eps, -1); + cur = ggml_add(ctx0, cur, build_mm(model.fusion_spk_kernel, act)); + cb(cur, "fused", -1); + + cur = ggml_add(ctx0, build_mm(model.mm_fc_w, cur), model.mm_fc_b); + cb(cur, "projected", -1); + + ggml_build_forward_expand(gf, cur); + + return gf; +} + +// indices of the k largest of v[0, n), ties to the lower index +static std::vector spk_topk(const float * v, int n, int k) { + std::vector idx(n); + std::iota(idx.begin(), idx.end(), 0); + k = std::clamp(k, 0, n); + std::partial_sort(idx.begin(), idx.begin() + k, idx.end(), [&](int a, int b) { + return v[a] > v[b] || (v[a] == v[b] && a < b); + }); + idx.resize(k); + return idx; +} + +// NeMo SortformerModules._compress_spkcache: keep the SPK_CACHE_LEN highest-scoring rows, speaker by speaker +static void spk_compress_cache(std::vector & emb, std::vector & preds, int n, int d, int n_spk, + const std::vector & sil_emb) { + const int per_spk = std::max(0, SPK_CACHE_LEN / n_spk - SPK_SIL_FRAMES); + const int strong = (int) std::floor(per_spk * SPK_STRONG_BOOST_RATE); + const int weak = (int) std::floor(per_spk * SPK_WEAK_BOOST_RATE); + const int min_pos = (int) std::floor(per_spk * SPK_MIN_POS_RATE); + const float disabled = 0.0f; + const float latest = std::exp(SPK_BOOST_LATEST); + + // speaker-major: each speaker's n rows, then its silence rows (+inf) + const int nt = n + SPK_SIL_FRAMES; + std::vector scores((size_t) n_spk * nt, std::numeric_limits::infinity()); + for (int s = 0; s < n_spk; s++) { + float * sc = scores.data() + (size_t) s * nt; + int n_pos = 0; + for (int t = 0; t < n; t++) { + const float * pt = preds.data() + (size_t) t * n_spk; + float prod = 1.0f; + for (int j = 0; j < n_spk; j++) { + if (j != s) { + prod *= std::max(1.0f - pt[j], SPK_PRED_SCORE_THR); + } + } + prod *= std::max(pt[s], SPK_PRED_SCORE_THR); + sc[t] = pt[s] > SPK_SPEECH_THR ? prod * SPK_SCORE_OFFSET : disabled; + n_pos += sc[t] > 1.0f; + } + if (n_pos >= min_pos) { + for (int t = 0; t < n; t++) { + if (!(sc[t] > 1.0f)) { + sc[t] = disabled; + } + } + } + for (int t = SPK_CACHE_LEN; t < n; t++) { + sc[t] *= latest; + } + for (int t : spk_topk(sc, n, strong)) { + sc[t] *= SPK_STRONG_BOOST; + } + for (int t : spk_topk(sc, n, weak)) { + sc[t] *= SPK_WEAK_BOOST; + } + } + + std::vector pick = spk_topk(scores.data(), (int) scores.size(), SPK_CACHE_LEN); + const int placeholder = (int) scores.size(); + for (int & i : pick) { + if (scores[i] == disabled) { + i = placeholder; + } + } + std::sort(pick.begin(), pick.end()); + + std::vector out_emb((size_t) SPK_CACHE_LEN * d); + std::vector out_preds((size_t) SPK_CACHE_LEN * n_spk, 0.0f); + for (int i = 0; i < SPK_CACHE_LEN; i++) { + const int t = pick[i] == placeholder ? -1 : pick[i] % nt; + float * e = out_emb.data() + (size_t) i * d; + if (t < 0 || t >= n) { + std::copy(sil_emb.begin(), sil_emb.end(), e); + } else { + std::copy_n(emb.data() + (size_t) t * d, d, e); + std::copy_n(preds.data() + (size_t) t * n_spk, n_spk, out_preds.data() + (size_t) i * n_spk); + } + } + emb.swap(out_emb); + preds.swap(out_preds); +} + +// speaker cache, with its predictions, and FIFO of one recording +struct spk_stream { + std::vector cache, cache_preds; + std::vector fifo; +}; + +// every update has a full window, which overflows the FIFO and the cache (NeMo's update period of 300 never applies) +static_assert(mtmd_graph_nemotron_transcribe::window_core - SPK_FIFO_LEN > SPK_CACHE_LEN); + +// NeMo SortformerModules.streaming_update: core rows join the FIFO, its overflow moves to the cache, which is compressed +static void update_spk_state(spk_stream & ss, const mtmd_subctx_enc_state & st, const mtmd_model & model, int64_t core_begin, + int64_t n_core) { + const int64_t d = model.hparams.spk_n_embd; + const int64_t n_spk = model.spk_head_fc2_w->ne[1]; + const int64_t n_cache = (int64_t) ss.cache.size() / d; + const int64_t n_fifo = (int64_t) ss.fifo.size() / d; + const int64_t n_rows = (int64_t) st.spk_chunk.size() / d; + GGML_ASSERT((n_cache + n_fifo + n_rows) * n_spk == (int64_t) st.spk_preds.size() && core_begin + n_core <= n_rows); + const float * preds = st.spk_preds.data(); + const float * core_preds = preds + (n_cache + n_fifo + core_begin) * n_spk; + + std::vector fifo_preds(preds + n_cache * n_spk, preds + (n_cache + n_fifo) * n_spk); + ss.fifo.insert(ss.fifo.end(), st.spk_chunk.begin() + core_begin * d, st.spk_chunk.begin() + (core_begin + n_core) * d); + fifo_preds.insert(fifo_preds.end(), core_preds, core_preds + n_core * n_spk); + + const int64_t n_pop = n_fifo + n_core - SPK_FIFO_LEN; + ss.cache.insert(ss.cache.end(), ss.fifo.begin(), ss.fifo.begin() + n_pop * d); + ss.cache_preds.insert(ss.cache_preds.end(), fifo_preds.begin(), fifo_preds.begin() + n_pop * n_spk); + ss.fifo.erase(ss.fifo.begin(), ss.fifo.begin() + n_pop * d); + spk_compress_cache(ss.cache, ss.cache_preds, (int) (ss.cache.size() / d), (int) d, (int) n_spk, model.spk_sil_emb); +} + +// NeMo ParallelExpertEncoder._forward_online: windows with left/right context, one graph each, the speaker state carried +bool mtmd_graph_nemotron_transcribe::encode_windows(mtmd_subcontext * ctx, const mtmd_model & model, mtmd_subctx_encode_params * params) { + GGML_ASSERT(params->imgs->entries.size() == 1 && params->out_embd != nullptr); + const mtmd_image_f32 & rec = params->imgs->entries[0]; + const auto & buf = rec.get_ro_buf(); + const int64_t n_frames = rec.nx(); + const int64_t n_rows = rec.ny(); + const int64_t n_embd = ctx->model.n_mmproj_embd(); + const int sub = model.hparams.subsampling_factor; + const int64_t core = (int64_t) window_core * sub; + const int64_t left = (int64_t) window_left * sub; + const int64_t right = (int64_t) window_right * sub; + const int64_t n_win = std::max(1, (n_frames + core - 1) / core); + GGML_ASSERT((int64_t) buf.size() == n_frames * n_rows); + + auto & out = *params->out_embd; + GGML_ASSERT(out.size() == (size_t) n_embd * ctx->model.n_output_tokens(&rec)); + + // the ASR expert's input: per-feature normalization over the whole recording (NeMo normalize_batch, per_feature) + std::vector mean(n_rows), denom(n_rows); + for (int64_t r = 0; r < n_rows; r++) { + const float * row = buf.data() + r * n_frames; + double sum = 0.0; + double sq_diff_sum = 0.0; + for (int64_t t = 0; t < n_frames; t++) { + sum += (double) row[t]; + } + mean[r] = sum / n_frames; + for (int64_t t = 0; t < n_frames; t++) { + const double diff = (double) row[t] - mean[r]; + sq_diff_sum += diff * diff; + } + denom[r] = std::sqrt(sq_diff_sum / (n_frames - 1.0)) + 1e-5; + } + + spk_stream ss; + mtmd_subctx_enc_state st; + size_t n_out = 0; + for (int64_t w = 0; w < n_win; w++) { + const int64_t core_begin = w * core; + const int64_t core_end = std::min(core_begin + core, n_frames); + const int64_t win_begin = std::max(core_begin - left, 0); + const int64_t win_end = std::min(core_end + right, n_frames); + const int64_t win_len = win_end - win_begin; + + std::vector data((size_t) win_len * n_rows); + st.mel_norm.resize(data.size()); + for (int64_t r = 0; r < n_rows; r++) { + const float * src = buf.data() + r * n_frames + win_begin; + std::copy_n(src, win_len, data.data() + r * win_len); + for (int64_t t = 0; t < win_len; t++) { + st.mel_norm[r * win_len + t] = (float) ((src[t] - mean[r]) / denom[r]); + } + } + mtmd_image_f32_batch batch; + batch.is_audio = true; + batch.entries.emplace_back(); + batch.entries[0].set_size({(int) win_len, (int) n_rows}, false, true); + batch.entries[0].cpy_buf(data); + + std::vector embd((size_t) n_embd * ctx->model.n_output_tokens(&batch.entries[0])); + mtmd_subctx_encode_params wp = *params; + wp.imgs = &batch; + wp.out_embd = &embd; + wp.enc_state = &st; + if (!ctx->encode(&wp)) { + return false; + } + + // output frames of the core, rounded up as NeMo's _asr_output_frame_boundary + const int64_t n_left = (core_begin - win_begin) / sub; + const int64_t n_core = (core_end - core_begin + sub - 1) / sub; + GGML_ASSERT((n_out + n_core) * n_embd <= out.size()); + std::copy_n(embd.data() + n_left * n_embd, n_core * n_embd, out.data() + n_out * n_embd); + n_out += n_core; + + if (w + 1 < n_win) { + update_spk_state(ss, st, model, n_left, n_core); + st.spk_prev = ss.cache; + st.spk_prev.insert(st.spk_prev.end(), ss.fifo.begin(), ss.fifo.end()); + } + } + GGML_ASSERT(n_out * n_embd == out.size()); + return true; +} diff --git a/tools/mtmd/models/nemotron-v2-vl.cpp b/tools/mtmd/models/nemotron-v2-vl.cpp index 03094be1b27d..6e644ea97b4e 100644 --- a/tools/mtmd/models/nemotron-v2-vl.cpp +++ b/tools/mtmd/models/nemotron-v2-vl.cpp @@ -1,6 +1,6 @@ #include "models.h" -ggml_cgraph * clip_graph_nemotron_v2_vl::build() { +ggml_cgraph * mtmd_graph_nemotron_v2_vl::build() { GGML_ASSERT(model.class_embedding != nullptr); GGML_ASSERT(model.position_embeddings != nullptr); diff --git a/tools/mtmd/models/paddleocr.cpp b/tools/mtmd/models/paddleocr.cpp index 0e6b3f4b6f62..1ad18693516b 100644 --- a/tools/mtmd/models/paddleocr.cpp +++ b/tools/mtmd/models/paddleocr.cpp @@ -1,6 +1,6 @@ #include "models.h" -ggml_cgraph * clip_graph_paddleocr::build() { +ggml_cgraph * mtmd_graph_paddleocr::build() { GGML_ASSERT(img.nx() % (patch_size * model.hparams.n_merge) == 0); GGML_ASSERT(img.ny() % (patch_size * model.hparams.n_merge) == 0); @@ -13,7 +13,7 @@ ggml_cgraph * clip_graph_paddleocr::build() { ggml_set_name(positions, "positions"); ggml_set_input(positions); - auto add_pos = [&](ggml_tensor * cur, const clip_layer &) { + auto add_pos = [&](ggml_tensor * cur, const mtmd_layer &) { return ggml_rope_multi( ctx0, cur, positions, nullptr, d_head/2, mrope_sections, GGML_ROPE_TYPE_VISION, diff --git a/tools/mtmd/models/parakeet.cpp b/tools/mtmd/models/parakeet.cpp index a6d7f0739943..4a2377b4cf87 100644 --- a/tools/mtmd/models/parakeet.cpp +++ b/tools/mtmd/models/parakeet.cpp @@ -4,7 +4,7 @@ static constexpr int PARAKEET_LOCAL_ATTN_THRESHOLD = 8192; static constexpr int PARAKEET_LOCAL_ATTN_WINDOW = 128; // conv subsampling + conformer encoder -ggml_cgraph * clip_graph_parakeet::build() { +ggml_cgraph * mtmd_graph_parakeet::build() { // Conv subsampling ggml_tensor * inp = build_inp_raw(1); diff --git a/tools/mtmd/models/pixtral.cpp b/tools/mtmd/models/pixtral.cpp index edfae0825360..6d3f01de062c 100644 --- a/tools/mtmd/models/pixtral.cpp +++ b/tools/mtmd/models/pixtral.cpp @@ -1,6 +1,6 @@ #include "models.h" -ggml_cgraph * clip_graph_pixtral::build() { +ggml_cgraph * mtmd_graph_pixtral::build() { const int n_merge = hparams.n_merge; // 2D input positions @@ -12,7 +12,7 @@ ggml_cgraph * clip_graph_pixtral::build() { ggml_set_name(pos_w, "pos_w"); ggml_set_input(pos_w); - auto add_pos = [&](ggml_tensor * cur, const clip_layer &) { + auto add_pos = [&](ggml_tensor * cur, const mtmd_layer &) { return build_rope_2d(ctx0, cur, pos_h, pos_w, hparams.rope_theta, true); }; diff --git a/tools/mtmd/models/pockettts-gen.cpp b/tools/mtmd/models/pockettts-gen.cpp index 662feff7fbfe..c5f340656d6a 100644 --- a/tools/mtmd/models/pockettts-gen.cpp +++ b/tools/mtmd/models/pockettts-gen.cpp @@ -9,13 +9,13 @@ // // there is no codebook anywhere, "codes" in the mtmd API are continuous features here -ggml_tensor * clip_graph_pockettts_gen::modulate(ggml_tensor * x, ggml_tensor * shift, ggml_tensor * scale) const { +ggml_tensor * mtmd_graph_pockettts_gen::modulate(ggml_tensor * x, ggml_tensor * shift, ggml_tensor * scale) const { ggml_tensor * cur = ggml_mul(ctx0, x, ggml_scale_bias(ctx0, scale, 1.0f, 1.0f)); return ggml_add(ctx0, cur, shift); } // see TimestepEmbedder in the reference -ggml_tensor * clip_graph_pockettts_gen::time_embed(const clip_flow_net::time_embd & te, float t) const { +ggml_tensor * mtmd_graph_pockettts_gen::time_embed(const mtmd_flow_net::time_embd & te, float t) const { // t is a graph-build constant, so the cos/sin table can be folded into a scaled copy ggml_tensor * args = ggml_scale(ctx0, te.freqs, t); ggml_tensor * emb = ggml_concat(ctx0, ggml_cos(ctx0, args), ggml_sin(ctx0, args), 0); @@ -42,7 +42,7 @@ ggml_tensor * clip_graph_pockettts_gen::time_embed(const clip_flow_net::time_emb } // one velocity evaluation: v(cond, s, t, x) -ggml_tensor * clip_graph_pockettts_gen::flow_forward(ggml_tensor * cond, ggml_tensor * x, float s, float t) const { +ggml_tensor * mtmd_graph_pockettts_gen::flow_forward(ggml_tensor * cond, ggml_tensor * x, float s, float t) const { const auto & flow = model.flow; ggml_tensor * cur = build_mm(flow.input_proj_w, x); @@ -99,7 +99,7 @@ ggml_tensor * clip_graph_pockettts_gen::flow_forward(ggml_tensor * cond, ggml_te // state carried between GEN_WAV calls: rope offset, per-layer KV window, conv left context // and the transposed-conv overlap tails -std::vector list_pockettts_state_slots(const clip_hparams & hparams, const clip_model & model) { +std::vector list_pockettts_state_slots(const mtmd_hparams & hparams, const mtmd_model & model) { std::vector slots; if (model.gen_upsample_w == nullptr) { return slots; // not a pocket-tts decoder @@ -136,8 +136,8 @@ std::vector list_pockettts_state_slots(const clip_hparams & hpar return slots; } -ggml_cgraph * clip_graph_pockettts_gen::build() { - if (gen_process == CLIP_GEN_PROCESS_GEN_CODE) { +ggml_cgraph * mtmd_graph_pockettts_gen::build() { + if (gen_process == MTMD_SUBCTX_GEN_PROCESS_GEN_CODE) { // the backbone hidden state arrives as the single batch entry ggml_tensor * h_state = build_inp_raw(1); h_state = ggml_reshape_2d(ctx0, h_state, n_mmproj_embd, 1); @@ -188,7 +188,7 @@ ggml_cgraph * clip_graph_pockettts_gen::build() { cur = build_mm(model.gen_quant_out_w, cur); cb(cur, "quant_out", -1); - clip_graph_pockettts_seanet seanet(*this); + mtmd_graph_pockettts_seanet seanet(*this); for (const auto & slot : list_pockettts_state_slots(hparams, model)) { ggml_tensor * t = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, slot.ne0, slot.ne1); ggml_set_name(t, ("state_in_" + slot.name).c_str()); @@ -225,7 +225,7 @@ ggml_cgraph * clip_graph_pockettts_gen::build() { keep = ggml_mul(ctx0, keep, ggml_step(ctx0, ggml_scale_bias(ctx0, ggml_add(ctx0, pos_k, base), 1.0f, 0.5f - (float) prefix))); ggml_tensor * kq_mask = ggml_reshape_4d(ctx0, ggml_log(ctx0, keep), n_kv, n_pos, 1, 1); - if (flash_attn_type == CLIP_FLASH_ATTN_TYPE_ENABLED) { + if (flash_attn_type == MTMD_FLASH_ATTN_TYPE_ENABLED) { kq_mask = ggml_cast(ctx0, kq_mask, GGML_TYPE_F16); } diff --git a/tools/mtmd/models/pockettts-seanet.cpp b/tools/mtmd/models/pockettts-seanet.cpp index c47207f569bb..4480aac9d69c 100644 --- a/tools/mtmd/models/pockettts-seanet.cpp +++ b/tools/mtmd/models/pockettts-seanet.cpp @@ -11,7 +11,7 @@ static int64_t div_ceil(int64_t a, int64_t b) { // x: [T, IC], w: [K, IC, OC] -> [T / stride, OC] // the convs are causal, so the whole K - stride padding goes on the left -ggml_tensor * clip_graph_pockettts_seanet::conv1d(ggml_tensor * x, ggml_tensor * w, ggml_tensor * b, int stride, int dilation, +ggml_tensor * mtmd_graph_pockettts_seanet::conv1d(ggml_tensor * x, ggml_tensor * w, ggml_tensor * b, int stride, int dilation, bool pad_replicate, const std::string & state_name) const { const int64_t k_size = (w->ne[0] - 1) * dilation + 1; const int64_t p_total = k_size - stride; @@ -48,7 +48,7 @@ ggml_tensor * clip_graph_pockettts_seanet::conv1d(ggml_tensor * x, ggml_tensor * // x: [T, IC], w: [K, OC/groups, IC] -> [T * stride, OC] // the K - stride overlap tail belongs to the next call: added to its head when streaming, else dropped -ggml_tensor * clip_graph_pockettts_seanet::conv_transpose1d(ggml_tensor * x, ggml_tensor * w, ggml_tensor * b, int stride, +ggml_tensor * mtmd_graph_pockettts_seanet::conv_transpose1d(ggml_tensor * x, ggml_tensor * w, ggml_tensor * b, int stride, const std::string & state_name) const { const int64_t K = w->ne[0]; const int64_t T = x->ne[0]; @@ -99,7 +99,7 @@ ggml_tensor * clip_graph_pockettts_seanet::conv_transpose1d(ggml_tensor * x, ggm return out; } -ggml_tensor * clip_graph_pockettts_seanet::res_unit(ggml_tensor * x, const clip_seanet::stage & stage, int dilation, +ggml_tensor * mtmd_graph_pockettts_seanet::res_unit(ggml_tensor * x, const mtmd_seanet::stage & stage, int dilation, const std::string & state_prefix) const { ggml_tensor * h = ggml_elu(ctx0, x); h = conv1d(h, stage.res_conv1_w, stage.res_conv1_b, 1, dilation, false, state_prefix); @@ -109,7 +109,7 @@ ggml_tensor * clip_graph_pockettts_seanet::res_unit(ggml_tensor * x, const clip_ return ggml_add(ctx0, x, h); } -ggml_tensor * clip_graph_pockettts_seanet::encode(ggml_tensor * x) const { +ggml_tensor * mtmd_graph_pockettts_seanet::encode(ggml_tensor * x) const { const auto & seanet = model.seanet; ggml_tensor * cur = conv1d(x, seanet.conv_in_w, seanet.conv_in_b, 1, 1); @@ -132,7 +132,7 @@ ggml_tensor * clip_graph_pockettts_seanet::encode(ggml_tensor * x) const { return cur; } -ggml_tensor * clip_graph_pockettts_seanet::decode(ggml_tensor * x) const { +ggml_tensor * mtmd_graph_pockettts_seanet::decode(ggml_tensor * x) const { const auto & seanet = model.seanet; const bool stream = !state_in.empty(); diff --git a/tools/mtmd/models/pockettts-spkenc.cpp b/tools/mtmd/models/pockettts-spkenc.cpp index 280f2a97ca36..f01727ad5f9e 100644 --- a/tools/mtmd/models/pockettts-spkenc.cpp +++ b/tools/mtmd/models/pockettts-spkenc.cpp @@ -4,7 +4,7 @@ // mimi encoder (SEANet + transformer + downsample), then flow_lm.speaker_proj_weight // pre-norm block with layer scale on both residual paths, see mimi_transformer.py -ggml_tensor * clip_graph_pockettts_spkenc::tfm_layer_forward(ggml_tensor * cur, const clip_layer & layer, ggml_tensor * inp_pos, ggml_tensor * kq_mask, int il) const { +ggml_tensor * mtmd_graph_pockettts_spkenc::tfm_layer_forward(ggml_tensor * cur, const mtmd_layer & layer, ggml_tensor * inp_pos, ggml_tensor * kq_mask, int il) const { ggml_tensor * inp = cur; cur = build_norm(cur, layer.ln_1_w, layer.ln_1_b, NORM_TYPE_NORMAL, eps, il); @@ -36,12 +36,12 @@ ggml_tensor * clip_graph_pockettts_spkenc::tfm_layer_forward(ggml_tensor * cur, return cur; } -ggml_cgraph * clip_graph_pockettts_spkenc::build() { +ggml_cgraph * mtmd_graph_pockettts_spkenc::build() { // the preprocessor hands over the waveform as a single-row "mel", already [n_samples, 1] ggml_tensor * inp_raw = build_inp_raw(1); ggml_tensor * cur = ggml_reshape_2d(ctx0, inp_raw, inp_raw->ne[0], inp_raw->ne[1]); - clip_graph_pockettts_seanet seanet(*this); + mtmd_graph_pockettts_seanet seanet(*this); cur = seanet.encode(cur); cb(cur, "mimi_enc", -1); diff --git a/tools/mtmd/models/qwen2vl.cpp b/tools/mtmd/models/qwen2vl.cpp index 7516aa23a986..8ecd59036d6a 100644 --- a/tools/mtmd/models/qwen2vl.cpp +++ b/tools/mtmd/models/qwen2vl.cpp @@ -1,6 +1,6 @@ #include "models.h" -ggml_cgraph * clip_graph_qwen2vl::build() { +ggml_cgraph * mtmd_graph_qwen2vl::build() { GGML_ASSERT(model.patch_bias == nullptr); GGML_ASSERT(model.class_embedding == nullptr); diff --git a/tools/mtmd/models/qwen3a.cpp b/tools/mtmd/models/qwen3a.cpp index 4de96955d969..0ed6b9a8f537 100644 --- a/tools/mtmd/models/qwen3a.cpp +++ b/tools/mtmd/models/qwen3a.cpp @@ -1,6 +1,6 @@ #include "models.h" -ggml_cgraph * clip_graph_qwen3a::build() { +ggml_cgraph * mtmd_graph_qwen3a::build() { // Ref implementation: https://github.com/QwenLM/Qwen3-ASR/blob/main/qwen_asr/core/transformers_backend/modeling_qwen3_asr.py // inp_raw: [n_frames, n_mel, 1] (nx=n_frames, ny=n_mel) diff --git a/tools/mtmd/models/qwen3tts-gen.cpp b/tools/mtmd/models/qwen3tts-gen.cpp index 9826190e68b4..59958fc4c3cd 100644 --- a/tools/mtmd/models/qwen3tts-gen.cpp +++ b/tools/mtmd/models/qwen3tts-gen.cpp @@ -3,7 +3,7 @@ #include // on-device sampling: top-k, top-p, then a random draw -ggml_tensor * clip_graph_qwen3tts_gen::code_gen::do_sampling(ggml_tensor * logits, ggml_tensor * inp_rand) const { +ggml_tensor * mtmd_graph_qwen3tts_gen::code_gen::do_sampling(ggml_tensor * logits, ggml_tensor * inp_rand) const { logits = ggml_reshape_1d(ctx0, logits, ggml_nelements(logits)); const int64_t n_vocab = logits->ne[0]; @@ -68,7 +68,7 @@ ggml_tensor * clip_graph_qwen3tts_gen::code_gen::do_sampling(ggml_tensor * logit } // returns a new cache with row row_idx set to value -ggml_tensor * clip_graph_qwen3tts_gen::code_gen::cache_set(ggml_tensor * cache, int row_idx, ggml_tensor * value) const { +ggml_tensor * mtmd_graph_qwen3tts_gen::code_gen::cache_set(ggml_tensor * cache, int row_idx, ggml_tensor * value) const { const int64_t n_embd = cache->ne[0]; const int64_t n_cache = cache->ne[1]; GGML_ASSERT(row_idx >= 0 && row_idx < n_cache); @@ -95,7 +95,7 @@ ggml_tensor * clip_graph_qwen3tts_gen::code_gen::cache_set(ggml_tensor * cache, } // builds a const i32 with no host upload: view a tensor, zero it via scale, add value, cast to i32 -ggml_tensor * clip_graph_qwen3tts_gen::code_gen::const_i32(ggml_tensor * anchor, float value) const { +ggml_tensor * mtmd_graph_qwen3tts_gen::code_gen::const_i32(ggml_tensor * anchor, float value) const { ggml_tensor * v = ggml_view_1d(ctx0, anchor, 1, 0); if (v->type != GGML_TYPE_F32) { v = ggml_cast(ctx0, v, GGML_TYPE_F32); @@ -104,20 +104,20 @@ ggml_tensor * clip_graph_qwen3tts_gen::code_gen::const_i32(ggml_tensor * anchor, } // causal keep-mask row for a query at position pos, window size n_kv_pad -ggml_tensor * clip_graph_qwen3tts_gen::code_gen::causal_mask_row(int64_t n_kv_pad, int pos) const { +ggml_tensor * mtmd_graph_qwen3tts_gen::code_gen::causal_mask_row(int64_t n_kv_pad, int pos) const { ggml_tensor * ones = ggml_fill(ctx0, ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_kv_pad, n_kv_pad), 1.0f); ggml_tensor * keep = ggml_tri(ctx0, ones, GGML_TRI_TYPE_LOWER_DIAG); ggml_tensor * row = ggml_view_1d(ctx0, keep, n_kv_pad, (size_t) pos * keep->nb[1]); ggml_tensor * mask = ggml_log(ctx0, row); // 0 = keep, -inf = masked mask = ggml_reshape_4d(ctx0, mask, n_kv_pad, 1, 1, 1); - if (flash_attn_type == CLIP_FLASH_ATTN_TYPE_ENABLED) { + if (flash_attn_type == MTMD_FLASH_ATTN_TYPE_ENABLED) { mask = ggml_cast(ctx0, mask, GGML_TYPE_F16); } return mask; } // talker hidden size -> predictor hidden size (small_to_mtp_projection) -ggml_tensor * clip_graph_qwen3tts_gen::code_gen::project_in(ggml_tensor * cur) const { +ggml_tensor * mtmd_graph_qwen3tts_gen::code_gen::project_in(ggml_tensor * cur) const { if (!model.gen_code_proj_in_w) { return cur; } @@ -129,9 +129,9 @@ ggml_tensor * clip_graph_qwen3tts_gen::code_gen::project_in(ggml_tensor * cur) c } // one transformer layer at position pos; writes k/v into k_cache_layer/v_cache_layer at row pos -ggml_tensor * clip_graph_qwen3tts_gen::code_gen::layer_forward( +ggml_tensor * mtmd_graph_qwen3tts_gen::code_gen::layer_forward( ggml_tensor * cur, - const clip_layer & layer, + const mtmd_layer & layer, ggml_tensor * inp_pos, ggml_tensor * kq_mask, ggml_tensor *& k_cache_layer, @@ -192,7 +192,7 @@ ggml_tensor * clip_graph_qwen3tts_gen::code_gen::layer_forward( // position 0: hidden bridge, seeds the k/v cache, no sampling // position 1: embed(code0), sample with lm_head[0], write out_code_cache[1] -void clip_graph_qwen3tts_gen::code_gen::prefill( +void mtmd_graph_qwen3tts_gen::code_gen::prefill( std::vector & k_cache, std::vector & v_cache, ggml_tensor *& out_code_cache, @@ -237,7 +237,7 @@ void clip_graph_qwen3tts_gen::code_gen::prefill( // - write new kv at cache row g+1, sample with lm_head[g] // - write result to out_code_cache[g+1] // step_idx must be in [1, n_acoustic - 1] -ggml_tensor * clip_graph_qwen3tts_gen::code_gen::step( +ggml_tensor * mtmd_graph_qwen3tts_gen::code_gen::step( std::vector & k_cache, std::vector & v_cache, ggml_tensor * out_code_cache, @@ -290,7 +290,7 @@ ggml_tensor * clip_graph_qwen3tts_gen::code_gen::step( // causal conv1d, stride 1: prepend persisted left-context instead of zero-padding, then a plain conv // x: [T, IC] (T-first). w: [K, IC, OC]. state_name empty means K == 1 (no left-context). returns [T, OC] -ggml_tensor * clip_graph_qwen3tts_gen::code2wav::causal_conv1d(ggml_tensor * x, ggml_tensor * w, ggml_tensor * b, int dilation, const std::string & state_name) const { +ggml_tensor * mtmd_graph_qwen3tts_gen::code2wav::causal_conv1d(ggml_tensor * x, ggml_tensor * w, ggml_tensor * b, int dilation, const std::string & state_name) const { const int K = (int) w->ne[0]; const int pad = (K - 1) * dilation; @@ -314,7 +314,7 @@ ggml_tensor * clip_graph_qwen3tts_gen::code2wav::causal_conv1d(ggml_tensor * x, // causal depthwise conv1d, stride 1, dilation 1, kernel from w's shape. // x: [T, C]. w: [K, 1, C]. returns [T, C]. see causal_conv1d for the state contract. -ggml_tensor * clip_graph_qwen3tts_gen::code2wav::causal_conv1d_dw(ggml_tensor * x, ggml_tensor * w, ggml_tensor * b, const std::string & state_name) const { +ggml_tensor * mtmd_graph_qwen3tts_gen::code2wav::causal_conv1d_dw(ggml_tensor * x, ggml_tensor * w, ggml_tensor * b, const std::string & state_name) const { const int K = (int) w->ne[0]; const int pad = K - 1; @@ -338,7 +338,7 @@ ggml_tensor * clip_graph_qwen3tts_gen::code2wav::causal_conv1d_dw(ggml_tensor * // causal ConvTranspose1d, the (kernel - stride) overlap tail is kept as state for the next call // x: [T, IC], w: [K, OC, IC]. state_name empty means K == stride (no overlap). returns [T * stride, OC] -ggml_tensor * clip_graph_qwen3tts_gen::code2wav::causal_conv_transpose1d(ggml_tensor * x, ggml_tensor * w, ggml_tensor * b, int stride, const std::string & state_name) const { +ggml_tensor * mtmd_graph_qwen3tts_gen::code2wav::causal_conv_transpose1d(ggml_tensor * x, ggml_tensor * w, ggml_tensor * b, int stride, const std::string & state_name) const { const int K = (int) w->ne[0]; const int OC = (int) w->ne[1]; const int trim = K - stride; @@ -372,7 +372,7 @@ ggml_tensor * clip_graph_qwen3tts_gen::code2wav::causal_conv_transpose1d(ggml_te // SnakeBeta activation: y = x + sin(alpha*x)^2 * inv_beta (alpha/inv_beta folded via exp/reciprocal at conversion time) // x: [T, C]. alpha/beta: [C], broadcasts over T -ggml_tensor * clip_graph_qwen3tts_gen::code2wav::snake(ggml_tensor * x, ggml_tensor * alpha, ggml_tensor * beta) const { +ggml_tensor * mtmd_graph_qwen3tts_gen::code2wav::snake(ggml_tensor * x, ggml_tensor * alpha, ggml_tensor * beta) const { ggml_tensor * a = ggml_reshape_2d(ctx0, alpha, 1, alpha->ne[0]); ggml_tensor * b = ggml_reshape_2d(ctx0, beta, 1, beta->ne[0]); @@ -388,7 +388,7 @@ ggml_tensor * clip_graph_qwen3tts_gen::code2wav::snake(ggml_tensor * x, ggml_ten // RVQ codebook decode: T frames of 16 codes -> 512-dim hidden (C-first, [512, T]) // codebook 0 (semantic) and 1..15 (acoustic) sum within their group, project separately, then add -ggml_tensor * clip_graph_qwen3tts_gen::code2wav::quant_decode(ggml_tensor * inp_codes) const { +ggml_tensor * mtmd_graph_qwen3tts_gen::code2wav::quant_decode(ggml_tensor * inp_codes) const { const auto & c2w = model.c2w; const int64_t T = inp_codes->ne[0]; @@ -418,7 +418,7 @@ ggml_tensor * clip_graph_qwen3tts_gen::code2wav::quant_decode(ggml_tensor * inp_ // one pre_transformer layer over a batch of N = sliding_window new frames // attention runs over [(W-1)-frame prefix from the last batch] + [N new frames] // RoPE positions come from a persisted counter, so phases line up across batches -ggml_tensor * clip_graph_qwen3tts_gen::code2wav::tfm_layer_forward(ggml_tensor * cur, const clip_layer & layer, int il) const { +ggml_tensor * mtmd_graph_qwen3tts_gen::code2wav::tfm_layer_forward(ggml_tensor * cur, const mtmd_layer & layer, int il) const { const int n_head = hparams.wav_tfm_n_head; const int n_head_kv = hparams.wav_tfm_n_head_kv; const int64_t d_head = layer.q_w->ne[1] / n_head; @@ -485,7 +485,7 @@ ggml_tensor * clip_graph_qwen3tts_gen::code2wav::tfm_layer_forward(ggml_tensor * keep = ggml_mul(ctx0, keep, warm); ggml_tensor * mask = ggml_reshape_4d(ctx0, ggml_log(ctx0, keep), total_kv, N, 1, 1); // 0 = keep, -inf = masked - if (flash_attn_type == CLIP_FLASH_ATTN_TYPE_ENABLED) { + if (flash_attn_type == MTMD_FLASH_ATTN_TYPE_ENABLED) { mask = ggml_cast(ctx0, mask, GGML_TYPE_F16); } @@ -515,7 +515,7 @@ ggml_tensor * clip_graph_qwen3tts_gen::code2wav::tfm_layer_forward(ggml_tensor * // dwconv -> LayerNorm -> pwconv1 -> GELU -> pwconv2 -> layer scale -> residual // x: [T, C] T-first; LayerNorm/pwconv need C on ne0, so this transposes in and back out -ggml_tensor * clip_graph_qwen3tts_gen::code2wav::convnext_block(ggml_tensor * x, const clip_code2wav::upsample_block & blk, const std::string & state_prefix) const { +ggml_tensor * mtmd_graph_qwen3tts_gen::code2wav::convnext_block(ggml_tensor * x, const mtmd_code2wav::upsample_block & blk, const std::string & state_prefix) const { ggml_tensor * residual = x; ggml_tensor * h = causal_conv1d_dw(x, blk.dwconv_w, blk.dwconv_b, state_prefix + "_dwconv"); // [T, C] @@ -527,7 +527,7 @@ ggml_tensor * clip_graph_qwen3tts_gen::code2wav::convnext_block(ggml_tensor * x, ggml_tensor * g = ggml_mul_mat(ctx0, blk.pw1_w, hc); g = ggml_add(ctx0, g, blk.pw1_b); - g = ggml_gelu(ctx0, g); + g = gelu_erf ? ggml_gelu_erf(ctx0, g) : ggml_gelu(ctx0, g); g = ggml_mul_mat(ctx0, blk.pw2_w, g); g = ggml_add(ctx0, g, blk.pw2_b); g = ggml_mul(ctx0, g, blk.gamma); @@ -538,7 +538,7 @@ ggml_tensor * clip_graph_qwen3tts_gen::code2wav::convnext_block(ggml_tensor * x, // SnakeBeta -> dilated causal conv (k=7) -> SnakeBeta -> pointwise causal conv (k=1) -> residual. // x: [T, C]. returns [T, C]. -ggml_tensor * clip_graph_qwen3tts_gen::code2wav::dac_res_unit(ggml_tensor * x, const clip_code2wav::dac_res & res, int dilation, const std::string & state_name) const { +ggml_tensor * mtmd_graph_qwen3tts_gen::code2wav::dac_res_unit(ggml_tensor * x, const mtmd_code2wav::dac_res & res, int dilation, const std::string & state_name) const { ggml_tensor * residual = x; ggml_tensor * h = snake(x, res.act1_alpha, res.act1_beta); h = causal_conv1d(h, res.conv1_w, res.conv1_b, dilation, state_name); @@ -548,7 +548,7 @@ ggml_tensor * clip_graph_qwen3tts_gen::code2wav::dac_res_unit(ggml_tensor * x, c } // RVQ codes -> raw PCM for a batch of N = sliding_window frames -ggml_tensor * clip_graph_qwen3tts_gen::code2wav::decode(ggml_tensor * inp_codes) const { +ggml_tensor * mtmd_graph_qwen3tts_gen::code2wav::decode(ggml_tensor * inp_codes) const { const auto & c2w = model.c2w; // 1. quantizer decode: N frames of 16 codes -> [512, N] (C-first) @@ -613,11 +613,11 @@ ggml_tensor * clip_graph_qwen3tts_gen::code2wav::decode(ggml_tensor * inp_codes) // code2wav's persisted state buffers: RoPE position counter, K/V per pre_transformer layer, // left-context/tail per stateful conv. shape lookup only, no graph needed -std::vector list_c2w_state_slots(const clip_hparams & hparams, const clip_model & model) { +std::vector list_c2w_state_slots(const mtmd_hparams & hparams, const mtmd_model & model) { const auto & c2w = model.c2w; std::vector slots; - if (c2w.pre_conv_w == nullptr) { + if (c2w.dac_entry_w == nullptr) { return slots; // not a code2wav model, it keeps no state between calls } @@ -632,7 +632,9 @@ std::vector list_c2w_state_slots(const clip_hparams & hparams, c slots.push_back({"tfm_v_" + std::to_string(il), kv_ch, prefix}); } - slots.push_back({"pre_conv", c2w.pre_conv_w->ne[0] - 1, c2w.pre_conv_w->ne[1]}); + if (c2w.pre_conv_w) { + slots.push_back({"pre_conv", c2w.pre_conv_w->ne[0] - 1, c2w.pre_conv_w->ne[1]}); + } for (size_t il = 0; il < c2w.upsample.size(); il++) { const auto & up = c2w.upsample[il]; @@ -661,20 +663,20 @@ std::vector list_c2w_state_slots(const clip_hparams & hparams, c // both sub-graphs are always built, so the topology stays constant // ggml_build_forward_select() then picks the one that actually runs -ggml_cgraph * clip_graph_qwen3tts_gen::build() { +ggml_cgraph * mtmd_graph_qwen3tts_gen::build() { GGML_ASSERT(n_batch == 1); // this module only ever processes one frame at a time int idx; switch (gen_process) { - case CLIP_GEN_PROCESS_GEN_CODE: idx = 0; break; - case CLIP_GEN_PROCESS_GEN_WAV: idx = 1; break; + case MTMD_SUBCTX_GEN_PROCESS_GEN_CODE: idx = 0; break; + case MTMD_SUBCTX_GEN_PROCESS_GEN_WAV: idx = 1; break; default: GGML_ABORT("unknown gen_process"); } - // ---- CLIP_GEN_PROCESS_GEN_CODE: backbone hidden state -> 16 RVQ codes + next-step embd ---- + // ---- MTMD_SUBCTX_GEN_PROCESS_GEN_CODE: backbone hidden state -> 16 RVQ codes + next-step embd ---- // not build_inp_raw(), a GEN_WAV call's `img` has no hidden-state data ggml_tensor * h_state = ggml_new_tensor_1d(ctx0, GGML_TYPE_F32, n_mmproj_embd); - ggml_set_name(h_state, "inp_raw"); // must keep this exact name, clip_encode() sets it by name + ggml_set_name(h_state, "inp_raw"); // must keep this exact name, mtmd_subcontext::encode() sets it by name ggml_set_input(h_state); ggml_tensor * code0 = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, 1); @@ -738,7 +740,7 @@ ggml_cgraph * clip_graph_qwen3tts_gen::build() { out_embd = ggml_reshape_2d(ctx0, out_embd, out_embd->ne[0], 1); cb(out_embd, "gen_audio_out", -1); - // ---- CLIP_GEN_PROCESS_GEN_WAV: 16 RVQ codes -> raw PCM ---- + // ---- MTMD_SUBCTX_GEN_PROCESS_GEN_WAV: 16 RVQ codes -> raw PCM ---- const int n_frames = hparams.wav_tfm_swa; // frames per batch, == the attention window ggml_tensor * inp_codes = ggml_new_tensor_2d(ctx0, GGML_TYPE_I32, n_frames, n_codes); @@ -762,7 +764,7 @@ ggml_cgraph * clip_graph_qwen3tts_gen::build() { ggml_set_output(slot.second); } - // out_embd goes last, clip_encode() reads it back via ggml_graph_node(gf, -1) + // out_embd goes last, mtmd_subcontext::encode() reads it back via ggml_graph_node(gf, -1) ggml_tensor * outs[2]; outs[0] = out_codes; outs[1] = out_audio; ggml_build_forward_select(gf, outs, 2, idx); diff --git a/tools/mtmd/models/qwen3tts-spkenc.cpp b/tools/mtmd/models/qwen3tts-spkenc.cpp index 405fbb9cbc20..1717eac6ed5a 100644 --- a/tools/mtmd/models/qwen3tts-spkenc.cpp +++ b/tools/mtmd/models/qwen3tts-spkenc.cpp @@ -5,7 +5,7 @@ static constexpr int SPK_DILATIONS[3] = { 2, 3, 4 }; // enc_dilations[1..3] // conv1d, kernel K, padding "same" (reflect), dilation d // x: [C, T] (ne[0]=C, ne[1]=T) -> [out_c, T] -ggml_tensor * clip_graph_qwen3tts_spkenc::conv1d_same(ggml_tensor * x, ggml_tensor * w, ggml_tensor * b, int dilation) const { +ggml_tensor * mtmd_graph_qwen3tts_spkenc::conv1d_same(ggml_tensor * x, ggml_tensor * w, ggml_tensor * b, int dilation) const { const int K = (int) w->ne[0]; const int IC = (int) w->ne[1]; const int OC = (int) w->ne[2]; @@ -27,7 +27,7 @@ ggml_tensor * clip_graph_qwen3tts_spkenc::conv1d_same(ggml_tensor * x, ggml_tens ggml_tensor * w2d = ggml_reshape_2d(ctx0, w, (int64_t) K * IC, OC); ggml_tensor * y = ggml_mul_mat(ctx0, w2d, col); // [OC, T_out] - ggml_prec_set_acc(y, GGML_PREC_F32); + ggml_prec_set_acc(y, GGML_PREC_BF16); ggml_tensor * b2d = ggml_reshape_2d(ctx0, b, OC, 1); y = ggml_add(ctx0, y, b2d); @@ -36,7 +36,7 @@ ggml_tensor * clip_graph_qwen3tts_spkenc::conv1d_same(ggml_tensor * x, ggml_tens // Res2Net: split channel axis into `scale` chunks, chain dilated conv1d branches // x: [C, T] -> [C, T] -ggml_tensor * clip_graph_qwen3tts_spkenc::res2net(ggml_tensor * x, const clip_layer & layer, int dilation, int scale) const { +ggml_tensor * mtmd_graph_qwen3tts_spkenc::res2net(ggml_tensor * x, const mtmd_layer & layer, int dilation, int scale) const { const int64_t C = x->ne[0]; const int64_t T = x->ne[1]; const int64_t Cs = C / scale; @@ -70,7 +70,7 @@ ggml_tensor * clip_graph_qwen3tts_spkenc::res2net(ggml_tensor * x, const clip_la } // squeeze-and-excitation gate. x: [C, T] -> [C, T] -ggml_tensor * clip_graph_qwen3tts_spkenc::se_block(ggml_tensor * x, const clip_layer & layer) const { +ggml_tensor * mtmd_graph_qwen3tts_spkenc::se_block(ggml_tensor * x, const mtmd_layer & layer) const { // temporal mean, keepdim: transpose so T is on ne[0], reduce, transpose back ggml_tensor * x_t = ggml_cont(ctx0, ggml_transpose(ctx0, x)); // [T, C] ggml_tensor * mean = ggml_mean(ctx0, x_t); // [1, C] @@ -85,7 +85,7 @@ ggml_tensor * clip_graph_qwen3tts_spkenc::se_block(ggml_tensor * x, const clip_l } // tdnn1 -> res2net -> tdnn2 -> se, plus residual. x: [C, T] -> [C, T] -ggml_tensor * clip_graph_qwen3tts_spkenc::se_res2net_block(ggml_tensor * x, const clip_layer & layer, int dilation, int scale) const { +ggml_tensor * mtmd_graph_qwen3tts_spkenc::se_res2net_block(ggml_tensor * x, const mtmd_layer & layer, int dilation, int scale) const { ggml_tensor * residual = x; ggml_tensor * h = conv1d_same(x, layer.conv_pw1_w, layer.conv_pw1_b, 1); // tdnn1 h = ggml_relu(ctx0, h); @@ -97,7 +97,7 @@ ggml_tensor * clip_graph_qwen3tts_spkenc::se_res2net_block(ggml_tensor * x, cons } // attentive statistics pooling. x: [C, T] -> [2*C, 1] -ggml_tensor * clip_graph_qwen3tts_spkenc::attentive_stats_pool(ggml_tensor * x) const { +ggml_tensor * mtmd_graph_qwen3tts_spkenc::attentive_stats_pool(ggml_tensor * x) const { const int64_t T = x->ne[1]; // mean over T: [C, 1] @@ -151,7 +151,7 @@ ggml_tensor * clip_graph_qwen3tts_spkenc::attentive_stats_pool(ggml_tensor * x) return ggml_concat(ctx0, w_mean, w_std, 0); // [2C, 1] } -ggml_cgraph * clip_graph_qwen3tts_spkenc::build() { +ggml_cgraph * mtmd_graph_qwen3tts_spkenc::build() { // inp_raw: [T, n_mel, 1, 1], from mtmd_audio_preprocessor_qwen3tts_spk ggml_tensor * inp = build_inp_raw(1); inp = ggml_reshape_2d(ctx0, inp, inp->ne[0], inp->ne[1]); diff --git a/tools/mtmd/models/qwen3vl.cpp b/tools/mtmd/models/qwen3vl.cpp index 2e49dcd019b7..ada7a129de86 100644 --- a/tools/mtmd/models/qwen3vl.cpp +++ b/tools/mtmd/models/qwen3vl.cpp @@ -1,6 +1,6 @@ #include "models.h" -ggml_cgraph * clip_graph_qwen3vl::build() { +ggml_cgraph * mtmd_graph_qwen3vl::build() { GGML_ASSERT(model.patch_bias != nullptr); GGML_ASSERT(model.position_embeddings != nullptr); GGML_ASSERT(model.class_embedding == nullptr); diff --git a/tools/mtmd/models/sensenova-u1.cpp b/tools/mtmd/models/sensenova-u1.cpp index 57875c6edbc8..8e685d74cd21 100644 --- a/tools/mtmd/models/sensenova-u1.cpp +++ b/tools/mtmd/models/sensenova-u1.cpp @@ -1,6 +1,6 @@ #include "models.h" -ggml_cgraph * clip_graph_sensenova_u1::build() { +ggml_cgraph * mtmd_graph_sensenova_u1::build() { GGML_ASSERT(model.patch_embeddings_0 != nullptr); GGML_ASSERT(model.patch_bias != nullptr); GGML_ASSERT(model.mm_0_w != nullptr); diff --git a/tools/mtmd/models/siglip.cpp b/tools/mtmd/models/siglip.cpp index 8cc901a28e47..84387f86d004 100644 --- a/tools/mtmd/models/siglip.cpp +++ b/tools/mtmd/models/siglip.cpp @@ -1,6 +1,6 @@ #include "models.h" -ggml_cgraph * clip_graph_siglip::build() { +ggml_cgraph * mtmd_graph_siglip::build() { ggml_tensor * inp = build_inp(); ggml_tensor * learned_pos_embd = model.position_embeddings; diff --git a/tools/mtmd/models/soprano.cpp b/tools/mtmd/models/soprano.cpp index bd16c69124be..43f34f86f5d5 100644 --- a/tools/mtmd/models/soprano.cpp +++ b/tools/mtmd/models/soprano.cpp @@ -2,7 +2,7 @@ #include "../mtmd-audio.h" // Soprano Vocos: normalized Qwen3 hidden states -> STFT coefficients. -ggml_cgraph * clip_graph_soprano::build() { +ggml_cgraph * mtmd_graph_soprano::build() { const auto & v = model.vocos; const int n_inp = hparams.projection_dim; const int n_up = upscale * (n_frames - 1) + 1; @@ -59,7 +59,7 @@ ggml_cgraph * clip_graph_soprano::build() { return gf; } -void clip_graph_soprano::decode_spectrum(const clip_model & model, const std::vector & spectrum, std::vector & pcm) { +void mtmd_graph_soprano::decode_spectrum(const mtmd_model & model, const std::vector & spectrum, std::vector & pcm) { const int n_frame_vals = model.vocos.output_w->ne[1]; const int n_fft = n_frame_vals - 2; const int hop_length = model.hparams.audio_hop_len; diff --git a/tools/mtmd/models/step3vl.cpp b/tools/mtmd/models/step3vl.cpp index 5142b0bba387..db0245a8910a 100644 --- a/tools/mtmd/models/step3vl.cpp +++ b/tools/mtmd/models/step3vl.cpp @@ -1,6 +1,6 @@ #include "models.h" -ggml_cgraph * clip_graph_step3vl::build() { +ggml_cgraph * mtmd_graph_step3vl::build() { GGML_ASSERT(model.class_embedding == nullptr); GGML_ASSERT(model.patch_embeddings_0 != nullptr); GGML_ASSERT(model.position_embeddings != nullptr); @@ -18,7 +18,7 @@ ggml_cgraph * clip_graph_step3vl::build() { ggml_tensor * inp = build_inp(); ggml_tensor * learned_pos_embd = resize_position_embeddings(); - auto add_pos = [&](ggml_tensor * cur, const clip_layer &) { + auto add_pos = [&](ggml_tensor * cur, const mtmd_layer &) { return build_rope_2d(ctx0, cur, pos_w, pos_h, hparams.rope_theta, false); }; diff --git a/tools/mtmd/models/voxtral-rt.cpp b/tools/mtmd/models/voxtral-rt.cpp index 4a01512d21b5..3f768d6a2e6d 100644 --- a/tools/mtmd/models/voxtral-rt.cpp +++ b/tools/mtmd/models/voxtral-rt.cpp @@ -2,7 +2,7 @@ // the input holds 2 context mel frames followed by the new frames, so the causal conv stem // sees the same left context as on the full spectrogram -ggml_cgraph * clip_graph_voxtral_rt::build() { +ggml_cgraph * mtmd_graph_voxtral_rt::build() { const int n_frames_in = img.nx(); const int n_new = (n_frames_in - 2) / 2; // encoder frames, the conv stem has stride 2 const int n_window = hparams.attn_window_size; diff --git a/tools/mtmd/models/whisper-enc.cpp b/tools/mtmd/models/whisper-enc.cpp index 49d5dd5add38..6225b707cec1 100644 --- a/tools/mtmd/models/whisper-enc.cpp +++ b/tools/mtmd/models/whisper-enc.cpp @@ -1,6 +1,6 @@ #include "models.h" -ggml_cgraph * clip_graph_whisper_enc::build() { +ggml_cgraph * mtmd_graph_whisper_enc::build() { const int n_frames = img.nx(); const int n_pos = n_frames / 2; GGML_ASSERT(model.position_embeddings->ne[1] >= n_pos); diff --git a/tools/mtmd/models/yasa2.cpp b/tools/mtmd/models/yasa2.cpp index e8cd3dacbf5a..07fa84e06fae 100644 --- a/tools/mtmd/models/yasa2.cpp +++ b/tools/mtmd/models/yasa2.cpp @@ -25,7 +25,7 @@ static ggml_tensor * mul_channel_weight( return ggml_mul(ctx0, x_whcb, w4); } -ggml_tensor * clip_graph_yasa2::layer_norm_channels(ggml_tensor * inp, ggml_tensor * w, ggml_tensor * b, float eps) { +ggml_tensor * mtmd_graph_yasa2::layer_norm_channels(ggml_tensor * inp, ggml_tensor * w, ggml_tensor * b, float eps) { // Match HF ConvNextLayerNorm(channels_first): // u = mean_c(x), s = mean_c((x-u)^2), x = (x-u)/sqrt(s+eps) // cast back to input dtype before affine. @@ -48,7 +48,7 @@ ggml_tensor * clip_graph_yasa2::layer_norm_channels(ggml_tensor * inp, ggml_tens return xhat; } -ggml_tensor * clip_graph_yasa2::convnext_grn(ggml_tensor * inp, ggml_tensor * w, ggml_tensor * b) { +ggml_tensor * mtmd_graph_yasa2::convnext_grn(ggml_tensor * inp, ggml_tensor * w, ggml_tensor * b) { // Exact ConvNeXtV2 GRN: // Gx = ||x||_2 over spatial dims (W,H), Nx = Gx / (mean_c(Gx) + eps) // y = w * (x * Nx) + b + x @@ -78,7 +78,7 @@ ggml_tensor * clip_graph_yasa2::convnext_grn(ggml_tensor * inp, ggml_tensor * w, return ggml_add(ctx0, inp, xnx); } -ggml_cgraph * clip_graph_yasa2::build() { +ggml_cgraph * mtmd_graph_yasa2::build() { ggml_tensor * cur = build_inp_raw(); // Patch embedding Conv2d(kernel=4, stride=4) diff --git a/tools/mtmd/models/youtuvl.cpp b/tools/mtmd/models/youtuvl.cpp index cda456c66b23..8cc714358e53 100644 --- a/tools/mtmd/models/youtuvl.cpp +++ b/tools/mtmd/models/youtuvl.cpp @@ -1,6 +1,6 @@ #include "models.h" -ggml_cgraph * clip_graph_youtuvl::build() { +ggml_cgraph * mtmd_graph_youtuvl::build() { GGML_ASSERT(model.class_embedding == nullptr); const int batch_size = 1; const bool use_window_attn = !hparams.wa_layer_indexes.empty(); diff --git a/tools/mtmd/mtmd-audio.cpp b/tools/mtmd/mtmd-audio.cpp index 14b5a721357d..aceaf5391d69 100644 --- a/tools/mtmd/mtmd-audio.cpp +++ b/tools/mtmd/mtmd-audio.cpp @@ -1,10 +1,10 @@ #include "mtmd-audio.h" +#include "mtmd-subcontext.h" -#define _USE_MATH_DEFINES // for M_PI +#include #include #include #include -#include #include #include #include @@ -270,100 +270,44 @@ static void ifft(const mtmd_audio_cache & cache, float * in, int N, float * out) fft_impl(cache, in, N, out); } -struct filter_params { - int64_t n_mel; - int64_t n_fft_bins; - int32_t hann_window_size; - int32_t hop_length; - int32_t sample_rate; - bool no_padding = false; - bool center_padding = false; - float preemph = 0.f; - bool use_natural_log = false; - bool norm_per_feature = false; - bool use_magnitude = false; // |X| instead of |X|^2 - float mel_floor = 5.960464477539063e-08f; - bool mel_floor_add = false; // log(x + floor) instead of log(max(x, floor)) - bool std_eps_after_sqrt = false; // std + eps instead of sqrt(var + eps) -}; - -static void log_mel_spectrogram_worker_thread(int ith, - const float * hann, - const std::vector & samples, - int n_samples, - int frame_size, - int frame_step, - int n_threads, - const filter_params & params, - const mtmd_audio_cache & cache, - mtmd_audio_mel & out) { - std::vector fft_in(frame_size * 2, 0.0); - std::vector fft_out(frame_size * 2 * 2 * 2); - - int64_t n_fft_bins = params.n_fft_bins; - int64_t i = ith; - - const auto & filters = cache.filters; - - // make sure n_fft == 1 + (WHISPER_N_FFT / 2), bin_0 to bin_nyquist - GGML_ASSERT(n_fft_bins == 1 + (frame_size / 2)); - GGML_ASSERT(cache.sin_vals.size() == cache.cos_vals.size()); - // calculate FFT only when fft_in are not all zero - for (; i < std::min((int64_t)(n_samples / frame_step + 1), out.n_len); i += n_threads) { - const int64_t offset = i * frame_step; - - // apply Hann window (~10% faster) - const int valid_len = std::min(frame_size, std::max(0, n_samples - (int)offset)); - for (int j = 0; j < valid_len; j++) { - fft_in[j] = hann[j] * samples[offset + j]; - } - - // fill the rest with zeros - if (valid_len < frame_size) { - std::fill(fft_in.begin() + valid_len, fft_in.end(), 0.0); - } - - // FFT - fft(cache, fft_in.data(), frame_size, fft_out.data()); - - // Calculate modulus^2 (power) or modulus (magnitude) - for (int j = 0; j < n_fft_bins; j++) { - float power = (fft_out[2 * j + 0] * fft_out[2 * j + 0] + fft_out[2 * j + 1] * fft_out[2 * j + 1]); - fft_out[j] = params.use_magnitude ? sqrtf(power) : power; - } - - // mel spectrogram - for (int64_t j = 0; j < out.n_mel; j++) { - double sum = 0.0; - // unroll loop (suggested by GH user @lunixbochs) - int k = 0; - for (k = 0; k < n_fft_bins - 3; k += 4) { - size_t idx = size_t(j) * size_t(n_fft_bins) + size_t(k); - sum += - fft_out[k + 0] * filters.data[idx + 0] + - fft_out[k + 1] * filters.data[idx + 1] + - fft_out[k + 2] * filters.data[idx + 2] + - fft_out[k + 3] * filters.data[idx + 3]; - } - // handle n_fft remainder - for (; k < n_fft_bins; k++) { - sum += fft_out[k] * filters.data[(size_t)j * n_fft_bins + k]; - } - sum = params.mel_floor_add ? sum + (double)params.mel_floor : std::max(sum, (double)params.mel_floor); - sum = params.use_natural_log - ? log(sum) - : log10(sum); - out.data[(size_t)j * out.n_len + i] = sum; +static void mtmd_audio_fill_entries(std::vector & mels, bool shape_only, std::vector & entries) { + entries.clear(); + for (auto & mel : mels) { + GGML_ASSERT(mel.n_len <= INT32_MAX && mel.n_len >= 0); + GGML_ASSERT(mel.n_mel <= INT32_MAX && mel.n_mel >= 0); + mtmd_image_f32 entry; + entry.set_size({(int) mel.n_len, (int) mel.n_mel}, shape_only || mel.data.empty(), /* is_audio */ true); + if (!entry.is_placeholder()) { + entry.cpy_buf(mel.data); } + entries.push_back(std::move(entry)); } +} - // Otherwise fft_out are all zero - double sum = params.use_natural_log ? log(1e-10) : log10(1e-10); - for (; i < out.n_len; i += n_threads) { - for (int64_t j = 0; j < out.n_mel; j++) { - out.data[(size_t)j * out.n_len + i] = sum; +mtmd_audio_preproc_out mtmd_audio_preprocessor::preprocess(const float * samples, size_t n_samples) const { + mtmd_audio_preproc_out out; + const bool deferred = mel_config.cache != nullptr; + std::vector mels; + if (process(samples, n_samples, mels, deferred)) { + mtmd_audio_fill_entries(mels, deferred, out.entries); + if (deferred) { + out.samples.assign(samples, samples + n_samples); } } + return out; +} + +bool mtmd_audio_preprocessor::compute(mtmd_audio_preproc_out & out) const { + if (!out.needs_compute()) { + return true; + } + std::vector mels; + if (!process(out.samples.data(), out.samples.size(), mels, false)) { + return false; + } + mtmd_audio_fill_entries(mels, false, out.entries); + out.samples = {}; + return true; } void mtmd_audio_log_mel_frame(const mtmd_audio_cache & cache, const float * samples, int n_fft, int n_mel, float * out) { @@ -395,15 +339,16 @@ void mtmd_audio_log_mel_frame(const mtmd_audio_cache & cache, const float * samp static bool log_mel_spectrogram( const float * samples, const int n_samples_in, - const int n_threads, - const filter_params & params, - const mtmd_audio_cache & cache, + const mtmd_audio_mel_config & params, + const bool shape_only, mtmd_audio_mel & out) { //const int64_t t_start_us = ggml_time_us(); out.n_len_org = n_samples_in; int n_samples = n_samples_in; + const auto & cache = *params.cache; + // Hann window const float * hann = cache.hann_window.data(); const int frame_size = (params.n_fft_bins - 1) * 2; @@ -486,20 +431,8 @@ static bool log_mel_spectrogram( } out.data.resize(total_size); - { - std::vector workers(n_threads - 1); - for (int iw = 0; iw < n_threads - 1; ++iw) { - workers[iw] = - std::thread(log_mel_spectrogram_worker_thread, iw + 1, hann, std::cref(samples_padded), n_samples, - frame_size, frame_step, n_threads, std::cref(params), std::cref(cache), std::ref(out)); - } - - // main thread - log_mel_spectrogram_worker_thread(0, hann, samples_padded, n_samples, frame_size, frame_step, n_threads, params, - cache, out); - for (int iw = 0; iw < n_threads - 1; ++iw) { - workers[iw].join(); - } + if (!shape_only && !mtmd_audio_log_mel(params, samples_padded.data(), hann, out.n_len, out.data.data())) { + return false; } const int64_t effective_n_len = n_samples_in / frame_step; @@ -572,11 +505,23 @@ void mtmd_audio_preprocessor_whisper::initialize() { cache.fill_sin_cos_table(hparams.audio_n_fft); cache.fill_hann_window(hparams.audio_window_len, true); cache.fill_mel_filterbank_matrix(hparams.n_mel_bins, hparams.audio_n_fft, hparams.audio_sample_rate); + + mel_config.n_mel = hparams.n_mel_bins; + mel_config.n_fft_bins = 1 + (hparams.audio_n_fft / 2); + mel_config.hann_window_size = hparams.audio_window_len; + mel_config.hop_length = hparams.audio_hop_len; + mel_config.sample_rate = hparams.audio_sample_rate; + mel_config.center_padding = false; + mel_config.preemph = 0.0f; // disabled + mel_config.use_natural_log = false; + mel_config.norm_per_feature = false; + mel_config.cache = &cache; } -bool mtmd_audio_preprocessor_whisper::preprocess(const float * samples, - size_t n_samples, - std::vector & output) const { +bool mtmd_audio_preprocessor_whisper::process(const float * samples, + size_t n_samples, + std::vector & output, + bool shape_only) const { if (n_samples == 0) { // empty audio return false; @@ -592,31 +537,18 @@ bool mtmd_audio_preprocessor_whisper::preprocess(const float * s n_samples = smpl.size(); } - filter_params params; - params.n_mel = hparams.n_mel_bins; - params.n_fft_bins = 1 + (hparams.audio_n_fft / 2); - params.hann_window_size = hparams.audio_window_len; - params.hop_length = hparams.audio_hop_len; - params.sample_rate = hparams.audio_sample_rate; - params.center_padding = false; - params.preemph = 0.0f; // disabled - params.use_natural_log = false; - params.norm_per_feature = false; - // make sure the cache is initialized GGML_ASSERT(!cache.sin_vals.empty()); GGML_ASSERT(!cache.cos_vals.empty()); GGML_ASSERT(!cache.filters.data.empty()); mtmd_audio_mel out_full; - bool ok = log_mel_spectrogram(samples, n_samples, - 4, // n_threads - params, cache, out_full); + bool ok = log_mel_spectrogram(samples, n_samples, mel_config, shape_only, out_full); if (!ok) { return false; } - // because the cgraph in clip.cpp only accepts 3000 frames each, we need to split the mel + // because the cgraph only accepts 3000 frames each, we need to split the mel // we always expect the mel to have 3000 silent frames at the end if (DEBUG) { printf("output: n_mel = %d, n_len = %d\n", (int) out_full.n_mel, (int) out_full.n_len); @@ -660,11 +592,21 @@ void mtmd_audio_preprocessor_qwen3a::initialize() { cache.fill_sin_cos_table(hparams.audio_n_fft); cache.fill_hann_window(hparams.audio_window_len, true); cache.fill_mel_filterbank_matrix(hparams.n_mel_bins, hparams.audio_n_fft, hparams.audio_sample_rate); + + mel_config.n_mel = hparams.n_mel_bins; + mel_config.n_fft_bins = 1 + (hparams.audio_n_fft / 2); + mel_config.hann_window_size = hparams.audio_window_len; + mel_config.hop_length = hparams.audio_hop_len; + mel_config.sample_rate = hparams.audio_sample_rate; + mel_config.no_padding = true; // reflection padding applied in preprocess() + mel_config.use_natural_log = false; // log10 + mel_config.cache = &cache; } -bool mtmd_audio_preprocessor_qwen3a::preprocess(const float * samples, - size_t n_samples, - std::vector & output) const { +bool mtmd_audio_preprocessor_qwen3a::process(const float * samples, + size_t n_samples, + std::vector & output, + bool shape_only) const { if (n_samples == 0) { return false; } @@ -689,17 +631,8 @@ bool mtmd_audio_preprocessor_qwen3a::preprocess(const float * sa padded[n_samples + pad + i] = (src >= 0) ? samples[src] : 0.0f; } - filter_params params; - params.n_mel = hparams.n_mel_bins; - params.n_fft_bins = 1 + (hparams.audio_n_fft / 2); - params.hann_window_size = hparams.audio_window_len; - params.hop_length = hparams.audio_hop_len; - params.sample_rate = hparams.audio_sample_rate; - params.no_padding = true; // reflection padding already applied above - params.use_natural_log = false; // log10 - mtmd_audio_mel mel_full; - bool ok = log_mel_spectrogram(padded.data(), (int)padded.size(), 4, params, cache, mel_full); + bool ok = log_mel_spectrogram(padded.data(), (int)padded.size(), mel_config, shape_only, mel_full); if (!ok) { return false; } @@ -762,11 +695,21 @@ void mtmd_audio_preprocessor_dots3note::initialize() { cache.fill_sin_cos_table(hparams.audio_n_fft); cache.fill_hann_window(hparams.audio_window_len, true); cache.fill_mel_filterbank_matrix(hparams.n_mel_bins, hparams.audio_n_fft, hparams.audio_sample_rate); + + mel_config.n_mel = hparams.n_mel_bins; + mel_config.n_fft_bins = 1 + (hparams.audio_n_fft / 2); + mel_config.hann_window_size = hparams.audio_window_len; + mel_config.hop_length = hparams.audio_hop_len; + mel_config.sample_rate = hparams.audio_sample_rate; + mel_config.no_padding = true; // padding applied in preprocess() + mel_config.use_natural_log = false; + mel_config.cache = &cache; } -bool mtmd_audio_preprocessor_dots3note::preprocess(const float * samples, - size_t n_samples, - std::vector & output) const { +bool mtmd_audio_preprocessor_dots3note::process(const float * samples, + size_t n_samples, + std::vector & output, + bool shape_only) const { if (n_samples == 0) { return false; } @@ -803,17 +746,8 @@ bool mtmd_audio_preprocessor_dots3note::preprocess(const float * } } - filter_params params; - params.n_mel = hparams.n_mel_bins; - params.n_fft_bins = 1 + (hparams.audio_n_fft / 2); - params.hann_window_size = hparams.audio_window_len; - params.hop_length = hop; - params.sample_rate = hparams.audio_sample_rate; - params.no_padding = true; // padding already applied above - params.use_natural_log = false; - mtmd_audio_mel mel_full; - if (!log_mel_spectrogram(padded.data(), (int) padded.size(), 4, params, cache, mel_full)) { + if (!log_mel_spectrogram(padded.data(), (int) padded.size(), mel_config, shape_only, mel_full)) { return false; } GGML_ASSERT(mel_full.n_len >= n_valid); @@ -862,11 +796,24 @@ void mtmd_audio_preprocessor_mimo_audio::initialize() { /*scale=*/ 1.0f, /*use_htk=*/ true ); + + mel_config.n_mel = hparams.n_mel_bins; + mel_config.n_fft_bins = 1 + (hparams.audio_n_fft / 2); + mel_config.hann_window_size = hparams.audio_window_len; + mel_config.hop_length = hparams.audio_hop_len; + mel_config.sample_rate = hparams.audio_sample_rate; + mel_config.no_padding = true; // reflect padding applied in preprocess() + mel_config.use_natural_log = true; + mel_config.use_magnitude = true; + mel_config.mel_floor = 1e-7f; + mel_config.norm_per_feature = false; + mel_config.cache = &cache; } -bool mtmd_audio_preprocessor_mimo_audio::preprocess(const float * samples, - size_t n_samples, - std::vector & output) const { +bool mtmd_audio_preprocessor_mimo_audio::process(const float * samples, + size_t n_samples, + std::vector & output, + bool shape_only) const { if (n_samples == 0) { return false; } @@ -888,20 +835,8 @@ bool mtmd_audio_preprocessor_mimo_audio::preprocess(const float * padded[n_samples + pad + i] = (src >= 0) ? samples[src] : 0.0f; } - filter_params params; - params.n_mel = hparams.n_mel_bins; - params.n_fft_bins = 1 + (hparams.audio_n_fft / 2); - params.hann_window_size = hparams.audio_window_len; - params.hop_length = hparams.audio_hop_len; - params.sample_rate = hparams.audio_sample_rate; - params.no_padding = true; // reflect padding already applied above - params.use_natural_log = true; - params.use_magnitude = true; - params.mel_floor = 1e-7f; - params.norm_per_feature = false; - mtmd_audio_mel out; - bool ok = log_mel_spectrogram(padded.data(), (int)padded.size(), 4, params, cache, out); + bool ok = log_mel_spectrogram(padded.data(), (int)padded.size(), mel_config, shape_only, out); if (!ok) { return false; } @@ -921,11 +856,23 @@ void mtmd_audio_preprocessor_qwen3tts_spk::initialize() { cache.fill_sin_cos_table(hparams.audio_n_fft); cache.fill_hann_window(hparams.audio_window_len, true); cache.fill_mel_filterbank_matrix(hparams.n_mel_bins, hparams.audio_n_fft, hparams.audio_sample_rate); + + mel_config.n_mel = hparams.n_mel_bins; + mel_config.n_fft_bins = 1 + (hparams.audio_n_fft / 2); + mel_config.hann_window_size = hparams.audio_window_len; + mel_config.hop_length = hparams.audio_hop_len; + mel_config.sample_rate = hparams.audio_sample_rate; + mel_config.no_padding = true; // reflect padding applied in preprocess() + mel_config.use_natural_log = true; + mel_config.use_magnitude = true; + mel_config.mel_floor = 1e-5f; + mel_config.cache = &cache; } -bool mtmd_audio_preprocessor_qwen3tts_spk::preprocess(const float * samples, - size_t n_samples, - std::vector & output) const { +bool mtmd_audio_preprocessor_qwen3tts_spk::process(const float * samples, + size_t n_samples, + std::vector & output, + bool shape_only) const { if (n_samples == 0) { return false; } @@ -949,19 +896,8 @@ bool mtmd_audio_preprocessor_qwen3tts_spk::preprocess(const float * padded[n_samples + pad + i] = samples[n_samples - 2 - i]; } - filter_params params; - params.n_mel = hparams.n_mel_bins; - params.n_fft_bins = 1 + (hparams.audio_n_fft / 2); - params.hann_window_size = hparams.audio_window_len; - params.hop_length = hparams.audio_hop_len; - params.sample_rate = hparams.audio_sample_rate; - params.no_padding = true; // reflect padding already applied above - params.use_natural_log = true; - params.use_magnitude = true; - params.mel_floor = 1e-5f; - mtmd_audio_mel out; - bool ok = log_mel_spectrogram(padded.data(), (int) padded.size(), 4, params, cache, out); + bool ok = log_mel_spectrogram(padded.data(), (int) padded.size(), mel_config, shape_only, out); if (!ok) { return false; } @@ -970,6 +906,481 @@ bool mtmd_audio_preprocessor_qwen3tts_spk::preprocess(const float * return true; } +// windowed frame of a centered, reflect padded signal: dst[i] = x[start + i] * win[i], zeros past the window +static void mtmd_audio_reflect_frame(const float * x, int n, int start, const std::vector & win, std::vector & dst) { + std::fill(dst.begin(), dst.end(), 0.0f); + for (int i = 0; i < (int) win.size(); i++) { + int idx = start + i; + if (idx < 0) { + idx = -idx; + } + if (idx >= n) { + idx = 2 * n - 2 - idx; + } + dst[(size_t) i] = x[idx] * win[(size_t) i]; + } +} + +// mel energies of the frame in fft_in: power (or magnitude) spectrum through cache.filters +static void mtmd_audio_mel_energies(const mtmd_audio_cache & cache, std::vector & fft_in, std::vector & fft_out, + bool magnitude, std::vector & mel) { + const int n_fft = (int) cache.filters.n_fft; + const int n_bins = n_fft / 2 + 1; + fft(cache, fft_in.data(), n_fft, fft_out.data()); + for (int k = 0; k < n_bins; k++) { + const float p = fft_out[2 * k + 0] * fft_out[2 * k + 0] + fft_out[2 * k + 1] * fft_out[2 * k + 1]; + fft_out[k] = magnitude ? sqrtf(p + 1e-9f) : p; + } + mel.assign(cache.filters.n_mel, 0.0); + for (int64_t m = 0; m < cache.filters.n_mel; m++) { + const float * w = cache.filters.data.data() + (size_t) m * n_bins; + for (int k = 0; k < n_bins; k++) { + mel[(size_t) m] += w[k] * fft_out[k]; + } + } +} + +// whisper style log-mel of the chatterbox s3 tokenizer (s3tokenizer.py), mel major +static bool mtmd_audio_s3tok_log_mel(const mtmd_audio_cache & cache, const float * samples, size_t n_samples, + std::vector & out, int & n_frames) { + const int n_fft = 400; + const int hop = 160; + const int n_mel = (int) cache.filters.n_mel; + const int n = (int) n_samples; + + n_frames = n / hop; + if (n_frames <= 0) { + return false; + } + + std::vector fft_in((size_t) n_fft * 2), fft_out((size_t) n_fft * 8); + std::vector e; + std::vector mel((size_t) n_mel * n_frames); + for (int fr = 0; fr < n_frames; fr++) { + mtmd_audio_reflect_frame(samples, n, fr * hop - n_fft / 2, cache.hann_window, fft_in); + mtmd_audio_mel_energies(cache, fft_in, fft_out, false, e); + for (int m = 0; m < n_mel; m++) { + mel[(size_t) m * n_frames + fr] = log10(std::max(e[(size_t) m], 1e-10)); + } + } + + const double mx = *std::max_element(mel.begin(), mel.end()); + out.resize(mel.size()); + for (size_t i = 0; i < mel.size(); i++) { + out[i] = (float) ((std::max(mel[i], mx - 8.0) + 4.0) / 4.0); + } + return true; +} + +// 16 kHz -> 24 kHz upsampler: hann windowed sinc cut just under the source nyquist +static void mtmd_audio_upsample_3_2(const float * samples, size_t n_samples, std::vector & out) { + const int W = 16; // sinc half width in source samples + const double fc = 0.495; // cutoff, normalized to the source rate + + const size_t n_out = n_samples * 3 / 2; + out.assign(n_out, 0.0f); + for (size_t n = 0; n < n_out; n++) { + const double t = (double) (2 * n) / 3.0; + const int k0 = (int) floor(t) - W + 1; + double acc = 0.0; + for (int k = k0; k < k0 + 2 * W; k++) { + if (k < 0 || k >= (int) n_samples) { + continue; + } + const double x = t - k; + const double s = x == 0.0 ? 1.0 : sin(2.0 * M_PI * fc * x) / (M_PI * x); + acc += samples[k] * s * 0.5 * (1.0 + cos(M_PI * x / (W + 1))); + } + out[n] = (float) acc; + } +} + +// matcha style log-mel of the s3gen prompt features (s3gen/utils/mel.py), frame major +static bool mtmd_audio_matcha_log_mel(const mtmd_audio_cache & cache, const float * samples, size_t n_samples, + std::vector & out, int & n_frames) { + const int n_fft = 1920; + const int hop = 480; + const int pad = (n_fft - hop) / 2; + const int n_mel = 80; + const int n = (int) n_samples; + + n_frames = 1 + (n + 2 * pad - n_fft) / hop; + if (n <= pad || n_frames <= 0) { + return false; + } + + std::vector fft_in((size_t) n_fft * 2), fft_out((size_t) n_fft * 8); + std::vector e; + out.resize((size_t) n_mel * n_frames); + for (int fr = 0; fr < n_frames; fr++) { + mtmd_audio_reflect_frame(samples, n, fr * hop - pad, cache.hann_window, fft_in); + mtmd_audio_mel_energies(cache, fft_in, fft_out, true, e); + for (int m = 0; m < n_mel; m++) { + out[(size_t) fr * n_mel + m] = logf(std::max((float) e[(size_t) m], 1e-5f)); + } + } + return true; +} + +// librosa.effects.trim: non-silent where the rms window is within top_db of the peak +static void mtmd_audio_trim_silence(const float * samples, size_t n_samples, float top_db, + size_t & start, size_t & end) { + const int win = 2048; + const int hop = 512; + const int n = (int) n_samples; + + const int n_fr = 1 + n / hop; + std::vector rms((size_t) n_fr); + double mx = 0.0; + for (int fr = 0; fr < n_fr; fr++) { + double acc = 0.0; + for (int i = 0; i < win; i++) { + const int idx = fr * hop - win / 2 + i; + if (idx >= 0 && idx < n) { + acc += (double) samples[idx] * samples[idx]; + } + } + rms[(size_t) fr] = sqrt(acc / win); + mx = std::max(mx, rms[(size_t) fr]); + } + + const double thr = mx * pow(10.0, -top_db / 20.0); + int first = -1, last = -1; + for (int fr = 0; fr < n_fr; fr++) { + if (rms[(size_t) fr] > thr) { + if (first < 0) { + first = fr; + } + last = fr; + } + } + if (first < 0) { + start = end = 0; + return; + } + start = (size_t) first * hop; + end = std::min((size_t) (last + 1) * hop, n_samples); +} + +// power mel of the voice encoder front-end, frame major, no log +static bool mtmd_audio_ve_mel(const mtmd_audio_cache & cache, const float * samples, size_t n_samples, + std::vector & out, int & n_frames) { + const int n_fft = 400; + const int hop = 160; + const int n_mel = 40; + const int n = (int) n_samples; + + n_frames = 1 + n / hop; + if (n < 2) { + return false; + } + + std::vector fft_in((size_t) n_fft * 2), fft_out((size_t) n_fft * 8); + std::vector e; + out.resize((size_t) n_mel * n_frames); + for (int fr = 0; fr < n_frames; fr++) { + mtmd_audio_reflect_frame(samples, n, fr * hop - n_fft / 2, cache.hann_window, fft_in); + mtmd_audio_mel_energies(cache, fft_in, fft_out, false, e); + for (int m = 0; m < n_mel; m++) { + out[(size_t) fr * n_mel + m] = (float) e[(size_t) m]; + } + } + return true; +} + +// ITU-R BS.1770 integrated loudness of a mono signal, matching pyloudnorm +static float mtmd_audio_lufs(const float * samples, size_t n_samples, int sample_rate) { + std::vector y(samples, samples + n_samples); + + auto biquad = [&](double b0, double b1, double b2, double a1, double a2) { + double x1 = 0.0, x2 = 0.0, y1 = 0.0, y2 = 0.0; + for (double & v : y) { + const double x0 = v; + v = b0 * x0 + b1 * x1 + b2 * x2 - a1 * y1 - a2 * y2; + x2 = x1; x1 = x0; + y2 = y1; y1 = v; + } + }; + + // stage 1: high shelf, f0 1681.9744509555319 Hz, +3.99984385397 dB, Q 0.7071752369554196 + { + const double A = pow(10.0, 3.99984385397 / 40.0); + const double w0 = 2.0 * M_PI * 1681.9744509555319 / sample_rate; + const double alpha = sin(w0) / (2.0 * 0.7071752369554196); + const double c = cos(w0); + const double sq = 2.0 * sqrt(A) * alpha; + const double b0 = A * ((A + 1.0) + (A - 1.0) * c + sq); + const double b1 = -2.0 * A * ((A - 1.0) + (A + 1.0) * c); + const double b2 = A * ((A + 1.0) + (A - 1.0) * c - sq); + const double a0 = (A + 1.0) - (A - 1.0) * c + sq; + const double a1 = 2.0 * ((A - 1.0) - (A + 1.0) * c); + const double a2 = (A + 1.0) - (A - 1.0) * c - sq; + biquad(b0 / a0, b1 / a0, b2 / a0, a1 / a0, a2 / a0); + } + // stage 2: high pass, f0 38.13547087602444 Hz, Q 0.5003270373238773 + { + const double w0 = 2.0 * M_PI * 38.13547087602444 / sample_rate; + const double alpha = sin(w0) / (2.0 * 0.5003270373238773); + const double c = cos(w0); + const double b0 = (1.0 + c) / 2.0; + const double b1 = -(1.0 + c); + const double b2 = (1.0 + c) / 2.0; + const double a0 = 1.0 + alpha; + const double a1 = -2.0 * c; + const double a2 = 1.0 - alpha; + biquad(b0 / a0, b1 / a0, b2 / a0, a1 / a0, a2 / a0); + } + + const int win = (int) (0.4 * sample_rate); + const int hop = win / 4; + if ((int) n_samples < win) { + return -HUGE_VALF; + } + const int n_blocks = 1 + ((int) n_samples - win) / hop; + std::vector z((size_t) n_blocks); + for (int b = 0; b < n_blocks; b++) { + double acc = 0.0; + for (int i = 0; i < win; i++) { + acc += y[(size_t) b * hop + i] * y[(size_t) b * hop + i]; + } + z[(size_t) b] = acc / win; + } + + auto gated_mean = [&](double thr_lufs) { + double acc = 0.0; + int n = 0; + for (const double v : z) { + if (-0.691 + 10.0 * log10(std::max(v, 1e-30)) > thr_lufs) { + acc += v; + n++; + } + } + return n > 0 ? acc / n : 0.0; + }; + + const double z_abs = gated_mean(-70.0); + if (z_abs <= 0.0) { + return -HUGE_VALF; + } + const double thr_rel = -0.691 + 10.0 * log10(z_abs) - 10.0; + const double z_rel = gated_mean(thr_rel); + if (z_rel <= 0.0) { + return -HUGE_VALF; + } + return (float) (-0.691 + 10.0 * log10(z_rel)); +} + +// +// mtmd_audio_preprocessor_chatterbox_ref +// + +void mtmd_audio_preprocessor_chatterbox_ref::initialize() { + // kaldi fbank: povey window, kaldi mel banks over the 512-point spectrum, the nyquist bin left out + { + const int frame_len = 400; + const int n_fft = 512; + const int n_bins = n_fft / 2 + 1; + const int n_mel = hparams.n_mel_bins; + const double sr = (double) hparams.audio_sample_rate; + + fbank_cache.fill_sin_cos_table(n_fft); + fbank_cache.hann_window.resize(frame_len); + for (int i = 0; i < frame_len; i++) { + fbank_cache.hann_window[(size_t) i] = (float) pow(0.5 - 0.5 * cos(2.0 * M_PI * i / (frame_len - 1)), 0.85); + } + + auto mel = [](double f) { return 1127.0 * log(1.0 + f / 700.0); }; + const double mel_lo = mel(20.0); + const double mel_hi = mel(sr / 2.0); + const double delta = (mel_hi - mel_lo) / (n_mel + 1); + const double bin_hz = sr / n_fft; + + auto & filters = fbank_cache.filters; + filters.n_mel = n_mel; + filters.n_fft = n_fft; + filters.data.assign((size_t) n_mel * n_bins, 0.0f); + for (int m = 0; m < n_mel; m++) { + const double left = mel_lo + m * delta; + const double center = left + delta; + const double right = center + delta; + for (int i = 0; i < n_bins - 1; i++) { + const double f = mel(bin_hz * i); + if (f > left && f < right) { + const double w = f <= center ? (f - left) / (center - left) + : (right - f) / (right - center); + filters.data[(size_t) m * n_bins + i] = (float) w; + } + } + } + } + + // s3 tokenizer: librosa slaney mel, 128 bands + s3tok_cache.fill_sin_cos_table(400); + s3tok_cache.fill_hann_window(400, true); + s3tok_cache.fill_mel_filterbank_matrix(128, 400, 16000); + + // voice encoder: slaney mel, 40 bands up to 8 kHz + ve_cache.fill_sin_cos_table(400); + ve_cache.fill_hann_window(400, true); + ve_cache.fill_mel_filterbank_matrix(40, 400, 16000, 0.0f, 8000.0f); + + // s3gen prompt features at the 24 kHz decoder rate + matcha_cache.fill_sin_cos_table(1920); + matcha_cache.fill_hann_window(1920, true); + matcha_cache.fill_mel_filterbank_matrix(80, 1920, 24000, 0.0f, 8000.0f); +} + +bool mtmd_audio_preprocessor_chatterbox_ref::process(const float * samples, + size_t n_samples, + std::vector & output, + bool /*shape_only*/) const { + output.clear(); + const int sr = (int) hparams.audio_sample_rate; + + // the turbo variant loudness-normalizes the whole clip before any feature + std::vector pcm(samples, samples + n_samples); + if (!is_mtl) { + const float lufs = mtmd_audio_lufs(pcm.data(), pcm.size(), sr); + if (lufs != -HUGE_VALF) { + const float gain = powf(10.0f, (-27.0f - lufs) / 20.0f); + if (std::isfinite(gain) && gain > 0.0f) { + for (float & v : pcm) { + v *= gain; + } + } + } + } + + // entry 0: torchaudio kaldi.fbank of CAMPPlus (snip edges, dc removal, preemphasis 0.97), mean subtracted over time + { + const int frame_len = 400; + const int hop = 160; + const int n_fft = 512; + const int n_mel = hparams.n_mel_bins; + + if ((int) pcm.size() < frame_len) { + return false; + } + const int n_frames = 1 + ((int) pcm.size() - frame_len) / hop; + + mtmd_audio_mel out; + out.n_len = n_frames; + out.n_len_org = n_frames; + out.n_mel = n_mel; + out.data.assign((size_t) n_mel * n_frames, 0.0f); + + const std::vector & win = fbank_cache.hann_window; + std::vector fft_in((size_t) n_fft * 2), fft_out((size_t) n_fft * 8); + std::vector e; + for (int fr = 0; fr < n_frames; fr++) { + const float * x = pcm.data() + (size_t) fr * hop; + + double mean = 0.0; + for (int i = 0; i < frame_len; i++) { + mean += x[i]; + } + mean /= frame_len; + + std::fill(fft_in.begin(), fft_in.end(), 0.0f); + fft_in[0] = (float) ((x[0] - mean) * (1.0 - 0.97) * win[0]); + for (int i = 1; i < frame_len; i++) { + fft_in[(size_t) i] = (float) (((x[i] - mean) - 0.97 * (x[i - 1] - mean)) * win[(size_t) i]); + } + mtmd_audio_mel_energies(fbank_cache, fft_in, fft_out, false, e); + for (int m = 0; m < n_mel; m++) { + out.data[(size_t) m * n_frames + fr] = (float) log(std::max(e[(size_t) m], (double) FLT_EPSILON)); + } + } + + for (int m = 0; m < n_mel; m++) { + float * row = out.data.data() + (size_t) m * n_frames; + double mean = 0.0; + for (int fr = 0; fr < n_frames; fr++) { + mean += row[fr]; + } + mean /= n_frames; + for (int fr = 0; fr < n_frames; fr++) { + row[fr] -= (float) mean; + } + } + + output.push_back(std::move(out)); + } + + // entries 1 and 2: s3 tokenizer log-mels at the flow and t3 caps, padded to whole 40 ms tokens + auto s3tok_entry = [&](size_t cap) -> bool { + std::vector clip(pcm.begin(), pcm.begin() + std::min(pcm.size(), cap)); + clip.resize((clip.size() + 639) / 640 * 640, 0.0f); + std::vector mel; + int n_frames = 0; + if (!mtmd_audio_s3tok_log_mel(s3tok_cache, clip.data(), clip.size(), mel, n_frames)) { + return false; + } + mtmd_audio_mel out; + out.n_len = n_frames; + out.n_len_org = n_frames; + out.n_mel = s3tok_cache.filters.n_mel; + out.data = std::move(mel); + output.push_back(std::move(out)); + return true; + }; + const size_t gen_cap = (size_t) 10 * sr; + const size_t t3_cap = (size_t) (is_mtl ? 6 : 15) * sr; + if (!s3tok_entry(gen_cap) || !s3tok_entry(t3_cap)) { + return false; + } + + // entry 3: s3gen prompt features of the flow-capped clip at 24 kHz + { + std::vector clip(pcm.begin(), pcm.begin() + std::min(pcm.size(), gen_cap)); + clip.resize((clip.size() + 639) / 640 * 640, 0.0f); + std::vector pcm24; + mtmd_audio_upsample_3_2(clip.data(), clip.size(), pcm24); + std::vector feat; + int n_frames = 0; + if (!mtmd_audio_matcha_log_mel(matcha_cache, pcm24.data(), pcm24.size(), feat, n_frames)) { + return false; + } + mtmd_audio_mel out; + out.n_len = n_frames; + out.n_len_org = n_frames; + out.n_mel = 80; + out.data = std::move(feat); + output.push_back(std::move(out)); + } + + // entry 4: voice encoder mel of the silence-trimmed clip, padded to the 160-frame partial grid (rate 1.3) + { + size_t t0 = 0, t1 = 0; + mtmd_audio_trim_silence(pcm.data(), pcm.size(), 20.0f, t0, t1); + if (t1 <= t0) { + return false; + } + std::vector mel; + int n_frames = 0; + if (!mtmd_audio_ve_mel(ve_cache, pcm.data() + t0, t1 - t0, mel, n_frames)) { + return false; + } + const int n_mel = 40; + const int n_partial = 160; + const int step = (int) lround((16000.0 / 1.3) / n_partial); + int n_wins = std::max(n_frames - n_partial + step, 0) / step; + const int rem = std::max(n_frames - n_partial + step, 0) % step; + if (n_wins == 0 || (double) (rem + n_partial - step) / n_partial >= 0.8) { + n_wins++; + } + const int target = n_partial + step * (n_wins - 1); + mel.resize((size_t) target * n_mel, 0.0f); + mtmd_audio_mel out; + out.n_len = target; + out.n_len_org = target; + out.n_mel = n_mel; + out.data = std::move(mel); + output.push_back(std::move(out)); + } + return true; +} + // // mtmd_audio_preprocessor_conformer // @@ -979,38 +1390,37 @@ void mtmd_audio_preprocessor_conformer::initialize() { // NeMo uses a symmetric window: torch.hann_window(periodic=False) cache.fill_hann_window(hparams.audio_window_len, false); cache.fill_mel_filterbank_matrix(hparams.n_mel_bins, hparams.audio_n_fft, hparams.audio_sample_rate); + + mel_config.n_mel = hparams.n_mel_bins; + mel_config.n_fft_bins = 1 + (hparams.audio_n_fft / 2); + mel_config.hann_window_size = hparams.audio_window_len; + mel_config.hop_length = hparams.audio_hop_len; + mel_config.sample_rate = hparams.audio_sample_rate; + mel_config.center_padding = true; + mel_config.preemph = 0.97f; + mel_config.use_natural_log = true; + mel_config.norm_per_feature = true; + mel_config.mel_floor_add = true; + mel_config.std_eps_after_sqrt = true; + mel_config.cache = &cache; } -bool mtmd_audio_preprocessor_conformer::preprocess(const float * samples, - size_t n_samples, - std::vector & output) const { +bool mtmd_audio_preprocessor_conformer::process(const float * samples, + size_t n_samples, + std::vector & output, + bool shape_only) const { // empty audio if (n_samples == 0) { return false; } - filter_params params; - params.n_mel = hparams.n_mel_bins; - params.n_fft_bins = 1 + (hparams.audio_n_fft / 2); - params.hann_window_size = hparams.audio_window_len; - params.hop_length = hparams.audio_hop_len; - params.sample_rate = hparams.audio_sample_rate; - params.center_padding = true; - params.preemph = 0.97f; - params.use_natural_log = true; - params.norm_per_feature = true; - params.mel_floor_add = true; - params.std_eps_after_sqrt = true; - // make sure the cache is initialized GGML_ASSERT(!cache.sin_vals.empty()); GGML_ASSERT(!cache.cos_vals.empty()); GGML_ASSERT(!cache.filters.data.empty()); mtmd_audio_mel out_full; - bool ok = log_mel_spectrogram(samples, n_samples, - 4, // n_threads - params, cache, out_full); + bool ok = log_mel_spectrogram(samples, n_samples, mel_config, shape_only, out_full); if (!ok) { return false; } @@ -1023,9 +1433,10 @@ bool mtmd_audio_preprocessor_conformer::preprocess(const float * // mtmd_audio_preprocessor_d1omni // -bool mtmd_audio_preprocessor_d1omni::preprocess(const float * samples, - size_t n_samples, - std::vector & output) const { +bool mtmd_audio_preprocessor_d1omni::process(const float * samples, + size_t n_samples, + std::vector & output, + bool shape_only) const { if (n_samples == 0) { return false; } @@ -1035,7 +1446,7 @@ bool mtmd_audio_preprocessor_d1omni::preprocess(const float * sa std::vector buf(samples, samples + std::min(n_samples, n_max)); buf.resize(std::max(buf.size(), n_min), 0.0f); - if (!mtmd_audio_preprocessor_conformer::preprocess(buf.data(), buf.size(), output)) { + if (!mtmd_audio_preprocessor_conformer::process(buf.data(), buf.size(), output, shape_only)) { return false; } @@ -1063,11 +1474,21 @@ void mtmd_audio_preprocessor_lfm3a::initialize() { cache.fill_sin_cos_table(hparams.audio_n_fft); cache.fill_hann_window(hparams.audio_window_len, true); cache.fill_mel_filterbank_matrix(hparams.n_mel_bins, hparams.audio_n_fft, hparams.audio_sample_rate); + + mel_config.n_mel = hparams.n_mel_bins; + mel_config.n_fft_bins = 1 + (hparams.audio_n_fft / 2); + mel_config.hann_window_size = hparams.audio_window_len; + mel_config.hop_length = hparams.audio_hop_len; + mel_config.sample_rate = hparams.audio_sample_rate; + mel_config.no_padding = true; + mel_config.mel_floor = 1e-12f; // log10 floor of -12 + mel_config.cache = &cache; } -bool mtmd_audio_preprocessor_lfm3a::preprocess(const float * samples, - size_t n_samples, - std::vector & output) const { +bool mtmd_audio_preprocessor_lfm3a::process(const float * samples, + size_t n_samples, + std::vector & output, + bool shape_only) const { if (n_samples == 0) { return false; } @@ -1083,21 +1504,12 @@ bool mtmd_audio_preprocessor_lfm3a::preprocess(const float * sam std::vector padded(samples, samples + n_samples); padded.resize(n_padded, 0.0f); - filter_params params; - params.n_mel = hparams.n_mel_bins; - params.n_fft_bins = 1 + (hparams.audio_n_fft / 2); - params.hann_window_size = hparams.audio_window_len; - params.hop_length = hparams.audio_hop_len; - params.sample_rate = hparams.audio_sample_rate; - params.no_padding = true; - params.mel_floor = 1e-12f; // log10 floor of -12 - GGML_ASSERT(!cache.sin_vals.empty()); GGML_ASSERT(!cache.cos_vals.empty()); GGML_ASSERT(!cache.filters.data.empty()); mtmd_audio_mel out; - bool ok = log_mel_spectrogram(padded.data(), (int) padded.size(), 4, params, cache, out); + bool ok = log_mel_spectrogram(padded.data(), (int) padded.size(), mel_config, shape_only, out); if (!ok) { return false; } @@ -1120,11 +1532,25 @@ void mtmd_audio_preprocessor_granite_speech::initialize() { cache.fill_mel_filterbank_matrix( hparams.n_mel_bins / 2, hparams.audio_n_fft, hparams.audio_sample_rate, 0.0f, -1.0f, false, 1.0f, true); + + mel_config.n_mel = hparams.n_mel_bins / 2; + mel_config.n_fft_bins = 1 + (hparams.audio_n_fft / 2); + mel_config.hann_window_size = hparams.audio_window_len; + mel_config.hop_length = hparams.audio_hop_len; + mel_config.sample_rate = hparams.audio_sample_rate; + mel_config.no_padding = true; + mel_config.center_padding = false; + mel_config.preemph = 0.0f; + mel_config.use_natural_log = false; + mel_config.norm_per_feature = false; + mel_config.mel_floor = 1e-10f; + mel_config.cache = &cache; } -bool mtmd_audio_preprocessor_granite_speech::preprocess(const float * samples, - size_t n_samples, - std::vector & output) const { +bool mtmd_audio_preprocessor_granite_speech::process(const float * samples, + size_t n_samples, + std::vector & output, + bool shape_only) const { if (n_samples == 0) { return false; } @@ -1155,21 +1581,8 @@ bool mtmd_audio_preprocessor_granite_speech::preprocess(const float * padded[pad + (int)n_samples + i] = samples[src]; } - filter_params params; - params.n_mel = hparams.n_mel_bins / 2; - params.n_fft_bins = 1 + (n_fft / 2); - params.hann_window_size = hparams.audio_window_len; - params.hop_length = hparams.audio_hop_len; - params.sample_rate = hparams.audio_sample_rate; - params.no_padding = true; - params.center_padding = false; - params.preemph = 0.0f; - params.use_natural_log = false; - params.norm_per_feature = false; - params.mel_floor = 1e-10f; - mtmd_audio_mel mel; - if (!log_mel_spectrogram(padded.data(), n_padded, 4, params, cache, mel)) { + if (!log_mel_spectrogram(padded.data(), n_padded, mel_config, shape_only, mel)) { return false; } @@ -1234,11 +1647,26 @@ void mtmd_audio_preprocessor_gemma4a::initialize() { /*scale=*/ 1.0f, /*use_htk=*/ true ); + + mel_config.n_mel = hparams.n_mel_bins; + mel_config.n_fft_bins = 1 + (hparams.audio_n_fft / 2); + mel_config.hann_window_size = hparams.audio_n_fft; // window is zero-padded to FFT size + mel_config.hop_length = hparams.audio_hop_len; + mel_config.sample_rate = hparams.audio_sample_rate; + mel_config.no_padding = true; + mel_config.center_padding = false; + mel_config.preemph = 0.0f; + mel_config.use_natural_log = true; + mel_config.use_magnitude = true; + mel_config.mel_floor = 0.001f; + mel_config.norm_per_feature = false; + mel_config.cache = &cache; } -bool mtmd_audio_preprocessor_gemma4a::preprocess(const float * samples, - size_t n_samples, - std::vector & output) const { +bool mtmd_audio_preprocessor_gemma4a::process(const float * samples, + size_t n_samples, + std::vector & output, + bool shape_only) const { if (n_samples == 0) { return false; } @@ -1247,22 +1675,8 @@ bool mtmd_audio_preprocessor_gemma4a::preprocess(const float * s GGML_ASSERT(!cache.cos_vals.empty()); GGML_ASSERT(!cache.filters.data.empty()); - filter_params params; - params.n_mel = hparams.n_mel_bins; - params.n_fft_bins = 1 + (hparams.audio_n_fft / 2); - params.hann_window_size = hparams.audio_n_fft; // window is zero-padded to FFT size - params.hop_length = hparams.audio_hop_len; - params.sample_rate = hparams.audio_sample_rate; - params.no_padding = true; - params.center_padding = false; - params.preemph = 0.0f; - params.use_natural_log = true; - params.use_magnitude = true; - params.mel_floor = 0.001f; - params.norm_per_feature = false; - // Split into 30-second chunks (model context limit, ~750 tokens each) - const size_t chunk_samples = 30 * hparams.audio_sample_rate; + const size_t chunk_samples = (size_t) segment_seconds * hparams.audio_sample_rate; for (size_t off = 0; off < n_samples; off += chunk_samples) { const float * chunk_ptr = samples + off; size_t chunk_len = std::min(chunk_samples, n_samples - off); @@ -1280,7 +1694,7 @@ bool mtmd_audio_preprocessor_gemma4a::preprocess(const float * s std::copy(chunk_ptr, chunk_ptr + chunk_len, padded_samples.data() + pad_left); mtmd_audio_mel out_chunk; - bool ok = log_mel_spectrogram(padded_samples.data(), padded_samples.size(), 4, params, cache, out_chunk); + bool ok = log_mel_spectrogram(padded_samples.data(), padded_samples.size(), mel_config, shape_only, out_chunk); if (!ok) { return false; } @@ -1298,79 +1712,6 @@ bool mtmd_audio_preprocessor_gemma4a::preprocess(const float * s // mtmd_audio_preprocessor_parakeet implementation // -void mtmd_audio_preprocessor_parakeet::worker_thread( - int ith, - const float * window_func, - int window_size, - const std::vector & samples, - int n_samples, - int frame_size, - int frame_step, - int n_threads, - int n_fft_bins, - const mtmd_audio_cache & cache, - mtmd_audio_mel & mel) { - std::vector fft_in(frame_size * 2, 0.0); - std::vector fft_out(frame_size * 2 * 2 * 2); - - int n_fb = n_fft_bins; - int i = ith; - - GGML_ASSERT(n_fb == 1 + (frame_size / 2)); - - const double eps = 5.960464477539063e-08; - - for (; i < std::min(n_samples / frame_step + 1, (int) mel.n_len); i += n_threads) { - const int offset = i * frame_step; - const int window_pad_left = (frame_size - window_size) / 2; - - // Zero-pad left. - std::fill(fft_in.begin(), fft_in.begin() + window_pad_left, 0.0f); - - // Apply windowed samples in the center. - const int n_to_process = std::min({window_size, n_samples - offset}); - for (int j = 0; j < n_to_process; j++) { - fft_in[window_pad_left + j] = window_func[j] * samples[offset + window_pad_left + j]; - } - - // Zero-pad right. - std::fill(fft_in.begin() + window_pad_left + n_to_process, fft_in.begin() + frame_size, 0.0f); - - // FFT. - fft(cache, fft_in.data(), frame_size, fft_out.data()); - - // Calculate modulus^2 of complex numbers. - for (int j = 0; j < n_fb; j++) { - fft_out[j] = (fft_out[2 * j + 0] * fft_out[2 * j + 0] + fft_out[2 * j + 1] * fft_out[2 * j + 1]); - } - - // mel spectrogram. - for (int j = 0; j < mel.n_mel; j++) { - double sum = 0.0; - int k = 0; - for (k = 0; k < n_fb - 3; k += 4) { - sum += - fft_out[k + 0] * cache.filters.data[j * n_fb + k + 0] + - fft_out[k + 1] * cache.filters.data[j * n_fb + k + 1] + - fft_out[k + 2] * cache.filters.data[j * n_fb + k + 2] + - fft_out[k + 3] * cache.filters.data[j * n_fb + k + 3]; - } - for (; k < n_fb; k++) { - sum += fft_out[k] * cache.filters.data[j * n_fb + k]; - } - mel.data[j * mel.n_len + i] = std::log(sum + eps); - } - } - - // Otherwise fft_out are all zero. - const double empty_sum = std::log(eps); - for (; i < mel.n_len; i += n_threads) { - for (int j = 0; j < mel.n_mel; j++) { - mel.data[j * mel.n_len + i] = empty_sum; - } - } -} - void mtmd_audio_preprocessor_parakeet::initialize() { cache.fill_sin_cos_table(hparams.audio_n_fft); @@ -1383,30 +1724,25 @@ void mtmd_audio_preprocessor_parakeet::initialize() { GGML_ASSERT(hparams.window.size() == (size_t)hparams.audio_window_len); GGML_ASSERT(hparams.window.size() <= (size_t) hparams.audio_n_fft); cache.hann_window = hparams.window; -} -bool mtmd_audio_preprocessor_parakeet::preprocess(const float * samples, - size_t n_samples_in, - std::vector & output) const { - if (n_samples_in == 0) { - return false; - } - - filter_params params; - params.n_mel = hparams.n_mel_bins; - params.n_fft_bins = 1 + (hparams.audio_n_fft / 2); - params.hann_window_size = hparams.audio_window_len; - params.hop_length = hparams.audio_hop_len; - params.sample_rate = hparams.audio_sample_rate; + mel_config.n_mel = hparams.n_mel_bins; + mel_config.n_fft_bins = 1 + (hparams.audio_n_fft / 2); + mel_config.hann_window_size = hparams.audio_window_len; + mel_config.hop_length = hparams.audio_hop_len; + mel_config.sample_rate = hparams.audio_sample_rate; + mel_config.no_padding = true; // preemphasis and padding already applied in log_mel() + mel_config.use_natural_log = true; + mel_config.mel_floor_add = true; + mel_config.cache = &cache; +} +// log-mel without normalization +bool mtmd_audio_preprocessor_parakeet::log_mel(const float * samples, size_t n_samples_in, mtmd_audio_mel & out_full, bool shape_only) const { GGML_ASSERT(!cache.sin_vals.empty()); GGML_ASSERT(!cache.cos_vals.empty()); GGML_ASSERT(!cache.filters.data.empty()); - const float * window_func = cache.hann_window.data(); - const int window_size = params.hann_window_size; - const int frame_size = (params.n_fft_bins - 1) * 2; - const int frame_step = params.hop_length; + const int frame_size = (mel_config.n_fft_bins - 1) * 2; // Apply preemphasis filter (high-pass): x[i] = x[i] - 0.97 * x[i-1] std::vector samples_preprocessed(samples, samples + n_samples_in); @@ -1422,50 +1758,30 @@ bool mtmd_audio_preprocessor_parakeet::preprocess(const float * samples, std::vector samples_padded(n_samples_in + 2 * pad, 0.0f); std::copy(samples_preprocessed.begin(), samples_preprocessed.end(), samples_padded.begin() + pad); - mtmd_audio_mel out_full; - out_full.n_mel = params.n_mel; - out_full.n_len = (samples_padded.size() - frame_size) / frame_step + 1; + if (!log_mel_spectrogram(samples_padded.data(), (int) samples_padded.size(), mel_config, shape_only, out_full)) { + return false; + } out_full.n_len_org = out_full.n_len; - out_full.data.resize(out_full.n_mel * out_full.n_len); - - const int n_threads = 4; - std::vector workers(n_threads - 1); - for (int iw = 0; iw < n_threads - 1; ++iw) { - workers[iw] = std::thread( - worker_thread, iw + 1, - window_func, - window_size, - std::cref(samples_padded), - samples_padded.size(), - frame_size, - frame_step, - n_threads, - params.n_fft_bins, - std::cref(cache), - std::ref(out_full) - ); - } - - worker_thread(0, - window_func, - window_size, - samples_padded, - samples_padded.size(), - frame_size, - frame_step, - n_threads, - params.n_fft_bins, - cache, - out_full); - - for (int iw = 0; iw < n_threads - 1; ++iw) { - workers[iw].join(); + return true; +} + +bool mtmd_audio_preprocessor_parakeet::process(const float * samples, + size_t n_samples_in, + std::vector & output, + bool shape_only) const { + if (n_samples_in == 0) { + return false; + } + + mtmd_audio_mel out_full; + if (!log_mel(samples, n_samples_in, out_full, shape_only)) { + return false; } // Per-feature normalization (only on valid frames) { const double eps = 1e-5; - int valid_frames = n_samples_in / frame_step; + int valid_frames = n_samples_in / hparams.audio_hop_len; for (int j = 0; j < out_full.n_mel; j++) { double sum = 0.0; @@ -1497,6 +1813,35 @@ bool mtmd_audio_preprocessor_parakeet::preprocess(const float * samples, return true; } +// +// mtmd_audio_preprocessor_nemotron_transcribe implementation +// + +bool mtmd_audio_preprocessor_nemotron_transcribe::process(const float * samples, + size_t n_samples, + std::vector & output, + bool shape_only) const { + if (n_samples < (size_t) hparams.audio_n_fft) { + return false; + } + + mtmd_audio_mel mel; + if (!log_mel(samples, n_samples, mel, shape_only)) { + return false; + } + + // frames past n_samples / hop are padding (NeMo get_seq_len); kept raw, encode_windows normalizes them for ASR + const int64_t n_frames = n_samples / hparams.audio_hop_len; + for (int64_t j = 1; j < mel.n_mel; j++) { + std::copy_n(mel.data.begin() + j * mel.n_len, n_frames, mel.data.begin() + j * n_frames); + } + mel.n_len = n_frames; + mel.n_len_org = n_frames; + mel.data.resize(mel.n_mel * n_frames); + output.push_back(std::move(mel)); + return true; +} + // mtmd_audio_preprocessor_gemma4ua // @@ -1505,9 +1850,10 @@ void mtmd_audio_preprocessor_gemma4ua::initialize() { // no-op: no FFT or filterbank needed } -bool mtmd_audio_preprocessor_gemma4ua::preprocess(const float * samples, - size_t n_samples, - std::vector & output) const { +bool mtmd_audio_preprocessor_gemma4ua::process(const float * samples, + size_t n_samples, + std::vector & output, + bool /*shape_only*/) const { if (n_samples == 0) { return false; } @@ -1646,9 +1992,10 @@ std::vector mtmd_audio_streaming_istft::flush() { // the samples are handed over as a single-row "mel", to reuse the normal chunk path // -bool mtmd_audio_preprocessor_pockettts::preprocess(const float * samples, - size_t n_samples, - std::vector & output) const { +bool mtmd_audio_preprocessor_pockettts::process(const float * samples, + size_t n_samples, + std::vector & output, + bool /*shape_only*/) const { // the encoder needs whole frames, see pad_for_conv1d() in the reference const int64_t frame_size = (int64_t) hparams.mimi_downsample * 120; if (n_samples == 0 || frame_size <= 0) { @@ -1656,10 +2003,10 @@ bool mtmd_audio_preprocessor_pockettts::preprocess(const float * } // the mimi transformer mask is dense, so cost is quadratic in the reference length - const int64_t max_samples = (int64_t) clip_hparams::pockettts_max_spk_seconds * hparams.audio_sample_rate; + const int64_t max_samples = (int64_t) mtmd_hparams::pockettts_max_spk_seconds * hparams.audio_sample_rate; if ((int64_t) n_samples > max_samples) { LOG_WRN("%s: speaker reference is %.1f s, truncating to the first %d s\n", __func__, - (double) n_samples / hparams.audio_sample_rate, clip_hparams::pockettts_max_spk_seconds); + (double) n_samples / hparams.audio_sample_rate, mtmd_hparams::pockettts_max_spk_seconds); n_samples = (size_t) max_samples; } @@ -1678,10 +2025,11 @@ bool mtmd_audio_preprocessor_pockettts::preprocess(const float * } // Muse Spark reads the raw waveform: pass the samples through, zero padded to a whole number of embeddings -bool mtmd_audio_preprocessor_muse_spark_audio::preprocess( - const float * samples, - size_t n_samples, - std::vector & output) const { +bool mtmd_audio_preprocessor_muse_spark_audio::process( + const float * samples, + size_t n_samples, + std::vector & output, + bool /*shape_only*/) const { const int64_t fold = hparams.audio_samples_per_token(); const int64_t n_padded = ((int64_t) n_samples + fold - 1) / fold * fold; diff --git a/tools/mtmd/mtmd-audio.h b/tools/mtmd/mtmd-audio.h index e9194ad5e339..1175ad624dc4 100644 --- a/tools/mtmd/mtmd-audio.h +++ b/tools/mtmd/mtmd-audio.h @@ -1,7 +1,8 @@ #pragma once #include "ggml.h" -#include "clip-model.h" +#include "ggml-backend.h" +#include "mtmd-model.h" #include #include @@ -53,37 +54,84 @@ struct mtmd_audio_cache { // compute one log10-mel frame from n_fft windowed samples, cache must be initialized for n_fft void mtmd_audio_log_mel_frame(const mtmd_audio_cache & cache, const float * samples, int n_fft, int n_mel, float * out); +struct mtmd_audio_mel_config { + int64_t n_mel = 0; + int64_t n_fft_bins = 0; + int32_t hann_window_size = 0; + int32_t hop_length = 0; + int32_t sample_rate = 0; + bool no_padding = false; + bool center_padding = false; + float preemph = 0.f; + bool use_natural_log = false; + bool norm_per_feature = false; + bool use_magnitude = false; // |X| instead of |X|^2 + float mel_floor = 5.960464477539063e-08f; + bool mel_floor_add = false; // log(x + floor) instead of log(max(x, floor)) + bool std_eps_after_sqrt = false; // std + eps instead of sqrt(var + eps) + int n_threads = 1; + + const mtmd_audio_cache * cache = nullptr; + ggml_backend_sched_t sched = nullptr; +}; + +struct mtmd_audio_preproc_out { + std::vector entries; // one per chunk, empty on failure + std::vector samples; // input of compute(), empty if entries already hold the data + + bool needs_compute() const { + return !samples.empty() && (entries.empty() || entries[0].is_placeholder()); + } +}; + struct mtmd_audio_preprocessor { - const clip_hparams & hparams; + const mtmd_hparams & hparams; + mtmd_audio_mel_config mel_config; // filled by initialize() of the preprocessors that use the log-mel graph - mtmd_audio_preprocessor(const clip_ctx * ctx): hparams(*clip_get_hparams(ctx)) {} - mtmd_audio_preprocessor(const clip_hparams & hparams): hparams(hparams) {} + // mel_config.sched must be set before the first compute(), see mtmd_context::init_audio() + mtmd_audio_preprocessor(const mtmd_model * model): hparams(model->hparams) {} + mtmd_audio_preprocessor(const mtmd_hparams & hparams, ggml_backend_sched_t sched): hparams(hparams) { + mel_config.sched = sched; + } + + // mel_config points to the cache of this instance + mtmd_audio_preprocessor(const mtmd_audio_preprocessor &) = delete; + mtmd_audio_preprocessor & operator=(const mtmd_audio_preprocessor &) = delete; virtual ~mtmd_audio_preprocessor() = default; virtual void initialize() = 0; // NOT thread-safe - virtual bool preprocess(const float * samples, size_t n_samples, std::vector & output) const = 0; + + // entries have the final shapes, but no data if needs_compute() + mtmd_audio_preproc_out preprocess(const float * samples, size_t n_samples) const; + + // run it on the thread that encodes, it uses mel_config.sched + bool compute(mtmd_audio_preproc_out & out) const; + + protected: + // shape_only skips the log-mel graph: the shapes are right, the data is zero-filled + virtual bool process(const float * samples, size_t n_samples, std::vector & output, bool shape_only) const = 0; }; struct mtmd_audio_preprocessor_whisper : mtmd_audio_preprocessor { - mtmd_audio_preprocessor_whisper(const clip_ctx * ctx) : mtmd_audio_preprocessor(ctx) {} + mtmd_audio_preprocessor_whisper(const mtmd_model * model) : mtmd_audio_preprocessor(model) {} void initialize() override; - bool preprocess(const float * samples, size_t n_samples, std::vector & output) const override; + bool process(const float * samples, size_t n_samples, std::vector & output, bool shape_only) const override; private: mtmd_audio_cache cache; }; struct mtmd_audio_preprocessor_muse_spark_audio : mtmd_audio_preprocessor { - mtmd_audio_preprocessor_muse_spark_audio(const clip_ctx * ctx) : mtmd_audio_preprocessor(ctx) {} + mtmd_audio_preprocessor_muse_spark_audio(const mtmd_model * model) : mtmd_audio_preprocessor(model) {} void initialize() override {} - bool preprocess(const float * samples, size_t n_samples, std::vector & output) const override; + bool process(const float * samples, size_t n_samples, std::vector & output, bool shape_only) const override; }; struct mtmd_audio_preprocessor_conformer : mtmd_audio_preprocessor { - mtmd_audio_preprocessor_conformer(const clip_ctx * ctx) : mtmd_audio_preprocessor(ctx) {} - mtmd_audio_preprocessor_conformer(const clip_hparams & hparams) : mtmd_audio_preprocessor(hparams) {} + mtmd_audio_preprocessor_conformer(const mtmd_model * model) : mtmd_audio_preprocessor(model) {} + mtmd_audio_preprocessor_conformer(const mtmd_hparams & hparams, ggml_backend_sched_t sched) : mtmd_audio_preprocessor(hparams, sched) {} void initialize() override; - bool preprocess(const float * samples, size_t n_samples, std::vector & output) const override; + bool process(const float * samples, size_t n_samples, std::vector & output, bool shape_only) const override; private: mtmd_audio_cache cache; @@ -92,73 +140,76 @@ struct mtmd_audio_preprocessor_conformer : mtmd_audio_preprocessor { // same as conformer, the audio is cut to 30 s and padded to 0.5 s (d1-omni audio.py) struct mtmd_audio_preprocessor_d1omni : mtmd_audio_preprocessor_conformer { using mtmd_audio_preprocessor_conformer::mtmd_audio_preprocessor_conformer; - bool preprocess(const float * samples, size_t n_samples, std::vector & output) const override; + bool process(const float * samples, size_t n_samples, std::vector & output, bool shape_only) const override; }; struct mtmd_audio_preprocessor_lfm3a : mtmd_audio_preprocessor { - mtmd_audio_preprocessor_lfm3a(const clip_ctx * ctx) : mtmd_audio_preprocessor(ctx) {} + mtmd_audio_preprocessor_lfm3a(const mtmd_model * model) : mtmd_audio_preprocessor(model) {} void initialize() override; - bool preprocess(const float * samples, size_t n_samples, std::vector & output) const override; + bool process(const float * samples, size_t n_samples, std::vector & output, bool shape_only) const override; private: mtmd_audio_cache cache; }; struct mtmd_audio_preprocessor_granite_speech : mtmd_audio_preprocessor { - mtmd_audio_preprocessor_granite_speech(const clip_ctx * ctx) : mtmd_audio_preprocessor(ctx) {} + mtmd_audio_preprocessor_granite_speech(const mtmd_model * model) : mtmd_audio_preprocessor(model) {} void initialize() override; - bool preprocess(const float * samples, size_t n_samples, std::vector & output) const override; + bool process(const float * samples, size_t n_samples, std::vector & output, bool shape_only) const override; private: mtmd_audio_cache cache; }; struct mtmd_audio_preprocessor_gemma4a : mtmd_audio_preprocessor { - mtmd_audio_preprocessor_gemma4a(const clip_ctx * ctx) : mtmd_audio_preprocessor(ctx) {} + // each segment is padded and encoded on its own + static constexpr int segment_seconds = 30; + + mtmd_audio_preprocessor_gemma4a(const mtmd_model * model) : mtmd_audio_preprocessor(model) {} void initialize() override; - bool preprocess(const float * samples, size_t n_samples, std::vector & output) const override; + bool process(const float * samples, size_t n_samples, std::vector & output, bool shape_only) const override; private: mtmd_audio_cache cache; }; struct mtmd_audio_preprocessor_gemma4ua : mtmd_audio_preprocessor { - mtmd_audio_preprocessor_gemma4ua(const clip_ctx * ctx) : mtmd_audio_preprocessor(ctx) {} + mtmd_audio_preprocessor_gemma4ua(const mtmd_model * model) : mtmd_audio_preprocessor(model) {} void initialize() override; - bool preprocess(const float * samples, size_t n_samples, std::vector & output) const override; + bool process(const float * samples, size_t n_samples, std::vector & output, bool shape_only) const override; }; struct mtmd_audio_preprocessor_qwen3a : mtmd_audio_preprocessor { - mtmd_audio_preprocessor_qwen3a(const clip_ctx * ctx) : mtmd_audio_preprocessor(ctx) {} + mtmd_audio_preprocessor_qwen3a(const mtmd_model * model) : mtmd_audio_preprocessor(model) {} void initialize() override; - bool preprocess(const float * samples, size_t n_samples, std::vector & output) const override; + bool process(const float * samples, size_t n_samples, std::vector & output, bool shape_only) const override; private: mtmd_audio_cache cache; }; struct mtmd_audio_preprocessor_dots3note : mtmd_audio_preprocessor { - mtmd_audio_preprocessor_dots3note(const clip_ctx * ctx) : mtmd_audio_preprocessor(ctx) {} + mtmd_audio_preprocessor_dots3note(const mtmd_model * model) : mtmd_audio_preprocessor(model) {} void initialize() override; - bool preprocess(const float * samples, size_t n_samples, std::vector & output) const override; + bool process(const float * samples, size_t n_samples, std::vector & output, bool shape_only) const override; private: mtmd_audio_cache cache; }; struct mtmd_audio_preprocessor_mimo_audio : mtmd_audio_preprocessor { - mtmd_audio_preprocessor_mimo_audio(const clip_ctx * ctx) : mtmd_audio_preprocessor(ctx) {} + mtmd_audio_preprocessor_mimo_audio(const mtmd_model * model) : mtmd_audio_preprocessor(model) {} void initialize() override; - bool preprocess(const float * samples, size_t n_samples, std::vector & output) const override; + bool process(const float * samples, size_t n_samples, std::vector & output, bool shape_only) const override; private: mtmd_audio_cache cache; }; struct mtmd_audio_preprocessor_qwen3tts_spk : mtmd_audio_preprocessor { - mtmd_audio_preprocessor_qwen3tts_spk(const clip_ctx * ctx) : mtmd_audio_preprocessor(ctx) {} + mtmd_audio_preprocessor_qwen3tts_spk(const mtmd_model * model) : mtmd_audio_preprocessor(model) {} void initialize() override; - bool preprocess(const float * samples, size_t n_samples, std::vector & output) const override; + bool process(const float * samples, size_t n_samples, std::vector & output, bool shape_only) const override; private: mtmd_audio_cache cache; @@ -166,24 +217,47 @@ struct mtmd_audio_preprocessor_qwen3tts_spk : mtmd_audio_preprocessor { // mimi convolves the waveform directly, so this only pads it to a whole number of frames struct mtmd_audio_preprocessor_pockettts : mtmd_audio_preprocessor { - mtmd_audio_preprocessor_pockettts(const clip_ctx * ctx) : mtmd_audio_preprocessor(ctx) {} + mtmd_audio_preprocessor_pockettts(const mtmd_model * model) : mtmd_audio_preprocessor(model) {} void initialize() override {} - bool preprocess(const float * samples, size_t n_samples, std::vector & output) const override; + bool process(const float * samples, size_t n_samples, std::vector & output, bool shape_only) const override; +}; + +// chatterbox reference front-end: one clip gives five entries, used by position in mtmd.cpp +// 0: CAMPPlus kaldi fbank [80 x n_frames] (mel major) +// 1: s3 tokenizer log-mel, flow cap [n_mel x n_frames] (mel major) +// 2: s3 tokenizer log-mel, t3 cap [n_mel x n_frames] (mel major) +// 3: s3gen prompt features [n_frames x 80] (frame major, 24 kHz mel rate) +// 4: voice encoder power mel [n_frames x 40] (frame major, trimmed, partial grid) +// turbo loudness-normalizes the clip to -27 LUFS and caps the t3 clip at 15 s instead of 6 s +struct mtmd_audio_preprocessor_chatterbox_ref : mtmd_audio_preprocessor { + mtmd_audio_preprocessor_chatterbox_ref(const mtmd_model * model, bool is_mtl) : mtmd_audio_preprocessor(model), is_mtl(is_mtl) {} + void initialize() override; + bool process(const float * samples, size_t n_samples, std::vector & output, bool shape_only) const override; + + private: + bool is_mtl; + mtmd_audio_cache fbank_cache; // entry 0, the window slot holds the povey window + mtmd_audio_cache s3tok_cache; // entries 1 and 2 + mtmd_audio_cache matcha_cache; // entry 3 + mtmd_audio_cache ve_cache; // entry 4 }; struct mtmd_audio_preprocessor_parakeet : mtmd_audio_preprocessor { - mtmd_audio_preprocessor_parakeet(clip_ctx * ctx) : mtmd_audio_preprocessor(ctx) { } + mtmd_audio_preprocessor_parakeet(const mtmd_model * model) : mtmd_audio_preprocessor(model) { } void initialize() override; - bool preprocess(const float * samples, size_t n_samples, std::vector & output) const override; + bool process(const float * samples, size_t n_samples, std::vector & output, bool shape_only) const override; + + protected: + bool log_mel(const float * samples, size_t n_samples, mtmd_audio_mel & out, bool shape_only) const; private: mtmd_audio_cache cache; +}; - static void worker_thread(int ith, const float * window_func, int window_size, - const std::vector & samples, int n_samples, - int frame_size, int frame_step, int n_threads, - int n_fft_bins, - const mtmd_audio_cache & cache, mtmd_audio_mel & mel); +// one chunk per recording: the raw log-mel of its valid frames +struct mtmd_audio_preprocessor_nemotron_transcribe : mtmd_audio_preprocessor_parakeet { + mtmd_audio_preprocessor_nemotron_transcribe(const mtmd_model * model) : mtmd_audio_preprocessor_parakeet(model) { } + bool process(const float * samples, size_t n_samples, std::vector & output, bool shape_only) const override; }; // diff --git a/tools/mtmd/mtmd-cli.cpp b/tools/mtmd/mtmd-cli.cpp index ba156f1c5b51..4534527fc8a7 100644 --- a/tools/mtmd/mtmd-cli.cpp +++ b/tools/mtmd/mtmd-cli.cpp @@ -153,7 +153,7 @@ struct mtmd_cli_context { } void init_vision_context(common_params & params) { - const char * clip_path = params.mmproj.path.c_str(); + const char * mmproj_path = params.mmproj.path.c_str(); mtmd_context_params mparams = mtmd_context_params_default(); mparams.use_gpu = params.mmproj_use_gpu; mparams.device = params.mmproj_device; @@ -166,7 +166,7 @@ struct mtmd_cli_context { { // non-causal models need the whole image in one ubatch const int n_ubatch = llama_n_ubatch(lctx); - auto mem = mtmd_get_memory_usage(clip_path, mparams); + auto mem = mtmd_get_memory_usage(mmproj_path, mparams); if (mem.use_non_causal && mem.image_max_tokens > n_ubatch) { LOG_WRN("%s: cap image_max_tokens (original=%d) to n_ubatch (%d) because model needs non-causal attention on image\n", __func__, mem.image_max_tokens, n_ubatch); LOG_WRN("%s: increase n_ubatch (-ub) to increase vision token budget\n", __func__); @@ -178,9 +178,9 @@ struct mtmd_cli_context { mparams.cb_eval_user_data = &cb_data; mparams.cb_eval = common_debug_cb_eval; } - ctx_vision.reset(mtmd_init_from_file(clip_path, model, mparams)); + ctx_vision.reset(mtmd_init_from_file(mmproj_path, model, mparams)); if (!ctx_vision.get()) { - LOG_ERR("Failed to load vision model from %s\n", clip_path); + LOG_ERR("Failed to load vision model from %s\n", mmproj_path); exit(1); } diff --git a/tools/mtmd/mtmd-graph.cpp b/tools/mtmd/mtmd-graph.cpp new file mode 100644 index 000000000000..738837152cc2 --- /dev/null +++ b/tools/mtmd/mtmd-graph.cpp @@ -0,0 +1,1065 @@ +#include "mtmd-graph.h" +#include "models/models.h" + +#include "ggml.h" +#include "ggml-cpp.h" +#include "ggml-backend.h" + +#include +#include +#include +#include +#include + +// +// mtmd_graph +// + +mtmd_graph::mtmd_graph(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : + model(ctx->model), + hparams(model.hparams), + proj_type(ctx->model.proj_type), + img(img), + patch_size(hparams.patch_size), + n_patches_x(img.nx() / patch_size), + n_patches_y(img.ny() / patch_size), + n_patches(n_patches_x * n_patches_y), + n_embd(hparams.n_embd), + n_head(hparams.n_head), + n_head_kv(hparams.n_head_kv), + d_head(hparams.n_embd_head > 0 ? hparams.n_embd_head : (n_head > 0 ? n_embd / n_head : 0)), + n_layer(hparams.n_layer), + n_mmproj_embd(ctx->model.n_mmproj_embd()), + eps(hparams.eps), + kq_scale(d_head > 0 ? 1.0f / sqrtf((float)d_head) : 0.0f), + flash_attn_type(ctx->flash_attn_type) { + struct ggml_init_params params = { + /*.mem_size =*/ ctx->buf_compute_meta.size(), + /*.mem_buffer =*/ ctx->buf_compute_meta.data(), + /*.no_alloc =*/ true, + }; + ctx0_ptr.reset(ggml_init(params)); + ctx0 = ctx0_ptr.get(); + gf = ggml_new_graph_custom(ctx0, ctx->max_nodes, false); +} + +mtmd_graph::mtmd_graph(const mtmd_graph & parent) : + model(parent.model), + hparams(parent.hparams), + proj_type(parent.proj_type), + img(parent.img), + patch_size(parent.patch_size), + n_patches_x(parent.n_patches_x), + n_patches_y(parent.n_patches_y), + n_patches(parent.n_patches), + n_embd(parent.n_embd), + n_head(parent.n_head), + n_head_kv(parent.n_head_kv), + d_head(parent.d_head), + n_layer(parent.n_layer), + n_mmproj_embd(parent.n_mmproj_embd), + eps(parent.eps), + kq_scale(parent.kq_scale), + flash_attn_type(parent.flash_attn_type) { + // reuse from parent + ctx0 = parent.ctx0; + gf = parent.gf; +} + +ggml_tensor * mtmd_graph::build_mm(ggml_tensor * w, ggml_tensor * x) const { + return ggml_mul_mat(ctx0, w, x); +} + +void mtmd_graph::cb(ggml_tensor * cur, const char * name, int il) const { + if (il >= 0) { + ggml_format_name(cur, "%s-%d", name, il); + } else { + ggml_set_name(cur, name); + } +} + +// siglip2 naflex +ggml_tensor * mtmd_graph::resize_position_embeddings(uint32_t interpolation_mode) { + ggml_tensor * pos_embd = model.position_embeddings; + const int height = img.ny() / patch_size; + const int width = img.nx() / patch_size; + const uint32_t mode = interpolation_mode; + const int n_per_side = (int)std::sqrt(pos_embd->ne[1]); + + GGML_ASSERT(pos_embd); + + if (height == n_per_side && width == n_per_side) { + return pos_embd; + } + + pos_embd = ggml_reshape_3d(ctx0, pos_embd, n_embd, n_per_side, n_per_side); // -> (n_embd, n_per_side, n_per_side) + pos_embd = ggml_permute(ctx0, pos_embd, 2, 0, 1, 3); // -> (n_per_side, n_per_side, n_embd) + pos_embd = ggml_interpolate(ctx0, pos_embd, width, height, n_embd, 1, mode); // -> (width, height, n_embd) + pos_embd = ggml_permute(ctx0, pos_embd, 1, 2, 0, 3); // -> (n_embd, width, height) + pos_embd = ggml_cont_2d(ctx0, pos_embd, n_embd, width * height); // -> (n_embd, width * height) + + return pos_embd; +} + +// build vision transformer (ViT) cgraph +// this function should cover most of the models +// if your model has specific features, you should probably duplicate this function +ggml_tensor * mtmd_graph::build_vit( + ggml_tensor * inp, + int64_t n_pos, + norm_type norm_t, + ffn_op_type ffn_t, + ggml_tensor * learned_pos_embd, + std::function add_pos, + const build_vit_opts & opts + ) { + // batch dim: inp is [n_embd, n_pos, B] + const int64_t B = inp->ne[2]; + + if (learned_pos_embd) { + inp = ggml_add(ctx0, inp, learned_pos_embd); + cb(inp, "pos_embed", -1); + } + + // flatten batch; unflatten again in attention + inp = ggml_reshape_2d(ctx0, inp, n_embd, n_pos * B); + + ggml_tensor * inpL = inp; + + // pre-layernorm + if (model.pre_ln_w) { + inpL = build_norm(inpL, model.pre_ln_w, model.pre_ln_b, norm_t, eps, -1); + cb(inpL, "pre_ln", -1); + } + + // loop over layers + for (int il = 0; il < n_layer; il++) { + auto & layer = model.layers[il]; + ggml_tensor * cur = inpL; // inpL = residual, cur = hidden_states + + ggml_tensor * attn_mask = opts.attn_mask; + if (opts.attn_mask_layers.size() > (size_t) il) { + attn_mask = opts.attn_mask_layers[il]; + } + + // layernorm1 + cur = build_norm(cur, layer.ln_1_w, layer.ln_1_b, norm_t, eps, il); + cb(cur, "layer_inp_normed", il); + + // self-attention + { + ggml_tensor * Qcur = nullptr; + ggml_tensor * Kcur = nullptr; + ggml_tensor * Vcur = nullptr; + if (layer.qkv_w != nullptr) { + // fused qkv + cur = build_mm(layer.qkv_w, cur); + if (layer.qkv_b != nullptr) { + cur = ggml_add(ctx0, cur, layer.qkv_b); + } + + // Q/K/V as [d_head, n_head, n_pos, B], the batch stride is cur->nb[1]*n_pos. + Qcur = ggml_view_4d(ctx0, cur, d_head, n_head, n_pos, B, + /* nb1 */ ggml_row_size(cur->type, d_head), + /* nb2 */ cur->nb[1], + /* nb3 */ cur->nb[1] * n_pos, + /* offset */ 0); + + Kcur = ggml_view_4d(ctx0, cur, d_head, n_head, n_pos, B, + /* nb1 */ ggml_row_size(cur->type, d_head), + /* nb2 */ cur->nb[1], + /* nb3 */ cur->nb[1] * n_pos, + /* offset */ ggml_row_size(cur->type, n_head * d_head)); + + Vcur = ggml_view_4d(ctx0, cur, d_head, n_head, n_pos, B, + /* nb1 */ ggml_row_size(cur->type, d_head), + /* nb2 */ cur->nb[1], + /* nb3 */ cur->nb[1] * n_pos, + /* offset */ ggml_row_size(cur->type, 2 * n_head * d_head)); + + if (layer.q_norm) { + GGML_ASSERT(layer.q_norm->ne[0] == Qcur->ne[0]); + Qcur = build_norm(Qcur, layer.q_norm, layer.q_norm_b, norm_t, eps, il); + cb(Qcur, "Qcur_norm", il); + } + + if (layer.k_norm) { + GGML_ASSERT(layer.k_norm->ne[0] == Kcur->ne[0]); + Kcur = build_norm(Kcur, layer.k_norm, layer.k_norm_b, norm_t, eps, il); + cb(Kcur, "Kcur_norm", il); + } + + } else { + // separate q, k, v + Qcur = build_mm(layer.q_w, cur); + if (layer.q_b) { + Qcur = ggml_add(ctx0, Qcur, layer.q_b); + } + + Kcur = build_mm(layer.k_w, cur); + if (layer.k_b) { + Kcur = ggml_add(ctx0, Kcur, layer.k_b); + } + + Vcur = build_mm(layer.v_w, cur); + if (layer.v_b) { + Vcur = ggml_add(ctx0, Vcur, layer.v_b); + } + + // if true, norm must be applied after reshaping to (d_head, n_head, n_pos) + bool norm_per_head = layer.q_norm && layer.q_norm->ne[0] == d_head; + + if (!norm_per_head) { + if (layer.q_norm) { + Qcur = build_norm(Qcur, layer.q_norm, layer.q_norm_b, norm_t, eps, il); + cb(Qcur, "Qcur_norm", il); + } + if (layer.k_norm) { + Kcur = build_norm(Kcur, layer.k_norm, layer.k_norm_b, norm_t, eps, il); + cb(Kcur, "Kcur_norm", il); + } + } + + Qcur = ggml_reshape_4d(ctx0, Qcur, d_head, n_head, n_pos, B); + Kcur = ggml_reshape_4d(ctx0, Kcur, d_head, n_head_kv, n_pos, B); + Vcur = ggml_reshape_4d(ctx0, Vcur, d_head, n_head_kv, n_pos, B); + + if (norm_per_head) { + if (layer.q_norm) { + Qcur = build_norm(Qcur, layer.q_norm, layer.q_norm_b, norm_t, eps, il); + cb(Qcur, "Qcur_norm_per_head", il); + } + if (layer.k_norm) { + Kcur = build_norm(Kcur, layer.k_norm, layer.k_norm_b, norm_t, eps, il); + cb(Kcur, "Kcur_norm_per_head", il); + } + } + } + + cb(Qcur, "Qcur", il); + cb(Kcur, "Kcur", il); + cb(Vcur, "Vcur", il); + + if (add_pos) { + Qcur = add_pos(Qcur, layer); + Kcur = add_pos(Kcur, layer); + cb(Qcur, "Qcur_pos", il); + cb(Kcur, "Kcur_pos", il); + } + + if (proj_type == PROJECTOR_TYPE_GEMMA4V) { + Vcur = ggml_rms_norm(ctx0, Vcur, eps); + cb(Vcur, "Vcur_normed", il); + } + + // build_attn returns a flat 2D [n_embd, n_pos*B] + cur = build_attn(layer.o_w, layer.o_b, + Qcur, Kcur, Vcur, attn_mask, kq_scale, il); + cb(cur, "attn_out", il); + } + + if (layer.ls_1_w) { + cur = ggml_mul(ctx0, cur, layer.ls_1_w); + cb(cur, "attn_out_scaled", il); + } + + if (layer.attn_post_norm_w) { + cur = build_norm(cur, layer.attn_post_norm_w, nullptr, norm_t, eps, il); + cb(cur, "attn_post_normed", il); + } + + // re-add the layer input, e.g., residual + cur = ggml_add(ctx0, cur, inpL); + + inpL = cur; // inpL = residual, cur = hidden_states + + if (opts.callback_layer_out) { + opts.callback_layer_out(cur, il); + } + + cb(cur, "ffn_inp", il); + + // layernorm2 (pre-ffn norm) + cur = build_norm(cur, layer.ln_2_w, layer.ln_2_b, norm_t, eps, il); + cb(cur, "ffn_inp_normed", il); + + // ffn + cur = layer.ff_gate_exps_w + ? build_moe_ffn(cur, layer, ffn_t, il) + : build_ffn(cur, + layer.ff_up_w, layer.ff_up_b, + layer.ff_gate_w, layer.ff_gate_b, + layer.ff_down_w, layer.ff_down_b, + ffn_t, il); + + cb(cur, "ffn_out", il); + + if (layer.ff_post_norm_w) { + cur = build_norm(cur, layer.ff_post_norm_w, nullptr, norm_t, eps, il); + cb(cur, "ffn_post_normed", il); + } + + if (layer.ls_2_w) { + cur = ggml_mul(ctx0, cur, layer.ls_2_w); + cb(cur, "ffn_out_scaled", il); + } + + // residual 2 + cur = ggml_add(ctx0, inpL, cur); + cb(cur, "layer_out", il); + + if (layer.ls_out_w) { + cur = ggml_mul(ctx0, cur, layer.ls_out_w); + cb(cur, "layer_out_scaled", il); + } + + inpL = cur; + } + + if (model.audio_has_avgpool()) { + ggml_tensor * cur = inpL; + cur = ggml_transpose(ctx0, cur); + cur = ggml_cont(ctx0, cur); + cur = ggml_pool_1d(ctx0, cur, GGML_OP_POOL_AVG, 2, 2, 0); + cur = ggml_transpose(ctx0, cur); + cur = ggml_cont(ctx0, cur); + inpL = cur; + } + + // post-layernorm + if (model.post_ln_w && !opts.skip_post_ln) { + inpL = build_norm(inpL, model.post_ln_w, model.post_ln_b, norm_t, eps, -1); + } + + // restore the batch dim + GGML_ASSERT(inpL->ne[1] % B == 0); + inpL = ggml_reshape_3d(ctx0, inpL, n_embd, inpL->ne[1] / B, B); + return inpL; +} + +// build the input after conv2d (inp_raw --> patches) +// returns tensor with shape [n_embd, n_patches] +ggml_tensor * mtmd_graph::build_inp() { + ggml_tensor * inp_raw = build_inp_raw(); + ggml_tensor * inp = ggml_conv_2d(ctx0, model.patch_embeddings_0, inp_raw, patch_size, patch_size, 0, 0, 1, 1); + inp = ggml_reshape_3d(ctx0, inp, n_patches, n_embd, n_batch); + inp = ggml_cont(ctx0, ggml_transpose(ctx0, inp)); + if (model.patch_bias) { + inp = ggml_add(ctx0, inp, model.patch_bias); + cb(inp, "patch_bias", -1); + } + return inp; +} + +ggml_tensor * mtmd_graph::build_inp_raw(int channels) { + ggml_tensor * inp_raw = ggml_new_tensor_4d(ctx0, GGML_TYPE_F32, img.nx(), img.ny(), channels, n_batch); + ggml_set_name(inp_raw, "inp_raw"); + ggml_set_input(inp_raw); + return inp_raw; +} + +// qwen-vl style temporal merge: Conv3D(kt=2) split into 2 Conv2D +ggml_tensor * mtmd_graph::build_inp_temporal_merge() { + ggml_tensor * inp_raw = build_inp_raw(); + + GGML_ASSERT(img.nx() % (patch_size * 2) == 0); + GGML_ASSERT(img.ny() % (patch_size * 2) == 0); + + const size_t nb1 = ggml_row_size(inp_raw->type, img.nx()); + const size_t nb2 = ggml_row_size(inp_raw->type, img.nx() * img.ny()); + + if (n_batch == 1) { + // still image input + return ggml_add(ctx0, + ggml_conv_2d(ctx0, model.patch_embeddings_0, inp_raw, patch_size, patch_size, 0, 0, 1, 1), + ggml_conv_2d(ctx0, model.patch_embeddings_1, inp_raw, patch_size, patch_size, 0, 0, 1, 1)); + } else if (n_batch == 2) { + // 2 frames input (video input) + ggml_tensor * inp_0 = ggml_view_3d(ctx0, inp_raw, + img.nx(), img.ny(), 3, nb1, nb2, 0); + ggml_tensor * inp_1 = ggml_view_3d(ctx0, inp_raw, + img.nx(), img.ny(), 3, nb1, nb2, + nb2 * 3); // move to the second frame + return ggml_add(ctx0, + ggml_conv_2d(ctx0, model.patch_embeddings_0, inp_0, patch_size, patch_size, 0, 0, 1, 1), + ggml_conv_2d(ctx0, model.patch_embeddings_1, inp_1, patch_size, patch_size, 0, 0, 1, 1)); + } else { + GGML_ASSERT(false && "n_batch > 2 is not supported"); + } +} + +ggml_tensor * mtmd_graph::build_inp_attn_mask(int64_t n_kv, int64_t n_q, int idx) { + const ggml_type type = flash_attn_type == MTMD_FLASH_ATTN_TYPE_ENABLED ? GGML_TYPE_F16 : GGML_TYPE_F32; + ggml_tensor * mask = ggml_new_tensor_2d(ctx0, type, n_kv, n_q); + ggml_set_name(mask, get_attn_mask_name(idx).c_str()); + ggml_set_input(mask); + return mask; +} + +ggml_tensor * mtmd_graph::build_norm( + ggml_tensor * cur, + ggml_tensor * mw, + ggml_tensor * mb, + norm_type type, + float norm_eps, + int il) const { + + cur = type == NORM_TYPE_RMS + ? ggml_rms_norm(ctx0, cur, norm_eps) + : ggml_norm(ctx0, cur, norm_eps); + + if (mw) { + cur = ggml_mul(ctx0, cur, mw); + cb(cur, "norm_w", il); + } + + if (mb) { + cur = ggml_add(ctx0, cur, mb); + cb(cur, "norm_b", il); + } + + return cur; +} + +ggml_tensor * mtmd_graph::build_ffn( + ggml_tensor * cur, + ggml_tensor * up, + ggml_tensor * up_b, + ggml_tensor * gate, + ggml_tensor * gate_b, + ggml_tensor * down, + ggml_tensor * down_b, + ffn_op_type type_op, + int il) const { + + ggml_tensor * tmp = up ? build_mm(up, cur) : cur; + cb(tmp, "ffn_up", il); + + if (up_b) { + tmp = ggml_add(ctx0, tmp, up_b); + cb(tmp, "ffn_up_b", il); + } + + if (gate) { + cur = build_mm(gate, cur); + cb(cur, "ffn_gate", il); + + if (gate_b) { + cur = ggml_add(ctx0, cur, gate_b); + cb(cur, "ffn_gate_b", il); + } + } else { + cur = tmp; + } + + // we only support parallel ffn for now + switch (type_op) { + case FFN_SILU: + if (gate) { + cur = hparams.swiglu_clamp > 0.0f + ? ggml_swiglu_clamp(ctx0, cur, tmp, hparams.swiglu_clamp) + : ggml_swiglu_split(ctx0, cur, tmp); + cb(cur, "ffn_swiglu", il); + } else { + cur = ggml_silu(ctx0, cur); + cb(cur, "ffn_silu", il); + } break; + case FFN_GELU: + if (gate) { + cur = ggml_geglu_split(ctx0, cur, tmp); + cb(cur, "ffn_geglu", il); + } else { + cur = ggml_gelu(ctx0, cur); + cb(cur, "ffn_gelu", il); + } break; + case FFN_GELU_ERF: + if (gate) { + cur = ggml_geglu_erf_split(ctx0, cur, tmp); + cb(cur, "ffn_geglu_erf", il); + } else { + cur = ggml_gelu_erf(ctx0, cur); + cb(cur, "ffn_gelu_erf", il); + } break; + case FFN_GELU_QUICK: + if (gate) { + cur = ggml_geglu_quick_split(ctx0, cur, tmp); + cb(cur, "ffn_geglu_quick", il); + } else { + cur = ggml_gelu_quick(ctx0, cur); + cb(cur, "ffn_gelu_quick", il); + } break; + case FFN_RELU_SQR: + { + cur = ggml_relu(ctx0, cur); + cur = ggml_sqr(ctx0, cur); + cb(cur, "ffn_relu_sqr", il); + } break; + } + + if (down) { + cur = build_mm(down, cur); + } + + if (down_b) { + cb(cur, "ffn_down", il); + } + + if (down_b) { + cur = ggml_add(ctx0, cur, down_b); + } + + return cur; +} + +// MoE FFN with sigmoid router and normalized top-k weights (dots3note vision) +// the router runs in fp32; exp_probs_b only affects expert selection, not the weights +ggml_tensor * mtmd_graph::build_moe_ffn(ggml_tensor * cur, const mtmd_layer & layer, ffn_op_type type_op, int il) const { + const int64_t n_tokens = cur->ne[1]; + const int64_t n_expert = layer.ff_gate_exps_w->ne[2]; + const int64_t n_expert_used = std::min((int64_t) hparams.n_expert_used, n_expert); + GGML_ASSERT(n_expert_used > 0); + GGML_ASSERT(type_op == FFN_SILU); + + ggml_tensor * probs = ggml_sigmoid(ctx0, build_mm(layer.ff_gate_inp_w, cur)); // [n_expert, n_tokens] + cb(probs, "ffn_moe_probs", il); + + ggml_tensor * sel = layer.ff_exp_probs_b + ? ggml_add(ctx0, probs, layer.ff_exp_probs_b) + : probs; + ggml_tensor * selected = ggml_top_k(ctx0, sel, n_expert_used); // [n_expert_used, n_tokens] + + ggml_tensor * weights = ggml_get_rows(ctx0, + ggml_reshape_3d(ctx0, probs, 1, n_expert, n_tokens), selected); + weights = ggml_reshape_2d(ctx0, weights, n_expert_used, n_tokens); + weights = ggml_div(ctx0, weights, ggml_sum_rows(ctx0, weights)); + weights = ggml_reshape_3d(ctx0, weights, 1, n_expert_used, n_tokens); + cb(weights, "ffn_moe_weights", il); + + cur = ggml_reshape_3d(ctx0, cur, cur->ne[0], 1, n_tokens); + ggml_tensor * gate = ggml_mul_mat_id(ctx0, layer.ff_gate_exps_w, cur, selected); // [n_ff, n_expert_used, n_tokens] + ggml_tensor * up = ggml_mul_mat_id(ctx0, layer.ff_up_exps_w, cur, selected); + cur = ggml_mul(ctx0, ggml_silu(ctx0, gate), up); + cur = ggml_mul_mat_id(ctx0, layer.ff_down_exps_w, cur, selected); // [n_embd, n_expert_used, n_tokens] + cur = ggml_mul(ctx0, cur, weights); + + // sum over the selected experts + ggml_tensor * out = nullptr; + for (int64_t i = 0; i < n_expert_used; i++) { + ggml_tensor * v = ggml_view_2d(ctx0, cur, cur->ne[0], n_tokens, cur->nb[2], i * cur->nb[1]); + out = out ? ggml_add(ctx0, out, v) : v; + } + if (n_expert_used == 1) { + out = ggml_cont(ctx0, out); + } + cb(out, "ffn_moe_out", il); + return out; +} + +ggml_tensor * mtmd_graph::build_attn( + ggml_tensor * wo, + ggml_tensor * wo_b, + ggml_tensor * q_cur, + ggml_tensor * k_cur, + ggml_tensor * v_cur, + ggml_tensor * kq_mask, + float kq_scale, + int il, + ggml_tensor * sinks) const { + // these nodes are added to the graph together so that they are not reordered + // by doing so, the number of splits in the graph is reduced + // the order is fixed without the compute flag, so an unselected branch stays out of the compute set + ggml_build_forward_order(gf, q_cur); + ggml_build_forward_order(gf, k_cur); + ggml_build_forward_order(gf, v_cur); + + ggml_tensor * q = ggml_permute(ctx0, q_cur, 0, 2, 1, 3); + //cb(q, "q", il); + + ggml_tensor * k = ggml_permute(ctx0, k_cur, 0, 2, 1, 3); + //cb(k, "k", il); + + ggml_tensor * cur; + + if (flash_attn_type == MTMD_FLASH_ATTN_TYPE_ENABLED) { + ggml_tensor * v = ggml_permute(ctx0, v_cur, 0, 2, 1, 3); + + k = ggml_cast(ctx0, k, GGML_TYPE_F16); + v = ggml_cast(ctx0, v, GGML_TYPE_F16); + // mask must be f16 here, use build_inp_attn_mask() + GGML_ASSERT(!kq_mask || kq_mask->type == GGML_TYPE_F16); + + cur = ggml_flash_attn_ext(ctx0, q, k, v, kq_mask, kq_scale, 0.0f, 0.0f); + ggml_prec_set_acc(cur, GGML_PREC_F32); + if (sinks != nullptr) { + ggml_flash_attn_ext_add_sinks(cur, sinks); + } + + cur = ggml_reshape_2d(ctx0, cur, cur->ne[0]*cur->ne[1], cur->ne[2]*cur->ne[3]); + + } else { + ggml_tensor * v = ggml_permute(ctx0, v_cur, 1, 2, 0, 3); + v = ggml_cont(ctx0, v); + + ggml_tensor * kq = ggml_mul_mat(ctx0, k, q); + // TODO: this most likely requires GGML_PREC_BF16, but will enable it later to avoid scope-creep + // ggml_prec_set_acc(kq, GGML_PREC_BF16); + + kq = ggml_soft_max_ext(ctx0, kq, kq_mask, kq_scale, 0.0f); + if (sinks != nullptr) { + ggml_soft_max_add_sinks(kq, sinks); + } + + ggml_tensor * kqv = ggml_mul_mat(ctx0, v, kq); + cur = ggml_permute(ctx0, kqv, 0, 2, 1, 3); + cur = ggml_cont_2d(ctx0, cur, cur->ne[0] * cur->ne[1], cur->ne[2] * cur->ne[3]); + } + + cb(cur, "kqv_out", il); + + if (wo) { + cur = build_mm(wo, cur); + } + + if (wo_b) { + cur = ggml_add(ctx0, cur, wo_b); + } + + return cur; +} + +// implementation of the 2D RoPE without adding a new op in ggml +ggml_tensor * mtmd_graph::build_rope_2d( + ggml_context * ctx0, + ggml_tensor * cur, + ggml_tensor * pos_a, // first half + ggml_tensor * pos_b, // second half + const float freq_base, + const bool interleave_freq +) { + const int64_t n_dim = cur->ne[0]; + + // for example, if we have cur tensor of shape (n_dim=8, n_head, n_pos) + // we will have a list of 4 inv_freq: 1e-0, 1e-1, 1e-2, 1e-3 + // first half of cur will use 1e-0, 1e-2 (even) + // second half of cur will use 1e-1, 1e-3 (odd) + // the trick here is to rotate just half of n_dim, so inv_freq will automatically be even + // ^ don't ask me why, it's math! -2(2i) / n_dim == -2i / (n_dim/2) + // then for the second half, we use freq_scale to shift the inv_freq + // ^ why? replace (2i) with (2i+1) in the above equation + const float freq_scale_odd = interleave_freq + ? std::pow(freq_base, (float)-2/n_dim) + : 1.0; + + // first half, dims [0, n_dim/2) + cur = ggml_rope_ext( + ctx0, + cur, + pos_a, // positions + nullptr, // freq factors + n_dim/2, // n_dims + 0, 0, freq_base, + 1.0f, 0.0f, 1.0f, 0.0f, 0.0f + ); + + // second half, dims [n_dim/2, n_dim) + cur = ggml_rope_ext( + ctx0, + cur, + pos_b, // positions + nullptr, // freq factors + n_dim/2, // n_dims + 0, 0, freq_base, + freq_scale_odd, + 0.0f, 1.0f, 0.0f, 0.0f + ); + cur = ggml_rope_set_offset(cur, n_dim/2); + + return cur; +} + +// Generic function to stack frames for audio processing +// Abstracts out the StackAudioFrames logic used by ultravox +ggml_tensor * mtmd_graph::build_stack(ggml_tensor * cur, int32_t stack_factor, int32_t n_embed) { + if (stack_factor <= 1) { + return cur; + } + + int64_t total_elements = ggml_nelements(cur); + int64_t stride = n_embed * stack_factor; + + // Calculate padded length + int64_t padded_len = GGML_PAD(total_elements, stride); + int64_t pad = padded_len - total_elements; + + if (pad > 0) { + // Pad the tensor to make it divisible by stride + cur = ggml_view_1d(ctx0, cur, total_elements, 0); + cur = ggml_pad(ctx0, cur, pad, 0, 0, 0); + } + + // Reshape to [stride, padded_len / stride] + cur = ggml_view_2d(ctx0, cur, stride, padded_len / stride, + ggml_row_size(cur->type, stride), 0); + return cur; +} + +// aka pixel_shuffle / pixel_unshuffle / patch_merger (Kimi-VL) +// support dynamic resolution +ggml_tensor * mtmd_graph::build_suffix(ggml_tensor * cur) { + for (int idx : mtmd_suffix_rows(img.suffix_type)) { + GGML_ASSERT(model.tok_embd_sep && idx < model.tok_embd_sep->ne[1]); + ggml_tensor * row = ggml_view_2d(ctx0, model.tok_embd_sep, model.tok_embd_sep->ne[0], 1, + model.tok_embd_sep->nb[1], idx * model.tok_embd_sep->nb[1]); + cur = ggml_concat(ctx0, cur, ggml_cast(ctx0, row, cur->type), 1); + } + return cur; +} + +ggml_tensor * mtmd_graph::build_patch_merge_permute(ggml_tensor * cur, int scale_factor) { + GGML_ASSERT(scale_factor > 1); + + const int n_embd = cur->ne[0]; + int width = img.nx() / patch_size; + int height = img.ny() / patch_size; + + // pad width and height to factor + const int64_t pad_width = MTMD_ALIGN(width, scale_factor) - width; + const int64_t pad_height = MTMD_ALIGN(height, scale_factor) - height; + cur = ggml_reshape_3d(ctx0, cur, n_embd, width, height); + if (pad_width || pad_height) { + cur = ggml_pad(ctx0, cur, 0, pad_width, pad_height, 0); + width += pad_width; + height += pad_height; + } + + // unshuffle h + cur = ggml_reshape_3d(ctx0, cur, n_embd * scale_factor, width / scale_factor, height); + cur = ggml_permute(ctx0, cur, 0, 2, 1, 3); + + // unshuffle w + cur = ggml_cont_3d(ctx0, cur, n_embd * scale_factor * scale_factor, height / scale_factor, width / scale_factor); + cur = ggml_permute(ctx0, cur, 0, 2, 1, 3); + + cur = ggml_cont_2d(ctx0, cur, cur->ne[0], cur->ne[1] * cur->ne[2]); + cb(cur, "pixel_shuffle", -1); + + return cur; +} + +std::unique_ptr mtmd_get_graph_builder(mtmd_subcontext * ctx, const mtmd_image_f32_batch & imgs, + const mtmd_subctx_encode_params * params) { + const mtmd_image_f32 & img = imgs.entries[0]; + std::unique_ptr builder; + + switch (ctx->model.proj_type) { + case PROJECTOR_TYPE_GEMMA3: + case PROJECTOR_TYPE_IDEFICS3: + case PROJECTOR_TYPE_COHERE2V: + case PROJECTOR_TYPE_LFM2: + case PROJECTOR_TYPE_D1OMNI_V: + case PROJECTOR_TYPE_JANUS_PRO: + case PROJECTOR_TYPE_PHI4: + { + builder = std::make_unique(ctx, img); + } break; + case PROJECTOR_TYPE_GEMMA3NV: + { + builder = std::make_unique(ctx, img); + } break; + case PROJECTOR_TYPE_GEMMA4V: + { + builder = std::make_unique(ctx, img); + } break; + case PROJECTOR_TYPE_GEMMA4UV: + { + builder = std::make_unique(ctx, img); + } break; + case PROJECTOR_TYPE_PIXTRAL: + case PROJECTOR_TYPE_LIGHTONOCR: + { + builder = std::make_unique(ctx, img); + } break; + case PROJECTOR_TYPE_DOTS_OCR: + case PROJECTOR_TYPE_DOTS3NOTE_V: // same ViT + merger; pyramid MoE is handled by build_vit + { + builder = std::make_unique(ctx, img); + } break; + case PROJECTOR_TYPE_DOTS3NOTE_A: + { + builder = std::make_unique(ctx, img); + } break; + case PROJECTOR_TYPE_QWEN2VL: + case PROJECTOR_TYPE_QWEN25VL: + { + builder = std::make_unique(ctx, img); + } break; + case PROJECTOR_TYPE_QWEN3VL: + { + builder = std::make_unique(ctx, img); + } break; + case PROJECTOR_TYPE_MAGEVL: + { + builder = std::make_unique(ctx, img); + } break; + case PROJECTOR_TYPE_SENSENOVA_U1: + { + builder = std::make_unique(ctx, img); + } break; + case PROJECTOR_TYPE_LFM3VL: + { + builder = std::make_unique(ctx, img); + } break; + case PROJECTOR_TYPE_LING3VL: + { + builder = std::make_unique(ctx, img); + } break; + case PROJECTOR_TYPE_EXAONE4_5: + { + builder = std::make_unique(ctx, img); + } break; + case PROJECTOR_TYPE_MIMOVL: + { + builder = std::make_unique(ctx, img); + } break; + case PROJECTOR_TYPE_MINIMAX_M3: + { + builder = std::make_unique(ctx, img); + } break; + case PROJECTOR_TYPE_MUSE_GLIMMER: + { + builder = std::make_unique(ctx, img); + } break; + case PROJECTOR_TYPE_STEP3VL: + { + builder = std::make_unique(ctx, img); + } break; + case PROJECTOR_TYPE_MINICPMV: + { + builder = std::make_unique(ctx, img); + } break; + case PROJECTOR_TYPE_MINICPMV4_6: + case PROJECTOR_TYPE_MINICPMV4_7: + case PROJECTOR_TYPE_MINICPM_OCR_1_0: + { + builder = std::make_unique(ctx, img); + } break; + case PROJECTOR_TYPE_INTERNVL: + { + builder = std::make_unique(ctx, img); + } break; + case PROJECTOR_TYPE_NEMOTRON_V2_VL: + { + builder = std::make_unique(ctx, img); + } break; + case PROJECTOR_TYPE_NEMOTRON_OMNI: + { + builder = std::make_unique(ctx, img); + } break; + case PROJECTOR_TYPE_LLAMA4: + { + builder = std::make_unique(ctx, img); + } break; + case PROJECTOR_TYPE_VOXTRAL_RT: + { + builder = std::make_unique(ctx, img, params ? params->enc_state : nullptr); + } break; + case PROJECTOR_TYPE_ULTRAVOX: + case PROJECTOR_TYPE_VOXTRAL: + case PROJECTOR_TYPE_QWEN2A: + case PROJECTOR_TYPE_GLMA: + case PROJECTOR_TYPE_MERALION: + case PROJECTOR_TYPE_MUSIC_FLAMINGO: + { + builder = std::make_unique(ctx, img); + } break; + case PROJECTOR_TYPE_MUSE_SPARK_AUDIO: + { + builder = std::make_unique(ctx, img); + } break; + case PROJECTOR_TYPE_KIMIVL: + { + builder = std::make_unique(ctx, img); + } break; + case PROJECTOR_TYPE_PADDLEOCR: + { + builder = std::make_unique(ctx, img); + } break; + case PROJECTOR_TYPE_KIMIK25: + case PROJECTOR_TYPE_KIMIK31: + { + builder = std::make_unique(ctx, img); + } break; + case PROJECTOR_TYPE_DEEPSEEK4V: + case PROJECTOR_TYPE_DEEPSEEK41V: + { + builder = std::make_unique(ctx, img); + } break; + case PROJECTOR_TYPE_COGVLM: + { + builder = std::make_unique(ctx, img); + } break; + case PROJECTOR_TYPE_HUNYUANVL: + { + builder = std::make_unique(ctx, img); + } break; + case PROJECTOR_TYPE_MLP: + case PROJECTOR_TYPE_MLP_NORM: + case PROJECTOR_TYPE_LDP: + case PROJECTOR_TYPE_LDPV2: + case PROJECTOR_TYPE_GLM_EDGE: + { + builder = std::make_unique(ctx, img); + } break; + case PROJECTOR_TYPE_DEEPSEEKOCR: + { + builder = std::make_unique(ctx, img); + } break; + case PROJECTOR_TYPE_DEEPSEEKOCR2: + { + builder = std::make_unique(ctx, img); + } break; + case PROJECTOR_TYPE_LFM2A: + case PROJECTOR_TYPE_D1OMNI_A: + { + builder = std::make_unique(ctx, img); + } break; + case PROJECTOR_TYPE_LFM3A: + { + builder = std::make_unique(ctx, img); + } break; + case PROJECTOR_TYPE_GEMMA4A: + { + builder = std::make_unique(ctx, img); + } break; + case PROJECTOR_TYPE_GEMMA4UA: + { + builder = std::make_unique(ctx, img); + } break; + case PROJECTOR_TYPE_GRANITE_SPEECH: + { + builder = std::make_unique(ctx, img); + } break; + case PROJECTOR_TYPE_GLM4V: + case PROJECTOR_TYPE_GLM5V: + { + builder = std::make_unique(ctx, img); + } break; + case PROJECTOR_TYPE_QWEN3A: + { + builder = std::make_unique(ctx, img); + } break; + case PROJECTOR_TYPE_MIMO_AUDIO: + { + builder = std::make_unique(ctx, img); + } break; + case PROJECTOR_TYPE_QWEN3TTS_SPKENC: + { + builder = std::make_unique(ctx, img); + } break; + case PROJECTOR_TYPE_POCKETTTS_SPKENC: + { + builder = std::make_unique(ctx, img); + } break; + case PROJECTOR_TYPE_POCKETTTS_GEN: + { + const auto gen_process = params ? params->gen_process : MTMD_SUBCTX_GEN_PROCESS_GEN_CODE; + const int n_step = ctx->model.hparams.flow_n_step; + const int64_t n_latent = ctx->model.gen_input_lin_w->ne[0]; + GGML_ASSERT(n_step > 0); + GGML_ASSERT(n_latent > 0); + // "inp_feats" takes the caller's buffer as-is, the graph must consume all of it + if (params && params->feats) { + GGML_ASSERT(params->feats->size() % (size_t) n_latent == 0); + GGML_ASSERT(params->feats->size() >= (size_t) n_latent); + } + const int n_frames = params && params->feats ? (int) (params->feats->size() / n_latent) : 1; + builder = std::make_unique(ctx, img, gen_process, n_step, n_frames); + } break; + case PROJECTOR_TYPE_CHATTERBOX_SPKENC: + { + builder = std::make_unique(ctx, img); + } break; + case PROJECTOR_TYPE_CHATTERBOX: + { + const auto gen_process = params ? params->gen_process : MTMD_SUBCTX_GEN_PROCESS_GEN_CODE; + int n_tokens = params && params->codes ? (int) params->codes->size() : 0; + int n_prompt_mel = 0; + if (n_tokens > 0) { + GGML_ASSERT(params->ref); + n_tokens += (int) params->ref->tokens.size(); + n_prompt_mel = (int) (params->ref->feat.size() / 80); + } + const int vnm = params ? params->vocode_n_mel : 0; + const int vns = params ? params->vocode_n_stft : 0; + builder = std::make_unique(ctx, img, gen_process, n_tokens, n_prompt_mel, vnm, vns); + } break; + case PROJECTOR_TYPE_LFM3A_GEN: + { + const auto gen_process = params ? params->gen_process : MTMD_SUBCTX_GEN_PROCESS_GEN_CODE; + const int64_t n_cb = ctx->model.gen_code_embd_w->ne[2]; + bool all_known = false; + int n_frames = 1; + if (params && params->codes) { + GGML_ASSERT(params->codes->size() % (size_t) n_cb == 0); + if (gen_process == MTMD_SUBCTX_GEN_PROCESS_GEN_WAV) { + n_frames = (int) (params->codes->size() / n_cb); + GGML_ASSERT(n_frames > 0); + } else { + // -1 marks a masked codebook, none masked means only the feedback embedding is needed + all_known = std::all_of(params->codes->begin(), params->codes->end(), [](int32_t c) { return c >= 0; }); + } + } + builder = std::make_unique(ctx, img, gen_process, all_known, n_frames); + } break; + case PROJECTOR_TYPE_DRAMA3_GEN: + { + const auto gen_process = params ? params->gen_process : MTMD_SUBCTX_GEN_PROCESS_GEN_CODE; + const int64_t n_cb = ctx->model.gen_code_embd_w->ne[2]; + const int n_frames = params && params->codes ? (int) (params->codes->size() / n_cb) : 1; + builder = std::make_unique(ctx, img, gen_process, params ? params->gen_state : nullptr, n_frames, + params ? params->top_k : 30, params ? params->top_p : 0.9f, + params ? params->temp : 1.0f); + } break; + case PROJECTOR_TYPE_QWEN3TTS_GEN: + { + const auto gen_process = params ? params->gen_process : MTMD_SUBCTX_GEN_PROCESS_GEN_CODE; + const int top_k = params ? params->top_k : 50; + const float top_p = params ? params->top_p : 1.0f; + builder = std::make_unique(ctx, img, gen_process, top_k, top_p); + } break; + case PROJECTOR_TYPE_SOPRANO: + { + const int n_frames = params && params->feats ? (int) (params->feats->size() / ctx->model.hparams.projection_dim) : 2; + builder = std::make_unique(ctx, img, n_frames); + } break; + case PROJECTOR_TYPE_YOUTUVL: + { + builder = std::make_unique(ctx, img); + } break; + case PROJECTOR_TYPE_YASA2: + { + builder = std::make_unique(ctx, img); + } break; + case PROJECTOR_TYPE_PARAKEET: + { + builder = std::make_unique(ctx, img); + } break; + case PROJECTOR_TYPE_NEMOTRON_TRANSCRIBE: + { + builder = std::make_unique(ctx, img, params ? params->enc_state : nullptr); + } break; + case PROJECTOR_TYPE_GRANITE4_VISION: + { + builder = std::make_unique(ctx, img); + } break; + default: + GGML_ABORT("missing cgraph builder"); + } + + builder->img_batch = &imgs; + + // TODO [QWEN_VIDEO]: improve this in the future + builder->n_batch = imgs.entries.size(); + + return builder; +} diff --git a/tools/mtmd/clip-graph.h b/tools/mtmd/mtmd-graph.h similarity index 84% rename from tools/mtmd/clip-graph.h rename to tools/mtmd/mtmd-graph.h index a0669d1a0c40..41765d03f8a8 100644 --- a/tools/mtmd/clip-graph.h +++ b/tools/mtmd/mtmd-graph.h @@ -2,9 +2,9 @@ #include "ggml.h" #include "ggml-cpp.h" -#include "clip.h" -#include "clip-impl.h" -#include "clip-model.h" +#include "mtmd-subcontext.h" +#include "mtmd-impl.h" +#include "mtmd-model.h" #include #include @@ -23,13 +23,13 @@ struct build_vit_opts { bool skip_post_ln = false; }; -struct clip_graph { - const clip_model & model; - const clip_hparams & hparams; +struct mtmd_graph { + const mtmd_model & model; + const mtmd_hparams & hparams; projector_type proj_type; - const clip_image_f32 & img; // for backward compat - const clip_image_f32_batch * img_batch = nullptr; + const mtmd_image_f32 & img; // for backward compat + const mtmd_image_f32_batch * img_batch = nullptr; const int patch_size; const int n_patches_x; @@ -43,7 +43,7 @@ struct clip_graph { const int n_mmproj_embd; const float eps; float kq_scale; // TODO: maybe move this to hparams - const clip_flash_attn_type flash_attn_type; + const mtmd_flash_attn_type flash_attn_type; // TODO [QWEN_VIDEO]: improve this in the future int n_batch = 1; @@ -52,12 +52,12 @@ struct clip_graph { ggml_context * ctx0; ggml_cgraph * gf; - clip_graph(clip_ctx * ctx, const clip_image_f32 & img); + mtmd_graph(mtmd_subcontext * ctx, const mtmd_image_f32 & img); // build sub-graph, reuse buf from parent - clip_graph(const clip_graph & parent); + mtmd_graph(const mtmd_graph & parent); - virtual ~clip_graph() = default; + virtual ~mtmd_graph() = default; virtual ggml_cgraph * build() = 0; // wrapper around ggml_mul_mat, allow hooking (e.g. LoRA, clamping) depending on the model @@ -74,7 +74,7 @@ struct clip_graph { // void cb(ggml_tensor * cur0, const char * name, int il) const; - const clip_image_f32 & get_img(size_t idx) const { + const mtmd_image_f32 & get_img(size_t idx) const { GGML_ASSERT(img_batch); GGML_ASSERT(idx < img_batch->entries.size()); return img_batch->entries[idx]; @@ -92,7 +92,7 @@ struct clip_graph { norm_type norm_t, ffn_op_type ffn_t, ggml_tensor * learned_pos_embd, - std::function add_pos, + std::function add_pos, const build_vit_opts & opts = {}); // build the input after conv2d (inp_raw --> patches) @@ -130,7 +130,7 @@ struct clip_graph { ggml_tensor * build_moe_ffn( ggml_tensor * cur, - const clip_layer & layer, + const mtmd_layer & layer, ffn_op_type type_op, int il) const; @@ -174,3 +174,11 @@ struct clip_graph { // append the separators of img.suffix_type after the image tokens ggml_tensor * build_suffix(ggml_tensor * cur); }; + +static inline std::string get_attn_mask_name(int idx) { + return idx == 0 ? "attn_mask" : "attn_mask_" + std::to_string(idx); +} + +// create the graph builder of the subcontext's projector, params is only needed by encode() +std::unique_ptr mtmd_get_graph_builder(mtmd_subcontext * ctx, const mtmd_image_f32_batch & imgs, + const mtmd_subctx_encode_params * params = nullptr); diff --git a/tools/mtmd/mtmd-helper-gen.cpp b/tools/mtmd/mtmd-helper-gen.cpp index bed8f3d8c6db..73e7e1b270b6 100644 --- a/tools/mtmd/mtmd-helper-gen.cpp +++ b/tools/mtmd/mtmd-helper-gen.cpp @@ -72,7 +72,7 @@ static void write_wav16_header(std::vector & buf, uint32_t data_sz, int32_ static void append_wav16_pcm(std::vector & buf, const float * pcm, size_t n) { for (size_t i = 0; i < n; i++) { - int16_t s = (int16_t) (std::max(-1.0f, std::min(1.0f, pcm[i])) * 32767.0f); + int16_t s = (int16_t) std::lround(std::max(-1.0f, std::min(1.0f, pcm[i])) * 32767.0f); put_bytes(buf, &s, 2); } } @@ -124,6 +124,39 @@ class mtmd_gen_audio_pipeline { return true; } + // encodes a speaker clip or a voice, out receives the encoder embeddings + bool encode_speaker(mtmd_bitmap * bitmap, std::vector & out) { + if (!mtmd_bitmap_is_voice(bitmap) && !mtmd_support_audio(mctx)) { + LOG_ERR("mtmd_helper_gen_audio: mmproj has no speaker/audio encoder\n"); + return false; + } + const std::string marker = mtmd_get_marker(mctx); + mtmd_input_text text{ marker.c_str(), marker.size(), false, true }; + mtmd_input_chunks * chunks = mtmd_input_chunks_init(); + const mtmd_bitmap * bptr = bitmap; + bool ok = mtmd_tokenize(mctx, chunks, &text, &bptr, 1) == 0; + if (ok) { + ok = false; + for (size_t i = 0; i < mtmd_input_chunks_size(chunks); i++) { + const mtmd_input_chunk * chunk = mtmd_input_chunks_get(chunks, i); + if (mtmd_input_chunk_get_type(chunk) != MTMD_INPUT_CHUNK_TYPE_AUDIO) { + continue; + } + if (mtmd_encode_chunk(mctx, chunk) != 0) { + LOG_ERR("mtmd_helper_gen_audio: speaker encode failed\n"); + break; + } + const float * embd = mtmd_get_output_embd(mctx); + const size_t n = (size_t) llama_model_n_embd_inp(model) * mtmd_input_chunk_get_n_tokens(chunk); + out.assign(embd, embd + n); + ok = true; + break; + } + } + mtmd_input_chunks_free(chunks); + return ok; + } + llama_context * lctx; mtmd_context * mctx; const llama_model * model; @@ -428,39 +461,6 @@ class qwen3tts_gen_audio_pipeline : public mtmd_gen_audio_pipeline { return true; } - // runs the reference wav through the speaker encoder, returns one x-vector embedding row - bool encode_speaker(mtmd_bitmap * bitmap, std::vector & out) { - if (!mtmd_bitmap_is_voice(bitmap) && !mtmd_support_audio(mctx)) { - LOG_ERR("mtmd_helper_gen_audio: mmproj has no speaker/audio encoder\n"); - return false; - } - const std::string marker = mtmd_get_marker(mctx); - mtmd_input_text text{ marker.c_str(), marker.size(), false, true }; - mtmd_input_chunks * chunks = mtmd_input_chunks_init(); - const mtmd_bitmap * bptr = bitmap; - bool ok = mtmd_tokenize(mctx, chunks, &text, &bptr, 1) == 0; - if (ok) { - ok = false; - for (size_t i = 0; i < mtmd_input_chunks_size(chunks); i++) { - const mtmd_input_chunk * chunk = mtmd_input_chunks_get(chunks, i); - if (mtmd_input_chunk_get_type(chunk) != MTMD_INPUT_CHUNK_TYPE_AUDIO) { - continue; - } - if (mtmd_encode_chunk(mctx, chunk) != 0) { - LOG_ERR("mtmd_helper_gen_audio: speaker encode failed\n"); - break; - } - const float * embd = mtmd_get_output_embd(mctx); - const size_t n = (size_t) llama_model_n_embd_inp(model) * mtmd_input_chunk_get_n_tokens(chunk); - out.assign(embd, embd + n); - ok = true; - break; - } - } - mtmd_input_chunks_free(chunks); - return ok; - } - // one GEN_WAV process() call over the buffered codes, state is carried across batches bool flush_gen_wav() { if (codes_buf.empty()) { @@ -494,7 +494,7 @@ class qwen3tts_gen_audio_pipeline : public mtmd_gen_audio_pipeline { llama_token tts_eos = LLAMA_TOKEN_NULL; std::vector tok_embd; // whole token embedding matrix, n_vocab * n_embd - // must match hparams.wav_tfm_swa hardcoded in clip.cpp + // must match hparams.wav_tfm_swa hardcoded in mtmd-model.cpp size_t window_frames = 72; // per-generation state, cleared by reset() @@ -521,19 +521,19 @@ class qwen3tts_gen_audio_pipeline : public mtmd_gen_audio_pipeline { // settings that only live in the reference's per-pack yaml, not in the checkpoint // the english packs share the same shapes and tokenizer, but disagree on these -// all three are 0 / false when the pack does not tune them, the model default is then used +// both are 0 / false when the pack does not tune them, the model default is then used struct pockettts_pack_settings { - float temp = 0.0f; - int frames_after_eos = 0; - bool pad_short_text = false; + int frames_after_eos = 0; + bool pad_short_text = false; }; static pockettts_pack_settings pockettts_pack(const char * variant) { static const std::unordered_map packs = { - { "english", { 0.3f, 0, false } }, - { "english_2026-01", { 0.7f, 0, true } }, - { "english_2026-04", { 0.3f, 0, false } }, - { "french_24l", { 0.7f, 8, false } }, + { "english", { 0, false } }, + { "english_2026-01", { 0, true } }, + { "english_2026-04", { 0, false } }, + { "french_24l", { 8, false } }, + { "italian", { 0, false } }, }; auto it = packs.find(variant ? variant : ""); if (it == packs.end()) { @@ -638,6 +638,7 @@ class pockettts_gen_audio_pipeline : public mtmd_gen_audio_pipeline { prompt_batch->set_position_normal(0, seq_id); prompt_pos = 0; + temp = inp->temp; out_type = inp->out_type; stream = inp->stream; if (!init_gen_state(inp->seed)) { @@ -680,8 +681,8 @@ class pockettts_gen_audio_pipeline : public mtmd_gen_audio_pipeline { mtmd_gen_inp inp = mtmd_gen_inp_default(mctx); inp.type = MTMD_GEN_PROCESS_TYPE_GEN_CODE; inp.embd = const_cast(h_state_in); - if (pack.temp > 0.0f) { - inp.temp = pack.temp; + if (temp >= 0.0f) { + inp.temp = temp; } mtmd_gen_out out{}; if (mtmd_gen_audio_process(mctx, gen_state.get(), &inp, &out) != 0) { @@ -996,39 +997,6 @@ class pockettts_gen_audio_pipeline : public mtmd_gen_audio_pipeline { return n; } - // runs the reference wav through the mimi encoder, returns one row per 12.5Hz frame - bool encode_speaker(mtmd_bitmap * bitmap, std::vector & out) { - if (!mtmd_bitmap_is_voice(bitmap) && !mtmd_support_audio(mctx)) { - LOG_ERR("mtmd_helper_gen_audio: mmproj has no voice encoder\n"); - return false; - } - const std::string marker = mtmd_get_marker(mctx); - mtmd_input_text text{ marker.c_str(), marker.size(), false, true }; - mtmd_input_chunks * chunks = mtmd_input_chunks_init(); - const mtmd_bitmap * bptr = bitmap; - bool ok = mtmd_tokenize(mctx, chunks, &text, &bptr, 1) == 0; - if (ok) { - ok = false; - for (size_t i = 0; i < mtmd_input_chunks_size(chunks); i++) { - const mtmd_input_chunk * chunk = mtmd_input_chunks_get(chunks, i); - if (mtmd_input_chunk_get_type(chunk) != MTMD_INPUT_CHUNK_TYPE_AUDIO) { - continue; - } - if (mtmd_encode_chunk(mctx, chunk) != 0) { - LOG_ERR("mtmd_helper_gen_audio: voice encode failed\n"); - break; - } - const float * embd = mtmd_get_output_embd(mctx); - const size_t n = (size_t) llama_model_n_embd_inp(model) * mtmd_input_chunk_get_n_tokens(chunk); - out.assign(embd, embd + n); - ok = true; - break; - } - } - mtmd_input_chunks_free(chunks); - return ok; - } - // decodes the buffered latents, the mimi decoder state carries over between calls bool flush_gen_wav() { if (feats_buf.empty()) { @@ -1049,6 +1017,7 @@ class pockettts_gen_audio_pipeline : public mtmd_gen_audio_pipeline { } pockettts_pack_settings pack; + float temp = -1.0f; // the caller's flow noise scale, < 0 for the model default bool specials_ok = false; llama_token bos_before_voice = LLAMA_TOKEN_NULL; llama_token audio_bos = LLAMA_TOKEN_NULL; @@ -1339,9 +1308,9 @@ class lfm3a_gen_audio_pipeline : public mtmd_gen_audio_pipeline { return 1; } - // the sampling defaults of the model, the caller's top-k/top-p are not used + // the sampling defaults of the model, the caller can only override the temperature const mtmd_gen_inp def = mtmd_gen_inp_default(mctx); - temp = def.temp; + temp = inp->temp >= 0.0f ? inp->temp : def.temp; top_k = def.top_k; rep_penalty = def.penalty_repeat; rep_window = def.penalty_last_n; @@ -1718,6 +1687,737 @@ class lfm3a_gen_audio_pipeline : public mtmd_gen_audio_pipeline { bool wav_header_sent = false; }; +// Chatterbox: the backbone emits s3 speech tokens, then one flow-matching mel decode and the NSF-iSTFT vocoder make the PCM +// the multilingual variant runs cfg with an uncond sequence (text rows zeroed), paired as seq_id and n_seq_max / 2 + seq_id +class chatterbox_gen_audio_pipeline : public mtmd_gen_audio_pipeline { +public: + using mtmd_gen_audio_pipeline::mtmd_gen_audio_pipeline; + + void reset() override { + seq_id = 0; + seq_uncond = 0; + pos = 0; + ar_idx = 0; + codes_buf.clear(); + audio_pcm.clear(); + h_state_buf.clear(); + out_buf.clear(); + prompt_embd_buf.clear(); + uncond_embd_buf.clear(); + cond_logits.clear(); + n_prompt = 0; + prompt_pos = 0; + decoded = false; + stream = false; + pcm_sent = 0; + wav_header_sent = false; + gen_state.reset(); + } + + int32_t set_input(const mtmd_helper_gen_audio_inp * inp) override { + reset(); + seq_id = inp->seq_id; + + if (!ensure_cache()) { + return 1; + } + + out_type = inp->out_type; + stream = inp->stream; + if (!init_gen_state(inp->seed)) { + return 1; + } + + if (mtl) { + // the uncond sequence lives half a seq space away from the cond one + const int32_t n_seq_max = (int32_t) llama_n_seq_max(lctx); + seq_uncond = (llama_seq_id) (n_seq_max / 2 + seq_id); + if (!llama_model_classifier_free_guidance(llama_get_model(lctx)) || seq_id >= n_seq_max / 2) { + LOG_ERR("mtmd_helper_gen_audio: multilingual chatterbox needs a backbone with classifier_free_guidance and a paired cfg sequence " + "(n_seq_max = %d, seq_id = %d)\n", n_seq_max, seq_id); + return 1; + } + } + llama_memory_t mem = llama_get_memory(lctx); + if (!llama_memory_seq_rm(mem, seq_id, 0, -1) || (mtl && !llama_memory_seq_rm(mem, seq_uncond, 0, -1))) { + LOG_ERR("mtmd_helper_gen_audio: cannot reset chatterbox sequences\n"); + return 1; + } + + // the encode gives the talker conditioning rows, set_ref keeps the flow decoder reference + if (!inp->speaker_ref) { + LOG_ERR("mtmd_helper_gen_audio: chatterbox requires a speaker reference or a voice\n"); + return 1; + } + if (!encode_speaker(inp->speaker_ref, prompt_embd_buf) || + mtmd_gen_state_set_ref(mctx, gen_state.get()) != 0) { + LOG_ERR("mtmd_helper_gen_audio: speaker reference encoding failed\n"); + return 1; + } + + const int n_e = n_embd; + auto push_row = [&](const float * src) { + prompt_embd_buf.insert(prompt_embd_buf.end(), src, src + n_e); + }; + auto add_pos = [&](float * r, const std::vector & tab, int idx) { + const float * p = tab.data() + (size_t) idx * n_e; + for (int j = 0; j < n_e; j++) { + r[j] += p[j]; + } + }; + + // prompt rows: [conditioning, text, speech bos] + // no text preprocessing here, the [SPACE] substitution and the case folding are baked in at conversion + std::string txt(inp->prompt, inp->prompt_len); + if (mtl && inp->lang && inp->lang[0]) { + // the reference prefixes the language tag, e.g. "[fr]" + std::string lang = inp->lang; + std::transform(lang.begin(), lang.end(), lang.begin(), [](unsigned char c) { return (char) std::tolower(c); }); + txt = "[" + lang + "]" + txt; + } + std::vector ids(txt.size() + 16); + int n_ids = llama_tokenize(vocab, txt.c_str(), (int32_t) txt.size(), ids.data(), (int32_t) ids.size(), + false, true); + if (n_ids < 1) { + LOG_ERR("mtmd_helper_gen_audio: tokenization failed\n"); + return 1; + } + ids.resize((size_t) n_ids); + if (mtl) { + // only the multilingual prompt wraps the text in start/stop tokens + ids.insert(ids.begin(), text_start); + ids.push_back(text_stop); + if (ids.size() > text_pos.size() / (size_t) n_e) { + LOG_ERR("mtmd_helper_gen_audio: prompt too long (%zu tokens, max %zu)\n", ids.size(), text_pos.size() / (size_t) n_e); + return 1; + } + } + const size_t text0 = prompt_embd_buf.size() / (size_t) n_e; + for (size_t i = 0; i < ids.size(); i++) { + push_row(tok_embd.data() + (size_t) ids[i] * n_e); + if (mtl) { + add_pos(prompt_embd_buf.data() + (text0 + i) * n_e, text_pos, (int) i); + } + } + + // speech bos opens the AR stream + push_row(tok_embd.data() + (size_t) llama_vocab_bos(vocab) * n_e); + if (mtl) { + add_pos(prompt_embd_buf.data() + prompt_embd_buf.size() - n_e, speech_pos, 0); + } + ar_idx = 1; + n_prompt = (int) (prompt_embd_buf.size() / (size_t) n_e); + + if (mtl) { + // cfg uncond prompt: text rows are zeroed but keep their learned positions, as in the reference + uncond_embd_buf = prompt_embd_buf; + for (size_t i = 0; i < ids.size(); i++) { + float * r = uncond_embd_buf.data() + (text0 + i) * n_e; + std::fill(r, r + n_e, 0.0f); + add_pos(r, text_pos, (int) i); + } + } + + prompt_pos = 0; + return 0; + } + + int32_t step_prompt(int32_t n_batch) override { + GGML_ASSERT(n_batch > 0); + // the uncond prompt follows the cond one on mtl + const int n_total = mtl ? 2 * n_prompt : n_prompt; + if (prompt_pos >= n_total) { + return 0; + } + const bool uncond = prompt_pos >= n_prompt; + const int p0 = uncond ? prompt_pos - n_prompt : prompt_pos; + const int32_t n_tokens_batch = std::min(n_batch, n_prompt - p0); + const bool last = p0 + n_tokens_batch == n_prompt; + + decode_embd_batch batch((uncond ? uncond_embd_buf : prompt_embd_buf).data(), n_prompt, 1, n_embd); + batch.set_position_normal(0, uncond ? seq_uncond : seq_id); + if (last) { + batch.logits[n_prompt - 1] = 1; + } + if (llama_process(lctx, LLAMA_PROCESS_TYPE_DECODE, batch.render(lctx, p0, n_tokens_batch)) != 0) { + LOG_ERR("mtmd_helper_gen_audio: prompt decode failed\n"); + return -1; + } + prompt_pos += n_tokens_batch; + + if (last && mtl) { + if (!uncond) { + cfg_read(); + } else { + cfg_apply(); + } + } + + if (prompt_pos >= n_total) { + pos = n_prompt; + prompt_embd_buf.clear(); + uncond_embd_buf.clear(); + return 0; + } + return n_total - prompt_pos; + } + + int32_t step_gen(llama_token sampled, const float * h_state_in, const float ** h_state_out, bool * out_stop) override { + GGML_UNUSED(h_state_in); + if (sampled < 0 || sampled >= llama_vocab_n_tokens(vocab)) { + LOG_ERR("mtmd_helper_gen_audio: chatterbox requires a token sampled from the backbone\n"); + return 1; + } + + // the speech stop token is the backbone eos, the multilingual speech positions end the stream too + if (llama_vocab_is_eog(vocab, sampled) || (mtl && (size_t) ar_idx >= speech_pos.size() / (size_t) n_embd)) { + *out_stop = true; + *h_state_out = nullptr; + return 0; + } + + // keep only the 6561 s3gen codes, dropping start/stop and oov ids + if (sampled >= speech_base && sampled - speech_base < 6561) { + codes_buf.push_back(sampled - speech_base); + } + + std::vector e(tok_embd.begin() + (size_t) sampled * n_embd, + tok_embd.begin() + (size_t) (sampled + 1) * n_embd); + if (mtl) { + // the multilingual backbone takes the token row plus the learned speech position + const float * p = speech_pos.data() + (size_t) ar_idx * n_embd; + for (int j = 0; j < n_embd; j++) { + e[(size_t) j] += p[j]; + } + ar_idx++; + } + + if (!decode_one(e.data(), seq_id)) { + return 1; + } + if (mtl) { + // cfg: the sampled token feeds both branches + cfg_read(); + if (!decode_one(e.data(), seq_uncond)) { + return 1; + } + cfg_apply(); + } + pos++; + + const float * h = llama_get_embeddings_ith(lctx, -1); + h_state_buf.assign(h, h + n_embd); + *h_state_out = h_state_buf.data(); + return 0; + } + + int32_t get_output(int32_t * out_sample_rate, const char ** out_data, size_t * out_data_len, int64_t * out_n_samples) override { + *out_sample_rate = info.sample_rate; + + if (!stream) { + if (!flush_gen_wav()) { + return 1; + } + if (out_n_samples) { + *out_n_samples = (int64_t) audio_pcm.size(); + } + if (out_type == MTMD_HELPER_GEN_AUDIO_OUTTYPE_PCM) { + *out_data = (const char *) audio_pcm.data(); + *out_data_len = audio_pcm.size() * sizeof(float); + return 0; + } + out_buf.clear(); + if (!write_wav16(out_buf, audio_pcm, info.sample_rate)) { + LOG_ERR("mtmd_helper_gen_audio: output too large for WAV\n"); + return 1; + } + *out_data = out_buf.data(); + *out_data_len = out_buf.size(); + return 0; + } + + // streaming: the flow decode is one-shot, so all the audio arrives after flush() + const size_t n_new = audio_pcm.size() - pcm_sent; + if (out_n_samples) { + *out_n_samples = (int64_t) n_new; + } + if (out_type == MTMD_HELPER_GEN_AUDIO_OUTTYPE_PCM) { + *out_data = (const char *) (audio_pcm.data() + pcm_sent); + *out_data_len = n_new * sizeof(float); + pcm_sent = audio_pcm.size(); + return 0; + } + out_buf.clear(); + if (!wav_header_sent) { + write_wav16_header(out_buf, UINT32_MAX, info.sample_rate); + wav_header_sent = true; + } + append_wav16_pcm(out_buf, audio_pcm.data() + pcm_sent, n_new); + pcm_sent = audio_pcm.size(); + *out_data = out_buf.data(); + *out_data_len = out_buf.size(); + return 0; + } + + int32_t flush() override { + return flush_gen_wav() ? 0 : 1; + } + +private: + bool decode_one(const float * e, llama_seq_id seq) { + decode_embd_batch batch(e, 1, 1, n_embd); + batch.set_position_normal(pos, seq); + batch.logits[0] = 1; + if (llama_process(lctx, LLAMA_PROCESS_TYPE_DECODE, batch.render(lctx, 0, 1)) != 0) { + LOG_ERR("mtmd_helper_gen_audio: step decode failed\n"); + return false; + } + return true; + } + + // cfg of the reference: logits = cond + 0.5 * (cond - uncond) + // the mix goes into the last output row (the uncond one), the caller samples from it + void cfg_read() { + const float * c = llama_get_logits_ith(lctx, -1); + cond_logits.assign(c, c + llama_vocab_n_tokens(vocab)); + } + void cfg_apply() { + float * u = llama_get_logits_ith(lctx, -1); + for (size_t i = 0; i < cond_logits.size(); i++) { + u[i] = cond_logits[i] + 0.5f * (cond_logits[i] - u[i]); + } + } + + // one GEN_WAV call over the whole utterance, the flow decode is not streamed + bool flush_gen_wav() { + if (decoded || codes_buf.empty()) { + return true; + } + if (!mtl) { + // turbo appends a short silence tail before vocoding + codes_buf.insert(codes_buf.end(), 3, 4299); + } + mtmd_gen_inp inp = mtmd_gen_inp_default(mctx); + inp.type = MTMD_GEN_PROCESS_TYPE_GEN_WAV; + inp.codes = codes_buf.data(); + inp.n_codes = codes_buf.size(); + mtmd_gen_out out{}; + if (mtmd_gen_audio_process(mctx, gen_state.get(), &inp, &out) != 0) { + LOG_ERR("mtmd_helper_gen_audio: gen_wav decode failed\n"); + return false; + } + audio_pcm.assign(out.audio, out.audio + out.n_samples); + decoded = true; + return true; + } + + bool ensure_cache() { + if (!tok_embd.empty()) { + return true; + } + // fused vocab layout: [text 0..speech_base) then the speech tokens + speech_base = find_special_token(vocab, "<|speech_0|>"); + if (speech_base == LLAMA_TOKEN_NULL) { + LOG_ERR("mtmd_helper_gen_audio: fused speech tokens not found in vocab\n"); + return false; + } + + const uint32_t n_tok_embd = llama_model_get_tok_embd(model, nullptr); + if (n_tok_embd != (uint32_t) llama_vocab_n_tokens(vocab) * (uint32_t) n_embd) { + LOG_ERR("mtmd_helper_gen_audio: unexpected token embedding size\n"); + return false; + } + tok_embd.resize(n_tok_embd); + llama_model_get_tok_embd(model, tok_embd.data()); + + auto read_table = [&](const char * name, std::vector & dst) { + const size_t n = mtmd_gen_audio_read_tensor(mctx, name, nullptr, 0); + dst.resize(n); + if (n == 0 || mtmd_gen_audio_read_tensor(mctx, name, dst.data(), n) != n) { + LOG_ERR("mtmd_helper_gen_audio: %s read failed\n", name); + return false; + } + return true; + }; + + // multilingual: the learned position tables + mtl = mtmd_gen_audio_read_tensor(mctx, "a.gen.t3.text_pos_emb", nullptr, 0) > 0; + if (mtl && (!read_table("a.gen.t3.text_pos_emb", text_pos) || !read_table("a.gen.t3.speech_pos_emb", speech_pos))) { + return false; + } + return true; + } + + // vocab and mmproj data fixed across the whole session, read once + std::vector tok_embd; + bool mtl = false; + std::vector text_pos; + std::vector speech_pos; + llama_token speech_base = LLAMA_TOKEN_NULL; + // start_text_token and stop_text_token of the reference, only the multilingual prompt uses them + static constexpr llama_token text_start = 255; + static constexpr llama_token text_stop = 0; + + // per-generation state, cleared by reset() + llama_seq_id seq_id = 0; + llama_seq_id seq_uncond = 0; + llama_pos pos = 0; + int ar_idx = 0; + std::vector prompt_embd_buf; + std::vector uncond_embd_buf; + int n_prompt = 0; + int prompt_pos = 0; + std::vector cond_logits; + std::vector codes_buf; + bool decoded = false; + std::vector audio_pcm; + std::vector h_state_buf; + mtmd_helper_gen_audio_outtype out_type = MTMD_HELPER_GEN_AUDIO_OUTTYPE_WAV; + std::vector out_buf; + bool stream = false; + size_t pcm_sent = 0; // samples already returned by get_output() + bool wav_header_sent = false; +}; + +// Drama 3: backbone samples the main token of each frame, codebook tower samples the other codes, Fish DAC makes the PCM +// the tower attends over the whole sequence, so it also runs on every prompt row +class drama3_gen_audio_pipeline : public mtmd_gen_audio_pipeline { +public: + using mtmd_gen_audio_pipeline::mtmd_gen_audio_pipeline; + + void reset() override { + seq_id = 0; + pos = 0; + prompt_tokens.clear(); + prompt_pos = 0; + frame.clear(); + hist.clear(); + codes_buf.clear(); + n_decoded = 0; + gen_state.reset(); + pcm_sent = 0; + wav_header_sent = false; + audio_pcm.clear(); + h_state_buf.clear(); + out_buf.clear(); + } + + int32_t set_input(const mtmd_helper_gen_audio_inp * inp) override { + reset(); + seq_id = inp->seq_id; + out_type = inp->out_type; + stream = inp->stream; + if (inp->speaker_ref) { + LOG_WRN("mtmd_helper_gen_audio: voice cloning is not supported yet, the speaker file is ignored\n"); + } + + sem_begin = find_special_token(vocab, "<|semantic:0|>"); + eos = find_special_token(vocab, "<|im_end|>"); + if (sem_begin == LLAMA_TOKEN_NULL || eos == LLAMA_TOKEN_NULL) { + LOG_ERR("mtmd_helper_gen_audio: missing Drama 3 tokens in vocab\n"); + return 1; + } + + // reference-bracket-v1 prompt without references, tokenize each segment alone as the reference does + std::string text(inp->prompt, inp->prompt_len); + if (text.rfind("<|speaker:", 0) != 0) { + text = "<|speaker:0|> " + text; + } + const char * segments[] = { + "<|im_start|>system\n", "convert the provided text to speech", "<|im_end|>\n", + "<|im_start|>user\n", text.c_str(), "<|im_end|>\n", "<|im_start|>assistant\n<|voice|>", + }; + for (const char * seg : segments) { + const int32_t len = (int32_t) strlen(seg); + std::vector ids(len + 16); + const int32_t n = llama_tokenize(vocab, seg, len, ids.data(), (int32_t) ids.size(), false, true); + if (n < 0) { + LOG_ERR("mtmd_helper_gen_audio: tokenization failed\n"); + return 1; + } + prompt_tokens.insert(prompt_tokens.end(), ids.begin(), ids.begin() + n); + } + if (prompt_tokens.size() >= llama_n_ctx_seq(lctx) || prompt_tokens.size() > n_tower_rows) { + LOG_ERR("mtmd_helper_gen_audio: prompt does not fit in the context\n"); + return 1; + } + if (!llama_memory_seq_rm(llama_get_memory(lctx), seq_id, 0, -1)) { + LOG_ERR("mtmd_helper_gen_audio: cannot reset the sequence\n"); + return 1; + } + rng.seed(inp->seed == UINT32_MAX ? std::random_device{}() : inp->seed); + return init_gen_state(inp->seed) ? 0 : 1; + } + + // run the tower on every prompt row to fill its KV + int32_t step_prompt(int32_t n_batch) override { + GGML_ASSERT(n_batch > 0); + const int n_prompt = (int) prompt_tokens.size(); + if (prompt_pos >= n_prompt) { + return 0; + } + // small batches: every row also outputs logits + const int n = std::min({ n_batch, n_prompt - prompt_pos, 64 }); + llama_batch batch = llama_batch_init(n, 0, 1); + for (int i = 0; i < n; i++) { + batch.token [i] = prompt_tokens[(size_t) (prompt_pos + i)]; + batch.pos [i] = pos + i; + batch.n_seq_id[i] = 1; + batch.seq_id [i][0] = seq_id; + batch.logits [i] = 1; + } + batch.n_tokens = n; + const int ret = llama_decode(lctx, batch); + llama_batch_free(batch); + if (ret != 0) { + LOG_ERR("mtmd_helper_gen_audio: prompt decode failed\n"); + return -1; + } + for (int i = 0; i < n; i++) { + const float * h = llama_get_embeddings_ith(lctx, i); + if (prompt_pos + i == n_prompt - 1) { + h_state_buf.assign(h, h + n_embd); // first frame comes from the last prompt row + break; + } + if (!run_tower(-1, h)) { + return -1; + } + } + pos += n; + prompt_pos += n; + return n_prompt - prompt_pos; + } + + // sample the main token here, ignore the token of the caller + int32_t step_gen(llama_token sampled, const float * h_state_in, const float ** h_state_out, bool * out_stop) override { + GGML_UNUSED(sampled); + GGML_UNUSED(h_state_in); + *h_state_out = nullptr; + *out_stop = false; + + const llama_token main = sample_main(llama_get_logits_ith(lctx, -1)); + const size_t n_rows = prompt_tokens.size() - 1 + codes_buf.size() / n_cb; + if (main == eos || (uint32_t) pos >= llama_n_ctx_seq(lctx) || n_rows >= n_tower_rows) { + *out_stop = true; + return 0; + } + hist.push_back(main); + if (hist.size() > ras_window) { + hist.erase(hist.begin()); + } + + const float * next_embd = run_tower(main - sem_begin, h_state_buf.data()); + if (!next_embd) { + return 1; + } + codes_buf.insert(codes_buf.end(), frame.begin(), frame.end()); + + // text/speech stream: all mrope sections use the same position + const bool mrope = llama_model_rope_type(model) == LLAMA_ROPE_TYPE_MROPE || + llama_model_rope_type(model) == LLAMA_ROPE_TYPE_IMROPE; + decode_embd_batch batch(next_embd, 1, mrope ? 4 : 1, n_embd); + if (mrope) { + batch.set_position_mrope_1d(pos, seq_id); + } else { + batch.set_position_normal(pos, seq_id); + } + batch.logits[0] = 1; + if (llama_process(lctx, LLAMA_PROCESS_TYPE_DECODE, batch.render(lctx, 0, 1)) != 0) { + LOG_ERR("mtmd_helper_gen_audio: decode failed\n"); + return 1; + } + pos++; + const float * h = llama_get_embeddings_ith(lctx, -1); + h_state_buf.assign(h, h + n_embd); + *h_state_out = h_state_buf.data(); + + if (stream && codes_buf.size() / n_cb - n_decoded >= stream_frames) { + return decode_wav() ? 0 : 1; + } + return 0; + } + + int32_t get_output(int32_t * out_sample_rate, const char ** out_data, size_t * out_data_len, int64_t * out_n_samples) override { + *out_sample_rate = info.sample_rate; + + if (!stream) { + if (flush() != 0) { + return 1; + } + if (out_n_samples) { + *out_n_samples = (int64_t) audio_pcm.size(); + } + if (out_type == MTMD_HELPER_GEN_AUDIO_OUTTYPE_PCM) { + *out_data = (const char *) audio_pcm.data(); + *out_data_len = audio_pcm.size() * sizeof(float); + return 0; + } + out_buf.clear(); + if (!write_wav16(out_buf, audio_pcm, info.sample_rate)) { + LOG_ERR("mtmd_helper_gen_audio: output too large for WAV\n"); + return 1; + } + *out_data = out_buf.data(); + *out_data_len = out_buf.size(); + return 0; + } + + // streaming: only the audio since the previous call + const size_t n_new = audio_pcm.size() - pcm_sent; + if (out_n_samples) { + *out_n_samples = (int64_t) n_new; + } + if (out_type == MTMD_HELPER_GEN_AUDIO_OUTTYPE_PCM) { + *out_data = (const char *) (audio_pcm.data() + pcm_sent); + *out_data_len = n_new * sizeof(float); + pcm_sent = audio_pcm.size(); + return 0; + } + out_buf.clear(); + if (!wav_header_sent) { + write_wav16_header(out_buf, UINT32_MAX, info.sample_rate); + wav_header_sent = true; + } + append_wav16_pcm(out_buf, audio_pcm.data() + pcm_sent, n_new); + pcm_sent = audio_pcm.size(); + *out_data = out_buf.data(); + *out_data_len = out_buf.size(); + return 0; + } + + int32_t flush() override { + return decode_wav() ? 0 : 1; + } + +private: + // repetition-aware sampling: if a semantic draw repeats one of the last ras_window main tokens, draw again with fixed settings + static constexpr size_t ras_window = 16; + static constexpr size_t n_cb = 10; + // codebook tower KV rows (mtmd_graph_drama3_gen::n_pos_max): one per prompt row but the last, one per frame + static constexpr size_t n_tower_rows = 4096; + // frames per codec call when streaming (0.33 s) + static constexpr size_t stream_frames = 7; + + // top-p keeps the token that crosses the threshold, as the reference does + llama_token draw(const std::vector & cand_logits, float temp, int32_t top_k, float top_p) { + std::vector> cand(cand_logits.size()); + for (size_t i = 0; i < cand.size(); i++) { + cand[i] = { cand_logits[i] / temp, (int) i }; + } + std::sort(cand.begin(), cand.end(), [](const auto & a, const auto & b) { return a.first > b.first; }); + const float lmax = cand[0].first; + double sum = 0.0; + std::vector prob(cand.size()); + for (size_t i = 0; i < cand.size(); i++) { + prob[i] = std::exp((double) (cand[i].first - lmax)); + sum += prob[i]; + } + size_t n_keep = cand.size(); + if (top_k > 0) { + n_keep = std::min(n_keep, (size_t) top_k); + } + double cum = 0.0; + for (size_t i = 0; i < n_keep; i++) { + if (cum > top_p * sum) { + n_keep = i; + break; + } + cum += prob[i]; + } + double kept = 0.0; + for (size_t i = 0; i < n_keep; i++) { + kept += prob[i]; + } + double r = std::uniform_real_distribution(0.0, 1.0)(rng) * kept; + for (size_t i = 0; i < n_keep; i++) { + r -= prob[i]; + if (r <= 0.0) { + return cand[i].second; + } + } + return cand[n_keep - 1].second; + } + + // candidates: eos, then semantic codes 1.. (the reference never samples code 0) + llama_token sample_main(const float * logits) { + std::vector cand(n_cb_vocab); + cand[0] = logits[eos]; + for (int i = 1; i < n_cb_vocab; i++) { + cand[i] = logits[sem_begin + i]; + } + auto to_token = [&](int i) { return i == 0 ? eos : sem_begin + i; }; + const llama_token first = to_token(draw(cand, 1.0f, 0, 0.9f)); + if (first != eos && std::find(hist.begin(), hist.end(), first) != hist.end()) { + return to_token(draw(cand, 1.0f, 30, 0.95f)); + } + return first; + } + + // code0 < 0 for a prompt row, else the main code of the next frame + // returns the next backbone input, the codes of the next frame go to frame + const float * run_tower(int32_t code0, const float * h) { + mtmd_gen_inp inp = mtmd_gen_inp_default(mctx); + inp.type = MTMD_GEN_PROCESS_TYPE_GEN_CODE; + inp.code0 = code0; + inp.embd = const_cast(h); + inp.codes = frame.empty() ? nullptr : frame.data(); + inp.n_codes = frame.size(); + mtmd_gen_out out{}; + if (mtmd_gen_audio_process(mctx, gen_state.get(), &inp, &out) != 0 || (code0 >= 0 && out.n_codes != n_cb)) { + LOG_ERR("mtmd_helper_gen_audio: codebook tower failed\n"); + return nullptr; + } + if (code0 >= 0) { + frame.assign(out.codes, out.codes + out.n_codes); + } + return out.embd; + } + + // the codec is stateful, so frames can be decoded in pieces + bool decode_wav() { + const size_t n_frames = codes_buf.size() / n_cb; + while (n_decoded < n_frames) { + const size_t n = std::min(n_frames - n_decoded, (size_t) 64); + mtmd_gen_inp inp = mtmd_gen_inp_default(mctx); + inp.type = MTMD_GEN_PROCESS_TYPE_GEN_WAV; + inp.codes = codes_buf.data() + n_decoded * n_cb; + inp.n_codes = n * n_cb; + mtmd_gen_out out{}; + if (mtmd_gen_audio_process(mctx, gen_state.get(), &inp, &out) != 0) { + LOG_ERR("mtmd_helper_gen_audio: codec decode failed\n"); + return false; + } + audio_pcm.insert(audio_pcm.end(), out.audio, out.audio + out.n_samples); + n_decoded += n; + } + return true; + } + + static constexpr int n_cb_vocab = 4096; // <|semantic:0..4095|> + + llama_token sem_begin = LLAMA_TOKEN_NULL; + llama_token eos = LLAMA_TOKEN_NULL; + + llama_seq_id seq_id = 0; + llama_pos pos = 0; + std::vector prompt_tokens; + int prompt_pos = 0; + + std::mt19937 rng; + std::vector frame; // codes of the last frame, input of the next tower row + std::vector hist; // last main tokens, for repetition-aware sampling + std::vector codes_buf; // frame-major + size_t n_decoded = 0; // frames already through the codec + + std::vector audio_pcm; + std::vector h_state_buf; + mtmd_helper_gen_audio_outtype out_type = MTMD_HELPER_GEN_AUDIO_OUTTYPE_WAV; + std::vector out_buf; + bool stream = false; + size_t pcm_sent = 0; // samples already returned by get_output() + bool wav_header_sent = false; +}; + static std::unique_ptr make_pipeline(llama_context * lctx, mtmd_context * mctx) { switch (mtmd_gen_audio_get_info(mctx).type) { case MTMD_GEN_AUDIO_TYPE_QWEN3TTS: @@ -1728,6 +2428,10 @@ static std::unique_ptr make_pipeline(llama_context * lc return std::unique_ptr(new soprano_gen_audio_pipeline(lctx, mctx)); case MTMD_GEN_AUDIO_TYPE_LFM3A: return std::unique_ptr(new lfm3a_gen_audio_pipeline(lctx, mctx)); + case MTMD_GEN_AUDIO_TYPE_CHATTERBOX: + return std::unique_ptr(new chatterbox_gen_audio_pipeline(lctx, mctx)); + case MTMD_GEN_AUDIO_TYPE_DRAMA3: + return std::unique_ptr(new drama3_gen_audio_pipeline(lctx, mctx)); default: return nullptr; } @@ -1757,6 +2461,7 @@ struct mtmd_helper_gen_audio_inp mtmd_helper_gen_audio_inp_default(void) { mtmd_helper_gen_audio_inp inp{}; inp.top_k = 50; inp.top_p = 1.0f; + inp.temp = -1.0f; // model default inp.seed = UINT32_MAX; // random inp.out_type = MTMD_HELPER_GEN_AUDIO_OUTTYPE_WAV; return inp; diff --git a/tools/mtmd/mtmd-helper.cpp b/tools/mtmd/mtmd-helper.cpp index 29812219c2b2..aefe504b2891 100644 --- a/tools/mtmd/mtmd-helper.cpp +++ b/tools/mtmd/mtmd-helper.cpp @@ -360,6 +360,74 @@ static bool decode_audio_from_buf(const unsigned char * buf_in, size_t len, int } // namespace audio_helpers +struct mtmd_helper_resampler { + ma_resampler rs; + bool passthrough = false; + std::vector out; +}; + +mtmd_helper_resampler * mtmd_helper_resampler_init(int32_t rate_in, int32_t rate_out) { + if (rate_in <= 0 || rate_out <= 0) { + LOG_ERR("%s: invalid sample rate %d -> %d\n", __func__, rate_in, rate_out); + return nullptr; + } + auto * ctx = new mtmd_helper_resampler; + if (rate_in == rate_out) { + ctx->passthrough = true; + return ctx; + } + ma_resampler_config config = ma_resampler_config_init(ma_format_f32, 1, (ma_uint32) rate_in, (ma_uint32) rate_out, + ma_resample_algorithm_linear); + config.linear.lpfOrder = MA_MAX_FILTER_ORDER; + if (ma_resampler_init(&config, nullptr, &ctx->rs) != MA_SUCCESS) { + LOG_ERR("%s: failed to initialize the resampler %d -> %d\n", __func__, rate_in, rate_out); + delete ctx; + return nullptr; + } + return ctx; +} + +void mtmd_helper_resampler_free(mtmd_helper_resampler * ctx) { + if (!ctx) { + return; + } + if (!ctx->passthrough) { + ma_resampler_uninit(&ctx->rs, nullptr); + } + delete ctx; +} + +int64_t mtmd_helper_resampler_process(mtmd_helper_resampler * ctx, const float * in, size_t n_in, const float ** out) { + if (ctx->passthrough) { + *out = in; + return (int64_t) n_in; + } + ma_uint64 n_expected = 0; + if (ma_resampler_get_expected_output_frame_count(&ctx->rs, n_in, &n_expected) != MA_SUCCESS) { + return -1; + } + ctx->out.resize(n_expected + 16); + size_t n_done_in = 0; + size_t n_done_out = 0; + while (n_done_in < n_in) { + if (n_done_out == ctx->out.size()) { + ctx->out.resize(ctx->out.size() * 2); + } + ma_uint64 cnt_in = n_in - n_done_in; + ma_uint64 cnt_out = ctx->out.size() - n_done_out; + if (ma_resampler_process_pcm_frames(&ctx->rs, in + n_done_in, &cnt_in, ctx->out.data() + n_done_out, &cnt_out) != MA_SUCCESS) { + return -1; + } + if (cnt_in == 0 && cnt_out == 0) { + break; + } + n_done_in += cnt_in; + n_done_out += cnt_out; + } + *out = ctx->out.data(); + return (int64_t) n_done_out; +} + static bool is_webp_file(const unsigned char * buf, size_t len) { // WEBP ref: https://developers.google.com/speed/webp/docs/riff_container return len >= 12 && memcmp(buf, "RIFF", 4) == 0 && memcmp(buf + 8, "WEBP", 4) == 0; @@ -508,8 +576,6 @@ mtmd_helper_bitmap_wrapper mtmd_helper_bitmap_init_from_file(const mtmd_context // Video input helpers // -#ifdef MTMD_VIDEO - // MSVC deprecates strdup(), use a std::string version directly static char * mtmd_strdup(const std::string & s) { size_t n = s.size() + 1; @@ -520,15 +586,218 @@ static char * mtmd_strdup(const std::string & s) { return p; } -struct mtmd_helper_video { +mtmd_helper_video_seq::~mtmd_helper_video_seq() { + mtmd_bitmap_free(pending_frame); + mtmd_bitmap_free(pending_audio); +} + +int32_t mtmd_helper_video_seq::ts_per_n_frames() const { + return tmpl == MTMD_VIDEO_TEMPLATE_LFM3 || tmpl == MTMD_VIDEO_TEMPLATE_LFM3_OMNI || tmpl == MTMD_VIDEO_TEMPLATE_QWEN3VL ? 2 : 0; +} + +void mtmd_helper_video_seq::configure(const mtmd_video_config & cfg, const mtmd_helper_video_src & src, float fps_target_arg) { + tmpl = cfg.tmpl; + uint32_t width = src.width; + uint32_t height = src.height; + float duration = src.duration; + if (duration < 0.0f && src.n_frames > 0) { + duration = (float)src.n_frames / src.fps; + } + + // the model's recommended sampling rate wins over the generic default + fps_target = cfg.fps > 0.0f ? cfg.fps + : fps_target_arg > 0.0f ? fps_target_arg : src.fps; + + if (cfg.n_frames > 0 && duration > 0.0f) { + const int32_t n = cfg.n_frames; + target_frames = n; + fps_target = (float) n / duration; + + // same frames as the reference processors: round(linspace(0, n_frames - 1, n)) + const double step = n > 1 ? (double) (src.n_frames - 1) / (n - 1) : 0.0; + frame_indices.resize(n); + for (int32_t i = 0; i < n; i++) { + frame_indices[i] = (int32_t) std::nearbyint(i * step); + } + src_fps = src.fps; + n_src_frames = src.n_frames; + + if (cfg.frame_width > 0 && cfg.frame_height > 0 && + (cfg.frame_width != (int32_t) width || cfg.frame_height != (int32_t) height)) { + out_width = cfg.frame_width; + out_height = cfg.frame_height; + } + } + + if (out_width > 0 && out_height > 0) { + width = (uint32_t) out_width; + height = (uint32_t) out_height; + } + + info.width = width; + info.height = height; + info.fps = fps_target; + LOG_DBG("%s: %ux%u fps=%.2f duration=%.2fs n_frames=%d target_frames=%d\n", + __func__, width, height, fps_target, duration, src.n_frames, target_frames); + info.n_frames = target_frames > 0 ? target_frames + : duration > 0.0f ? (int32_t)(duration * fps_target + 0.5f) : -1; + frame_buf.resize((size_t)width * height * 3); +} + +// the soundtrack of the group of frames that ends with frame idx, nullptr if the soundtrack has ended +mtmd_bitmap * mtmd_helper_video_seq::soundtrack_slice(int32_t idx) const { + const int32_t ts_per_n_frames = this->ts_per_n_frames(); + const int32_t group = idx / ts_per_n_frames; + const int32_t n_groups = ((int32_t) frame_indices.size() + ts_per_n_frames - 1) / ts_per_n_frames; + auto group_start = [&](int32_t g) { + const double t = frame_indices[g * ts_per_n_frames] / src_fps; + return std::min(soundtrack.size(), (size_t) std::nearbyint(t * audio_rate)); + }; + const size_t begin = group == 0 ? 0 : group_start(group); + const size_t end = group == n_groups - 1 ? soundtrack.size() : group_start(group + 1); + if (begin >= end) { + return nullptr; + } + mtmd_bitmap * slice = mtmd_bitmap_init_from_audio(end - begin, soundtrack.data() + begin); + if (!id.empty()) { + mtmd_bitmap_set_id(slice, (id + "+audio" + std::to_string(group)).c_str()); + } + return slice; +} + +mtmd_bitmap * mtmd_helper_video_seq::read_next_frame() { + if (!src_alive) return nullptr; + + if (!frame_indices.empty()) { + if (next_index >= frame_indices.size()) { + return nullptr; + } + // skip to the wanted source frame; a repeated index reuses the frame in frame_buf + const int32_t want = frame_indices[next_index]; + while (src_frame <= want) { + if (!read_raw_frame()) { + src_alive = false; + return nullptr; + } + src_frame++; + } + next_index++; + } else if (!read_raw_frame()) { + src_alive = false; + return nullptr; + } + + LOG_DBG("%s: frame %d read OK\n", __func__, current_frame); + mtmd_bitmap * frame = mtmd_bitmap_init(info.width, info.height, frame_buf.data()); + mtmd_bitmap_set_mergeable(frame, true); + mtmd_bitmap_set_is_video(frame, true); + if (!id.empty()) { + // each frame gets a unique id in the form of {hash}+{frame}, so that it can be identified in cache + std::string frame_id = id + "+" + std::to_string(current_frame); + mtmd_bitmap_set_id(frame, frame_id.c_str()); + } + current_frame++; + return frame; +} + +int32_t mtmd_helper_video_seq::read_next(mtmd_bitmap ** out_bitmap, char ** out_text) { + *out_bitmap = nullptr; + *out_text = nullptr; + + if (!pending_text.empty()) { + *out_text = mtmd_strdup(pending_text); + pending_text.clear(); + return *out_text ? 0 : -2; + } + + if (pending_frame) { + *out_bitmap = pending_frame; + pending_frame = nullptr; + return 0; + } + + if (pending_audio) { + *out_bitmap = pending_audio; + pending_audio = nullptr; + return 0; + } + + LOG_DBG("%s: proc_alive=%d start_emitted=%d current_frame=%d\n", + __func__, (int)src_alive, (int)start_emitted, current_frame); + + if (!src_alive) { + return (current_frame == 0) ? -2 : -1; + } + + if (!start_emitted) { + start_emitted = true; + // model templates have no prompt prefix, the video begin/end tokens are added by mtmd_tokenize + if (tmpl == MTMD_VIDEO_TEMPLATE_DEFAULT && !prompt_start.empty()) { + *out_text = mtmd_strdup(prompt_start); + return *out_text ? 0 : -2; + } + } + + mtmd_bitmap * frame = read_next_frame(); + if (!frame) { + return -1; + } + + const int32_t ts_per_n_frames = this->ts_per_n_frames(); + if (ts_per_n_frames > 0) { + // emit the group timestamp before the group's first frame; the value is the + // midpoint time of the group, matching HF processors that stamp each temporal group + const int32_t idx = current_frame - 1; // read_next_frame() already incremented + // the soundtrack of a group follows its last frame + const bool group_ends = idx % ts_per_n_frames == ts_per_n_frames - 1 || idx + 1 == (int32_t) frame_indices.size(); + if (!soundtrack.empty() && !frame_indices.empty() && group_ends) { + pending_audio = soundtrack_slice(idx); + } + if (idx % ts_per_n_frames == 0) { + char ts_buf[64]; + double ts_s = ((double)idx + 0.5*(double)(ts_per_n_frames - 1)) / info.fps; + if (!frame_indices.empty()) { + // stamp from the source frames; a short last group repeats its last frame + const size_t first = idx; + const size_t last = std::min(first + ts_per_n_frames - 1, frame_indices.size() - 1); + ts_s = (frame_indices[first] / src_fps + frame_indices[last] / src_fps) / 2.0; + } + snprintf(ts_buf, sizeof(ts_buf), "<%.1f seconds>", ts_s); + pending_frame = frame; + *out_text = mtmd_strdup(ts_buf); + return *out_text ? 0 : -2; + } + *out_bitmap = frame; + return 0; + } + + *out_bitmap = frame; + + if (timestamp_interval_ms > 0) { + // current_frame was already incremented by read_next_frame(); undo for elapsed calc + float elapsed_ms = (float)(current_frame - 1) / info.fps * 1000.0f; + if (elapsed_ms >= next_timestamp_ms) { + char ts_buf[32]; + float elapsed_s = elapsed_ms / 1000.0f; + int minutes = (int)(elapsed_s / 60); + float seconds = elapsed_s - minutes * 60.0f; + snprintf(ts_buf, sizeof(ts_buf), "[%dm%.2fs]", minutes, seconds); + pending_text = ts_buf; + next_timestamp_ms += (float)timestamp_interval_ms; + } + } + + return 0; +} + +#ifdef MTMD_VIDEO + +struct mtmd_helper_video : mtmd_helper_video_seq { const mtmd_context * mctx; std::string path; std::vector input_buf; // non-empty when initialized from buffer std::string ffmpeg_bin; std::string ffprobe_bin; - float fps_target = 0.0f; - mtmd_helper_video_info info = {}; - std::string id; // hash of the input video // RAII wrapper for managing subprocess struct subprocess_handle { @@ -596,46 +865,15 @@ struct mtmd_helper_video { }; subprocess_handle sp; - int32_t current_frame = 0; - - std::string prompt_start = "Video:"; - int32_t timestamp_interval_ms = 5000; // emit a timestamp text every N ms (0 = disabled) - float next_timestamp_ms = 0.0f; // next elapsed-ms threshold at which to emit // apply mtmd_get_video_config(); false for still images decoded via ffmpeg - bool use_video_config = false; - mtmd_video_template tmpl = MTMD_VIDEO_TEMPLATE_DEFAULT; - - int32_t target_frames = 0; // exact frame count to emit (0 = no limit) - int32_t out_width = 0; // frame size from mtmd_get_video_config (0 = source size) - int32_t out_height = 0; - - // source frames to emit, as picked by the reference processors (empty = use the fps filter) - std::vector frame_indices; - size_t next_index = 0; - int32_t src_frame = 0; // source frames read from ffmpeg so far - double src_fps = 0.0; - - std::vector frame_buf; - std::string pending_text; // text queued to be returned before the next frame - mtmd_bitmap * pending_frame = nullptr; // frame queued to be returned after its group timestamp - - // soundtrack: after each group of frames comes the audio from its first frame to the next group's - double video_start_s = 0.0; - int32_t n_src_frames = 0; - std::vector soundtrack; - mtmd_bitmap * pending_audio = nullptr; - bool start_emitted = false; + bool use_video_config = false; + double video_start_s = 0.0; // start time of the video stream, to align the soundtrack bool is_buf_input() const { return !input_buf.empty(); } - // number of frames per timestamp group (0 = timestamp every timestamp_interval_ms) - int32_t ts_per_n_frames() const { - return tmpl == MTMD_VIDEO_TEMPLATE_LFM3 || tmpl == MTMD_VIDEO_TEMPLATE_LFM3_OMNI ? 2 : 0; - } - bool probe(float fps_target_arg) { // cache: makes stdin seekable; without it ffprobe reports width/height 0 // for mp4 files whose moov atom sits at the end @@ -671,11 +909,7 @@ struct mtmd_helper_video { probe_sp.start_feeder(input_buf); } - uint32_t width = 0; - uint32_t height = 0; - float orig_fps = 0.0f; - float duration = -1.0f; - int32_t n_frames_orig = -1; + mtmd_helper_video_src src; char line[256]; FILE * fp = probe_sp.stdout_pipe(); @@ -689,15 +923,15 @@ struct mtmd_helper_video { if (nl) *nl = '\0'; if (strcmp(key, "width") == 0) { - width = (uint32_t)atoi(val); + src.width = (uint32_t)atoi(val); } else if (strcmp(key, "height") == 0) { - height = (uint32_t)atoi(val); + src.height = (uint32_t)atoi(val); } else if (strcmp(key, "r_frame_rate") == 0) { - orig_fps = parse_rational(val); + src.fps = parse_rational(val); } else if (strcmp(key, "nb_frames") == 0 && strcmp(val, "N/A") != 0) { - n_frames_orig = atoi(val); + src.n_frames = atoi(val); } else if (strcmp(key, "duration") == 0 && strcmp(val, "N/A") != 0) { - duration = (float)atof(val); + src.duration = (float)atof(val); } else if (strcmp(key, "start_time") == 0 && strcmp(val, "N/A") != 0) { video_start_s = atof(val); } @@ -705,58 +939,15 @@ struct mtmd_helper_video { probe_sp.stop(); - if (width == 0 || height == 0 || orig_fps <= 0.0f) { + if (src.width == 0 || src.height == 0 || src.fps <= 0.0f) { return false; } - if (duration < 0.0f && n_frames_orig > 0) { - duration = (float)n_frames_orig / orig_fps; - } - mtmd_video_config cfg{MTMD_VIDEO_TEMPLATE_DEFAULT, 0.0f, 0, 0, 0}; if (use_video_config) { - cfg = mtmd_get_video_config(mctx, width, height, n_frames_orig, orig_fps); - tmpl = cfg.tmpl; - } - - // the model's recommended sampling rate wins over the generic default - fps_target = cfg.fps > 0.0f ? cfg.fps - : fps_target_arg > 0.0f ? fps_target_arg : orig_fps; - - if (cfg.n_frames > 0 && duration > 0.0f) { - const int32_t n = cfg.n_frames; - target_frames = n; - fps_target = (float) n / duration; - - // same frames as the reference processors: round(linspace(0, n_frames - 1, n)) - const double step = n > 1 ? (double) (n_frames_orig - 1) / (n - 1) : 0.0; - frame_indices.resize(n); - for (int32_t i = 0; i < n; i++) { - frame_indices[i] = (int32_t) std::nearbyint(i * step); - } - src_fps = orig_fps; - n_src_frames = n_frames_orig; - - if (cfg.frame_width > 0 && cfg.frame_height > 0 && - (cfg.frame_width != (int32_t) width || cfg.frame_height != (int32_t) height)) { - out_width = cfg.frame_width; - out_height = cfg.frame_height; - } - } - - if (out_width > 0 && out_height > 0) { - width = (uint32_t) out_width; - height = (uint32_t) out_height; + cfg = mtmd_get_video_config(mctx, src.width, src.height, src.n_frames, src.fps); } - - info.width = width; - info.height = height; - info.fps = fps_target; - LOG_DBG("%s: %ux%u fps=%.2f duration=%.2fs n_frames=%d target_frames=%d\n", - __func__, width, height, fps_target, duration, n_frames_orig, target_frames); - info.n_frames = target_frames > 0 ? target_frames - : duration > 0.0f ? (int32_t)(duration * fps_target + 0.5f) : -1; - frame_buf.resize((size_t)width * height * 3); + configure(cfg, src, fps_target_arg); return true; } @@ -800,31 +991,10 @@ struct mtmd_helper_video { LOG_WRN("%s: the video has no soundtrack\n", __func__); return; } + audio_rate = rate; soundtrack.assign(pcm.begin() + start, pcm.begin() + std::min(pcm.size(), start + len)); } - // the soundtrack of the group of frames that ends with frame idx, nullptr if the soundtrack has ended - mtmd_bitmap * soundtrack_slice(int32_t idx) const { - const int32_t ts_per_n_frames = this->ts_per_n_frames(); - const int32_t group = idx / ts_per_n_frames; - const int32_t n_groups = ((int32_t) frame_indices.size() + ts_per_n_frames - 1) / ts_per_n_frames; - const int rate = mtmd_get_audio_sample_rate(mctx); - auto group_start = [&](int32_t g) { - const double t = frame_indices[g * ts_per_n_frames] / src_fps; - return std::min(soundtrack.size(), (size_t) std::nearbyint(t * rate)); - }; - const size_t begin = group == 0 ? 0 : group_start(group); - const size_t end = group == n_groups - 1 ? soundtrack.size() : group_start(group + 1); - if (begin >= end) { - return nullptr; - } - mtmd_bitmap * slice = mtmd_bitmap_init_from_audio(end - begin, soundtrack.data() + begin); - if (!id.empty()) { - mtmd_bitmap_set_id(slice, (id + "+audio" + std::to_string(group)).c_str()); - } - return slice; - } - bool start_ffmpeg(float seek_seconds) { char seek_buf[64]; char fps_buf[64]; @@ -917,6 +1087,7 @@ struct mtmd_helper_video { sp.start_feeder(input_buf); } + src_alive = sp.alive; return sp.alive; } @@ -925,7 +1096,7 @@ struct mtmd_helper_video { } // read one raw frame from ffmpeg into frame_buf - bool read_raw_frame() { + bool read_raw_frame() override { FILE * fp = sp.stdout_pipe(); const size_t frame_size = (size_t)info.width * info.height * 3; LOG_DBG("%s: reading frame %d, expecting %zu bytes (%ux%u)\n", @@ -938,7 +1109,6 @@ struct mtmd_helper_video { // clean EOF only if no bytes read yet; partial frame is an error LOG_DBG("%s: fread returned 0 after %zu/%zu bytes (ferror=%d)\n", __func__, total_read, frame_size, ferror(fp)); - sp.alive = false; return false; } total_read += n; @@ -946,129 +1116,6 @@ struct mtmd_helper_video { return true; } - mtmd_bitmap * read_next_frame() { - if (!sp.alive) return nullptr; - - if (!frame_indices.empty()) { - if (next_index >= frame_indices.size()) { - return nullptr; - } - // skip to the wanted source frame; a repeated index reuses the frame in frame_buf - const int32_t want = frame_indices[next_index]; - while (src_frame <= want) { - if (!read_raw_frame()) { - return nullptr; - } - src_frame++; - } - next_index++; - } else if (!read_raw_frame()) { - return nullptr; - } - - LOG_DBG("%s: frame %d read OK\n", __func__, current_frame); - mtmd_bitmap * frame = mtmd_bitmap_init(info.width, info.height, frame_buf.data()); - mtmd_bitmap_set_mergeable(frame, true); - mtmd_bitmap_set_is_video(frame, true); - if (!id.empty()) { - // each frame gets a unique id in the form of {hash}+{frame}, so that it can be identified in cache - std::string frame_id = id + "+" + std::to_string(current_frame); - mtmd_bitmap_set_id(frame, frame_id.c_str()); - } - current_frame++; - return frame; - } - - int32_t read_next(mtmd_bitmap ** out_bitmap, char ** out_text) { - *out_bitmap = nullptr; - *out_text = nullptr; - - if (!pending_text.empty()) { - *out_text = mtmd_strdup(pending_text); - pending_text.clear(); - return *out_text ? 0 : -2; - } - - if (pending_frame) { - *out_bitmap = pending_frame; - pending_frame = nullptr; - return 0; - } - - if (pending_audio) { - *out_bitmap = pending_audio; - pending_audio = nullptr; - return 0; - } - - LOG_DBG("%s: proc_alive=%d start_emitted=%d current_frame=%d\n", - __func__, (int)sp.alive, (int)start_emitted, current_frame); - - if (!sp.alive) { - return (current_frame == 0) ? -2 : -1; - } - - if (!start_emitted) { - start_emitted = true; - // model templates have no prompt prefix, the video begin/end tokens are added by mtmd_tokenize - if (tmpl == MTMD_VIDEO_TEMPLATE_DEFAULT && !prompt_start.empty()) { - *out_text = mtmd_strdup(prompt_start); - return *out_text ? 0 : -2; - } - } - - mtmd_bitmap * frame = read_next_frame(); - if (!frame) { - return -1; - } - - const int32_t ts_per_n_frames = this->ts_per_n_frames(); - if (ts_per_n_frames > 0) { - // emit the group timestamp before the group's first frame; the value is the - // midpoint time of the group, matching HF processors that stamp each temporal group - const int32_t idx = current_frame - 1; // read_next_frame() already incremented - // the soundtrack of a group follows its last frame - const bool group_ends = idx % ts_per_n_frames == ts_per_n_frames - 1 || idx + 1 == (int32_t) frame_indices.size(); - if (!soundtrack.empty() && !frame_indices.empty() && group_ends) { - pending_audio = soundtrack_slice(idx); - } - if (idx % ts_per_n_frames == 0) { - char ts_buf[64]; - double ts_s = ((double)idx + 0.5*(double)(ts_per_n_frames - 1)) / info.fps; - if (!frame_indices.empty()) { - // stamp from the source frames; a short last group repeats its last frame - const size_t first = idx; - const size_t last = std::min(first + ts_per_n_frames - 1, frame_indices.size() - 1); - ts_s = (frame_indices[first] / src_fps + frame_indices[last] / src_fps) / 2.0; - } - snprintf(ts_buf, sizeof(ts_buf), "<%.1f seconds>", ts_s); - pending_frame = frame; - *out_text = mtmd_strdup(ts_buf); - return *out_text ? 0 : -2; - } - *out_bitmap = frame; - return 0; - } - - *out_bitmap = frame; - - if (timestamp_interval_ms > 0) { - // current_frame was already incremented by read_next_frame(); undo for elapsed calc - float elapsed_ms = (float)(current_frame - 1) / info.fps * 1000.0f; - if (elapsed_ms >= next_timestamp_ms) { - char ts_buf[32]; - float elapsed_s = elapsed_ms / 1000.0f; - int minutes = (int)(elapsed_s / 60); - float seconds = elapsed_s - minutes * 60.0f; - snprintf(ts_buf, sizeof(ts_buf), "[%dm%.2fs]", minutes, seconds); - pending_text = ts_buf; - next_timestamp_ms += (float)timestamp_interval_ms; - } - } - - return 0; - } - static float parse_rational(const char * s) { int num = 0, den = 1; if (sscanf(s, "%d/%d", &num, &den) == 2 && den > 0) { @@ -1233,8 +1280,6 @@ void mtmd_helper_video_free(mtmd_helper_video * ctx) { #ifdef MTMD_VIDEO if (!ctx) return; ctx->stop_ffmpeg(); - mtmd_bitmap_free(ctx->pending_frame); - mtmd_bitmap_free(ctx->pending_audio); delete ctx; #else GGML_UNUSED(ctx); diff --git a/tools/mtmd/mtmd-helper.h b/tools/mtmd/mtmd-helper.h index e562aae1823c..7d72fdb994f2 100644 --- a/tools/mtmd/mtmd-helper.h +++ b/tools/mtmd/mtmd-helper.h @@ -78,6 +78,19 @@ MTMD_API struct mtmd_helper_bitmap_wrapper mtmd_helper_bitmap_init_from_buf( bool placeholder, struct mtmd_helper_init_opt opt); +// streaming resampler for raw mono F32 PCM (e.g. realtime input), to match mtmd_get_audio_sample_rate() +// it keeps state between calls, use one per stream +struct mtmd_helper_resampler; +typedef struct mtmd_helper_resampler mtmd_helper_resampler; + +// returns NULL on failure +MTMD_API mtmd_helper_resampler * mtmd_helper_resampler_init(int32_t rate_in, int32_t rate_out); +MTMD_API void mtmd_helper_resampler_free(mtmd_helper_resampler * ctx); + +// *out is valid until the next call +// returns the number of output samples (can be 0), < 0 on error +MTMD_API int64_t mtmd_helper_resampler_process(mtmd_helper_resampler * ctx, const float * in, size_t n_in, const float ** out); + // helper to count the total number of tokens from a list of chunks, useful to keep track of KV cache MTMD_API size_t mtmd_helper_get_n_tokens(const mtmd_input_chunks * chunks); @@ -248,6 +261,7 @@ struct mtmd_helper_gen_audio_inp { int32_t top_k; float top_p; + float temp; // audio frame temperature if the model supports it, < 0 = model default, 0 = greedy uint32_t seed; // UINT32_MAX for random (default: random) enum mtmd_helper_gen_audio_outtype out_type; @@ -318,6 +332,7 @@ MTMD_API int32_t mtmd_helper_gen_audio_flush(mtmd_helper_gen_audio * ctx); #ifdef __cplusplus #include #include +#include #include namespace mtmd_helper { @@ -332,6 +347,12 @@ struct mtmd_helper_video_deleter { }; using video_ptr = std::unique_ptr; +// audio input-related C++ wrappers +struct mtmd_helper_resampler_deleter { + void operator()(mtmd_helper_resampler * val) { mtmd_helper_resampler_free(val); } +}; +using resampler_ptr = std::unique_ptr; + // audio generation-related C++ wrappers struct mtmd_helper_gen_audio_deleter { void operator()(mtmd_helper_gen_audio * val) { mtmd_helper_gen_audio_free(val); } @@ -408,6 +429,68 @@ inline std::vector audio_stream_logit_bias(const mtmd_context } } // namespace mtmd_helper + +// +// test API (C++ only) +// internals of mtmd_helper_video, do not use in applications +// + +struct mtmd_helper_video_src { + uint32_t width = 0; + uint32_t height = 0; + float fps = 0.0f; + int32_t n_frames = -1; + float duration = -1.0f; +}; + +struct mtmd_helper_video_seq { + float fps_target = 0.0f; + mtmd_helper_video_info info = {}; + std::string id; + + bool src_alive = false; + int32_t current_frame = 0; + + std::string prompt_start = "Video:"; + int32_t timestamp_interval_ms = 5000; + float next_timestamp_ms = 0.0f; + + mtmd_video_template tmpl = MTMD_VIDEO_TEMPLATE_DEFAULT; + + int32_t target_frames = 0; + int32_t out_width = 0; + int32_t out_height = 0; + + std::vector frame_indices; + size_t next_index = 0; + int32_t src_frame = 0; + double src_fps = 0.0; + + std::vector frame_buf; + std::string pending_text; + mtmd_bitmap * pending_frame = nullptr; + + int32_t n_src_frames = 0; + int audio_rate = 0; + std::vector soundtrack; + mtmd_bitmap * pending_audio = nullptr; + bool start_emitted = false; + + mtmd_helper_video_seq() = default; + mtmd_helper_video_seq(const mtmd_helper_video_seq &) = delete; + mtmd_helper_video_seq & operator=(const mtmd_helper_video_seq &) = delete; + virtual ~mtmd_helper_video_seq(); + + virtual bool read_raw_frame() = 0; + + void configure(const mtmd_video_config & cfg, const mtmd_helper_video_src & src, float fps_target_arg); + + int32_t ts_per_n_frames() const; + + mtmd_bitmap * soundtrack_slice(int32_t idx) const; + mtmd_bitmap * read_next_frame(); + int32_t read_next(mtmd_bitmap ** out_bitmap, char ** out_text); +}; #endif #endif diff --git a/tools/mtmd/mtmd-image.cpp b/tools/mtmd/mtmd-image.cpp index 2750bc3fc61b..33ce243d5291 100644 --- a/tools/mtmd/mtmd-image.cpp +++ b/tools/mtmd/mtmd-image.cpp @@ -4,8 +4,8 @@ #include #include -void mtmd_image_preproc_out::append(const clip_hparams & hparams, const clip_image_u8 & img, bool normalized) { - clip_image_f32 dst; +void mtmd_image_preproc_out::append(const mtmd_hparams & hparams, const mtmd_image_u8 & img, bool normalized) { + mtmd_image_f32 dst; dst.from_u8(img); if (normalized) { dst.normalize(hparams.image_mean, hparams.image_std); @@ -13,20 +13,20 @@ void mtmd_image_preproc_out::append(const clip_hparams & hparams, const clip_ima entries.push_back(std::move(dst)); } -void mtmd_image_preproc_out::append(const clip_hparams & hparams, const std::vector & imgs, bool normalized) { +void mtmd_image_preproc_out::append(const mtmd_hparams & hparams, const std::vector & imgs, bool normalized) { for (const auto & img : imgs) { append(hparams, img, normalized); } } -void mtmd_image_preproc_out::append(const clip_hparams & hparams, clip_image_f32 & img, bool normalized) { +void mtmd_image_preproc_out::append(const mtmd_hparams & hparams, mtmd_image_f32 & img, bool normalized) { if (normalized) { img.normalize(hparams.image_mean, hparams.image_std); } entries.push_back(std::move(img)); } -void mtmd_image_preproc_out::append_overview(const clip_hparams & hparams, const clip_image_u8 & img, bool normalized) { +void mtmd_image_preproc_out::append_overview(const mtmd_hparams & hparams, const mtmd_image_u8 & img, bool normalized) { overview.from_u8(img); if (normalized) { overview.normalize(hparams.image_mean, hparams.image_std); @@ -37,9 +37,9 @@ void mtmd_image_preproc_out::append_overview(const clip_hparams & hparams, const // in the future, we can have HW acceleration by allowing this struct to access 3rd party lib like imagick or opencv struct img_tool { static void resize( - const clip_image_u8 & src, - clip_image_u8 & dst, - const clip_image_size & target_resolution, + const mtmd_image_u8 & src, + mtmd_image_u8 & dst, + const mtmd_image_size & target_resolution, resize_algo algo, pad_style padding = PAD_CEIL, std::array pad_color = {0, 0, 0}) { @@ -61,7 +61,7 @@ struct img_tool { resize_pillow(src, dst, target_resolution.width, target_resolution.height, algo); } else { // resize with padding - clip_image_u8 resized_image; + mtmd_image_u8 resized_image; float scale_w = static_cast(target_resolution.width) / src.get_size().width; float scale_h = static_cast(target_resolution.height) / src.get_size().height; float scale = std::min(scale_w, scale_h); @@ -92,7 +92,7 @@ struct img_tool { } } - static void crop(const clip_image_u8 & image, clip_image_u8 & dst, int x, int y, int w, int h) { + static void crop(const mtmd_image_u8 & image, mtmd_image_u8 & dst, int x, int y, int w, int h) { GGML_ASSERT(x >= 0 && y >= 0 && w > 0 && h > 0); GGML_ASSERT(x + w <= image.get_size().width && y + h <= image.get_size().height); dst.set_size({w, h}, image.is_placeholder()); @@ -120,7 +120,7 @@ struct img_tool { // calculate the size of the **resized** image, while preserving the aspect ratio and // aligning to the nearest multiple of align_size ("smart_resize" in transformers code) - static clip_image_size calc_size_preserved_ratio(const clip_image_size & inp_size, const calc_size_opt & opts) { + static mtmd_image_size calc_size_preserved_ratio(const mtmd_image_size & inp_size, const calc_size_opt & opts) { GGML_ASSERT(opts.align_size > 0); const int width = inp_size.width; const int height = inp_size.height; @@ -161,7 +161,7 @@ struct img_tool { } // draw src image into dst image at offset (offset_x, offset_y) - static void composite(clip_image_u8 & dst, const clip_image_u8 & src, int offset_x, int offset_y) { + static void composite(mtmd_image_u8 & dst, const mtmd_image_u8 & src, int offset_x, int offset_y) { if (src.is_placeholder()) { // no-op for placeholder image return; @@ -183,7 +183,7 @@ struct img_tool { } // fill the image with a solid color - static void fill(clip_image_u8 & img, const std::array & color) { + static void fill(mtmd_image_u8 & img, const std::array & color) { if (img.is_placeholder()) { // no-op for placeholder image return; @@ -208,8 +208,8 @@ struct img_tool { // // RESIZE_ALGO_FLAG_TORCH is the same, except for the int16 weights of torchvision (aten/src/ATen/native/cpu/UpSampleKernel.cpp) static bool resize_pillow( - const clip_image_u8 & img, - clip_image_u8 & dst, + const mtmd_image_u8 & img, + mtmd_image_u8 & dst, int target_width, int target_height, resize_algo algo) { @@ -516,8 +516,8 @@ struct img_tool { // mtmd_image_preprocessor_llava_uhd // -mtmd_image_preproc_out mtmd_image_preprocessor_llava_uhd::preprocess(const clip_image_u8 & img) const { - const clip_image_size original_size = img.get_size(); +mtmd_image_preproc_out mtmd_image_preprocessor_llava_uhd::preprocess(const mtmd_image_u8 & img) const { + const mtmd_image_size original_size = img.get_size(); auto const inst = get_slice_instructions(original_size); auto sliced = slice_image(img, inst); @@ -530,7 +530,7 @@ mtmd_image_preproc_out mtmd_image_preprocessor_llava_uhd::preprocess(const clip_ return output; } -mtmd_image_preprocessor_llava_uhd::slice_instructions mtmd_image_preprocessor_llava_uhd::get_slice_instructions(const clip_image_size & original_size) const { +mtmd_image_preprocessor_llava_uhd::slice_instructions mtmd_image_preprocessor_llava_uhd::get_slice_instructions(const mtmd_image_size & original_size) const { mtmd_image_preprocessor_llava_uhd::slice_instructions res; const int patch_size = get_slice_align(); const int slice_size = hparams.image_size; @@ -542,9 +542,9 @@ mtmd_image_preprocessor_llava_uhd::slice_instructions mtmd_image_preprocessor_ll if (!has_slices) { // skip slicing logic - res.overview_size = clip_image_size{slice_size, slice_size}; - res.refined_size = clip_image_size{0, 0}; - res.grid_size = clip_image_size{0, 0}; + res.overview_size = mtmd_image_size{slice_size, slice_size}; + res.refined_size = mtmd_image_size{0, 0}; + res.grid_size = mtmd_image_size{0, 0}; return res; } @@ -554,9 +554,9 @@ mtmd_image_preprocessor_llava_uhd::slice_instructions mtmd_image_preprocessor_ll auto refine_size = select_best_resolution( original_size, hparams.image_res_candidates); - res.overview_size = clip_image_size{slice_size, slice_size}; + res.overview_size = mtmd_image_size{slice_size, slice_size}; res.refined_size = refine_size; - res.grid_size = clip_image_size{0, 0}; + res.grid_size = mtmd_image_size{0, 0}; LOG_DBG("%s: using pinpoints for slicing\n", __func__); LOG_DBG("%s: original size: %d x %d, overview size: %d x %d, refined size: %d x %d\n", @@ -633,7 +633,7 @@ mtmd_image_preprocessor_llava_uhd::slice_instructions mtmd_image_preprocessor_ll return res; } -mtmd_image_preprocessor_llava_uhd::slice_output mtmd_image_preprocessor_llava_uhd::slice_image(const clip_image_u8 & img, const mtmd_image_preprocessor_llava_uhd::slice_instructions & inst) const { +mtmd_image_preprocessor_llava_uhd::slice_output mtmd_image_preprocessor_llava_uhd::slice_image(const mtmd_image_u8 & img, const mtmd_image_preprocessor_llava_uhd::slice_instructions & inst) const { slice_output output; // resize to overview size @@ -646,7 +646,7 @@ mtmd_image_preprocessor_llava_uhd::slice_output mtmd_image_preprocessor_llava_uh } // resize to refined size - clip_image_u8 refined_img; + mtmd_image_u8 refined_img; img_tool::resize(img, refined_img, inst.refined_size, hparams.image_resize_algo_rf, hparams.image_pad_rf, hparams.image_pad_color_rf); @@ -657,7 +657,7 @@ mtmd_image_preprocessor_llava_uhd::slice_output mtmd_image_preprocessor_llava_uh int w = slice.size.width; int h = slice.size.height; - clip_image_u8 img_slice; + mtmd_image_u8 img_slice; img_tool::crop(refined_img, img_slice, x, y, w, h); output.slices.push_back(std::move(img_slice)); } @@ -665,7 +665,7 @@ mtmd_image_preprocessor_llava_uhd::slice_output mtmd_image_preprocessor_llava_uh return output; } -clip_image_size mtmd_image_preprocessor_llava_uhd::get_best_resize(const clip_image_size & original_size, int scale_resolution, int patch_size, bool allow_upscale) const { +mtmd_image_size mtmd_image_preprocessor_llava_uhd::get_best_resize(const mtmd_image_size & original_size, int scale_resolution, int patch_size, bool allow_upscale) const { int width = original_size.width; int height = original_size.height; if ((width * height > scale_resolution * scale_resolution) || allow_upscale) { @@ -673,28 +673,28 @@ clip_image_size mtmd_image_preprocessor_llava_uhd::get_best_resize(const clip_im height = static_cast(scale_resolution / std::sqrt(r)); width = static_cast(height * r); } - clip_image_size res; + mtmd_image_size res; res.width = ensure_divide(width, patch_size); res.height = ensure_divide(height, patch_size); return res; } -clip_image_size mtmd_image_preprocessor_llava_uhd::resize_maintain_aspect_ratio(const clip_image_size & orig, const clip_image_size & target_max) const { +mtmd_image_size mtmd_image_preprocessor_llava_uhd::resize_maintain_aspect_ratio(const mtmd_image_size & orig, const mtmd_image_size & target_max) const { float scale_width = static_cast(target_max.width) / orig.width; float scale_height = static_cast(target_max.height) / orig.height; float scale = std::min(scale_width, scale_height); - return clip_image_size{ + return mtmd_image_size{ static_cast(orig.width * scale), static_cast(orig.height * scale), }; } -clip_image_size mtmd_image_preprocessor_llava_uhd::select_best_resolution(const clip_image_size & original_size, const std::vector & possible_resolutions) const { - clip_image_size best_fit; +mtmd_image_size mtmd_image_preprocessor_llava_uhd::select_best_resolution(const mtmd_image_size & original_size, const std::vector & possible_resolutions) const { + mtmd_image_size best_fit; int min_wasted_area = std::numeric_limits::max(); int max_effective_resolution = 0; - for (const clip_image_size & candidate : possible_resolutions) { + for (const mtmd_image_size & candidate : possible_resolutions) { auto target_size = resize_maintain_aspect_ratio(original_size, candidate); int effective_resolution = std::min( target_size.width * target_size.height, @@ -717,7 +717,7 @@ int mtmd_image_preprocessor_llava_uhd::ensure_divide(int length, int patch_size) return std::max(align_round(static_cast(length) / patch_size) * patch_size, patch_size); } -clip_image_size mtmd_image_preprocessor_llava_uhd::get_refine_size(const clip_image_size & original_size, const clip_image_size & grid, int scale_resolution, int patch_size, bool allow_upscale) const { +mtmd_image_size mtmd_image_preprocessor_llava_uhd::get_refine_size(const mtmd_image_size & original_size, const mtmd_image_size & grid, int scale_resolution, int patch_size, bool allow_upscale) const { int width = original_size.width; int height = original_size.height; int grid_x = grid.width; @@ -726,7 +726,7 @@ clip_image_size mtmd_image_preprocessor_llava_uhd::get_refine_size(const clip_im int refine_width = ensure_divide(width, grid_x); int refine_height = ensure_divide(height, grid_y); - clip_image_size grid_size; + mtmd_image_size grid_size; grid_size.width = refine_width / grid_x; grid_size.height = refine_height / grid_y; @@ -734,13 +734,13 @@ clip_image_size mtmd_image_preprocessor_llava_uhd::get_refine_size(const clip_im int best_grid_width = best_grid_size.width; int best_grid_height = best_grid_size.height; - clip_image_size refine_size; + mtmd_image_size refine_size; refine_size.width = best_grid_width * grid_x; refine_size.height = best_grid_height * grid_y; return refine_size; } -clip_image_size mtmd_image_preprocessor_llava_uhd::get_best_grid(const int max_slice_nums, const int multiple, const float log_ratio) const { +mtmd_image_size mtmd_image_preprocessor_llava_uhd::get_best_grid(const int max_slice_nums, const int multiple, const float log_ratio) const { std::vector candidate_split_grids_nums; for (int i : {multiple - 1, multiple, multiple + 1}) { if (i == 1 || i > max_slice_nums) { @@ -749,18 +749,18 @@ clip_image_size mtmd_image_preprocessor_llava_uhd::get_best_grid(const int max_s candidate_split_grids_nums.push_back(i); } - std::vector candidate_grids; + std::vector candidate_grids; for (int split_grids_nums : candidate_split_grids_nums) { int m = 1; while (m <= split_grids_nums) { if (split_grids_nums % m == 0) { - candidate_grids.push_back(clip_image_size{m, split_grids_nums / m}); + candidate_grids.push_back(mtmd_image_size{m, split_grids_nums / m}); } ++m; } } - clip_image_size best_grid{1, 1}; + mtmd_image_size best_grid{1, 1}; float min_error = std::numeric_limits::infinity(); for (const auto& grid : candidate_grids) { float error = std::abs(log_ratio - std::log(1.0 * grid.width / grid.height)); @@ -776,8 +776,8 @@ clip_image_size mtmd_image_preprocessor_llava_uhd::get_best_grid(const int max_s // mtmd_image_preprocessor_fixed_size // -mtmd_image_preproc_out mtmd_image_preprocessor_fixed_size::preprocess(const clip_image_u8 & img) const { - clip_image_u8 resized_image; +mtmd_image_preproc_out mtmd_image_preprocessor_fixed_size::preprocess(const mtmd_image_u8 & img) const { + mtmd_image_u8 resized_image; int sz = hparams.image_size; img_tool::resize(img, resized_image, {sz, sz}, hparams.image_resize_algo, @@ -792,7 +792,7 @@ mtmd_image_preproc_out mtmd_image_preprocessor_fixed_size::preprocess(const clip // mtmd_image_preprocessor_dyn_size // -clip_image_size mtmd_image_preprocessor_dyn_size::get_target_size(const clip_hparams & hparams, const clip_image_size & original_size) { +mtmd_image_size mtmd_image_preprocessor_dyn_size::get_target_size(const mtmd_hparams & hparams, const mtmd_image_size & original_size) { GGML_ASSERT(hparams.image_min_pixels > 0 && hparams.image_max_pixels > 0); // the original pixtral model doesn't have n_merge const int cur_merge = hparams.n_merge; @@ -807,8 +807,8 @@ clip_image_size mtmd_image_preprocessor_dyn_size::get_target_size(const clip_hpa }); } -mtmd_image_preproc_out mtmd_image_preprocessor_dyn_size::preprocess(const clip_image_u8 & img) const { - clip_image_u8 resized_image; +mtmd_image_preproc_out mtmd_image_preprocessor_dyn_size::preprocess(const mtmd_image_u8 & img) const { + mtmd_image_u8 resized_image; img_tool::resize(img, resized_image, target_size(img.get_size()), hparams.image_resize_algo, hparams.image_resize_pad, @@ -819,12 +819,51 @@ mtmd_image_preproc_out mtmd_image_preprocessor_dyn_size::preprocess(const clip_i } // -// mtmd_image_preprocessor_lfm3vl +// mtmd_image_preprocessor_kimik25 +// + +mtmd_image_preproc_out mtmd_image_preprocessor_kimik25::preprocess(const mtmd_image_u8 & img) const { + GGML_ASSERT(hparams.image_max_pixels > 0); + // patch_limit_on_one_side from the preprocessor config, same for K2.5 and K3.1 + const int side_limit = 512; + const int ps = hparams.patch_size; + const int64_t in_patch_limit = hparams.image_max_pixels / (ps * ps); + const int width = img.get_size().width; + const int height = img.get_size().height; + + const double s1 = std::sqrt((double) in_patch_limit + / (std::max(1.0, (double) (width / ps)) * std::max(1.0, (double) (height / ps)))); + const double s2 = (double) side_limit * ps / width; + const double s3 = (double) side_limit * ps / height; + const double scale = std::min({1.0, s1, s2, s3}); + const int new_w = std::min(std::max(1, (int) (width * scale)), side_limit * ps); + const int new_h = std::min(std::max(1, (int) (height * scale)), side_limit * ps); + + mtmd_image_u8 content; + if (new_w == width && new_h == height) { + content = img; + } else { + img_tool::resize(img, content, mtmd_image_size{new_w, new_h}, hparams.image_resize_algo, PAD_NONE); + } + + const int align = ps * hparams.n_merge; + mtmd_image_u8 canvas; + canvas.set_size(mtmd_image_size{MTMD_ALIGN(new_w, align), MTMD_ALIGN(new_h, align)}, img.is_placeholder()); + img_tool::fill(canvas, {0, 0, 0}); + img_tool::composite(canvas, content, 0, 0); + + mtmd_image_preproc_out output; + output.append(hparams, canvas, true); + return output; +} + +// +// mtmd_image_preprocessor_qwen3vl // // rounds before the pixel checks (an edge may round to 0) and computes in double, as transformers does -clip_image_size mtmd_image_preprocessor_lfm3vl::smart_resize(int width, int height, int factor, int64_t min_pixels, int64_t max_pixels, - int n_frames, int temporal_factor) { +mtmd_image_size mtmd_image_preprocessor_qwen3vl::smart_resize(int width, int height, int factor, int64_t min_pixels, int64_t max_pixels, + int n_frames, int temporal_factor) { double h = height; double w = width; // the video variant first upscales edges below factor @@ -850,18 +889,13 @@ clip_image_size mtmd_image_preprocessor_lfm3vl::smart_resize(int width, int heig return {std::max(factor, w_bar), std::max(factor, h_bar)}; } -clip_image_size mtmd_image_preprocessor_lfm3vl::target_size(const clip_image_size & original_size) const { - return smart_resize(original_size.width, original_size.height, hparams.patch_size * hparams.n_merge, - hparams.image_min_pixels, hparams.image_max_pixels); -} - -bool mtmd_image_preprocessor_lfm3vl::get_video_frame_size(int n_frames, int width, int height, clip_image_size & out) const { - // Lfm3VlVideoProcessor size: shortest_edge 4096, longest_edge 25165824, temporal_patch_size 2 +bool mtmd_image_preprocessor_qwen3vl::get_video_frame_size(int n_frames, int width, int height, mtmd_image_size & out) const { + // Qwen3VLVideoProcessor (and Lfm3VlVideoProcessor) size: shortest_edge 4096, longest_edge 25165824, temporal_patch_size 2 out = smart_resize(width, height, hparams.patch_size * hparams.n_merge, 4096, 25165824, n_frames, 2); return true; } -mtmd_image_preproc_out mtmd_image_preprocessor_lfm3vl::preprocess_video_frame(const clip_image_u8 & img) const { +mtmd_image_preproc_out mtmd_image_preprocessor_qwen3vl::preprocess_video_frame(const mtmd_image_u8 & img) const { const int f = hparams.patch_size * hparams.n_merge; const auto size = img.get_size(); return resize_to(img, { @@ -870,8 +904,8 @@ mtmd_image_preproc_out mtmd_image_preprocessor_lfm3vl::preprocess_video_frame(co }); } -mtmd_image_preproc_out mtmd_image_preprocessor_lfm3vl::resize_to(const clip_image_u8 & img, const clip_image_size & size) const { - clip_image_u8 resized_image; +mtmd_image_preproc_out mtmd_image_preprocessor_qwen3vl::resize_to(const mtmd_image_u8 & img, const mtmd_image_size & size) const { + mtmd_image_u8 resized_image; img_tool::resize(img, resized_image, size, hparams.image_resize_algo, hparams.image_resize_pad, @@ -881,13 +915,22 @@ mtmd_image_preproc_out mtmd_image_preprocessor_lfm3vl::resize_to(const clip_imag return output; } +// +// mtmd_image_preprocessor_lfm3vl +// + +mtmd_image_size mtmd_image_preprocessor_lfm3vl::target_size(const mtmd_image_size & original_size) const { + return smart_resize(original_size.width, original_size.height, hparams.patch_size * hparams.n_merge, + hparams.image_min_pixels, hparams.image_max_pixels); +} + // // mtmd_image_preprocessor_glm5v // // The canvas is ceil-aligned to patch_size*n_merge and fitted to the token budget. // Only rescaled to meet the budget and sits top-left, with black padding on the right and bottom -mtmd_image_preproc_out mtmd_image_preprocessor_glm5v::preprocess(const clip_image_u8 & img) const { +mtmd_image_preproc_out mtmd_image_preprocessor_glm5v::preprocess(const mtmd_image_u8 & img) const { GGML_ASSERT(hparams.image_min_pixels > 0 && hparams.image_max_pixels > 0); const int64_t factor = hparams.patch_size * hparams.n_merge; @@ -937,11 +980,11 @@ mtmd_image_preproc_out mtmd_image_preprocessor_glm5v::preprocess(const clip_imag const int content_h = (int) std::max(1, std::min(canvas_h, (int64_t) std::floor(height * scale))); const int content_w = (int) std::max(1, std::min(canvas_w, (int64_t) std::floor(width * scale))); - clip_image_u8 content; - img_tool::resize(img, content, clip_image_size{content_w, content_h}, hparams.image_resize_algo, PAD_NONE); + mtmd_image_u8 content; + img_tool::resize(img, content, mtmd_image_size{content_w, content_h}, hparams.image_resize_algo, PAD_NONE); - clip_image_u8 canvas; - canvas.set_size(clip_image_size{(int) canvas_w, (int) canvas_h}, img.is_placeholder()); + mtmd_image_u8 canvas; + canvas.set_size(mtmd_image_size{(int) canvas_w, (int) canvas_h}, img.is_placeholder()); img_tool::fill(canvas, {0, 0, 0}); img_tool::composite(canvas, content, 0, 0); @@ -954,9 +997,9 @@ mtmd_image_preproc_out mtmd_image_preprocessor_glm5v::preprocess(const clip_imag // mtmd_image_preprocessor_longest_edge // -clip_image_size mtmd_image_preprocessor_longest_edge::get_target_size(const clip_hparams & hparams, const clip_image_size & original_size) { +mtmd_image_size mtmd_image_preprocessor_longest_edge::get_target_size(const mtmd_hparams & hparams, const mtmd_image_size & original_size) { GGML_ASSERT(hparams.image_longest_edge > 0); - clip_image_size size = original_size; + mtmd_image_size size = original_size; if (hparams.image_floor_before_align) { const double ratio = double(std::max(size.width, size.height)) / hparams.image_longest_edge; if (ratio > 1.0) { @@ -981,9 +1024,9 @@ clip_image_size mtmd_image_preprocessor_longest_edge::get_target_size(const clip }); } -mtmd_image_preproc_out mtmd_image_preprocessor_longest_edge::preprocess(const clip_image_u8 & img) const { - clip_image_u8 resized_image; - const clip_image_size target_size = get_target_size(hparams, img.get_size()); +mtmd_image_preproc_out mtmd_image_preprocessor_longest_edge::preprocess(const mtmd_image_u8 & img) const { + mtmd_image_u8 resized_image; + const mtmd_image_size target_size = get_target_size(hparams, img.get_size()); img_tool::resize(img, resized_image, target_size, hparams.image_resize_algo, hparams.image_resize_pad, @@ -997,15 +1040,15 @@ mtmd_image_preproc_out mtmd_image_preprocessor_longest_edge::preprocess(const cl // mtmd_image_preprocessor_minicpmv // -mtmd_image_preprocessor_llava_uhd::slice_instructions mtmd_image_preprocessor_minicpmv::get_slice_instructions(const clip_image_size & original_size) const { +mtmd_image_preprocessor_llava_uhd::slice_instructions mtmd_image_preprocessor_minicpmv::get_slice_instructions(const mtmd_image_size & original_size) const { // overview only for small images, unlike generic llava-uhd which slices once one side exceeds scale resolution const int slice_size = hparams.image_size; const float ratio = (float) original_size.width * original_size.height / (slice_size * slice_size); if (ratio <= 1.0f) { mtmd_image_preprocessor_llava_uhd::slice_instructions inst; inst.overview_size = get_best_resize(original_size, slice_size, get_slice_align(), true); - inst.refined_size = clip_image_size{0, 0}; - inst.grid_size = clip_image_size{0, 0}; + inst.refined_size = mtmd_image_size{0, 0}; + inst.grid_size = mtmd_image_size{0, 0}; return inst; } return mtmd_image_preprocessor_llava_uhd::get_slice_instructions(original_size); @@ -1015,7 +1058,7 @@ mtmd_image_preprocessor_llava_uhd::slice_instructions mtmd_image_preprocessor_mi // mtmd_image_preprocessor_lfm2 // -mtmd_image_preproc_out mtmd_image_preprocessor_lfm2::preprocess(const clip_image_u8 & img) const { +mtmd_image_preproc_out mtmd_image_preprocessor_lfm2::preprocess(const mtmd_image_u8 & img) const { auto const inst = get_slice_instructions(img.get_size()); if (!inst.slices.empty()) { return mtmd_image_preprocessor_llava_uhd::preprocess(img); @@ -1030,8 +1073,8 @@ mtmd_image_preproc_out mtmd_image_preprocessor_lfm2::preprocess(const clip_image } bool mtmd_image_preprocessor_lfm2::should_tile( - const clip_hparams & hparams, - const clip_image_size & original_size) { + const mtmd_hparams & hparams, + const mtmd_image_size & original_size) { const int align_size = hparams.patch_size * hparams.n_merge; const auto round_by_factor = [align_size](float x) { @@ -1048,9 +1091,9 @@ bool mtmd_image_preprocessor_lfm2::should_tile( } // same as Lfm2VlImageProcessor.smart_resize: Python's round half to even, double precision -clip_image_size mtmd_image_preprocessor_lfm2::smart_resize( - const clip_hparams & hparams, - const clip_image_size & original_size) { +mtmd_image_size mtmd_image_preprocessor_lfm2::smart_resize( + const mtmd_hparams & hparams, + const mtmd_image_size & original_size) { const int align_size = hparams.patch_size * hparams.n_merge; const double height = original_size.height; const double width = original_size.width; @@ -1071,7 +1114,7 @@ clip_image_size mtmd_image_preprocessor_lfm2::smart_resize( return {w_bar, h_bar}; } -mtmd_image_preprocessor_llava_uhd::slice_instructions mtmd_image_preprocessor_lfm2::get_slice_instructions(const clip_image_size & original_size) const { +mtmd_image_preprocessor_llava_uhd::slice_instructions mtmd_image_preprocessor_lfm2::get_slice_instructions(const mtmd_image_size & original_size) const { mtmd_image_preprocessor_llava_uhd::slice_instructions inst; inst.overview_size = smart_resize(hparams, original_size); @@ -1079,15 +1122,15 @@ mtmd_image_preprocessor_llava_uhd::slice_instructions mtmd_image_preprocessor_lf const int tile_size = hparams.preproc_tile_size; if (!needs_tiling) { - inst.refined_size = clip_image_size{0, 0}; - inst.grid_size = clip_image_size{0, 0}; + inst.refined_size = mtmd_image_size{0, 0}; + inst.grid_size = mtmd_image_size{0, 0}; return inst; } - const clip_image_size grid = get_grid_layout(hparams, original_size.height, original_size.width); + const mtmd_image_size grid = get_grid_layout(hparams, original_size.height, original_size.width); inst.grid_size = grid; - inst.refined_size = clip_image_size{tile_size * grid.width, tile_size * grid.height}; + inst.refined_size = mtmd_image_size{tile_size * grid.width, tile_size * grid.height}; LOG_DBG("%s: original size: %d x %d, overview size: %d x %d, refined size: %d x %d, grid size: %d x %d\n", __func__, @@ -1101,7 +1144,7 @@ mtmd_image_preprocessor_llava_uhd::slice_instructions mtmd_image_preprocessor_lf mtmd_image_preprocessor_llava_uhd::slice_coordinates slice; slice.x = col * tile_size; slice.y = row * tile_size; - slice.size = clip_image_size{tile_size, tile_size}; + slice.size = mtmd_image_size{tile_size, tile_size}; inst.slices.push_back(slice); LOG_DBG("%s: slice %d: x=%d, y=%d, size=%d x %d\n", __func__, (int)inst.slices.size() - 1, @@ -1113,13 +1156,13 @@ mtmd_image_preprocessor_llava_uhd::slice_instructions mtmd_image_preprocessor_lf } // double precision as in Python, float breaks exact ties such as 720/1600 between 2/4 and 2/5 -clip_image_size mtmd_image_preprocessor_lfm2::find_closest_aspect_ratio( - const clip_hparams & hparams, +mtmd_image_size mtmd_image_preprocessor_lfm2::find_closest_aspect_ratio( + const mtmd_hparams & hparams, double aspect_ratio, - const std::vector & target_ratios, + const std::vector & target_ratios, int width, int height) { double best_ratio_diff = std::numeric_limits::max(); - clip_image_size best_ratio = {1, 1}; + mtmd_image_size best_ratio = {1, 1}; const double area = static_cast(width) * height; for (const auto & ratio : target_ratios) { @@ -1139,10 +1182,10 @@ clip_image_size mtmd_image_preprocessor_lfm2::find_closest_aspect_ratio( return best_ratio; } -std::vector mtmd_image_preprocessor_lfm2::get_target_ratios(const clip_hparams & hparams) { +std::vector mtmd_image_preprocessor_lfm2::get_target_ratios(const mtmd_hparams & hparams) { const int min_tiles = hparams.preproc_min_tiles; const int max_tiles = hparams.preproc_max_tiles; - std::vector ratios; + std::vector ratios; for (int n = min_tiles; n <= max_tiles; n++) { for (int w = 1; w <= n; w++) { for (int h = 1; h <= n; h++) { @@ -1161,13 +1204,13 @@ std::vector mtmd_image_preprocessor_lfm2::get_target_ratios(con } } } - std::sort(ratios.begin(), ratios.end(), [](const clip_image_size & a, const clip_image_size & b) { + std::sort(ratios.begin(), ratios.end(), [](const mtmd_image_size & a, const mtmd_image_size & b) { return a.width * a.height < b.width * b.height; }); return ratios; } -clip_image_size mtmd_image_preprocessor_lfm2::get_grid_layout(const clip_hparams & hparams, int height, int width) { +mtmd_image_size mtmd_image_preprocessor_lfm2::get_grid_layout(const mtmd_hparams & hparams, int height, int width) { const double aspect_ratio = static_cast(width) / height; const auto ratios = get_target_ratios(hparams); return find_closest_aspect_ratio(hparams, aspect_ratio, ratios, width, height); @@ -1177,7 +1220,7 @@ clip_image_size mtmd_image_preprocessor_lfm2::get_grid_layout(const clip_hparams // mtmd_image_preprocessor_idefics3 // -mtmd_image_preproc_out mtmd_image_preprocessor_idefics3::preprocess(const clip_image_u8 & img) const { +mtmd_image_preproc_out mtmd_image_preprocessor_idefics3::preprocess(const mtmd_image_u8 & img) const { // The refined size has two steps: // 1. Resize w/ aspect-ratio preserving such that the longer side is // the preprocessor longest size @@ -1185,7 +1228,7 @@ mtmd_image_preproc_out mtmd_image_preprocessor_idefics3::preprocess(const clip_i // multiples of image_size (always rounding up) // // CITE: https://github.com/huggingface/transformers/blob/main/src/transformers/models/idefics3/image_processing_idefics3.py#L737 - const clip_image_size original_size = img.get_size(); + const mtmd_image_size original_size = img.get_size(); // old gguf files have no preprocessor longest size, custom token limits also need the generic size below if (hparams.image_longest_edge > 0 && hparams.image_min_pixels <= 0 && hparams.image_max_pixels <= 0) { @@ -1193,7 +1236,7 @@ mtmd_image_preproc_out mtmd_image_preprocessor_idefics3::preprocess(const clip_i const int longest_edge = hparams.image_longest_edge; const double aspect_ratio = (double) original_size.width / original_size.height; - clip_image_size resized_size; + mtmd_image_size resized_size; if (original_size.width >= original_size.height) { resized_size.width = longest_edge; resized_size.height = (int) (longest_edge / aspect_ratio); @@ -1206,21 +1249,21 @@ mtmd_image_preproc_out mtmd_image_preprocessor_idefics3::preprocess(const clip_i const int grid_x = (resized_size.width + tile_size - 1) / tile_size; const int grid_y = (resized_size.height + tile_size - 1) / tile_size; - const clip_image_size refined_size = clip_image_size{grid_x * tile_size, grid_y * tile_size}; + const mtmd_image_size refined_size = mtmd_image_size{grid_x * tile_size, grid_y * tile_size}; - clip_image_u8 resized_img; + mtmd_image_u8 resized_img; img_tool::resize(img, resized_img, resized_size, hparams.image_resize_algo, PAD_NONE); - clip_image_u8 refined_img; + mtmd_image_u8 refined_img; img_tool::resize(resized_img, refined_img, refined_size, hparams.image_resize_algo, PAD_NONE); - clip_image_u8 overview; + mtmd_image_u8 overview; img_tool::resize(refined_img, overview, {tile_size, tile_size}, hparams.image_resize_algo, PAD_NONE); - std::vector slices; + std::vector slices; for (int y = 0; y < grid_y; y++) { for (int x = 0; x < grid_x; x++) { - clip_image_u8 slice; + mtmd_image_u8 slice; img_tool::crop(refined_img, slice, x * tile_size, y * tile_size, tile_size, tile_size); slices.push_back(std::move(slice)); } @@ -1236,7 +1279,7 @@ mtmd_image_preproc_out mtmd_image_preprocessor_idefics3::preprocess(const clip_i return output; } - const clip_image_size refined_size = img_tool::calc_size_preserved_ratio( + const mtmd_image_size refined_size = img_tool::calc_size_preserved_ratio( original_size, { hparams.image_size, std::max(0, hparams.image_min_pixels), std::max(0, hparams.image_max_pixels), hparams.image_longest_edge }); // LOG_INF("%s: original size: %d x %d, refined size: %d x %d\n", @@ -1244,9 +1287,9 @@ mtmd_image_preproc_out mtmd_image_preprocessor_idefics3::preprocess(const clip_i // refined_size.width, refined_size.height); mtmd_image_preprocessor_llava_uhd::slice_instructions instructions; - instructions.overview_size = clip_image_size{hparams.image_size, hparams.image_size}; + instructions.overview_size = mtmd_image_size{hparams.image_size, hparams.image_size}; instructions.refined_size = refined_size; - instructions.grid_size = clip_image_size{ + instructions.grid_size = mtmd_image_size{ static_cast(std::ceil(static_cast(refined_size.width) / hparams.image_size)), static_cast(std::ceil(static_cast(refined_size.height) / hparams.image_size)), }; @@ -1256,7 +1299,7 @@ mtmd_image_preproc_out mtmd_image_preprocessor_idefics3::preprocess(const clip_i instructions.slices.push_back(mtmd_image_preprocessor_llava_uhd::slice_coordinates{ /* x */x, /* y */y, - /* size */clip_image_size{ + /* size */mtmd_image_size{ std::min(hparams.image_size, refined_size.width - x), std::min(hparams.image_size, refined_size.height - y) } @@ -1277,7 +1320,7 @@ mtmd_image_preproc_out mtmd_image_preprocessor_idefics3::preprocess(const clip_i // mtmd_image_preprocessor_cohere2v // -mtmd_image_preproc_out mtmd_image_preprocessor_cohere2v::preprocess(const clip_image_u8 & img) const { +mtmd_image_preproc_out mtmd_image_preprocessor_cohere2v::preprocess(const mtmd_image_u8 & img) const { const auto inst = get_slice_instructions(img.get_size()); auto sliced = slice_image(img, inst); @@ -1294,21 +1337,21 @@ mtmd_image_preproc_out mtmd_image_preprocessor_cohere2v::preprocess(const clip_i return output; } -mtmd_image_preprocessor_llava_uhd::slice_instructions mtmd_image_preprocessor_cohere2v::get_slice_instructions(const clip_image_size & original_size) const { +mtmd_image_preprocessor_llava_uhd::slice_instructions mtmd_image_preprocessor_cohere2v::get_slice_instructions(const mtmd_image_size & original_size) const { const int tile = hparams.image_size; // pick the grid with the least upscale; if all grids need downscale, pick the one with the least downscale // grids are visited by tile count, then by width, same order as HF for ties double best_down = -1.0; double best_up = std::numeric_limits::max(); - clip_image_size grid_down = { 1, 1 }; - clip_image_size grid_up = { 0, 0 }; + mtmd_image_size grid_down = { 1, 1 }; + mtmd_image_size grid_up = { 0, 0 }; for (int n = 1; n <= hparams.preproc_max_tiles; n++) { for (int w = 1; w <= n; w++) { if (n % w != 0) { continue; } - const clip_image_size g = { w, n / w }; + const mtmd_image_size g = { w, n / w }; const double scale = std::min( (double) (g.width * tile) / original_size.width, (double) (g.height * tile) / original_size.height); @@ -1323,7 +1366,7 @@ mtmd_image_preprocessor_llava_uhd::slice_instructions mtmd_image_preprocessor_co } } } - const clip_image_size grid = grid_up.width > 0 ? grid_up : grid_down; + const mtmd_image_size grid = grid_up.width > 0 ? grid_up : grid_down; slice_instructions inst; inst.overview_size = { tile, tile }; @@ -1343,9 +1386,9 @@ mtmd_image_preprocessor_llava_uhd::slice_instructions mtmd_image_preprocessor_co // mtmd_image_preprocessor_internvl // -mtmd_image_preproc_out mtmd_image_preprocessor_internvl::preprocess(const clip_image_u8 & img) const { +mtmd_image_preproc_out mtmd_image_preprocessor_internvl::preprocess(const mtmd_image_u8 & img) const { GGML_ASSERT(!hparams.image_res_candidates.empty()); - const clip_image_size original_size = img.get_size(); + const mtmd_image_size original_size = img.get_size(); auto const inst = get_slice_instructions(original_size); auto sliced = slice_image(img, inst); @@ -1362,8 +1405,8 @@ mtmd_image_preproc_out mtmd_image_preprocessor_internvl::preprocess(const clip_i // mtmd_image_preprocessor_deepseekocr // -std::vector mtmd_image_preprocessor_deepseekocr::get_target_ratios() const { - std::vector ratios; +std::vector mtmd_image_preprocessor_deepseekocr::get_target_ratios() const { + std::vector ratios; for (int n = min_tiles; n <= max_tiles; n++) { for (int w = 1; w <= n; w++) { for (int h = 1; h <= n; h++) { @@ -1383,19 +1426,19 @@ std::vector mtmd_image_preprocessor_deepseekocr::get_target_rat } } } - std::sort(ratios.begin(), ratios.end(), [](const clip_image_size & a, const clip_image_size & b) { + std::sort(ratios.begin(), ratios.end(), [](const mtmd_image_size & a, const mtmd_image_size & b) { return a.width * a.height < b.width * b.height; }); return ratios; } -clip_image_size mtmd_image_preprocessor_deepseekocr::find_closest_aspect_ratio( +mtmd_image_size mtmd_image_preprocessor_deepseekocr::find_closest_aspect_ratio( float aspect_ratio, - const std::vector & target_ratios, + const std::vector & target_ratios, int width, int height) const { float best_ratio_diff = std::numeric_limits::max(); - clip_image_size best_ratio = { 1, 1 }; + mtmd_image_size best_ratio = { 1, 1 }; const float area = static_cast(width * height); for (const auto & ratio : target_ratios) { @@ -1478,7 +1521,7 @@ void mtmd_image_preprocessor_deepseek4v::safe_resize(int height, int width, int } // ref: load_image() -mtmd_image_preproc_out mtmd_image_preprocessor_deepseek4v::preprocess(const clip_image_u8 & img) const { +mtmd_image_preproc_out mtmd_image_preprocessor_deepseek4v::preprocess(const mtmd_image_u8 & img) const { mtmd_image_preproc_out out; const int p = hparams.patch_size; @@ -1486,7 +1529,7 @@ mtmd_image_preproc_out mtmd_image_preprocessor_deepseek4v::preprocess(const clip const int max_n_token = hparams.dsv4_max_n_token; const int max_wh = hparams.dsv4_max_wh_ratio; - const clip_image_size orig = img.get_size(); + const mtmd_image_size orig = img.get_size(); int width = orig.width; int height = orig.height; if (max_wh > 0 && width > height * max_wh) { @@ -1498,11 +1541,11 @@ mtmd_image_preproc_out mtmd_image_preprocessor_deepseek4v::preprocess(const clip width = (int) (width * up); height = (int) (height * up); } - int best_width = CLIP_ALIGN(width, p); - int best_height = CLIP_ALIGN(height, p); + int best_width = MTMD_ALIGN(width, p); + int best_height = MTMD_ALIGN(height, p); safe_resize(height, width, best_height, best_width, p, r, max_n_token); - clip_image_u8 resized; + mtmd_image_u8 resized; if (max_wh > 0 && orig.width >= max_wh * orig.height) { // extreme aspect ratio: plain stretch resize, no padding img_tool::resize(img, resized, {best_width, best_height}, hparams.image_resize_algo, PAD_NONE); @@ -1536,8 +1579,8 @@ void mtmd_image_preprocessor_deepseek41v::plan_grid(int width, int height, int p width = (int) (width * ratio); height = (int) (height * ratio); } - best_width = CLIP_ALIGN(width, p); - best_height = CLIP_ALIGN(height, p); + best_width = MTMD_ALIGN(width, p); + best_height = MTMD_ALIGN(height, p); if (n_tokens(best_height, best_width) > max_n_token) { // solve_resize_ratio: largest aspect-preserving size whose token block fits const double ratio = (double) height / width; @@ -1559,11 +1602,11 @@ void mtmd_image_preprocessor_deepseek41v::plan_grid(int width, int height, int p } } -mtmd_image_preproc_out mtmd_image_preprocessor_deepseek41v::preprocess(const clip_image_u8 & img) const { +mtmd_image_preproc_out mtmd_image_preprocessor_deepseek41v::preprocess(const mtmd_image_u8 & img) const { mtmd_image_preproc_out out; const int p = hparams.patch_size; - const clip_image_size orig = img.get_size(); + const mtmd_image_size orig = img.get_size(); int best_width = 0, best_height = 0; plan_grid(orig.width, orig.height, p, hparams.n_merge, hparams.dsv4_max_n_token, hparams.image_min_pixels, best_width, best_height); @@ -1581,10 +1624,10 @@ mtmd_image_preproc_out mtmd_image_preprocessor_deepseek41v::preprocess(const cli new_w = round_half_even((double) orig.width / orig.height * best_height); } } - clip_image_u8 resized; + mtmd_image_u8 resized; img_tool::resize(img, resized, {new_w, new_h}, hparams.image_resize_algo, PAD_NONE); - clip_image_u8 padded; + mtmd_image_u8 padded; if (new_w == best_width && new_h == best_height) { padded = std::move(resized); } else { @@ -1610,14 +1653,14 @@ mtmd_image_preproc_out mtmd_image_preprocessor_deepseek41v::preprocess(const cli return out; } -mtmd_image_preproc_out mtmd_image_preprocessor_deepseekocr::preprocess(const clip_image_u8 & img) const { +mtmd_image_preproc_out mtmd_image_preprocessor_deepseekocr::preprocess(const mtmd_image_u8 & img) const { mtmd_image_preproc_out output; int grid_w = 0; int grid_h = 0; const auto img_size = img.get_size(); // global view: aspect-preserving fit-and-pad to base_size - clip_image_u8 padded; + mtmd_image_u8 padded; img_tool::resize(img, padded, { base_size, base_size }, RESIZE_ALGO_BICUBIC, @@ -1630,12 +1673,12 @@ mtmd_image_preproc_out mtmd_image_preprocessor_deepseekocr::preprocess(const cli if (img_size.width > tile_size || img_size.height > tile_size) { const float aspect_ratio = static_cast(img_size.width) / img_size.height; const auto target_ratios = get_target_ratios(); - const clip_image_size grid = + const mtmd_image_size grid = find_closest_aspect_ratio(aspect_ratio, target_ratios, img_size.width, img_size.height); grid_w = grid.width; grid_h = grid.height; - clip_image_u8 refined; + mtmd_image_u8 refined; img_tool::resize(img, refined, { tile_size * grid_w, tile_size * grid_h }, RESIZE_ALGO_BICUBIC, PAD_NONE); @@ -1644,7 +1687,7 @@ mtmd_image_preproc_out mtmd_image_preprocessor_deepseekocr::preprocess(const cli // concat all tiles in this row into a single image, along the H axis // output image size: w = tile_size, h = tile_size * grid_w // this is to ensure the whole row is always processed together - clip_image_u8 row_img; + mtmd_image_u8 row_img; row_img.set_size({tile_size, tile_size * grid_w}, false); for (int col = 0; col < grid_w; col++) { for (int py = 0; py < tile_size; py++) { @@ -1657,7 +1700,7 @@ mtmd_image_preproc_out mtmd_image_preprocessor_deepseekocr::preprocess(const cli output.append(hparams, row_img, true); } else { for (int col = 0; col < grid_w; col++) { - clip_image_u8 tile; + mtmd_image_u8 tile; img_tool::crop(refined, tile, col * tile_size, row * tile_size, tile_size, tile_size); output.append(hparams, tile, true); } @@ -1681,8 +1724,8 @@ mtmd_image_preproc_out mtmd_image_preprocessor_deepseekocr::preprocess(const cli // void mtmd_image_preprocessor_step3vl::img_u8_resize_bilinear_to_f32( - const clip_image_u8 & src, - clip_image_f32 & dst, + const mtmd_image_u8 & src, + mtmd_image_f32 & dst, int target_width, int target_height, const float mean[3], @@ -1741,11 +1784,11 @@ void mtmd_image_preprocessor_step3vl::img_u8_resize_bilinear_to_f32( dst.cpy_buf(local_buf); } -int mtmd_image_preprocessor_step3vl::get_image_longest_edge(const clip_hparams & params) { +int mtmd_image_preprocessor_step3vl::get_image_longest_edge(const mtmd_hparams & params) { return params.image_longest_edge > 0 ? params.image_longest_edge : default_image_longest_edge; } -int mtmd_image_preprocessor_step3vl::determine_window_size(const clip_hparams & params, int longer, int shorter) { +int mtmd_image_preprocessor_step3vl::determine_window_size(const mtmd_hparams & params, int longer, int shorter) { const int image_size = params.image_size; const int crop_size = default_image_crop_size; const float aspect_ratio = static_cast(longer) / shorter; @@ -1787,15 +1830,15 @@ std::vector mtmd_image_preprocessor_step3vl::calc_grid(int length, int wind return starts; } -clip_image_u8 mtmd_image_preprocessor_step3vl::prepare_image(const clip_image_u8 & img, const clip_hparams & params) { - clip_image_u8 resized = img; +mtmd_image_u8 mtmd_image_preprocessor_step3vl::prepare_image(const mtmd_image_u8 & img, const mtmd_hparams & params) { + mtmd_image_u8 resized = img; const auto img_size = img.get_size(); const float aspect_ratio = img_size.height > 0 ? static_cast(img_size.width) / img_size.height : 1.0f; if (std::min(img_size.width, img_size.height) < 32 && (aspect_ratio > wide_aspect_ratio_limit || aspect_ratio < 1.0f / wide_aspect_ratio_limit)) { const int square_size = std::max(img_size.width, img_size.height); - clip_image_u8 padded; + mtmd_image_u8 padded; padded.set_size({square_size, square_size}, false); img_tool::fill(padded, {0, 0, 0}); img_tool::composite(padded, img, 0, 0); @@ -1806,11 +1849,11 @@ clip_image_u8 mtmd_image_preprocessor_step3vl::prepare_image(const clip_image_u8 const auto resized_size = resized.get_size(); if (std::max(resized_size.width, resized_size.height) > max_image_size) { const float scale = static_cast(max_image_size) / std::max(resized_size.width, resized_size.height); - const clip_image_size new_size = { + const mtmd_image_size new_size = { std::max(1, static_cast(std::floor(resized_size.width * scale))), std::max(1, static_cast(std::floor(resized_size.height * scale))), }; - clip_image_u8 scaled; + mtmd_image_u8 scaled; img_tool::resize(resized, scaled, new_size, RESIZE_ALGO_BILINEAR, PAD_NONE); resized = std::move(scaled); } @@ -1818,8 +1861,8 @@ clip_image_u8 mtmd_image_preprocessor_step3vl::prepare_image(const clip_image_u8 return resized; } -clip_image_u8 mtmd_image_preprocessor_step3vl::crop_with_black_padding(const clip_image_u8 & image, int x, int y, int w, int h) { - clip_image_u8 dst; +mtmd_image_u8 mtmd_image_preprocessor_step3vl::crop_with_black_padding(const mtmd_image_u8 & image, int x, int y, int w, int h) { + mtmd_image_u8 dst; dst.set_size({w, h}, false); img_tool::fill(dst, {0, 0, 0}); @@ -1846,8 +1889,8 @@ clip_image_u8 mtmd_image_preprocessor_step3vl::crop_with_black_padding(const cli } mtmd_image_preprocessor_step3vl::slice_instructions mtmd_image_preprocessor_step3vl::build_slice_instructions( - const clip_hparams & params, - const clip_image_size & prepared_size) { + const mtmd_hparams & params, + const mtmd_image_size & prepared_size) { slice_instructions instructions; instructions.overview_size = prepared_size; @@ -1856,18 +1899,18 @@ mtmd_image_preprocessor_step3vl::slice_instructions mtmd_image_preprocessor_step std::max(prepared_size.width, prepared_size.height), std::min(prepared_size.width, prepared_size.height)); if (window_size <= 0) { - instructions.refined_size = clip_image_size{0, 0}; - instructions.grid_size = clip_image_size{0, 0}; + instructions.refined_size = mtmd_image_size{0, 0}; + instructions.grid_size = mtmd_image_size{0, 0}; return instructions; } const int crop_width = calc_crop_extent(prepared_size.width, window_size); const int crop_height = calc_crop_extent(prepared_size.height, window_size); - instructions.refined_size = clip_image_size{crop_width, crop_height}; + instructions.refined_size = mtmd_image_size{crop_width, crop_height}; const auto xs = calc_grid(crop_width, window_size); const auto ys = calc_grid(crop_height, window_size); - instructions.grid_size = clip_image_size{ + instructions.grid_size = mtmd_image_size{ static_cast(xs.size()), static_cast(ys.size()), }; @@ -1877,7 +1920,7 @@ mtmd_image_preprocessor_step3vl::slice_instructions mtmd_image_preprocessor_step instructions.slices.push_back(slice_coordinates{ /* x */ x, /* y */ y, - /* size */ clip_image_size{window_size, window_size}, + /* size */ mtmd_image_size{window_size, window_size}, }); } } @@ -1885,8 +1928,8 @@ mtmd_image_preprocessor_step3vl::slice_instructions mtmd_image_preprocessor_step return instructions; } -mtmd_image_preproc_out mtmd_image_preprocessor_step3vl::preprocess(const clip_image_u8 & img) const { - clip_image_u8 prepared = prepare_image(img, hparams); +mtmd_image_preproc_out mtmd_image_preprocessor_step3vl::preprocess(const mtmd_image_u8 & img) const { + mtmd_image_u8 prepared = prepare_image(img, hparams); const auto instructions = build_slice_instructions(hparams, prepared.get_size()); mtmd_image_preproc_out output; @@ -1905,10 +1948,10 @@ mtmd_image_preproc_out mtmd_image_preprocessor_step3vl::preprocess(const clip_im return output; } - clip_image_u8 img_for_crop = prepared; + mtmd_image_u8 img_for_crop = prepared; const auto prepared_size = prepared.get_size(); if (instructions.refined_size.width != prepared_size.width || instructions.refined_size.height != prepared_size.height) { - clip_image_u8 refined; + mtmd_image_u8 refined; img_tool::resize(prepared, refined, instructions.refined_size, RESIZE_ALGO_BILINEAR, PAD_NONE); img_for_crop = std::move(refined); } @@ -1916,9 +1959,9 @@ mtmd_image_preproc_out mtmd_image_preprocessor_step3vl::preprocess(const clip_im const int crop_size = default_image_crop_size; for (const auto & slice : instructions.slices) { // If the requested patch extends past the source image, pad the out-of-bounds area with black. - clip_image_u8 patch = crop_with_black_padding(img_for_crop, slice.x, slice.y, slice.size.width, slice.size.height); + mtmd_image_u8 patch = crop_with_black_padding(img_for_crop, slice.x, slice.y, slice.size.width, slice.size.height); - clip_image_f32 patch_f32; + mtmd_image_f32 patch_f32; img_u8_resize_bilinear_to_f32( patch, patch_f32, @@ -1939,7 +1982,7 @@ mtmd_image_preproc_out mtmd_image_preprocessor_step3vl::preprocess(const clip_im // mtmd_image_preprocessor_youtuvl // -mtmd_image_preproc_out mtmd_image_preprocessor_youtuvl::preprocess(const clip_image_u8 & img) const { +mtmd_image_preproc_out mtmd_image_preprocessor_youtuvl::preprocess(const mtmd_image_u8 & img) const { const int patch_size = hparams.patch_size; // typically 16 const int merge_size = hparams.n_merge; // typically 2 const int align_size = patch_size * merge_size; // 32 @@ -1977,10 +2020,10 @@ mtmd_image_preproc_out mtmd_image_preprocessor_youtuvl::preprocess(const clip_im } } - clip_image_size new_size = {target_width, target_height}; + mtmd_image_size new_size = {target_width, target_height}; // Resize the image - clip_image_u8 resized; + mtmd_image_u8 resized; img_tool::resize(img, resized, new_size, hparams.image_resize_algo, hparams.image_resize_pad); mtmd_image_preproc_out output; @@ -1988,33 +2031,33 @@ mtmd_image_preproc_out mtmd_image_preprocessor_youtuvl::preprocess(const clip_im return output; } -mtmd_image_preproc_out mtmd_image_preprocessor_granite::preprocess(const clip_image_u8 & img) const { +mtmd_image_preproc_out mtmd_image_preprocessor_granite::preprocess(const mtmd_image_u8 & img) const { GGML_ASSERT(!hparams.image_res_candidates.empty()); - const clip_image_size orig_size = img.get_size(); + const mtmd_image_size orig_size = img.get_size(); const int tile_size = hparams.image_size; GGML_ASSERT(tile_size > 0); // llava-next always encodes an overview plus a grid of tiles, even for small images - const clip_image_size refined_size = select_best_resolution(orig_size, hparams.image_res_candidates); + const mtmd_image_size refined_size = select_best_resolution(orig_size, hparams.image_res_candidates); const int grid_x = refined_size.width / tile_size; const int grid_y = refined_size.height / tile_size; // the tiles are stacked on the Y axis, a big grid overflows the stacked image height GGML_ASSERT(grid_x >= 0 && grid_x <= 1024 && grid_y >= 0 && grid_y <= 1024); - clip_image_u8 overview; + mtmd_image_u8 overview; img_tool::resize(img, overview, {tile_size, tile_size}, hparams.image_resize_algo_ov, hparams.image_pad_ov, hparams.image_pad_color_ov); - clip_image_u8 refined; + mtmd_image_u8 refined; img_tool::resize(img, refined, refined_size, hparams.image_resize_algo_rf, hparams.image_pad_rf, hparams.image_pad_color_rf); // stack the overview and the tiles on the Y axis, so the whole grid goes through one graph - clip_image_u8 stacked; + mtmd_image_u8 stacked; stacked.set_size({tile_size, tile_size * (1 + grid_x * grid_y)}, false); - auto copy_tile = [&](const clip_image_u8 & src, int src_x, int src_y, int dst_idx) { + auto copy_tile = [&](const mtmd_image_u8 & src, int src_x, int src_y, int dst_idx) { for (int py = 0; py < tile_size; py++) { for (int px = 0; px < tile_size; px++) { stacked.set_pixel(px, dst_idx * tile_size + py, src.get_pixel(src_x + px, src_y + py)); @@ -2045,7 +2088,7 @@ mtmd_image_preproc_out mtmd_image_preprocessor_granite::preprocess(const clip_im // // Replicates transformers' get_aspect_ratio_preserving_size -static clip_image_size muse_glimmer_grid_size(int img_w, int img_h, int patch_hw, int max_tokens) { +static mtmd_image_size muse_glimmer_grid_size(int img_w, int img_h, int patch_hw, int max_tokens) { double i_nph = (double) img_h / patch_hw; double i_npw = (double) img_w / patch_hw; const double ratio = i_nph > 0.0 ? i_npw / i_nph : 1.0; @@ -2080,24 +2123,140 @@ static clip_image_size muse_glimmer_grid_size(int img_w, int img_h, int patch_hw best_nph = std::max(1, (int) std::lround(i_nph)); best_npw = std::max(1, (int) std::lround(i_npw)); } - return clip_image_size{ best_npw * patch_hw, best_nph * patch_hw }; + return mtmd_image_size{ best_npw * patch_hw, best_nph * patch_hw }; } -mtmd_image_preproc_out mtmd_image_preprocessor_muse_glimmer::preprocess(const clip_image_u8 & img) const { +mtmd_image_preproc_out mtmd_image_preprocessor_muse_glimmer::preprocess(const mtmd_image_u8 & img) const { const int patch_hw = hparams.patch_size * hparams.n_merge; const int patch_area = hparams.patch_size * hparams.patch_size * hparams.n_merge * hparams.n_merge; GGML_ASSERT(patch_area > 0 && hparams.image_max_pixels > 0); const int max_tokens = hparams.image_max_pixels / patch_area; - const clip_image_size original_size = img.get_size(); - const clip_image_size target_size = muse_glimmer_grid_size( + const mtmd_image_size original_size = img.get_size(); + const mtmd_image_size target_size = muse_glimmer_grid_size( original_size.width, original_size.height, patch_hw, max_tokens); // PIL resizes directly to (target_w, target_h) -- a stretch, no padding. - clip_image_u8 resized_image; + mtmd_image_u8 resized_image; img_tool::resize(img, resized_image, target_size, hparams.image_resize_algo, PAD_NONE); mtmd_image_preproc_out output; output.append(hparams, resized_image, true); return output; } + +//#define MTMD_DEBUG_FUNCTIONS + +#ifdef MTMD_DEBUG_FUNCTIONS +static void mtmd_image_write_image_to_ppm(const mtmd_image_u8& img, const std::string& filename) { + std::ofstream file(filename, std::ios::binary); + if (!file.is_open()) { + LOG_ERR("Failed to open file for writing: %s\n", filename.c_str()); + return; + } + + // PPM header: P6 format, width, height, and max color value + const auto ppm_size = img.get_size(); + file << "P6\n" << ppm_size.width << " " << ppm_size.height << "\n255\n"; + + // Write pixel data + const auto & ppm_buf = img.get_ro_buf(); + for (size_t i = 0; i < ppm_buf.size(); i += 3) { + // PPM expects binary data in RGB format, which matches our image buffer + file.write(reinterpret_cast(&ppm_buf[i]), 3); + } + + file.close(); +} + +static void mtmd_image_save_to_bmp(const mtmd_image_u8& img, const std::string& filename) { + std::ofstream file(filename, std::ios::binary); + if (!file.is_open()) { + LOG_ERR("Failed to open file for writing: %s\n", filename.c_str()); + return; + } + + const auto bmp_size = img.get_size(); + int fileSize = 54 + 3 * bmp_size.width * bmp_size.height; // File header + info header + pixel data + int bytesPerPixel = 3; + int widthInBytes = bmp_size.width * bytesPerPixel; + int paddingAmount = (4 - (widthInBytes % 4)) % 4; + int stride = widthInBytes + paddingAmount; + + // Bitmap file header + unsigned char fileHeader[14] = { + 'B','M', // Signature + 0,0,0,0, // Image file size in bytes + 0,0,0,0, // Reserved + 54,0,0,0 // Start of pixel array + }; + + // Total file size + fileSize = 54 + (stride * bmp_size.height); + fileHeader[2] = (unsigned char)(fileSize); + fileHeader[3] = (unsigned char)(fileSize >> 8); + fileHeader[4] = (unsigned char)(fileSize >> 16); + fileHeader[5] = (unsigned char)(fileSize >> 24); + + // Bitmap information header (BITMAPINFOHEADER) + unsigned char infoHeader[40] = { + 40,0,0,0, // Size of this header (40 bytes) + 0,0,0,0, // Image width + 0,0,0,0, // Image height + 1,0, // Number of color planes + 24,0, // Bits per pixel + 0,0,0,0, // No compression + 0,0,0,0, // Image size (can be 0 for no compression) + 0,0,0,0, // X pixels per meter (not specified) + 0,0,0,0, // Y pixels per meter (not specified) + 0,0,0,0, // Total colors (color table not used) + 0,0,0,0 // Important colors (all are important) + }; + + // Width and height in the information header + infoHeader[4] = (unsigned char)(bmp_size.width); + infoHeader[5] = (unsigned char)(bmp_size.width >> 8); + infoHeader[6] = (unsigned char)(bmp_size.width >> 16); + infoHeader[7] = (unsigned char)(bmp_size.width >> 24); + infoHeader[8] = (unsigned char)(bmp_size.height); + infoHeader[9] = (unsigned char)(bmp_size.height >> 8); + infoHeader[10] = (unsigned char)(bmp_size.height >> 16); + infoHeader[11] = (unsigned char)(bmp_size.height >> 24); + + // Write file headers + file.write(reinterpret_cast(fileHeader), sizeof(fileHeader)); + file.write(reinterpret_cast(infoHeader), sizeof(infoHeader)); + + // Pixel data + std::vector padding(3, 0); // Max padding size to be added to each row + for (int y = bmp_size.height - 1; y >= 0; --y) { // BMP files are stored bottom-to-top + for (int x = 0; x < bmp_size.width; ++x) { + // Each pixel + const auto px = img.get_pixel(x, y); + unsigned char pixel[3] = { + px[2], // BMP stores pixels in BGR format + px[1], + px[0] + }; + file.write(reinterpret_cast(pixel), 3); + } + // Write padding for the row + file.write(reinterpret_cast(padding.data()), paddingAmount); + } + + file.close(); +} + +// debug function to convert f32 to u8 +static void mtmd_image_convert_f32_to_u8(const mtmd_image_f32& src, mtmd_image_u8& dst) { + dst.set_size(src.get_size(), false); + const auto & src_buf = src.get_ro_buf(); + std::vector dst_buf(src.n_elements()); + for (size_t i = 0; i < src.n_elements(); ++i) { + dst_buf[i] = static_cast(std::min(std::max(int(src_buf[i] * 255.0f), 0), 255)); + } + dst.cpy_buf(dst_buf); +} +#endif + + diff --git a/tools/mtmd/mtmd-image.h b/tools/mtmd/mtmd-image.h index 07feae4b2305..3e0b127e9e2f 100644 --- a/tools/mtmd/mtmd-image.h +++ b/tools/mtmd/mtmd-image.h @@ -1,7 +1,7 @@ #pragma once #include "ggml.h" -#include "clip-model.h" +#include "mtmd-model.h" #include #include @@ -9,18 +9,18 @@ #define MTMD_INTERNAL_HEADER struct mtmd_image_preproc_out { - std::vector entries; + std::vector entries; // grid size is required for llava-uhd style models - clip_image_f32 overview; // overview image (downscaled image) + mtmd_image_f32 overview; // overview image (downscaled image) int grid_x = 0; int grid_y = 0; - void append(const clip_hparams & hparams, const clip_image_u8 & img, bool normalized = true); - void append(const clip_hparams & hparams, const std::vector & imgs, bool normalized = true); - void append(const clip_hparams & hparams, clip_image_f32 & img, bool normalized = true); + void append(const mtmd_hparams & hparams, const mtmd_image_u8 & img, bool normalized = true); + void append(const mtmd_hparams & hparams, const std::vector & imgs, bool normalized = true); + void append(const mtmd_hparams & hparams, mtmd_image_f32 & img, bool normalized = true); - void append_overview(const clip_hparams & hparams, const clip_image_u8 & img, bool normalized = true); + void append_overview(const mtmd_hparams & hparams, const mtmd_image_u8 & img, bool normalized = true); bool has_overview() const { return overview.nx() > 0 || overview.ny() > 0; } @@ -28,17 +28,17 @@ struct mtmd_image_preproc_out { // base class, models must inherit from this class struct mtmd_image_preprocessor { - const clip_hparams & hparams; + const mtmd_hparams & hparams; - mtmd_image_preprocessor(const clip_ctx * ctx): hparams(*clip_get_hparams(ctx)) {} - mtmd_image_preprocessor(const clip_hparams & hparams): hparams(hparams) {} + mtmd_image_preprocessor(const mtmd_model * model): hparams(model->hparams) {} + mtmd_image_preprocessor(const mtmd_hparams & hparams): hparams(hparams) {} virtual ~mtmd_image_preprocessor() = default; - virtual mtmd_image_preproc_out preprocess(const clip_image_u8 & img) const = 0; + virtual mtmd_image_preproc_out preprocess(const mtmd_image_u8 & img) const = 0; // frames of a video, which the video helper may have already sized - virtual mtmd_image_preproc_out preprocess_video_frame(const clip_image_u8 & img) const { return preprocess(img); } + virtual mtmd_image_preproc_out preprocess_video_frame(const mtmd_image_u8 & img) const { return preprocess(img); } // frame size of a video with n_frames sampled frames, as the reference video processor; false if none - virtual bool get_video_frame_size(int /*n_frames*/, int /*width*/, int /*height*/, clip_image_size & /*out*/) const { return false; } + virtual bool get_video_frame_size(int /*n_frames*/, int /*width*/, int /*height*/, mtmd_image_size & /*out*/) const { return false; } }; /** @@ -63,30 +63,30 @@ struct mtmd_image_preprocessor { * NOTE: for the ordering of overview, set "ov_img_first" on the mtmd_context */ struct mtmd_image_preprocessor_llava_uhd : mtmd_image_preprocessor { - mtmd_image_preprocessor_llava_uhd(const clip_ctx * ctx) : mtmd_image_preprocessor(ctx) {} - mtmd_image_preprocessor_llava_uhd(const clip_hparams & hparams) : mtmd_image_preprocessor(hparams) {} - mtmd_image_preproc_out preprocess(const clip_image_u8 & img) const override; + mtmd_image_preprocessor_llava_uhd(const mtmd_model * model) : mtmd_image_preprocessor(model) {} + mtmd_image_preprocessor_llava_uhd(const mtmd_hparams & hparams) : mtmd_image_preprocessor(hparams) {} + mtmd_image_preproc_out preprocess(const mtmd_image_u8 & img) const override; struct slice_coordinates { int x; int y; - clip_image_size size; + mtmd_image_size size; }; struct slice_instructions { - clip_image_size overview_size; // size of downscaled image - clip_image_size refined_size; // size of image right before slicing (must be multiple of slice size) - clip_image_size grid_size; // grid_size.width * grid_size.height = number of slices + mtmd_image_size overview_size; // size of downscaled image + mtmd_image_size refined_size; // size of image right before slicing (must be multiple of slice size) + mtmd_image_size grid_size; // grid_size.width * grid_size.height = number of slices std::vector slices; }; - virtual slice_instructions get_slice_instructions(const clip_image_size & original_size) const; + virtual slice_instructions get_slice_instructions(const mtmd_image_size & original_size) const; struct slice_output { - clip_image_u8 overview; - std::vector slices; + mtmd_image_u8 overview; + std::vector slices; }; - slice_output slice_image(const clip_image_u8 & img, const slice_instructions & inst) const; + slice_output slice_image(const mtmd_image_u8 & img, const slice_instructions & inst) const; protected: // align slices to a multiple of the merger factor (integer merger tokens per slice) @@ -100,7 +100,7 @@ struct mtmd_image_preprocessor_llava_uhd : mtmd_image_preprocessor { return static_cast(std::round(v)); } - clip_image_size get_best_resize(const clip_image_size & original_size, int scale_resolution, int patch_size, bool allow_upscale = false) const; + mtmd_image_size get_best_resize(const mtmd_image_size & original_size, int scale_resolution, int patch_size, bool allow_upscale = false) const; /** * Selects the best resolution from a list of possible resolutions based on the original size. @@ -117,62 +117,74 @@ struct mtmd_image_preprocessor_llava_uhd : mtmd_image_preprocessor { * @param possible_resolutions A list of possible resolutions * @return The best fit resolution */ - clip_image_size select_best_resolution(const clip_image_size & original_size, const std::vector & possible_resolutions) const; + mtmd_image_size select_best_resolution(const mtmd_image_size & original_size, const std::vector & possible_resolutions) const; private: - clip_image_size resize_maintain_aspect_ratio(const clip_image_size & orig, const clip_image_size & target_max) const; + mtmd_image_size resize_maintain_aspect_ratio(const mtmd_image_size & orig, const mtmd_image_size & target_max) const; int ensure_divide(int length, int patch_size) const; - clip_image_size get_refine_size(const clip_image_size & original_size, const clip_image_size & grid, int scale_resolution, int patch_size, bool allow_upscale = false) const; - clip_image_size get_best_grid(const int max_slice_nums, const int multiple, const float log_ratio) const; + mtmd_image_size get_refine_size(const mtmd_image_size & original_size, const mtmd_image_size & grid, int scale_resolution, int patch_size, bool allow_upscale = false) const; + mtmd_image_size get_best_grid(const int max_slice_nums, const int multiple, const float log_ratio) const; }; // downscale or upscale the input image to fixed size struct mtmd_image_preprocessor_fixed_size : mtmd_image_preprocessor { - mtmd_image_preprocessor_fixed_size(const clip_ctx * ctx) : mtmd_image_preprocessor(ctx) {} - mtmd_image_preproc_out preprocess(const clip_image_u8 & img) const override; + mtmd_image_preprocessor_fixed_size(const mtmd_model * model) : mtmd_image_preprocessor(model) {} + mtmd_image_preproc_out preprocess(const mtmd_image_u8 & img) const override; }; // resize image to multiple of patch_size*n_merge, while preserving aspect ratio // if image_resize_pad is true, the resized image will be padded, otherwise it will be either stretched or center-cropped depending on image_resize_pad // this is used by models with native support for dynamic image size, for example: Qwen-VL, Pixtral, Kimi-VL, etc struct mtmd_image_preprocessor_dyn_size : mtmd_image_preprocessor { - mtmd_image_preprocessor_dyn_size(const clip_ctx * ctx) : mtmd_image_preprocessor(ctx) {} - static clip_image_size get_target_size(const clip_hparams & hparams, const clip_image_size & original_size); - mtmd_image_preproc_out preprocess(const clip_image_u8 & img) const override; - virtual clip_image_size target_size(const clip_image_size & original_size) const { return get_target_size(hparams, original_size); } + mtmd_image_preprocessor_dyn_size(const mtmd_model * model) : mtmd_image_preprocessor(model) {} + mtmd_image_preprocessor_dyn_size(const mtmd_hparams & hparams) : mtmd_image_preprocessor(hparams) {} + static mtmd_image_size get_target_size(const mtmd_hparams & hparams, const mtmd_image_size & original_size); + mtmd_image_preproc_out preprocess(const mtmd_image_u8 & img) const override; + virtual mtmd_image_size target_size(const mtmd_image_size & original_size) const { return get_target_size(hparams, original_size); } }; -// LFM3-VL, same as dyn_size but sized exactly as the transformers Qwen2-VL smart_resize -struct mtmd_image_preprocessor_lfm3vl : mtmd_image_preprocessor_dyn_size { - mtmd_image_preprocessor_lfm3vl(const clip_ctx * ctx) : mtmd_image_preprocessor_dyn_size(ctx) {} - clip_image_size target_size(const clip_image_size & original_size) const override; +// Kimi-K2.5 / K3.1 navit_resize_image: scale down to the patch limits, then pad bottom-right with black to a multiple of patch_size*n_merge +struct mtmd_image_preprocessor_kimik25 : mtmd_image_preprocessor { + mtmd_image_preprocessor_kimik25(const mtmd_model * model) : mtmd_image_preprocessor(model) {} + mtmd_image_preproc_out preprocess(const mtmd_image_u8 & img) const override; +}; + +// Qwen3-VL, same as dyn_size for images, video frames are sized as Qwen3VLVideoProcessor +struct mtmd_image_preprocessor_qwen3vl : mtmd_image_preprocessor_dyn_size { + using mtmd_image_preprocessor_dyn_size::mtmd_image_preprocessor_dyn_size; // the video helper sizes the frames (get_video_frame_size), they are only snapped to the grid - mtmd_image_preproc_out preprocess_video_frame(const clip_image_u8 & img) const override; - bool get_video_frame_size(int n_frames, int width, int height, clip_image_size & out) const override; + mtmd_image_preproc_out preprocess_video_frame(const mtmd_image_u8 & img) const override; + bool get_video_frame_size(int n_frames, int width, int height, mtmd_image_size & out) const override; - mtmd_image_preproc_out resize_to(const clip_image_u8 & img, const clip_image_size & size) const; + mtmd_image_preproc_out resize_to(const mtmd_image_u8 & img, const mtmd_image_size & size) const; // with temporal_factor > 1 it is the Qwen3-VL video variant over n_frames - static clip_image_size smart_resize(int width, int height, int factor, int64_t min_pixels, int64_t max_pixels, + static mtmd_image_size smart_resize(int width, int height, int factor, int64_t min_pixels, int64_t max_pixels, int n_frames = 1, int temporal_factor = 1); }; +// LFM3-VL, same as qwen3vl but images are sized exactly as the transformers Qwen2-VL smart_resize +struct mtmd_image_preprocessor_lfm3vl : mtmd_image_preprocessor_qwen3vl { + using mtmd_image_preprocessor_qwen3vl::mtmd_image_preprocessor_qwen3vl; + mtmd_image_size target_size(const mtmd_image_size & original_size) const override; +}; + // GLM 5.3 flash, similar to dyn_size, but each edge is aligned up to patch_size*n_merge, and max budget is met by a search over the height with the width scaled proportionally struct mtmd_image_preprocessor_glm5v : mtmd_image_preprocessor { - mtmd_image_preprocessor_glm5v(const clip_ctx * ctx) : mtmd_image_preprocessor(ctx) {} - mtmd_image_preproc_out preprocess(const clip_image_u8 & img) const override; + mtmd_image_preprocessor_glm5v(const mtmd_model * model) : mtmd_image_preprocessor(model) {} + mtmd_image_preproc_out preprocess(const mtmd_image_u8 & img) const override; }; // similar to mtmd_image_preprocessor_dyn_size, but cap the longest edge at hparams.image_longest_edge, while preserving aspect ratio struct mtmd_image_preprocessor_longest_edge : mtmd_image_preprocessor { - mtmd_image_preprocessor_longest_edge(const clip_ctx * ctx) : mtmd_image_preprocessor(ctx) {} - static clip_image_size get_target_size(const clip_hparams & hparams, const clip_image_size & original_size); - mtmd_image_preproc_out preprocess(const clip_image_u8 & img) const override; + mtmd_image_preprocessor_longest_edge(const mtmd_model * model) : mtmd_image_preprocessor(model) {} + static mtmd_image_size get_target_size(const mtmd_hparams & hparams, const mtmd_image_size & original_size); + mtmd_image_preproc_out preprocess(const mtmd_image_u8 & img) const override; }; // ref: inference/image_processor.py in the HF repo (DeepSeek-V4-Flash-Vision) struct mtmd_image_preprocessor_deepseek4v : mtmd_image_preprocessor { - mtmd_image_preprocessor_deepseek4v(const clip_ctx * ctx) : mtmd_image_preprocessor(ctx) {} - mtmd_image_preproc_out preprocess(const clip_image_u8 & img) const override; + mtmd_image_preprocessor_deepseek4v(const mtmd_model * model) : mtmd_image_preprocessor(model) {} + mtmd_image_preproc_out preprocess(const mtmd_image_u8 & img) const override; private: struct grid_info { @@ -187,8 +199,8 @@ struct mtmd_image_preprocessor_deepseek4v : mtmd_image_preprocessor { // ref: inference/image_processor.py in the HF repo (DeepSeek-V4.1-Flash): plan_image_grid / load_image struct mtmd_image_preprocessor_deepseek41v : mtmd_image_preprocessor { - mtmd_image_preprocessor_deepseek41v(const clip_ctx * ctx) : mtmd_image_preprocessor(ctx) {} - mtmd_image_preproc_out preprocess(const clip_image_u8 & img) const override; + mtmd_image_preprocessor_deepseek41v(const mtmd_model * model) : mtmd_image_preprocessor(model) {} + mtmd_image_preproc_out preprocess(const mtmd_image_u8 & img) const override; // target pixel size (multiples of patch_size) for an image of the given size, exposed for tests static void plan_grid(int width, int height, int p, int r, int max_n_token, int min_pixels, int & best_width, int & best_height); @@ -197,7 +209,7 @@ struct mtmd_image_preprocessor_deepseek41v : mtmd_image_preprocessor { // custom llava-uhd slicing logic for MiniCPM-V struct mtmd_image_preprocessor_minicpmv : mtmd_image_preprocessor_llava_uhd { using mtmd_image_preprocessor_llava_uhd::mtmd_image_preprocessor_llava_uhd; - slice_instructions get_slice_instructions(const clip_image_size & original_size) const override; + slice_instructions get_slice_instructions(const mtmd_image_size & original_size) const override; protected: // always patch_size * 4, even in 4x mode (the 2x2 vit_merger slot stays) @@ -236,50 +248,50 @@ struct mtmd_image_preprocessor_minicpm_ocr_1_0 : mtmd_image_preprocessor_minicpm // ref: https://github.com/huggingface/transformers/blob/v5.1.0/src/transformers/models/lfm2_vl/image_processing_lfm2_vl_fast.py struct mtmd_image_preprocessor_lfm2 : mtmd_image_preprocessor_llava_uhd { using mtmd_image_preprocessor_llava_uhd::mtmd_image_preprocessor_llava_uhd; - mtmd_image_preproc_out preprocess(const clip_image_u8 & img) const override; - slice_instructions get_slice_instructions(const clip_image_size & original_size) const override; + mtmd_image_preproc_out preprocess(const mtmd_image_u8 & img) const override; + slice_instructions get_slice_instructions(const mtmd_image_size & original_size) const override; - static bool should_tile(const clip_hparams & hparams, const clip_image_size & original_size); - static clip_image_size smart_resize(const clip_hparams & hparams, const clip_image_size & original_size); - static clip_image_size get_grid_layout(const clip_hparams & hparams, int height, int width); + static bool should_tile(const mtmd_hparams & hparams, const mtmd_image_size & original_size); + static mtmd_image_size smart_resize(const mtmd_hparams & hparams, const mtmd_image_size & original_size); + static mtmd_image_size get_grid_layout(const mtmd_hparams & hparams, int height, int width); private: - static clip_image_size find_closest_aspect_ratio( - const clip_hparams & hparams, + static mtmd_image_size find_closest_aspect_ratio( + const mtmd_hparams & hparams, double aspect_ratio, - const std::vector & target_ratios, + const std::vector & target_ratios, int width, int height); - static std::vector get_target_ratios(const clip_hparams & hparams); + static std::vector get_target_ratios(const mtmd_hparams & hparams); }; struct mtmd_image_preprocessor_idefics3 : mtmd_image_preprocessor_llava_uhd { - mtmd_image_preprocessor_idefics3(const clip_ctx * ctx) : mtmd_image_preprocessor_llava_uhd(ctx) {} - mtmd_image_preproc_out preprocess(const clip_image_u8 & img) const override; + mtmd_image_preprocessor_idefics3(const mtmd_model * model) : mtmd_image_preprocessor_llava_uhd(model) {} + mtmd_image_preproc_out preprocess(const mtmd_image_u8 & img) const override; }; struct mtmd_image_preprocessor_internvl : mtmd_image_preprocessor_llava_uhd { - mtmd_image_preprocessor_internvl(const clip_ctx * ctx) : mtmd_image_preprocessor_llava_uhd(ctx) {} - mtmd_image_preproc_out preprocess(const clip_image_u8 & img) const override; + mtmd_image_preprocessor_internvl(const mtmd_model * model) : mtmd_image_preprocessor_llava_uhd(model) {} + mtmd_image_preproc_out preprocess(const mtmd_image_u8 & img) const override; }; // stretch the image to a grid of tiles, add a thumbnail if there is more than 1 tile // ref: https://github.com/huggingface/transformers/blob/main/src/transformers/models/cohere2_vision/image_processing_cohere2_vision.py struct mtmd_image_preprocessor_cohere2v : mtmd_image_preprocessor_llava_uhd { - mtmd_image_preprocessor_cohere2v(const clip_ctx * ctx) : mtmd_image_preprocessor_llava_uhd(ctx) {} - mtmd_image_preproc_out preprocess(const clip_image_u8 & img) const override; - slice_instructions get_slice_instructions(const clip_image_size & original_size) const override; + mtmd_image_preprocessor_cohere2v(const mtmd_model * model) : mtmd_image_preprocessor_llava_uhd(model) {} + mtmd_image_preproc_out preprocess(const mtmd_image_u8 & img) const override; + slice_instructions get_slice_instructions(const mtmd_image_size & original_size) const override; }; // DeepSeek-OCR (v1/v2) global view + optional local tile grid struct mtmd_image_preprocessor_deepseekocr : mtmd_image_preprocessor { - mtmd_image_preprocessor_deepseekocr(const clip_ctx * ctx) - : mtmd_image_preprocessor(ctx), - fuse_row(clip_get_projector_type(ctx) == PROJECTOR_TYPE_DEEPSEEKOCR), + mtmd_image_preprocessor_deepseekocr(const mtmd_model * model) + : mtmd_image_preprocessor(model), + fuse_row(model->proj_type == PROJECTOR_TYPE_DEEPSEEKOCR), base_size(hparams.image_size), tile_size(hparams.preproc_tile_size), min_tiles(hparams.preproc_min_tiles), max_tiles(hparams.preproc_max_tiles) {} - mtmd_image_preproc_out preprocess(const clip_image_u8 & img) const override; + mtmd_image_preproc_out preprocess(const mtmd_image_u8 & img) const override; private: bool fuse_row; // v1 fuses a tile-row into one image; v2 keeps tiles separate @@ -288,19 +300,19 @@ struct mtmd_image_preprocessor_deepseekocr : mtmd_image_preprocessor { int min_tiles; int max_tiles; - std::vector get_target_ratios() const; - clip_image_size find_closest_aspect_ratio( + std::vector get_target_ratios() const; + mtmd_image_size find_closest_aspect_ratio( float aspect_ratio, - const std::vector & target_ratios, + const std::vector & target_ratios, int width, int height) const; }; // custom image preprocessing for Step3VL // ref: https://huggingface.co/stepfun-ai/Step3-VL-10B/blob/main/processing_step3.py struct mtmd_image_preprocessor_step3vl : mtmd_image_preprocessor_llava_uhd { - mtmd_image_preprocessor_step3vl(const clip_ctx * ctx) : mtmd_image_preprocessor_llava_uhd(ctx) {} - mtmd_image_preproc_out preprocess(const clip_image_u8 & img) const override; - static slice_instructions build_slice_instructions(const clip_hparams & params, const clip_image_size & prepared_size); + mtmd_image_preprocessor_step3vl(const mtmd_model * model) : mtmd_image_preprocessor_llava_uhd(model) {} + mtmd_image_preproc_out preprocess(const mtmd_image_u8 & img) const override; + static slice_instructions build_slice_instructions(const mtmd_hparams & params, const mtmd_image_size & prepared_size); private: static constexpr int default_image_longest_edge = 3024; @@ -310,33 +322,33 @@ struct mtmd_image_preprocessor_step3vl : mtmd_image_preprocessor_llava_uhd { static constexpr float crop_rounding_threshold = 0.2f; void img_u8_resize_bilinear_to_f32( - const clip_image_u8 & src, - clip_image_f32 & dst, + const mtmd_image_u8 & src, + mtmd_image_f32 & dst, int target_width, int target_height, const float mean[3], const float std[3]) const; - static int get_image_longest_edge(const clip_hparams & params); - static int determine_window_size(const clip_hparams & params, int longer, int shorter); + static int get_image_longest_edge(const mtmd_hparams & params); + static int determine_window_size(const mtmd_hparams & params, int longer, int shorter); static int calc_crop_extent(int length, int window_size); static std::vector calc_grid(int length, int window_size); - static clip_image_u8 prepare_image(const clip_image_u8 & img, const clip_hparams & params); - static clip_image_u8 crop_with_black_padding(const clip_image_u8 & image, int x, int y, int w, int h); + static mtmd_image_u8 prepare_image(const mtmd_image_u8 & img, const mtmd_hparams & params); + static mtmd_image_u8 crop_with_black_padding(const mtmd_image_u8 & image, int x, int y, int w, int h); }; struct mtmd_image_preprocessor_youtuvl : mtmd_image_preprocessor { - mtmd_image_preprocessor_youtuvl(const clip_ctx * ctx) : mtmd_image_preprocessor(ctx) {} - mtmd_image_preproc_out preprocess(const clip_image_u8 & img) const override; + mtmd_image_preprocessor_youtuvl(const mtmd_model * model) : mtmd_image_preprocessor(model) {} + mtmd_image_preproc_out preprocess(const mtmd_image_u8 & img) const override; }; -// llava-next "anyres": stacks the overview and all tiles into one image, assembled by clip in a single graph +// llava-next "anyres": stacks the overview and all tiles into one image, assembled in a single graph struct mtmd_image_preprocessor_granite : mtmd_image_preprocessor_llava_uhd { - mtmd_image_preprocessor_granite(const clip_ctx * ctx) : mtmd_image_preprocessor_llava_uhd(ctx) {} - mtmd_image_preproc_out preprocess(const clip_image_u8 & img) const override; + mtmd_image_preprocessor_granite(const mtmd_model * model) : mtmd_image_preprocessor_llava_uhd(model) {} + mtmd_image_preproc_out preprocess(const mtmd_image_u8 & img) const override; }; // pick the patch grid closest to the input aspect ratio under the per-image token cap, stretch-resize. struct mtmd_image_preprocessor_muse_glimmer : mtmd_image_preprocessor { - mtmd_image_preprocessor_muse_glimmer(const clip_ctx * ctx) : mtmd_image_preprocessor(ctx) {} - mtmd_image_preproc_out preprocess(const clip_image_u8 & img) const override; + mtmd_image_preprocessor_muse_glimmer(const mtmd_model * model) : mtmd_image_preprocessor(model) {} + mtmd_image_preproc_out preprocess(const mtmd_image_u8 & img) const override; }; diff --git a/tools/mtmd/clip-impl.h b/tools/mtmd/mtmd-impl.h similarity index 91% rename from tools/mtmd/clip-impl.h rename to tools/mtmd/mtmd-impl.h index 0504e8fdcc4c..9d447a05e09d 100644 --- a/tools/mtmd/clip-impl.h +++ b/tools/mtmd/mtmd-impl.h @@ -3,7 +3,7 @@ #include "ggml.h" #include "ggml-cpp.h" #include "gguf.h" -#include "clip.h" +#include "mtmd.h" #include #include @@ -25,7 +25,7 @@ #include #endif -// Internal header for clip.cpp +// !!! Internal header, to be used by mtmd only !!! #define MTMD_INTERNAL_HEADER @@ -113,6 +113,11 @@ #define KEY_GEN_AUDIO_PENALTY_LAST_N "clip.gen.audio.sampling.penalty_last_n" #define KEY_AUDIO_SUBSMPL_FACTOR "clip.audio.subsampling_factor" +// nemotron transcribe +#define KEY_A_ROPE_DIMS "clip.audio.rope_dims" +#define KEY_A_SPK_N_BLOCK "clip.audio.spk.block_count" +#define KEY_A_SPK_N_HEAD "clip.audio.spk.head_count" + // // tensor name constants // @@ -122,6 +127,7 @@ #define TN_PATCH_EMBD "v.patch_embd.weight" // not rename tensor with ".0" postfix for backward compat #define TN_PATCH_EMBD_1 "v.patch_embd.weight.1" #define TN_PATCH_BIAS "v.patch_embd.bias" +#define TN_ROPE_FREQS "v.rope_freqs.weight" #define TN_NORM_EMBD "v.norm_embd.%s" #define TN_PATCH_NORM "v.patch_norm.%d.%s" #define TN_ATTN_QKV "%s.blk.%d.attn_qkv.%s" @@ -256,6 +262,13 @@ #define TN_A_GEN_CODE_MASK_EMBD "a.gen.code.mask_embd" #define TN_ATTN_GATE "%s.blk.%d.attn_gate.%s" +// drama3 tower, the rest of the block uses the generic names with prefix "a.gen.code" +#define TN_A_GEN_CODE_ENORM "a.gen.code.blk.%d.enorm.%s" +#define TN_A_GEN_CODE_HNORM "a.gen.code.blk.%d.hnorm.%s" +#define TN_A_GEN_CODE_EH_PROJ "a.gen.code.blk.%d.eh_proj.%s" +#define TN_A_GEN_CODE_BLK_EMBD "a.gen.code.blk.%d.embd.%s" +#define TN_A_GEN_CODE_BLK_OUT_NORM "a.gen.code.blk.%d.out_norm.%s" + // qwen3tts code2wav (RVQ codes -> raw PCM) // pre_transformer layers use the generic TN_ATTN_*/TN_FFN_*/TN_LN_*/TN_LS_* macros, prefix "a.gen.wav.tfm" #define TN_A_GEN_WAV_QUANT_FIRST_IN "a.gen.wav.quant.first.in_proj.%s" @@ -469,38 +482,15 @@ #define TN_CONV_NORM_MEAN "%s.blk.%d.conv_norm_mean" #define TN_CONV_NORM_VAR "%s.blk.%d.conv_norm_var" -// align x to upper multiple of n -#define CLIP_ALIGN(x, n) ((((x) + (n) - 1) / (n)) * (n)) - -// sliding window KV cache of a causal audio encoder, see clip_enc_state_init() -struct clip_enc_state { - int32_t n_ctx = 0; // ring slots: attention window + frames per call - int32_t n_past = 0; // absolute position of the next frame - - std::vector slot_pos; // absolute position held by each slot, -1 if empty - std::vector stem_tail; // last conv stem output frame, empty on a cold start - - ggml_context_ptr ctx_data; - ggml_backend_buffer_ptr buf; - std::vector k; // per layer [n_embd_kv, n_ctx] - std::vector v; +// nemotron transcribe +#define TN_A_SPK_INP_PROJ "a.spk.input_projection.%s" +#define TN_A_SPK_HEAD "a.spk.head.%s.%s" // proj, upsample, fc1, fc2 +#define TN_A_SPK_SIL_EMB "a.spk.head.sil_emb" +#define TN_A_FUSION_NORM "a.fusion.%s.%s" // asr_norm, spk_norm +#define TN_A_FUSION_KERNEL "a.fusion.spk_kernel" - // lfm3a: same order as list_lfm3a_state_slots(), F32 [ne0, ne1], updated in place - std::vector slots; -}; - -// per-sequence state of audio generation, see clip_gen_state_init() -struct clip_gen_state { - std::mt19937 rng; - - ggml_context_ptr ctx_data; - ggml_backend_buffer_ptr buf; - std::vector slots; // same order as list_gen_state_slots(), F32 [ne0, ne1] -}; - -// forward declaration -// TODO: improve this later -struct clip_ctx; +// align x to upper multiple of n +#define MTMD_ALIGN(x, n) ((((x) + (n) - 1) / (n)) * (n)) enum projector_type { PROJECTOR_TYPE_MLP, @@ -530,7 +520,7 @@ enum projector_type { PROJECTOR_TYPE_QWEN2A, PROJECTOR_TYPE_QWEN3A, PROJECTOR_TYPE_GLMA, - PROJECTOR_TYPE_QWEN25O, // will be replaced by QWEN2A or QWEN25VL depending on clip_ctx + PROJECTOR_TYPE_QWEN25O, // will be replaced by QWEN2A or QWEN25VL depending on mtmd_subcontext PROJECTOR_TYPE_VOXTRAL, PROJECTOR_TYPE_VOXTRAL_RT, PROJECTOR_TYPE_MERALION, @@ -558,6 +548,7 @@ enum projector_type { PROJECTOR_TYPE_YOUTUVL, PROJECTOR_TYPE_YASA2, PROJECTOR_TYPE_KIMIK25, + PROJECTOR_TYPE_KIMIK31, PROJECTOR_TYPE_NEMOTRON_V2_VL, PROJECTOR_TYPE_NEMOTRON_OMNI, PROJECTOR_TYPE_HUNYUANVL, @@ -567,7 +558,9 @@ enum projector_type { PROJECTOR_TYPE_MINICPMV4_7, PROJECTOR_TYPE_MINICPM_OCR_1_0, PROJECTOR_TYPE_GRANITE_SPEECH, + PROJECTOR_TYPE_NEMOTRON_TRANSCRIBE, PROJECTOR_TYPE_MIMOVL, + PROJECTOR_TYPE_MAGEVL, PROJECTOR_TYPE_MINIMAX_M3, PROJECTOR_TYPE_GRANITE4_VISION, PROJECTOR_TYPE_MIMO_AUDIO, @@ -577,10 +570,13 @@ enum projector_type { PROJECTOR_TYPE_POCKETTTS_GEN, PROJECTOR_TYPE_SOPRANO, PROJECTOR_TYPE_LFM3A_GEN, + PROJECTOR_TYPE_DRAMA3_GEN, PROJECTOR_TYPE_MUSE_GLIMMER, PROJECTOR_TYPE_SENSENOVA_U1, PROJECTOR_TYPE_COHERE2V, PROJECTOR_TYPE_MUSE_SPARK_AUDIO, + PROJECTOR_TYPE_CHATTERBOX, + PROJECTOR_TYPE_CHATTERBOX_SPKENC, PROJECTOR_TYPE_UNKNOWN, }; @@ -639,6 +635,7 @@ static std::map PROJECTOR_TYPE_NAMES = { { PROJECTOR_TYPE_YOUTUVL, "youtuvl"}, { PROJECTOR_TYPE_YASA2, "yasa2"}, { PROJECTOR_TYPE_KIMIK25, "kimik25"}, + { PROJECTOR_TYPE_KIMIK31, "kimik31"}, { PROJECTOR_TYPE_NEMOTRON_V2_VL, "nemotron_v2_vl"}, { PROJECTOR_TYPE_NEMOTRON_OMNI, "nemotron_omni"}, { PROJECTOR_TYPE_EXAONE4_5, "exaone4_5"}, @@ -647,7 +644,9 @@ static std::map PROJECTOR_TYPE_NAMES = { { PROJECTOR_TYPE_MINICPMV4_7, "minicpmv4_7"}, { PROJECTOR_TYPE_MINICPM_OCR_1_0, "minicpm_ocr_1_0"}, { PROJECTOR_TYPE_GRANITE_SPEECH, "granite_speech"}, + { PROJECTOR_TYPE_NEMOTRON_TRANSCRIBE, "nemotron_transcribe"}, { PROJECTOR_TYPE_MIMOVL, "mimovl"}, + { PROJECTOR_TYPE_MAGEVL, "magevl"}, { PROJECTOR_TYPE_MINIMAX_M3, "minimax_m3"}, { PROJECTOR_TYPE_GRANITE4_VISION, "granite4_vision"}, { PROJECTOR_TYPE_MIMO_AUDIO, "mimo_audio"}, @@ -658,13 +657,16 @@ static std::map PROJECTOR_TYPE_NAMES = { { PROJECTOR_TYPE_POCKETTTS_GEN, "pockettts_gen"}, { PROJECTOR_TYPE_SOPRANO, "soprano"}, { PROJECTOR_TYPE_LFM3A_GEN, "lfm3a_gen"}, + { PROJECTOR_TYPE_DRAMA3_GEN, "drama3_gen"}, { PROJECTOR_TYPE_MUSE_GLIMMER, "muse-glimmer"}, { PROJECTOR_TYPE_SENSENOVA_U1, "sensenova_u1"}, { PROJECTOR_TYPE_COHERE2V, "cohere2v"}, { PROJECTOR_TYPE_MUSE_SPARK_AUDIO, "muse-spark-audio"}, + { PROJECTOR_TYPE_CHATTERBOX, "chatterbox"}, + { PROJECTOR_TYPE_CHATTERBOX_SPKENC, "chatterbox_spkenc"}, }; -static projector_type clip_projector_type_from_string(const std::string & str) { +static projector_type projector_type_from_string(const std::string & str) { for (const auto & pair : PROJECTOR_TYPE_NAMES) { if (pair.second == str) { return pair.first; @@ -673,13 +675,30 @@ static projector_type clip_projector_type_from_string(const std::string & str) { return PROJECTOR_TYPE_UNKNOWN; } +struct mtmd_image_size { + int width; + int height; + bool operator==(const mtmd_image_size & other) const { + return width == other.width && height == other.height; + } + bool operator!=(const mtmd_image_size & other) const { + return !(*this == other); + } + int area() const { + // avoid overflow when computing area + GGML_ASSERT(width >= 0 && width <= 46000); + GGML_ASSERT(height >= 0 && height <= 46000); + return width * height; + } +}; + // RGB uint8 image -struct clip_image_u8 { - clip_image_size get_size() const { +struct mtmd_image_u8 { + mtmd_image_size get_size() const { return { nx, ny }; } - void set_size(clip_image_size size, bool is_placeholder) { + void set_size(mtmd_image_size size, bool is_placeholder) { nx = size.width; ny = size.height; if (is_placeholder) { @@ -695,7 +714,7 @@ struct clip_image_u8 { const std::vector & get_ro_buf() const { if (is_placeholder()) { - throw std::runtime_error("this clip_image_u8 is a placeholder"); + throw std::runtime_error("this mtmd_image_u8 is a placeholder"); } return buf; } @@ -742,20 +761,20 @@ struct clip_image_u8 { struct mtmd_serialization; // forward declaration // separators appended after the image tokens of one entry, as rows of v.tok_embd_sep -enum clip_suffix_type : int32_t { - CLIP_SUFFIX_NONE = 0, +enum mtmd_suffix_type : int32_t { + MTMD_SUFFIX_NONE = 0, // MiniCPM-V 4.7 tiles - CLIP_SUFFIX_MINICPMV_OV, // - CLIP_SUFFIX_MINICPMV_OV_SLICE, // - CLIP_SUFFIX_MINICPMV_SLICE, // - CLIP_SUFFIX_MINICPMV_ROW_END, // \n - CLIP_SUFFIX_MINICPMV_LAST, // - CLIP_SUFFIX_COUNT, + MTMD_SUFFIX_MINICPMV_OV, // + MTMD_SUFFIX_MINICPMV_OV_SLICE, // + MTMD_SUFFIX_MINICPMV_SLICE, // + MTMD_SUFFIX_MINICPMV_ROW_END, // \n + MTMD_SUFFIX_MINICPMV_LAST, // + MTMD_SUFFIX_COUNT, }; // rows of v.tok_embd_sep for each suffix type // MiniCPM-V 4.7 rows (set by the converter): 0 = , 1 = , 2 = , 3 = \n -static inline const std::vector & clip_suffix_rows(clip_suffix_type type) { +static inline const std::vector & mtmd_suffix_rows(mtmd_suffix_type type) { static const std::vector none; static const std::vector minicpmv_ov = { 0 }; static const std::vector minicpmv_ov_slice = { 0, 1 }; @@ -763,12 +782,12 @@ static inline const std::vector & clip_suffix_rows(clip_suffix_type type) { static const std::vector minicpmv_row_end = { 2, 3, 1 }; static const std::vector minicpmv_last = { 2 }; switch (type) { - case CLIP_SUFFIX_NONE: return none; - case CLIP_SUFFIX_MINICPMV_OV: return minicpmv_ov; - case CLIP_SUFFIX_MINICPMV_OV_SLICE: return minicpmv_ov_slice; - case CLIP_SUFFIX_MINICPMV_SLICE: return minicpmv_slice; - case CLIP_SUFFIX_MINICPMV_ROW_END: return minicpmv_row_end; - case CLIP_SUFFIX_MINICPMV_LAST: return minicpmv_last; + case MTMD_SUFFIX_NONE: return none; + case MTMD_SUFFIX_MINICPMV_OV: return minicpmv_ov; + case MTMD_SUFFIX_MINICPMV_OV_SLICE: return minicpmv_ov_slice; + case MTMD_SUFFIX_MINICPMV_SLICE: return minicpmv_slice; + case MTMD_SUFFIX_MINICPMV_ROW_END: return minicpmv_row_end; + case MTMD_SUFFIX_MINICPMV_LAST: return minicpmv_last; default: GGML_ABORT("invalid suffix type"); } } @@ -779,7 +798,7 @@ static inline const std::vector & clip_suffix_rows(clip_suffix_type type) { // Memory layout: RGBRGB...RGBRGB... (nt times) // For audio, only one channel is used, buf.size() == nx*ny // nx will be n_frames and ny will be n_mel -struct clip_image_f32 { +struct mtmd_image_f32 { // marks the global view in e.g., DeepSeek-OCR Models bool add_viewsep = false; // appends a learned newline (or EOI) token after the image @@ -789,7 +808,7 @@ struct clip_image_f32 { // depends on the chunk position, set at tokenize time (see mtmd_tokenizer::add_media) int32_t lead_pad = 0; // separators appended after the image tokens - clip_suffix_type suffix_type = CLIP_SUFFIX_NONE; + mtmd_suffix_type suffix_type = MTMD_SUFFIX_NONE; // tile grid of the image group this entry belongs to // llava-next "anyres" (Granite4 Vision): the whole grid is encoded and assembled in a single graph @@ -806,14 +825,14 @@ struct clip_image_f32 { }; anyres_info anyres; - clip_image_size get_size() const { + mtmd_image_size get_size() const { return { nx_, ny_ }; } int nx() const { return nx_; } int ny() const { return ny_; } - void set_size(clip_image_size size, bool is_placeholder, bool is_audio) { + void set_size(mtmd_image_size size, bool is_placeholder, bool is_audio) { nx_ = size.width; ny_ = size.height; if (is_placeholder) { @@ -831,7 +850,7 @@ struct clip_image_f32 { buf = new_buf; } - void from_u8(const clip_image_u8 & img) { + void from_u8(const mtmd_image_u8 & img) { auto size = img.get_size(); nx_ = size.width; ny_ = size.height; @@ -863,7 +882,7 @@ struct clip_image_f32 { const std::vector & get_ro_buf() const { if (is_placeholder()) { - throw std::runtime_error("this clip_image_f32 is a placeholder"); + throw std::runtime_error("this mtmd_image_f32 is a placeholder"); } return buf; } @@ -889,7 +908,7 @@ struct clip_image_f32 { // token area kept after removing the padding added by the anyres resize // ref: https://github.com/huggingface/transformers/blob/v5.0.0/src/transformers/models/llava_next/modeling_llava_next.py#L109 -static inline void clip_anyres_unpad(int cur_w, int cur_h, int orig_w, int orig_h, +static inline void mtmd_anyres_unpad(int cur_w, int cur_h, int orig_w, int orig_h, int & off_x, int & off_y, int & out_w, int & out_h) { off_x = 0; off_y = 0; @@ -931,21 +950,21 @@ static inline dsv4_block_layout dsv4_get_block_layout(int n_llm_w, int n_llm_h, // logging // -static void clip_log_callback_default(enum ggml_log_level level, const char * text, void * user_data) { +static void mtmd_log_callback_default(enum ggml_log_level level, const char * text, void * user_data) { (void) level; (void) user_data; fputs(text, stderr); fflush(stderr); } -struct clip_logger_state { +struct mtmd_logger_state { ggml_log_callback log_callback; void * log_callback_user_data; }; -extern struct clip_logger_state g_logger_state; +extern struct mtmd_logger_state g_logger_state; -static void clip_log_internal_v(enum ggml_log_level level, const char * format, va_list args) { +static void mtmd_log_internal_v(enum ggml_log_level level, const char * format, va_list args) { if (format == NULL) { return; } @@ -965,31 +984,31 @@ static void clip_log_internal_v(enum ggml_log_level level, const char * format, va_end(args_copy); } -static void clip_log_internal(enum ggml_log_level level, const char * format, ...) { +static void mtmd_log_internal(enum ggml_log_level level, const char * format, ...) { va_list args; va_start(args, format); - clip_log_internal_v(level, format, args); + mtmd_log_internal_v(level, format, args); va_end(args); } -#define LOG_TRC(...) clip_log_internal(GGML_LOG_LEVEL_DEBUG, __VA_ARGS__) -#define LOG_DBG(...) clip_log_internal(GGML_LOG_LEVEL_DEBUG, __VA_ARGS__) -#define LOG_INF(...) clip_log_internal(GGML_LOG_LEVEL_INFO, __VA_ARGS__) -#define LOG_WRN(...) clip_log_internal(GGML_LOG_LEVEL_WARN, __VA_ARGS__) -#define LOG_ERR(...) clip_log_internal(GGML_LOG_LEVEL_ERROR, __VA_ARGS__) -#define LOG_CNT(...) clip_log_internal(GGML_LOG_LEVEL_CONT, __VA_ARGS__) +#define LOG_TRC(...) mtmd_log_internal(GGML_LOG_LEVEL_DEBUG, __VA_ARGS__) +#define LOG_DBG(...) mtmd_log_internal(GGML_LOG_LEVEL_DEBUG, __VA_ARGS__) +#define LOG_INF(...) mtmd_log_internal(GGML_LOG_LEVEL_INFO, __VA_ARGS__) +#define LOG_WRN(...) mtmd_log_internal(GGML_LOG_LEVEL_WARN, __VA_ARGS__) +#define LOG_ERR(...) mtmd_log_internal(GGML_LOG_LEVEL_ERROR, __VA_ARGS__) +#define LOG_CNT(...) mtmd_log_internal(GGML_LOG_LEVEL_CONT, __VA_ARGS__) // // cpp wrappers // -struct clip_image_f32_batch { - std::vector entries; +struct mtmd_image_f32_batch { + std::vector entries; bool is_audio = false; bool is_video = false; - clip_image_f32_batch clone() const { - clip_image_f32_batch new_batch{ + mtmd_image_f32_batch clone() const { + mtmd_image_f32_batch new_batch{ /* entries */ {}, /* is_audio */ is_audio, /* is_video */ is_video, @@ -1025,24 +1044,11 @@ static std::ifstream open_ifstream_binary(const std::string & fname) { } #endif +std::string mtmd_format(const char * fmt, ...); + // in test-mtmd-impl, we include woth common.h and this file, and these functions are duplicated // this is a quick fix to avoid compilation errors #ifndef DIRECTORY_SEPARATOR -static std::string string_format(const char * fmt, ...) { - va_list ap; - va_list ap2; - va_start(ap, fmt); - va_copy(ap2, ap); - int size = vsnprintf(NULL, 0, fmt, ap); - GGML_ASSERT(size >= 0 && size < INT_MAX); // NOLINT - std::vector buf(size + 1); - int size2 = vsnprintf(buf.data(), size + 1, fmt, ap2); - GGML_ASSERT(size2 == size); - va_end(ap2); - va_end(ap); - return std::string(buf.data(), buf.size()); -} - static void string_replace_all(std::string & s, const std::string & search, const std::string & replace) { if (search.empty()) { return; @@ -1104,7 +1110,7 @@ static std::string gguf_data_to_str(enum gguf_type type, const void * data, int case GGUF_TYPE_FLOAT32: return std::to_string(((const float *)data)[i]); case GGUF_TYPE_FLOAT64: return std::to_string(((const double *)data)[i]); case GGUF_TYPE_BOOL: return ((const int8_t *)data)[i] != 0 ? "true" : "false"; - default: return string_format("unknown type %d", type); + default: return mtmd_format("unknown type %d", type); } } @@ -1209,9 +1215,3 @@ static void print_tensor_data(ggml_tensor * t, uint8_t * data, int64_t n) { } } -// -// API used internally with mtmd -// - -projector_type clip_get_projector_type(const struct clip_ctx * ctx); -void clip_set_debug_output_embeddings(struct clip_ctx * ctx, bool debug); diff --git a/tools/mtmd/mtmd-internal.h b/tools/mtmd/mtmd-internal.h index e7c62773e38f..eb961d31edca 100644 --- a/tools/mtmd/mtmd-internal.h +++ b/tools/mtmd/mtmd-internal.h @@ -19,3 +19,10 @@ struct mtmd_internal_part { // [QWEN_VIDEO] merged parts are erased from `parts`, so one group always maps to one part std::vector> mtmd_group_mergeable_bitmaps(std::vector & parts, int n_merge); + +struct mtmd_image_preprocessor; + +// mtmd_get_video_config() for a given template, without a context +mtmd_video_config mtmd_video_config_init(mtmd_video_template tmpl, const mtmd_image_preprocessor * preproc, + int32_t src_width, int32_t src_height, + int32_t src_n_frames, float src_fps); diff --git a/tools/mtmd/mtmd-model.cpp b/tools/mtmd/mtmd-model.cpp new file mode 100644 index 000000000000..da74ed383340 --- /dev/null +++ b/tools/mtmd/mtmd-model.cpp @@ -0,0 +1,4372 @@ +#include "mtmd-impl.h" +#include "mtmd-model.h" +#include "models/models.h" + +#include "ggml.h" +#include "ggml-cpp.h" +#include "ggml-alloc.h" +#include "ggml-backend.h" +#include "gguf.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +// +// mtmd_model_loader +// + +struct mtmd_model_loader { + ggml_context_ptr ctx_meta; + gguf_context_ptr ctx_gguf; + + std::string fname; + + size_t model_size = 0; // in bytes + + bool has_vision = false; + bool has_audio = false; + bool has_gen_audio = false; + + mtmd_progress_callback progress_callback = nullptr; + void * progress_callback_user_data = nullptr; + + mtmd_model_loader(const char * fname, + bool skip_tensors = false, + mtmd_progress_callback progress_cb = nullptr, + void * progress_user_data = nullptr) + : fname(fname), + progress_callback(progress_cb), + progress_callback_user_data(progress_user_data) { + struct ggml_context * meta = nullptr; + + struct gguf_init_params params = { + /*.no_alloc = */ true, + /*.ctx = */ &meta, + }; + + ctx_gguf = gguf_context_ptr(gguf_init_from_file(fname, params)); + if (!ctx_gguf.get()) { + throw std::runtime_error(mtmd_format("%s: failed to load mmproj model from %s. Does this file exist?\n", __func__, fname)); + } + + ctx_meta.reset(meta); + + const int n_tensors = gguf_get_n_tensors(ctx_gguf.get()); + + // print gguf info + { + std::string name; + get_string(KEY_NAME, name, false); + std::string description; + get_string(KEY_DESCRIPTION, description, false); + LOG_INF("%s: model name: %s\n", __func__, name.c_str()); + LOG_INF("%s: description: %s\n", __func__, description.c_str()); + LOG_INF("%s: GGUF version: %d\n", __func__, gguf_get_version(ctx_gguf.get())); + LOG_INF("%s: alignment: %zu\n", __func__, gguf_get_alignment(ctx_gguf.get())); + LOG_INF("%s: n_tensors: %d\n", __func__, n_tensors); + LOG_INF("%s: n_kv: %d\n", __func__, (int)gguf_get_n_kv(ctx_gguf.get())); + LOG_INF("\n"); + } + + // modalities + { + get_bool(KEY_HAS_VISION_ENC, has_vision, false); + get_bool(KEY_HAS_AUDIO_ENC, has_audio, false); + get_bool(KEY_HAS_GEN_AUDIO_ENC, has_gen_audio, false); + + if (has_vision) { + LOG_INF("%s: has vision encoder\n", __func__); + } + if (has_audio) { + LOG_INF("%s: has audio encoder\n", __func__); + } + if (has_gen_audio) { + LOG_INF("%s: has audio generation (gen) encoder\n", __func__); + } + } + + // tensors + if (!skip_tensors) { + for (int i = 0; i < n_tensors; ++i) { + const char * name = gguf_get_tensor_name(ctx_gguf.get(), i); + const size_t offset = gguf_get_tensor_offset(ctx_gguf.get(), i); + enum ggml_type type = gguf_get_tensor_type(ctx_gguf.get(), i); + ggml_tensor * cur = ggml_get_tensor(meta, name); + size_t tensor_size = ggml_nbytes(cur); + model_size += tensor_size; + LOG_DBG("%s: tensor[%d]: n_dims = %d, name = %s, tensor_size=%zu, offset=%zu, shape:[%" PRIu64 ", %" PRIu64 ", %" PRIu64 ", %" PRIu64 "], type = %s\n", + __func__, i, ggml_n_dims(cur), cur->name, tensor_size, offset, cur->ne[0], cur->ne[1], cur->ne[2], cur->ne[3], ggml_type_name(type)); + } + } + } + + void load_hparams(mtmd_model & model, mtmd_modality modality) { + auto & hparams = model.hparams; + std::string log_ffn_op; // for logging + + // sanity check + if (modality == MTMD_MODALITY_VISION) { + GGML_ASSERT(has_vision); + } else if (modality == MTMD_MODALITY_AUDIO) { + GGML_ASSERT(has_audio); + } else if (modality == MTMD_MODALITY_GEN_AUDIO) { + GGML_ASSERT(has_gen_audio); + } + model.modality = modality; + + + // projector type + std::string proj_type; + { + // default key + get_string(KEY_PROJ_TYPE, proj_type, false); + + // for models with mixed modalities + if (proj_type.empty()) { + if (modality == MTMD_MODALITY_VISION) { + get_string(KEY_VISION_PROJ_TYPE, proj_type, false); + } else if (modality == MTMD_MODALITY_AUDIO) { + get_string(KEY_AUDIO_PROJ_TYPE, proj_type, false); + } else if (modality == MTMD_MODALITY_GEN_AUDIO) { + get_string(KEY_GEN_AUDIO_PROJ_TYPE, proj_type, false); + } else { + GGML_ABORT("unknown modality"); + } + } + + model.proj_type = projector_type_from_string(proj_type); + + if (model.proj_type == PROJECTOR_TYPE_UNKNOWN) { + throw std::runtime_error(mtmd_format("%s: unknown projector type: %s\n", __func__, proj_type.c_str())); + } + + // correct arch for multimodal models (legacy method) + if (model.proj_type == PROJECTOR_TYPE_QWEN25O) { + model.proj_type = modality == MTMD_MODALITY_VISION + ? PROJECTOR_TYPE_QWEN25VL + : PROJECTOR_TYPE_QWEN2A; + } + } + + const bool is_vision = model.modality == MTMD_MODALITY_VISION; + const bool is_audio = model.modality == MTMD_MODALITY_AUDIO; + const bool is_gen_audio = model.modality == MTMD_MODALITY_GEN_AUDIO; + + // other hparams + { + const char * prefix = is_vision ? "vision" : (is_audio ? "audio" : "gen.audio"); + get_u32(mtmd_format(KEY_N_EMBD, prefix), hparams.n_embd); + get_u32(mtmd_format(KEY_N_HEAD, prefix), hparams.n_head); + get_u32(mtmd_format(KEY_N_EMBD_HEAD, prefix), hparams.n_embd_head, false); + get_u32(mtmd_format(KEY_N_FF, prefix), hparams.n_ff); + get_u32(mtmd_format(KEY_N_BLOCK, prefix), hparams.n_layer); + get_u32(mtmd_format(KEY_PROJ_DIM, prefix), hparams.projection_dim); + get_f32(mtmd_format(KEY_LAYER_NORM_EPS, prefix), hparams.eps); + + // n_head_kv is optional (for GQA), default to n_head + hparams.n_head_kv = hparams.n_head; + get_u32(mtmd_format(KEY_N_HEAD_KV, prefix), hparams.n_head_kv, false); + + if (is_vision) { + get_u32(KEY_IMAGE_SIZE, hparams.image_size); + get_u32(KEY_PATCH_SIZE, hparams.patch_size); + get_u32(KEY_MAX_SLICE_NUMS, hparams.max_slice_nums, false); + get_i32(KEY_MINICPMV_VERSION, hparams.minicpmv_version, false); // legacy + get_u32(KEY_MINICPMV_QUERY_NUM, hparams.minicpmv_query_num, false); + if (hparams.minicpmv_query_num == 0) { + // Fallback to hardcoded values for legacy models + if (hparams.minicpmv_version == 3) { + hparams.minicpmv_query_num = 64; + } else if (hparams.minicpmv_version == 4) { + hparams.minicpmv_query_num = 64; + } else if (hparams.minicpmv_version == 5) { + hparams.minicpmv_query_num = 64; + } else if (hparams.minicpmv_version == 6) { + hparams.minicpmv_query_num = 64; + } else if (hparams.minicpmv_version == 100045) { + hparams.minicpmv_query_num = 64; + } else { + hparams.minicpmv_query_num = 96; + } + } + } else if (is_audio) { + get_u32(KEY_A_NUM_MEL_BINS, hparams.n_mel_bins); + // some hparams are unused, but still need to set to avoid issues + hparams.image_size = 0; + hparams.patch_size = 1; + + } else if (is_gen_audio) { + // these are unused, but still need to be set to avoid issues + hparams.image_size = 0; + hparams.patch_size = 1; + get_string(KEY_GEN_AUDIO_VARIANT, hparams.gen_model_variant, false); + + } else { + GGML_ASSERT(false && "unknown modality"); + } + + // for pinpoints, we need to convert it into a list of resolution candidates + { + std::vector pinpoints; + get_arr_int(KEY_IMAGE_GRID_PINPOINTS, pinpoints, false); + if (pinpoints.size() % 2 != 0) { + throw std::runtime_error(mtmd_format("%s: image_grid_pinpoints must have an even number of elements, got %zu\n", __func__, pinpoints.size())); + } + if (!pinpoints.empty()) { + for (size_t i = 0; i < pinpoints.size(); i += 2) { + hparams.image_res_candidates.push_back({ + pinpoints[i], + pinpoints[i+1], + }); + } + } + } + + // default warmup value + hparams.warmup_image_size = hparams.image_size; + + { + bool use_gelu = false; + bool use_silu = false; + get_bool(KEY_USE_GELU, use_gelu, false); + get_bool(KEY_USE_SILU, use_silu, false); + if (use_gelu && use_silu) { + throw std::runtime_error(mtmd_format("%s: both use_gelu and use_silu are set to true\n", __func__)); + } + if (use_gelu) { + hparams.ffn_op = FFN_GELU; + log_ffn_op = "gelu"; + } else if (use_silu) { + hparams.ffn_op = FFN_SILU; + log_ffn_op = "silu"; + } else { + hparams.ffn_op = FFN_GELU_QUICK; + log_ffn_op = "gelu_quick"; + } + } + + { + std::string mm_patch_merge_type; + get_string(KEY_MM_PATCH_MERGE_TYPE, mm_patch_merge_type, false); + if (mm_patch_merge_type == "spatial_unpad") { + hparams.mm_patch_merge_type = PATCH_MERGE_SPATIAL_UNPAD; + } + } + + if (is_vision) { + std::vector image_mean; + std::vector image_std; + get_arr_f32(KEY_IMAGE_MEAN, image_mean, false); + get_arr_f32(KEY_IMAGE_STD , image_std, false); + if (image_mean.size() < 3 || image_std.size() < 3) { + throw std::runtime_error(mtmd_format("%s: image_mean/image_std arrays must have at least 3 elements, got %zu and %zu\n", __func__, image_mean.size(), image_std.size())); + } + for (int i = 0; i < 3; ++i) { + hparams.image_mean[i] = image_mean[i]; + hparams.image_std[i] = image_std[i]; + } + } + + // Load the vision/audio feature layer indices if they are explicitly provided + // NOTE: gguf conversions should standardize the values of the vision feature layer to be non-negative, since we use -1 to mark values as unset here. + get_arr_int(mtmd_format(KEY_FEATURE_LAYERS, prefix), hparams.feature_layers, false); + for (const auto & v : hparams.feature_layers) { + if (v > (int) hparams.n_layer) { + throw std::runtime_error(mtmd_format("%s: feature layer index %d is out of range (n_layer: %d)", + __func__, v, hparams.n_layer)); + } + } + + // model-specific params + switch (model.proj_type) { + case PROJECTOR_TYPE_MLP: + case PROJECTOR_TYPE_MLP_NORM: + case PROJECTOR_TYPE_LDP: + case PROJECTOR_TYPE_LDPV2: + case PROJECTOR_TYPE_COGVLM: + { + hparams.has_llava_projector = model.proj_type != PROJECTOR_TYPE_COGVLM; + hparams.image_pad_color = {122, 116, 104}; + if (!hparams.image_res_candidates.empty()) { + hparams.image_resize_pad = PAD_CEIL; + hparams.image_resize_algo = RESIZE_ALGO_BICUBIC; + } else { + // llava-1.6 default params + hparams.image_pad_ov = PAD_NONE; + hparams.image_pad_rf = PAD_CEIL; + hparams.image_pad_color_rf = {122, 116, 104}; + } + } break; + case PROJECTOR_TYPE_GLM_EDGE: + { + hparams.image_resize_pad = PAD_CEIL; + hparams.image_resize_algo = RESIZE_ALGO_BICUBIC; + } break; + case PROJECTOR_TYPE_MINICPMV: + { + // use default llava-uhd preprocessing params + if (hparams.minicpmv_version == 0) { + hparams.minicpmv_version = 2; // default to 2 if not set + } + } break; + case PROJECTOR_TYPE_MINICPMV4_6: + case PROJECTOR_TYPE_MINICPMV4_7: + case PROJECTOR_TYPE_MINICPM_OCR_1_0: + { + // MiniCPM-V 4.6/4.7 unified merger projector + // ViT merger 2x2 + final merger 2x2 = 4x spatial merge per dimension + hparams.n_merge = 4; + get_u32(KEY_PROJ_SCALE_FACTOR, hparams.n_merge, false); + GGML_ASSERT(hparams.n_merge == 2 || hparams.n_merge == 4); + + // no padding: the reference stretches the refined image to the target size + hparams.image_pad_ov = PAD_NONE; + hparams.image_pad_rf = PAD_NONE; + + // borrow wa_layer_indexes for vit_merger insertion point + std::vector wa_layer_indexes_vec; + get_arr_int(KEY_WIN_ATTN_LAYER_INDEXES, wa_layer_indexes_vec, false); + if (!wa_layer_indexes_vec.empty()) { + const int insert_lid = wa_layer_indexes_vec[0]; + if (insert_lid < 0 || insert_lid >= (int) hparams.n_layer) { + throw std::runtime_error(mtmd_format("%s: layer index %d is out of range (n_layer: %d)", + __func__, insert_lid, hparams.n_layer)); + } + hparams.insert_layer_id = insert_lid; + } + } break; + case PROJECTOR_TYPE_INTERNVL: + { + // use default llava-uhd preprocessing params + // older version of internvl doesn't have min/max tiles, we need to provide default values for them to avoid issues + hparams.preproc_min_tiles = 1; + hparams.preproc_max_tiles = 12; + get_u32(KEY_PROJ_SCALE_FACTOR, hparams.n_merge, false); + get_u32(KEY_PREPROC_MIN_TILES, hparams.preproc_min_tiles, false); + get_u32(KEY_PREPROC_MAX_TILES, hparams.preproc_max_tiles, false); + GGML_ASSERT(hparams.preproc_min_tiles <= hparams.preproc_max_tiles && hparams.preproc_max_tiles < INT32_MAX); + set_internvl_dhr_res_candidates(model); + } break; + case PROJECTOR_TYPE_NEMOTRON_V2_VL: + { + get_u32(KEY_PROJ_SCALE_FACTOR, hparams.n_merge, false); + } break; + case PROJECTOR_TYPE_NEMOTRON_OMNI: + { + hparams.ffn_op = FFN_GELU_ERF; + log_ffn_op = "gelu_erf"; + hparams.image_resize_pad = PAD_NONE; + get_u32(KEY_PROJ_SCALE_FACTOR, hparams.n_merge); + get_u32(KEY_IMAGE_MIN_PIXELS, hparams.image_min_pixels); + get_u32(KEY_IMAGE_MAX_PIXELS, hparams.image_max_pixels); + hparams.warmup_image_size = hparams.image_size; + } break; + case PROJECTOR_TYPE_PARAKEET: + { + get_u32(KEY_AUDIO_SUBSMPL_FACTOR, hparams.subsampling_factor); + GGML_ASSERT(hparams.subsampling_factor == 8 && + "subsampling_factor must match the conv strides in mtmd_graph_parakeet::build()"); + get_u32(KEY_A_CONV_KERNEL_SIZE, hparams.audio_conv_kernel_size); + GGML_ASSERT(hparams.audio_conv_kernel_size > 0 && hparams.audio_conv_kernel_size % 2 == 1 && + "audio_conv_kernel_size must be a positive odd integer"); + hparams.audio_chunk_len = 0; + hparams.audio_sample_rate = 16000; + hparams.audio_n_fft = 512; + hparams.audio_window_len = 400; + hparams.audio_hop_len = 160; + } break; + case PROJECTOR_TYPE_NEMOTRON_TRANSCRIBE: + { + get_u32(KEY_AUDIO_SUBSMPL_FACTOR, hparams.subsampling_factor); + get_u32(KEY_A_ROPE_DIMS, hparams.audio_rope_dims); + get_u32(KEY_A_SPK_N_BLOCK, hparams.spk_n_layer); + get_u32(KEY_A_SPK_N_HEAD, hparams.spk_n_head); + if (hparams.subsampling_factor != 8) { + throw std::runtime_error(mtmd_format("%s: unsupported subsampling factor %d\n", __func__, hparams.subsampling_factor)); + } + if (hparams.spk_n_layer <= 0 || hparams.spk_n_layer > (int) gguf_get_n_tensors(ctx_gguf.get())) { + throw std::runtime_error(mtmd_format("%s: invalid speaker block count %d\n", __func__, hparams.spk_n_layer)); + } + hparams.audio_chunk_len = 0; + hparams.audio_sample_rate = 16000; + hparams.audio_n_fft = 512; + hparams.audio_window_len = 400; + hparams.audio_hop_len = 160; + // one full window + using nt = mtmd_graph_nemotron_transcribe; + hparams.warmup_audio_size = (nt::window_left + nt::window_core + nt::window_right) * hparams.subsampling_factor; + } break; + case PROJECTOR_TYPE_IDEFICS3: + { + // use default llava-uhd preprocessing params + hparams.image_resize_algo = RESIZE_ALGO_LANCZOS; + get_u32(KEY_PROJ_SCALE_FACTOR, hparams.n_merge, false); + get_u32(KEY_PREPROC_IMAGE_SIZE, hparams.image_longest_edge, false); + hparams.set_limit_image_tokens(); + } break; + case PROJECTOR_TYPE_COHERE2V: + { + hparams.image_pad_rf = PAD_NONE; + get_u32(KEY_PROJ_SCALE_FACTOR, hparams.n_merge); + get_u32(KEY_PREPROC_MAX_TILES, hparams.preproc_max_tiles); + if (hparams.preproc_max_tiles <= 0 || hparams.preproc_max_tiles > 256) { + throw std::runtime_error(mtmd_format("%s: preproc_max_tiles (%d) must be in range [1, 256]\n", __func__, hparams.preproc_max_tiles)); + } + } break; + case PROJECTOR_TYPE_LFM2: + case PROJECTOR_TYPE_D1OMNI_V: + { + // default for older GGUFs + std::string resize_algo = "bilinear"; + get_string(KEY_IMAGE_RESIZE_ALGO, resize_algo, false); + // Lfm2VlImageProcessor and d1-omni vision.py resize bilinear and bicubic with torchvision + if (resize_algo == "bilinear") { + hparams.set_image_resize_algo(RESIZE_ALGO_BILINEAR, true); + } else if (resize_algo == "bicubic") { + hparams.set_image_resize_algo(RESIZE_ALGO_BICUBIC, true); + } else if (resize_algo == "lanczos") { + hparams.set_image_resize_algo(RESIZE_ALGO_LANCZOS); + } else { + throw std::runtime_error(mtmd_format("%s: unsupported image resize algo: %s\n", __func__, resize_algo.c_str())); + } + hparams.image_resize_algo_rf = hparams.image_resize_algo; + hparams.image_resize_algo_ov = hparams.image_resize_algo; + // the tiles stretch the image to the grid (Lfm2VlImageProcessor, d1-omni vision.py) + hparams.image_pad_rf = PAD_NONE; + get_u32(KEY_PROJ_SCALE_FACTOR, hparams.n_merge, false); + // defaults for GGUFs converted before tiling params were written + // ref: https://huggingface.co/LiquidAI/LFM2.5-VL-1.6B/blob/main/processor_config.json + hparams.preproc_min_tiles = 2; + hparams.preproc_max_tiles = 10; + hparams.preproc_tile_size = 512; + hparams.preproc_max_pixels_tolerance = 2.0f; + get_u32(KEY_PREPROC_MIN_TILES, hparams.preproc_min_tiles, false); + get_u32(KEY_PREPROC_MAX_TILES, hparams.preproc_max_tiles, false); + get_u32(KEY_PREPROC_TILE_SIZE, hparams.preproc_tile_size, false); + get_f32(KEY_PREPROC_MAX_PIXELS_TOLERANCE, hparams.preproc_max_pixels_tolerance, false); + GGML_ASSERT(hparams.preproc_min_tiles >= 1 && hparams.preproc_min_tiles <= hparams.preproc_max_tiles); + hparams.set_limit_image_tokens(64, 256); + } break; + case PROJECTOR_TYPE_PHI4: + { + hparams.n_merge = 1; + hparams.image_resize_algo = RESIZE_ALGO_BILINEAR; + get_u32(KEY_IMAGE_MIN_PIXELS, hparams.image_min_pixels); + get_u32(KEY_IMAGE_MAX_PIXELS, hparams.image_max_pixels); + hparams.set_warmup_n_tokens(16*16); + } break; + case PROJECTOR_TYPE_PIXTRAL: + { + // ref: https://huggingface.co/mistral-community/pixtral-12b/blob/main/preprocessor_config.json + // TODO: verify the image_min_tokens + hparams.n_merge = 1; // the original pixtral does not use patch merging + hparams.image_resize_algo = RESIZE_ALGO_BICUBIC; + hparams.rope_theta = 10000.0f; + get_u32(KEY_SPATIAL_MERGE_SIZE, hparams.n_merge, false); + get_u32(KEY_PREPROC_IMAGE_SIZE, hparams.image_longest_edge, false); + if (hparams.image_longest_edge > 0) { + const int64_t patch_size = int64_t(hparams.patch_size) * hparams.n_merge; + if (patch_size <= 0) { + throw std::runtime_error("Pixtral merged patch size must be positive"); + } + const int64_t n_tokens_per_side = + (hparams.image_longest_edge + patch_size - 1) / patch_size; + if (n_tokens_per_side * patch_size > std::sqrt(std::numeric_limits::max())) { + throw std::runtime_error("Pixtral preprocessing image size is too large"); + } + hparams.image_resize_pad = PAD_NONE; + hparams.image_floor_before_align = true; + hparams.set_limit_image_tokens(0, n_tokens_per_side * n_tokens_per_side); + } else { + hparams.set_limit_image_tokens(8, 1024); + } + hparams.set_warmup_n_tokens(256); // avoid OOM on warmup + } break; + case PROJECTOR_TYPE_LIGHTONOCR: + { + hparams.n_merge = 1; + hparams.image_resize_algo = RESIZE_ALGO_BICUBIC; + hparams.rope_theta = 10000.0f; + get_u32(KEY_SPATIAL_MERGE_SIZE, hparams.n_merge, false); + hparams.image_longest_edge = hparams.image_size; + get_u32(KEY_PREPROC_IMAGE_SIZE, hparams.image_longest_edge, false); + hparams.set_limit_image_tokens(); + hparams.set_warmup_n_tokens(256); // avoid OOM on warmup + } break; + case PROJECTOR_TYPE_DOTS_OCR: + { + hparams.rope_theta = 10000.0f; + hparams.image_resize_pad = PAD_NONE; // the HF processor resizes to the smart_resize size, no padding + hparams.image_round_half_even = true; // smart_resize uses Python round() + get_u32(KEY_PROJ_SCALE_FACTOR, hparams.n_merge); + get_u32(KEY_IMAGE_MIN_PIXELS, hparams.image_min_pixels); + get_u32(KEY_IMAGE_MAX_PIXELS, hparams.image_max_pixels); + hparams.set_warmup_n_tokens(46*46); // avoid OOM on warmup + } break; + case PROJECTOR_TYPE_DOTS3NOTE_V: + { + hparams.rope_theta = 10000.0f; + hparams.image_resize_algo = RESIZE_ALGO_BICUBIC; + get_u32(KEY_SPATIAL_MERGE_SIZE, hparams.n_merge); + get_u32(KEY_IMAGE_MIN_PIXELS, hparams.image_min_pixels); + get_u32(KEY_IMAGE_MAX_PIXELS, hparams.image_max_pixels); + get_u32(KEY_VISION_N_EXPERT_USED, hparams.n_expert_used); + hparams.set_warmup_n_tokens(46*46); // avoid OOM on warmup + } break; + case PROJECTOR_TYPE_DOTS3NOTE_A: + { + hparams.rope_theta = 10000.0f; + hparams.audio_chunk_len = 60; // in seconds + hparams.audio_sample_rate = 16000; + hparams.audio_n_fft = 400; + hparams.audio_window_len = 400; + hparams.audio_hop_len = 160; + } break; + case PROJECTOR_TYPE_KIMIVL: + { + hparams.image_resize_algo = RESIZE_ALGO_BICUBIC; + hparams.rope_theta = 10000.0f; + get_u32(KEY_PROJ_SCALE_FACTOR, hparams.n_merge, false); + // TODO: check kimivl preprocessor for exact values + hparams.set_limit_image_tokens(8, 1024); + hparams.set_warmup_n_tokens(256); // avoid OOM on warmup + } break; + case PROJECTOR_TYPE_KIMIK25: + case PROJECTOR_TYPE_KIMIK31: + { + hparams.image_resize_algo = RESIZE_ALGO_BICUBIC; + hparams.rope_theta = 10000.0f; + get_u32(KEY_PROJ_SCALE_FACTOR, hparams.n_merge, false); + + int min_pixels = 0, max_pixels = 0; + get_u32(KEY_IMAGE_MIN_PIXELS, min_pixels, false); + get_u32(KEY_IMAGE_MAX_PIXELS, max_pixels, false); + if (min_pixels > 0 && max_pixels > 0) { + hparams.image_min_pixels = min_pixels; + hparams.image_max_pixels = max_pixels; + hparams.warmup_image_size = static_cast(std::sqrt(max_pixels)); + } else { + hparams.set_limit_image_tokens(2, 4096); + } + if (model.proj_type == PROJECTOR_TYPE_KIMIK31) { + hparams.set_warmup_n_tokens(46*46); // avoid OOM on warmup + } + } break; + case PROJECTOR_TYPE_DEEPSEEK4V: + { + hparams.image_resize_algo = RESIZE_ALGO_BICUBIC; + hparams.image_pad_color = {127, 127, 127}; + hparams.rope_theta = 10000.0f; + get_u32(KEY_PROJ_SCALE_FACTOR, hparams.n_merge); + get_u32(KEY_IMAGE_MIN_PIXELS, hparams.image_min_pixels); + hparams.dsv4_max_n_token = 384; + hparams.dsv4_max_wh_ratio = 8; + const int patch_area = hparams.patch_size * hparams.patch_size * hparams.n_merge * hparams.n_merge; + // handle min/max token counts from CLI + if (hparams.custom_image_min_tokens > 0) { + hparams.image_min_pixels = hparams.custom_image_min_tokens * patch_area; + } + if (hparams.custom_image_max_tokens > 0) { + // the cap is on the whole token block, keep some room for the resize solver + hparams.dsv4_max_n_token = std::max(hparams.custom_image_max_tokens, 16); + } + hparams.image_max_pixels = hparams.dsv4_max_n_token * patch_area; + // a small custom max token count also lowers the min-pixel upscale threshold + hparams.image_min_pixels = std::min(hparams.image_min_pixels, hparams.image_max_pixels); + // avoid OOM on warmup + const int warmup_side = (int) std::sqrt((double) std::min(256, hparams.dsv4_max_n_token)); + hparams.set_warmup_n_tokens(warmup_side * warmup_side); + } break; + case PROJECTOR_TYPE_DEEPSEEK41V: + { + // ref: load_image in inference/image_processor.py (V4.1), PIL ImageOps.pad on gray 127 + hparams.image_resize_algo = RESIZE_ALGO_BICUBIC; + hparams.image_pad_color = {127, 127, 127}; + hparams.rope_theta = 10000.0f; + get_u32(KEY_PROJ_SCALE_FACTOR, hparams.n_merge); + get_u32(KEY_IMAGE_MIN_PIXELS, hparams.image_min_pixels); + // vision_max_n_token = 1024, vision_max_wh_ratio = null (converter asserts both) + hparams.dsv4_max_n_token = 1024; + hparams.dsv4_max_wh_ratio = 0; + const int patch_area = hparams.patch_size * hparams.patch_size * hparams.n_merge * hparams.n_merge; + if (hparams.custom_image_min_tokens > 0) { + hparams.image_min_pixels = hparams.custom_image_min_tokens * patch_area; + } + if (hparams.custom_image_max_tokens > 0) { + // the cap is on the whole token block (newlines + start/end included) + hparams.dsv4_max_n_token = std::max(hparams.custom_image_max_tokens, 16); + } + hparams.image_max_pixels = hparams.dsv4_max_n_token * patch_area; + hparams.image_min_pixels = std::min(hparams.image_min_pixels, hparams.image_max_pixels); + const int warmup_side = (int) std::sqrt((double) std::min(256, hparams.dsv4_max_n_token)); + hparams.set_warmup_n_tokens(warmup_side * warmup_side); + } break; + case PROJECTOR_TYPE_GEMMA3: + { + // default value (used by all model sizes in gemma 3 family) + // number of patches for each **side** is reduced by a factor of 4 + hparams.n_merge = 4; + hparams.image_resize_algo = RESIZE_ALGO_BILINEAR; + // test model (tinygemma3) has a different value, we optionally read it + get_u32(KEY_PROJ_SCALE_FACTOR, hparams.n_merge, false); + } break; + + case PROJECTOR_TYPE_GEMMA4V: + case PROJECTOR_TYPE_GEMMA4UV: + { + hparams.rope_theta = 100.0f; + hparams.ffn_op = FFN_GELU; + log_ffn_op = "gelu"; + hparams.n_merge = 3; // pooling_kernel_size + hparams.image_resize_algo = RESIZE_ALGO_BICUBIC; + hparams.image_resize_pad = PAD_NONE; + get_u32(KEY_PROJ_SCALE_FACTOR, hparams.n_merge, false); + if (model.proj_type == PROJECTOR_TYPE_GEMMA4UV) { + // for "unified" variant, we directly use a bigger patch size, because the "token merging" is done directly on conv layer + hparams.patch_size = hparams.patch_size * hparams.n_merge; + hparams.n_merge = 1; + } + hparams.set_limit_image_tokens(70, 1120); + hparams.set_warmup_n_tokens(256); // avoid OOM on warmup + } break; + + case PROJECTOR_TYPE_GEMMA3NV: + { + // Gemma3n uses MobileNetV5 which produces 256 tokens (16x16) + // Similar configuration to Gemma3 + hparams.n_merge = 1; // MobileNetV5 handles resizing internally + hparams.image_resize_algo = RESIZE_ALGO_BILINEAR; + get_u32(KEY_PROJ_SCALE_FACTOR, hparams.n_merge, false); + } break; + case PROJECTOR_TYPE_QWEN2VL: + case PROJECTOR_TYPE_QWEN25VL: + case PROJECTOR_TYPE_QWEN3VL: + case PROJECTOR_TYPE_LING3VL: + { + hparams.n_merge = 2; // default value for Qwen 2 and 2.5 + hparams.image_resize_algo = RESIZE_ALGO_BICUBIC; + hparams.image_resize_pad = PAD_NONE; // the HF processor resizes to the smart_resize size, no padding + hparams.image_round_half_even = true; // smart_resize uses Python round() + get_u32(KEY_SPATIAL_MERGE_SIZE, hparams.n_merge, false); + if (hparams.n_merge != 2) { + // the temporal merge input and the merger use a 2x2 patch grid + throw std::runtime_error(mtmd_format("%s: spatial_merge_size (%d) must be 2\n", __func__, hparams.n_merge)); + } + get_u32(KEY_WIN_ATTN_PATTERN, hparams.n_wa_pattern, model.proj_type == PROJECTOR_TYPE_QWEN25VL); // only 2.5 requires it + // ref: https://huggingface.co/Qwen/Qwen2.5-VL-7B-Instruct/blob/main/preprocessor_config.json + hparams.set_limit_image_tokens(8, 4096); + // optional limits of the model, the custom values take precedence + if (hparams.custom_image_min_tokens <= 0) { + get_u32(KEY_IMAGE_MIN_PIXELS, hparams.image_min_pixels, false); + } + if (hparams.custom_image_max_tokens <= 0) { + get_u32(KEY_IMAGE_MAX_PIXELS, hparams.image_max_pixels, false); + } + hparams.set_warmup_n_tokens(46*46); // avoid OOM on warmup + const int warn_min_pixels = 1024 * hparams.n_merge * hparams.n_merge * hparams.patch_size * hparams.patch_size; + if (hparams.image_min_pixels < warn_min_pixels) { + LOG_WRN("%s: Qwen-VL models require at minimum 1024 image tokens to function correctly on grounding tasks\n", __func__); + LOG_WRN("%s: if you encounter problems with accuracy, try adding --image-min-tokens 1024\n", __func__); + LOG_WRN("%s: more info: https://github.com/ggml-org/llama.cpp/issues/16842\n\n", __func__); + } + } break; + case PROJECTOR_TYPE_MAGEVL: + { + hparams.ffn_op = FFN_GELU_ERF; + log_ffn_op = "gelu_erf"; + hparams.image_resize_algo = RESIZE_ALGO_BICUBIC; + hparams.image_resize_pad = PAD_NONE; // the HF processor resizes to the smart_resize size, no padding + hparams.image_round_half_even = true; // smart_resize uses Python round() + get_u32(KEY_SPATIAL_MERGE_SIZE, hparams.n_merge); + get_u32(KEY_IMAGE_MIN_PIXELS, hparams.image_min_pixels); + get_u32(KEY_IMAGE_MAX_PIXELS, hparams.image_max_pixels); + hparams.set_warmup_n_tokens(46*46); // avoid OOM on warmup + } break; + case PROJECTOR_TYPE_SENSENOVA_U1: + { + hparams.n_merge = 2; + hparams.image_resize_algo = RESIZE_ALGO_BICUBIC; + get_u32(KEY_SPATIAL_MERGE_SIZE, hparams.n_merge, false); + get_u32(KEY_IMAGE_MIN_PIXELS, hparams.image_min_pixels); + get_u32(KEY_IMAGE_MAX_PIXELS, hparams.image_max_pixels); + hparams.rope_theta = 10000.0f; + hparams.set_limit_image_tokens(16, 4096); + if (hparams.n_merge != 2) { + throw std::runtime_error(mtmd_format("%s: SenseNova U1 requires spatial_merge_size=2, got %u", __func__, hparams.n_merge)); + } + hparams.set_warmup_n_tokens(16*16); + } break; + case PROJECTOR_TYPE_LFM3VL: + { + std::string resize_algo; + get_string(KEY_IMAGE_RESIZE_ALGO, resize_algo); + // Lfm3VlImageProcessor resizes with torchvision + if (resize_algo == "bilinear") { + hparams.set_image_resize_algo(RESIZE_ALGO_BILINEAR, true); + } else if (resize_algo == "bicubic") { + hparams.set_image_resize_algo(RESIZE_ALGO_BICUBIC, true); + } else { + throw std::runtime_error(mtmd_format("%s: unsupported image resize algo: %s\n", __func__, resize_algo.c_str())); + } + // smart_resize stretches to the aligned size, no letterbox + hparams.image_resize_pad = PAD_NONE; + get_u32(KEY_SPATIAL_MERGE_SIZE, hparams.n_merge); + // token counts and the projector graph assume a 2x2 merge + if (hparams.n_merge != 2) { + throw std::runtime_error(mtmd_format("%s: unsupported spatial_merge_size %d for lfm3vl, only 2 is supported\n", __func__, hparams.n_merge)); + } + get_u32(mtmd_format(KEY_N_HEAD_KV, "vision"), hparams.n_head_kv); + hparams.rope_theta = 1000000.0f; // vision_rope_theta + // HF processor size; the max is also capped at 4096 tokens as an OOM guard + int min_pixels = 0; + int max_pixels = 0; + get_u32(KEY_IMAGE_MIN_PIXELS, min_pixels); + get_u32(KEY_IMAGE_MAX_PIXELS, max_pixels); + if (min_pixels <= 0 || max_pixels < min_pixels) { + throw std::runtime_error(mtmd_format("%s: invalid image min/max pixels %d/%d\n", __func__, min_pixels, max_pixels)); + } + const int patch_area = hparams.patch_size * hparams.patch_size * hparams.n_merge * hparams.n_merge; + hparams.set_limit_image_tokens(min_pixels / patch_area, std::min(4096, max_pixels / patch_area)); + hparams.set_warmup_n_tokens(46*46); // avoid OOM on warmup + } break; + case PROJECTOR_TYPE_MINIMAX_M3: + { + hparams.n_merge = 2; // spatial_merge_size + hparams.image_resize_algo = RESIZE_ALGO_BICUBIC; + hparams.image_resize_pad = PAD_NONE; + get_u32(KEY_SPATIAL_MERGE_SIZE, hparams.n_merge, false); + // n_merge is used as a divisor in mtmd_subcontext::encode() + // (gh / n_merge); reject 0 to avoid int div-by-zero (DoS). + GGML_ASSERT(hparams.n_merge > 0); + hparams.rope_theta = 10000.0f; // vision_config.rope_theta + // MiniMax-M3: max_pixels 451584 (=672^2) -> 576 merged tokens (image_seq_length) + hparams.set_limit_image_tokens(8, 576); + hparams.set_warmup_n_tokens(16*16); + } break; + case PROJECTOR_TYPE_MUSE_SPARK_AUDIO: + { + hparams.audio_sample_rate = 24000; // mono waveform, no frontend + get_u32(KEY_A_CONV_KERNEL_SIZE, hparams.audio_conv_kernel_size, true); + get_arr_int(KEY_A_HIDDEN_SIZES, hparams.audio_hidden_sizes, true); + get_arr_int(KEY_A_PATCH_SIZES, hparams.audio_patch_sizes, true); + get_arr_int(KEY_A_DEPTHS, hparams.audio_depths, true); + + // the graph indexes the three arrays by stage and model.layers by the running depth sum + const size_t n_stage = hparams.audio_patch_sizes.size(); + GGML_ASSERT(n_stage > 0); + GGML_ASSERT(hparams.audio_hidden_sizes.size() == n_stage); + GGML_ASSERT(hparams.audio_depths.size() == n_stage); + GGML_ASSERT(hparams.n_mel_bins == 1); + GGML_ASSERT(hparams.audio_conv_kernel_size >= 1); + + int64_t n_blocks = 0; + for (size_t i = 0; i < n_stage; ++i) { + GGML_ASSERT(hparams.audio_hidden_sizes[i] >= 1); + GGML_ASSERT(hparams.audio_depths[i] >= 0); + n_blocks += hparams.audio_depths[i]; + } + if (n_blocks != hparams.n_layer) { + throw std::runtime_error(mtmd_format("%s: sum of audio depths (%" PRId64 ") does not match n_layer (%d)\n", __func__, n_blocks, hparams.n_layer)); + } + hparams.warmup_audio_size = hparams.audio_samples_per_token(); + } break; + case PROJECTOR_TYPE_MUSE_GLIMMER: + { + hparams.n_merge = 2; // pixel-shuffle downsample after the ViT + hparams.image_resize_algo = RESIZE_ALGO_LANCZOS; + hparams.rope_theta = 10000.0f; + hparams.muse_glimmer_patch_temporal = 2; + hparams.muse_glimmer_sparse_factor = 4; // 3 sparse layers + 1 global, repeating + get_u32(KEY_SPATIAL_MERGE_SIZE, hparams.n_merge, false); + hparams.set_limit_image_tokens(1, 4096); + hparams.set_warmup_n_tokens(32*32); + } break; + case PROJECTOR_TYPE_MIMOVL: + { + hparams.n_merge = 2; // spatial_merge_size + hparams.image_resize_algo = RESIZE_ALGO_BICUBIC; + hparams.image_resize_pad = PAD_NONE; // the HF processor resizes to the smart_resize size, no padding + hparams.image_round_half_even = true; // smart_resize uses Python round() + get_u32(KEY_SPATIAL_MERGE_SIZE, hparams.n_merge, false); + get_u32(mtmd_format(KEY_N_HEAD_KV, "vision"), hparams.n_head_kv); + // 1D banded sliding-window radius (visual_token_window_size); required + get_u32(KEY_ATTN_WINDOW_SIZE, hparams.attn_window_size); + std::vector pat; + get_arr_int(KEY_WA_PATTERN_MODE, pat, true); + GGML_ASSERT((int) pat.size() == hparams.n_layer && "mimovl wa_pattern_mode length must equal n_layer"); + hparams.wa_pattern_mode.assign(pat.begin(), pat.end()); + get_u32(KEY_IMAGE_MIN_PIXELS, hparams.image_min_pixels); + get_u32(KEY_IMAGE_MAX_PIXELS, hparams.image_max_pixels); + hparams.set_warmup_n_tokens(46*46); // avoid OOM on warmup + } break; + case PROJECTOR_TYPE_STEP3VL: + { + hparams.n_merge = 4; // two stride-2 downsamplers after patching + get_u32(KEY_PROJ_SCALE_FACTOR, hparams.n_merge, false); + hparams.rope_theta = 10000.0f; + get_u32(KEY_PREPROC_IMAGE_SIZE, hparams.image_longest_edge, false); + if (hparams.image_longest_edge == 0) { + hparams.image_longest_edge = 3024; + } + // note: the step3vl preprocessor slices based on a fixed window grid, so it does not support custom min/max image tokens + hparams.warmup_image_size = hparams.image_size; + } break; + case PROJECTOR_TYPE_YOUTUVL: + { + hparams.n_merge = 2; + hparams.image_resize_algo = RESIZE_ALGO_BILINEAR; + hparams.image_resize_pad = PAD_NONE; + get_u32(KEY_SPATIAL_MERGE_SIZE, hparams.n_merge, false); + get_u32(KEY_ATTN_WINDOW_SIZE, hparams.attn_window_size, true); + std::vector wa_layer_indexes_vec; + get_arr_int(KEY_WIN_ATTN_LAYER_INDEXES, wa_layer_indexes_vec, true); + for (auto & layer : wa_layer_indexes_vec) { + hparams.wa_layer_indexes.insert(layer); + } + // support max_height * max_width = 8000 * 8000. 8000/16/2 = 250 image tokens + hparams.set_limit_image_tokens(1, 62500); + hparams.set_warmup_n_tokens(16*16); // avoid OOM on warmup + } break; + case PROJECTOR_TYPE_YASA2: + { + hparams.ffn_op = FFN_GELU_ERF; + log_ffn_op = "gelu_erf"; + hparams.image_resize_algo = RESIZE_ALGO_BICUBIC; + + // reka model performs better when the image is stretched to fit + // fixed square size (no padding) + hparams.image_resize_pad = PAD_NONE; + } break; + case PROJECTOR_TYPE_GLM4V: + { + hparams.rope_theta = 10000.0f; + hparams.n_merge = 2; // default value for GLM4-V + hparams.image_resize_algo = RESIZE_ALGO_BICUBIC; + get_u32(KEY_SPATIAL_MERGE_SIZE, hparams.n_merge, false); + hparams.set_limit_image_tokens(8, 4096); + hparams.set_warmup_n_tokens(46*46); // avoid OOM on warmup + } break; + case PROJECTOR_TYPE_GLM5V: + { + // glm4v tower with clamped SwiGLU, ceil-aligned resize and its own token budget + hparams.rope_theta = 10000.0f; + hparams.n_merge = 2; + hparams.image_resize_algo = RESIZE_ALGO_BICUBIC; + get_u32(KEY_SPATIAL_MERGE_SIZE, hparams.n_merge, false); + get_f32(KEY_SWIGLU_CLAMP, hparams.swiglu_clamp, true); + get_u32(KEY_IMAGE_MIN_PIXELS, hparams.image_min_pixels); + get_u32(KEY_IMAGE_MAX_PIXELS, hparams.image_max_pixels); + hparams.set_limit_image_tokens(); + hparams.set_warmup_n_tokens(46*46); // avoid OOM on warmup + } break; + case PROJECTOR_TYPE_LLAMA4: + { + hparams.rope_theta = 10000.0f; + hparams.image_resize_algo = RESIZE_ALGO_BILINEAR; + get_u32(KEY_PROJ_SCALE_FACTOR, hparams.n_merge, false); + set_llava_uhd_res_candidates(model, 3); + } break; + case PROJECTOR_TYPE_ULTRAVOX: + case PROJECTOR_TYPE_QWEN2A: + case PROJECTOR_TYPE_QWEN3A: + case PROJECTOR_TYPE_GLMA: + case PROJECTOR_TYPE_VOXTRAL: + case PROJECTOR_TYPE_MERALION: + case PROJECTOR_TYPE_MUSIC_FLAMINGO: + { + bool require_stack = model.proj_type == PROJECTOR_TYPE_ULTRAVOX || + model.proj_type == PROJECTOR_TYPE_VOXTRAL || + model.proj_type == PROJECTOR_TYPE_MERALION || + model.proj_type == PROJECTOR_TYPE_GLMA; + get_u32(KEY_A_PROJ_STACK_FACTOR, hparams.proj_stack_factor, require_stack); + hparams.ffn_op = FFN_GELU_ERF; + log_ffn_op = "gelu_erf"; // temporary solution for logging + + // audio preprocessing params + hparams.audio_chunk_len = 30; // in seconds + hparams.audio_sample_rate = 16000; + hparams.audio_n_fft = 400; + hparams.audio_window_len = 400; + hparams.audio_hop_len = 160; + } break; + case PROJECTOR_TYPE_VOXTRAL_RT: + { + get_u32(KEY_A_PROJ_STACK_FACTOR, hparams.proj_stack_factor); + get_u32(KEY_A_ATTN_WINDOW_SIZE, hparams.attn_window_size); + hparams.rope_theta = 1000000.0f; // params.json + hparams.ffn_op = FFN_SILU; + log_ffn_op = "silu"; + + // audio preprocessing params + hparams.audio_sample_rate = 16000; + hparams.audio_n_fft = 400; + hparams.audio_window_len = 400; + hparams.audio_hop_len = 160; + // 2 context frames + 8 s of new frames, see mtmd_subcontext::enc_state_init() + hparams.warmup_audio_size = 2 + 2*400; + } break; + case PROJECTOR_TYPE_MIMO_AUDIO: + { + get_u32(KEY_A_RVQ_NUM_QUANTIZERS, hparams.rvq_num_quantizers, false); + get_arr_int(KEY_A_RVQ_CODEBOOK_SIZE, hparams.rvq_codebook_size, false); + if (hparams.rvq_num_quantizers <= 0) { + throw std::runtime_error(mtmd_format("%s: mimo_audio: missing %s\n", __func__, KEY_A_RVQ_NUM_QUANTIZERS)); + } + if ((int) hparams.rvq_codebook_size.size() != hparams.rvq_num_quantizers) { + throw std::runtime_error(mtmd_format( + "%s: mimo_audio: %s length (%zu) must equal %s (%d)\n", __func__, + KEY_A_RVQ_CODEBOOK_SIZE, hparams.rvq_codebook_size.size(), + KEY_A_RVQ_NUM_QUANTIZERS, hparams.rvq_num_quantizers)); + } + hparams.ffn_op = FFN_GELU_ERF; // PyTorch F.gelu default (approximate="none") + hparams.rope_theta = 10000.0f; + + // audio preprocessing params (mel spectrogram) + hparams.audio_sample_rate = 24000; + hparams.audio_n_fft = 960; + hparams.audio_window_len = 960; + hparams.audio_hop_len = 240; + + get_u32(KEY_A_ATTN_WINDOW_SIZE, hparams.attn_window_size); + std::vector wa_pattern; + get_arr_int(KEY_A_WA_PATTERN_MODE, wa_pattern, true); + if ((int) wa_pattern.size() != hparams.n_layer) { + throw std::runtime_error(mtmd_format( + "%s: mimo_audio: %s length (%zu) must equal n_layer (%d)\n", __func__, + KEY_A_WA_PATTERN_MODE, wa_pattern.size(), hparams.n_layer)); + } + hparams.wa_pattern_mode.assign(wa_pattern.begin(), wa_pattern.end()); + + get_u32(KEY_A_LOCAL_BLOCK_COUNT, hparams.audio_local_n_layer); + get_u32(KEY_A_LOCAL_GROUP_SIZE, hparams.audio_local_group_size); + if (hparams.audio_local_group_size <= 0) { + throw std::runtime_error(mtmd_format( + "%s: mimo_audio: %s must be > 0\n", __func__, KEY_A_LOCAL_GROUP_SIZE)); + } + } break; + case PROJECTOR_TYPE_QWEN3TTS_SPKENC: + { + // ECAPA-TDNN speaker encoder, mel front-end uses the Slaney default (fmin=0, fmax=sr/2) + hparams.audio_sample_rate = 24000; + hparams.audio_n_fft = 1024; + hparams.audio_window_len = 1024; + hparams.audio_hop_len = 256; + } break; + case PROJECTOR_TYPE_CHATTERBOX_SPKENC: + { + // kaldi fbank front-end of CAMPPlus: 25 ms / 10 ms framing, 512-point spectrum + hparams.audio_sample_rate = 16000; + hparams.audio_n_fft = 512; + hparams.audio_window_len = 400; + hparams.audio_hop_len = 160; + } break; + case PROJECTOR_TYPE_LFM3A_GEN: + { + // TODO: CUDA has no flash attention kernel for the depthformer head dim 32, enable it once it does + model.flash_attn_auto = false; + + // fixed by LiquidCodecConfig, head counts come from the tensors + hparams.rope_theta = 1000000.0f; + + hparams.wav_tfm_n_layer = 8; + hparams.wav_tfm_n_embd = 512; + hparams.wav_tfm_eps = 1e-5f; + hparams.wav_tfm_rope_theta = 1000000.0f; + hparams.wav_tfm_swa = 30; // decoder_window + hparams.wav_frame_stack = 6; // stft_frame_stack, STFT frames per codec frame + // the ISTFT runs on the host, see mtmd_gen_audio_process() + hparams.audio_sample_rate = 24000; + hparams.audio_n_fft = 1280; + hparams.audio_window_len = 1280; + hparams.audio_hop_len = 320; + + // speech-code sampling, the converter writes the model card values; else the LFM3-Omni card + hparams.gen_temp = 0.4f; + hparams.gen_top_k = 0; + hparams.gen_penalty_repeat = 1.2f; + hparams.gen_penalty_last_n = 64; + get_f32(KEY_GEN_AUDIO_TEMP, hparams.gen_temp, false); + get_i32(KEY_GEN_AUDIO_TOP_K, hparams.gen_top_k, false); + get_f32(KEY_GEN_AUDIO_PENALTY_REPEAT, hparams.gen_penalty_repeat, false); + get_i32(KEY_GEN_AUDIO_PENALTY_LAST_N, hparams.gen_penalty_last_n, false); + } break; + case PROJECTOR_TYPE_QWEN3TTS_GEN: + { + // TODO: hardcoded for now, read from code_predictor_config instead + hparams.rope_theta = 1000000.0f; + + // code2wav params + hparams.wav_tfm_n_layer = 8; + hparams.wav_tfm_n_embd = 512; + hparams.wav_tfm_n_ff = 1024; + hparams.wav_tfm_n_head = 16; + hparams.wav_tfm_n_head_kv = 16; + hparams.wav_tfm_eps = 1e-5f; + hparams.wav_tfm_rope_theta = 10000.0f; + hparams.wav_upsample_n_block = 2; + hparams.wav_dac_n_block = 4; + hparams.wav_dac_n_res = 3; + // matches the reference decoder's sliding_window (speech_tokenizer/config.json) + hparams.wav_tfm_swa = 72; + } break; + case PROJECTOR_TYPE_DRAMA3_GEN: + { + // Fish DAC decoder, codec.pth has no config + hparams.audio_sample_rate = 44100; + hparams.wav_tfm_n_layer = 8; + hparams.wav_tfm_n_embd = 1024; + hparams.wav_tfm_n_ff = 3072; + hparams.wav_tfm_n_head = 16; + hparams.wav_tfm_n_head_kv = 16; + hparams.wav_tfm_eps = 1e-5f; + hparams.wav_tfm_rope_theta = 10000.0f; + hparams.wav_tfm_swa = 128; + hparams.wav_upsample_n_block = 2; + hparams.wav_dac_n_block = 4; + hparams.wav_dac_n_res = 3; + } break; + case PROJECTOR_TYPE_POCKETTTS_SPKENC: + case PROJECTOR_TYPE_POCKETTTS_GEN: + { + // mimi front-end takes the raw waveform, no mel + hparams.audio_sample_rate = 24000; + // seanet ratios are [6,5,4] in the config, the encoder reverses them + hparams.seanet_ratios = { 4, 5, 6 }; + hparams.seanet_n_stage = (int32_t) hparams.seanet_ratios.size(); + hparams.mimi_downsample = 16; + // matches the reference transformer's "context" + hparams.mimi_tfm_context = 250; + hparams.rope_theta = 10000.0f; + // flow_lm defaults, see pocket_tts/default_parameters.py + hparams.flow_n_step = 1; + hparams.gen_eos_threshold = -4.0f; + } break; + case PROJECTOR_TYPE_CHATTERBOX: + { + // the classic cfm solver unrolls up to 10 steps x 2 cfg estimator evaluations in one graph + model.max_nodes = 65536; + + // s3gen output rate + hparams.audio_sample_rate = 24000; + } break; + case PROJECTOR_TYPE_SOPRANO: + { + // SopranoDecoder defaults, n_fft comes from the head tensor + hparams.audio_sample_rate = 32000; + hparams.audio_hop_len = 512; + } break; + case PROJECTOR_TYPE_PADDLEOCR: + { + hparams.n_merge = 2; + hparams.image_resize_algo = RESIZE_ALGO_BICUBIC; + get_u32(KEY_IMAGE_MIN_PIXELS, hparams.image_min_pixels); + get_u32(KEY_IMAGE_MAX_PIXELS, hparams.image_max_pixels); + + hparams.set_warmup_n_tokens(28*28); // avoid OOM on warmup + } break; + case PROJECTOR_TYPE_DEEPSEEKOCR: + case PROJECTOR_TYPE_DEEPSEEKOCR2: + { + hparams.patch_size = 16; + hparams.image_size = 1024; + hparams.warmup_image_size = 1024; + hparams.image_resize_algo = RESIZE_ALGO_BICUBIC; + hparams.image_pad_color = {127, 127, 127}; + + get_u32(KEY_SAM_N_BLOCK, hparams.sam_n_layer, true); + get_u32(KEY_SAM_N_HEAD, hparams.sam_n_head, true); + get_u32(KEY_SAM_N_EMBD, hparams.sam_n_embd, true); + get_u32(KEY_ATTN_WINDOW_SIZE, hparams.attn_window_size, true); + hparams.preproc_min_tiles = 2; + if (model.proj_type == PROJECTOR_TYPE_DEEPSEEKOCR) { + hparams.preproc_max_tiles = 9; + hparams.preproc_tile_size = 640; + // the CLIP/ViT body runs its layernorms at 1e-5 (the SAM stage uses 1e-6) + hparams.eps = 1e-5f; + } + if (model.proj_type == PROJECTOR_TYPE_DEEPSEEKOCR2) { + hparams.preproc_max_tiles = 6; + hparams.preproc_tile_size = 768; + // qwen2 encoder is GQA, requires KEY_N_HEAD_KV + get_u32(mtmd_format(KEY_N_HEAD_KV, "vision"), hparams.n_head_kv); + } + // unlimited-ocr shares the v1 projector but tiles up to 32 + get_u32(KEY_PREPROC_MIN_TILES, hparams.preproc_min_tiles, false); + get_u32(KEY_PREPROC_MAX_TILES, hparams.preproc_max_tiles, false); + GGML_ASSERT(hparams.preproc_min_tiles >= 0 + && hparams.preproc_min_tiles <= hparams.preproc_max_tiles + && hparams.preproc_max_tiles <= 256); + } break; + case PROJECTOR_TYPE_HUNYUANVL: + { + hparams.n_merge = 2; + hparams.image_resize_algo = RESIZE_ALGO_LANCZOS; + hparams.image_resize_pad = PAD_NONE; + hparams.ffn_op = FFN_GELU; + hparams.set_limit_image_tokens(256, 16384); + get_u32(KEY_SPATIAL_MERGE_SIZE, hparams.n_merge, false); + get_u32(KEY_IMAGE_MIN_PIXELS, hparams.image_min_pixels, false); + get_u32(KEY_IMAGE_MAX_PIXELS, hparams.image_max_pixels, false); + hparams.set_warmup_n_tokens(32*32); + } break; + case PROJECTOR_TYPE_LFM2A: + case PROJECTOR_TYPE_D1OMNI_A: + { + // audio preprocessing params + hparams.audio_chunk_len = 1; // in seconds + hparams.audio_sample_rate = 16000; + hparams.audio_n_fft = 512; + hparams.audio_window_len = 400; + hparams.audio_hop_len = 160; + } break; + case PROJECTOR_TYPE_LFM3A: + { + get_u32(KEY_A_ATTN_WINDOW_SIZE, hparams.attn_window_size); + if (hparams.attn_window_size <= 0) { + throw std::runtime_error(mtmd_format("%s: invalid audio attention window %d\n", __func__, hparams.attn_window_size)); + } + hparams.rope_theta = 1000000.0f; // checked by the converter + // audio preprocessing params, see mtmd_audio_preprocessor_lfm3a + hparams.audio_chunk_len = 0; + hparams.audio_sample_rate = 16000; + hparams.audio_n_fft = 400; + hparams.audio_window_len = 400; + hparams.audio_hop_len = 160; + } break; + case PROJECTOR_TYPE_EXAONE4_5: + { + hparams.n_merge = 2; + hparams.image_resize_pad = PAD_NONE; // the HF processor resizes to the smart_resize size, no padding + hparams.image_round_half_even = true; // smart_resize uses Python round() + get_u32(KEY_SPATIAL_MERGE_SIZE, hparams.n_merge, false); + get_u32(KEY_WIN_ATTN_PATTERN, hparams.n_wa_pattern, false); + get_u32(KEY_IMAGE_MIN_PIXELS, hparams.image_min_pixels); + get_u32(KEY_IMAGE_MAX_PIXELS, hparams.image_max_pixels); + hparams.set_warmup_n_tokens(46 * 46); + if (hparams.rope_theta <= 0.0f) { + hparams.rope_theta = 10000.0f; + } + get_u32(mtmd_format(KEY_N_HEAD_KV, "vision"), hparams.n_head_kv); + } break; + case PROJECTOR_TYPE_GEMMA4A: + { + // Gemma4 feature_extraction_gemma4.py: + // frame_length_ms=20 -> 320 samples, n_fft=512, hop=10ms -> 160 + hparams.audio_chunk_len = 0; // no fixed-length padding + hparams.audio_sample_rate = 16000; + hparams.audio_n_fft = 512; + hparams.audio_window_len = 320; // 20ms frame (NOT 25ms/400) + hparams.audio_hop_len = 160; + // due to a mistake in the original conversion code, rms_norm_eps is set to a wrong value + // since all gemma4a models use 1e-6, we just hardcode it here to avoid re-conversion + hparams.eps = 1e-6f; + } break; + case PROJECTOR_TYPE_GEMMA4UA: + { + // Encoder-free: raw 16 kHz waveform chunked into 640-sample frames. + hparams.audio_chunk_len = 0; + hparams.audio_sample_rate = 16000; + hparams.eps = 1e-6f; + hparams.n_mel_bins = 640; + } break; + case PROJECTOR_TYPE_GRANITE_SPEECH: + { + hparams.audio_chunk_len = 0; + hparams.audio_sample_rate = 16000; + hparams.audio_n_fft = 512; + hparams.audio_window_len = 400; + hparams.audio_hop_len = 160; + get_u32(KEY_A_CHUNK_SIZE, hparams.audio_chunk_size); + // context_size is squared for the attn_dists/mask buffers; cap to prevent int32 overflow + // (legitimate values are small, e.g. 12-200; 8192^2 = 67M still fits int32) + GGML_ASSERT(hparams.audio_chunk_size > 0 && hparams.audio_chunk_size <= 8192); + get_u32(KEY_A_CONV_KERNEL_SIZE, hparams.audio_conv_kernel_size); + get_u32(KEY_A_MAX_POS_EMB, hparams.audio_max_pos_emb); + get_u32(KEY_A_PROJ_WINDOW_SIZE, hparams.audio_proj_window_size); + get_u32(KEY_A_PROJ_DOWNSAMPLE_RATE, hparams.audio_proj_downsample_rate); + get_u32(KEY_A_PROJ_HEAD_COUNT, hparams.audio_proj_head_count); + // NOTE: feature layers loaded above in common path + } break; + case PROJECTOR_TYPE_JANUS_PRO: + { + hparams.image_pad_color = {127, 127, 127}; + hparams.image_resize_algo = RESIZE_ALGO_BICUBIC; + } break; + case PROJECTOR_TYPE_GRANITE4_VISION: + { + // SigLIP tower. + hparams.image_resize_algo = RESIZE_ALGO_BICUBIC; + hparams.image_resize_pad = PAD_CEIL; + + // NOTE: feature_layers loaded in common path as optional + get_arr_int(KEY_PROJ_SPATIAL_OFFSETS, hparams.proj_spatial_offsets); + if (hparams.feature_layers.size() != hparams.proj_spatial_offsets.size()) { + throw std::runtime_error(mtmd_format("%s: feature_layers.size() %d != proj_spatial_offsets.size() %d", + hparams.feature_layers.size(), hparams.proj_spatial_offsets.size())); + } + + get_u32(KEY_PROJ_SAMPLE_QUERY_SIDE, hparams.downsample_query_side); + get_u32(KEY_PROJ_SAMPLE_WINDOW_SIDE, hparams.downsample_window_side); + hparams.warmup_image_size = hparams.image_size; + } break; + default: + throw std::runtime_error(mtmd_format("%s: unknown vision projector type %s\n", __func__, proj_type.c_str())); + } + + // sanity check + { + if (hparams.image_size < 0) { + // note: some models having hparams.image_size == 0, which means the image size is dynamic + throw std::runtime_error(mtmd_format("%s: image_size (%d) cannot be negative\n", __func__, hparams.image_size)); + } + if (hparams.image_size > 8192) { + // cap prevents int32 overflow in n_patches = (image_size/patch_size)^2 + throw std::runtime_error(mtmd_format("%s: image_size (%d) is too large (max 8192)\n", __func__, hparams.image_size)); + } + if (hparams.patch_size <= 0 || hparams.patch_size >= 65536) { + throw std::runtime_error(mtmd_format("%s: patch_size (%d) must be positive and less than 65536\n", __func__, hparams.patch_size)); + } + if (hparams.n_embd <= 0) { + throw std::runtime_error(mtmd_format("%s: n_embd (%d) must be greater than 0\n", __func__, hparams.n_embd)); + } + if (hparams.image_max_pixels < hparams.image_min_pixels) { + throw std::runtime_error(mtmd_format("%s: image_max_pixels (%d) is less than image_min_pixels (%d)\n", __func__, hparams.image_max_pixels, hparams.image_min_pixels)); + } + if (hparams.n_merge <= 0 || hparams.n_merge >= 65536) { + throw std::runtime_error(mtmd_format("%s: n_merge (%d) must be greater than 0 and less than 65536\n", __func__, hparams.n_merge)); + } + if (hparams.attn_window_size > 4096) { + throw std::runtime_error(mtmd_format("%s: attn_window_size (%d) is too large (max 4096)\n", __func__, hparams.attn_window_size)); + } + } + + LOG_INF("%s: projector: %s\n", __func__, proj_type.c_str()); + LOG_INF("%s: n_embd: %d\n", __func__, hparams.n_embd); + LOG_INF("%s: n_head: %d\n", __func__, hparams.n_head); + LOG_INF("%s: n_ff: %d\n", __func__, hparams.n_ff); + LOG_INF("%s: n_layer: %d\n", __func__, hparams.n_layer); + LOG_INF("%s: ffn_op: %s\n", __func__, log_ffn_op.c_str()); + LOG_INF("%s: projection_dim: %d\n", __func__, hparams.projection_dim); + if (is_vision) { + LOG_INF("\n--- vision hparams ---\n"); + LOG_INF("%s: image_size: %d\n", __func__, hparams.image_size); + LOG_INF("%s: patch_size: %d\n", __func__, hparams.patch_size); + LOG_INF("%s: has_llava_proj: %d\n", __func__, hparams.has_llava_projector); + LOG_INF("%s: minicpmv_version: %d\n", __func__, hparams.minicpmv_version); + LOG_INF("%s: n_merge: %d\n", __func__, hparams.n_merge); + LOG_INF("%s: n_wa_pattern: %d\n", __func__, hparams.n_wa_pattern); + if (!hparams.wa_layer_indexes.empty()) { + LOG_INF("%s: wa_layer_indexes: ", __func__); + for (auto & layer : hparams.wa_layer_indexes) { + LOG_INF("%d ", layer); + } + LOG_INF("\n"); + } + if (hparams.image_min_pixels > 0) { + LOG_INF("%s: image_min_pixels: %d%s\n", __func__, hparams.image_min_pixels, hparams.custom_image_min_tokens > 0 ? " (custom value)" : ""); + } + if (hparams.image_max_pixels > 0) { + LOG_INF("%s: image_max_pixels: %d%s\n", __func__, hparams.image_max_pixels, hparams.custom_image_max_tokens > 0 ? " (custom value)" : ""); + } + if (hparams.preproc_max_tiles > 0) { + LOG_INF("%s: preproc_tiles: %d - %d\n", __func__, hparams.preproc_min_tiles, hparams.preproc_max_tiles); + } + } else if (is_audio) { + LOG_INF("\n--- audio hparams ---\n"); + LOG_INF("%s: n_mel_bins: %d\n", __func__, hparams.n_mel_bins); + LOG_INF("%s: proj_stack_factor: %d\n", __func__, hparams.proj_stack_factor); + LOG_INF("%s: audio_chunk_len: %d\n", __func__, hparams.audio_chunk_len); + LOG_INF("%s: audio_sample_rate: %d\n", __func__, hparams.audio_sample_rate); + LOG_INF("%s: audio_n_fft: %d\n", __func__, hparams.audio_n_fft); + LOG_INF("%s: audio_window_len: %d\n", __func__, hparams.audio_window_len); + LOG_INF("%s: audio_hop_len: %d\n", __func__, hparams.audio_hop_len); + + // GEMMA4UA is encoder-free: it uses n_mel_bins as a raw-waveform frame size (640) and has no FFT/filterbank, so the mel-range and FFT + // checks below do not apply to it. + // pocket-tts is encoder-free in the same sense: mimi convolves the raw waveform + // Muse Spark is encoder-free in the same sense: its conv pyramid reads the raw waveform, one channel takes the place of the mel bins + const bool fft_based = model.proj_type != PROJECTOR_TYPE_GEMMA4UA && + model.proj_type != PROJECTOR_TYPE_MUSE_SPARK_AUDIO && + model.proj_type != PROJECTOR_TYPE_POCKETTTS_SPKENC; + + // Validate audio hparams loaded from GGUF metadata + if (hparams.n_mel_bins <= 0 || (fft_based && hparams.n_mel_bins > 256)) { + throw std::runtime_error(mtmd_format("%s: n_mel_bins (%d) must be in range [1, 256]\n", __func__, hparams.n_mel_bins)); + } + if (fft_based && (hparams.audio_sample_rate <= 0 || hparams.audio_n_fft <= 0 || hparams.audio_hop_len <= 0 || hparams.audio_window_len <= 0)) { + throw std::runtime_error(mtmd_format("%s: audio hparams invalid: sample_rate=%d n_fft=%d window_len=%d hop_len=%d\n", + __func__, hparams.audio_sample_rate, hparams.audio_n_fft, hparams.audio_window_len, hparams.audio_hop_len)); + } + } + LOG_INF("\n"); + LOG_INF("%s: model size: %.2f MiB\n", __func__, model_size / 1024.0 / 1024.0); + LOG_INF("%s: metadata size: %.2f MiB\n", __func__, ggml_get_mem_size(ctx_meta.get()) / 1024.0 / 1024.0); + } + } + + void load_tensors(mtmd_model & model, ggml_backend_buffer_type_t buft, bool no_alloc) { + auto & hparams = model.hparams; + std::map tensor_offset; + std::vector tensors_to_load; + + auto fin = open_ifstream_binary(fname); + if (!fin) { + throw std::runtime_error(mtmd_format("%s: failed to open %s\n", __func__, fname.c_str())); + } + + // TODO @ngxson : support both audio and video in the future + const char * prefix = model.modality == MTMD_MODALITY_AUDIO ? "a" + : model.modality == MTMD_MODALITY_GEN_AUDIO ? "a.gen.code" + : "v"; + + // get offsets + for (int64_t i = 0; i < gguf_get_n_tensors(ctx_gguf.get()); ++i) { + const char * name = gguf_get_tensor_name(ctx_gguf.get(), i); + tensor_offset[name] = gguf_get_data_offset(ctx_gguf.get()) + gguf_get_tensor_offset(ctx_gguf.get(), i); + } + + // create data context + struct ggml_init_params params = { + /*.mem_size =*/ static_cast(gguf_get_n_tensors(ctx_gguf.get()) + 1) * ggml_tensor_overhead(), + /*.mem_buffer =*/ NULL, + /*.no_alloc =*/ true, + }; + model.ctx_data.reset(ggml_init(params)); + if (!model.ctx_data) { + throw std::runtime_error(mtmd_format("%s: failed to init ggml context\n", __func__)); + } + + // helper function + std::unordered_set loaded_tensor_names; + auto get_tensor = [&](const std::string & name, bool required = true) { + // Each tensor should only be loaded once; duplicates indicate a bug + if (loaded_tensor_names.count(name)) { + throw std::runtime_error(mtmd_format("%s: tensor already loaded: %s\n", __func__, name.c_str())); + } + ggml_tensor * cur = ggml_get_tensor(ctx_meta.get(), name.c_str()); + if (!cur && required) { + throw std::runtime_error(mtmd_format("%s: unable to find tensor %s\n", __func__, name.c_str())); + } + if (cur) { + tensors_to_load.push_back(cur); + ggml_tensor * data_tensor = ggml_dup_tensor(model.ctx_data.get(), cur); + ggml_set_name(data_tensor, cur->name); + loaded_tensor_names.insert(name); + cur = data_tensor; + // add to weight memory counter + model.mem_weights[ggml_backend_buft_get_device(buft)] += ggml_nbytes(cur); + } + return cur; + }; + + // pocket-tts: the encoder and the decoder share the same layout, only the prefix differs + auto load_seanet = [&](mtmd_seanet & seanet, bool is_decoder) { + const char * conv_in = is_decoder ? TN_A_GEN_WAV_SEANET_CONV_IN : TN_A_SEANET_CONV_IN; + const char * conv_out = is_decoder ? TN_A_GEN_WAV_SEANET_CONV_OUT : TN_A_SEANET_CONV_OUT; + const char * res1 = is_decoder ? TN_A_GEN_WAV_SEANET_RES_CONV1 : TN_A_SEANET_RES_CONV1; + const char * res2 = is_decoder ? TN_A_GEN_WAV_SEANET_RES_CONV2 : TN_A_SEANET_RES_CONV2; + const char * scale = is_decoder ? TN_A_GEN_WAV_SEANET_SCALE_CONV : TN_A_SEANET_SCALE_CONV; + + seanet.conv_in_w = get_tensor(mtmd_format(conv_in, "weight")); + seanet.conv_in_b = get_tensor(mtmd_format(conv_in, "bias")); + seanet.conv_out_w = get_tensor(mtmd_format(conv_out, "weight")); + seanet.conv_out_b = get_tensor(mtmd_format(conv_out, "bias")); + + seanet.stages.resize(hparams.seanet_n_stage); + for (int i = 0; i < hparams.seanet_n_stage; i++) { + auto & stage = seanet.stages[i]; + stage.res_conv1_w = get_tensor(mtmd_format(res1, i, "weight")); + stage.res_conv1_b = get_tensor(mtmd_format(res1, i, "bias")); + stage.res_conv2_w = get_tensor(mtmd_format(res2, i, "weight")); + stage.res_conv2_b = get_tensor(mtmd_format(res2, i, "bias")); + stage.scale_conv_w = get_tensor(mtmd_format(scale, i, "weight")); + stage.scale_conv_b = get_tensor(mtmd_format(scale, i, "bias")); + } + }; + + auto get_vector = [&](const std::string & name) { + std::vector result; + auto it = tensor_offset.find(name); + if (it == tensor_offset.end()) { + return result; + } + + const int64_t idx = gguf_find_tensor(ctx_gguf.get(), name.c_str()); + if (idx < 0) { + throw std::runtime_error(mtmd_format("%s: failed to find tensor %s\n", __func__, name.c_str())); + } + + if (const auto type = gguf_get_tensor_type(ctx_gguf.get(), idx); type != GGML_TYPE_F32) { + throw std::runtime_error(mtmd_format("%s: %s must be %s, was %s\n", __func__, + name.c_str(), ggml_type_name(GGML_TYPE_F32), ggml_type_name(type))); + } + + const size_t n_bytes = gguf_get_tensor_size(ctx_gguf.get(), idx); + if (n_bytes == 0) { + throw std::runtime_error(mtmd_format("%s: tensor %s is empty\n", __func__, name.c_str())); + } + + const size_t n_elems = n_bytes / sizeof(float); + result.resize(n_elems); + fin.seekg(it->second, std::ios::beg); + fin.read(reinterpret_cast(result.data()), n_bytes); + return result; + }; + + auto get_scalar = [&](const std::string & name, float default_val) { + auto v = get_vector(name); + if (v.empty()) { + return default_val; + } + if (v.size() != 1) { + throw std::runtime_error(mtmd_format("%s: expected scalar tensor '%s' but got %d elements\n", + __func__, name.c_str(), (int) v.size())); + } + + return v[0]; + }; + + model.class_embedding = get_tensor(TN_CLASS_EMBD, false); + + model.pre_ln_w = get_tensor(mtmd_format(TN_LN_PRE, prefix, "weight"), false); + model.pre_ln_b = get_tensor(mtmd_format(TN_LN_PRE, prefix, "bias"), false); + + model.post_ln_w = get_tensor(mtmd_format(TN_LN_POST, prefix, "weight"), false); + model.post_ln_b = get_tensor(mtmd_format(TN_LN_POST, prefix, "bias"), false); + + // every vision encoder starts with a patch embedding, except the ones with their own stem + const bool has_patch_embd = ( + model.modality == MTMD_MODALITY_VISION && + model.proj_type != PROJECTOR_TYPE_GEMMA3NV && + model.proj_type != PROJECTOR_TYPE_DEEPSEEKOCR && + model.proj_type != PROJECTOR_TYPE_DEEPSEEKOCR2); + + model.patch_bias = get_tensor(TN_PATCH_BIAS, false); + model.patch_embeddings_0 = get_tensor(TN_PATCH_EMBD, has_patch_embd); + model.patch_embeddings_1 = get_tensor(TN_PATCH_EMBD_1, false); + + model.norm_embd_w = get_tensor(mtmd_format(TN_NORM_EMBD, "weight"), false); + model.norm_embd_b = get_tensor(mtmd_format(TN_NORM_EMBD, "bias"), false); + + model.position_embeddings = get_tensor(mtmd_format(TN_POS_EMBD, prefix), false); + + // lfm3vl + model.lfm3vl_embedding_norm_w = get_tensor(mtmd_format(TN_LFM3VL_EMBEDDING_NORM, prefix, "weight"), false); + + const bool has_standard_layers = ( + model.proj_type != PROJECTOR_TYPE_GEMMA3NV && + model.proj_type != PROJECTOR_TYPE_QWEN3TTS_SPKENC && + model.proj_type != PROJECTOR_TYPE_POCKETTTS_GEN && + model.proj_type != PROJECTOR_TYPE_SOPRANO && + model.proj_type != PROJECTOR_TYPE_MUSE_SPARK_AUDIO && + model.proj_type != PROJECTOR_TYPE_CHATTERBOX && + model.proj_type != PROJECTOR_TYPE_CHATTERBOX_SPKENC); + + // layers + const int n_layers_to_load = has_standard_layers ? hparams.n_layer : 0; + model.layers.resize(n_layers_to_load); + for (int il = 0; il < n_layers_to_load; ++il) { + auto & layer = model.layers[il]; + // lfm3vl / lfm3a conv layers have no attention tensors + const bool o_w_required = model.proj_type != PROJECTOR_TYPE_LFM3VL + && model.proj_type != PROJECTOR_TYPE_LFM3A; + layer.k_w = get_tensor(mtmd_format(TN_ATTN_K, prefix, il, "weight"), false); + layer.q_w = get_tensor(mtmd_format(TN_ATTN_Q, prefix, il, "weight"), false); + layer.v_w = get_tensor(mtmd_format(TN_ATTN_V, prefix, il, "weight"), false); + layer.o_w = get_tensor(mtmd_format(TN_ATTN_OUTPUT, prefix, il, "weight"), o_w_required); + layer.qkv_w = get_tensor(mtmd_format(TN_ATTN_QKV, prefix, il, "weight"), false); + layer.k_norm = get_tensor(mtmd_format(TN_ATTN_K_NORM, prefix, il, "weight"), false); + layer.q_norm = get_tensor(mtmd_format(TN_ATTN_Q_NORM, prefix, il, "weight"), false); + layer.ln_1_w = get_tensor(mtmd_format(TN_LN_1, prefix, il, "weight"), false); + layer.ln_2_w = get_tensor(mtmd_format(TN_LN_2, prefix, il, "weight"), false); + layer.ls_1_w = get_tensor(mtmd_format(TN_LS_1, prefix, il, "weight"), false); // no bias + layer.ls_2_w = get_tensor(mtmd_format(TN_LS_2, prefix, il, "weight"), false); // no bias + layer.ls_out_w = get_tensor(mtmd_format(TN_LS_OUT, prefix, il, "weight"), false); // no bias + layer.attn_post_norm_w = get_tensor(mtmd_format(TN_ATTN_POST_NORM, prefix, il, "weight"), false); // no bias + layer.ff_post_norm_w = get_tensor(mtmd_format(TN_FFN_POST_NORM, prefix, il, "weight"), false); // no bias + + layer.k_b = get_tensor(mtmd_format(TN_ATTN_K, prefix, il, "bias"), false); + layer.q_b = get_tensor(mtmd_format(TN_ATTN_Q, prefix, il, "bias"), false); + layer.v_b = get_tensor(mtmd_format(TN_ATTN_V, prefix, il, "bias"), false); + layer.o_b = get_tensor(mtmd_format(TN_ATTN_OUTPUT, prefix, il, "bias"), false); + layer.qkv_b = get_tensor(mtmd_format(TN_ATTN_QKV, prefix, il, "bias"), false); + layer.ln_1_b = get_tensor(mtmd_format(TN_LN_1, prefix, il, "bias"), false); + layer.ln_2_b = get_tensor(mtmd_format(TN_LN_2, prefix, il, "bias"), false); + + // MoE ffn (dots3note vision pyramid blocks); replaces the dense ffn when present + layer.ff_gate_inp_w = get_tensor(mtmd_format(TN_FFN_GATE_INP, prefix, il, "weight"), false); + layer.ff_gate_exps_w = get_tensor(mtmd_format(TN_FFN_GATE_EXPS, prefix, il, "weight"), false); + layer.ff_up_exps_w = get_tensor(mtmd_format(TN_FFN_UP_EXPS, prefix, il, "weight"), false); + layer.ff_down_exps_w = get_tensor(mtmd_format(TN_FFN_DOWN_EXPS, prefix, il, "weight"), false); + layer.ff_exp_probs_b = get_tensor(mtmd_format(TN_FFN_EXP_PROBS_B, prefix, il, "weight"), false); + const bool is_moe = layer.ff_gate_exps_w != nullptr; + + // ffn + layer.ff_up_w = get_tensor(mtmd_format(TN_FFN_UP, prefix, il, "weight"), !is_moe); + layer.ff_up_b = get_tensor(mtmd_format(TN_FFN_UP, prefix, il, "bias"), false); + layer.ff_gate_w = get_tensor(mtmd_format(TN_FFN_GATE, prefix, il, "weight"), false); + layer.ff_gate_b = get_tensor(mtmd_format(TN_FFN_GATE, prefix, il, "bias"), false); + layer.ff_down_w = get_tensor(mtmd_format(TN_FFN_DOWN, prefix, il, "weight"), !is_moe); + layer.ff_down_b = get_tensor(mtmd_format(TN_FFN_DOWN, prefix, il, "bias"), false); + + // mimovl per-head attention sink bias + layer.attn_sinks = get_tensor(mtmd_format(TN_ATTN_SINKS, prefix, il), false); + + // qwen3vl deepstack layer + layer.deepstack_norm_w = get_tensor(mtmd_format(TN_DEEPSTACK_NORM, il, "weight"), false); + layer.deepstack_norm_b = get_tensor(mtmd_format(TN_DEEPSTACK_NORM, il, "bias"), false); + layer.deepstack_fc1_w = get_tensor(mtmd_format(TN_DEEPSTACK_FC1, il, "weight"), false); + layer.deepstack_fc1_b = get_tensor(mtmd_format(TN_DEEPSTACK_FC1, il, "bias"), false); + layer.deepstack_fc2_w = get_tensor(mtmd_format(TN_DEEPSTACK_FC2, il, "weight"), false); + layer.deepstack_fc2_b = get_tensor(mtmd_format(TN_DEEPSTACK_FC2, il, "bias"), false); + if (layer.has_deepstack()) { + model.n_deepstack_layers++; + } + + // lfm3vl conv layers + layer.conv2d_in_proj_w = get_tensor(mtmd_format(TN_LFM3VL_CONV2D_IN_PROJ, prefix, il, "weight"), false); + layer.conv2d_spatial_w = get_tensor(mtmd_format(TN_LFM3VL_CONV2D_SPATIAL, prefix, il, "weight"), false); + layer.conv2d_out_proj_w = get_tensor(mtmd_format(TN_LFM3VL_CONV2D_OUT_PROJ, prefix, il, "weight"), false); + + // some models already exported with legacy (incorrect) naming which is quite messy, let's fix it here + // note: Qwen model converted from the old surgery script has n_ff = 0, so we cannot use n_ff to check! + bool is_ffn_swapped = ( + // only old models need this fix + model.proj_type == PROJECTOR_TYPE_MLP + || model.proj_type == PROJECTOR_TYPE_MLP_NORM + || model.proj_type == PROJECTOR_TYPE_LDP + || model.proj_type == PROJECTOR_TYPE_LDPV2 + || model.proj_type == PROJECTOR_TYPE_QWEN2VL + || model.proj_type == PROJECTOR_TYPE_QWEN25VL + || model.proj_type == PROJECTOR_TYPE_EXAONE4_5 + || model.proj_type == PROJECTOR_TYPE_GLM_EDGE + || model.proj_type == PROJECTOR_TYPE_GEMMA3 + || model.proj_type == PROJECTOR_TYPE_IDEFICS3 + || model.proj_type == PROJECTOR_TYPE_MINICPMV + || model.proj_type == PROJECTOR_TYPE_MINICPMV4_6 + || model.proj_type == PROJECTOR_TYPE_MINICPMV4_7 + || model.proj_type == PROJECTOR_TYPE_MINICPM_OCR_1_0 + ) && layer.ff_up_w && layer.ff_down_w && layer.ff_down_w->ne[0] == hparams.n_embd; + if (is_ffn_swapped) { + // swap up and down weights + ggml_tensor * tmp = layer.ff_up_w; + layer.ff_up_w = layer.ff_down_w; + layer.ff_down_w = tmp; + // swap up and down biases + tmp = layer.ff_up_b; + layer.ff_up_b = layer.ff_down_b; + layer.ff_down_b = tmp; + if (il == 0) { + LOG_WRN("%s: ffn up/down are swapped\n", __func__); + } + } + } + + // code2wav: RVQ codes -> raw PCM, drama3 has no quant/pre_conv/tfm projections + auto load_code2wav = [&](bool with_proj) { + auto & c2w = model.c2w; + + c2w.quant_first_in_w = get_tensor(mtmd_format(TN_A_GEN_WAV_QUANT_FIRST_IN, "weight"), with_proj); + c2w.quant_first_out_w = get_tensor(mtmd_format(TN_A_GEN_WAV_QUANT_FIRST_OUT, "weight"), with_proj); + c2w.quant_first_cb_w = get_tensor(mtmd_format(TN_A_GEN_WAV_QUANT_FIRST_CB, "weight")); + c2w.quant_rest_in_w = get_tensor(mtmd_format(TN_A_GEN_WAV_QUANT_REST_IN, "weight"), with_proj); + c2w.quant_rest_out_w = get_tensor(mtmd_format(TN_A_GEN_WAV_QUANT_REST_OUT, "weight"), with_proj); + c2w.quant_rest_cb_w = get_tensor(mtmd_format(TN_A_GEN_WAV_QUANT_REST_CB, "weight")); + + c2w.pre_conv_w = get_tensor(mtmd_format(TN_A_GEN_WAV_PRE_CONV, "weight"), with_proj); + c2w.pre_conv_b = get_tensor(mtmd_format(TN_A_GEN_WAV_PRE_CONV, "bias"), with_proj); + + c2w.tfm_in_proj_w = get_tensor(mtmd_format(TN_A_GEN_WAV_TFM_IN_PROJ, "weight"), with_proj); + c2w.tfm_in_proj_b = get_tensor(mtmd_format(TN_A_GEN_WAV_TFM_IN_PROJ, "bias"), with_proj); + c2w.tfm_out_proj_w = get_tensor(mtmd_format(TN_A_GEN_WAV_TFM_OUT_PROJ, "weight"), with_proj); + c2w.tfm_out_proj_b = get_tensor(mtmd_format(TN_A_GEN_WAV_TFM_OUT_PROJ, "bias"), with_proj); + c2w.tfm_output_norm_w = get_tensor(mtmd_format(TN_A_GEN_WAV_TFM_OUT_NORM, "weight")); + + // loaded manually, the generic model.layers loop is taken by code_predictor + c2w.tfm_layers.resize(hparams.wav_tfm_n_layer); + for (int il = 0; il < hparams.wav_tfm_n_layer; il++) { + auto & layer = c2w.tfm_layers[il]; + const char * p = "a.gen.wav.tfm"; + layer.q_w = get_tensor(mtmd_format(TN_ATTN_Q, p, il, "weight")); + layer.k_w = get_tensor(mtmd_format(TN_ATTN_K, p, il, "weight")); + layer.v_w = get_tensor(mtmd_format(TN_ATTN_V, p, il, "weight")); + layer.o_w = get_tensor(mtmd_format(TN_ATTN_OUTPUT, p, il, "weight")); + layer.ln_1_w = get_tensor(mtmd_format(TN_LN_1, p, il, "weight")); + layer.ln_2_w = get_tensor(mtmd_format(TN_LN_2, p, il, "weight")); + layer.ls_1_w = get_tensor(mtmd_format(TN_LS_1, p, il, "weight")); + layer.ls_2_w = get_tensor(mtmd_format(TN_LS_2, p, il, "weight")); + layer.ff_gate_w = get_tensor(mtmd_format(TN_FFN_GATE, p, il, "weight")); + layer.ff_up_w = get_tensor(mtmd_format(TN_FFN_UP, p, il, "weight")); + layer.ff_down_w = get_tensor(mtmd_format(TN_FFN_DOWN, p, il, "weight")); + } + + // upsample: 2x (causal ConvTranspose1d + ConvNeXt block) + c2w.upsample.resize(hparams.wav_upsample_n_block); + for (int il = 0; il < hparams.wav_upsample_n_block; il++) { + auto & up = c2w.upsample[il]; + up.conv_w = get_tensor(mtmd_format(TN_A_GEN_WAV_UP_CONV, il, "weight")); + up.conv_b = get_tensor(mtmd_format(TN_A_GEN_WAV_UP_CONV, il, "bias")); + up.dwconv_w = get_tensor(mtmd_format(TN_A_GEN_WAV_UP_DWCONV, il, "weight")); + up.dwconv_b = get_tensor(mtmd_format(TN_A_GEN_WAV_UP_DWCONV, il, "bias")); + up.norm_w = get_tensor(mtmd_format(TN_A_GEN_WAV_UP_NORM, il, "weight")); + up.norm_b = get_tensor(mtmd_format(TN_A_GEN_WAV_UP_NORM, il, "bias")); + up.pw1_w = get_tensor(mtmd_format(TN_A_GEN_WAV_UP_PW1, il, "weight")); + up.pw1_b = get_tensor(mtmd_format(TN_A_GEN_WAV_UP_PW1, il, "bias")); + up.pw2_w = get_tensor(mtmd_format(TN_A_GEN_WAV_UP_PW2, il, "weight")); + up.pw2_b = get_tensor(mtmd_format(TN_A_GEN_WAV_UP_PW2, il, "bias")); + up.gamma = get_tensor(mtmd_format(TN_A_GEN_WAV_UP_GAMMA, il)); + } + + // DAC decoder: conv_pre + n upsample blocks (each with n_res residual units) + conv_post + c2w.dac_entry_w = get_tensor(mtmd_format(TN_A_GEN_WAV_DAC_ENTRY, "weight")); + c2w.dac_entry_b = get_tensor(mtmd_format(TN_A_GEN_WAV_DAC_ENTRY, "bias")); + + c2w.dac.resize(hparams.wav_dac_n_block); + for (int il = 0; il < hparams.wav_dac_n_block; il++) { + auto & blk = c2w.dac[il]; + blk.snake_alpha = get_tensor(mtmd_format(TN_A_GEN_WAV_DAC_SNAKE, il, "alpha")); + blk.snake_beta = get_tensor(mtmd_format(TN_A_GEN_WAV_DAC_SNAKE, il, "beta")); + blk.conv_w = get_tensor(mtmd_format(TN_A_GEN_WAV_DAC_CONV, il, "weight")); + blk.conv_b = get_tensor(mtmd_format(TN_A_GEN_WAV_DAC_CONV, il, "bias")); + + blk.res.resize(hparams.wav_dac_n_res); + for (int ir = 0; ir < hparams.wav_dac_n_res; ir++) { + auto & res = blk.res[ir]; + res.act1_alpha = get_tensor(mtmd_format(TN_A_GEN_WAV_DAC_RES_ACT1, il, ir, "alpha")); + res.act1_beta = get_tensor(mtmd_format(TN_A_GEN_WAV_DAC_RES_ACT1, il, ir, "beta")); + res.conv1_w = get_tensor(mtmd_format(TN_A_GEN_WAV_DAC_RES_CONV1, il, ir, "weight")); + res.conv1_b = get_tensor(mtmd_format(TN_A_GEN_WAV_DAC_RES_CONV1, il, ir, "bias")); + res.act2_alpha = get_tensor(mtmd_format(TN_A_GEN_WAV_DAC_RES_ACT2, il, ir, "alpha")); + res.act2_beta = get_tensor(mtmd_format(TN_A_GEN_WAV_DAC_RES_ACT2, il, ir, "beta")); + res.conv2_w = get_tensor(mtmd_format(TN_A_GEN_WAV_DAC_RES_CONV2, il, ir, "weight")); + res.conv2_b = get_tensor(mtmd_format(TN_A_GEN_WAV_DAC_RES_CONV2, il, ir, "bias")); + } + } + + c2w.dac_post_snake_alpha = get_tensor(mtmd_format(TN_A_GEN_WAV_DAC_POST_SNAKE, "alpha")); + c2w.dac_post_snake_beta = get_tensor(mtmd_format(TN_A_GEN_WAV_DAC_POST_SNAKE, "beta")); + c2w.dac_post_conv_w = get_tensor(mtmd_format(TN_A_GEN_WAV_DAC_POST_CONV, "weight")); + c2w.dac_post_conv_b = get_tensor(mtmd_format(TN_A_GEN_WAV_DAC_POST_CONV, "bias")); + }; + + switch (model.proj_type) { + case PROJECTOR_TYPE_MLP: + case PROJECTOR_TYPE_MLP_NORM: + { + // LLaVA projection: mm.0 -> gelu -> mm.2 + // Yi-type llava (MLP_NORM): mm.0 -> norm mm.1 -> gelu -> mm.3 -> norm mm.4 + model.mm_0_w = get_tensor(mtmd_format(TN_LLAVA_PROJ, 0, "weight")); + model.mm_0_b = get_tensor(mtmd_format(TN_LLAVA_PROJ, 0, "bias")); + model.mm_3_w = get_tensor(mtmd_format(TN_LLAVA_PROJ, 3, "weight"), model.proj_type == PROJECTOR_TYPE_MLP_NORM); + if (model.mm_3_w) { + // TODO: this is a hack to support Yi-type llava + model.proj_type = PROJECTOR_TYPE_MLP_NORM; + model.mm_1_w = get_tensor(mtmd_format(TN_LLAVA_PROJ, 1, "weight")); + model.mm_1_b = get_tensor(mtmd_format(TN_LLAVA_PROJ, 1, "bias")); + model.mm_3_b = get_tensor(mtmd_format(TN_LLAVA_PROJ, 3, "bias")); + model.mm_4_w = get_tensor(mtmd_format(TN_LLAVA_PROJ, 4, "weight")); + model.mm_4_b = get_tensor(mtmd_format(TN_LLAVA_PROJ, 4, "bias")); + } else { + model.mm_2_w = get_tensor(mtmd_format(TN_LLAVA_PROJ, 2, "weight")); + model.mm_2_b = get_tensor(mtmd_format(TN_LLAVA_PROJ, 2, "bias")); + } + model.image_newline = get_tensor(TN_IMAGE_NEWLINE, false); + } break; + case PROJECTOR_TYPE_LDP: + { + // MobileVLM projection + model.mm_model_mlp_1_w = get_tensor(mtmd_format(TN_MVLM_PROJ_MLP, 1, "weight")); + model.mm_model_mlp_1_b = get_tensor(mtmd_format(TN_MVLM_PROJ_MLP, 1, "bias")); + model.mm_model_mlp_3_w = get_tensor(mtmd_format(TN_MVLM_PROJ_MLP, 3, "weight")); + model.mm_model_mlp_3_b = get_tensor(mtmd_format(TN_MVLM_PROJ_MLP, 3, "bias")); + model.mm_model_block_1_block_0_0_w = get_tensor(mtmd_format(TN_MVLM_PROJ_BLOCK, 1, 0, "0.weight")); + model.mm_model_block_1_block_0_1_w = get_tensor(mtmd_format(TN_MVLM_PROJ_BLOCK, 1, 0, "1.weight")); + model.mm_model_block_1_block_0_1_b = get_tensor(mtmd_format(TN_MVLM_PROJ_BLOCK, 1, 0, "1.bias")); + model.mm_model_block_1_block_1_fc1_w = get_tensor(mtmd_format(TN_MVLM_PROJ_BLOCK, 1, 1, "fc1.weight")); + model.mm_model_block_1_block_1_fc1_b = get_tensor(mtmd_format(TN_MVLM_PROJ_BLOCK, 1, 1, "fc1.bias")); + model.mm_model_block_1_block_1_fc2_w = get_tensor(mtmd_format(TN_MVLM_PROJ_BLOCK, 1, 1, "fc2.weight")); + model.mm_model_block_1_block_1_fc2_b = get_tensor(mtmd_format(TN_MVLM_PROJ_BLOCK, 1, 1, "fc2.bias")); + model.mm_model_block_1_block_2_0_w = get_tensor(mtmd_format(TN_MVLM_PROJ_BLOCK, 1, 2, "0.weight")); + model.mm_model_block_1_block_2_1_w = get_tensor(mtmd_format(TN_MVLM_PROJ_BLOCK, 1, 2, "1.weight")); + model.mm_model_block_1_block_2_1_b = get_tensor(mtmd_format(TN_MVLM_PROJ_BLOCK, 1, 2, "1.bias")); + model.mm_model_block_2_block_0_0_w = get_tensor(mtmd_format(TN_MVLM_PROJ_BLOCK, 2, 0, "0.weight")); + model.mm_model_block_2_block_0_1_w = get_tensor(mtmd_format(TN_MVLM_PROJ_BLOCK, 2, 0, "1.weight")); + model.mm_model_block_2_block_0_1_b = get_tensor(mtmd_format(TN_MVLM_PROJ_BLOCK, 2, 0, "1.bias")); + model.mm_model_block_2_block_1_fc1_w = get_tensor(mtmd_format(TN_MVLM_PROJ_BLOCK, 2, 1, "fc1.weight")); + model.mm_model_block_2_block_1_fc1_b = get_tensor(mtmd_format(TN_MVLM_PROJ_BLOCK, 2, 1, "fc1.bias")); + model.mm_model_block_2_block_1_fc2_w = get_tensor(mtmd_format(TN_MVLM_PROJ_BLOCK, 2, 1, "fc2.weight")); + model.mm_model_block_2_block_1_fc2_b = get_tensor(mtmd_format(TN_MVLM_PROJ_BLOCK, 2, 1, "fc2.bias")); + model.mm_model_block_2_block_2_0_w = get_tensor(mtmd_format(TN_MVLM_PROJ_BLOCK, 2, 2, "0.weight")); + model.mm_model_block_2_block_2_1_w = get_tensor(mtmd_format(TN_MVLM_PROJ_BLOCK, 2, 2, "1.weight")); + model.mm_model_block_2_block_2_1_b = get_tensor(mtmd_format(TN_MVLM_PROJ_BLOCK, 2, 2, "1.bias")); + } break; + case PROJECTOR_TYPE_LDPV2: + { + // MobilVLM_V2 projection + model.mm_model_mlp_0_w = get_tensor(mtmd_format(TN_MVLM_PROJ_MLP, 0, "weight")); + model.mm_model_mlp_0_b = get_tensor(mtmd_format(TN_MVLM_PROJ_MLP, 0, "bias")); + model.mm_model_mlp_2_w = get_tensor(mtmd_format(TN_MVLM_PROJ_MLP, 2, "weight")); + model.mm_model_mlp_2_b = get_tensor(mtmd_format(TN_MVLM_PROJ_MLP, 2, "bias")); + model.mm_model_peg_0_w = get_tensor(mtmd_format(TN_MVLM_PROJ_PEG, 0, "weight")); + model.mm_model_peg_0_b = get_tensor(mtmd_format(TN_MVLM_PROJ_PEG, 0, "bias")); + } break; + case PROJECTOR_TYPE_MINICPMV: + { + // model.mm_model_pos_embed = get_tensor(new_clip->ctx_data, TN_MINICPMV_POS_EMBD); + model.mm_model_pos_embed_k = get_tensor(TN_MINICPMV_POS_EMBD_K); + model.mm_model_query = get_tensor(TN_MINICPMV_QUERY); + model.mm_model_proj = get_tensor(TN_MINICPMV_PROJ); + model.mm_model_kv_proj = get_tensor(TN_MINICPMV_KV_PROJ); + model.mm_model_attn_q_w = get_tensor(mtmd_format(TN_MINICPMV_ATTN, "q", "weight")); + model.mm_model_attn_k_w = get_tensor(mtmd_format(TN_MINICPMV_ATTN, "k", "weight")); + model.mm_model_attn_v_w = get_tensor(mtmd_format(TN_MINICPMV_ATTN, "v", "weight")); + model.mm_model_attn_q_b = get_tensor(mtmd_format(TN_MINICPMV_ATTN, "q", "bias")); + model.mm_model_attn_k_b = get_tensor(mtmd_format(TN_MINICPMV_ATTN, "k", "bias")); + model.mm_model_attn_v_b = get_tensor(mtmd_format(TN_MINICPMV_ATTN, "v", "bias")); + model.mm_model_attn_o_w = get_tensor(mtmd_format(TN_MINICPMV_ATTN, "out", "weight")); + model.mm_model_attn_o_b = get_tensor(mtmd_format(TN_MINICPMV_ATTN, "out", "bias")); + model.mm_model_ln_q_w = get_tensor(mtmd_format(TN_MINICPMV_LN, "q", "weight")); + model.mm_model_ln_q_b = get_tensor(mtmd_format(TN_MINICPMV_LN, "q", "bias")); + model.mm_model_ln_kv_w = get_tensor(mtmd_format(TN_MINICPMV_LN, "kv", "weight")); + model.mm_model_ln_kv_b = get_tensor(mtmd_format(TN_MINICPMV_LN, "kv", "bias")); + model.mm_model_ln_post_w = get_tensor(mtmd_format(TN_MINICPMV_LN, "post", "weight")); + model.mm_model_ln_post_b = get_tensor(mtmd_format(TN_MINICPMV_LN, "post", "bias")); + } break; + case PROJECTOR_TYPE_MINICPMV4_6: + case PROJECTOR_TYPE_MINICPMV4_7: + case PROJECTOR_TYPE_MINICPM_OCR_1_0: + { + const bool merger_required = hparams.n_merge == 4; + auto get_merger_tensor = [&](const std::string & name, bool required = true) { + return get_tensor(name, merger_required && required); + }; + + // ViT merger: window self-attention + model.vit_merger_ln1_w = get_merger_tensor(mtmd_format(TN_VIT_MERGER_LN1, "weight")); + model.vit_merger_ln1_b = get_merger_tensor(mtmd_format(TN_VIT_MERGER_LN1, "bias")); + model.vit_merger_attn_q_w = get_merger_tensor(mtmd_format(TN_VIT_MERGER_ATTN_Q, "weight")); + model.vit_merger_attn_q_b = get_merger_tensor(mtmd_format(TN_VIT_MERGER_ATTN_Q, "bias"), false); + model.vit_merger_attn_k_w = get_merger_tensor(mtmd_format(TN_VIT_MERGER_ATTN_K, "weight")); + model.vit_merger_attn_k_b = get_merger_tensor(mtmd_format(TN_VIT_MERGER_ATTN_K, "bias"), false); + model.vit_merger_attn_v_w = get_merger_tensor(mtmd_format(TN_VIT_MERGER_ATTN_V, "weight")); + model.vit_merger_attn_v_b = get_merger_tensor(mtmd_format(TN_VIT_MERGER_ATTN_V, "bias"), false); + model.vit_merger_attn_o_w = get_merger_tensor(mtmd_format(TN_VIT_MERGER_ATTN_O, "weight")); + model.vit_merger_attn_o_b = get_merger_tensor(mtmd_format(TN_VIT_MERGER_ATTN_O, "bias"), false); + // ViT merger: MLP downsample + model.vit_merger_ds_ln_w = get_merger_tensor(mtmd_format(TN_VIT_MERGER_DS_LN, "weight")); + model.vit_merger_ds_ln_b = get_merger_tensor(mtmd_format(TN_VIT_MERGER_DS_LN, "bias")); + model.vit_merger_ds_up_w = get_merger_tensor(mtmd_format(TN_VIT_MERGER_DS_UP, "weight")); + model.vit_merger_ds_up_b = get_merger_tensor(mtmd_format(TN_VIT_MERGER_DS_UP, "bias"), false); + model.vit_merger_ds_down_w = get_merger_tensor(mtmd_format(TN_VIT_MERGER_DS_DOWN, "weight")); + model.vit_merger_ds_down_b = get_merger_tensor(mtmd_format(TN_VIT_MERGER_DS_DOWN, "bias"), false); + // Final Merger (DownsampleMLP) + model.mm_input_norm_w = get_tensor(TN_MM_INP_NORM); + model.mm_input_norm_b = get_tensor(TN_MM_INP_NORM_B, false); + model.mm_ffn_up_w = get_tensor(mtmd_format(TN_MM_UP, "weight")); + model.mm_ffn_up_b = get_tensor(mtmd_format(TN_MM_UP, "bias"), false); + model.mm_ffn_down_w = get_tensor(mtmd_format(TN_MM_DOWN, "weight")); + model.mm_ffn_down_b = get_tensor(mtmd_format(TN_MM_DOWN, "bias"), false); + model.tok_embd_sep = get_tensor(TN_TOK_EMBD_SEP, model.proj_type == PROJECTOR_TYPE_MINICPMV4_7); + } break; + case PROJECTOR_TYPE_GLM_EDGE: + { + model.mm_model_adapter_conv_w = get_tensor(mtmd_format(TN_GLM_ADAPER_CONV, "weight")); + model.mm_model_adapter_conv_b = get_tensor(mtmd_format(TN_GLM_ADAPER_CONV, "bias")); + model.mm_model_mlp_0_w = get_tensor(mtmd_format(TN_GLM_ADAPTER_LINEAR, "weight")); + model.mm_model_ln_q_w = get_tensor(mtmd_format(TN_GLM_ADAPTER_NORM_1, "weight")); + model.mm_model_ln_q_b = get_tensor(mtmd_format(TN_GLM_ADAPTER_NORM_1, "bias")); + model.mm_model_mlp_1_w = get_tensor(mtmd_format(TN_GLM_ADAPTER_D_H_2_4H, "weight")); + model.mm_model_mlp_2_w = get_tensor(mtmd_format(TN_GLM_ADAPTER_GATE, "weight")); + model.mm_model_mlp_3_w = get_tensor(mtmd_format(TN_GLM_ADAPTER_D_4H_2_H, "weight")); + model.mm_boi = get_tensor(mtmd_format(TN_TOK_GLM_BOI)); + model.mm_eoi = get_tensor(mtmd_format(TN_TOK_GLM_EOI)); + } break; + case PROJECTOR_TYPE_QWEN2VL: + case PROJECTOR_TYPE_QWEN25VL: + case PROJECTOR_TYPE_EXAONE4_5: + { + model.mm_0_w = get_tensor(mtmd_format(TN_LLAVA_PROJ, 0, "weight")); + model.mm_0_b = get_tensor(mtmd_format(TN_LLAVA_PROJ, 0, "bias")); + model.mm_1_w = get_tensor(mtmd_format(TN_LLAVA_PROJ, 2, "weight")); + model.mm_1_b = get_tensor(mtmd_format(TN_LLAVA_PROJ, 2, "bias")); + } break; + case PROJECTOR_TYPE_MAGEVL: + { + // d_head rope factors: the first half drives the even outputs, the second half the odd ones + model.rope_freqs = get_tensor(TN_ROPE_FREQS); + const int d_head = hparams.n_head > 0 ? hparams.n_embd / hparams.n_head : 0; + if (d_head <= 0 || d_head * hparams.n_head != hparams.n_embd || d_head % 32 != 0 || + model.rope_freqs->type != GGML_TYPE_F32 || ggml_nelements(model.rope_freqs) != d_head) { + throw std::runtime_error(mtmd_format("%s: hparams do not match the tensor shapes\n", __func__)); + } + model.mm_0_w = get_tensor(mtmd_format(TN_LLAVA_PROJ, 0, "weight")); + model.mm_0_b = get_tensor(mtmd_format(TN_LLAVA_PROJ, 0, "bias")); + model.mm_1_w = get_tensor(mtmd_format(TN_LLAVA_PROJ, 2, "weight")); + model.mm_1_b = get_tensor(mtmd_format(TN_LLAVA_PROJ, 2, "bias")); + } break; + case PROJECTOR_TYPE_QWEN3VL: + { + model.mm_0_w = get_tensor(mtmd_format(TN_LLAVA_PROJ, 0, "weight")); + model.mm_0_b = get_tensor(mtmd_format(TN_LLAVA_PROJ, 0, "bias")); + model.mm_1_w = get_tensor(mtmd_format(TN_LLAVA_PROJ, 2, "weight")); + model.mm_1_b = get_tensor(mtmd_format(TN_LLAVA_PROJ, 2, "bias")); + } break; + case PROJECTOR_TYPE_LFM3VL: + { + model.mm_input_norm_w = get_tensor(TN_MM_INP_NORM); + model.mm_input_norm_b = get_tensor(TN_MM_INP_NORM_B); + model.mm_0_w = get_tensor(mtmd_format(TN_LLAVA_PROJ, 0, "weight")); + model.mm_0_b = get_tensor(mtmd_format(TN_LLAVA_PROJ, 0, "bias")); + model.mm_1_w = get_tensor(mtmd_format(TN_LLAVA_PROJ, 2, "weight")); + model.mm_1_b = get_tensor(mtmd_format(TN_LLAVA_PROJ, 2, "bias")); + // each layer must be either a conv block or a complete attention block + for (size_t il = 0; il < model.layers.size(); ++il) { + const auto & layer = model.layers[il]; + const bool is_conv = layer.conv2d_in_proj_w != nullptr; + const bool is_attn = layer.q_w && layer.k_w && layer.v_w && layer.o_w; + if (is_conv == is_attn) { + throw std::runtime_error(mtmd_format("%s: lfm3vl layer %zu is neither a complete conv nor a complete attention block\n", __func__, il)); + } + } + } break; + case PROJECTOR_TYPE_LING3VL: + { + model.mm_input_norm_w = get_tensor(TN_MM_INP_NORM); // merger.norm + model.mm_input_norm_b = get_tensor(TN_MM_INP_NORM_B); // merger.norm + model.mm_0_w = get_tensor(mtmd_format(TN_LLAVA_PROJ, 0, "weight")); // linear_proj.0 + model.mm_0_b = get_tensor(mtmd_format(TN_LLAVA_PROJ, 0, "bias")); + model.mm_1_w = get_tensor(mtmd_format(TN_LLAVA_PROJ, 2, "weight")); // linear_proj.2 + model.mm_1_b = get_tensor(mtmd_format(TN_LLAVA_PROJ, 2, "bias")); + } break; + case PROJECTOR_TYPE_MIMOVL: + { + model.mm_0_w = get_tensor(mtmd_format(TN_LLAVA_PROJ, 0, "weight")); + model.mm_0_b = get_tensor(mtmd_format(TN_LLAVA_PROJ, 0, "bias"), false); + model.mm_1_w = get_tensor(mtmd_format(TN_LLAVA_PROJ, 2, "weight")); + model.mm_1_b = get_tensor(mtmd_format(TN_LLAVA_PROJ, 2, "bias"), false); + } break; + case PROJECTOR_TYPE_MINIMAX_M3: + { + // per-patch MLP: mm.1 -> gelu -> mm.2 + model.mm_1_w = get_tensor(mtmd_format(TN_LLAVA_PROJ, 1, "weight")); + model.mm_1_b = get_tensor(mtmd_format(TN_LLAVA_PROJ, 1, "bias")); + model.mm_2_w = get_tensor(mtmd_format(TN_LLAVA_PROJ, 2, "weight")); + model.mm_2_b = get_tensor(mtmd_format(TN_LLAVA_PROJ, 2, "bias")); + // 2x2 merge MLP: mm.merge.fc1 -> gelu -> mm.merge.fc2 + model.mm_merger_fc1_w = get_tensor(mtmd_format(TN_MM_MERGER_FC1, "weight")); + model.mm_merger_fc1_b = get_tensor(mtmd_format(TN_MM_MERGER_FC1, "bias")); + model.mm_merger_fc2_w = get_tensor(mtmd_format(TN_MM_MERGER_FC2, "weight")); + model.mm_merger_fc2_b = get_tensor(mtmd_format(TN_MM_MERGER_FC2, "bias")); + } break; + case PROJECTOR_TYPE_MUSE_SPARK_AUDIO: + { + model.layers.resize(hparams.n_layer); + model.a_patches.resize(hparams.audio_patch_sizes.size()); + for (size_t i = 0; i < model.a_patches.size(); ++i) { + auto & pe = model.a_patches[i]; + pe.proj_w = get_tensor(mtmd_format(TN_A_PATCH, (int) i, "weight")); + pe.proj_b = get_tensor(mtmd_format(TN_A_PATCH, (int) i, "bias")); + pe.norm_w = get_tensor(mtmd_format(TN_A_PATCH_NORM, (int) i, "weight")); + if (pe.proj_w->ne[1] != hparams.audio_hidden_sizes[i]) { + throw std::runtime_error(mtmd_format("%s: audio stage %zu projection has width %" PRId64 ", expected %d\n", __func__, i, pe.proj_w->ne[1], hparams.audio_hidden_sizes[i])); + } + } + + for (int il = 0; il < hparams.n_layer; ++il) { + auto & layer = model.layers[il]; + layer.conv_w = get_tensor(mtmd_format(TN_CONV1D, il, "weight")); + layer.conv_b = get_tensor(mtmd_format(TN_CONV1D, il, "bias")); + layer.conv_norm_w = get_tensor(mtmd_format(TN_A_CONV1D_NORM, il, "weight")); + layer.ff_up_w = get_tensor(mtmd_format(TN_FFN_UP, "a", il, "weight")); + layer.ff_up_b = get_tensor(mtmd_format(TN_FFN_UP, "a", il, "bias")); + layer.ff_down_w = get_tensor(mtmd_format(TN_FFN_DOWN, "a", il, "weight")); + layer.ff_down_b = get_tensor(mtmd_format(TN_FFN_DOWN, "a", il, "bias")); + } + + // pre_ln and post_ln come from the generic path above, where they are optional + if (!model.pre_ln_w || !model.post_ln_w) { + throw std::runtime_error(mtmd_format("%s: missing pre_ln or post_ln weight\n", __func__)); + } + model.mm_fc_w = get_tensor(mtmd_format(TN_MM_AUDIO_FC, "weight")); + model.mm_fc_b = get_tensor(mtmd_format(TN_MM_AUDIO_FC, "bias")); + } break; + case PROJECTOR_TYPE_MUSE_GLIMMER: + { + // 3-linear MLP: fc -> erf-GELU -> proj -> erf-GELU -> vision_proj (into LLM residual dim) + model.mm_0_w = get_tensor(mtmd_format(TN_LLAVA_PROJ, 0, "weight")); + model.mm_1_w = get_tensor(mtmd_format(TN_LLAVA_PROJ, 1, "weight")); + model.mm_2_w = get_tensor(mtmd_format(TN_LLAVA_PROJ, 2, "weight")); + } break; + case PROJECTOR_TYPE_STEP3VL: + { + model.mm_0_w = get_tensor(mtmd_format(TN_LLAVA_PROJ, 0, "weight")); + model.mm_0_b = get_tensor(mtmd_format(TN_LLAVA_PROJ, 0, "bias"), false); + model.mm_1_w = get_tensor(mtmd_format(TN_LLAVA_PROJ, 1, "weight")); + model.mm_1_b = get_tensor(mtmd_format(TN_LLAVA_PROJ, 1, "bias"), false); + model.mm_model_proj = get_tensor(mtmd_format(TN_MM_PROJECTOR, "weight")); + } break; + case PROJECTOR_TYPE_YOUTUVL: + { + model.mm_input_norm_w = get_tensor(TN_MM_INP_NORM); // merger.ln_q (RMS norm) + model.mm_0_w = get_tensor(mtmd_format(TN_LLAVA_PROJ, 0, "weight")); // merger.mlp.0 + model.mm_0_b = get_tensor(mtmd_format(TN_LLAVA_PROJ, 0, "bias")); + model.mm_1_w = get_tensor(mtmd_format(TN_LLAVA_PROJ, 2, "weight")); // merger.mlp.2 + model.mm_1_b = get_tensor(mtmd_format(TN_LLAVA_PROJ, 2, "bias")); + } break; + case PROJECTOR_TYPE_YASA2: + { + // reuse tensors already loaded by the common section + // (TN_PATCH_EMBD and TN_PATCH_BIAS have the same tensor names) + model.yasa_patch_w = model.patch_embeddings_0; + model.yasa_patch_b = model.patch_bias; + model.yasa_patch_ln_w = get_tensor(TN_YASA_PATCH_LN_W, false); + model.yasa_patch_ln_b = get_tensor(TN_YASA_PATCH_LN_B, false); + model.yasa_backbone_ln_w = get_tensor(TN_YASA_BACKBONE_LN_W, false); + model.yasa_backbone_ln_b = get_tensor(TN_YASA_BACKBONE_LN_B, false); + model.yasa_vision_pos_embed = get_tensor(TN_YASA_POS_EMBD, false); + model.mm_0_w = get_tensor(mtmd_format(TN_LLAVA_PROJ, 0, "weight")); + model.mm_0_b = get_tensor(mtmd_format(TN_LLAVA_PROJ, 0, "bias"), false); + model.mm_2_w = get_tensor(mtmd_format(TN_LLAVA_PROJ, 2, "weight")); + model.mm_2_b = get_tensor(mtmd_format(TN_LLAVA_PROJ, 2, "bias"), false); + + model.yasa_stages.clear(); + for (int s = 0; ; ++s) { + yasa2_stage stage; + stage.down_ln_w = get_tensor(mtmd_format(TN_YASA_STAGE_DOWN_LN, s, "weight"), false); + stage.down_ln_b = get_tensor(mtmd_format(TN_YASA_STAGE_DOWN_LN, s, "bias"), false); + stage.down_conv_w = get_tensor(mtmd_format(TN_YASA_STAGE_DOWN_CONV, s, "weight"), false); + stage.down_conv_b = get_tensor(mtmd_format(TN_YASA_STAGE_DOWN_CONV, s, "bias"), false); + + for (int bi = 0; ; ++bi) { + yasa2_block blk; + blk.dw_w = get_tensor(mtmd_format(TN_YASA_STAGE_BLK, s, bi, "dw", "weight"), false); + if (!blk.dw_w) { + break; + } + blk.dw_b = get_tensor(mtmd_format(TN_YASA_STAGE_BLK, s, bi, "dw", "bias"), false); + blk.ln_w = get_tensor(mtmd_format(TN_YASA_STAGE_BLK, s, bi, "ln", "weight"), false); + blk.ln_b = get_tensor(mtmd_format(TN_YASA_STAGE_BLK, s, bi, "ln", "bias"), false); + blk.pw1_w = get_tensor(mtmd_format(TN_YASA_STAGE_BLK, s, bi, "pw1", "weight"), false); + blk.pw1_b = get_tensor(mtmd_format(TN_YASA_STAGE_BLK, s, bi, "pw1", "bias"), false); + blk.grn_w = get_tensor(mtmd_format(TN_YASA_STAGE_BLK, s, bi, "grn", "weight"), false); + blk.grn_b = get_tensor(mtmd_format(TN_YASA_STAGE_BLK, s, bi, "grn", "bias"), false); + blk.pw2_w = get_tensor(mtmd_format(TN_YASA_STAGE_BLK, s, bi, "pw2", "weight"), false); + blk.pw2_b = get_tensor(mtmd_format(TN_YASA_STAGE_BLK, s, bi, "pw2", "bias"), false); + stage.blocks.push_back(blk); + } + + if (!stage.down_conv_w && stage.blocks.empty()) { + break; + } + model.yasa_stages.push_back(std::move(stage)); + } + } break; + case PROJECTOR_TYPE_GLM4V: + case PROJECTOR_TYPE_GLM5V: + { + model.mm_fc_w = get_tensor(mtmd_format(TN_MM_PROJECTOR, "weight")); + model.mm_ffn_up_w = get_tensor(mtmd_format(TN_MM_UP, "weight")); + model.mm_ffn_up_b = get_tensor(mtmd_format(TN_MM_UP, "bias"), false); + model.mm_ffn_gate_w = get_tensor(mtmd_format(TN_MM_GATE, "weight")); + model.mm_ffn_gate_b = get_tensor(mtmd_format(TN_MM_GATE, "bias"), false); + model.mm_ffn_down_w = get_tensor(mtmd_format(TN_MM_DOWN, "weight")); + model.mm_ffn_down_b = get_tensor(mtmd_format(TN_MM_DOWN, "bias"), false); + model.mm_post_norm_w = get_tensor(mtmd_format(TN_MM_POST_NORM, "weight")); + model.mm_post_norm_b = get_tensor(mtmd_format(TN_MM_POST_NORM, "bias"), false); + model.mm_patch_merger_w = get_tensor(mtmd_format(TN_MM_PATCH_MERGER, "weight")); + model.mm_patch_merger_b = get_tensor(mtmd_format(TN_MM_PATCH_MERGER, "bias")); + } break; + case PROJECTOR_TYPE_GEMMA3: + { + model.mm_input_proj_w = get_tensor(TN_MM_INP_PROJ); + model.mm_soft_emb_norm_w = get_tensor(TN_MM_SOFT_EMB_N); + } break; + case PROJECTOR_TYPE_GEMMA4V: + { + model.mm_input_proj_w = get_tensor(TN_MM_INP_PROJ); + model.std_bias = get_tensor(TN_STD_BIAS, false); + model.std_scale = get_tensor(TN_STD_SCALE, false); + // load scalar for Gemma4ClippableLinear + for (auto * tensor : tensors_to_load) { + std::string name = tensor->name; + if (string_ends_with(name, ".weight")) { + std::string name_inp_max = name; + std::string name_inp_min = name; + std::string name_out_max = name; + std::string name_out_min = name; + string_replace_all(name_inp_max, ".weight", ".input_max"); + string_replace_all(name_inp_min, ".weight", ".input_min"); + string_replace_all(name_out_max, ".weight", ".output_max"); + string_replace_all(name_out_min, ".weight", ".output_min"); + model.clamp_info_map[name] = { + get_scalar(name_inp_max, FLT_MAX), + get_scalar(name_inp_min, -FLT_MAX), + get_scalar(name_out_max, FLT_MAX), + get_scalar(name_out_min, -FLT_MAX) + }; + } + } + } break; + case PROJECTOR_TYPE_GEMMA4UV: + { + model.mm_input_proj_w = get_tensor(TN_MM_INP_PROJ); + model.patch_norm_1_w = get_tensor(mtmd_format(TN_PATCH_NORM, 1, "weight")); + model.patch_norm_1_b = get_tensor(mtmd_format(TN_PATCH_NORM, 1, "bias")); + model.patch_norm_2_w = get_tensor(mtmd_format(TN_PATCH_NORM, 2, "weight")); + model.patch_norm_2_b = get_tensor(mtmd_format(TN_PATCH_NORM, 2, "bias")); + model.patch_norm_3_w = get_tensor(mtmd_format(TN_PATCH_NORM, 3, "weight")); // pos_norm + model.patch_norm_3_b = get_tensor(mtmd_format(TN_PATCH_NORM, 3, "bias")); // pos_norm + } break; + case PROJECTOR_TYPE_GEMMA3NV: + { + model.mobilenet_stem_conv_w = get_tensor(TN_MNV5_STEM_CONV, false); + model.mobilenet_stem_conv_b = get_tensor(TN_MNV5_STEM_BIAS, false); + model.mobilenet_stem_norm_w = get_tensor(TN_MNV5_STEM_BN, false); + + model.msfa_ffn_expand_w = get_tensor(TN_MNV5_MSFA_FFN_EXP_W, false); + model.msfa_ffn_expand_bn = get_tensor(TN_MNV5_MSFA_FFN_EXP_BN, false); // Consume BN if present but likely folded + model.msfa_ffn_project_w = get_tensor(TN_MNV5_MSFA_FFN_PROJ_W, false); + model.msfa_ffn_project_bn = get_tensor(TN_MNV5_MSFA_FFN_PROJ_BN, false); + + model.msfa_concat_norm_w = get_tensor(TN_MNV5_MSFA_NORM, false); + + // Dynamically load blocks stage by stage + for (int stage = 0; stage < 4; ++stage) { + int blocks_found_in_stage = 0; + + for (int blk_idx = 0; ; ++blk_idx) { + bool found_block = false; + mobilenetv5_block block; + + // 1. Check for Edge Residual (S0) + block.s0_conv_exp_w = get_tensor(mtmd_format(TN_MNV5_BLK_S0_EXP_W, stage, blk_idx), false); + if (block.s0_conv_exp_w) { + found_block = true; + block.s0_bn1_w = get_tensor(mtmd_format(TN_MNV5_BLK_S0_BN1_W, stage, blk_idx), false); + block.s0_conv_pwl_w = get_tensor(mtmd_format(TN_MNV5_BLK_S0_PWL_W, stage, blk_idx), false); + block.s0_bn2_w = get_tensor(mtmd_format(TN_MNV5_BLK_S0_BN2_W, stage, blk_idx), false); + } + // 2. Check for UIR (Universal Inverted Residual) + else { + // Check for dw_start OR pw_exp (some UIR blocks skip dw_start) + block.dw_start_w = get_tensor(mtmd_format(TN_MNV5_BLK_DW_START_W, stage, blk_idx), false); + block.pw_exp_w = get_tensor(mtmd_format(TN_MNV5_BLK_PW_EXP_W, stage, blk_idx), false); + + if (block.dw_start_w || block.pw_exp_w) { + found_block = true; + if (block.dw_start_w) { + block.dw_start_bn_w = get_tensor(mtmd_format(TN_MNV5_BLK_DW_START_BN, stage, blk_idx), false); + } + if (block.pw_exp_w) { + block.pw_exp_bn_w = get_tensor(mtmd_format(TN_MNV5_BLK_PW_EXP_BN, stage, blk_idx), false); + } + block.dw_mid_w = get_tensor(mtmd_format(TN_MNV5_BLK_DW_MID_W, stage, blk_idx), false); + if (block.dw_mid_w) { + block.dw_mid_bn_w = get_tensor(mtmd_format(TN_MNV5_BLK_DW_MID_BN, stage, blk_idx), false); + } + block.pw_proj_w = get_tensor(mtmd_format(TN_MNV5_BLK_PW_PROJ_W, stage, blk_idx), false); + if (block.pw_proj_w) { + block.pw_proj_bn_w = get_tensor(mtmd_format(TN_MNV5_BLK_PW_PROJ_BN, stage, blk_idx), false); + } + block.layer_scale_w = get_tensor(mtmd_format(TN_MNV5_BLK_LAYER_SCALE, stage, blk_idx), false); + } + } + + // 3. Check for Attention (MQA) + // Even if UIR/Edge check failed, this might be a pure attention block + ggml_tensor* attn_q_check = get_tensor(mtmd_format(TN_MNV5_ATTN_Q_W, stage, blk_idx), false); + if (attn_q_check) { + found_block = true; + block.attn_q_w = attn_q_check; + block.attn_k_w = get_tensor(mtmd_format(TN_MNV5_ATTN_K_W, stage, blk_idx), false); + block.attn_v_w = get_tensor(mtmd_format(TN_MNV5_ATTN_V_W, stage, blk_idx), false); + block.attn_o_w = get_tensor(mtmd_format(TN_MNV5_ATTN_O_W, stage, blk_idx), false); + block.attn_k_dw_w = get_tensor(mtmd_format(TN_MNV5_ATTN_K_DW, stage, blk_idx), false); + block.attn_k_norm_w = get_tensor(mtmd_format(TN_MNV5_ATTN_K_NORM, stage, blk_idx), false); + block.attn_v_dw_w = get_tensor(mtmd_format(TN_MNV5_ATTN_V_DW, stage, blk_idx), false); + block.attn_v_norm_w = get_tensor(mtmd_format(TN_MNV5_ATTN_V_NORM, stage, blk_idx), false); + block.attn_norm_w = get_tensor(mtmd_format(TN_MNV5_ATTN_NORM, stage, blk_idx), false); + // Note: Attention blocks also have layer_scale, load it if not already loaded by UIR check + if (!block.layer_scale_w) { + block.layer_scale_w = get_tensor(mtmd_format(TN_MNV5_BLK_LAYER_SCALE, stage, blk_idx), false); + } + } + + if (found_block) { + model.mobilenet_blocks.push_back(block); + blocks_found_in_stage++; + } else { + // End of blocks for this stage + break; + } + } + + // Track where this stage ends in the flat vector + if (blocks_found_in_stage > 0) { + model.mobilenet_stage_ends.push_back(model.mobilenet_blocks.size() - 1); + LOG_INF("%s: Stage %d ended at global block index %zu\n", __func__, stage, model.mobilenet_blocks.size() - 1); + } + } + model.mm_input_proj_w = get_tensor(TN_MM_INP_PROJ); + model.mm_soft_emb_norm_w = get_tensor(TN_MM_SOFT_EMB_N); + } break; + case PROJECTOR_TYPE_IDEFICS3: + { + model.mm_fc_w = get_tensor(mtmd_format(TN_MM_PROJECTOR, "weight")); + } break; + case PROJECTOR_TYPE_COHERE2V: + { + model.mm_1_w = get_tensor(mtmd_format(TN_LLAVA_PROJ, 1, "weight")); + model.mm_1_b = get_tensor(mtmd_format(TN_LLAVA_PROJ, 1, "bias")); + model.mm_2_w = get_tensor(mtmd_format(TN_LLAVA_PROJ, 2, "weight")); + model.mm_2_b = get_tensor(mtmd_format(TN_LLAVA_PROJ, 2, "bias")); + } break; + case PROJECTOR_TYPE_LFM2: + case PROJECTOR_TYPE_D1OMNI_V: + { + model.mm_input_norm_w = get_tensor(TN_MM_INP_NORM, false); + model.mm_input_norm_b = get_tensor(TN_MM_INP_NORM_B, false); + model.mm_1_w = get_tensor(mtmd_format(TN_LLAVA_PROJ, 1, "weight")); + model.mm_1_b = get_tensor(mtmd_format(TN_LLAVA_PROJ, 1, "bias")); + model.mm_2_w = get_tensor(mtmd_format(TN_LLAVA_PROJ, 2, "weight")); + model.mm_2_b = get_tensor(mtmd_format(TN_LLAVA_PROJ, 2, "bias")); + } break; + case PROJECTOR_TYPE_KIMIVL: + case PROJECTOR_TYPE_PADDLEOCR: + case PROJECTOR_TYPE_KIMIK25: + { + model.mm_input_norm_w = get_tensor(TN_MM_INP_NORM); + model.mm_input_norm_b = get_tensor(TN_MM_INP_NORM_B); + model.mm_1_w = get_tensor(mtmd_format(TN_LLAVA_PROJ, 1, "weight")); + model.mm_1_b = get_tensor(mtmd_format(TN_LLAVA_PROJ, 1, "bias")); + model.mm_2_w = get_tensor(mtmd_format(TN_LLAVA_PROJ, 2, "weight")); + model.mm_2_b = get_tensor(mtmd_format(TN_LLAVA_PROJ, 2, "bias")); + } break; + case PROJECTOR_TYPE_KIMIK31: + { + model.mm_1_w = get_tensor(mtmd_format(TN_LLAVA_PROJ, 1, "weight")); + model.mm_2_w = get_tensor(mtmd_format(TN_LLAVA_PROJ, 2, "weight")); + model.mm_post_norm_w = get_tensor(mtmd_format(TN_MM_POST_NORM, "weight")); + } break; + case PROJECTOR_TYPE_DEEPSEEK4V: + { + model.mm_1_w = get_tensor(mtmd_format(TN_LLAVA_PROJ, 1, "weight")); + model.mm_1_b = get_tensor(mtmd_format(TN_LLAVA_PROJ, 1, "bias")); + model.mm_2_w = get_tensor(mtmd_format(TN_LLAVA_PROJ, 2, "weight")); + model.mm_2_b = get_tensor(mtmd_format(TN_LLAVA_PROJ, 2, "bias")); + // sentinel token embeddings written into the output block + model.image_newline = get_tensor(TN_IMAGE_NEWLINE); + model.token_embd_img_start = get_tensor(TN_TOK_IMG_START); + model.token_embd_img_end = get_tensor(TN_TOK_IMG_END); + model.token_embd_img_pad = get_tensor(TN_TOK_IMG_PAD); + } break; + case PROJECTOR_TYPE_DEEPSEEK41V: + { + model.mm_1_w = get_tensor(mtmd_format(TN_LLAVA_PROJ, 1, "weight")); + model.mm_1_b = get_tensor(mtmd_format(TN_LLAVA_PROJ, 1, "bias")); + model.mm_2_w = get_tensor(mtmd_format(TN_LLAVA_PROJ, 2, "weight")); + model.mm_2_b = get_tensor(mtmd_format(TN_LLAVA_PROJ, 2, "bias")); + // image_start / image_end / image_newline, written into the output block + model.image_newline = get_tensor(TN_IMAGE_NEWLINE); + model.token_embd_img_start = get_tensor(TN_TOK_IMG_START); + model.token_embd_img_end = get_tensor(TN_TOK_IMG_END); + } break; + case PROJECTOR_TYPE_PIXTRAL: + { + model.mm_1_w = get_tensor(mtmd_format(TN_LLAVA_PROJ, 1, "weight")); + model.mm_1_b = get_tensor(mtmd_format(TN_LLAVA_PROJ, 1, "bias"), false); + model.mm_2_w = get_tensor(mtmd_format(TN_LLAVA_PROJ, 2, "weight")); + model.mm_2_b = get_tensor(mtmd_format(TN_LLAVA_PROJ, 2, "bias"), false); + // [IMG_BREAK] token embedding + model.token_embd_img_break = get_tensor(TN_TOK_IMG_BREAK); + // for mistral small 3.1 + model.mm_input_norm_w = get_tensor(TN_MM_INP_NORM, false); + model.mm_patch_merger_w = get_tensor(mtmd_format(TN_MM_PATCH_MERGER, "weight"), false); + } break; + case PROJECTOR_TYPE_LIGHTONOCR: + { + model.mm_1_w = get_tensor(mtmd_format(TN_LLAVA_PROJ, 1, "weight")); + model.mm_1_b = get_tensor(mtmd_format(TN_LLAVA_PROJ, 1, "bias"), false); + model.mm_2_w = get_tensor(mtmd_format(TN_LLAVA_PROJ, 2, "weight")); + model.mm_2_b = get_tensor(mtmd_format(TN_LLAVA_PROJ, 2, "bias"), false); + model.mm_input_norm_w = get_tensor(TN_MM_INP_NORM, false); + model.mm_patch_merger_w = get_tensor(mtmd_format(TN_MM_PATCH_MERGER, "weight"), false); + } break; + case PROJECTOR_TYPE_DOTS_OCR: + case PROJECTOR_TYPE_DOTS3NOTE_V: + { + model.mm_0_w = get_tensor(mtmd_format(TN_LLAVA_PROJ, 0, "weight")); + model.mm_0_b = get_tensor(mtmd_format(TN_LLAVA_PROJ, 0, "bias")); + model.mm_2_w = get_tensor(mtmd_format(TN_LLAVA_PROJ, 2, "weight")); + model.mm_2_b = get_tensor(mtmd_format(TN_LLAVA_PROJ, 2, "bias")); + model.mm_input_norm_w = get_tensor(TN_MM_INP_NORM); + model.mm_input_norm_b = get_tensor(TN_MM_INP_NORM_B); + // post_trunk_norm: applied after all ViT blocks, before the merger + model.post_ln_w = get_tensor(mtmd_format(TN_MM_POST_NORM, "weight")); + } break; + case PROJECTOR_TYPE_DOTS3NOTE_A: + { + model.conv2d_1_w = get_tensor(mtmd_format(TN_CONV2D, 1, "weight")); + model.conv2d_1_b = get_tensor(mtmd_format(TN_CONV2D, 1, "bias")); + model.conv2d_2_w = get_tensor(mtmd_format(TN_CONV2D, 2, "weight")); + model.conv2d_2_b = get_tensor(mtmd_format(TN_CONV2D, 2, "bias")); + model.conv2d_3_w = get_tensor(mtmd_format(TN_CONV2D, 3, "weight")); + model.conv2d_3_b = get_tensor(mtmd_format(TN_CONV2D, 3, "bias")); + model.conv_out_w = get_tensor(mtmd_format(TN_CONV_OUT, "weight")); // no bias + // adapter: LayerNorm -> Linear -> GELU -> Linear + model.mm_norm_pre_w = get_tensor(mtmd_format(TN_MM_NORM_PRE, "weight")); + model.mm_norm_pre_b = get_tensor(mtmd_format(TN_MM_NORM_PRE, "bias")); + model.mm_1_w = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 1, "weight")); + model.mm_1_b = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 1, "bias")); + model.mm_2_w = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 3, "weight")); + model.mm_2_b = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 3, "bias")); + } break; + case PROJECTOR_TYPE_ULTRAVOX: + { + model.conv1d_1_w = get_tensor(mtmd_format(TN_CONV1D, 1, "weight")); + model.conv1d_1_b = get_tensor(mtmd_format(TN_CONV1D, 1, "bias")); + model.conv1d_2_w = get_tensor(mtmd_format(TN_CONV1D, 2, "weight")); + model.conv1d_2_b = get_tensor(mtmd_format(TN_CONV1D, 2, "bias")); + model.mm_1_w = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 1, "weight")); + model.mm_2_w = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 2, "weight")); + model.mm_norm_pre_w = get_tensor(mtmd_format(TN_MM_NORM_PRE, "weight")); + model.mm_norm_mid_w = get_tensor(mtmd_format(TN_MM_NORM_MID, "weight")); + } break; + case PROJECTOR_TYPE_MERALION: + { + // Whisper encoder conv layers + model.conv1d_1_w = get_tensor(mtmd_format(TN_CONV1D, 1, "weight")); + model.conv1d_1_b = get_tensor(mtmd_format(TN_CONV1D, 1, "bias")); + model.conv1d_2_w = get_tensor(mtmd_format(TN_CONV1D, 2, "weight")); + model.conv1d_2_b = get_tensor(mtmd_format(TN_CONV1D, 2, "bias")); + // MERaLiON adaptor: 4 linear layers + ln_pre + // linear_0 = frame compression (19200->6400) + SiLU + // linear_1 = gate_proj (6400->6400) for GLU + // linear_2 = pool_proj (6400->6400) for GLU + // linear_3 = out_proj (6400->3584) + model.mm_0_w = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 0, "weight")); + model.mm_0_b = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 0, "bias")); + model.mm_1_w = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 1, "weight")); + model.mm_1_b = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 1, "bias")); + model.mm_2_w = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 2, "weight")); + model.mm_2_b = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 2, "bias")); + model.mm_3_w = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 3, "weight")); + model.mm_3_b = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 3, "bias")); + // ln_speech (LayerNorm before adaptor) + model.mm_norm_pre_w = get_tensor(mtmd_format(TN_MM_NORM_PRE, "weight")); + model.mm_norm_pre_b = get_tensor(mtmd_format(TN_MM_NORM_PRE, "bias")); + } break; + case PROJECTOR_TYPE_QWEN2A: + { + model.conv1d_1_w = get_tensor(mtmd_format(TN_CONV1D, 1, "weight")); + model.conv1d_1_b = get_tensor(mtmd_format(TN_CONV1D, 1, "bias")); + model.conv1d_2_w = get_tensor(mtmd_format(TN_CONV1D, 2, "weight")); + model.conv1d_2_b = get_tensor(mtmd_format(TN_CONV1D, 2, "bias")); + model.mm_fc_w = get_tensor(mtmd_format(TN_MM_AUDIO_FC, "weight")); + model.mm_fc_b = get_tensor(mtmd_format(TN_MM_AUDIO_FC, "bias")); + } break; + case PROJECTOR_TYPE_QWEN3A: + { + model.conv2d_1_w = get_tensor(mtmd_format(TN_CONV2D, 1, "weight")); + model.conv2d_1_b = get_tensor(mtmd_format(TN_CONV2D, 1, "bias")); + model.conv2d_2_w = get_tensor(mtmd_format(TN_CONV2D, 2, "weight")); + model.conv2d_2_b = get_tensor(mtmd_format(TN_CONV2D, 2, "bias")); + model.conv2d_3_w = get_tensor(mtmd_format(TN_CONV2D, 3, "weight")); + model.conv2d_3_b = get_tensor(mtmd_format(TN_CONV2D, 3, "bias")); + model.conv_out_w = get_tensor(mtmd_format(TN_CONV_OUT, "weight")); // no bias + model.mm_1_w = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 1, "weight")); + model.mm_1_b = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 1, "bias")); + model.mm_2_w = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 2, "weight")); + model.mm_2_b = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 2, "bias")); + } break; + case PROJECTOR_TYPE_MIMO_AUDIO: + { + model.conv1d_1_w = get_tensor(mtmd_format(TN_CONV1D, 1, "weight")); + model.conv1d_1_b = get_tensor(mtmd_format(TN_CONV1D, 1, "bias")); + model.conv1d_2_w = get_tensor(mtmd_format(TN_CONV1D, 2, "weight")); + model.conv1d_2_b = get_tensor(mtmd_format(TN_CONV1D, 2, "bias")); + model.downsample_conv_w = get_tensor(mtmd_format(TN_A_DOWNSAMPLE_CONV, "weight")); + model.downsample_norm_w = get_tensor(mtmd_format(TN_A_DOWNSAMPLE_NORM, "weight")); + model.downsample_norm_b = get_tensor(mtmd_format(TN_A_DOWNSAMPLE_NORM, "bias")); + model.rvq_codebook = get_tensor(mtmd_format(TN_A_RVQ_CODEBOOK, "weight"), false); + model.mm_a_code_embd = get_tensor(mtmd_format(TN_MM_A_CODE_EMBD, "weight"), false); + if (!model.rvq_codebook || !model.mm_a_code_embd) { + throw std::runtime_error(mtmd_format("%s: mimo_audio: missing %s or %s\n", __func__, + TN_A_RVQ_CODEBOOK, TN_MM_A_CODE_EMBD)); + } + // hparams.rvq_codebook_size comes from GGUF metadata and is independent of the + // tensors' actual shapes - bound it so codebook/code_embd views built from it + // (mimo-audio.cpp) can never read past either tensor's allocated bins/vocab. + for (int32_t bins : hparams.rvq_codebook_size) { + if (bins <= 0 || bins > model.rvq_codebook->ne[1] || bins > model.mm_a_code_embd->ne[1]) { + throw std::runtime_error(mtmd_format( + "%s: mimo_audio: %s entry (%d) out of range for codebook/code_embd tensors\n", + __func__, KEY_A_RVQ_CODEBOOK_SIZE, bins)); + } + } + + // LLM-side connector: input_local_transformer + projection + model.mm_a_local_layers.resize(hparams.audio_local_n_layer); + for (int il = 0; il < hparams.audio_local_n_layer; il++) { + auto & layer = model.mm_a_local_layers[il]; + layer.q_w = get_tensor(mtmd_format(TN_MM_A_LOCAL_ATTN_Q, il, "weight")); + layer.q_b = get_tensor(mtmd_format(TN_MM_A_LOCAL_ATTN_Q, il, "bias")); + layer.k_w = get_tensor(mtmd_format(TN_MM_A_LOCAL_ATTN_K, il, "weight")); + layer.k_b = get_tensor(mtmd_format(TN_MM_A_LOCAL_ATTN_K, il, "bias")); + layer.v_w = get_tensor(mtmd_format(TN_MM_A_LOCAL_ATTN_V, il, "weight")); + layer.v_b = get_tensor(mtmd_format(TN_MM_A_LOCAL_ATTN_V, il, "bias")); + layer.o_w = get_tensor(mtmd_format(TN_MM_A_LOCAL_ATTN_OUT, il, "weight")); + layer.ff_gate_w = get_tensor(mtmd_format(TN_MM_A_LOCAL_FFN_GATE, il, "weight")); + layer.ff_up_w = get_tensor(mtmd_format(TN_MM_A_LOCAL_FFN_UP, il, "weight")); + layer.ff_down_w = get_tensor(mtmd_format(TN_MM_A_LOCAL_FFN_DOWN, il, "weight")); + layer.ln_1_w = get_tensor(mtmd_format(TN_MM_A_LOCAL_LN1, il, "weight")); + layer.ln_2_w = get_tensor(mtmd_format(TN_MM_A_LOCAL_LN2, il, "weight")); + } + model.mm_a_local_norm_w = get_tensor(mtmd_format(TN_MM_A_LOCAL_NORM, "weight")); + + model.mm_1_w = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 1, "weight")); + model.mm_2_w = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 2, "weight")); + } break; + case PROJECTOR_TYPE_QWEN3TTS_SPKENC: + { + // stem TDNN (block 0) + model.conv1d_1_w = get_tensor(mtmd_format(TN_CONV1D, 0, "weight")); + model.conv1d_1_b = get_tensor(mtmd_format(TN_CONV1D, 0, "bias")); + + // SE-Res2Net blocks (GGUF bid 1..3, one per hparams.n_layer) + model.layers.resize(hparams.n_layer); + for (int il = 0; il < hparams.n_layer; il++) { + auto & layer = model.layers[il]; + int bid = il + 1; + layer.conv_pw1_w = get_tensor(mtmd_format(TN_CONV_PW1, prefix, bid, "weight")); + layer.conv_pw1_b = get_tensor(mtmd_format(TN_CONV_PW1, prefix, bid, "bias")); + layer.conv_pw2_w = get_tensor(mtmd_format(TN_CONV_PW2, prefix, bid, "weight")); + layer.conv_pw2_b = get_tensor(mtmd_format(TN_CONV_PW2, prefix, bid, "bias")); + layer.se_conv1_w = get_tensor(mtmd_format(TN_A_SE_CONV1, bid, "weight")); + layer.se_conv1_b = get_tensor(mtmd_format(TN_A_SE_CONV1, bid, "bias")); + layer.se_conv2_w = get_tensor(mtmd_format(TN_A_SE_CONV2, bid, "weight")); + layer.se_conv2_b = get_tensor(mtmd_format(TN_A_SE_CONV2, bid, "bias")); + layer.res2_conv_w.resize(7); + layer.res2_conv_b.resize(7); + for (int xid = 0; xid < 7; xid++) { + layer.res2_conv_w[xid] = get_tensor(mtmd_format(TN_A_CONV_RES2, bid, xid, "weight")); + layer.res2_conv_b[xid] = get_tensor(mtmd_format(TN_A_CONV_RES2, bid, xid, "bias")); + } + } + + // multi-layer feature aggregation + model.conv_out_w = get_tensor(mtmd_format(TN_CONV_OUT, "weight")); + model.conv_out_b = get_tensor(mtmd_format(TN_CONV_OUT, "bias")); + + // attentive statistics pooling + model.spk_asp_attn_w = get_tensor(mtmd_format(TN_A_ASP_ATTN, "weight")); + model.spk_asp_attn_b = get_tensor(mtmd_format(TN_A_ASP_ATTN, "bias")); + model.spk_asp_tdnn_w = get_tensor(mtmd_format(TN_A_ASP_TDNN, "weight")); + model.spk_asp_tdnn_b = get_tensor(mtmd_format(TN_A_ASP_TDNN, "bias")); + + // final speaker embedding projection + model.mm_fc_w = get_tensor(mtmd_format(TN_MM_AUDIO_FC, "weight")); + model.mm_fc_b = get_tensor(mtmd_format(TN_MM_AUDIO_FC, "bias")); + } break; + case PROJECTOR_TYPE_POCKETTTS_SPKENC: + { + load_seanet(model.seanet, false); + model.downsample_w = get_tensor(mtmd_format(TN_A_DOWNSAMPLE_CONV, "weight")); + model.spk_proj_w = get_tensor(mtmd_format(TN_A_SPEAKER_PROJ, "weight")); + } break; + case PROJECTOR_TYPE_POCKETTTS_GEN: + { + auto & flow = model.flow; + flow.input_proj_w = get_tensor(mtmd_format(TN_A_GEN_FLOW_INPUT_PROJ, "weight")); + flow.input_proj_b = get_tensor(mtmd_format(TN_A_GEN_FLOW_INPUT_PROJ, "bias")); + flow.cond_embd_w = get_tensor(mtmd_format(TN_A_GEN_FLOW_COND_EMBD, "weight")); + flow.cond_embd_b = get_tensor(mtmd_format(TN_A_GEN_FLOW_COND_EMBD, "bias")); + flow.final_ada_w = get_tensor(mtmd_format(TN_A_GEN_FLOW_FINAL_ADA, "weight")); + flow.final_ada_b = get_tensor(mtmd_format(TN_A_GEN_FLOW_FINAL_ADA, "bias")); + flow.final_proj_w = get_tensor(mtmd_format(TN_A_GEN_FLOW_FINAL_PROJ, "weight")); + flow.final_proj_b = get_tensor(mtmd_format(TN_A_GEN_FLOW_FINAL_PROJ, "bias")); + + flow.time.resize(2); + for (size_t i = 0; i < flow.time.size(); i++) { + auto & t = flow.time[i]; + t.freqs = get_tensor(mtmd_format(TN_A_GEN_FLOW_TIME_FREQS, (int) i)); + t.up_w = get_tensor(mtmd_format(TN_A_GEN_FLOW_TIME_UP, (int) i, "weight")); + t.up_b = get_tensor(mtmd_format(TN_A_GEN_FLOW_TIME_UP, (int) i, "bias")); + t.down_w = get_tensor(mtmd_format(TN_A_GEN_FLOW_TIME_DOWN, (int) i, "weight")); + t.down_b = get_tensor(mtmd_format(TN_A_GEN_FLOW_TIME_DOWN, (int) i, "bias")); + t.norm = get_tensor(mtmd_format(TN_A_GEN_FLOW_TIME_NORM, (int) i)); + } + + // one AdaLN block per flow depth, the count is only known from the tensors + for (int il = 0; ; il++) { + ggml_tensor * probe = get_tensor(mtmd_format(TN_A_GEN_FLOW_BLK_NORM, il, "weight"), false); + if (probe == nullptr) { + break; + } + mtmd_flow_net::block blk; + blk.norm_w = probe; + blk.norm_b = get_tensor(mtmd_format(TN_A_GEN_FLOW_BLK_NORM, il, "bias")); + blk.up_w = get_tensor(mtmd_format(TN_A_GEN_FLOW_BLK_UP, il, "weight")); + blk.up_b = get_tensor(mtmd_format(TN_A_GEN_FLOW_BLK_UP, il, "bias")); + blk.down_w = get_tensor(mtmd_format(TN_A_GEN_FLOW_BLK_DOWN, il, "weight")); + blk.down_b = get_tensor(mtmd_format(TN_A_GEN_FLOW_BLK_DOWN, il, "bias")); + blk.ada_w = get_tensor(mtmd_format(TN_A_GEN_FLOW_BLK_ADA, il, "weight")); + blk.ada_b = get_tensor(mtmd_format(TN_A_GEN_FLOW_BLK_ADA, il, "bias")); + flow.blocks.push_back(blk); + } + + model.gen_out_eos_w = get_tensor(mtmd_format(TN_A_GEN_OUT_EOS, "weight")); + model.gen_out_eos_b = get_tensor(mtmd_format(TN_A_GEN_OUT_EOS, "bias")); + model.gen_input_lin_w = get_tensor(mtmd_format(TN_A_GEN_INPUT_LINEAR, "weight")); + model.gen_emb_mean = get_tensor(TN_A_GEN_EMB_MEAN); + model.gen_emb_std = get_tensor(TN_A_GEN_EMB_STD); + + // mimi decoder + model.gen_quant_out_w = get_tensor(mtmd_format(TN_A_GEN_WAV_QUANT_OUT, "weight")); + model.gen_upsample_w = get_tensor(mtmd_format(TN_A_GEN_WAV_UPSAMPLE, "weight")); + load_seanet(model.seanet, true); + model.gen_tfm_layers.resize(hparams.n_layer); + for (int il = 0; il < hparams.n_layer; il++) { + auto & layer = model.gen_tfm_layers[il]; + const char * p = "a.gen.wav.tfm"; + layer.ln_1_w = get_tensor(mtmd_format(TN_LN_1, p, il, "weight")); + layer.ln_1_b = get_tensor(mtmd_format(TN_LN_1, p, il, "bias")); + layer.q_w = get_tensor(mtmd_format(TN_ATTN_Q, p, il, "weight")); + layer.k_w = get_tensor(mtmd_format(TN_ATTN_K, p, il, "weight")); + layer.v_w = get_tensor(mtmd_format(TN_ATTN_V, p, il, "weight")); + layer.o_w = get_tensor(mtmd_format(TN_ATTN_OUTPUT, p, il, "weight")); + layer.ls_1_w = get_tensor(mtmd_format(TN_LS_1, p, il, "weight")); + layer.ln_2_w = get_tensor(mtmd_format(TN_LN_2, p, il, "weight")); + layer.ln_2_b = get_tensor(mtmd_format(TN_LN_2, p, il, "bias")); + layer.ff_up_w = get_tensor(mtmd_format(TN_FFN_UP, p, il, "weight")); + layer.ff_down_w = get_tensor(mtmd_format(TN_FFN_DOWN, p, il, "weight")); + layer.ls_2_w = get_tensor(mtmd_format(TN_LS_2, p, il, "weight")); + } + + for (int64_t i = 0; i < gguf_get_n_tensors(ctx_gguf.get()); ++i) { + const char * name = gguf_get_tensor_name(ctx_gguf.get(), i); + if (strncmp(name, "voice.", 6) != 0) { + continue; + } + const ggml_tensor * t = ggml_get_tensor(ctx_meta.get(), name); + if (t->type != GGML_TYPE_F32 || ggml_n_dims(t) > 2 || t->ne[0] != model.gen_input_lin_w->ne[1]) { + throw std::runtime_error(mtmd_format("%s: invalid voice tensor %s\n", __func__, name)); + } + auto & embd = model.gen_voices[name + 6].rows; + embd.resize(ggml_nelements(t)); + fin.seekg(tensor_offset.at(name), std::ios::beg); + if (!fin.read(reinterpret_cast(embd.data()), ggml_nbytes(t))) { + throw std::runtime_error(mtmd_format("%s: failed to read %s\n", __func__, name)); + } + } + if (!model.gen_voices.empty()) { + LOG_INF("%s: %zu predefined voices\n", __func__, model.gen_voices.size()); + } + } break; + case PROJECTOR_TYPE_LFM3A_GEN: + { + // must match the "tts" chat template, see conversion/lfm3.py + // note for contributor: this is one-off fix, do not repeat this for future models + model.gen_voices = { {"female", {}}, {"male", {}} }; + + // depthformer, the generic loop above loaded the a.gen.code.blk.* tensors + for (int il = 0; il < hparams.n_layer; ++il) { + model.layers[il].attn_gate_w = get_tensor(mtmd_format(TN_ATTN_GATE, prefix, il, "weight")); + } + model.gen_code_proj_in_w = get_tensor(mtmd_format(TN_A_GEN_CODE_PROJ_IN, "weight")); + model.gen_code_proj_in_b = get_tensor(mtmd_format(TN_A_GEN_CODE_PROJ_IN, "bias")); + model.gen_code_embd_w = get_tensor(mtmd_format(TN_A_GEN_CODE_EMBD, "weight")); // tied head + model.gen_code_out_embd_w = get_tensor(mtmd_format(TN_A_GEN_CODE_OUT_EMBD, "weight")); + model.gen_code_norm_w = get_tensor(mtmd_format(TN_A_GEN_CODE_NORM, "weight")); + model.gen_code_head_norm_w = get_tensor(mtmd_format(TN_A_GEN_CODE_HEAD_NORM, "weight")); + model.gen_code_mask_embd = get_tensor(TN_A_GEN_CODE_MASK_EMBD); + + // codec decoder: FSQ level table, one projection over all codebooks, LFM3 decoder -> polar STFT frames + auto & c2w = model.c2w; + c2w.quant_first_cb_w = get_tensor(mtmd_format(TN_A_GEN_WAV_QUANT_FIRST_CB, "weight")); + c2w.quant_first_out_w = get_tensor(mtmd_format(TN_A_GEN_WAV_QUANT_FIRST_OUT, "weight")); + c2w.quant_first_out_b = get_tensor(mtmd_format(TN_A_GEN_WAV_QUANT_FIRST_OUT, "bias")); + c2w.tfm_in_proj_w = get_tensor(mtmd_format(TN_A_GEN_WAV_TFM_IN_PROJ, "weight")); + c2w.tfm_in_proj_b = get_tensor(mtmd_format(TN_A_GEN_WAV_TFM_IN_PROJ, "bias")); + c2w.tfm_out_proj_w = get_tensor(mtmd_format(TN_A_GEN_WAV_TFM_OUT_PROJ, "weight")); + c2w.tfm_out_proj_b = get_tensor(mtmd_format(TN_A_GEN_WAV_TFM_OUT_PROJ, "bias")); + c2w.tfm_output_norm_w = get_tensor(mtmd_format(TN_A_GEN_WAV_TFM_OUT_NORM, "weight")); + + // each layer is a short conv block or a gated attention block, both with sandwich norms + c2w.tfm_layers.resize(hparams.wav_tfm_n_layer); + for (int il = 0; il < hparams.wav_tfm_n_layer; il++) { + auto & layer = c2w.tfm_layers[il]; + const char * p = "a.gen.wav.tfm"; + layer.ln_1_w = get_tensor(mtmd_format(TN_LN_1, p, il, "weight")); + layer.attn_post_norm_w = get_tensor(mtmd_format(TN_ATTN_POST_NORM, p, il, "weight")); + layer.ln_2_w = get_tensor(mtmd_format(TN_LN_2, p, il, "weight")); + layer.ff_post_norm_w = get_tensor(mtmd_format(TN_FFN_POST_NORM, p, il, "weight")); + layer.ff_gate_w = get_tensor(mtmd_format(TN_FFN_GATE, p, il, "weight")); + layer.ff_up_w = get_tensor(mtmd_format(TN_FFN_UP, p, il, "weight")); + layer.ff_down_w = get_tensor(mtmd_format(TN_FFN_DOWN, p, il, "weight")); + layer.conv_pw1_w = get_tensor(mtmd_format(TN_CONV_PW1, p, il, "weight"), false); + layer.conv_dw_w = get_tensor(mtmd_format(TN_CONV_DW, p, il, "weight"), false); + layer.conv_pw2_w = get_tensor(mtmd_format(TN_CONV_PW2, p, il, "weight"), false); + layer.q_w = get_tensor(mtmd_format(TN_ATTN_Q, p, il, "weight"), false); + layer.k_w = get_tensor(mtmd_format(TN_ATTN_K, p, il, "weight"), false); + layer.v_w = get_tensor(mtmd_format(TN_ATTN_V, p, il, "weight"), false); + layer.o_w = get_tensor(mtmd_format(TN_ATTN_OUTPUT, p, il, "weight"), false); + layer.q_norm = get_tensor(mtmd_format(TN_ATTN_Q_NORM, p, il, "weight"), false); + layer.k_norm = get_tensor(mtmd_format(TN_ATTN_K_NORM, p, il, "weight"), false); + layer.attn_gate_w = get_tensor(mtmd_format(TN_ATTN_GATE, p, il, "weight"), false); + const bool is_conv = layer.conv_pw1_w && layer.conv_dw_w && layer.conv_pw2_w; + const bool is_attn = layer.q_w && layer.k_w && layer.v_w && layer.o_w && layer.q_norm && layer.k_norm && layer.attn_gate_w; + if (is_conv == is_attn) { + throw std::runtime_error(mtmd_format("%s: codec layer %d is neither a conv nor an attention block\n", __func__, il)); + } + if (is_attn) { + const int d_head = layer.q_norm->ne[0]; + hparams.wav_tfm_n_head = layer.q_w->ne[1] / d_head; + hparams.wav_tfm_n_head_kv = layer.k_w->ne[1] / d_head; + } + } + } break; + case PROJECTOR_TYPE_QWEN3TTS_GEN: + { + // code_predictor, proj_in is absent when the talker and the predictor share the hidden size + model.gen_code_proj_in_w = get_tensor(mtmd_format(TN_A_GEN_CODE_PROJ_IN, "weight"), false); + model.gen_code_proj_in_b = get_tensor(mtmd_format(TN_A_GEN_CODE_PROJ_IN, "bias"), false); + model.gen_code_embd_w = get_tensor(mtmd_format(TN_A_GEN_CODE_EMBD, "weight")); + model.gen_code_head_w = get_tensor(mtmd_format(TN_A_GEN_CODE_HEAD, "weight")); + model.gen_code_out_embd_w = get_tensor(mtmd_format(TN_A_GEN_CODE_OUT_EMBD, "weight")); + model.gen_code_norm_w = get_tensor(mtmd_format(TN_A_GEN_CODE_NORM, "weight")); + + // code2wav: RVQ codes -> raw PCM, lives in the same ctx as code_predictor + load_code2wav(true); + } break; + case PROJECTOR_TYPE_DRAMA3_GEN: + { + // generic a.gen.code.blk.* tensors are loaded by the loop above + model.drama3_layers.resize(hparams.n_layer); + for (int il = 0; il < hparams.n_layer; ++il) { + auto & tl = model.drama3_layers[il]; + model.layers[il].attn_gate_w = get_tensor(mtmd_format(TN_ATTN_GATE, prefix, il, "weight")); + tl.enorm_w = get_tensor(mtmd_format(TN_A_GEN_CODE_ENORM, il, "weight")); + tl.hnorm_w = get_tensor(mtmd_format(TN_A_GEN_CODE_HNORM, il, "weight")); + tl.eh_proj_w = get_tensor(mtmd_format(TN_A_GEN_CODE_EH_PROJ, il, "weight")); + tl.embd_w = get_tensor(mtmd_format(TN_A_GEN_CODE_BLK_EMBD, il, "weight")); + tl.out_norm_w = get_tensor(mtmd_format(TN_A_GEN_CODE_BLK_OUT_NORM, il, "weight")); + } + model.gen_code_embd_w = get_tensor(mtmd_format(TN_A_GEN_CODE_EMBD, "weight")); // merged 3D, one table per codebook + model.gen_code_head_w = get_tensor(mtmd_format(TN_A_GEN_CODE_HEAD, "weight")); // merged 3D, codebooks 1.. + model.gen_code_out_embd_w = get_tensor(mtmd_format(TN_A_GEN_CODE_OUT_EMBD, "weight")); // backbone rows of the semantic band + load_code2wav(false); + } break; + case PROJECTOR_TYPE_CHATTERBOX: + { + auto & c = model.cbx; + auto has = [&](const std::string & name) { + return gguf_find_tensor(ctx_gguf.get(), name.c_str()) >= 0; + }; + auto load_enc_layer = [&](const std::string & p, mtmd_chatterbox::enc_layer & l) { + l.norm_mha_w = get_tensor(p + ".norm_mha.weight"); + l.norm_mha_b = get_tensor(p + ".norm_mha.bias"); + l.attn_q_w = get_tensor(p + ".self_attn.linear_q.weight"); + l.attn_q_b = get_tensor(p + ".self_attn.linear_q.bias"); + l.attn_k_w = get_tensor(p + ".self_attn.linear_k.weight"); + l.attn_k_b = get_tensor(p + ".self_attn.linear_k.bias"); + l.attn_v_w = get_tensor(p + ".self_attn.linear_v.weight"); + l.attn_v_b = get_tensor(p + ".self_attn.linear_v.bias"); + l.attn_pos_w = get_tensor(p + ".self_attn.linear_pos.weight"); + l.attn_pos_bias_u = get_tensor(p + ".self_attn.pos_bias_u"); + l.attn_pos_bias_v = get_tensor(p + ".self_attn.pos_bias_v"); + l.attn_out_w = get_tensor(p + ".self_attn.linear_out.weight"); + l.attn_out_b = get_tensor(p + ".self_attn.linear_out.bias"); + l.norm_ff_w = get_tensor(p + ".norm_ff.weight"); + l.norm_ff_b = get_tensor(p + ".norm_ff.bias"); + l.ffn_1_w = get_tensor(p + ".feed_forward.w_1.weight"); + l.ffn_1_b = get_tensor(p + ".feed_forward.w_1.bias"); + l.ffn_2_w = get_tensor(p + ".feed_forward.w_2.weight"); + l.ffn_2_b = get_tensor(p + ".feed_forward.w_2.bias"); + }; + auto load_causal = [&](const std::string & p, mtmd_chatterbox::causal_block & b) { + b.conv_w = get_tensor(p + ".block.0.weight"); + b.conv_b = get_tensor(p + ".block.0.bias"); + b.norm_w = get_tensor(p + ".block.2.weight"); + b.norm_b = get_tensor(p + ".block.2.bias"); + }; + auto load_resnet = [&](const std::string & p, mtmd_chatterbox::resnet & r) { + load_causal(p + ".block1", r.block1); + load_causal(p + ".block2", r.block2); + r.mlp_w = get_tensor(p + ".mlp.1.weight"); + r.mlp_b = get_tensor(p + ".mlp.1.bias"); + r.res_conv_w = get_tensor(p + ".res_conv.weight"); + r.res_conv_b = get_tensor(p + ".res_conv.bias"); + }; + auto load_tfm = [&](const std::string & p, mtmd_chatterbox::tfm_block & b) { + b.norm1_w = get_tensor(p + ".norm1.weight"); + b.norm1_b = get_tensor(p + ".norm1.bias"); + b.attn_q_w = get_tensor(p + ".attn1.to_q.weight"); + b.attn_k_w = get_tensor(p + ".attn1.to_k.weight"); + b.attn_v_w = get_tensor(p + ".attn1.to_v.weight"); + b.attn_out_w = get_tensor(p + ".attn1.to_out.0.weight"); + b.attn_out_b = get_tensor(p + ".attn1.to_out.0.bias"); + b.norm3_w = get_tensor(p + ".norm3.weight"); + b.norm3_b = get_tensor(p + ".norm3.bias"); + b.ff_in_w = get_tensor(p + ".ff.net.0.proj.weight"); + b.ff_in_b = get_tensor(p + ".ff.net.0.proj.bias"); + b.ff_out_w = get_tensor(p + ".ff.net.2.weight"); + b.ff_out_b = get_tensor(p + ".ff.net.2.bias"); + }; + auto load_stage = [&](const std::string & p, mtmd_chatterbox::est_stage & s, bool boundary) { + load_resnet(p + ".0", s.res); + for (int j = 0; has(p + ".1." + std::to_string(j) + ".norm1.weight"); j++) { + mtmd_chatterbox::tfm_block b; + load_tfm(p + ".1." + std::to_string(j), b); + s.tfm.push_back(b); + } + if (boundary) { + s.conv_w = get_tensor(p + ".2.weight"); + s.conv_b = get_tensor(p + ".2.bias"); + } + }; + auto load_hres = [&](const std::string & p, mtmd_chatterbox::hift_res & r) { + for (int j = 0; has(p + ".convs1." + std::to_string(j) + ".weight"); j++) { + const std::string js = std::to_string(j); + mtmd_chatterbox::hift_res_unit u; + u.act1_alpha = get_tensor(p + ".activations1." + js + ".alpha"); + u.act2_alpha = get_tensor(p + ".activations2." + js + ".alpha"); + u.conv1_w = get_tensor(p + ".convs1." + js + ".weight"); + u.conv1_b = get_tensor(p + ".convs1." + js + ".bias"); + u.conv2_w = get_tensor(p + ".convs2." + js + ".weight"); + u.conv2_b = get_tensor(p + ".convs2." + js + ".bias"); + r.units.push_back(u); + } + }; + + // flow encoder + c.input_embedding_w = get_tensor("a.gen.flow.input_embedding.weight"); + c.spk_affine_w = get_tensor("a.gen.flow.spk_embed_affine_layer.weight"); + c.spk_affine_b = get_tensor("a.gen.flow.spk_embed_affine_layer.bias"); + c.embed_linear_w = get_tensor("a.gen.fenc.embed.out.0.weight"); + c.embed_linear_b = get_tensor("a.gen.fenc.embed.out.0.bias"); + c.embed_norm_w = get_tensor("a.gen.fenc.embed.out.1.weight"); + c.embed_norm_b = get_tensor("a.gen.fenc.embed.out.1.bias"); + c.pre_conv1_w = get_tensor("a.gen.fenc.pre_lookahead_layer.conv1.weight"); + c.pre_conv1_b = get_tensor("a.gen.fenc.pre_lookahead_layer.conv1.bias"); + c.pre_conv2_w = get_tensor("a.gen.fenc.pre_lookahead_layer.conv2.weight"); + c.pre_conv2_b = get_tensor("a.gen.fenc.pre_lookahead_layer.conv2.bias"); + for (int i = 0; has("a.gen.fenc.encoders." + std::to_string(i) + ".norm_mha.weight"); i++) { + mtmd_chatterbox::enc_layer l; + load_enc_layer("a.gen.fenc.encoders." + std::to_string(i), l); + c.enc.push_back(l); + } + c.up_conv_w = get_tensor("a.gen.fenc.up_layer.conv.weight"); + c.up_conv_b = get_tensor("a.gen.fenc.up_layer.conv.bias"); + c.up_embed_linear_w = get_tensor("a.gen.fenc.up_embed.out.0.weight"); + c.up_embed_linear_b = get_tensor("a.gen.fenc.up_embed.out.0.bias"); + c.up_embed_norm_w = get_tensor("a.gen.fenc.up_embed.out.1.weight"); + c.up_embed_norm_b = get_tensor("a.gen.fenc.up_embed.out.1.bias"); + for (int i = 0; has("a.gen.fenc.up_encoders." + std::to_string(i) + ".norm_mha.weight"); i++) { + mtmd_chatterbox::enc_layer l; + load_enc_layer("a.gen.fenc.up_encoders." + std::to_string(i), l); + c.up_enc.push_back(l); + } + c.after_norm_w = get_tensor("a.gen.fenc.after_norm.weight"); + c.after_norm_b = get_tensor("a.gen.fenc.after_norm.bias"); + c.encoder_proj_w = get_tensor("a.gen.flow.encoder_proj.weight"); + c.encoder_proj_b = get_tensor("a.gen.flow.encoder_proj.bias"); + + // cfm estimator + c.time_mlp_1_w = get_tensor("a.gen.est.time_mlp.linear_1.weight"); + c.time_mlp_1_b = get_tensor("a.gen.est.time_mlp.linear_1.bias"); + c.time_mlp_2_w = get_tensor("a.gen.est.time_mlp.linear_2.weight"); + c.time_mlp_2_b = get_tensor("a.gen.est.time_mlp.linear_2.bias"); + c.time_embed_mixer_w = get_tensor("a.gen.est.time_embed_mixer.weight", false); + load_stage("a.gen.est.down_blocks.0", c.est_down, true); + for (int i = 0; has("a.gen.est.mid_blocks." + std::to_string(i) + ".0.block1.block.0.weight"); i++) { + mtmd_chatterbox::est_stage s; + load_stage("a.gen.est.mid_blocks." + std::to_string(i), s, false); + c.est_mid.push_back(std::move(s)); + } + load_stage("a.gen.est.up_blocks.0", c.est_up, true); + load_causal("a.gen.est.final_block", c.est_final_block); + c.est_final_proj_w = get_tensor("a.gen.est.final_proj.weight"); + c.est_final_proj_b = get_tensor("a.gen.est.final_proj.bias"); + + // hift vocoder + c.hift_pre_w = get_tensor("a.gen.hift.conv_pre.weight"); + c.hift_pre_b = get_tensor("a.gen.hift.conv_pre.bias"); + c.hift_post_w = get_tensor("a.gen.hift.conv_post.weight"); + c.hift_post_b = get_tensor("a.gen.hift.conv_post.bias"); + for (int i = 0; has("a.gen.hift.ups." + std::to_string(i) + ".weight"); i++) { + const std::string is = std::to_string(i); + mtmd_chatterbox::hift_up up; + up.up_w = get_tensor("a.gen.hift.ups." + is + ".weight"); + up.up_b = get_tensor("a.gen.hift.ups." + is + ".bias"); + up.source_down_w = get_tensor("a.gen.hift.source_downs." + is + ".weight"); + up.source_down_b = get_tensor("a.gen.hift.source_downs." + is + ".bias"); + load_hres("a.gen.hift.source_resblocks." + is, up.source_res); + load_hres("a.gen.hift.resblocks." + std::to_string(3 * i), up.res_0); + load_hres("a.gen.hift.resblocks." + std::to_string(3 * i + 1), up.res_1); + load_hres("a.gen.hift.resblocks." + std::to_string(3 * i + 2), up.res_2); + c.hift_ups.push_back(std::move(up)); + } + for (int i = 0; has("a.gen.hift.f0_predictor.condnet." + std::to_string(i) + ".weight"); i += 2) { + const std::string is = std::to_string(i); + mtmd_chatterbox::f0_conv fc; + fc.w = get_tensor("a.gen.hift.f0_predictor.condnet." + is + ".weight"); + fc.b = get_tensor("a.gen.hift.f0_predictor.condnet." + is + ".bias"); + c.f0_condnet.push_back(fc); + } + c.f0_classifier_w = get_tensor("a.gen.hift.f0_predictor.classifier.weight"); + c.f0_classifier_b = get_tensor("a.gen.hift.f0_predictor.classifier.bias"); + + // s3 tokenizer + c.s3tok_conv1_w = get_tensor("a.s3tok.encoder.conv1.weight"); + c.s3tok_conv1_b = get_tensor("a.s3tok.encoder.conv1.bias"); + c.s3tok_conv2_w = get_tensor("a.s3tok.encoder.conv2.weight"); + c.s3tok_conv2_b = get_tensor("a.s3tok.encoder.conv2.bias"); + for (int i = 0; has("a.s3tok.encoder.blocks." + std::to_string(i) + ".attn_ln.weight"); i++) { + const std::string p = "a.s3tok.encoder.blocks." + std::to_string(i); + mtmd_chatterbox::s3tok_block b; + b.attn_ln_w = get_tensor(p + ".attn_ln.weight"); + b.attn_ln_b = get_tensor(p + ".attn_ln.bias"); + b.attn_q_w = get_tensor(p + ".attn.query.weight"); + b.attn_q_b = get_tensor(p + ".attn.query.bias"); + b.attn_k_w = get_tensor(p + ".attn.key.weight"); + b.attn_v_w = get_tensor(p + ".attn.value.weight"); + b.attn_v_b = get_tensor(p + ".attn.value.bias"); + b.fsmn_w = get_tensor(p + ".attn.fsmn_block.weight"); + b.attn_out_w = get_tensor(p + ".attn.out.weight"); + b.attn_out_b = get_tensor(p + ".attn.out.bias"); + b.mlp_ln_w = get_tensor(p + ".mlp_ln.weight"); + b.mlp_ln_b = get_tensor(p + ".mlp_ln.bias"); + b.mlp_in_w = get_tensor(p + ".mlp.0.weight"); + b.mlp_in_b = get_tensor(p + ".mlp.0.bias"); + b.mlp_out_w = get_tensor(p + ".mlp.2.weight"); + b.mlp_out_b = get_tensor(p + ".mlp.2.bias"); + c.s3tok_blocks.push_back(b); + } + c.s3tok_down_w = get_tensor("a.s3tok.quantizer._codebook.project_down.weight"); + c.s3tok_down_b = get_tensor("a.s3tok.quantizer._codebook.project_down.bias"); + + // read on the host: embedding tables, source module + for (ggml_tensor * t = ggml_get_first_tensor(ctx_meta.get()); t; t = ggml_get_next_tensor(ctx_meta.get(), t)) { + const std::string name = t->name; + if (name.rfind("a.gen.code.", 0) == 0 || name.rfind("a.gen.t3.", 0) == 0 || + name.rfind("a.gen.hift.m_source.", 0) == 0) { + model.cbx_tensors[name] = get_tensor(name); + } + } + + // voice. holds the talker rows, voice..ref_* the flow decoder reference + const int64_t n_embd_talker = model.cbx_tensors.at("a.gen.code.out_embd.weight")->ne[0]; + for (int64_t i = 0; i < gguf_get_n_tensors(ctx_gguf.get()); ++i) { + const std::string name = gguf_get_tensor_name(ctx_gguf.get(), i); + if (name.rfind("voice.", 0) != 0) { + continue; + } + const ggml_tensor * t = ggml_get_tensor(ctx_meta.get(), name.c_str()); + std::string voice = name.substr(6); + std::vector ids; // the converter stores 1D tensors as F32, ids are exact + void * dst = nullptr; + bool ok = false; + auto take = [&](const char * suffix) { + const size_t n = strlen(suffix); + if (voice.size() > n && voice.compare(voice.size() - n, n, suffix) == 0) { + voice.resize(voice.size() - n); + return true; + } + return false; + }; + if (take(".ref_tokens")) { + ids.resize(ggml_nelements(t)); + dst = ids.data(); + ok = t->type == GGML_TYPE_F32; + } else if (take(".ref_feat")) { + auto & v = model.gen_voices[voice].ref.feat; + v.resize(ggml_nelements(t)); + dst = v.data(); + ok = t->type == GGML_TYPE_F32 && t->ne[0] == 80; + } else if (take(".ref_spk")) { + auto & v = model.gen_voices[voice].ref.spk; + v.resize(ggml_nelements(t)); + dst = v.data(); + ok = t->type == GGML_TYPE_F32 && t->ne[0] == 192; + } else { + auto & v = model.gen_voices[voice].rows; + v.resize(ggml_nelements(t)); + dst = v.data(); + ok = t->type == GGML_TYPE_F32 && ggml_n_dims(t) <= 2 && t->ne[0] == n_embd_talker; + } + if (!ok) { + throw std::runtime_error(mtmd_format("%s: invalid voice tensor %s\n", __func__, name.c_str())); + } + fin.seekg(tensor_offset.at(name), std::ios::beg); + if (!fin.read(reinterpret_cast(dst), ggml_nbytes(t))) { + throw std::runtime_error(mtmd_format("%s: failed to read %s\n", __func__, name.c_str())); + } + if (!ids.empty()) { + model.gen_voices[voice].ref.tokens.assign(ids.begin(), ids.end()); + } + } + for (const auto & [voice, v] : model.gen_voices) { + if (v.rows.empty() || v.ref.tokens.empty() || v.ref.feat.size() != 2 * 80 * v.ref.tokens.size() || v.ref.spk.empty()) { + throw std::runtime_error(mtmd_format("%s: incomplete voice %s\n", __func__, voice.c_str())); + } + } + } break; + case PROJECTOR_TYPE_CHATTERBOX_SPKENC: + { + auto & c = model.cbx; + auto has = [&](const std::string & name) { + return gguf_find_tensor(ctx_gguf.get(), name.c_str()) >= 0; + }; + auto load_bn = [&](const std::string & p, mtmd_chatterbox::bn & n) { + n.mean = get_tensor(p + ".running_mean"); + n.var = get_tensor(p + ".running_var"); + n.w = get_tensor(p + ".weight", false); + n.b = get_tensor(p + ".bias", false); + }; + auto load_res2d = [&](const std::string & p, mtmd_chatterbox::spk_res2d & r) { + r.conv1_w = get_tensor(p + ".conv1.weight"); + load_bn(p + ".bn1", r.bn1); + r.conv2_w = get_tensor(p + ".conv2.weight"); + load_bn(p + ".bn2", r.bn2); + r.shortcut_w = get_tensor(p + ".shortcut.0.weight", false); + if (r.shortcut_w) { + load_bn(p + ".shortcut.1", r.shortcut_bn); + } + }; + auto load_cam_block = [&](const std::string & bp, const std::string & tp, mtmd_chatterbox::spk_cam_block & blk) { + for (int li = 1; has(bp + ".tdnnd" + std::to_string(li) + ".linear1.weight"); li++) { + const std::string p = bp + ".tdnnd" + std::to_string(li); + mtmd_chatterbox::spk_cam_layer l; + load_bn(p + ".nonlinear1.batchnorm", l.nl1_bn); + l.linear1_w = get_tensor(p + ".linear1.weight"); + load_bn(p + ".nonlinear2.batchnorm", l.nl2_bn); + l.local_w = get_tensor(p + ".cam_layer.linear_local.weight"); + l.ctx1_w = get_tensor(p + ".cam_layer.linear1.weight"); + l.ctx1_b = get_tensor(p + ".cam_layer.linear1.bias"); + l.ctx2_w = get_tensor(p + ".cam_layer.linear2.weight"); + l.ctx2_b = get_tensor(p + ".cam_layer.linear2.bias"); + blk.layers.push_back(l); + } + load_bn(tp + ".nonlinear.batchnorm", blk.transit_bn); + blk.transit_w = get_tensor(tp + ".linear.weight"); + }; + + c.spk_conv1_w = get_tensor("a.spk.head.conv1.weight"); + load_bn("a.spk.head.bn1", c.spk_bn1); + load_res2d("a.spk.head.layer1.0", c.spk_layer1_0); + load_res2d("a.spk.head.layer1.1", c.spk_layer1_1); + load_res2d("a.spk.head.layer2.0", c.spk_layer2_0); + load_res2d("a.spk.head.layer2.1", c.spk_layer2_1); + c.spk_conv2_w = get_tensor("a.spk.head.conv2.weight"); + load_bn("a.spk.head.bn2", c.spk_bn2); + c.spk_tdnn_w = get_tensor("a.spk.xvector.tdnn.linear.weight"); + load_bn("a.spk.xvector.tdnn.nonlinear.batchnorm", c.spk_tdnn_bn); + load_cam_block("a.spk.xvector.block1", "a.spk.xvector.transit1", c.spk_block1); + load_cam_block("a.spk.xvector.block2", "a.spk.xvector.transit2", c.spk_block2); + load_cam_block("a.spk.xvector.block3", "a.spk.xvector.transit3", c.spk_block3); + load_bn("a.spk.xvector.out_nonlinear.batchnorm", c.spk_out_bn); + c.spk_dense_w = get_tensor("a.spk.xvector.dense.linear.weight"); + load_bn("a.spk.xvector.dense.nonlinear.batchnorm", c.spk_dense_bn); + + // read on the host: voice encoder lstm, conditioning encoder + for (ggml_tensor * t = ggml_get_first_tensor(ctx_meta.get()); t; t = ggml_get_next_tensor(ctx_meta.get(), t)) { + const std::string name = t->name; + if (name.rfind("a.ve.", 0) == 0 || name.rfind("a.cenc.", 0) == 0) { + model.cbx_tensors[name] = get_tensor(name); + } + } + } break; + case PROJECTOR_TYPE_SOPRANO: + { + auto & v = model.vocos; + v.input_w = get_tensor(mtmd_format(TN_A_GEN_WAV_INPUT, "weight")); + v.input_b = get_tensor(mtmd_format(TN_A_GEN_WAV_INPUT, "bias")); + v.norm_w = get_tensor(mtmd_format(TN_A_GEN_WAV_NORM, "weight")); + v.norm_b = get_tensor(mtmd_format(TN_A_GEN_WAV_NORM, "bias")); + v.output_norm_w = get_tensor(mtmd_format(TN_A_GEN_WAV_OUTPUT_NORM, "weight")); + v.output_norm_b = get_tensor(mtmd_format(TN_A_GEN_WAV_OUTPUT_NORM, "bias")); + v.output_w = get_tensor(mtmd_format(TN_A_GEN_WAV_OUTPUT, "weight")); + v.output_b = get_tensor(mtmd_format(TN_A_GEN_WAV_OUTPUT, "bias")); + v.blocks.resize(hparams.n_layer); + for (int il = 0; il < hparams.n_layer; ++il) { + auto & b = v.blocks[il]; + b.dwconv_w = get_tensor(mtmd_format(TN_A_GEN_WAV_UP_DWCONV, il, "weight")); + b.dwconv_b = get_tensor(mtmd_format(TN_A_GEN_WAV_UP_DWCONV, il, "bias")); + b.norm_w = get_tensor(mtmd_format(TN_A_GEN_WAV_UP_NORM, il, "weight")); + b.norm_b = get_tensor(mtmd_format(TN_A_GEN_WAV_UP_NORM, il, "bias")); + b.pw1_w = get_tensor(mtmd_format(TN_A_GEN_WAV_UP_PW1, il, "weight")); + b.pw1_b = get_tensor(mtmd_format(TN_A_GEN_WAV_UP_PW1, il, "bias")); + b.pw2_w = get_tensor(mtmd_format(TN_A_GEN_WAV_UP_PW2, il, "weight")); + b.pw2_b = get_tensor(mtmd_format(TN_A_GEN_WAV_UP_PW2, il, "bias")); + b.gamma = get_tensor(mtmd_format(TN_A_GEN_WAV_UP_GAMMA, il)); + } + } break; + case PROJECTOR_TYPE_VOXTRAL: + case PROJECTOR_TYPE_VOXTRAL_RT: + { + model.conv1d_1_w = get_tensor(mtmd_format(TN_CONV1D, 1, "weight")); + model.conv1d_1_b = get_tensor(mtmd_format(TN_CONV1D, 1, "bias")); + model.conv1d_2_w = get_tensor(mtmd_format(TN_CONV1D, 2, "weight")); + model.conv1d_2_b = get_tensor(mtmd_format(TN_CONV1D, 2, "bias")); + model.mm_1_w = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 1, "weight")); + model.mm_2_w = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 2, "weight")); + } break; + case PROJECTOR_TYPE_MUSIC_FLAMINGO: + { + model.conv1d_1_w = get_tensor(mtmd_format(TN_CONV1D, 1, "weight")); + model.conv1d_1_b = get_tensor(mtmd_format(TN_CONV1D, 1, "bias")); + model.conv1d_2_w = get_tensor(mtmd_format(TN_CONV1D, 2, "weight")); + model.conv1d_2_b = get_tensor(mtmd_format(TN_CONV1D, 2, "bias")); + model.mm_1_w = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 1, "weight")); + model.mm_1_b = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 1, "bias")); + model.mm_2_w = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 2, "weight")); + model.mm_2_b = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 2, "bias")); + } break; + case PROJECTOR_TYPE_INTERNVL: + { + model.mm_0_w = get_tensor(mtmd_format(TN_MVLM_PROJ_MLP, 0, "weight")); + model.mm_0_b = get_tensor(mtmd_format(TN_MVLM_PROJ_MLP, 0, "bias")); + model.mm_1_w = get_tensor(mtmd_format(TN_MVLM_PROJ_MLP, 1, "weight")); + model.mm_1_b = get_tensor(mtmd_format(TN_MVLM_PROJ_MLP, 1, "bias")); + model.mm_3_w = get_tensor(mtmd_format(TN_MVLM_PROJ_MLP, 3, "weight")); + model.mm_3_b = get_tensor(mtmd_format(TN_MVLM_PROJ_MLP, 3, "bias")); + } break; + case PROJECTOR_TYPE_NEMOTRON_V2_VL: + case PROJECTOR_TYPE_NEMOTRON_OMNI: + { + model.mm_0_w = get_tensor(mtmd_format(TN_MVLM_PROJ_MLP, 0, "weight")); + model.mm_1_w = get_tensor(mtmd_format(TN_MVLM_PROJ_MLP, 1, "weight")); + model.mm_3_w = get_tensor(mtmd_format(TN_MVLM_PROJ_MLP, 3, "weight")); + } break; + case PROJECTOR_TYPE_GLMA: + { + model.conv1d_1_w = get_tensor(mtmd_format(TN_CONV1D, 1, "weight")); + model.conv1d_1_b = get_tensor(mtmd_format(TN_CONV1D, 1, "bias")); + model.conv1d_2_w = get_tensor(mtmd_format(TN_CONV1D, 2, "weight")); + model.conv1d_2_b = get_tensor(mtmd_format(TN_CONV1D, 2, "bias")); + model.mm_1_w = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 1, "weight")); + model.mm_1_b = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 1, "bias")); + model.mm_2_w = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 2, "weight")); + model.mm_2_b = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 2, "bias")); + model.mm_norm_pre_w = get_tensor(mtmd_format(TN_MM_NORM_PRE, "weight")); + model.mm_norm_pre_b = get_tensor(mtmd_format(TN_MM_NORM_PRE, "bias")); + model.mm_boi = get_tensor(mtmd_format(TN_TOK_BOI)); + model.mm_eoi = get_tensor(mtmd_format(TN_TOK_EOI)); + } break; + case PROJECTOR_TYPE_LLAMA4: + { + model.mm_model_proj = get_tensor(mtmd_format(TN_MM_PROJECTOR, "weight")); + model.mm_model_mlp_1_w = get_tensor(mtmd_format(TN_MVLM_PROJ_MLP, 1, "weight")); + model.mm_model_mlp_2_w = get_tensor(mtmd_format(TN_MVLM_PROJ_MLP, 2, "weight")); + } break; + case PROJECTOR_TYPE_COGVLM: + { + model.mm_model_proj = get_tensor(mtmd_format(TN_MM_PROJECTOR, "weight")); + model.mm_post_fc_norm_w = get_tensor(mtmd_format(TN_MM_POST_FC_NORM, "weight")); + model.mm_post_fc_norm_b = get_tensor(mtmd_format(TN_MM_POST_FC_NORM, "bias")); + model.mm_h_to_4h_w = get_tensor(mtmd_format(TN_MM_H_TO_4H, "weight")); + model.mm_gate_w = get_tensor(mtmd_format(TN_MM_GATE, "weight")); + model.mm_4h_to_h_w = get_tensor(mtmd_format(TN_MM_4H_TO_H, "weight")); + model.mm_boi = get_tensor(TN_TOK_BOI); + model.mm_eoi = get_tensor(TN_TOK_EOI); + } break; + case PROJECTOR_TYPE_HUNYUANVL: + { + // proj.0 -> mm.0 (conv1), proj.2 -> mm.2 (conv2), mlp -> mm.model.fc (linear) + model.mm_0_w = get_tensor(mtmd_format(TN_LLAVA_PROJ, 0, "weight")); + model.mm_0_b = get_tensor(mtmd_format(TN_LLAVA_PROJ, 0, "bias")); + model.mm_1_w = get_tensor(mtmd_format(TN_LLAVA_PROJ, 2, "weight")); + model.mm_1_b = get_tensor(mtmd_format(TN_LLAVA_PROJ, 2, "bias")); + model.mm_model_proj = get_tensor(mtmd_format(TN_MM_PROJECTOR, "weight")); + model.mm_model_proj_b = get_tensor(mtmd_format(TN_MM_PROJECTOR, "bias")); + model.mm_pre_norm_w = get_tensor(mtmd_format(TN_MM_PRE_NORM, "weight")); + model.mm_post_norm_w = get_tensor(mtmd_format(TN_MM_POST_NORM, "weight")); + model.mm_img_begin = get_tensor(TN_MM_IMG_BEGIN); + model.mm_img_end = get_tensor(TN_MM_IMG_END); + model.image_newline = get_tensor(TN_IMAGE_NEWLINE); + model.view_seperator = get_tensor(TN_IMAGE_SEPERATOR, false); + // the position embedding is resampled on the host as F32 + if (!model.position_embeddings || model.position_embeddings->type != GGML_TYPE_F32) { + throw std::runtime_error(mtmd_format("%s: HunyuanVL requires an F32 %s\n", __func__, mtmd_format(TN_POS_EMBD, prefix).c_str())); + } + } break; + case PROJECTOR_TYPE_JANUS_PRO: + { + model.mm_0_w = get_tensor(mtmd_format(TN_LLAVA_PROJ, 0, "weight")); + model.mm_0_b = get_tensor(mtmd_format(TN_LLAVA_PROJ, 0, "bias")); + model.mm_1_w = get_tensor(mtmd_format(TN_LLAVA_PROJ, 1, "weight")); + model.mm_1_b = get_tensor(mtmd_format(TN_LLAVA_PROJ, 1, "bias")); + } break; + case PROJECTOR_TYPE_PHI4: + { + model.mm_0_w = get_tensor(mtmd_format(TN_LLAVA_PROJ, 0, "weight")); + model.mm_0_b = get_tensor(mtmd_format(TN_LLAVA_PROJ, 0, "bias")); + model.mm_2_w = get_tensor(mtmd_format(TN_LLAVA_PROJ, 2, "weight")); + model.mm_2_b = get_tensor(mtmd_format(TN_LLAVA_PROJ, 2, "bias")); + } break; + case PROJECTOR_TYPE_DEEPSEEKOCR: + case PROJECTOR_TYPE_DEEPSEEKOCR2: + { + model.pos_embed = get_tensor(mtmd_format(TN_SAM_POS_EMBD, "weight")); + model.patch_embed_proj_w = get_tensor(mtmd_format(TN_SAM_PATCH_EMBD, "weight")); + model.patch_embed_proj_b = get_tensor(mtmd_format(TN_SAM_PATCH_EMBD, "bias")); + model.n_sam_layers = hparams.sam_n_layer; + model.sam_layers.resize(model.n_sam_layers); + for (int il = 0; il < model.n_sam_layers; ++il) { + auto & layer = model.sam_layers[il]; + layer.qkv_w = get_tensor(mtmd_format(TN_SAM_ATTN_QKV, il, "weight")); + layer.qkv_b = get_tensor(mtmd_format(TN_SAM_ATTN_QKV, il, "bias")); + layer.o_w = get_tensor(mtmd_format(TN_SAM_ATTN_OUT, il, "weight")); + layer.o_b = get_tensor(mtmd_format(TN_SAM_ATTN_OUT, il, "bias")); + layer.ln_1_w = get_tensor(mtmd_format(TN_SAM_PRE_NORM, il, "weight")); + layer.ln_1_b = get_tensor(mtmd_format(TN_SAM_PRE_NORM, il, "bias")); + layer.ln_2_w = get_tensor(mtmd_format(TN_SAM_POST_NORM, il, "weight")); + layer.ln_2_b = get_tensor(mtmd_format(TN_SAM_POST_NORM, il, "bias")); + layer.rel_pos_h = get_tensor(mtmd_format(TN_SAM_ATTN_POS_H, il, "weight")); + layer.rel_pos_w = get_tensor(mtmd_format(TN_SAM_ATTN_POS_W, il, "weight")); + layer.ff_up_w = get_tensor(mtmd_format(TN_SAM_FFN_UP, il, "weight")); + layer.ff_up_b = get_tensor(mtmd_format(TN_SAM_FFN_UP, il, "bias")); + layer.ff_down_w = get_tensor(mtmd_format(TN_SAM_FFN_DOWN, il, "weight")); + layer.ff_down_b = get_tensor(mtmd_format(TN_SAM_FFN_DOWN, il, "bias")); + } + model.neck_0_w = get_tensor(mtmd_format(TN_SAM_NECK, 0, "weight")); + model.neck_1_b = get_tensor(mtmd_format(TN_SAM_NECK, 1, "bias")); + model.neck_1_w = get_tensor(mtmd_format(TN_SAM_NECK, 1, "weight")); + model.neck_2_w = get_tensor(mtmd_format(TN_SAM_NECK, 2, "weight")); + model.neck_3_b = get_tensor(mtmd_format(TN_SAM_NECK, 3, "bias")); + model.neck_3_w = get_tensor(mtmd_format(TN_SAM_NECK, 3, "weight")); + model.net_2 = get_tensor(mtmd_format(TN_SAM_NET, 2, "weight")); + model.net_3 = get_tensor(mtmd_format(TN_SAM_NET, 3, "weight")); + model.image_newline = get_tensor(TN_IMAGE_NEWLINE, false); + model.view_seperator = get_tensor(TN_IMAGE_SEPERATOR); + model.mm_fc_w = get_tensor(mtmd_format(TN_MM_PROJECTOR, "weight")); + model.mm_fc_b = get_tensor(mtmd_format(TN_MM_PROJECTOR, "bias")); + model.resample_query_768 = get_tensor(mtmd_format(TN_RESMPL_QUERY, 768, "weight"), false); + model.resample_query_1024 = get_tensor(mtmd_format(TN_RESMPL_QUERY, 1024, "weight"), false); + } break; + case PROJECTOR_TYPE_GEMMA4A: + { + for (int i = 0; i < 2; i++) { + model.sscp_conv_w[i] = get_tensor(mtmd_format(TN_A_CONV1D, i, "weight")); + model.sscp_conv_b[i] = get_tensor(mtmd_format(TN_A_CONV1D, i, "bias"), false); + model.sscp_norm_w[i] = get_tensor(mtmd_format(TN_A_CONV1D_NORM, i, "weight"), false); + } + model.sscp_inp_proj_w = get_tensor(mtmd_format(TN_A_INP_PROJ, "weight")); + model.sscp_inp_proj_b = get_tensor(mtmd_format(TN_A_INP_PROJ, "bias"), false); + model.audio_out_proj_w = get_tensor(mtmd_format(TN_A_OUT_PROJ, "weight"), false); + model.audio_out_proj_b = get_tensor(mtmd_format(TN_A_OUT_PROJ, "bias"), false); + // audio multimodal embedder (mm.a.* namespace, not mm.*) + model.mm_soft_emb_norm_w = get_tensor(mtmd_format(TN_A_MM_SOFT_EMB_N, "weight"), false); + model.mm_input_proj_w = get_tensor(mtmd_format(TN_A_MM_INP_PROJ, "weight"), false); + + // Per-layer tensors NOT loaded by the generic loop above + for (int il = 0; il < hparams.n_layer; ++il) { + auto & layer = model.layers[il]; + + // Gemma4 audio conformer-specific tensors + layer.ff_norm_w = get_tensor(mtmd_format(TN_FFN_NORM, prefix, il, "weight")); + layer.attn_pre_norm_w = get_tensor(mtmd_format(TN_A_ATTN_PRE_NORM, prefix, il, "weight"), false); + layer.per_dim_scale_w = get_tensor(mtmd_format(TN_A_PER_DIM_SCALE, prefix, il, "weight"), false); + layer.per_dim_k_scale_w = get_tensor(mtmd_format(TN_A_PER_DIM_K_SCALE, prefix, il, "weight"), false); + layer.attn_k_rel_w = get_tensor(mtmd_format(TN_A_ATTN_K_REL, prefix, il, "weight"), false); + + // Convolution module + // Note: conv_norm / norm_conv are swapped in GGUF due to + // upstream tensor_mapping.py, so we load them in reverse order + layer.norm_conv_w = get_tensor(mtmd_format(TN_CONV_NORM, prefix, il, "weight"), false); + layer.norm_conv_b = get_tensor(mtmd_format(TN_CONV_NORM, prefix, il, "bias"), false); + layer.conv_pw1_w = get_tensor(mtmd_format(TN_CONV_PW1, prefix, il, "weight")); + layer.conv_pw1_b = get_tensor(mtmd_format(TN_CONV_PW1, prefix, il, "bias"), false); + layer.conv_dw_w = get_tensor(mtmd_format(TN_CONV_DW, prefix, il, "weight")); + layer.conv_dw_b = get_tensor(mtmd_format(TN_CONV_DW, prefix, il, "bias"), false); + layer.conv_norm_w = get_tensor(mtmd_format(TN_NORM_CONV, prefix, il, "weight"), false); + layer.conv_norm_b = get_tensor(mtmd_format(TN_NORM_CONV, prefix, il, "bias"), false); + layer.conv_pw2_w = get_tensor(mtmd_format(TN_CONV_PW2, prefix, il, "weight")); + layer.conv_pw2_b = get_tensor(mtmd_format(TN_CONV_PW2, prefix, il, "bias"), false); + + // FFN2 (second half-step) + layer.ff_norm_1_w = get_tensor(mtmd_format(TN_FFN_NORM_1, prefix, il, "weight")); + layer.ff_up_1_w = get_tensor(mtmd_format(TN_FFN_UP_1, prefix, il, "weight")); + layer.ff_up_1_b = get_tensor(mtmd_format(TN_FFN_UP_1, prefix, il, "bias"), false); + layer.ff_down_1_w = get_tensor(mtmd_format(TN_FFN_DOWN_1, prefix, il, "weight")); + layer.ff_down_1_b = get_tensor(mtmd_format(TN_FFN_DOWN_1, prefix, il, "bias"), false); + layer.ff_post_norm_1_w = get_tensor(mtmd_format(TN_A_FFN_POST_NORM_1, prefix, il, "weight"), false); + } + + // Load clamp info for ClippableLinear AFTER all tensors are loaded + for (auto * tensor : tensors_to_load) { + std::string name = tensor->name; + if (string_ends_with(name, ".weight")) { + std::string name_inp_max = name; + std::string name_inp_min = name; + std::string name_out_max = name; + std::string name_out_min = name; + string_replace_all(name_inp_max, ".weight", ".input_max"); + string_replace_all(name_inp_min, ".weight", ".input_min"); + string_replace_all(name_out_max, ".weight", ".output_max"); + string_replace_all(name_out_min, ".weight", ".output_min"); + model.clamp_info_map[name] = { + get_scalar(name_inp_max, FLT_MAX), + get_scalar(name_inp_min, -FLT_MAX), + get_scalar(name_out_max, FLT_MAX), + get_scalar(name_out_min, -FLT_MAX) + }; + } + } + } break; + case PROJECTOR_TYPE_GEMMA4UA: + { + model.mm_input_proj_w = get_tensor(mtmd_format(TN_A_MM_INP_PROJ, "weight")); + } break; + case PROJECTOR_TYPE_LFM2A: + case PROJECTOR_TYPE_D1OMNI_A: + { + for (int i : {0, 2, 3, 5, 6}) { + model.pre_encode_conv_X_w[i] = get_tensor(mtmd_format(TN_CONV1D, i, "weight")); + model.pre_encode_conv_X_b[i] = get_tensor(mtmd_format(TN_CONV1D, i, "bias")); + } + model.pre_encode_out_w = get_tensor(mtmd_format(TN_PRE_ENCODE_OUT, "weight")); + model.pre_encode_out_b = get_tensor(mtmd_format(TN_PRE_ENCODE_OUT, "bias")); + + model.mm_0_w = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 0, "weight")); + model.mm_0_b = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 0, "bias")); + model.mm_1_w = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 1, "weight")); + model.mm_1_b = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 1, "bias")); + model.mm_3_w = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 3, "weight")); + model.mm_3_b = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 3, "bias")); + + // residual block after the projector: norm, down, up + if (model.proj_type == PROJECTOR_TYPE_D1OMNI_A) { + model.mm_4_w = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 4, "weight")); + model.mm_4_b = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 4, "bias")); + model.mm_5_w = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 5, "weight")); + model.mm_5_b = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 5, "bias")); + model.mm_6_w = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 6, "weight")); + model.mm_6_b = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 6, "bias")); + } + + for (int il = 0; il < hparams.n_layer; ++il) { + auto & layer = model.layers[il]; + + layer.ff_norm_w = get_tensor(mtmd_format(TN_FFN_NORM, prefix, il, "weight")); + layer.ff_norm_b = get_tensor(mtmd_format(TN_FFN_NORM, prefix, il, "bias")); + layer.ff_norm_1_w = get_tensor(mtmd_format(TN_FFN_NORM_1, prefix, il, "weight")); + layer.ff_norm_1_b = get_tensor(mtmd_format(TN_FFN_NORM_1, prefix, il, "bias")); + layer.ff_up_1_w = get_tensor(mtmd_format(TN_FFN_UP_1, prefix, il, "weight")); + layer.ff_up_1_b = get_tensor(mtmd_format(TN_FFN_UP_1, prefix, il, "bias")); + layer.ff_down_1_w = get_tensor(mtmd_format(TN_FFN_DOWN_1, prefix, il, "weight")); + layer.ff_down_1_b = get_tensor(mtmd_format(TN_FFN_DOWN_1, prefix, il, "bias")); + + layer.pos_bias_u = get_tensor(mtmd_format(TN_POS_BIAS_U, prefix, il)); + layer.pos_bias_v = get_tensor(mtmd_format(TN_POS_BIAS_V, prefix, il)); + + layer.norm_conv_w = get_tensor(mtmd_format(TN_NORM_CONV, prefix, il, "weight")); + layer.norm_conv_b = get_tensor(mtmd_format(TN_NORM_CONV, prefix, il, "bias")); + + layer.linear_pos_w = get_tensor(mtmd_format(TN_LINEAR_POS, prefix, il, "weight")); + + layer.conv_norm_w = get_tensor(mtmd_format(TN_CONV_NORM, prefix, il, "weight")); + layer.conv_norm_b = get_tensor(mtmd_format(TN_CONV_NORM, prefix, il, "bias")); + layer.conv_dw_w = get_tensor(mtmd_format(TN_CONV_DW, prefix, il, "weight")); + layer.conv_dw_b = get_tensor(mtmd_format(TN_CONV_DW, prefix, il, "bias")); + layer.conv_pw1_w = get_tensor(mtmd_format(TN_CONV_PW1, prefix, il, "weight")); + layer.conv_pw1_b = get_tensor(mtmd_format(TN_CONV_PW1, prefix, il, "bias")); + layer.conv_pw2_w = get_tensor(mtmd_format(TN_CONV_PW2, prefix, il, "weight")); + layer.conv_pw2_b = get_tensor(mtmd_format(TN_CONV_PW2, prefix, il, "bias")); + } + } break; + case PROJECTOR_TYPE_LFM3A: + { + for (int i : {0, 2, 3, 5, 6}) { + model.pre_encode_conv_X_w[i] = get_tensor(mtmd_format(TN_CONV1D, i, "weight")); + model.pre_encode_conv_X_b[i] = get_tensor(mtmd_format(TN_CONV1D, i, "bias")); + } + model.pre_encode_out_w = get_tensor(mtmd_format(TN_PRE_ENCODE_OUT, "weight")); + model.pre_encode_out_b = get_tensor(mtmd_format(TN_PRE_ENCODE_OUT, "bias")); + + model.mm_0_w = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 0, "weight")); + model.mm_0_b = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 0, "bias")); + model.mm_1_w = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 1, "weight")); + model.mm_1_b = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 1, "bias")); + model.mm_3_w = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 3, "weight")); + model.mm_3_b = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 3, "bias")); + + if (!model.post_ln_w) { + throw std::runtime_error(mtmd_format("%s: missing tensor %s\n", __func__, mtmd_format(TN_LN_POST, prefix, "weight").c_str())); + } + + // each layer is either a short conv block or a complete attention block + for (int il = 0; il < hparams.n_layer; ++il) { + auto & layer = model.layers[il]; + layer.conv_pw1_w = get_tensor(mtmd_format(TN_CONV_PW1, prefix, il, "weight"), false); + layer.conv_dw_w = get_tensor(mtmd_format(TN_CONV_DW, prefix, il, "weight"), false); + layer.conv_pw2_w = get_tensor(mtmd_format(TN_CONV_PW2, prefix, il, "weight"), false); + const bool is_conv = layer.conv_pw1_w && layer.conv_dw_w && layer.conv_pw2_w; + const bool is_attn = layer.q_w && layer.k_w && layer.v_w && layer.o_w && layer.q_norm && layer.k_norm; + if (is_conv == is_attn) { + throw std::runtime_error(mtmd_format("%s: lfm3a layer %d is neither a complete conv nor a complete attention block\n", __func__, il)); + } + } + } break; + case PROJECTOR_TYPE_PARAKEET: + { + + hparams.mel_filters = get_vector(TN_MEL_FILTERS); + hparams.window = get_vector(TN_WINDOW); + + // Subsampling layers (conv1d) + for (int i : {0, 2, 3, 5, 6}) { + model.pre_encode_conv_X_w[i] = get_tensor(mtmd_format(TN_CONV1D, i, "weight")); + model.pre_encode_conv_X_b[i] = get_tensor(mtmd_format(TN_CONV1D, i, "bias")); + } + model.pre_encode_out_w = get_tensor(mtmd_format(TN_PRE_ENCODE_OUT, "weight")); + model.pre_encode_out_b = get_tensor(mtmd_format(TN_PRE_ENCODE_OUT, "bias")); + + // Projection layers + model.mm_norm_pre_w = get_tensor(mtmd_format(TN_MM_NORM_PRE, "weight"), false); + model.mm_0_w = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 1, "weight"), false); + model.mm_1_w = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 2, "weight"), false); + + // Encoder layers + for (int il = 0; il < hparams.n_layer; ++il) { + auto & layer = model.layers[il]; + + // Attention (from shared above) + + // Relative position encoding + layer.linear_pos_w = get_tensor(mtmd_format(TN_LINEAR_POS, prefix, il, "weight")); + layer.pos_bias_u = get_tensor(mtmd_format(TN_POS_BIAS_U, prefix, il)); + layer.pos_bias_v = get_tensor(mtmd_format(TN_POS_BIAS_V, prefix, il)); + + // Convolution module + layer.conv_pw1_w = get_tensor(mtmd_format(TN_CONV_PW1, prefix, il, "weight")); + layer.conv_pw1_b = get_tensor(mtmd_format(TN_CONV_PW1, prefix, il, "bias"), false); + layer.conv_dw_w = get_tensor(mtmd_format(TN_CONV_DW, prefix, il, "weight")); + layer.conv_dw_b = get_tensor(mtmd_format(TN_CONV_DW, prefix, il, "bias"), false); + layer.conv_norm_w = get_tensor(mtmd_format(TN_CONV_NORM, prefix, il, "weight")); + layer.conv_norm_b = get_tensor(mtmd_format(TN_CONV_NORM, prefix, il, "bias")); + layer.conv_norm_mean = get_tensor(mtmd_format(TN_CONV_NORM_MEAN, prefix, il)); + layer.conv_norm_var = get_tensor(mtmd_format(TN_CONV_NORM_VAR, prefix, il)); + layer.conv_pw2_w = get_tensor(mtmd_format(TN_CONV_PW2, prefix, il, "weight")); + layer.conv_pw2_b = get_tensor(mtmd_format(TN_CONV_PW2, prefix, il, "bias"), false); + + // Feed-forward networks + layer.ff_norm_w = get_tensor(mtmd_format(TN_FFN_NORM, prefix, il, "weight")); + layer.ff_norm_b = get_tensor(mtmd_format(TN_FFN_NORM, prefix, il, "bias")); + + layer.ff_norm_1_w = get_tensor(mtmd_format(TN_FFN_NORM_1, prefix, il, "weight")); + layer.ff_norm_1_b = get_tensor(mtmd_format(TN_FFN_NORM_1, prefix, il, "bias")); + layer.ff_up_1_w = get_tensor(mtmd_format(TN_FFN_UP_1, prefix, il, "weight")); + layer.ff_up_1_b = get_tensor(mtmd_format(TN_FFN_UP_1, prefix, il, "bias"), false); + layer.ff_down_1_w = get_tensor(mtmd_format(TN_FFN_DOWN_1, prefix, il, "weight")); + layer.ff_down_1_b = get_tensor(mtmd_format(TN_FFN_DOWN_1, prefix, il, "bias"), false); + + // Layer norms + layer.norm_conv_w = get_tensor(mtmd_format(TN_NORM_CONV, prefix, il, "weight")); + layer.norm_conv_b = get_tensor(mtmd_format(TN_NORM_CONV, prefix, il, "bias")); + } + + model.mm_model_mlp_1_w = get_tensor(mtmd_format(TN_MVLM_PROJ_MLP, 0, "weight")); + model.mm_model_mlp_2_w = get_tensor(mtmd_format(TN_MVLM_PROJ_MLP, 1, "weight")); + model.mm_model_mlp_3_w = get_tensor(mtmd_format(TN_MVLM_PROJ_MLP, 3, "weight")); + } break; + case PROJECTOR_TYPE_NEMOTRON_TRANSCRIBE: + { + hparams.mel_filters = get_vector(TN_MEL_FILTERS); + hparams.window = get_vector(TN_WINDOW); + + // ASR expert: blocks, pre_ln and post_ln come from the generic loader + model.inp_proj_w = get_tensor(mtmd_format(TN_INP_PROJ, "weight")); + for (int il = 0; il < hparams.n_layer; ++il) { + auto & layer = model.layers[il]; + layer.q_norm_b = get_tensor(mtmd_format(TN_ATTN_Q_NORM, prefix, il, "bias")); + layer.k_norm_b = get_tensor(mtmd_format(TN_ATTN_K_NORM, prefix, il, "bias")); + } + + // speaker expert + const char * sp = "a.spk"; + model.spk_inp_proj_w = get_tensor(mtmd_format(TN_A_SPK_INP_PROJ, "weight")); + model.spk_pre_ln_w = get_tensor(mtmd_format(TN_LN_PRE, sp, "weight")); + model.spk_pre_ln_b = get_tensor(mtmd_format(TN_LN_PRE, sp, "bias")); + model.spk_post_ln_w = get_tensor(mtmd_format(TN_LN_POST, sp, "weight")); + model.spk_post_ln_b = get_tensor(mtmd_format(TN_LN_POST, sp, "bias")); + model.spk_layers.resize(hparams.spk_n_layer); + for (int il = 0; il < hparams.spk_n_layer; ++il) { + auto & layer = model.spk_layers[il]; + layer.ln_1_w = get_tensor(mtmd_format(TN_LN_1, sp, il, "weight")); + layer.ln_1_b = get_tensor(mtmd_format(TN_LN_1, sp, il, "bias")); + layer.qkv_w = get_tensor(mtmd_format(TN_ATTN_QKV, sp, il, "weight")); + layer.o_w = get_tensor(mtmd_format(TN_ATTN_OUTPUT, sp, il, "weight")); + layer.o_b = get_tensor(mtmd_format(TN_ATTN_OUTPUT, sp, il, "bias")); + layer.ln_2_w = get_tensor(mtmd_format(TN_LN_2, sp, il, "weight")); + layer.ln_2_b = get_tensor(mtmd_format(TN_LN_2, sp, il, "bias")); + layer.ff_up_w = get_tensor(mtmd_format(TN_FFN_UP, sp, il, "weight")); + layer.ff_up_b = get_tensor(mtmd_format(TN_FFN_UP, sp, il, "bias")); + layer.ff_down_w = get_tensor(mtmd_format(TN_FFN_DOWN, sp, il, "weight")); + layer.ff_down_b = get_tensor(mtmd_format(TN_FFN_DOWN, sp, il, "bias")); + } + + // Sortformer head, fusion + model.spk_head_proj_w = get_tensor(mtmd_format(TN_A_SPK_HEAD, "proj", "weight")); + model.spk_head_proj_b = get_tensor(mtmd_format(TN_A_SPK_HEAD, "proj", "bias")); + model.spk_head_up_w = get_tensor(mtmd_format(TN_A_SPK_HEAD, "upsample", "weight")); + model.spk_head_up_b = get_tensor(mtmd_format(TN_A_SPK_HEAD, "upsample", "bias")); + model.spk_head_fc1_w = get_tensor(mtmd_format(TN_A_SPK_HEAD, "fc1", "weight")); + model.spk_head_fc1_b = get_tensor(mtmd_format(TN_A_SPK_HEAD, "fc1", "bias")); + model.spk_head_fc2_w = get_tensor(mtmd_format(TN_A_SPK_HEAD, "fc2", "weight")); + model.spk_head_fc2_b = get_tensor(mtmd_format(TN_A_SPK_HEAD, "fc2", "bias")); + model.fusion_asr_norm_w = get_tensor(mtmd_format(TN_A_FUSION_NORM, "asr_norm", "weight")); + model.fusion_asr_norm_b = get_tensor(mtmd_format(TN_A_FUSION_NORM, "asr_norm", "bias")); + model.fusion_spk_norm_w = get_tensor(mtmd_format(TN_A_FUSION_NORM, "spk_norm", "weight")); + model.fusion_spk_norm_b = get_tensor(mtmd_format(TN_A_FUSION_NORM, "spk_norm", "bias")); + model.fusion_spk_kernel = get_tensor(TN_A_FUSION_KERNEL); + + model.spk_sil_emb = get_vector(TN_A_SPK_SIL_EMB); + + // the head layouts and the widths the graph and the speaker state use must match the tensors + hparams.spk_n_embd = model.spk_inp_proj_w->ne[1]; + const int64_t d_head = hparams.n_head > 0 ? hparams.n_embd / hparams.n_head : 0; + const int64_t d_head_spk = hparams.spk_n_head > 0 ? hparams.spk_n_embd / hparams.spk_n_head : 0; + const int64_t n_spk = model.spk_head_fc2_w->ne[1]; + // the graph splits each fused QKV into three views of the layer width + bool qkv_ok = true; + for (const auto & layer : model.layers) { + qkv_ok = qkv_ok && layer.qkv_w && layer.qkv_w->ne[0] == hparams.n_embd && layer.qkv_w->ne[1] == 3 * hparams.n_embd; + } + for (const auto & layer : model.spk_layers) { + qkv_ok = qkv_ok && layer.qkv_w->ne[0] == hparams.spk_n_embd && layer.qkv_w->ne[1] == 3 * hparams.spk_n_embd; + } + if (!qkv_ok || model.inp_proj_w->ne[1] != hparams.n_embd || d_head <= 0 || d_head * hparams.n_head != hparams.n_embd || + (hparams.n_embd_head > 0 && hparams.n_embd_head != d_head) || + hparams.audio_rope_dims <= 0 || hparams.audio_rope_dims % 2 != 0 || hparams.audio_rope_dims > d_head || + d_head_spk <= 0 || d_head_spk % 2 != 0 || d_head_spk * hparams.spk_n_head != hparams.spk_n_embd || + (int64_t) model.spk_sil_emb.size() != hparams.spk_n_embd || n_spk <= 0 || + model.fusion_spk_kernel->ne[0] != n_spk || model.fusion_spk_kernel->ne[1] != hparams.n_embd || + hparams.mel_filters.size() != (size_t) hparams.n_mel_bins * (hparams.audio_n_fft / 2 + 1) || + hparams.window.size() != (size_t) hparams.audio_window_len) { + throw std::runtime_error(mtmd_format("%s: hparams do not match the tensor shapes\n", __func__)); + } + + // projection to the text model + model.mm_fc_w = get_tensor(mtmd_format(TN_MM_AUDIO_FC, "weight")); + model.mm_fc_b = get_tensor(mtmd_format(TN_MM_AUDIO_FC, "bias")); + } break; + case PROJECTOR_TYPE_GRANITE_SPEECH: + { + model.inp_proj_w = get_tensor(mtmd_format(TN_INP_PROJ, "weight")); + model.inp_proj_b = get_tensor(mtmd_format(TN_INP_PROJ, "bias")); + model.ctc_out_w = get_tensor(mtmd_format(TN_CTC_OUT, "weight")); + model.ctc_out_b = get_tensor(mtmd_format(TN_CTC_OUT, "bias")); + model.ctc_out_mid_w = get_tensor(mtmd_format(TN_CTC_OUT_MID, "weight")); + model.ctc_out_mid_b = get_tensor(mtmd_format(TN_CTC_OUT_MID, "bias")); + + // per-layer tensors not loaded by the generic loop above + for (int il = 0; il < hparams.n_layer; ++il) { + auto & layer = model.layers[il]; + + layer.attn_rel_pos_emb = get_tensor(mtmd_format(TN_ATTN_REL_POS_EMB, prefix, il)); + + layer.ff_norm_w = get_tensor(mtmd_format(TN_FFN_NORM, prefix, il, "weight")); + layer.ff_norm_b = get_tensor(mtmd_format(TN_FFN_NORM, prefix, il, "bias")); + + layer.ff_norm_1_w = get_tensor(mtmd_format(TN_FFN_NORM_1, prefix, il, "weight")); + layer.ff_norm_1_b = get_tensor(mtmd_format(TN_FFN_NORM_1, prefix, il, "bias")); + layer.ff_up_1_w = get_tensor(mtmd_format(TN_FFN_UP_1, prefix, il, "weight")); + layer.ff_up_1_b = get_tensor(mtmd_format(TN_FFN_UP_1, prefix, il, "bias")); + layer.ff_down_1_w = get_tensor(mtmd_format(TN_FFN_DOWN_1, prefix, il, "weight")); + layer.ff_down_1_b = get_tensor(mtmd_format(TN_FFN_DOWN_1, prefix, il, "bias")); + + layer.norm_conv_w = get_tensor(mtmd_format(TN_NORM_CONV, prefix, il, "weight")); + layer.norm_conv_b = get_tensor(mtmd_format(TN_NORM_CONV, prefix, il, "bias")); + layer.conv_norm_w = get_tensor(mtmd_format(TN_CONV_NORM, prefix, il, "weight")); + layer.conv_norm_b = get_tensor(mtmd_format(TN_CONV_NORM, prefix, il, "bias")); + layer.conv_dw_w = get_tensor(mtmd_format(TN_CONV_DW, prefix, il, "weight")); + layer.conv_pw1_w = get_tensor(mtmd_format(TN_CONV_PW1, prefix, il, "weight")); + layer.conv_pw1_b = get_tensor(mtmd_format(TN_CONV_PW1, prefix, il, "bias")); + layer.conv_pw2_w = get_tensor(mtmd_format(TN_CONV_PW2, prefix, il, "weight")); + layer.conv_pw2_b = get_tensor(mtmd_format(TN_CONV_PW2, prefix, il, "bias")); + } + + model.qf_proj_blocks.resize(1); + auto & qf = model.qf_proj_blocks[0]; + qf.qf_proj_query = get_tensor(mtmd_format(TN_QF_PROJ_QUERY, prefix)); + qf.qf_proj_norm_w = get_tensor(mtmd_format(TN_QF_PROJ_NORM, prefix, "weight")); + qf.qf_proj_norm_b = get_tensor(mtmd_format(TN_QF_PROJ_NORM, prefix, "bias")); + qf.qf_proj_linear_w = get_tensor(mtmd_format(TN_QF_PROJ_LINEAR, prefix, "weight")); + qf.qf_proj_linear_b = get_tensor(mtmd_format(TN_QF_PROJ_LINEAR, prefix, "bias")); + + const int n_proj_layers = 2; + qf.qf_proj_layers.resize(n_proj_layers); + for (int il = 0; il < n_proj_layers; ++il) { + auto & pl = qf.qf_proj_layers[il]; + + pl.q_w = get_tensor(mtmd_format(TN_QF_SELF_ATTN_Q, prefix, il, "weight")); + pl.q_b = get_tensor(mtmd_format(TN_QF_SELF_ATTN_Q, prefix, il, "bias")); + pl.k_w = get_tensor(mtmd_format(TN_QF_SELF_ATTN_K, prefix, il, "weight")); + pl.k_b = get_tensor(mtmd_format(TN_QF_SELF_ATTN_K, prefix, il, "bias")); + pl.v_w = get_tensor(mtmd_format(TN_QF_SELF_ATTN_V, prefix, il, "weight")); + pl.v_b = get_tensor(mtmd_format(TN_QF_SELF_ATTN_V, prefix, il, "bias")); + pl.o_w = get_tensor(mtmd_format(TN_QF_SELF_ATTN_O, prefix, il, "weight")); + pl.o_b = get_tensor(mtmd_format(TN_QF_SELF_ATTN_O, prefix, il, "bias")); + pl.ln_1_w = get_tensor(mtmd_format(TN_QF_SELF_ATTN_N, prefix, il, "weight")); + pl.ln_1_b = get_tensor(mtmd_format(TN_QF_SELF_ATTN_N, prefix, il, "bias")); + + pl.cross_attn_q_w = get_tensor(mtmd_format(TN_QF_CROSS_ATTN_Q, prefix, il, "weight")); + pl.cross_attn_q_b = get_tensor(mtmd_format(TN_QF_CROSS_ATTN_Q, prefix, il, "bias")); + pl.cross_attn_k_w = get_tensor(mtmd_format(TN_QF_CROSS_ATTN_K, prefix, il, "weight")); + pl.cross_attn_k_b = get_tensor(mtmd_format(TN_QF_CROSS_ATTN_K, prefix, il, "bias")); + pl.cross_attn_v_w = get_tensor(mtmd_format(TN_QF_CROSS_ATTN_V, prefix, il, "weight")); + pl.cross_attn_v_b = get_tensor(mtmd_format(TN_QF_CROSS_ATTN_V, prefix, il, "bias")); + pl.cross_attn_o_w = get_tensor(mtmd_format(TN_QF_CROSS_ATTN_O, prefix, il, "weight")); + pl.cross_attn_o_b = get_tensor(mtmd_format(TN_QF_CROSS_ATTN_O, prefix, il, "bias")); + pl.cross_attn_norm_w = get_tensor(mtmd_format(TN_QF_CROSS_ATTN_N, prefix, il, "weight")); + pl.cross_attn_norm_b = get_tensor(mtmd_format(TN_QF_CROSS_ATTN_N, prefix, il, "bias")); + + pl.ff_up_w = get_tensor(mtmd_format(TN_QF_FFN_UP, prefix, il, "weight")); + pl.ff_up_b = get_tensor(mtmd_format(TN_QF_FFN_UP, prefix, il, "bias")); + pl.ff_down_w = get_tensor(mtmd_format(TN_QF_FFN_DOWN, prefix, il, "weight")); + pl.ff_down_b = get_tensor(mtmd_format(TN_QF_FFN_DOWN, prefix, il, "bias")); + pl.ln_2_w = get_tensor(mtmd_format(TN_QF_FFN_NORM, prefix, il, "weight")); + pl.ln_2_b = get_tensor(mtmd_format(TN_QF_FFN_NORM, prefix, il, "bias")); + } + } break; + case PROJECTOR_TYPE_GRANITE4_VISION: + { + // image_newline lives at the top-level. + model.image_newline = get_tensor(TN_IMAGE_NEWLINE); + + // Load separate layerwise and spatial projector tensors + const auto projector_count = hparams.feature_layers.size(); + model.qf_proj_blocks.resize(projector_count); + for (size_t bid = 0; bid < projector_count; ++bid) { + auto & b = model.qf_proj_blocks[bid]; + + // non-layerwise tensors + b.qf_proj_img_pos = get_tensor(mtmd_format(TN_MULTI_PROJ_IMG_POS, bid)); + b.qf_proj_query = get_tensor(mtmd_format(TN_MULTI_PROJ_QUERY, prefix, bid)); + b.qf_proj_linear_w = get_tensor(mtmd_format(TN_MULTI_PROJ_LINEAR, prefix, bid, "weight")); + b.qf_proj_linear_b = get_tensor(mtmd_format(TN_MULTI_PROJ_LINEAR, prefix, bid, "bias")); + b.qf_proj_norm_w = get_tensor(mtmd_format(TN_MULTI_PROJ_NORM, prefix, bid, "weight")); + b.qf_proj_norm_b = get_tensor(mtmd_format(TN_MULTI_PROJ_NORM, prefix, bid, "bias")); + b.qf_proj_post_norm_w = get_tensor(mtmd_format(TN_MULTI_PROJ_POST_NORM, prefix, bid, "weight")); + b.qf_proj_post_norm_b = get_tensor(mtmd_format(TN_MULTI_PROJ_POST_NORM, prefix, bid, "bias")); + + // laywerwise tensors + // NOTE: If any model uses multi-layer qformers, this will need to change + b.qf_proj_layers.resize(1); + auto & pl = b.qf_proj_layers[0]; + + pl.q_w = get_tensor(mtmd_format(TN_QF_SELF_ATTN_Q, prefix, bid, "weight")); + pl.q_b = get_tensor(mtmd_format(TN_QF_SELF_ATTN_Q, prefix, bid, "bias")); + pl.k_w = get_tensor(mtmd_format(TN_QF_SELF_ATTN_K, prefix, bid, "weight")); + pl.k_b = get_tensor(mtmd_format(TN_QF_SELF_ATTN_K, prefix, bid, "bias")); + pl.v_w = get_tensor(mtmd_format(TN_QF_SELF_ATTN_V, prefix, bid, "weight")); + pl.v_b = get_tensor(mtmd_format(TN_QF_SELF_ATTN_V, prefix, bid, "bias")); + pl.o_w = get_tensor(mtmd_format(TN_QF_SELF_ATTN_O, prefix, bid, "weight")); + pl.o_b = get_tensor(mtmd_format(TN_QF_SELF_ATTN_O, prefix, bid, "bias")); + pl.ln_1_w = get_tensor(mtmd_format(TN_QF_SELF_ATTN_N, prefix, bid, "weight")); + pl.ln_1_b = get_tensor(mtmd_format(TN_QF_SELF_ATTN_N, prefix, bid, "bias")); + + pl.cross_attn_q_w = get_tensor(mtmd_format(TN_QF_CROSS_ATTN_Q, prefix, bid, "weight")); + pl.cross_attn_q_b = get_tensor(mtmd_format(TN_QF_CROSS_ATTN_Q, prefix, bid, "bias")); + pl.cross_attn_k_w = get_tensor(mtmd_format(TN_QF_CROSS_ATTN_K, prefix, bid, "weight")); + pl.cross_attn_k_b = get_tensor(mtmd_format(TN_QF_CROSS_ATTN_K, prefix, bid, "bias")); + pl.cross_attn_v_w = get_tensor(mtmd_format(TN_QF_CROSS_ATTN_V, prefix, bid, "weight")); + pl.cross_attn_v_b = get_tensor(mtmd_format(TN_QF_CROSS_ATTN_V, prefix, bid, "bias")); + pl.cross_attn_o_w = get_tensor(mtmd_format(TN_QF_CROSS_ATTN_O, prefix, bid, "weight")); + pl.cross_attn_o_b = get_tensor(mtmd_format(TN_QF_CROSS_ATTN_O, prefix, bid, "bias")); + pl.cross_attn_norm_w = get_tensor(mtmd_format(TN_QF_CROSS_ATTN_N, prefix, bid, "weight")); + pl.cross_attn_norm_b = get_tensor(mtmd_format(TN_QF_CROSS_ATTN_N, prefix, bid, "bias")); + + pl.ff_up_w = get_tensor(mtmd_format(TN_QF_FFN_UP, prefix, bid, "weight")); + pl.ff_up_b = get_tensor(mtmd_format(TN_QF_FFN_UP, prefix, bid, "bias")); + pl.ff_down_w = get_tensor(mtmd_format(TN_QF_FFN_DOWN, prefix, bid, "weight")); + pl.ff_down_b = get_tensor(mtmd_format(TN_QF_FFN_DOWN, prefix, bid, "bias")); + pl.ln_2_w = get_tensor(mtmd_format(TN_QF_FFN_NORM, prefix, bid, "weight")); + pl.ln_2_b = get_tensor(mtmd_format(TN_QF_FFN_NORM, prefix, bid, "bias")); + } + + } break; + case PROJECTOR_TYPE_SENSENOVA_U1: + { + model.mm_0_w = get_tensor(mtmd_format(TN_LLAVA_PROJ, 0, "weight")); + model.mm_0_b = get_tensor(mtmd_format(TN_LLAVA_PROJ, 0, "bias")); + } break; + default: + GGML_ASSERT(false && "unknown projector type"); + } + + // load data + { + std::vector read_buf; + + // start loading event + if (progress_callback){ + progress_callback(0.0, progress_callback_user_data); + } + + // compute total tensor data size for progress reporting + size_t total_data_size = 0; + for (auto & t : tensors_to_load) { + total_data_size += ggml_nbytes(t); + } + + // alloc memory and offload data + model.buf.reset(ggml_backend_alloc_ctx_tensors_from_buft(model.ctx_data.get(), buft)); + ggml_backend_buffer_set_usage(model.buf.get(), GGML_BACKEND_BUFFER_USAGE_WEIGHTS); + // read the weight from file + if (!no_alloc) { + size_t data_loaded = 0; + for (auto & t : tensors_to_load) { + ggml_tensor * cur = ggml_get_tensor(model.ctx_data.get(), t->name); + GGML_ASSERT(cur && "tensor not found in ctx_data"); + auto it_off = tensor_offset.find(t->name); + GGML_ASSERT(it_off != tensor_offset.end() && "no offset for tensor"); + const size_t offset = it_off->second; + fin.seekg(offset, std::ios::beg); + if (!fin) { + throw std::runtime_error(mtmd_format("%s: failed to seek for tensor %s\n", __func__, t->name)); + } + size_t num_bytes = ggml_nbytes(cur); + if (ggml_backend_buft_is_host(buft)) { + // for the CPU and Metal backend, we can read directly into the tensor + fin.read(reinterpret_cast(cur->data), num_bytes); + } else { + // read into a temporary buffer first, then copy to device memory + read_buf.resize(num_bytes); + fin.read(reinterpret_cast(read_buf.data()), num_bytes); + ggml_backend_tensor_set(cur, read_buf.data(), 0, num_bytes); + } + data_loaded += num_bytes; + if (progress_callback && total_data_size > 0) { + const float progress = (float)data_loaded / (float)total_data_size; + if (!progress_callback(progress, progress_callback_user_data)) { + throw std::runtime_error(mtmd_format("%s: model loading cancelled by progress_callback\n", __func__)); + } + } + } + LOG_DBG("%s: loaded %zu tensors from %s\n", __func__, tensors_to_load.size(), fname.c_str()); + } else { + LOG_DBG("%s: no_alloc is set, skipping tensor data loading (%zu tensors)\n", __func__, tensors_to_load.size()); + } + fin.close(); + } + + } + + void get_bool(const std::string & key, bool & output, bool required = true) const { + const int i = gguf_find_key(ctx_gguf.get(), key.c_str()); + if (i < 0) { + if (required) { + throw std::runtime_error("Key not found: " + key); + } + return; + } + output = gguf_get_val_bool(ctx_gguf.get(), i); + } + + void get_i32(const std::string & key, int & output, bool required = true) const { + const int i = gguf_find_key(ctx_gguf.get(), key.c_str()); + if (i < 0) { + if (required) { + throw std::runtime_error("Key not found: " + key); + } + return; + } + output = gguf_get_val_i32(ctx_gguf.get(), i); + } + + void get_u32(const std::string & key, int & output, bool required = true) const { + const int i = gguf_find_key(ctx_gguf.get(), key.c_str()); + if (i < 0) { + if (required) { + throw std::runtime_error("Key not found: " + key); + } + return; + } + const uint32_t val = gguf_get_val_u32(ctx_gguf.get(), i); + // sanity check + if (val > (uint32_t) INT32_MAX) { + throw std::runtime_error(mtmd_format("%s: value %u for key '%s' exceeds INT32_MAX\n", + __func__, val, key.c_str())); + } + output = (int) val; + } + + void get_f32(const std::string & key, float & output, bool required = true) const { + const int i = gguf_find_key(ctx_gguf.get(), key.c_str()); + if (i < 0) { + if (required) { + throw std::runtime_error("Key not found: " + key); + } + return; + } + output = gguf_get_val_f32(ctx_gguf.get(), i); + } + + void get_arr_f32(const std::string & key, std::vector & output, bool required = true) const { + const int i = gguf_find_key(ctx_gguf.get(), key.c_str()); + if (i < 0) { + if (required) { + throw std::runtime_error("Key not found: " + key); + } + return; + } + if (gguf_get_kv_type(ctx_gguf.get(), i) != GGUF_TYPE_ARRAY) { + throw std::runtime_error(mtmd_format("%s: key '%s' is not an array\n", __func__, key.c_str())); + } + const auto type = gguf_get_arr_type(ctx_gguf.get(), i); + if (type != GGUF_TYPE_FLOAT32) { + throw std::runtime_error(mtmd_format("%s: array '%s' has type %d, expected %d (GGUF_TYPE_FLOAT32)\n", __func__, key.c_str(), type, GGUF_TYPE_FLOAT32)); + } + const size_t n = gguf_get_arr_n(ctx_gguf.get(), i); + if (n > (size_t) std::numeric_limits::max()) { + throw std::runtime_error(mtmd_format("%s: array '%s' is too large (%zu elements)\n", __func__, key.c_str(), n)); + } + output.resize(n); + const float * values = (const float *)gguf_get_arr_data(ctx_gguf.get(), i); + for (size_t j = 0; j < n; ++j) { + output[j] = values[j]; + } + } + + void get_string(const std::string & key, std::string & output, bool required = true) const { + const int i = gguf_find_key(ctx_gguf.get(), key.c_str()); + if (i < 0) { + if (required) { + throw std::runtime_error("Key not found: " + key); + } + return; + } + output = std::string(gguf_get_val_str(ctx_gguf.get(), i)); + } + + void get_arr_int(const std::string & key, std::vector & output, bool required = true) const { + const int i = gguf_find_key(ctx_gguf.get(), key.c_str()); + if (i < 0) { + if (required) { + throw std::runtime_error("Key not found: " + key); + } + return; + } + if (gguf_get_kv_type(ctx_gguf.get(), i) != GGUF_TYPE_ARRAY) { + throw std::runtime_error(mtmd_format("%s: key '%s' is not an array\n", __func__, key.c_str())); + } + const auto type = gguf_get_arr_type(ctx_gguf.get(), i); + if (type != GGUF_TYPE_INT32) { + throw std::runtime_error(mtmd_format("%s: array '%s' has type %d, expected %d (GGUF_TYPE_INT32)\n", __func__, key.c_str(), type, GGUF_TYPE_INT32)); + } + const size_t n = gguf_get_arr_n(ctx_gguf.get(), i); + if (n > (size_t) std::numeric_limits::max()) { + throw std::runtime_error(mtmd_format("%s: array '%s' is too large (%zu elements)\n", __func__, key.c_str(), n)); + } + output.resize(n); + const int32_t * values = (const int32_t *)gguf_get_arr_data(ctx_gguf.get(), i); + for (size_t j = 0; j < n; ++j) { + output[j] = values[j]; + } + } + + static void set_llava_uhd_res_candidates(mtmd_model & model, const int max_patches_per_side) { + auto & hparams = model.hparams; + for (int x = 1; x <= max_patches_per_side; x++) { + for (int y = 1; y <= max_patches_per_side; y++) { + if (x == 1 && y == 1) { + continue; // skip the first point + } + hparams.image_res_candidates.push_back(mtmd_image_size{ + x*hparams.image_size, + y*hparams.image_size, + }); + } + } + } + + static void set_internvl_dhr_res_candidates(mtmd_model & model) { + auto & hparams = model.hparams; + int min_num = hparams.preproc_min_tiles; + int max_num = hparams.preproc_max_tiles; + if (min_num < 1) { + return; // avoid divide by 0 + } + for (int a = min_num; a <= max_num; ++a) { + int b_lo = (min_num + a - 1) / a; + int b_hi = max_num / a; + b_lo = std::max(b_lo, min_num); + b_hi = std::min(b_hi, max_num); + for (int b = b_lo; b <= b_hi; ++b) { + hparams.image_res_candidates.push_back(mtmd_image_size { + a*hparams.image_size, + b*hparams.image_size, + }); + } + } + } +}; + +// +// loading +// + +mtmd_models mtmd_model_load(const char * fname, const mtmd_model_params & params, ggml_backend_buffer_type_t buft) { + mtmd_model_loader loader(fname, + /* skip_tensors */ false, + params.progress_callback, + params.progress_callback_user_data); + + auto load = [&](mtmd_modality modality) { + auto model = std::make_unique(); + model->hparams.custom_image_min_tokens = params.image_min_tokens; + model->hparams.custom_image_max_tokens = params.image_max_tokens; + loader.load_hparams(*model, modality); + loader.load_tensors(*model, buft, params.no_alloc); + return model; + }; + + mtmd_models res; + if (loader.has_vision) { + res.v = load(MTMD_MODALITY_VISION); + } + + // TODO: we don't support audio for Gemma 3N, but GGUF contains audio tensors + // we can remove this check when we implement audio support for Gemma 3N + const bool skip_audio = res.v && res.v->proj_type == PROJECTOR_TYPE_GEMMA3NV; + if (loader.has_audio && !skip_audio) { + res.a = load(MTMD_MODALITY_AUDIO); + } + + if (loader.has_gen_audio) { + res.gen_a = load(MTMD_MODALITY_GEN_AUDIO); + } + + return res; +} + +mtmd_model_caps mtmd_model_get_caps(const char * fname) { + mtmd_model_caps res; + mtmd_model_loader loader(fname, /* skip_tensors= */ true); + res.has_vision = loader.has_vision; + res.has_audio = loader.has_audio; + res.has_gen_audio = loader.has_gen_audio; + return res; +} + +// +// mtmd_model +// + +int mtmd_model::image_max_tokens() const { + if (proj_type == PROJECTOR_TYPE_DEEPSEEK4V || proj_type == PROJECTOR_TYPE_DEEPSEEK41V) { + return hparams.dsv4_max_n_token; + } + if (hparams.image_max_pixels <= 0) { + return -1; + } + const int patch_area = hparams.patch_size * hparams.patch_size * hparams.n_merge * hparams.n_merge; + return hparams.image_max_pixels / patch_area; +} + +int mtmd_model::n_output_tokens_x(const mtmd_image_f32 * img) const { + const auto & params = hparams; + const int n_total = n_output_tokens(img); + const auto & proj = proj_type; + switch (proj) { + case PROJECTOR_TYPE_QWEN2VL: + case PROJECTOR_TYPE_QWEN25VL: + case PROJECTOR_TYPE_QWEN3VL: + case PROJECTOR_TYPE_MAGEVL: + case PROJECTOR_TYPE_SENSENOVA_U1: + case PROJECTOR_TYPE_LING3VL: + case PROJECTOR_TYPE_EXAONE4_5: + case PROJECTOR_TYPE_MIMOVL: + case PROJECTOR_TYPE_GLM4V: + case PROJECTOR_TYPE_GLM5V: + case PROJECTOR_TYPE_PADDLEOCR: + case PROJECTOR_TYPE_HUNYUANVL: + case PROJECTOR_TYPE_YOUTUVL: + case PROJECTOR_TYPE_MUSE_GLIMMER: + case PROJECTOR_TYPE_LFM3VL: + return (img->nx() / params.patch_size) / 2; + case PROJECTOR_TYPE_STEP3VL: + case PROJECTOR_TYPE_MINICPMV4_6: + case PROJECTOR_TYPE_MINICPMV4_7: + case PROJECTOR_TYPE_MINICPM_OCR_1_0: + return img->nx() / (params.patch_size * params.n_merge); + case PROJECTOR_TYPE_DEEPSEEKOCR: + case PROJECTOR_TYPE_DEEPSEEKOCR2: + return (img->nx() / params.patch_size) / 4; + default: + break; + } + return n_total; +} + +int mtmd_model::n_output_tokens_y(const mtmd_image_f32 * img) const { + const auto & params = hparams; + const auto & proj = proj_type; + switch (proj) { + case PROJECTOR_TYPE_QWEN2VL: + case PROJECTOR_TYPE_QWEN25VL: + case PROJECTOR_TYPE_QWEN3VL: + case PROJECTOR_TYPE_MAGEVL: + case PROJECTOR_TYPE_SENSENOVA_U1: + case PROJECTOR_TYPE_LING3VL: + case PROJECTOR_TYPE_EXAONE4_5: + case PROJECTOR_TYPE_MIMOVL: + case PROJECTOR_TYPE_GLM4V: + case PROJECTOR_TYPE_GLM5V: + case PROJECTOR_TYPE_PADDLEOCR: + case PROJECTOR_TYPE_HUNYUANVL: + case PROJECTOR_TYPE_YOUTUVL: + case PROJECTOR_TYPE_MUSE_GLIMMER: + case PROJECTOR_TYPE_LFM3VL: + return (img->ny() / params.patch_size) / 2; + case PROJECTOR_TYPE_STEP3VL: + case PROJECTOR_TYPE_MINICPMV4_6: + case PROJECTOR_TYPE_MINICPMV4_7: + case PROJECTOR_TYPE_MINICPM_OCR_1_0: + return img->ny() / (params.patch_size * params.n_merge); + default: + break; + } + return 1; +} + +int mtmd_model::n_output_tokens(const mtmd_image_f32 * img) const { + const auto & params = hparams; + + // for models with fixed size image, the input image is already pre-processed and resized to square + int patch_size = params.patch_size; + int n_patches = (img->nx() / patch_size) * (img->ny() / patch_size); + + projector_type proj = proj_type; + + switch (proj) { + case PROJECTOR_TYPE_MLP: + case PROJECTOR_TYPE_MLP_NORM: + case PROJECTOR_TYPE_JANUS_PRO: + case PROJECTOR_TYPE_PHI4: + { + // do nothing + } break; + case PROJECTOR_TYPE_YASA2: + { + n_patches = 64; // adaptive average pooling to 8x8 tokens + } break; + case PROJECTOR_TYPE_LDP: + case PROJECTOR_TYPE_LDPV2: + case PROJECTOR_TYPE_GLM_EDGE: + { + n_patches /= 4; + if (mm_boi) { + n_patches += 2; // for BOI and EOI token embeddings + } + } break; + case PROJECTOR_TYPE_MINICPMV: + { + // Use actual config value if available, otherwise fall back to hardcoded values + if (params.minicpmv_query_num > 0) { + n_patches = params.minicpmv_query_num; + } else { + // Fallback to hardcoded values for legacy models + if (params.minicpmv_version == 2) { + n_patches = 96; + } else if (params.minicpmv_version == 3) { + n_patches = 64; + } else if (params.minicpmv_version == 4) { + n_patches = 64; + } else if (params.minicpmv_version == 5) { + // MiniCPM-V 4.0 + n_patches = 64; + } else if (params.minicpmv_version == 6) { + // MiniCPM-V 4.5 + n_patches = 64; + } else if (params.minicpmv_version == 100045) { + // MiniCPM-o 4.5 + n_patches = 64; + } else { + GGML_ABORT("Unknown minicpmv version"); + } + } + } break; + case PROJECTOR_TYPE_MINICPMV4_6: + case PROJECTOR_TYPE_MINICPMV4_7: + case PROJECTOR_TYPE_MINICPM_OCR_1_0: + { + n_patches /= params.n_merge * params.n_merge; + } break; + case PROJECTOR_TYPE_QWEN2VL: + case PROJECTOR_TYPE_QWEN25VL: + case PROJECTOR_TYPE_QWEN3VL: + case PROJECTOR_TYPE_MAGEVL: + case PROJECTOR_TYPE_SENSENOVA_U1: + case PROJECTOR_TYPE_LING3VL: + case PROJECTOR_TYPE_EXAONE4_5: + case PROJECTOR_TYPE_MIMOVL: + case PROJECTOR_TYPE_MINIMAX_M3: + case PROJECTOR_TYPE_GLM4V: + case PROJECTOR_TYPE_GLM5V: + case PROJECTOR_TYPE_YOUTUVL: + case PROJECTOR_TYPE_MUSE_GLIMMER: + case PROJECTOR_TYPE_LFM3VL: + { + // dynamic size (2 conv, so double patch size) + int x_patch = img->nx() / (params.patch_size * 2); + int y_patch = img->ny() / (params.patch_size * 2); + n_patches = x_patch * y_patch; + } break; + case PROJECTOR_TYPE_STEP3VL: + { + int x_patch = img->nx() / (params.patch_size * params.n_merge); + int y_patch = img->ny() / (params.patch_size * params.n_merge); + n_patches = x_patch * y_patch; + } break; + case PROJECTOR_TYPE_GEMMA3: + case PROJECTOR_TYPE_GEMMA4V: + case PROJECTOR_TYPE_GEMMA4UV: + case PROJECTOR_TYPE_IDEFICS3: + case PROJECTOR_TYPE_COHERE2V: + case PROJECTOR_TYPE_INTERNVL: + case PROJECTOR_TYPE_NEMOTRON_V2_VL: + case PROJECTOR_TYPE_NEMOTRON_OMNI: + case PROJECTOR_TYPE_LLAMA4: + { + // both X and Y are downscaled by the scale factor + int scale_factor = hparams.n_merge; + n_patches /= (scale_factor * scale_factor); + } break; + case PROJECTOR_TYPE_GEMMA3NV: + { + // MobileNetV5 MSFA adapter always outputs fixed 16x16 resolution + // regardless of input size (see architecture description) + n_patches = hparams.image_size / hparams.patch_size; + } break; + case PROJECTOR_TYPE_LFM2: + case PROJECTOR_TYPE_D1OMNI_V: + case PROJECTOR_TYPE_KIMIVL: + case PROJECTOR_TYPE_KIMIK25: + case PROJECTOR_TYPE_KIMIK31: + { + // dynamic size + int out_patch_size = params.patch_size * hparams.n_merge; + int x_patch = MTMD_ALIGN(img->nx(), out_patch_size) / out_patch_size; + int y_patch = MTMD_ALIGN(img->ny(), out_patch_size) / out_patch_size; + n_patches = x_patch * y_patch; + } break; + case PROJECTOR_TYPE_DEEPSEEK4V: + { + const int out_patch_size = params.patch_size * params.n_merge; + const int n_llm_w = MTMD_ALIGN(img->nx(), out_patch_size) / out_patch_size; + const int n_llm_h = MTMD_ALIGN(img->ny(), out_patch_size) / out_patch_size; + n_patches = dsv4_get_block_layout(n_llm_w, n_llm_h, img->lead_pad).n_out; + } break; + case PROJECTOR_TYPE_DEEPSEEK41V: + { + const int out_patch_size = params.patch_size * params.n_merge; + const int n_llm_w = MTMD_ALIGN(img->nx(), out_patch_size) / out_patch_size; + const int n_llm_h = MTMD_ALIGN(img->ny(), out_patch_size) / out_patch_size; + n_patches = dsv41_n_block_tokens(n_llm_w, n_llm_h); + } break; + case PROJECTOR_TYPE_PADDLEOCR: + case PROJECTOR_TYPE_DOTS_OCR: + case PROJECTOR_TYPE_DOTS3NOTE_V: + { + // dynamic size + int n_merge = hparams.n_merge; + int stride = n_merge * n_merge; + n_patches = MTMD_ALIGN(n_patches, stride) / stride; + } break; + case PROJECTOR_TYPE_DOTS3NOTE_A: + { + // 3x stride-2 conv2d over mel frames + n_patches = (img->nx() + 7) / 8; + } break; + case PROJECTOR_TYPE_PIXTRAL: + case PROJECTOR_TYPE_LIGHTONOCR: + { + // dynamic size + int n_merge = hparams.n_merge; + int n_patches_x = img->nx() / patch_size / n_merge; + int n_patches_y = img->ny() / patch_size / n_merge; + if (token_embd_img_break) { + n_patches = n_patches_y * n_patches_x + n_patches_y - 1; // + one [IMG_BREAK] per row, except the last row + } else { + n_patches = n_patches_y * n_patches_x; + } + } break; + case PROJECTOR_TYPE_VOXTRAL_RT: + { + // 2 context frames, conv stride 2, then stacked frames + n_patches = (img->nx() - 2) / 2 / hparams.proj_stack_factor; + } break; + case PROJECTOR_TYPE_MUSE_SPARK_AUDIO: + { + const int64_t fold = hparams.audio_samples_per_token(); + n_patches = (int) ((img->nx() + fold - 1) / fold); + } break; + case PROJECTOR_TYPE_VOXTRAL: + case PROJECTOR_TYPE_ULTRAVOX: + case PROJECTOR_TYPE_QWEN2A: + case PROJECTOR_TYPE_MERALION: + case PROJECTOR_TYPE_MUSIC_FLAMINGO: + { + n_patches = img->nx(); + + const int proj_stack_factor = hparams.proj_stack_factor; + if (audio_has_stack_frames()) { + GGML_ASSERT(proj_stack_factor > 0); + const int n_len = MTMD_ALIGN(n_patches, proj_stack_factor); + n_patches = n_len / proj_stack_factor; + } + + // whisper downscales input token by half after conv1d + n_patches /= 2; + + if (audio_has_avgpool()) { + // divide by 2 because of nn.AvgPool1d(2, stride=2) + n_patches /= 2; + } + } break; + case PROJECTOR_TYPE_QWEN3A: + { + // chunk_size=100 frames --> 3x stride-2 conv2d --> 13 tokens per chunk + const int chunk_size = 100; + const int tokens_per_chunk = 13; + n_patches = (img->nx() / chunk_size) * tokens_per_chunk; + } break; + case PROJECTOR_TYPE_GLMA: + { + n_patches = img->nx(); + // whisper downscales input token by half after conv1d + n_patches /= 2; + // reshape by merge_factor + n_patches /= hparams.proj_stack_factor; + // for BOI and EOI token embeddings + n_patches += 2; + } break; + case PROJECTOR_TYPE_COGVLM: + { + n_patches += 2; // for BOI and EOI token embeddings + } break; + case PROJECTOR_TYPE_DEEPSEEKOCR: + { + // SAM encoder applies two stride-2 convolutions (net_2 and net_3) + // that reduce spatial dimensions by 4x in each direction (16x total) + // E.g., 64x64 -> 16x16 patches + n_patches /= 16; + + if (img->add_viewsep) { + // global view: one image-newline per token-row + trailing view separator + const int h = static_cast(std::sqrt(static_cast(n_patches))); + n_patches = h * (h + 1) + 1; + } else if (img->ny() >= img->nx() && img->ny() % img->nx() == 0) { + // tile row: one image-newline per token-row + const int grid_w = img->ny() / img->nx(); + const int tile_patches = img->nx() / (patch_size * 4); // patches per tile side (SAM divides by 4) + const int h = tile_patches; + n_patches = (tile_patches * grid_w + 1) * h; + } + } break; + case PROJECTOR_TYPE_HUNYUANVL: + { + int merge = hparams.n_merge; + int ow = (img->nx() / patch_size) / merge; + int oh = (img->ny() / patch_size) / merge; + n_patches = (ow + 1) * oh + 2; + } break; + case PROJECTOR_TYPE_DEEPSEEKOCR2: + { + // 1024 global view -> 256 query tokens + 1 view separator = 257; + // 768 local tile -> 144 query tokens, no separator. + n_patches /= 16; + if (img->add_viewsep) { + n_patches += 1; // view separator, appended only after the global view + } + } break; + case PROJECTOR_TYPE_LFM2A: + case PROJECTOR_TYPE_D1OMNI_A: + case PROJECTOR_TYPE_LFM3A: + { + n_patches = ((((img->nx() + 1) / 2) + 1) / 2 + 1) / 2; + } break; + case PROJECTOR_TYPE_GEMMA4A: + { + // Two Conv2D stride-2: O = floor((I + 2p - k) / s) + 1, p=1, k=3, s=2 + // O = floor((I - 1) / 2) + 1 + int n = img->nx(); + for (int i = 0; i < 2; i++) { + n = (n - 1) / 2 + 1; + } + n_patches = n; + } break; + case PROJECTOR_TYPE_NEMOTRON_TRANSCRIBE: + case PROJECTOR_TYPE_PARAKEET: + { + n_patches = (img->nx() + (params.subsampling_factor - 1)) / params.subsampling_factor; + } break; + case PROJECTOR_TYPE_GEMMA4UA: + { + n_patches = img->nx(); // no downsampling: one token per raw waveform frame + } break; + case PROJECTOR_TYPE_MIMO_AUDIO: + { + // conv1(s=1) + conv2(s=2) -> RVQ-encoder downsample conv(k=2,s=2) + int n = img->nx(); + n = (n - 1) / 2 + 1; // conv1 + conv2 + n = (n - 2) / 2 + 1; // downsample conv + const int group_size = params.audio_local_group_size; + n_patches = (n + group_size - 1) / group_size; + } break; + case PROJECTOR_TYPE_GRANITE_SPEECH: + { + const int ws = hparams.audio_proj_window_size; + const int ds = hparams.audio_proj_downsample_rate; + n_patches = ((img->nx() + ws - 1) / ws) * (ws / ds); + } break; + case PROJECTOR_TYPE_QWEN3TTS_SPKENC: + { + // pooling gives one speaker embedding, whatever the clip length is + n_patches = 1; + } break; + case PROJECTOR_TYPE_CHATTERBOX_SPKENC: + { + // statistics pooling gives one speaker embedding, whatever the clip length is + n_patches = 1; + } break; + case PROJECTOR_TYPE_QWEN3TTS_GEN: + case PROJECTOR_TYPE_LFM3A_GEN: + case PROJECTOR_TYPE_DRAMA3_GEN: + { + // one hidden-state vector fed back to the talker per call + n_patches = 1; + } break; + case PROJECTOR_TYPE_POCKETTTS_SPKENC: + { + // one conditioning row per 12.5Hz frame + const int hop = hparams.mimi_downsample * 120; + n_patches = img->nx() / hop; + } break; + case PROJECTOR_TYPE_POCKETTTS_GEN: + { + // one latent per call for GEN_CODE, GEN_WAV sizes its input from the caller + n_patches = 1; + } break; + case PROJECTOR_TYPE_GRANITE4_VISION: + { + // Per-tile output token count: each projector block outputs + // query_side^2 tokens per window x n^2 windows. + // For 384x384 input: n = 24/8 = 3, query_side = 4 -> 144. + const int window_side = hparams.downsample_window_side; + const int query_side = hparams.downsample_query_side; + const int side = img->nx() / params.patch_size; + const int n = side / window_side; + const int out_side = query_side * n; + n_patches = out_side * out_side; + if (img->anyres.is_tiled()) { + // overview tile, then the unpadded tile grid with one newline per row + int off_x, off_y, w, h; + mtmd_anyres_unpad(img->anyres.grid_x * out_side, img->anyres.grid_y * out_side, + img->anyres.orig_nx, img->anyres.orig_ny, off_x, off_y, w, h); + n_patches += h * (w + 1); + } + } break; + default: + GGML_ABORT("unsupported projector type"); + } + + n_patches += (int) mtmd_suffix_rows(img->suffix_type).size(); + + return n_patches; +} + +size_t mtmd_model::cbx_read_tensor(const char * name, float * out, size_t n_max) const { + const auto & tensors = cbx_tensors; + auto it = tensors.find(name); + if (it == tensors.end() || !it->second) { + return 0; + } + ggml_tensor * t = it->second; + const size_t n = (size_t) ggml_nelements(t); + if (!out) { + return n; + } + if (n_max < n) { + return 0; + } + if (t->type == GGML_TYPE_F32) { + ggml_backend_tensor_get(t, out, 0, n * sizeof(float)); + } else if (t->type == GGML_TYPE_F16) { + std::vector tmp(n); + ggml_backend_tensor_get(t, tmp.data(), 0, n * sizeof(ggml_fp16_t)); + for (size_t i = 0; i < n; i++) out[i] = ggml_fp16_to_fp32(tmp[i]); + } else { + return 0; + } + return n; +} + +int mtmd_model::n_mmproj_embd() const { + switch (proj_type) { + case PROJECTOR_TYPE_LDP: + return mm_model_block_1_block_2_1_b->ne[0]; + case PROJECTOR_TYPE_LDPV2: + return mm_model_peg_0_b->ne[0]; + case PROJECTOR_TYPE_MLP: + case PROJECTOR_TYPE_PHI4: + case PROJECTOR_TYPE_PIXTRAL: + case PROJECTOR_TYPE_LIGHTONOCR: + case PROJECTOR_TYPE_DOTS_OCR: + case PROJECTOR_TYPE_DOTS3NOTE_V: + case PROJECTOR_TYPE_DOTS3NOTE_A: + return mm_2_w->ne[1]; + case PROJECTOR_TYPE_MLP_NORM: + return mm_3_b->ne[0]; + case PROJECTOR_TYPE_MINICPMV: + return mm_model_proj->ne[1]; + case PROJECTOR_TYPE_MINICPMV4_6: + case PROJECTOR_TYPE_MINICPMV4_7: + case PROJECTOR_TYPE_MINICPM_OCR_1_0: + return mm_ffn_down_w->ne[1]; + case PROJECTOR_TYPE_GLM_EDGE: + return mm_model_mlp_3_w->ne[1]; + case PROJECTOR_TYPE_MINIMAX_M3: + return mm_merger_fc2_b->ne[0]; + case PROJECTOR_TYPE_MUSE_GLIMMER: + return mm_2_w->ne[1]; + case PROJECTOR_TYPE_QWEN2VL: + case PROJECTOR_TYPE_QWEN25VL: + case PROJECTOR_TYPE_MAGEVL: + case PROJECTOR_TYPE_LFM3VL: + case PROJECTOR_TYPE_EXAONE4_5: + case PROJECTOR_TYPE_JANUS_PRO: + case PROJECTOR_TYPE_YOUTUVL: + return mm_1_b->ne[0]; + case PROJECTOR_TYPE_SENSENOVA_U1: + return hparams.projection_dim; + case PROJECTOR_TYPE_QWEN3VL: + // main path + deepstack paths + return mm_1_b->ne[0] * (1 + n_deepstack_layers); + case PROJECTOR_TYPE_LING3VL: + return mm_1_b->ne[0]; + case PROJECTOR_TYPE_MIMOVL: + return mm_1_w->ne[1]; + case PROJECTOR_TYPE_STEP3VL: + return mm_model_proj->ne[1]; + case PROJECTOR_TYPE_GEMMA3: + case PROJECTOR_TYPE_GEMMA3NV: + return mm_input_proj_w->ne[0]; + case PROJECTOR_TYPE_GEMMA4V: + case PROJECTOR_TYPE_GEMMA4UV: + case PROJECTOR_TYPE_GEMMA4A: + case PROJECTOR_TYPE_GEMMA4UA: + return mm_input_proj_w->ne[1]; + case PROJECTOR_TYPE_IDEFICS3: + case PROJECTOR_TYPE_MUSE_SPARK_AUDIO: + return mm_fc_w->ne[1]; + case PROJECTOR_TYPE_ULTRAVOX: + case PROJECTOR_TYPE_VOXTRAL: + case PROJECTOR_TYPE_VOXTRAL_RT: + case PROJECTOR_TYPE_MUSIC_FLAMINGO: + return mm_2_w->ne[1]; + case PROJECTOR_TYPE_MERALION: + return mm_3_w->ne[1]; // out_proj output dim + case PROJECTOR_TYPE_INTERNVL: + case PROJECTOR_TYPE_NEMOTRON_V2_VL: + case PROJECTOR_TYPE_NEMOTRON_OMNI: + return mm_3_w->ne[1]; + case PROJECTOR_TYPE_LLAMA4: + return mm_model_proj->ne[1]; + case PROJECTOR_TYPE_QWEN2A: + case PROJECTOR_TYPE_NEMOTRON_TRANSCRIBE: + return mm_fc_w->ne[1]; + case PROJECTOR_TYPE_QWEN3A: + return mm_2_w->ne[1]; + case PROJECTOR_TYPE_GLMA: + case PROJECTOR_TYPE_LFM2: + case PROJECTOR_TYPE_D1OMNI_V: + case PROJECTOR_TYPE_KIMIVL: + case PROJECTOR_TYPE_PADDLEOCR: + case PROJECTOR_TYPE_KIMIK25: + case PROJECTOR_TYPE_KIMIK31: + case PROJECTOR_TYPE_YASA2: + case PROJECTOR_TYPE_DEEPSEEK4V: + case PROJECTOR_TYPE_DEEPSEEK41V: + case PROJECTOR_TYPE_COHERE2V: + return mm_2_w->ne[1]; + case PROJECTOR_TYPE_HUNYUANVL: + return mm_model_proj->ne[1]; + case PROJECTOR_TYPE_COGVLM: + return mm_4h_to_h_w->ne[1]; + case PROJECTOR_TYPE_DEEPSEEKOCR: + case PROJECTOR_TYPE_DEEPSEEKOCR2: + return mm_fc_w->ne[1]; + case PROJECTOR_TYPE_LFM2A: + return position_embeddings->ne[0]; + case PROJECTOR_TYPE_D1OMNI_A: + case PROJECTOR_TYPE_LFM3A: + return mm_3_w->ne[1]; + case PROJECTOR_TYPE_GRANITE_SPEECH: + return qf_proj_blocks[0].qf_proj_linear_w->ne[1]; + case PROJECTOR_TYPE_GRANITE4_VISION: + return qf_proj_blocks.size() * hparams.projection_dim; + case PROJECTOR_TYPE_GLM4V: + case PROJECTOR_TYPE_GLM5V: + return mm_ffn_down_w->ne[1]; + case PROJECTOR_TYPE_MIMO_AUDIO: + return mm_2_w->ne[1]; + case PROJECTOR_TYPE_QWEN3TTS_SPKENC: + return mm_fc_w->ne[2]; + case PROJECTOR_TYPE_QWEN3TTS_GEN: + case PROJECTOR_TYPE_LFM3A_GEN: + case PROJECTOR_TYPE_DRAMA3_GEN: + return gen_code_out_embd_w->ne[0]; + case PROJECTOR_TYPE_POCKETTTS_SPKENC: + return spk_proj_w->ne[1]; + case PROJECTOR_TYPE_POCKETTTS_GEN: + return gen_input_lin_w->ne[1]; + case PROJECTOR_TYPE_CHATTERBOX: + // talker width, carried by the speech table + return cbx_tensors.at("a.gen.code.out_embd.weight")->ne[0]; + case PROJECTOR_TYPE_CHATTERBOX_SPKENC: + // talker conditioning rows, the speaker projection carries their width + return cbx_tensors.at("a.cenc.spkr_enc.weight")->ne[1]; + case PROJECTOR_TYPE_SOPRANO: + return hparams.projection_dim; + case PROJECTOR_TYPE_PARAKEET: + return mm_1_w->ne[1]; + default: + GGML_ABORT("Unknown projector type"); + } +} + +// TODO @ngxson : this is no longer correct with mtmd_batch API +// this was only meant to be used by qwen-vl-based models, to fuse 2 input images into one (qwen-vl video support) +// this logic should be refactored in near future to distinctly handle "merge frames" and "batching" +int mtmd_model::n_temporal_merge() const { + switch (proj_type) { + case PROJECTOR_TYPE_QWEN2VL: + case PROJECTOR_TYPE_QWEN25VL: + case PROJECTOR_TYPE_QWEN3VL: + case PROJECTOR_TYPE_LFM3VL: + case PROJECTOR_TYPE_NEMOTRON_OMNI: + case PROJECTOR_TYPE_LING3VL: + return 2; + default: + return 1; + } +} diff --git a/tools/mtmd/clip-model.h b/tools/mtmd/mtmd-model.h similarity index 72% rename from tools/mtmd/clip-model.h rename to tools/mtmd/mtmd-model.h index 2810a5cc665d..0d975ba36147 100644 --- a/tools/mtmd/clip-model.h +++ b/tools/mtmd/mtmd-model.h @@ -1,16 +1,36 @@ #pragma once #include "ggml.h" -#include "clip.h" -#include "clip-impl.h" +#include "ggml-backend.h" +#include "mtmd-impl.h" #include #include +#include #include #include #include #include +enum mtmd_modality { + MTMD_MODALITY_VISION, + MTMD_MODALITY_AUDIO, + MTMD_MODALITY_GEN_AUDIO, +}; + +// gen-side voice reference (chatterbox): speech tokens, mel-rate features and the speaker vector +struct mtmd_gen_ref { + std::vector tokens; + std::vector feat; + std::vector spk; +}; + +// a predefined voice: backbone rows, plus the gen-side reference when the gen stack needs one +struct mtmd_gen_voice { + std::vector rows; + mtmd_gen_ref ref; +}; + enum ffn_op_type { FFN_GELU, FFN_GELU_ERF, @@ -49,7 +69,7 @@ enum pad_style { PAD_NEAREST, }; -struct clip_hparams { +struct mtmd_hparams { int32_t image_size = 0; int32_t patch_size = 0; int32_t n_embd = 0; @@ -73,7 +93,7 @@ struct clip_hparams { std::array image_pad_color = {0, 0, 0}; // (preprocessor) for llava-uhd style models - std::vector image_res_candidates; + std::vector image_res_candidates; int32_t preproc_min_tiles = 0; int32_t preproc_max_tiles = 0; int32_t max_slice_nums = 9; // llava-uhd slice cap; per-model, carried in the GGUF @@ -165,6 +185,12 @@ struct clip_hparams { std::vector mel_filters; std::vector window; + // nemotron transcribe + int32_t audio_rope_dims = 0; + int32_t spk_n_embd = 0; + int32_t spk_n_layer = 0; + int32_t spk_n_head = 0; + // mimo-audio-tokenizer: residual vector quantizer int32_t rvq_num_quantizers = 0; std::vector rvq_codebook_size; // per-quantizer bin count (ragged, e.g. 1024/1024/256/128x17) @@ -270,7 +296,7 @@ struct clip_hparams { } }; -struct clip_layer { +struct mtmd_layer { // layernorm 1 (or layer input norm, or pre-attention norm) ggml_tensor * ln_1_w = nullptr; ggml_tensor * ln_1_b = nullptr; @@ -292,6 +318,8 @@ struct clip_layer { ggml_tensor * k_norm = nullptr; ggml_tensor * q_norm = nullptr; + ggml_tensor * k_norm_b = nullptr; + ggml_tensor * q_norm_b = nullptr; // lfm3: per-head sigmoid gate on the attention output ggml_tensor * attn_gate_w = nullptr; @@ -475,13 +503,13 @@ struct qf_block { ggml_tensor * qf_proj_post_norm_w = nullptr; ggml_tensor * qf_proj_post_norm_b = nullptr; ggml_tensor * qf_proj_img_pos = nullptr; // Vision only - std::vector qf_proj_layers; + std::vector qf_proj_layers; }; // pocket-tts SEANet stack, used in both directions: // encoder = conv_in -> per stage (residual unit, strided conv) -> conv_out // decoder = conv_in -> per stage (strided convtr, residual unit) -> conv_out -struct clip_seanet { +struct mtmd_seanet { // one residual unit: ELU -> dilated conv -> ELU -> pointwise conv, added to the input struct stage { ggml_tensor * res_conv1_w = nullptr; @@ -500,7 +528,7 @@ struct clip_seanet { }; // pocket-tts flow-matching decoder (SimpleMLPAdaLN) -struct clip_flow_net { +struct mtmd_flow_net { // AdaLN res block: in_ln -> modulate -> Linear -> SiLU -> Linear, gated residual struct block { ggml_tensor * norm_w = nullptr; @@ -536,7 +564,7 @@ struct clip_flow_net { }; // qwen3tts code2wav: RVQ codes -> raw PCM -struct clip_code2wav { +struct mtmd_code2wav { // "upsample" stage: one ConvNeXt block plus the causal ConvTranspose1d before it struct upsample_block { ggml_tensor * conv_w = nullptr; // causal ConvTranspose1d, 2x @@ -590,7 +618,7 @@ struct clip_code2wav { ggml_tensor * tfm_out_proj_w = nullptr; ggml_tensor * tfm_out_proj_b = nullptr; ggml_tensor * tfm_output_norm_w = nullptr; - std::vector tfm_layers; // reuses the generic block fields (ln_1/attn/ln_2/ffn/ls_1/ls_2) + std::vector tfm_layers; // reuses the generic block fields (ln_1/attn/ln_2/ffn/ls_1/ls_2) std::vector upsample; @@ -603,7 +631,245 @@ struct clip_code2wav { ggml_tensor * dac_post_conv_b = nullptr; }; -struct clip_vocos { +struct mtmd_chatterbox { + // espnet rel-pos conformer layer of the flow encoder + struct enc_layer { + ggml_tensor * norm_mha_w = nullptr; + ggml_tensor * norm_mha_b = nullptr; + ggml_tensor * attn_q_w = nullptr; + ggml_tensor * attn_q_b = nullptr; + ggml_tensor * attn_k_w = nullptr; + ggml_tensor * attn_k_b = nullptr; + ggml_tensor * attn_v_w = nullptr; + ggml_tensor * attn_v_b = nullptr; + ggml_tensor * attn_pos_w = nullptr; // linear_pos, no bias + ggml_tensor * attn_pos_bias_u = nullptr; + ggml_tensor * attn_pos_bias_v = nullptr; + ggml_tensor * attn_out_w = nullptr; + ggml_tensor * attn_out_b = nullptr; + ggml_tensor * norm_ff_w = nullptr; + ggml_tensor * norm_ff_b = nullptr; + ggml_tensor * ffn_1_w = nullptr; + ggml_tensor * ffn_1_b = nullptr; + ggml_tensor * ffn_2_w = nullptr; + ggml_tensor * ffn_2_b = nullptr; + }; + + // causal conv block of the estimator (conv k3 left padded + layer norm) + struct causal_block { + ggml_tensor * conv_w = nullptr; + ggml_tensor * conv_b = nullptr; + ggml_tensor * norm_w = nullptr; + ggml_tensor * norm_b = nullptr; + }; + + // time conditioned resnet of the estimator + struct resnet { + causal_block block1; + causal_block block2; + ggml_tensor * mlp_w = nullptr; // time projection (mlp.1) + ggml_tensor * mlp_b = nullptr; + ggml_tensor * res_conv_w = nullptr; + ggml_tensor * res_conv_b = nullptr; + }; + + // diffusers style transformer block of the estimator + struct tfm_block { + ggml_tensor * norm1_w = nullptr; + ggml_tensor * norm1_b = nullptr; + ggml_tensor * attn_q_w = nullptr; // no bias on qkv + ggml_tensor * attn_k_w = nullptr; + ggml_tensor * attn_v_w = nullptr; + ggml_tensor * attn_out_w = nullptr; // to_out.0 + ggml_tensor * attn_out_b = nullptr; + ggml_tensor * norm3_w = nullptr; + ggml_tensor * norm3_b = nullptr; + ggml_tensor * ff_in_w = nullptr; // ff.net.0.proj + ggml_tensor * ff_in_b = nullptr; + ggml_tensor * ff_out_w = nullptr; // ff.net.2 + ggml_tensor * ff_out_b = nullptr; + }; + + // one estimator stage: resnet, transformer stack, boundary causal conv (null on mid stages) + struct est_stage { + resnet res; + std::vector tfm; + ggml_tensor * conv_w = nullptr; + ggml_tensor * conv_b = nullptr; + }; + + // snake resblock unit of the hift vocoder + struct hift_res_unit { + ggml_tensor * act1_alpha = nullptr; + ggml_tensor * act2_alpha = nullptr; + ggml_tensor * conv1_w = nullptr; + ggml_tensor * conv1_b = nullptr; + ggml_tensor * conv2_w = nullptr; + ggml_tensor * conv2_b = nullptr; + }; + struct hift_res { + std::vector units; // dilations 1/3/5 on conv1 + }; + + // one hift upsample stage: conv transpose, source injection, 3 resblocks + struct hift_up { + ggml_tensor * up_w = nullptr; + ggml_tensor * up_b = nullptr; + ggml_tensor * source_down_w = nullptr; + ggml_tensor * source_down_b = nullptr; + hift_res source_res; + hift_res res_0; + hift_res res_1; + hift_res res_2; + }; + + // f0 predictor conv of the hift vocoder + struct f0_conv { + ggml_tensor * w = nullptr; + ggml_tensor * b = nullptr; + }; + + // s3 tokenizer attention block (neox rope on q/k, fsmn memory on v) + struct s3tok_block { + ggml_tensor * attn_ln_w = nullptr; + ggml_tensor * attn_ln_b = nullptr; + ggml_tensor * attn_q_w = nullptr; + ggml_tensor * attn_q_b = nullptr; + ggml_tensor * attn_k_w = nullptr; // no bias + ggml_tensor * attn_v_w = nullptr; + ggml_tensor * attn_v_b = nullptr; + ggml_tensor * fsmn_w = nullptr; + ggml_tensor * attn_out_w = nullptr; + ggml_tensor * attn_out_b = nullptr; + ggml_tensor * mlp_ln_w = nullptr; + ggml_tensor * mlp_ln_b = nullptr; + ggml_tensor * mlp_in_w = nullptr; + ggml_tensor * mlp_in_b = nullptr; + ggml_tensor * mlp_out_w = nullptr; + ggml_tensor * mlp_out_b = nullptr; + }; + + // folded batchnorm (w/b stay null on the affine free variant) + struct bn { + ggml_tensor * mean = nullptr; + ggml_tensor * var = nullptr; + ggml_tensor * w = nullptr; + ggml_tensor * b = nullptr; + }; + + // fcm residual 2d block of the speaker encoder, the shortcut is null on stride 1 blocks + struct spk_res2d { + ggml_tensor * conv1_w = nullptr; + bn bn1; + ggml_tensor * conv2_w = nullptr; + bn bn2; + ggml_tensor * shortcut_w = nullptr; + bn shortcut_bn; + }; + + // cam dense tdnn layer of the speaker encoder + struct spk_cam_layer { + bn nl1_bn; + ggml_tensor * linear1_w = nullptr; + bn nl2_bn; + ggml_tensor * local_w = nullptr; // cam_layer.linear_local + ggml_tensor * ctx1_w = nullptr; // cam_layer.linear1 + ggml_tensor * ctx1_b = nullptr; + ggml_tensor * ctx2_w = nullptr; // cam_layer.linear2 + ggml_tensor * ctx2_b = nullptr; + }; + struct spk_cam_block { + std::vector layers; + bn transit_bn; + ggml_tensor * transit_w = nullptr; + }; + + // flow encoder + ggml_tensor * input_embedding_w = nullptr; // flow.input_embedding + ggml_tensor * embed_linear_w = nullptr; // fenc.embed.out.0 + ggml_tensor * embed_linear_b = nullptr; + ggml_tensor * embed_norm_w = nullptr; // fenc.embed.out.1 + ggml_tensor * embed_norm_b = nullptr; + ggml_tensor * pre_conv1_w = nullptr; // pre_lookahead_layer + ggml_tensor * pre_conv1_b = nullptr; + ggml_tensor * pre_conv2_w = nullptr; + ggml_tensor * pre_conv2_b = nullptr; + std::vector enc; + ggml_tensor * up_conv_w = nullptr; // fenc.up_layer.conv + ggml_tensor * up_conv_b = nullptr; + ggml_tensor * up_embed_linear_w = nullptr; // fenc.up_embed.out.0 + ggml_tensor * up_embed_linear_b = nullptr; + ggml_tensor * up_embed_norm_w = nullptr; // fenc.up_embed.out.1 + ggml_tensor * up_embed_norm_b = nullptr; + std::vector up_enc; + ggml_tensor * after_norm_w = nullptr; + ggml_tensor * after_norm_b = nullptr; + ggml_tensor * encoder_proj_w = nullptr; // flow.encoder_proj + ggml_tensor * encoder_proj_b = nullptr; + + // cfm estimator + ggml_tensor * time_mlp_1_w = nullptr; + ggml_tensor * time_mlp_1_b = nullptr; + ggml_tensor * time_mlp_2_w = nullptr; + ggml_tensor * time_mlp_2_b = nullptr; + ggml_tensor * time_embed_mixer_w = nullptr; // meanflow variant only + est_stage est_down; + std::vector est_mid; + est_stage est_up; + causal_block est_final_block; + ggml_tensor * est_final_proj_w = nullptr; + ggml_tensor * est_final_proj_b = nullptr; + + // hift vocoder + ggml_tensor * hift_pre_w = nullptr; + ggml_tensor * hift_pre_b = nullptr; + std::vector hift_ups; + ggml_tensor * hift_post_w = nullptr; + ggml_tensor * hift_post_b = nullptr; + std::vector f0_condnet; + ggml_tensor * f0_classifier_w = nullptr; + ggml_tensor * f0_classifier_b = nullptr; + + // s3 tokenizer + ggml_tensor * s3tok_conv1_w = nullptr; + ggml_tensor * s3tok_conv1_b = nullptr; + ggml_tensor * s3tok_conv2_w = nullptr; + ggml_tensor * s3tok_conv2_b = nullptr; + std::vector s3tok_blocks; + ggml_tensor * s3tok_down_w = nullptr; // quantizer._codebook.project_down + ggml_tensor * s3tok_down_b = nullptr; + + // CAMPPlus speaker encoder + ggml_tensor * spk_conv1_w = nullptr; + bn spk_bn1; + spk_res2d spk_layer1_0; + spk_res2d spk_layer1_1; + spk_res2d spk_layer2_0; + spk_res2d spk_layer2_1; + ggml_tensor * spk_conv2_w = nullptr; + bn spk_bn2; + ggml_tensor * spk_tdnn_w = nullptr; + bn spk_tdnn_bn; + spk_cam_block spk_block1; + spk_cam_block spk_block2; + spk_cam_block spk_block3; + bn spk_out_bn; + ggml_tensor * spk_dense_w = nullptr; + bn spk_dense_bn; + ggml_tensor * spk_affine_w = nullptr; + ggml_tensor * spk_affine_b = nullptr; +}; + +// drama3 tower: extra tensors around the generic block +struct mtmd_drama3_layer { + ggml_tensor * enorm_w = nullptr; + ggml_tensor * hnorm_w = nullptr; + ggml_tensor * eh_proj_w = nullptr; // [code embd; hidden] -> hidden + ggml_tensor * embd_w = nullptr; + ggml_tensor * out_norm_w = nullptr; +}; + +struct mtmd_vocos { ggml_tensor * input_w = nullptr; ggml_tensor * input_b = nullptr; ggml_tensor * norm_w = nullptr; @@ -612,13 +878,23 @@ struct clip_vocos { ggml_tensor * output_norm_b = nullptr; ggml_tensor * output_w = nullptr; ggml_tensor * output_b = nullptr; - std::vector blocks; + std::vector blocks; }; -struct clip_model { - clip_modality modality = CLIP_MODALITY_VISION; +// the weights and hparams of one modality of an mmproj file, read-only once loaded +struct mtmd_model { + mtmd_modality modality = MTMD_MODALITY_VISION; projector_type proj_type = PROJECTOR_TYPE_MLP; - clip_hparams hparams; + mtmd_hparams hparams; + + // runtime requirements declared by the model, applied by mtmd_subcontext + int max_nodes = 8192; + bool flash_attn_auto = true; // false: MTMD_FLASH_ATTN_TYPE_AUTO resolves to disabled + + // weights + ggml_context_ptr ctx_data; + ggml_backend_buffer_ptr buf; + std::map mem_weights; // embeddings ggml_tensor * class_embedding = nullptr; @@ -626,6 +902,7 @@ struct clip_model { ggml_tensor * patch_embeddings_1 = nullptr; // additional patch kernel; interpretation is model-specific (Qwen2VL, Nemotron Omni) ggml_tensor * patch_bias = nullptr; ggml_tensor * position_embeddings = nullptr; + ggml_tensor * rope_freqs = nullptr; ggml_tensor * norm_embd_w = nullptr; ggml_tensor * norm_embd_b = nullptr; @@ -640,7 +917,7 @@ struct clip_model { ggml_tensor * pre_ln_w = nullptr; ggml_tensor * pre_ln_b = nullptr; - std::vector layers; + std::vector layers; // one patch embedder per audio pyramid stage (muse-spark) struct audio_patch { @@ -650,7 +927,7 @@ struct clip_model { }; std::vector a_patches; - int32_t n_deepstack_layers = 0; // used by Qwen3-VL, calculated from clip_layer + int32_t n_deepstack_layers = 0; // used by Qwen3-VL, calculated from mtmd_layer ggml_tensor * post_ln_w; ggml_tensor * post_ln_b; @@ -837,7 +1114,7 @@ struct clip_model { // mimo-v2.5: LLM-side connector (input_local_transformer, separate from the // audio_tokenizer's own encoder `layers`) - std::vector mm_a_local_layers; + std::vector mm_a_local_layers; ggml_tensor * mm_a_local_norm_w = nullptr; // qwen3a @@ -868,17 +1145,18 @@ struct clip_model { ggml_tensor * gen_code_mask_embd = nullptr; // qwen3tts code2wav: RVQ codes -> raw PCM - clip_code2wav c2w; + mtmd_code2wav c2w; + std::vector drama3_layers; // pocket-tts: SEANet stack, shared by the encoder (speaker path) and the decoder (gen path) - clip_seanet seanet; + mtmd_seanet seanet; // pocket-tts: voice latent -> backbone embd (speaker path) ggml_tensor * spk_proj_w = nullptr; ggml_tensor * downsample_w = nullptr; // pocket-tts: flow-matching decoder, backbone hidden state -> next latent - clip_flow_net flow; + mtmd_flow_net flow; ggml_tensor * gen_out_eos_w = nullptr; ggml_tensor * gen_out_eos_b = nullptr; ggml_tensor * gen_input_lin_w = nullptr; // latent -> backbone embd @@ -886,13 +1164,18 @@ struct clip_model { ggml_tensor * gen_emb_std = nullptr; ggml_tensor * gen_quant_out_w = nullptr; // latent -> decoder dim ggml_tensor * gen_upsample_w = nullptr; // depthwise convtr, frame rate -> encoder frame rate - std::vector gen_tfm_layers; // mimi decoder_transformer + std::vector gen_tfm_layers; // mimi decoder_transformer + // chatterbox flow encoder, cfm estimator, hift vocoder, s3 tokenizer and CAMPPlus speaker encoder + mtmd_chatterbox cbx; + + // chatterbox tensors read on the host, see cbx_read_tensor() + std::map cbx_tensors; // predefined voices, no rows if the voice is named in the prompt (lfm3a) - std::map> gen_voices; + std::map gen_voices; // soprano: Vocos decoder, hidden states -> STFT coefficients - clip_vocos vocos; + mtmd_vocos vocos; // cogvlm ggml_tensor * mm_post_fc_norm_w = nullptr; @@ -924,7 +1207,29 @@ struct clip_model { int32_t n_sam_layers = 12; // used by deepseek-ocr sam encoder - std::vector sam_layers; + std::vector sam_layers; + + // nemotron transcribe: speaker expert, Sortformer head, fusion + std::vector spk_layers; + ggml_tensor * spk_inp_proj_w = nullptr; + ggml_tensor * spk_pre_ln_w = nullptr; + ggml_tensor * spk_pre_ln_b = nullptr; + ggml_tensor * spk_post_ln_w = nullptr; + ggml_tensor * spk_post_ln_b = nullptr; + ggml_tensor * spk_head_proj_w = nullptr; + ggml_tensor * spk_head_proj_b = nullptr; + ggml_tensor * spk_head_up_w = nullptr; + ggml_tensor * spk_head_up_b = nullptr; + ggml_tensor * spk_head_fc1_w = nullptr; + ggml_tensor * spk_head_fc1_b = nullptr; + ggml_tensor * spk_head_fc2_w = nullptr; + ggml_tensor * spk_head_fc2_b = nullptr; + ggml_tensor * fusion_asr_norm_w = nullptr; + ggml_tensor * fusion_asr_norm_b = nullptr; + ggml_tensor * fusion_spk_norm_w = nullptr; + ggml_tensor * fusion_spk_norm_b = nullptr; + ggml_tensor * fusion_spk_kernel = nullptr; + std::vector spk_sil_emb; // deepseek-ocr-2 ggml_tensor * resample_query_768 = nullptr; @@ -978,6 +1283,51 @@ struct clip_model { || proj_type == PROJECTOR_TYPE_VOXTRAL || proj_type == PROJECTOR_TYPE_MERALION; } + + // this should be equal to the embedding dimension of the text model + int n_mmproj_embd() const; + + int n_output_tokens(const mtmd_image_f32 * img) const; + + // for M-RoPE, this will be the number of token positions in X and Y directions + // for other models, X will be the total number of tokens and Y will be 1 + int n_output_tokens_x(const mtmd_image_f32 * img) const; + int n_output_tokens_y(const mtmd_image_f32 * img) const; + + // max number of output tokens per image, -1 if not dynamic size + int image_max_tokens() const; + + // TODO @ngxson : this is no longer correct with mtmd_batch API + int n_temporal_merge() const; + + // read a chatterbox tensor by name as F32, returns the element count (0 if not found), out may be null to query the size + size_t cbx_read_tensor(const char * name, float * out, size_t n_max) const; +}; + +struct mtmd_model_params { + // custom value provided by user, -1 if not set + int32_t image_min_tokens = -1; + int32_t image_max_tokens = -1; + bool no_alloc = false; // only create the tensors, do not read their data + mtmd_progress_callback progress_callback = nullptr; + void * progress_callback_user_data = nullptr; +}; + +// the modalities of an mmproj file, a missing one is nullptr +struct mtmd_models { + std::unique_ptr v; + std::unique_ptr a; + std::unique_ptr gen_a; +}; + +// the weights are allocated in buft, throws on error +mtmd_models mtmd_model_load(const char * fname, const mtmd_model_params & params, ggml_backend_buffer_type_t buft); + +struct mtmd_model_caps { + bool has_vision; + bool has_audio; + bool has_gen_audio; }; -const clip_hparams * clip_get_hparams(const struct clip_ctx * ctx); +// read the modalities of an mmproj file without loading its tensors +mtmd_model_caps mtmd_model_get_caps(const char * fname); diff --git a/tools/mtmd/mtmd-subcontext.cpp b/tools/mtmd/mtmd-subcontext.cpp new file mode 100644 index 000000000000..7f13b7f62948 --- /dev/null +++ b/tools/mtmd/mtmd-subcontext.cpp @@ -0,0 +1,2673 @@ +#include "mtmd-subcontext.h" +#include "mtmd-graph.h" +#include "models/models.h" +#include "mtmd-audio.h" + +#include "ggml.h" +#include "ggml-cpp.h" +#include "ggml-alloc.h" +#include "ggml-backend.h" +#include "ggml-cpu.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +// ggml_backend_cpu_set_n_threads(backend_cpu, n_threads); +static void mtmd_backend_cpu_set_n_threads(ggml_backend_t backend_cpu, int n_threads) { + ggml_backend_dev_t dev = ggml_backend_get_device(backend_cpu); + ggml_backend_reg_t reg = dev ? ggml_backend_dev_backend_reg(dev) : nullptr; + if (reg) { + auto ggml_backend_set_n_threads_fn = (ggml_backend_set_n_threads_t) ggml_backend_reg_get_proc_address(reg, "ggml_backend_set_n_threads"); + if (ggml_backend_set_n_threads_fn) { + ggml_backend_set_n_threads_fn(backend_cpu, n_threads); + } + } +} + +static std::shared_ptr mtmd_threadpool_init(int n_threads) { + ggml_backend_dev_t dev = ggml_backend_dev_by_type(GGML_BACKEND_DEVICE_TYPE_CPU); + ggml_backend_reg_t reg = dev ? ggml_backend_dev_backend_reg(dev) : nullptr; + if (!reg || n_threads < 2) { + return nullptr; + } + auto * new_fn = (decltype(ggml_threadpool_new) *) ggml_backend_reg_get_proc_address(reg, "ggml_threadpool_new"); + auto * free_fn = (decltype(ggml_threadpool_free) *) ggml_backend_reg_get_proc_address(reg, "ggml_threadpool_free"); + if (!new_fn || !free_fn) { + return nullptr; + } + ggml_threadpool_params tpp = ggml_threadpool_params_default(n_threads); + tpp.poll = 0; // the pool is idle most of the time, don't take CPU away from the text model + ggml_threadpool_t threadpool = new_fn(&tpp); + if (!threadpool) { + throw std::runtime_error("failed to create threadpool"); + } + return std::shared_ptr(threadpool, free_fn); +} + +ggml_backend_dev_t mtmd_select_device(bool use_gpu, ggml_backend_dev_t device) { + if (!use_gpu) { + return nullptr; + } + if (device != nullptr) { + return device; + } + ggml_backend_dev_t dev = ggml_backend_dev_by_type(GGML_BACKEND_DEVICE_TYPE_GPU); + return dev ? dev : ggml_backend_dev_by_type(GGML_BACKEND_DEVICE_TYPE_IGPU); +} + +mtmd_backends::mtmd_backends(ggml_backend_dev_t device, int n_threads, int max_nodes, + ggml_backend_sched_eval_callback cb_eval, void * cb_eval_user_data) { + threadpool = mtmd_threadpool_init(n_threads); + + backend_cpu = ggml_backend_init_by_type(GGML_BACKEND_DEVICE_TYPE_CPU, nullptr); + if (!backend_cpu) { + throw std::runtime_error("failed to initialize CPU backend"); + } + if (threadpool) { + ggml_backend_reg_t reg = ggml_backend_dev_backend_reg(ggml_backend_get_device(backend_cpu)); + auto * set_threadpool_fn = (decltype(ggml_backend_cpu_set_threadpool) *) ggml_backend_reg_get_proc_address(reg, "ggml_backend_cpu_set_threadpool"); + if (set_threadpool_fn) { + set_threadpool_fn(backend_cpu, threadpool.get()); + } + } + if (device != nullptr) { + backend = ggml_backend_dev_init(device, nullptr); + if (!backend) { + ggml_backend_free(backend_cpu); + throw std::runtime_error(mtmd_format("%s: failed to initialize \"%s\" backend\n", + __func__, ggml_backend_dev_name(device))); + } + } + + if (backend) { + LOG_INF("%s: using %s backend\n", __func__, ggml_backend_name(backend)); + ptrs.push_back(backend); + bufts.push_back(ggml_backend_get_default_buffer_type(backend)); + } else { + backend = backend_cpu; + LOG_INF("%s: using CPU backend\n", __func__); + } + + ptrs.push_back(backend_cpu); + bufts.push_back(ggml_backend_get_default_buffer_type(backend_cpu)); + + sched.reset(ggml_backend_sched_new(ptrs.data(), bufts.data(), ptrs.size(), max_nodes, false, true)); + if (cb_eval != nullptr) { + ggml_backend_sched_set_eval_callback(sched.get(), cb_eval, cb_eval_user_data); + } +} + +mtmd_backends::~mtmd_backends() { + // before the backends it uses + sched.reset(); + ggml_backend_free(backend); + if (backend != backend_cpu) { + ggml_backend_free(backend_cpu); + } +} + +std::map mtmd_backends::get_mem_compute() const { + std::map res; + for (ggml_backend_t b : ptrs) { + res[ggml_backend_get_device(b)] += ggml_backend_sched_get_buffer_size(sched.get(), b); + } + return res; +} + +mtmd_subcontext::mtmd_subcontext(const mtmd_model & model, mtmd_backends & backends, const mtmd_subctx_params & params) : + model(model), + backends(backends), + max_nodes(model.max_nodes), + flash_attn_type(params.flash_attn_type) { + if (flash_attn_type == MTMD_FLASH_ATTN_TYPE_AUTO && !model.flash_attn_auto) { + flash_attn_type = MTMD_FLASH_ATTN_TYPE_DISABLED; + } + + debug_output_embeddings = std::getenv("MTMD_DEBUG_EMBEDDINGS") != nullptr; + + buf_compute_meta.resize(max_nodes * ggml_tensor_overhead() + ggml_graph_overhead_custom(max_nodes, false)); + + auto batch = get_dummy_batch(); + if (model.modality != MTMD_MODALITY_GEN_AUDIO) { + // check batching support + support_batch = mtmd_get_graph_builder(this, batch)->support_batch(); + } + if (params.warmup) { + warmup(batch); + } +} + +mtmd_image_f32_batch mtmd_subcontext::get_dummy_batch() const { + // create a fake batch + const auto & hparams = model.hparams; + mtmd_image_f32_batch batch; + mtmd_image_f32 img; + if (model.modality == MTMD_MODALITY_VISION) { + const int sz = hparams.warmup_image_size; + img.set_size({sz, sz}, false, false); + LOG_INF("%s: warmup with image size = %d x %d\n", __func__, sz, sz); + } else if (model.modality == MTMD_MODALITY_GEN_AUDIO) { + // TODO: improve this, move to per-model impl + const int n_embd = model.n_mmproj_embd(); + img.set_size({n_embd, 1}, false, true); + LOG_INF("%s: warmup with one gen-audio step\n", __func__); + } else { + // GEMMA4UA uses n_mel_bins as a raw-waveform frame size (640), not a mel-bin count, + // so the [1, 256] bound only applies to FFT-based models. + const bool fft_based = model.proj_type != PROJECTOR_TYPE_GEMMA4UA; + if (hparams.n_mel_bins <= 0 || (fft_based && hparams.n_mel_bins > 256)) { + throw std::runtime_error(mtmd_format("%s: invalid n_mel_bins (%d), must be in [1, 256]\n", __func__, hparams.n_mel_bins)); + } + img.set_size({hparams.warmup_audio_size, hparams.n_mel_bins}, false, false); + LOG_INF("%s: warmup with audio size = %d\n", __func__, hparams.warmup_audio_size); + } + batch.entries.push_back(img); + return batch; +} + +void mtmd_subcontext::warmup(const mtmd_image_f32_batch & batch) { + support_info_graph info; + + if (flash_attn_type == MTMD_FLASH_ATTN_TYPE_AUTO) { + // try to enable flash attention to see if it's supported + flash_attn_type = MTMD_FLASH_ATTN_TYPE_ENABLED; + info = reserve_compute_meta(batch); + if (!info.fattn && info.fattn_op) { + auto op = info.fattn_op; + LOG_WRN("%s: *****************************************************************\n", __func__); + LOG_WRN("%s: WARNING: flash attention not supported by %s, memory usage will increase\n", __func__, ggml_backend_name(backends.backend)); + LOG_WRN("%s: op params: \n", __func__); + static auto print_shape = [](const char * fn, const char * name, ggml_tensor * t) { + LOG_WRN("%s: %s: type = %s, ne = [%d %d %d %d], nb = [%d %d %d %d]\n", fn, + name, ggml_type_name(t->type), + t->ne[0], t->ne[1], t->ne[2], t->ne[3], + t->nb[0], t->nb[1], t->nb[2], t->nb[3]); + }; + print_shape(__func__, " dst", op); + print_shape(__func__, "src0", op->src[0]); + print_shape(__func__, "src1", op->src[1]); + print_shape(__func__, "src2", op->src[2]); + LOG_WRN("%s: please report this on github as an issue\n", __func__); + LOG_WRN("%s: *****************************************************************\n", __func__); + flash_attn_type = MTMD_FLASH_ATTN_TYPE_DISABLED; + reserve_compute_meta(batch); + } + } else { + info = reserve_compute_meta(batch); + if (!info.fattn && flash_attn_type == MTMD_FLASH_ATTN_TYPE_ENABLED) { + LOG_WRN("%s: flash attention is not supported by the current backend; falling back to CPU (performance will be degraded)\n", __func__); + } + } + + is_allocated = true; // mark buffers as allocated + + LOG_INF("%s: flash attention is %s\n", __func__, + (flash_attn_type == MTMD_FLASH_ATTN_TYPE_ENABLED) ? "enabled" : "disabled"); + + // print ops that are not supported by the GPU backend (if there is one) + if (backends.backend != backends.backend_cpu) { + std::vector unsupported_ops; + for (const auto & op : info.ops) { + if (!op.is_accel) { + unsupported_ops.push_back(op); + } + } + if (!unsupported_ops.empty()) { + LOG_WRN("%s: *****************************************************************\n", __func__); + LOG_WRN("%s: WARNING: the mmproj graph uses unsupported operators by the backend\n", __func__); + LOG_WRN("%s: the performance will be suboptimal \n", __func__); + LOG_WRN("%s: list of unsupported ops (backend=%s):\n", __func__, ggml_backend_name(backends.backend)); + for (const auto & op : unsupported_ops) { + LOG_WRN("%s: %16s: type = %s, ne = [%d %d %d %d]\n", __func__, + ggml_op_name(op.op->op), + ggml_type_name(op.op->type), + op.op->ne[0], op.op->ne[1], op.op->ne[2], op.op->ne[3]); + } + LOG_WRN("%s: flash attention is %s\n", __func__, + (flash_attn_type == MTMD_FLASH_ATTN_TYPE_ENABLED) ? "enabled" : "disabled"); + LOG_WRN("%s: please report this on github as an issue\n", __func__); + LOG_WRN("%s: ref: https://github.com/ggml-org/llama.cpp/pull/16837#issuecomment-3461676118\n", __func__); + LOG_WRN("%s: *****************************************************************\n", __func__); + } + } +} + +mtmd_subcontext::support_info_graph mtmd_subcontext::reserve_compute_meta(const mtmd_image_f32_batch & batch) { + ggml_cgraph * gf = mtmd_get_graph_builder(this, batch)->build(); + ggml_backend_sched_reserve(backends.sched.get(), gf); + + const int n_splits = ggml_backend_sched_get_n_splits(backends.sched.get()); + const int n_nodes = ggml_graph_n_nodes(gf); + + LOG_INF("%s: graph splits = %d, nodes = %d\n", __func__, n_splits, n_nodes); + + support_info_graph res { + /*.fattn = */ true, + /*.fattn_op = */ nullptr, + /*.ops = */ {}, + }; + + // check op support + for (int i = 0; i < ggml_graph_n_nodes(gf); i++) { + ggml_tensor * node = ggml_graph_node(gf, i); + res.ops.push_back({node, true}); + if (!ggml_backend_supports_op(backends.backend, node)) { + res.ops.back().is_accel = false; + if (node->op == GGML_OP_FLASH_ATTN_EXT) { + res.fattn = false; + res.fattn_op = node; + } + } + } + + return res; +} + +bool mtmd_audio_log_mel(const mtmd_audio_mel_config & config, const float * samples, const float * window, int64_t n_frames, float * out) { + ggml_backend_sched_t sched = config.sched; + ggml_backend_t backend_cpu = ggml_backend_sched_get_backend(sched, ggml_backend_sched_get_n_backends(sched) - 1); + mtmd_backend_cpu_set_n_threads(backend_cpu, std::max(1, config.n_threads)); + + const auto & cache = *config.cache; + const auto & filters = cache.filters.data; + const int64_t n_fft_bins = config.n_fft_bins; + const int64_t frame_size = (n_fft_bins - 1) * 2; + const int64_t n_mel = config.n_mel; + GGML_ASSERT(filters.size() == (size_t) (n_mel * n_fft_bins)); + GGML_ASSERT(cache.cos_vals.size() == (size_t) frame_size && cache.sin_vals.size() == (size_t) frame_size); + + // DFT matrix with the window folded in, the first n_fft_bins rows give the real part + std::vector dft((size_t) 2 * n_fft_bins * frame_size); + for (int64_t k = 0; k < n_fft_bins; k++) { + for (int64_t j = 0; j < frame_size; j++) { + const int64_t q = (j * k) % frame_size; + dft[k * frame_size + j] = window[j] * cache.cos_vals[q]; + dft[(k + n_fft_bins) * frame_size + j] = window[j] * cache.sin_vals[q]; + } + } + + // the spectrum is several times larger than the result, so long audio is processed in blocks + const int64_t n_frames_max = 8192; + + for (int64_t off = 0; off < n_frames; off += n_frames_max) { + const int64_t n_cur = std::min(n_frames_max, n_frames - off); + + ggml_init_params params = { + /*.mem_size =*/ ggml_tensor_overhead() * 32 + ggml_graph_overhead(), + /*.mem_buffer =*/ nullptr, + /*.no_alloc =*/ true, + }; + ggml_context_ptr ctx_ptr(ggml_init(params)); + ggml_context * ctx0 = ctx_ptr.get(); + + ggml_tensor * inp_samples = ggml_new_tensor_1d(ctx0, GGML_TYPE_F32, (n_cur - 1) * config.hop_length + frame_size); + ggml_tensor * inp_dft = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, frame_size, 2 * n_fft_bins); + ggml_tensor * inp_filters = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_fft_bins, n_mel); + ggml_set_input(inp_samples); + ggml_set_input(inp_dft); + ggml_set_input(inp_filters); + + ggml_tensor * cur = ggml_cont(ctx0, ggml_view_2d(ctx0, inp_samples, frame_size, n_cur, config.hop_length * ggml_element_size(inp_samples), 0)); + cur = ggml_mul_mat(ctx0, inp_dft, cur); // [2 * n_fft_bins, n_cur] + ggml_prec_set_src(cur, GGML_PREC_F32, 1); + + cur = ggml_sqr(ctx0, cur); + cur = ggml_add(ctx0, + ggml_view_2d(ctx0, cur, n_fft_bins, n_cur, cur->nb[1], 0), + ggml_view_2d(ctx0, cur, n_fft_bins, n_cur, cur->nb[1], n_fft_bins * ggml_element_size(cur))); + if (config.use_magnitude) { + cur = ggml_sqrt(ctx0, cur); + } + + cur = ggml_mul_mat(ctx0, cur, inp_filters); // [n_cur, n_mel] + ggml_prec_set_src(cur, GGML_PREC_F32, 1); + cur = config.mel_floor_add + ? ggml_scale_bias(ctx0, cur, 1.0f, config.mel_floor) + : ggml_clamp(ctx0, cur, config.mel_floor, FLT_MAX); + cur = ggml_log(ctx0, cur); + if (!config.use_natural_log) { + cur = ggml_scale(ctx0, cur, 1.0f / logf(10.0f)); + } + ggml_set_output(cur); + + ggml_cgraph * gf = ggml_new_graph(ctx0); + ggml_build_forward_expand(gf, cur); + + ggml_backend_sched_reset(sched); + if (!ggml_backend_sched_alloc_graph(sched, gf)) { + LOG_ERR("%s: failed to allocate compute graph\n", __func__); + return false; + } + + ggml_backend_tensor_set(inp_samples, samples + off * config.hop_length, 0, ggml_nbytes(inp_samples)); + ggml_backend_tensor_set(inp_dft, dft.data(), 0, ggml_nbytes(inp_dft)); + ggml_backend_tensor_set(inp_filters, filters.data(), 0, ggml_nbytes(inp_filters)); + + auto status = ggml_backend_sched_graph_compute(sched, gf); + if (status != GGML_STATUS_SUCCESS) { + LOG_ERR("%s: ggml_backend_sched_graph_compute failed with error %d\n", __func__, status); + return false; + } + + for (int64_t j = 0; j < n_mel; j++) { + ggml_backend_tensor_get(cur, out + j * n_frames + off, j * cur->nb[1], n_cur * ggml_element_size(cur)); + } + } + + return true; +} + +bool mtmd_subcontext::encode_image(int n_threads, const mtmd_image_f32 * img, std::vector & out_vec) { + mtmd_image_f32_batch imgs; + mtmd_image_f32 img_copy = *img; + imgs.entries.push_back(std::move(img_copy)); + + return encode_batch(n_threads, &imgs, out_vec); +} + +bool mtmd_subcontext::encode_batch(int n_threads, const mtmd_image_f32_batch * imgs_c_ptr, std::vector & out_batch_embd) { + mtmd_subctx_encode_params params; + params.imgs = imgs_c_ptr; + params.n_threads = n_threads; + params.out_embd = &out_batch_embd; + + if (model.proj_type == PROJECTOR_TYPE_NEMOTRON_TRANSCRIBE) { + return mtmd_graph_nemotron_transcribe::encode_windows(this, model, ¶ms); + } + return encode(¶ms); +} + +// persisted state slots of the gen-audio decoders and the streaming audio encoders +static std::vector list_gen_state_slots(const mtmd_hparams & hparams, const mtmd_model & model) { + switch (model.proj_type) { + case PROJECTOR_TYPE_QWEN3TTS_GEN: return list_c2w_state_slots(hparams, model); + case PROJECTOR_TYPE_DRAMA3_GEN: return list_drama3_state_slots(hparams, model); + case PROJECTOR_TYPE_POCKETTTS_GEN: return list_pockettts_state_slots(hparams, model); + case PROJECTOR_TYPE_LFM3A_GEN: return list_lfm3a_gen_state_slots(hparams, model); + default: return {}; + } +} + +// state slots of the streaming audio encoders +static std::vector list_enc_state_slots(const mtmd_hparams & hparams, const mtmd_model & model) { + switch (model.proj_type) { + case PROJECTOR_TYPE_LFM3A: return list_lfm3a_state_slots(hparams, model); + case PROJECTOR_TYPE_GEMMA4A: return list_gemma4a_state_slots(hparams, model); + default: return {}; + } +} + +// state_out can have another shape than its slot, only the byte size must match +static void copy_gen_state_out(ggml_tensor * src, ggml_tensor * dst) { + GGML_ASSERT(ggml_nbytes(src) == ggml_nbytes(dst)); + if (src->type == dst->type && ggml_are_same_shape(src, dst) && ggml_is_contiguous(src)) { + ggml_backend_tensor_copy(src, dst); + return; + } + std::vector tmp(ggml_nbytes(src)); + ggml_backend_tensor_get(src, tmp.data(), 0, tmp.size()); + ggml_backend_tensor_set(dst, tmp.data(), 0, tmp.size()); +} + +mtmd_subctx_gen_state * mtmd_subcontext::gen_state_init(uint32_t seed) { + auto st = std::make_unique(); + st->rng.seed(seed == UINT32_MAX ? std::random_device{}() : seed); + + const auto slots = list_gen_state_slots(model.hparams, model); + if (slots.empty()) { + return st.release(); + } + + ggml_init_params params = { + /*.mem_size =*/ ggml_tensor_overhead() * slots.size(), + /*.mem_buffer =*/ nullptr, + /*.no_alloc =*/ true, + }; + st->ctx_data.reset(ggml_init(params)); + for (const auto & slot : slots) { + ggml_tensor * t = ggml_new_tensor_2d(st->ctx_data.get(), GGML_TYPE_F32, slot.ne0, slot.ne1); + ggml_format_name(t, "gen_state_%s", slot.name.c_str()); + st->slots.push_back(t); + } + st->buf.reset(ggml_backend_alloc_ctx_tensors(st->ctx_data.get(), backends.backend)); + if (!st->buf) { + LOG_ERR("%s: failed to allocate the gen state\n", __func__); + return nullptr; + } + // a zero state is the cold start + ggml_backend_buffer_clear(st->buf.get(), 0); + + return st.release(); +} + +mtmd_subctx_enc_state * mtmd_subcontext::enc_state_init(int32_t n_frames_max) { + const auto & hparams = model.hparams; + if (model.proj_type == PROJECTOR_TYPE_LFM3A || model.proj_type == PROJECTOR_TYPE_GEMMA4A) { + // see list_enc_state_slots() + const auto slots = list_enc_state_slots(hparams, model); + auto st = std::make_unique(); + ggml_init_params params = { + /*.mem_size =*/ ggml_tensor_overhead() * slots.size(), + /*.mem_buffer =*/ nullptr, + /*.no_alloc =*/ true, + }; + st->ctx_data.reset(ggml_init(params)); + for (const auto & slot : slots) { + ggml_tensor * t = ggml_new_tensor_2d(st->ctx_data.get(), GGML_TYPE_F32, slot.ne0, slot.ne1); + ggml_format_name(t, "enc_state_%s", slot.name.c_str()); + st->slots.push_back(t); + } + st->buf.reset(ggml_backend_alloc_ctx_tensors(st->ctx_data.get(), backends.backend)); + if (!st->buf) { + LOG_ERR("%s: failed to allocate the encoder state\n", __func__); + return nullptr; + } + // a zero state is the cold start + ggml_backend_buffer_clear(st->buf.get(), 0); + return st.release(); + } + if (model.proj_type != PROJECTOR_TYPE_VOXTRAL_RT || hparams.attn_window_size <= 0) { + LOG_ERR("%s: the encoder does not support streaming input\n", __func__); + return nullptr; + } + if (n_frames_max <= 0) { + LOG_ERR("%s: invalid n_frames_max %d\n", __func__, n_frames_max); + return nullptr; + } + + const int n_layer = hparams.n_layer; + const int d_head = hparams.n_embd_head > 0 ? hparams.n_embd_head : hparams.n_embd / hparams.n_head; + const int n_embd_kv = hparams.n_head_kv * d_head; + + auto st = std::make_unique(); + st->n_ctx = hparams.attn_window_size + n_frames_max; + st->slot_pos.assign(st->n_ctx, -1); + + ggml_init_params params = { + /*.mem_size =*/ ggml_tensor_overhead() * 2 * n_layer, + /*.mem_buffer =*/ nullptr, + /*.no_alloc =*/ true, + }; + st->ctx_data.reset(ggml_init(params)); + for (int il = 0; il < n_layer; il++) { + ggml_tensor * k = ggml_new_tensor_2d(st->ctx_data.get(), GGML_TYPE_F16, n_embd_kv, st->n_ctx); + ggml_tensor * v = ggml_new_tensor_2d(st->ctx_data.get(), GGML_TYPE_F16, n_embd_kv, st->n_ctx); + ggml_format_name(k, "enc_state_k_%d", il); + ggml_format_name(v, "enc_state_v_%d", il); + st->k.push_back(k); + st->v.push_back(v); + } + st->buf.reset(ggml_backend_alloc_ctx_tensors(st->ctx_data.get(), backends.backend)); + if (!st->buf) { + LOG_ERR("%s: failed to allocate the encoder state\n", __func__); + return nullptr; + } + // empty slots are masked out, but they must not hold NaN + ggml_backend_buffer_clear(st->buf.get(), 0); + + LOG_INF("%s: encoder state: %d slots, %.2f MiB\n", __func__, st->n_ctx, ggml_backend_buffer_get_size(st->buf.get()) / 1024.0 / 1024.0); + + return st.release(); +} + +bool mtmd_subcontext::encode(mtmd_subctx_encode_params * params) { + const mtmd_image_f32_batch & imgs = *params->imgs; + int n_batch_cur = imgs.entries.size(); + + // [QWEN_VIDEO] for video models, the batch dimension is used as temporal dimension for merged frames + if (!support_batch && n_batch_cur > model.n_temporal_merge()) { + LOG_ERR("%s: batch size %d exceeds maximum supported batch/temporal-merge size %d\n", __func__, n_batch_cur, model.n_temporal_merge()); + return false; + } + + // if buffers are not allocated, we need to do a warmup run to allocate them + if (!is_allocated) { + warmup(*params->imgs); + } + + const bool needs_gen_state = params->gen_process == MTMD_SUBCTX_GEN_PROCESS_GEN_CODE || + params->gen_process == MTMD_SUBCTX_GEN_PROCESS_GEN_WAV || + params->gen_process == MTMD_SUBCTX_GEN_PROCESS_TTS; + if (needs_gen_state && params->gen_state == nullptr) { + LOG_ERR("%s: audio gen requires a gen state\n", __func__); + return false; + } + if (model.proj_type == PROJECTOR_TYPE_DRAMA3_GEN && params->gen_process == MTMD_SUBCTX_GEN_PROCESS_GEN_CODE) { + float pos = 0.0f; + ggml_backend_tensor_get(params->gen_state->slots[0], &pos, 0, sizeof(float)); + if ((int) pos >= mtmd_graph_drama3_gen::n_pos_max) { + LOG_ERR("%s: the codebook tower holds at most %d rows\n", __func__, mtmd_graph_drama3_gen::n_pos_max); + return false; + } + } + + // build the inference graph + ggml_backend_sched_reset(backends.sched.get()); + ggml_cgraph * gf = mtmd_get_graph_builder(this, imgs, params)->build(); + + // the last node is the embedding tensor, code2wav has no out_embd + // read it before scheduling: backends may reorder the nodes of graphs with several outputs (Metal) + ggml_tensor * embeddings = params->out_embd ? ggml_graph_node(gf, -1) : nullptr; + + if (!ggml_backend_sched_alloc_graph(backends.sched.get(), gf)) { + LOG_ERR("%s: failed to allocate compute graph\n", __func__); + return false; + } + + // set inputs + const auto & hparams = model.hparams; + + const int image_size_width = imgs.entries[0].nx(); + const int image_size_height = imgs.entries[0].ny(); + + const int patch_size = hparams.patch_size; + const int num_patches = ((image_size_width / patch_size) * (image_size_height / patch_size)); + const int n_pos = num_patches + (model.class_embedding ? 1 : 0); + const int pos_w = image_size_width / patch_size; + const int pos_h = image_size_height / patch_size; + + + auto get_inp_tensor = [&gf](const char * name) { + ggml_tensor * inp = ggml_graph_get_tensor(gf, name); + if (inp == nullptr) { + GGML_ABORT("Failed to get tensor %s", name); + } + if (!(inp->flags & GGML_TENSOR_FLAG_INPUT)) { + GGML_ABORT("Tensor %s is not an input tensor", name); + } + return inp; + }; + + auto set_input_f32 = [&get_inp_tensor](const char * name, const std::vector & values) { + ggml_tensor * cur = get_inp_tensor(name); + GGML_ASSERT(cur->type == GGML_TYPE_F32); + GGML_ASSERT(ggml_nelements(cur) == (int64_t)values.size()); + ggml_backend_tensor_set(cur, values.data(), 0, ggml_nbytes(cur)); + }; + + // mask from build_inp_attn_mask(), f16 if flash attn is enabled + auto set_input_attn_mask = [&get_inp_tensor](const std::vector & values, int idx = 0) { + ggml_tensor * cur = get_inp_tensor(get_attn_mask_name(idx).c_str()); + GGML_ASSERT(ggml_nelements(cur) == (int64_t)values.size()); + if (cur->type == GGML_TYPE_F16) { + std::vector values_f16(values.size()); + ggml_fp32_to_fp16_row(values.data(), values_f16.data(), values.size()); + ggml_backend_tensor_set(cur, values_f16.data(), 0, ggml_nbytes(cur)); + } else { + GGML_ASSERT(cur->type == GGML_TYPE_F32); + ggml_backend_tensor_set(cur, values.data(), 0, ggml_nbytes(cur)); + } + }; + + auto set_input_i32 = [&get_inp_tensor](const char * name, std::vector & values) { + ggml_tensor * cur = get_inp_tensor(name); + GGML_ASSERT(cur->type == GGML_TYPE_I32); + GGML_ASSERT(ggml_nelements(cur) == (int64_t)values.size()); + ggml_backend_tensor_set(cur, values.data(), 0, ggml_nbytes(cur)); + }; + + // state of the previous call, zero on a cold start + auto set_gen_state_in = [&]() { + const auto & slots = params->gen_state->slots; + size_t i = 0; + for (const auto & slot : list_gen_state_slots(hparams, model)) { + GGML_ASSERT(i < slots.size()); + if (slot.in_place) { + i++; + continue; + } + ggml_backend_tensor_copy(slots[i++], get_inp_tensor(("state_in_" + slot.name).c_str())); + } + }; + + // returns the "pos" slot; without a stream, the state is zero + auto set_enc_state_in = [&]() -> int { + const auto state_slots = list_enc_state_slots(hparams, model); + GGML_ASSERT(!state_slots.empty() && state_slots[0].name == "pos"); + if (params->enc_state == nullptr) { + for (const auto & slot : state_slots) { + ggml_tensor * t = get_inp_tensor(("state_in_" + slot.name).c_str()); + ggml_backend_tensor_memset(t, 0, 0, ggml_nbytes(t)); + } + return 0; + } + const auto & slots = params->enc_state->slots; + GGML_ASSERT(slots.size() == state_slots.size()); + float base_f = 0.0f; + ggml_backend_tensor_get(slots[0], &base_f, 0, sizeof(float)); + size_t i = 0; + for (const auto & slot : state_slots) { + ggml_backend_tensor_copy(slots[i++], get_inp_tensor(("state_in_" + slot.name).c_str())); + } + return (int) base_f; + }; + + // lfm3a encoder and codec: causal sliding window over [cached window | this call] + auto set_lfm3a_window_inputs = [&](int base, int n_pos, int window) { + std::vector positions(n_pos); + for (int i = 0; i < n_pos; i++) { + positions[i] = base + i; + } + set_input_i32("positions", positions); + + // key j holds position base - prefix + j + const int prefix = window - 1; + const int n_kv = prefix + n_pos; + std::vector mask((size_t) n_kv * n_pos, -INFINITY); + for (int q = 0; q < n_pos; q++) { + for (int j = std::max(0, q + prefix - window + 1); j <= q + prefix; j++) { + if (base - prefix + j >= 0) { + mask[(size_t) q * n_kv + j] = 0.0f; + } + } + } + set_input_attn_mask(mask); + }; + + // rope positions and attention mask of the mimi transformers (pocket-tts). + // the mask is causal with a sliding window, see _build_attention_mask() in the reference + auto set_pockettts_tfm_inputs = [&]() { + const int64_t n_pos = ggml_nelements(get_inp_tensor("inp_pos")); + GGML_ASSERT(n_pos > 0); + std::vector positions((size_t) n_pos); + for (int64_t i = 0; i < n_pos; i++) { + positions[(size_t) i] = (int32_t) i; + } + set_input_i32("inp_pos", positions); + + // the preprocessor truncates the waveform to keep this mask bounded + const int64_t max_pos = (int64_t) mtmd_hparams::pockettts_max_spk_seconds * hparams.audio_sample_rate / 120; + GGML_ASSERT(n_pos <= max_pos && "pocket-tts speaker reference too long for a dense mask"); + + const int64_t context = hparams.mimi_tfm_context; + std::vector mask((size_t) n_pos * n_pos, -INFINITY); + for (int64_t q = 0; q < n_pos; q++) { + for (int64_t k = 0; k < n_pos; k++) { + const int64_t delta = q - k; + if (delta >= 0 && delta < context) { + mask[(size_t) q * n_pos + k] = 0.0f; + } + } + } + set_input_attn_mask(mask); + }; + + // set input pixel values + if (!imgs.is_audio) { + size_t nelem = 0; + for (const auto & img : imgs.entries) { + nelem += img.nx() * img.ny() * 3; + } + std::vector inp_raw(nelem); + + // layout of data (note: the channel dim is unrolled to better visualize the layout): + // + // ┌──W──┐ + // │ H │ channel = R + // ├─────┤ │ + // │ H │ channel = G + // ├─────┤ │ + // │ H │ channel = B + // └─────┘ │ + // ──────┘ x B + + // IMPORTANT: [QWEN_VIDEO] the batch dim is currently used for temporal dim in Qwen-VL models + // All entries must have the same spatial size (enforced by can_batch_with() during merging) + { + const int nx = imgs.entries[0].nx(); + const int ny = imgs.entries[0].ny(); + const int n = nx * ny; + + for (int b = 0; b < n_batch_cur; b++) { + LOG_DBG("%s: copying image %d/%d to input buffer (nx=%d, ny=%d)\n", __func__, b+1, n_batch_cur, nx, ny); + const auto & buf = imgs.entries[b].get_ro_buf(); + float * batch_entry = inp_raw.data() + b * (3*n); + for (int y = 0; y < ny; y++) { + for (int x = 0; x < nx; x++) { + size_t base_src = 3*(y * nx + x); + size_t base_dst = y * nx + x; + batch_entry[ base_dst] = buf[base_src ]; + batch_entry[1*n + base_dst] = buf[base_src + 1]; + batch_entry[2*n + base_dst] = buf[base_src + 2]; + } + } + } + } + set_input_f32("inp_raw", inp_raw); + + } else if (params->gen_process != MTMD_SUBCTX_GEN_PROCESS_GEN_WAV && + params->gen_process != MTMD_SUBCTX_GEN_PROCESS_TTS && + params->gen_process != MTMD_SUBCTX_GEN_PROCESS_TTS_VOCODE && + ggml_graph_get_tensor(gf, "inp_raw") != nullptr) { + // audio input. GEN_WAV and the chatterbox TTS stages are not here: they take codes or feats, set in the switch below. + // a GEN_CODE graph may have no hidden-state input (lfm3a_gen feedback embedding) + GGML_ASSERT(imgs.entries.size() == 1); + + const auto & mel_inp = imgs.entries[0]; + const auto & buf = mel_inp.get_ro_buf(); + const int n_step = mel_inp.nx(); + const int n_mel = mel_inp.ny(); + GGML_ASSERT((size_t)n_step * n_mel == buf.size()); + + set_input_f32("inp_raw", buf); + } + + // set input per projector + switch (model.proj_type) { + case PROJECTOR_TYPE_VOXTRAL_RT: + { + mtmd_subctx_enc_state * st = params->enc_state; + if (!st) { + LOG_ERR("%s: this encoder needs a mtmd_subctx_enc_state\n", __func__); + return false; + } + const int n_window = hparams.attn_window_size; + const int n_new = (imgs.entries[0].nx() - 2) / 2; + GGML_ASSERT(n_new <= st->n_ctx - n_window); + + std::vector pos(n_new); + std::vector idx(n_new); + std::vector slot_pos = st->slot_pos; // including the frames written by this call + for (int i = 0; i < n_new; i++) { + pos[i] = st->n_past + i; + idx[i] = pos[i] % st->n_ctx; + slot_pos[idx[i]] = pos[i]; + } + set_input_i32("inp_pos", pos); + ggml_backend_tensor_set(get_inp_tensor("inp_kv_idx"), idx.data(), 0, idx.size() * sizeof(int64_t)); + + // each frame sees itself and the n_window - 1 frames before it + std::vector mask((size_t) st->n_ctx * n_new, -INFINITY); + for (int i = 0; i < n_new; i++) { + for (int s = 0; s < st->n_ctx; s++) { + const int32_t p = slot_pos[s]; + if (p >= 0 && p <= pos[i] && p > pos[i] - n_window) { + mask[(size_t) i * st->n_ctx + s] = 0.0f; + } + } + } + set_input_attn_mask(mask); + + std::vector tail = st->stem_tail; + tail.resize(hparams.n_embd, 0.0f); + set_input_f32("inp_stem_tail", tail); + } break; + case PROJECTOR_TYPE_MUSE_GLIMMER: + { + const int grid_w = pos_w; // image_size_width / patch_size + const int grid_h = pos_h; // image_size_height / patch_size + const int n_tok = grid_w * grid_h; + const int pgrid = (int) std::sqrt((double) model.position_embeddings->ne[1]); // 32 + const int f = hparams.n_merge; // downsample 2 + + // pixel patchify runs inside the graph via build_inp() (ggml_conv_2d); + // pos-emb bilinear interp via resize_position_embeddings(). + + // --- sparse window grouping (pgrid x pgrid windows) --- + const int win = pgrid; + const int nwin_h = (grid_h + win - 1) / win; + const int nwin_w = (grid_w + win - 1) / win; + std::vector sp_perm; sp_perm.reserve(n_tok); + std::vector sp_slens; + for (int wy = 0; wy < nwin_h; wy++) { + for (int wx = 0; wx < nwin_w; wx++) { + int cnt = 0; + for (int hh = 0; hh < win; hh++) { + for (int ww = 0; ww < win; ww++) { + const int gy = wy * win + hh; + const int gx = wx * win + ww; + if (gy < grid_h && gx < grid_w) { sp_perm.push_back(gy * grid_w + gx); cnt++; } + } + } + if (cnt > 0) sp_slens.push_back(cnt); + } + } + std::vector rpos_w(n_tok), rpos_h(n_tok), inv_perm(n_tok); + for (int i = 0; i < n_tok; i++) { + const int orig = sp_perm[i]; + rpos_w[i] = (orig % grid_w) + 1; // 1-indexed + rpos_h[i] = (orig / grid_w) + 1; + inv_perm[orig] = i; + } + set_input_i32("muse_glimmer_sp_perm", sp_perm); + set_input_i32("muse_glimmer_inv_perm", inv_perm); + set_input_i32("muse_glimmer_pos_w", rpos_w); + set_input_i32("muse_glimmer_pos_h", rpos_h); + + // block-diagonal window mask (permuted order) + std::vector sp_mask((size_t) n_tok * n_tok, -INFINITY); + { + int off = 0; + for (int s : sp_slens) { + for (int a = 0; a < s; a++) + for (int b = 0; b < s; b++) + sp_mask[(size_t) (off + a) * n_tok + (off + b)] = 0.0f; + off += s; + } + } + set_input_attn_mask(sp_mask); + + // pixel-shuffle gather (original order): f*f spatial neighbours grouped + std::vector dsp; dsp.reserve(n_tok); + for (int oy = 0; oy < grid_h / f; oy++) + for (int ox = 0; ox < grid_w / f; ox++) + for (int ry = 0; ry < f; ry++) + for (int rx = 0; rx < f; rx++) + dsp.push_back((oy * f + ry) * grid_w + (ox * f + rx)); + set_input_i32("muse_glimmer_ds_perm", dsp); + } break; + case PROJECTOR_TYPE_MINICPMV: + { + // inspired from siglip: + // -> https://huggingface.co/HuggingFaceM4/siglip-so400m-14-980-flash-attn2-navit + // -> https://huggingface.co/HuggingFaceM4/siglip-so400m-14-980-flash-attn2-navit/blob/d66538faeba44480d0bfaa42145eef26f9423199/modeling_siglip.py#L316 + std::vector positions(pos_h * pos_w); + // note: sized by the actual patch counts; a tall/wide image produces more + // than 1024 patches per side and a fixed [1024] array would be overrun + std::vector bucket_coords_h(pos_h); + std::vector bucket_coords_w(pos_w); + for (int i = 0; i < pos_h; i++){ + bucket_coords_h[i] = std::floor(70.0*i/pos_h); + } + for (int i = 0; i < pos_w; i++){ + bucket_coords_w[i] = std::floor(70.0*i/pos_w); + } + for (int i = 0, id = 0; i < pos_h; i++){ + for (int j = 0; j < pos_w; j++){ + positions[id++] = bucket_coords_h[i]*70 + bucket_coords_w[j]; + } + } + set_input_i32("positions", positions); + + // inputs for resampler projector + // set the 2D positions (using float for sinusoidal embedding) + int n_patches_per_col = image_size_width / patch_size; + std::vector pos_data(n_pos); + // dimension H + for (int i = 0; i < n_pos; i++) { + pos_data[i] = static_cast(i / n_patches_per_col); + } + set_input_f32("pos_h", pos_data); + // dimension W + for (int i = 0; i < n_pos; i++) { + pos_data[i] = static_cast(i % n_patches_per_col); + } + set_input_f32("pos_w", pos_data); + // base frequency omega + const float base_freq = 10000.0f; + const int n_embd_proj = model.n_mmproj_embd(); + std::vector omega(n_embd_proj / 4); + for (int i = 0; i < n_embd_proj / 4; ++i) { + omega[i] = 1.0f / std::pow(base_freq, static_cast(i) / (n_embd_proj / 4)); + } + set_input_f32("omega", omega); + } break; + case PROJECTOR_TYPE_MINICPMV4_6: + case PROJECTOR_TYPE_MINICPMV4_7: + case PROJECTOR_TYPE_MINICPM_OCR_1_0: + { + const bool is_4x = hparams.n_merge == 2; + + // SigLIP position buckets (same as resampler path) + std::vector positions(pos_h * pos_w); + std::vector bucket_coords_h(pos_h); + std::vector bucket_coords_w(pos_w); + for (int i = 0; i < pos_h; i++){ + bucket_coords_h[i] = std::floor(70.0*i/pos_h); + } + for (int i = 0; i < pos_w; i++){ + bucket_coords_w[i] = std::floor(70.0*i/pos_w); + } + for (int i = 0, id = 0; i < pos_h; i++){ + for (int j = 0; j < pos_w; j++){ + positions[id++] = bucket_coords_h[i]*70 + bucket_coords_w[j]; + } + } + set_input_i32("positions", positions); + + const int half_h = pos_h / 2; + const int half_w = pos_w / 2; + + auto make_ds_idx = [](int off_r, int off_c, int ds_h, int ds_w, int stride_w) { + std::vector idx(ds_h * ds_w); + for (int i = 0; i < ds_h; i++) { + for (int j = 0; j < ds_w; j++) { + idx[i * ds_w + j] = (2*i + off_r) * stride_w + (2*j + off_c); + } + } + return idx; + }; + + if (!is_4x) { + // window reorder indices for 2x2 windows + std::vector window_idx(n_pos); + std::vector inv_window_idx(n_pos); + { + int k = 0; + for (int wi = 0; wi < half_h; wi++) { + for (int wj = 0; wj < half_w; wj++) { + window_idx[k++] = (2*wi ) * pos_w + (2*wj ); + window_idx[k++] = (2*wi ) * pos_w + (2*wj + 1); + window_idx[k++] = (2*wi + 1) * pos_w + (2*wj ); + window_idx[k++] = (2*wi + 1) * pos_w + (2*wj + 1); + } + } + for (int i = 0; i < n_pos; i++) { + inv_window_idx[window_idx[i]] = i; + } + } + set_input_i32("vit_merger_window_idx", window_idx); + set_input_i32("vit_merger_inv_window_idx", inv_window_idx); + + // block-diagonal attention mask: tokens in the same 4-token + // window attend to each other (mask = 0), all other positions + // are masked out (-inf). matches the window-major reorder above. + std::vector window_mask_data(n_pos * n_pos, std::numeric_limits::lowest()); + for (int wi = 0; wi < n_pos / 4; wi++) { + for (int i = 0; i < 4; i++) { + for (int j = 0; j < 4; j++) { + window_mask_data[(wi*4 + i) * n_pos + (wi*4 + j)] = 0.0f; + } + } + } + set_input_attn_mask(window_mask_data); + + // ViT merger 2x2 downsample indices + auto vit_merger_ds_0 = make_ds_idx(0, 0, half_h, half_w, pos_w); + auto vit_merger_ds_1 = make_ds_idx(0, 1, half_h, half_w, pos_w); + auto vit_merger_ds_2 = make_ds_idx(1, 0, half_h, half_w, pos_w); + auto vit_merger_ds_3 = make_ds_idx(1, 1, half_h, half_w, pos_w); + set_input_i32("vit_merger_ds_idx_0", vit_merger_ds_0); + set_input_i32("vit_merger_ds_idx_1", vit_merger_ds_1); + set_input_i32("vit_merger_ds_idx_2", vit_merger_ds_2); + set_input_i32("vit_merger_ds_idx_3", vit_merger_ds_3); + } + + const int merger_h = is_4x ? pos_h : half_h; + const int merger_w = is_4x ? pos_w : half_w; + auto m_ds_0 = make_ds_idx(0, 0, merger_h / 2, merger_w / 2, merger_w); + auto m_ds_1 = make_ds_idx(0, 1, merger_h / 2, merger_w / 2, merger_w); + auto m_ds_2 = make_ds_idx(1, 0, merger_h / 2, merger_w / 2, merger_w); + auto m_ds_3 = make_ds_idx(1, 1, merger_h / 2, merger_w / 2, merger_w); + set_input_i32("merger_ds_idx_0", m_ds_0); + set_input_i32("merger_ds_idx_1", m_ds_1); + set_input_i32("merger_ds_idx_2", m_ds_2); + set_input_i32("merger_ds_idx_3", m_ds_3); + } break; + case PROJECTOR_TYPE_LFM3VL: + { + // patches in raster-scan order (no interleave before encoder layers) + const int pw = image_size_width / patch_size; + const int ph = image_size_height / patch_size; + std::vector positions(n_pos * 4); + int ptr = 0; + for (int y = 0; y < ph; y++) { + for (int x = 0; x < pw; x++) { + positions[ ptr] = y; + positions[ num_patches + ptr] = x; + positions[2 * num_patches + ptr] = y; + positions[3 * num_patches + ptr] = x; + ptr++; + } + } + set_input_i32("positions", positions); + } break; + case PROJECTOR_TYPE_MAGEVL: + { + // t, h, w in 2x2 block order, t is always 0 for an image + const int merge_ratio = hparams.n_merge; + const int pw = image_size_width / patch_size / merge_ratio; + const int ph = image_size_height / patch_size / merge_ratio; + std::vector positions(n_pos * 4, 0); + int ptr = 0; + for (int ty = 0; ty < ph; ty++) { + for (int tx = 0; tx < pw; tx++) { + for (int dy = 0; dy < merge_ratio; dy++) { + for (int dx = 0; dx < merge_ratio; dx++) { + positions[ num_patches + ptr] = ty * merge_ratio + dy; + positions[2 * num_patches + ptr] = tx * merge_ratio + dx; + ptr++; + } + } + } + } + set_input_i32("positions", positions); + } break; + case PROJECTOR_TYPE_QWEN2VL: + case PROJECTOR_TYPE_QWEN3VL: + case PROJECTOR_TYPE_LING3VL: + case PROJECTOR_TYPE_GLM4V: + case PROJECTOR_TYPE_GLM5V: + { + const int merge_ratio = hparams.n_merge; + const int pw = image_size_width / patch_size / merge_ratio; + const int ph = image_size_height / patch_size / merge_ratio; + std::vector positions(n_pos * 4); + int ptr = 0; + for (int ty = 0; ty < ph; ty++) { + for (int tx = 0; tx < pw; tx++) { + for (int dy = 0; dy < merge_ratio; dy++) { + for (int dx = 0; dx < merge_ratio; dx++) { + const int y = ty * merge_ratio + dy; + const int x = tx * merge_ratio + dx; + positions[ ptr] = y; + positions[ num_patches + ptr] = x; + positions[2 * num_patches + ptr] = y; + positions[3 * num_patches + ptr] = x; + ptr++; + } + } + } + } + + set_input_i32("positions", positions); + } break; + case PROJECTOR_TYPE_STEP3VL: + { + std::vector pos_data(n_pos); + for (int i = 0; i < n_pos; i++) { + pos_data[i] = i / pos_w; + } + set_input_i32("pos_h", pos_data); + for (int i = 0; i < n_pos; i++) { + pos_data[i] = i % pos_w; + } + set_input_i32("pos_w", pos_data); + } break; + case PROJECTOR_TYPE_PADDLEOCR: + { + const int merge_ratio = hparams.n_merge; + const int pw = image_size_width / patch_size / merge_ratio; + const int ph = image_size_height / patch_size / merge_ratio; + std::vector positions(n_pos * 4); + int ptr = 0; + // NOTE: same as Qwen-VL, but x and y are swapped + for (int ty = 0; ty < ph; ty++) { + for (int dy = 0; dy < merge_ratio; dy++) { + for (int tx = 0; tx < pw; tx++) { + for (int dx = 0; dx < merge_ratio; dx++) { + const int y = ty * merge_ratio + dy; + const int x = tx * merge_ratio + dx; + positions[ ptr] = y; + positions[ num_patches + ptr] = x; + positions[2 * num_patches + ptr] = y; + positions[3 * num_patches + ptr] = x; + ptr++; + } + } + } + } + + set_input_i32("positions", positions); + } break; + case PROJECTOR_TYPE_MINIMAX_M3: + { + const int n_merge = hparams.n_merge; + const int gh = image_size_height / patch_size; + const int gw = image_size_width / patch_size; + std::vector pos_h, pos_w; + pos_h.reserve(gh * gw); + pos_w.reserve(gh * gw); + for (int bh = 0; bh < gh / n_merge; bh++) + for (int bw = 0; bw < gw / n_merge; bw++) + for (int mh = 0; mh < n_merge; mh++) + for (int mw = 0; mw < n_merge; mw++) { + pos_h.push_back(bh * n_merge + mh); + pos_w.push_back(bw * n_merge + mw); + } + set_input_i32("minimax_pos_h", pos_h); + set_input_i32("minimax_pos_w", pos_w); + } break; + case PROJECTOR_TYPE_DOTS_OCR: + case PROJECTOR_TYPE_DOTS3NOTE_V: + { + const int pw = image_size_width / patch_size; + const int ph = image_size_height / patch_size; + const int n_pos = ph * pw; + std::vector positions(n_pos * 4); + int ptr = 0; + + // flat layout: [h, w, h, w] for each patch + // patches are in raster order (matching conv2d output) + for (int y = 0; y < ph; y++) { + for (int x = 0; x < pw; x++) { + positions[ ptr] = y; + positions[ n_pos + ptr] = x; + positions[2*n_pos + ptr] = y; + positions[3*n_pos + ptr] = x; + ptr++; + } + } + + set_input_i32("positions", positions); + } break; + case PROJECTOR_TYPE_QWEN25VL: + case PROJECTOR_TYPE_EXAONE4_5: + case PROJECTOR_TYPE_YOUTUVL: + { + // pw * ph = number of tokens output by ViT after apply patch merger + // ipw * ipw = number of vision token been processed inside ViT + const bool use_window_attn = + (model.proj_type == PROJECTOR_TYPE_QWEN25VL || model.proj_type == PROJECTOR_TYPE_EXAONE4_5) + ? hparams.n_wa_pattern > 0 + : !hparams.wa_layer_indexes.empty(); + const int merge_ratio = 2; + const int pw = image_size_width / patch_size / merge_ratio; + const int ph = image_size_height / patch_size / merge_ratio; + const int ipw = image_size_width / patch_size; + const int iph = image_size_height / patch_size; + + std::vector idx (ph * pw); + std::vector inv_idx(ph * pw); + + if (use_window_attn) { + const int attn_window_size = hparams.attn_window_size > 0 ? hparams.attn_window_size : 112; + const int grid_window = attn_window_size / patch_size / merge_ratio; + int dst = 0; + // [num_vision_tokens, num_vision_tokens] attention mask tensor + std::vector mask(pow(ipw * iph, 2), std::numeric_limits::lowest()); + int mask_row = 0; + + for (int y = 0; y < ph; y += grid_window) { + for (int x = 0; x < pw; x += grid_window) { + const int win_h = std::min(grid_window, ph - y); + const int win_w = std::min(grid_window, pw - x); + const int dst_0 = dst; + // group all tokens belong to the same window togather (to a continue range) + for (int dy = 0; dy < win_h; dy++) { + for (int dx = 0; dx < win_w; dx++) { + const int src = (y + dy) * pw + (x + dx); + GGML_ASSERT(src < (int)idx.size()); + GGML_ASSERT(dst < (int)inv_idx.size()); + idx [src] = dst; + inv_idx[dst] = src; + dst++; + } + } + + for (int r=0; r < win_h * win_w * merge_ratio * merge_ratio; r++) { + int row_offset = mask_row * (ipw * iph); + std::fill( + mask.begin() + row_offset + (dst_0 * merge_ratio * merge_ratio), + mask.begin() + row_offset + (dst * merge_ratio * merge_ratio), + 0.0); + mask_row++; + } + } + } + + set_input_i32("window_idx", idx); + set_input_i32("inv_window_idx", inv_idx); + set_input_attn_mask(mask); + } else { + for (int i = 0; i < ph * pw; i++) { + idx[i] = i; + } + } + + const int mpow = merge_ratio * merge_ratio; + std::vector positions(n_pos * 4); + + int ptr = 0; + for (int ty = 0; ty < ph; ty++) { + for (int tx = 0; tx < pw; tx++) { + for (int dy = 0; dy < merge_ratio; dy++) { + for (int dx = 0; dx < merge_ratio; dx++) { + auto remap = idx[ptr / mpow]; + remap = (remap * mpow) + (ptr % mpow); + + const int y = ty * merge_ratio + dy; + const int x = tx * merge_ratio + dx; + positions[ remap] = y; + positions[ num_patches + remap] = x; + positions[2 * num_patches + remap] = y; + positions[3 * num_patches + remap] = x; + ptr++; + } + } + } + } + + set_input_i32("positions", positions); + } break; + case PROJECTOR_TYPE_MIMOVL: + { + const int merge = hparams.n_merge; // 2 + const int merge_unit = merge * merge; // 4 + const int patch = hparams.patch_size; // 16 + const int H = image_size_height / patch; + const int W = image_size_width / patch; + const int n_pos_full = H * W; + const int llm_h = H / merge; + const int llm_w = W / merge; + const int n_units = llm_h * llm_w; // n_pos / merge_unit + + // Row-major merge-tile-ordered (h, w) positions + std::vector pos_h_row(n_pos_full); + std::vector pos_w_row(n_pos_full); + { + int idx = 0; + for (int ty = 0; ty < llm_h; ty++) { + for (int tx = 0; tx < llm_w; tx++) { + for (int dy = 0; dy < merge; dy++) { + for (int dx = 0; dx < merge; dx++) { + pos_h_row[idx] = ty * merge + dy; + pos_w_row[idx] = tx * merge + dx; + idx++; + } + } + } + } + } + + // Col-major merge-unit permutation + std::vector idx_col(n_units); + for (int r = 0; r < llm_h; r++) { + for (int c = 0; c < llm_w; c++) { + int u_row = r * llm_w + c; + int u_col = c * llm_h + r; + idx_col[u_col] = (float) u_row; + } + } + + // Col-mode positions: permute pos_*_row by idx_col + std::vector pos_h_col(n_pos_full); + std::vector pos_w_col(n_pos_full); + for (int u = 0; u < n_units; u++) { + int src = (int) idx_col[u]; + for (int k = 0; k < merge_unit; k++) { + pos_h_col[u * merge_unit + k] = pos_h_row[src * merge_unit + k]; + pos_w_col[u * merge_unit + k] = pos_w_row[src * merge_unit + k]; + } + } + + // Pack into ggml_rope_multi VISION-mode layout. The non-CPU kernels + // only read slots 0 and 1, so pack h in slot 0, w in slot 1: + // positions[0..n_pos) = h + // positions[n_pos..2*n_pos) = w + // positions[2*n_pos..3*n_pos) = 0 + // positions[3*n_pos..4*n_pos) = 0 + std::vector positions_row(static_cast(n_pos_full) * 4, 0); + std::vector positions_col(static_cast(n_pos_full) * 4, 0); + for (int i = 0; i < n_pos_full; i++) { + positions_row[0 * n_pos_full + i] = pos_h_row[i]; + positions_row[1 * n_pos_full + i] = pos_w_row[i]; + positions_col[0 * n_pos_full + i] = pos_h_col[i]; + positions_col[1 * n_pos_full + i] = pos_w_col[i]; + } + + // Banded 1D sliding-window mask + const int window = hparams.attn_window_size; + GGML_ASSERT(window > 0); + std::vector mask(static_cast(n_pos_full) * n_pos_full, std::numeric_limits::lowest()); + for (int q = 0; q < n_pos_full; q++) { + int lo = std::max(0, q - window); + int hi = std::min(n_pos_full - 1, q + window); + for (int k = lo; k <= hi; k++) { + mask[static_cast(q) * n_pos_full + k] = 0.0f; + } + } + + set_input_i32("mimovl_positions_row", positions_row); + set_input_i32("mimovl_positions_col", positions_col); + set_input_f32("mimovl_idx_col", idx_col); + set_input_attn_mask(mask); + } break; + case PROJECTOR_TYPE_PIXTRAL: + case PROJECTOR_TYPE_KIMIVL: + case PROJECTOR_TYPE_KIMIK25: + case PROJECTOR_TYPE_KIMIK31: + case PROJECTOR_TYPE_LIGHTONOCR: + { + // set the 2D positions + int n_patches_per_col = image_size_width / patch_size; + std::vector pos_data(n_pos); + // dimension H + for (int i = 0; i < n_pos; i++) { + pos_data[i] = i / n_patches_per_col; + } + set_input_i32("pos_h", pos_data); + // dimension W + for (int i = 0; i < n_pos; i++) { + pos_data[i] = i % n_patches_per_col; + } + set_input_i32("pos_w", pos_data); + } break; + case PROJECTOR_TYPE_SENSENOVA_U1: + { + const int n_patches_per_row = image_size_width / patch_size; + std::vector pos_data(n_pos); + for (int i = 0; i < n_pos; i++) { + pos_data[i] = i / n_patches_per_row; + } + set_input_i32("pos_y", pos_data); + for (int i = 0; i < n_pos; i++) { + pos_data[i] = i % n_patches_per_row; + } + set_input_i32("pos_x", pos_data); + } break; + case PROJECTOR_TYPE_DEEPSEEK4V: + { + // set the 2D positions (mrope layout, only the first 2 channels are used) + int n_patches_per_row = image_size_width / patch_size; + std::vector positions(n_pos * 4, 0); + for (int i = 0; i < n_pos; i++) { + positions[i] = i / n_patches_per_row; // row + positions[n_pos + i] = i % n_patches_per_row; // col + } + set_input_i32("positions", positions); + + // token block layout index (see mtmd_graph_deepseek4v::build) + // rows [0, n_grid) are the aligner output, the sentinels follow + const int n_merge = hparams.n_merge; + const int n_llm_w = MTMD_ALIGN(pos_w, n_merge) / n_merge; + const int n_llm_h = MTMD_ALIGN(pos_h, n_merge) / n_merge; + const int n_grid = n_llm_w * n_llm_h; + const int idx_start = n_grid; + const int idx_end = n_grid + 1; + const int idx_newline = n_grid + 2; + const int idx_pad = n_grid + 3; + + const int lead_pad = imgs.entries[0].lead_pad; + const auto bl = dsv4_get_block_layout(n_llm_w, n_llm_h, lead_pad); + + std::vector idx; + idx.reserve(bl.n_out); + for (int i = 0; i < lead_pad; i++) { + idx.push_back(idx_pad); + } + idx.push_back(idx_start); + // pairs of adjacent rows are interleaved column-wise ("N-layout") + // ref: build_image_block in inference/image_processor.py + for (int t = 0; t < bl.rows * bl.row_len; t++) { + const int g = t / (2 * bl.row_len); + const int rem = t % (2 * bl.row_len); + const int c = rem / 2; // column + const int r = 2 * g + rem % 2; // row + if (r >= n_llm_h) { + idx.push_back(idx_pad); + } else if (c == n_llm_w) { + idx.push_back(idx_newline); + } else { + idx.push_back(r * n_llm_w + c); + } + } + for (int i = 0; i < bl.pad_last; i++) { + idx.push_back(idx_pad); + } + idx.push_back(idx_end); + set_input_i32("layout_idx", idx); + } break; + case PROJECTOR_TYPE_DEEPSEEK41V: + { + // mrope layout, only the first 2 channels are used + const int n_patches_per_row = image_size_width / patch_size; + std::vector positions(n_pos * 4, 0); + for (int i = 0; i < n_pos; i++) { + positions[i] = i / n_patches_per_row; + positions[n_pos + i] = i % n_patches_per_row; + } + set_input_i32("positions", positions); + + // reading-order block: rows [0, n_grid) are the aligner output, then start / end / newline + const int n_merge = hparams.n_merge; + const int n_llm_w = MTMD_ALIGN(pos_w, n_merge) / n_merge; + const int n_llm_h = MTMD_ALIGN(pos_h, n_merge) / n_merge; + const int n_grid = n_llm_w * n_llm_h; + std::vector idx; + idx.reserve(dsv41_n_block_tokens(n_llm_w, n_llm_h)); + idx.push_back(n_grid); // IMAGE_START + for (int r = 0; r < n_llm_h; r++) { + for (int c = 0; c < n_llm_w; c++) { + idx.push_back(r * n_llm_w + c); + } + idx.push_back(n_grid + 2); // IMAGE_NEW_LINE + } + idx.push_back(n_grid + 1); // IMAGE_END + set_input_i32("layout_idx", idx); + } break; + case PROJECTOR_TYPE_GLM_EDGE: + { + // llava and other models + std::vector positions(n_pos); + for (int i = 0; i < n_pos; i++) { + positions[i] = i; + } + set_input_i32("positions", positions); + } break; + case PROJECTOR_TYPE_MLP: + case PROJECTOR_TYPE_MLP_NORM: + case PROJECTOR_TYPE_LDP: + case PROJECTOR_TYPE_LDPV2: + { + // llava and other models + std::vector positions(n_pos); + for (int i = 0; i < n_pos; i++) { + positions[i] = i; + } + set_input_i32("positions", positions); + + // The patches vector is used to get rows to index into the embeds with; + // we should skip dim 0 only if we have CLS to avoid going out of bounds + // when retrieving the rows. + int patch_offset = model.class_embedding ? 1 : 0; + std::vector patches(num_patches); + for (int i = 0; i < num_patches; i++) { + patches[i] = i + patch_offset; + } + set_input_i32("patches", patches); + } break; + case PROJECTOR_TYPE_POCKETTTS_SPKENC: + { + set_pockettts_tfm_inputs(); + } break; + case PROJECTOR_TYPE_POCKETTTS_GEN: + { + if (params->gen_process == MTMD_SUBCTX_GEN_PROCESS_GEN_WAV) { + GGML_ASSERT(params->feats != nullptr); + set_input_f32("inp_feats", *params->feats); + // positions and mask are derived in-graph from the persisted counter + set_gen_state_in(); + } else { + // flow matching starts from gaussian noise, std = sqrt(temp), no noise at temp 0 + ggml_tensor * t = get_inp_tensor("inp_noise"); + std::vector noise(ggml_nelements(t), 0.0f); + if (params->temp > 0.0f) { + std::normal_distribution dist(0.0f, std::sqrt(params->temp)); + for (auto & v : noise) { + v = dist(params->gen_state->rng); + } + } + set_input_f32("inp_noise", noise); + } + } break; + case PROJECTOR_TYPE_SOPRANO: + { + set_input_f32("inp_feats", *params->feats); + } break; + case PROJECTOR_TYPE_GEMMA4V: + case PROJECTOR_TYPE_GEMMA4UV: + { + // set (col, row) patch positions for learned positional embedding + const int n_cols = image_size_width / patch_size; + std::vector pos_x(num_patches), pos_y(num_patches); + for (int i = 0; i < num_patches; i++) { + pos_x[i] = i % n_cols; + pos_y[i] = i / n_cols; + } + set_input_i32("pos_x", pos_x); + set_input_i32("pos_y", pos_y); + } break; + case PROJECTOR_TYPE_DEEPSEEKOCR: + case PROJECTOR_TYPE_DEEPSEEKOCR2: + { + GGML_ASSERT( + (pos_w == pos_h) // overview image + || (pos_h >= pos_w && pos_h % pos_w == 0) // tile images + ); + + const int window = hparams.attn_window_size; + const int pos = pos_w; + std::vector rel_pos_indices_local(window * window); + std::vector rel_pos_indices_global(pos * pos); + + for (int q = 0; q < window; q++) { + for (int k = 0; k < window; k++) { + rel_pos_indices_local[q * window + k] = q - k + window - 1; + } + } + + for (int q = 0; q < pos; q++) { + for (int k = 0; k < pos; k++) { + rel_pos_indices_global[q * pos + k] = q - k + pos - 1; + } + } + + set_input_i32("rel_pos_indices_local", rel_pos_indices_local); + set_input_i32("rel_pos_indices_global", rel_pos_indices_global); + + if (model.proj_type == PROJECTOR_TYPE_DEEPSEEKOCR2) { + + // qwen2 encoder attention mask + + // num_image_tokens = num_patches / 16 + // 256 for 1024 global view + // 144 for 768 tile views + const int num_image_tokens = num_patches / 16; + const int seq_len = num_image_tokens * 2; + std::vector qwen2_mask(static_cast(seq_len) * seq_len, 0.0f); + + // attention mask layout + // +--------------+---------------+ + // | all 0 | all -inf | + // +--------------+---------------+ + // | all 0 | lower tri 0 | + // +--------------+---------------+ + for (int i = 0; i < seq_len; i++) { + for (int j = 0; j < seq_len; j++) { + const bool zero = i < num_image_tokens ? + j < num_image_tokens : + j < num_image_tokens || j <= i; + qwen2_mask[static_cast(i) * seq_len + j] = zero ? 0.0f : -1e9f; + } + } + set_input_attn_mask(qwen2_mask); + } + } break; + case PROJECTOR_TYPE_CHATTERBOX: + { + if (params->gen_process == MTMD_SUBCTX_GEN_PROCESS_TTS_VOCODE) { + set_input_f32("inp_mel", *params->mel_in); + set_input_f32("inp_sstft", *params->sstft_in); + break; + } + if (params->gen_process == MTMD_SUBCTX_GEN_PROCESS_TOKENIZE) { + const int T1 = (imgs.entries[0].nx() - 1) / 2 + 1; + const int T2 = (T1 - 1) / 2 + 1; + std::vector pos(T2); + for (int i = 0; i < T2; i++) { + pos[(size_t) i] = i; + } + set_input_i32("inp_pos", pos); + break; + } + if (params->gen_process != MTMD_SUBCTX_GEN_PROCESS_TTS) { + break; + } + const int n_gen = (int) params->codes->size(); + GGML_ASSERT(params->ref); + const int n_prompt = (int) params->ref->tokens.size(); + std::vector tokens = params->ref->tokens; + tokens.resize((size_t) n_prompt + n_gen); + memcpy(tokens.data() + n_prompt, params->codes->data(), (size_t) n_gen * sizeof(int32_t)); + const int64_t n_codes = model.cbx.input_embedding_w->ne[1]; + for (int32_t t : tokens) { + if (t < 0 || t >= n_codes) { + LOG_ERR("%s: speech token out of range (%d, codebook size %lld)\n", __func__, t, (long long) n_codes); + return false; + } + } + const int T1 = n_prompt + n_gen; + const int T2 = 2 * T1; + set_input_i32("inp_tokens", tokens); + + // mel-rate reference conditioning of the voice + set_input_f32("inp_prompt_feat", params->ref->feat); + set_input_f32("inp_xvec", params->ref->spk); + + // espnet relative positional encoding, entry k holds the sinusoid of relative position (T-1) - k + auto fill_pos = [&](const char * name, int T) { + const int d = 512; + std::vector pos((size_t) (2 * T - 1) * d); + for (int k = 0; k < 2 * T - 1; k++) { + const double rel = (double) (T - 1 - k); + for (int i = 0; i < d / 2; i++) { + const double div = exp(-(double) (2 * i) * log(10000.0) / d); + pos[(size_t) k * d + 2 * i ] = (float) sin(rel * div); + pos[(size_t) k * d + 2 * i + 1] = (float) cos(rel * div); + } + } + set_input_f32(name, pos); + }; + fill_pos("inp_pos1", T1); + fill_pos("inp_pos2", T2); + + // noise and the time embeddings of the solver span points, matcha layout (sines then cosines, scale 1000) + std::vector noise((size_t) 80 * T2); + std::normal_distribution nd(0.0f, 1.0f); + for (auto & f : noise) f = nd(params->gen_state->rng); + set_input_f32("inp_noise", noise); + + const bool meanflow = model.cbx.time_embed_mixer_w != nullptr; + const int n_steps = meanflow ? 2 : 10; + std::vector temb((size_t) 320 * (n_steps + 1)); + for (int s = 0; s <= n_steps; s++) { + const double u = (double) s / n_steps; + const double t = meanflow ? u : 1.0 - cos(u * M_PI / 2.0); + for (int i = 0; i < 160; i++) { + const double div = exp(-(double) i * log(10000.0) / 159.0); + temb[(size_t) s * 320 + i ] = (float) sin(1000.0 * t * div); + temb[(size_t) s * 320 + i + 160] = (float) cos(1000.0 * t * div); + } + } + set_input_f32("inp_temb", temb); + } break; + case PROJECTOR_TYPE_GEMMA3: + case PROJECTOR_TYPE_GEMMA3NV: + case PROJECTOR_TYPE_IDEFICS3: + case PROJECTOR_TYPE_COHERE2V: + case PROJECTOR_TYPE_INTERNVL: + case PROJECTOR_TYPE_NEMOTRON_V2_VL: + case PROJECTOR_TYPE_NEMOTRON_OMNI: + case PROJECTOR_TYPE_QWEN2A: + case PROJECTOR_TYPE_QWEN3A: + case PROJECTOR_TYPE_GLMA: + case PROJECTOR_TYPE_ULTRAVOX: + case PROJECTOR_TYPE_LFM2: + case PROJECTOR_TYPE_D1OMNI_V: + case PROJECTOR_TYPE_VOXTRAL: + case PROJECTOR_TYPE_MERALION: + case PROJECTOR_TYPE_MUSIC_FLAMINGO: + case PROJECTOR_TYPE_JANUS_PRO: + case PROJECTOR_TYPE_PHI4: + case PROJECTOR_TYPE_COGVLM: + case PROJECTOR_TYPE_YASA2: + case PROJECTOR_TYPE_GEMMA4UA: + case PROJECTOR_TYPE_MUSE_SPARK_AUDIO: + case PROJECTOR_TYPE_QWEN3TTS_SPKENC: + { + // do nothing + } break; + case PROJECTOR_TYPE_CHATTERBOX_SPKENC: + { + // the ceil-mode seg pooling divides by the full segment length, rescale the last partial segment + const int T1 = (imgs.entries[0].nx() - 1) / 2 + 1; + const int S = (T1 + 99) / 100; + std::vector segfix((size_t) S, 1.0f); + const int last = T1 - (S - 1) * 100; + segfix[(size_t) S - 1] = 100.0f / (float) last; + set_input_f32("inp_segfix", segfix); + } break; + case PROJECTOR_TYPE_LFM3A_GEN: + { + const int64_t n_cb = model.gen_code_embd_w->ne[2]; + const int64_t n_vocab = model.gen_code_embd_w->ne[1]; + if (params->gen_process == MTMD_SUBCTX_GEN_PROCESS_GEN_WAV) { + GGML_ASSERT(params->codes != nullptr); + // codes index the FSQ table via ggml_get_rows, the end-of-audio code has no waveform + const int64_t n_codes = model.c2w.quant_first_cb_w->ne[1]; + for (int32_t c : *params->codes) { + if (c < 0 || c >= n_codes) { + LOG_ERR("%s: code out of range (%d, codebook size %lld)\n", __func__, c, (long long) n_codes); + return false; + } + } + std::vector codes = *params->codes; + set_input_i32("inp_codes", codes); + set_gen_state_in(); + // the "pos" slot leads the state + float base = 0.0f; + ggml_backend_tensor_get(params->gen_state->slots[0], &base, 0, sizeof(float)); + set_lfm3a_window_inputs((int) base, (int) ggml_nelements(get_inp_tensor("positions")), hparams.wav_tfm_swa); + } else { + // rows of the merged per-codebook tables; a masked codebook (-1) takes row 0, the mask drops it + std::vector codes(n_cb, -1); + if (params->codes) { + GGML_ASSERT(params->codes->size() == (size_t) n_cb); + codes = *params->codes; + } + std::vector mask(n_cb, 0.0f); + for (int64_t i = 0; i < n_cb; i++) { + if (codes[i] >= n_vocab) { + LOG_ERR("%s: code out of range (%d, vocab %lld)\n", __func__, codes[i], (long long) n_vocab); + return false; + } + if (codes[i] < 0) { + codes[i] = 0; + } else { + mask[i] = 1.0f; + } + codes[i] += (int32_t) (i * n_vocab); + } + set_input_i32("inp_codes", codes); + if (ggml_graph_get_tensor(gf, "inp_mask") != nullptr) { + set_input_f32("inp_mask", mask); + } + } + } break; + case PROJECTOR_TYPE_DRAMA3_GEN: + { + const int64_t n_cb = model.gen_code_embd_w->ne[2]; + const int64_t n_vocab = model.gen_code_embd_w->ne[1]; + if (params->gen_process == MTMD_SUBCTX_GEN_PROCESS_GEN_WAV) { + // frame-major to group-major, clamp: codec residual codebooks are smaller than the tower vocab + GGML_ASSERT(params->codes != nullptr && params->codes->size() % n_cb == 0); + const int64_t n_frames = (int64_t) params->codes->size() / n_cb; + const int64_t n_first = model.c2w.quant_first_cb_w->ne[1]; + const int64_t n_rest = model.c2w.quant_rest_cb_w->ne[1]; + std::vector codes(params->codes->size()); + for (int64_t f = 0; f < n_frames; f++) { + for (int64_t g = 0; g < n_cb; g++) { + const int64_t n = g == 0 ? n_first : n_rest; + codes[g * n_frames + f] = std::clamp((*params->codes)[f * n_cb + g], 0, (int32_t) n - 1); + } + } + set_input_i32("inp_codes", codes); + set_gen_state_in(); + } else { + // code0 < 0: next row is text (prompt), the tower only fills its KV + // codes: input frame codes for a speech row, empty for a text row + const bool next_speech = params->code0 >= 0; + const bool cur_speech = params->codes && (int64_t) params->codes->size() == n_cb; + if (params->code0 >= n_vocab) { + LOG_ERR("%s: code0 out of range (%d, vocab %lld)\n", __func__, params->code0, (long long) n_vocab); + return false; + } + std::vector code0 = { std::max(params->code0, 0) }; + set_input_i32("inp_code0", code0); + + std::vector extra(n_cb - 2); + for (int64_t g = 2; g < n_cb; g++) { + const int32_t c = cur_speech ? std::clamp((*params->codes)[g], 0, (int32_t) n_vocab - 1) : 0; + extra[g - 2] = c + (int32_t) (g * n_vocab); + } + set_input_i32("inp_extra", extra); + set_input_f32("inp_scale", { next_speech ? 1.0f : 0.0f, cur_speech ? 1.0f : 0.0f }); + + float pos = 0.0f; + ggml_backend_tensor_get(params->gen_state->slots[0], &pos, 0, sizeof(float)); + std::vector inp_pos = { (int32_t) pos }; + set_input_i32("inp_pos", inp_pos); + + std::uniform_real_distribution dist(0.0f, 1.0f); + std::vector r(model.gen_code_head_w->ne[2]); + for (auto & v : r) { + v = dist(params->gen_state->rng); + } + set_input_f32("inp_rand", r); + } + } break; + case PROJECTOR_TYPE_QWEN3TTS_GEN: + { + if (params->gen_process == MTMD_SUBCTX_GEN_PROCESS_GEN_WAV) { + GGML_ASSERT(params->codes != nullptr); + + // frame-major input to group-major, rear-padded with code 0 up to one window + const int64_t n_codes = model.gen_code_head_w->ne[2] + 1; + const int64_t n_frames_w = hparams.wav_tfm_swa; + const int64_t n_frames = (int64_t) params->codes->size() / n_codes; + GGML_ASSERT(n_frames > 0 && n_frames <= n_frames_w); + + // codes are used as ggml_get_rows indices, so check them against the codebook vocab + const int64_t vocab_first = model.c2w.quant_first_cb_w->ne[1]; + const int64_t vocab_rest = model.c2w.quant_rest_cb_w->ne[1]; + for (int64_t f = 0; f < n_frames; f++) { + for (int64_t g = 0; g < n_codes; g++) { + const int32_t c = (*params->codes)[f * n_codes + g]; + const int64_t vocab = (g == 0) ? vocab_first : vocab_rest; + if (c < 0 || (int64_t) c >= vocab) { + LOG_ERR("%s: code out of range (frame %lld, group %lld, code %d, vocab %lld)\n", + __func__, (long long) f, (long long) g, c, (long long) vocab); + return false; + } + } + } + + std::vector codes(n_frames_w * n_codes, 0); + for (int64_t f = 0; f < n_frames; f++) { + for (int64_t g = 0; g < n_codes; g++) { + codes[g * n_frames_w + f] = (*params->codes)[f * n_codes + g]; + } + } + set_input_i32("inp_codes", codes); + set_gen_state_in(); + } else { + // code0 indexes gen_code_out_embd_w via ggml_get_rows; bound it + const int64_t vocab0 = model.gen_code_out_embd_w->ne[1]; + if (params->code0 < 0 || (int64_t) params->code0 >= vocab0) { + LOG_ERR("%s: code0 out of range (%d, vocab %lld)\n", __func__, params->code0, (long long) vocab0); + return false; + } + std::vector code0 = { params->code0 }; + set_input_i32("inp_code0", code0); + + // one uniform(0,1) draw per codebook, used by do_sampling() + std::uniform_real_distribution dist(0.0f, 1.0f); + const int64_t n_acoustic = model.gen_code_head_w->ne[2]; + for (int64_t g = 0; g < n_acoustic; g++) { + std::vector r = { dist(params->gen_state->rng) }; + set_input_f32(("inp_rand_" + std::to_string(g)).c_str(), r); + } + } + } break; + case PROJECTOR_TYPE_HUNYUANVL: + { + // Compute the HunyuanVL 2D position embedding on CPU (with the + // custom sf=(target+0.1)/n_grid bilinear sampling that the + // reference implementation uses) and upload it to the graph + // input declared in mtmd_graph_hunyuanvl::build(). + GGML_ASSERT(model.position_embeddings != nullptr); + ggml_tensor * src_t = model.position_embeddings; + const int64_t n_embd = src_t->ne[0]; + const int64_t n_pos = src_t->ne[1]; // = n_grid * n_grid + const int n_grid = (int)std::lround(std::sqrt((double)n_pos)); + GGML_ASSERT((int64_t)n_grid * n_grid == n_pos); + const int out_w = pos_w; // pw + const int out_h = pos_h; // ph + + // Pull weight to host. + std::vector src(n_embd * n_pos); + ggml_backend_tensor_get(src_t, src.data(), 0, ggml_nbytes(src_t)); + + // Output layout matches ggml_new_tensor_2d(F32, n_embd, out_h*out_w): + // ne[0] = n_embd (fastest), ne[1] = out_h*out_w + // dst[(y*out_w + x) * n_embd + c] + std::vector dst((size_t)n_embd * out_h * out_w); + + const float sx = (float)(out_w + 0.1f) / (float)n_grid; + const float sy = (float)(out_h + 0.1f) / (float)n_grid; + + for (int y = 0; y < out_h; ++y) { + // Match ggml_compute_forward_upscale_f32 pixel-center + // convention (align_corners=False): src_y = (y+0.5)/sy - 0.5. + const float fy = ((float)y + 0.5f) / sy - 0.5f; + int y0 = (int)std::floor(fy); + int y1 = y0 + 1; + y0 = std::clamp(y0, 0, n_grid - 1); + y1 = std::clamp(y1, 0, n_grid - 1); + float wy1 = std::clamp(fy - (float)y0, 0.0f, 1.0f); + const float wy0 = 1.0f - wy1; + for (int x = 0; x < out_w; ++x) { + const float fx = ((float)x + 0.5f) / sx - 0.5f; + int x0 = (int)std::floor(fx); + int x1 = x0 + 1; + x0 = std::clamp(x0, 0, n_grid - 1); + x1 = std::clamp(x1, 0, n_grid - 1); + float wx1 = std::clamp(fx - (float)x0, 0.0f, 1.0f); + const float wx0 = 1.0f - wx1; + + const float w00 = wy0 * wx0; + const float w01 = wy0 * wx1; + const float w10 = wy1 * wx0; + const float w11 = wy1 * wx1; + + const float * s00 = &src[((size_t)y0 * n_grid + x0) * n_embd]; + const float * s01 = &src[((size_t)y0 * n_grid + x1) * n_embd]; + const float * s10 = &src[((size_t)y1 * n_grid + x0) * n_embd]; + const float * s11 = &src[((size_t)y1 * n_grid + x1) * n_embd]; + float * d = &dst[((size_t)y * out_w + x) * n_embd]; + for (int c = 0; c < n_embd; ++c) { + d[c] = w00 * s00[c] + w01 * s01[c] + w10 * s10[c] + w11 * s11[c]; + } + } + } + + set_input_f32("hunyuanvl_pos_embd", dst); + } break; + case PROJECTOR_TYPE_LLAMA4: + { + // set the 2D positions + int n_patches_per_col = image_size_width / patch_size; + std::vector pos_data(num_patches + 1, 0); // +1 for the [CLS] token + // last pos is always kept 0, it's for CLS + // dimension H + for (int i = 0; i < num_patches; i++) { + pos_data[i] = (i / n_patches_per_col) + 1; + } + set_input_i32("pos_h", pos_data); + // dimension W + for (int i = 0; i < num_patches; i++) { + pos_data[i] = (i % n_patches_per_col) + 1; + } + set_input_i32("pos_w", pos_data); + } break; + case PROJECTOR_TYPE_DOTS3NOTE_A: + { + GGML_ASSERT(imgs.entries.size() == 1); + const int n_pos = (imgs.entries.front().nx() + 7) / 8; // 3x stride-2 conv2d + std::vector positions(n_pos); + for (int i = 0; i < n_pos; i++) { + positions[i] = i; + } + set_input_i32("positions", positions); + } break; + case PROJECTOR_TYPE_GEMMA4A: + { + GGML_ASSERT(imgs.entries.size() == 1); + const auto & img0 = imgs.entries.front(); + // Compute n_pos matching SSCP output: two stride-2 convs + int n_pos = img0.nx(); + for (int i = 0; i < 2; i++) { n_pos = (n_pos - 1) / 2 + 1; } + + // Chunked local attention: blocked causal mask and RPE + const int chunk_size = mtmd_graph_gemma4a::chunk_size; + const int max_past = mtmd_graph_gemma4a::max_past; + const int context_size = chunk_size + max_past; + const int num_blocks = (n_pos + chunk_size - 1) / chunk_size; + + // cached keys of a stream are at positions -n_hist .. -1 + const int n_hist = std::min(set_enc_state_in(), max_past); + + // Blocked causal attention mask: [context_size, chunk_size, num_blocks] + { + std::vector mask(context_size * chunk_size * num_blocks, -1e9f); + for (int b = 0; b < num_blocks; b++) { + for (int q = 0; q < chunk_size; q++) { + int gq = b * chunk_size + q; + for (int k = 0; k < context_size; k++) { + int gk = b * chunk_size - max_past + k; + if (gq < n_pos && gk >= -n_hist && gk < n_pos && gk <= gq && (gq - gk) < max_past) { + mask[k + q * context_size + b * context_size * chunk_size] = 0.0f; + } + } + } + } + set_input_f32("kq_mask", mask); + } + + // Sinusoidal RPE: 13 positions [12, 11, ..., 0] + { + const int n_embd = model.hparams.n_embd; + const int num_timescales = n_embd / 2; + const float log_timescale_increment = logf(10000.0f) / std::max(num_timescales - 1, 1); + const int rpe_len = max_past + 1; + std::vector pos_emb(n_embd * rpe_len, 0.0f); + for (int p = 0; p < rpe_len; p++) { + float position = (float)(max_past - p); + for (int i = 0; i < num_timescales; i++) { + float inv_ts = expf(-(float)i * log_timescale_increment); + float scaled = position * inv_ts; + pos_emb[p * n_embd + i] = sinf(scaled); + pos_emb[p * n_embd + i + num_timescales] = cosf(scaled); + } + } + set_input_f32("pos_emb", pos_emb); + } + } break; + case PROJECTOR_TYPE_MIMO_AUDIO: + { + GGML_ASSERT(imgs.entries.size() == 1); + const int n_frames = imgs.entries.front().nx(); + const int n_pos = (n_frames - 1) / 2 + 1; // matches conv1(s=1)+conv2(s=2) output length + + std::vector positions(n_pos); + for (int i = 0; i < n_pos; i++) { + positions[i] = i; + } + set_input_i32("mimo_audio_positions", positions); + + const int window = hparams.attn_window_size; + GGML_ASSERT(window > 0); + + const float neg_inf = std::numeric_limits::lowest(); + std::vector full_mask((size_t) n_pos * n_pos); + std::vector window_mask((size_t) n_pos * n_pos); + for (int q = 0; q < n_pos; q++) { + for (int k = 0; k < n_pos; k++) { + const bool causal_ok = k <= q; + full_mask[(size_t) q * n_pos + k] = causal_ok ? 0.0f : neg_inf; + window_mask[(size_t) q * n_pos + k] = (causal_ok && (q - k) <= window) ? 0.0f : neg_inf; + } + } + set_input_attn_mask(full_mask, 0); + set_input_attn_mask(window_mask, 1); + + // input_local_transformer: block-diagonal mask + in-group positions + { + const int n_pos_ds = (n_pos - 2) / 2 + 1; // matches downsample conv (k=2,s=2,p=0) + const int group_size = hparams.audio_local_group_size; + GGML_ASSERT(group_size > 0); + const int n_groups = (n_pos_ds + group_size - 1) / group_size; + const int n_padded = n_groups * group_size; + + std::vector local_positions(n_padded); + for (int i = 0; i < n_padded; i++) { + local_positions[i] = i % group_size; + } + set_input_i32("mimo_audio_local_positions", local_positions); + + std::vector local_mask((size_t) n_padded * n_padded); + for (int q = 0; q < n_padded; q++) { + for (int k = 0; k < n_padded; k++) { + const bool same_group = (q / group_size) == (k / group_size); + local_mask[(size_t) q * n_padded + k] = same_group ? 0.0f : neg_inf; + } + } + set_input_attn_mask(local_mask, 2); + } + } break; + case PROJECTOR_TYPE_LFM2A: + case PROJECTOR_TYPE_D1OMNI_A: + { + GGML_ASSERT(imgs.entries.size() == 1); + const auto n_frames = model.n_output_tokens(&imgs.entries.front()); + + auto d_model = 512; + auto seq_len = n_frames * 2 - 1; + std::vector pos_emb(d_model*seq_len); + std::vector inv_freq(d_model / 2); + for (size_t i = 0; i < inv_freq.size(); ++i) { + inv_freq[i] = std::exp(-(std::log(10000.0) / (float)d_model) * (2.0f * (float)(i))); + } + for (int64_t pos = 0; pos < seq_len; ++pos) { + for (size_t i = 0; i < inv_freq.size(); ++i) { + const float ang = (n_frames - pos - 1) * inv_freq[i]; + pos_emb[pos*d_model + 2*i + 0] = sinf(ang); // even + pos_emb[pos*d_model + 2*i + 1] = cosf(ang); // odd + } + } + set_input_f32("pos_emb", pos_emb); + } break; + case PROJECTOR_TYPE_LFM3A: + { + GGML_ASSERT(imgs.entries.size() == 1); + const int n_pos = model.n_output_tokens(&imgs.entries.front()); + + const int base = set_enc_state_in(); + set_lfm3a_window_inputs(base, n_pos, hparams.attn_window_size); + } break; + case PROJECTOR_TYPE_NEMOTRON_TRANSCRIBE: + { + GGML_ASSERT(imgs.entries.size() == 1); + std::vector positions(ggml_nelements(get_inp_tensor("positions"))); + for (size_t i = 0; i < positions.size(); i++) { + positions[i] = (int32_t) i; + } + set_input_i32("positions", positions); + GGML_ASSERT(params->enc_state != nullptr); + set_input_f32("inp_norm", params->enc_state->mel_norm); + if (ggml_graph_get_tensor(gf, "spk_prev") != nullptr) { + set_input_f32("spk_prev", params->enc_state->spk_prev); + } + } break; + case PROJECTOR_TYPE_PARAKEET: + { + GGML_ASSERT(imgs.entries.size() == 1); + struct ggml_tensor * attn_mask = ggml_graph_get_tensor(gf, "attn_mask"); + const int n_q = attn_mask->ne[1]; + const int n_k = attn_mask->ne[0]; + const int n_frames = imgs.entries.front().nx(); + const int n_tokens_real = (n_frames + hparams.subsampling_factor-1) / hparams.subsampling_factor; + const float mask_value = -1e30f; + + std::vector mask_data(n_q * n_k); + if (n_k == n_q) { + // full attention: mask keys that are padding + for (int q = 0; q < n_q; ++q) { + for (int k = 0; k < n_k; ++k) { + mask_data[q * n_k + k] = (k >= n_tokens_real) ? mask_value : 0.0f; + } + } + } else { + // local attention: mask keys outside the valid window + const int att_left = n_k / 2; + for (int q = 0; q < n_q; ++q) { + for (int k = 0; k < n_k; ++k) { + const int key = q - att_left + k; + mask_data[q * n_k + k] = (key >= 0 && key < n_tokens_real) ? 0.0f : mask_value; + } + } + } + set_input_f32(attn_mask->name, mask_data); + + // local attention skew mask: zeroes out the probs that were + // computed for keys outside the valid sliding window. + if (struct ggml_tensor * local_mask = ggml_graph_get_tensor(gf, "local_mask")) { + const int lm_k = local_mask->ne[0]; + const int lm_q = local_mask->ne[1]; + const int window_size = lm_k - lm_q + 1; + std::vector lm_data(lm_q * lm_k); + for (int q = 0; q < lm_q; ++q) { + for (int k = 0; k < lm_k; ++k) { + const int rel = k - q; + lm_data[q * lm_k + k] = (rel >= 0 && rel < window_size) ? 1.0f : 0.0f; + } + } + set_input_f32(local_mask->name, lm_data); + } + + // Generate rotation frequencies for relative positional encoding. + { + const int n_state = hparams.n_embd; + const int d_half = n_state / 2; + const float log_10000 = logf(10000.0f); + std::vector freqs(d_half); + for (int k = 0; k < d_half; ++k) { + freqs[k] = expf(-(float(k * 2) * log_10000 / float(n_state))); + } + set_input_f32("pos_freqs", freqs); + } + + // Generate relative positional distance values which scaled by + // the frequency to produce the angles for sin/cos. + { + // window_size is only known after graph construction since it depends on + // n_time from the conv output, so we read it back from the graph tensor. + struct ggml_tensor * rel_pos = ggml_graph_get_tensor(gf, "rel_positions"); + const int window_size = rel_pos->ne[1]; + std::vector pos(window_size); + // local attention: window is fixed at [att_left, att_right] + // full attention: window covers the full sequence, centered + if (ggml_graph_get_tensor(gf, "local_mask")) { + const int att_left = window_size / 2; + for (int t = 0; t < window_size; ++t) { + pos[t] = float(att_left - t); + } + } else { + const int n_time = (window_size + 1) / 2; + for (int t = 0; t < window_size; ++t) { + pos[t] = float(n_time - 1 - t); + } + } + set_input_f32(rel_pos->name, pos); + } + } break; + case PROJECTOR_TYPE_GRANITE_SPEECH: + { + const int context_size = model.hparams.audio_chunk_size; + const int max_pos_emb = model.hparams.audio_max_pos_emb; + + std::vector dists((size_t) context_size * (size_t) context_size); + for (int i = 0; i < context_size; i++) { + for (int j = 0; j < context_size; j++) { + int d = i - j; + if (d < -context_size) d = -context_size; + if (d > context_size) d = context_size; + dists[(size_t) i * (size_t) context_size + (size_t) j] = d + max_pos_emb; + } + } + set_input_i32("attn_dists", dists); + + const int n_frames = image_size_width; + const int remainder = n_frames % context_size; + if (remainder > 0) { + const int num_blocks = (n_frames + context_size - 1) / context_size; + std::vector mask((size_t) context_size * (size_t) context_size * (size_t) num_blocks, 0.0f); + const float neg_inf = -INFINITY; + const size_t last_block_offset = (size_t) (num_blocks - 1) * (size_t) context_size * (size_t) context_size; + for (int q = 0; q < context_size; q++) { + for (int k = 0; k < context_size; k++) { + if (q >= remainder || k >= remainder) { + mask[last_block_offset + (size_t) q * (size_t) context_size + (size_t) k] = neg_inf; + } + } + } + set_input_f32("attn_mask", mask); + } + } break; + case PROJECTOR_TYPE_GRANITE4_VISION: + { + // Granite Vision 4.1 uses precomputed permutation index + // tensors to express the _win / _unwin / spatial sampling + // reshapes as ggml_get_rows gathers. The names are set + // by g4v_gather() in models/granite4-vision.cpp. + const int patch_size = model.hparams.patch_size; + const int image_side = imgs.entries.front().nx() / patch_size; + const int window_side = hparams.downsample_window_side; + const int query_side = hparams.downsample_query_side; + const int n = image_side / window_side; + const int new_side = n * query_side; + + // Builds the raster→window permutation indices for a + // (side, side) grid split into (n × n) windows of (win × win) + // tokens each. dst[w * win*win + p] = source raster index. + auto make_win_idx = [](int side, int win) { + const int nn = side / win; + std::vector idx(static_cast(side) * side); + for (int wy = 0; wy < nn; ++wy) { + for (int wx = 0; wx < nn; ++wx) { + for (int iy = 0; iy < win; ++iy) { + for (int ix = 0; ix < win; ++ix) { + const int w = wy * nn + wx; + const int p = iy * win + ix; + const int y = wy * win + iy; + const int x = wx * win + ix; + idx[static_cast(w) * (win*win) + p] = y * side + x; + } + } + } + } + return idx; + }; + + auto make_unwin_idx = [&](int side, int win) { + const std::vector fwd = make_win_idx(side, win); + std::vector inv(fwd.size()); + for (size_t i = 0; i < fwd.size(); ++i) { + inv[fwd[i]] = static_cast(i); + } + return inv; + }; + + auto make_spatial_idx = [](int side, int offset) { + const int off_y = (offset >> 1) & 1; + const int off_x = offset & 1; + const int new_s = side / 2; + std::vector idx(static_cast(new_s) * new_s); + for (int y = 0; y < new_s; ++y) { + for (int x = 0; x < new_s; ++x) { + idx[y * new_s + x] = (y * 2 + off_y) * side + (x * 2 + off_x); + } + } + return idx; + }; + + // the same permutation is applied to every tile of the stacked image + auto upload = [&](const std::string & name, const std::vector & idx) { + ggml_tensor * t = ggml_graph_get_tensor(gf, name.c_str()); + GGML_ASSERT(t); + GGML_ASSERT(ggml_nelements(t) % (int64_t) idx.size() == 0); + const int n_rep = ggml_nelements(t) / idx.size(); + std::vector buf; + buf.reserve(idx.size() * n_rep); + for (int i = 0; i < n_rep; ++i) { + buf.insert(buf.end(), idx.begin(), idx.end()); + } + ggml_backend_tensor_set(t, buf.data(), 0, ggml_nbytes(t)); + }; + + // Stage 1b only uses block 0's permutations; future stages + // will upload all blocks. + for (size_t bid = 0; bid < hparams.feature_layers.size(); ++bid) { + const std::string prefix = "g4v_blk" + std::to_string(bid) + "_"; + upload(prefix + "win_idx", make_win_idx(image_side, window_side)); + upload(prefix + "qwin_idx", make_win_idx(new_side, query_side)); + upload(prefix + "unwin_idx", make_unwin_idx(new_side, query_side)); + const auto spatial_offset = hparams.proj_spatial_offsets[bid]; + if (spatial_offset >= 0) { + upload(prefix + "spatial_idx", make_spatial_idx(image_side,spatial_offset)); + } + } + } break; + default: + GGML_ABORT("Unknown projector type"); + } + + mtmd_backend_cpu_set_n_threads(backends.backend_cpu, params->n_threads); + + auto status = ggml_backend_sched_graph_compute(backends.sched.get(), gf); + if (status != GGML_STATUS_SUCCESS) { + LOG_ERR("%s: ggml_backend_sched_graph_compute failed with error %d\n", __func__, status); + return false; + } + + if (embeddings != nullptr) { + // sanity check (assuming that all images in batch have the same number of tokens, so we only check the first one) + const int n_tokens_out = embeddings->ne[1]; + const int expected_n_tokens_out = model.n_output_tokens(&imgs.entries[0]); + if (n_tokens_out != expected_n_tokens_out) { + LOG_ERR("%s: expected output %d tokens, got %d\n", __func__, expected_n_tokens_out, n_tokens_out); + GGML_ABORT("Invalid number of output tokens"); + } + + LOG_DBG("%s: output embedding shape [%d, %d, %d]\n", __func__, + (int)embeddings->ne[0], (int)embeddings->ne[1], (int)embeddings->ne[2]); + + // copy output to user buffer if provided + // if output is empty, skip the copy + auto & out_batch_embd = *params->out_embd; + if (!out_batch_embd.empty()) { + if (out_batch_embd.size() != (size_t)ggml_nelements(embeddings)) { + LOG_ERR("%s: output buffer has %zu elements but expected %zu\n", __func__, out_batch_embd.size(), (size_t)ggml_nelements(embeddings)); + GGML_ABORT("Output buffer size mismatch"); + } + ggml_backend_tensor_get(embeddings, out_batch_embd.data(), 0, ggml_nbytes(embeddings)); + } else { + LOG_WRN("%s: output buffer is empty, skipping copy\n", __func__); + } + } + + // + // for audio gen models + // + + if (model.proj_type == PROJECTOR_TYPE_CHATTERBOX && params->gen_process == MTMD_SUBCTX_GEN_PROCESS_TOKENIZE) { + ggml_tensor * fsq = ggml_graph_get_tensor(gf, "out_fsq"); + GGML_ASSERT(fsq != nullptr && params->out_codes); + const int n_tok = (int) fsq->ne[1]; + std::vector h((size_t) 8 * n_tok); + ggml_backend_tensor_get(fsq, h.data(), 0, ggml_nbytes(fsq)); + + // fsq round to base 3: h in (-1, 1) maps to digits {0, 1, 2} + params->out_codes->resize(n_tok); + for (int t = 0; t < n_tok; t++) { + int32_t code = 0; + for (int i = 7; i >= 0; i--) { + const int32_t d = (int32_t) roundf(h[(size_t) t * 8 + i] * 0.9990000128746033f) + 1; + code = code * 3 + d; + } + (*params->out_codes)[(size_t) t] = code; + } + + // speech table rows of the codes, the multilingual variant adds its learned positions as its perceiver expects + if (params->out_code_embd) { + const auto & tensors = model.cbx_tensors; + auto tab_it = tensors.find("a.gen.code.out_embd.weight"); + GGML_ASSERT(tab_it != tensors.end()); + ggml_tensor * tab = tab_it->second; + const int n_e = (int) tab->ne[0]; + auto pos_it = tensors.find("a.gen.t3.speech_pos_emb"); + ggml_tensor * pos = pos_it == tensors.end() ? nullptr : pos_it->second; + if ((tab->type != GGML_TYPE_F16 && tab->type != GGML_TYPE_F32) || + (pos && (pos->type != GGML_TYPE_F32 || pos->ne[0] != n_e || pos->ne[1] < n_tok))) { + LOG_ERR("%s: unexpected speech table or positions\n", __func__); + return false; + } + + params->out_code_embd->resize((size_t) n_tok * n_e); + std::vector h16(n_e); + std::vector pr(n_e); + for (int t = 0; t < n_tok; t++) { + float * dst = params->out_code_embd->data() + (size_t) t * n_e; + if ((*params->out_codes)[(size_t) t] >= tab->ne[1]) { + LOG_ERR("%s: speech code beyond the speech table\n", __func__); + return false; + } + const size_t r = (size_t) (*params->out_codes)[(size_t) t] * n_e; + if (tab->type == GGML_TYPE_F16) { + ggml_backend_tensor_get(tab, h16.data(), r * sizeof(ggml_fp16_t), (size_t) n_e * sizeof(ggml_fp16_t)); + for (int j = 0; j < n_e; j++) { + dst[j] = ggml_fp16_to_fp32(h16[j]); + } + } else { + ggml_backend_tensor_get(tab, dst, r * sizeof(float), (size_t) n_e * sizeof(float)); + } + if (pos) { + ggml_backend_tensor_get(pos, pr.data(), (size_t) t * n_e * sizeof(float), (size_t) n_e * sizeof(float)); + for (int j = 0; j < n_e; j++) { + dst[j] += pr[j]; + } + } + } + } + return true; + } + + if (model.proj_type == PROJECTOR_TYPE_CHATTERBOX && params->gen_process == MTMD_SUBCTX_GEN_PROCESS_TTS_VOCODE) { + ggml_tensor * sp = ggml_graph_get_tensor(gf, "out_spec"); + GGML_ASSERT(sp != nullptr && params->out_spec); + params->out_spec->resize(ggml_nelements(sp)); + ggml_backend_tensor_get(sp, params->out_spec->data(), 0, ggml_nbytes(sp)); + return true; + } + + if (model.proj_type == PROJECTOR_TYPE_CHATTERBOX && params->gen_process == MTMD_SUBCTX_GEN_PROCESS_TTS) { + ggml_tensor * mel = ggml_graph_get_tensor(gf, "out_mel"); + GGML_ASSERT(mel != nullptr); + std::vector mel_data(ggml_nelements(mel)); + ggml_backend_tensor_get(mel, mel_data.data(), 0, ggml_nbytes(mel)); + + // hift: f0 -> harmonic source -> source stft on the host, then the vocoder graph and the istft + ggml_tensor * f0_t = ggml_graph_get_tensor(gf, "out_f0"); + GGML_ASSERT(f0_t != nullptr); + const int n_mel_out = (int) ggml_nelements(f0_t); + std::vector f0(n_mel_out); + ggml_backend_tensor_get(f0_t, f0.data(), 0, ggml_nbytes(f0_t)); + + // SineGen: f0 upsampled x480, 9 harmonics with cumulative phase, uv gating and noise, merged by l_linear + tanh + const int ups_total = 480; + const double sr = 24000.0; + const int64_t n_wav = (int64_t) n_mel_out * ups_total; + std::vector lw(9); + float lb = 0.0f; + if (model.cbx_read_tensor("a.gen.hift.m_source.l_linear.weight", lw.data(), lw.size()) != 9 || + model.cbx_read_tensor("a.gen.hift.m_source.l_linear.bias", &lb, 1) != 1) { + LOG_ERR("%s: unexpected hift source module\n", __func__); + return false; + } + + std::mt19937 & srng = params->gen_state->rng; + std::uniform_real_distribution ud(-M_PI, M_PI); + std::normal_distribution snd(0.0f, 1.0f); + double phase[9]; + for (int h = 0; h < 9; h++) { + phase[h] = h == 0 ? 0.0 : ud(srng); + } + std::vector src((size_t) n_wav); + double cum[9] = {0.0}; + for (int64_t t = 0; t < n_wav; t++) { + const float f = f0[(size_t) (t / ups_total)]; + const float uv = f > 10.0f ? 1.0f : 0.0f; // nsf_voiced_threshold + const float namp = uv * 0.003f + (1.0f - uv) * 0.1f / 3.0f; + float merged = lb; + for (int h = 0; h < 9; h++) { + cum[h] += (double) f * (h + 1) / sr; + cum[h] -= floor(cum[h]); + float sine = 0.1f * (float) sin(2.0 * M_PI * cum[h] + phase[h]); + sine = sine * uv + namp * snd(srng); + merged += lw[(size_t) h] * sine; + } + src[(size_t) t] = tanhf(merged); + } + + // stft of the source: n_fft 16, hop 4, hann window, centered + const int n_fft = 16, hop = 4, n_bins = 9; + std::vector win(n_fft); + for (int i = 0; i < n_fft; i++) win[(size_t) i] = 0.5f - 0.5f * cosf(2.0f * (float) M_PI * i / n_fft); + const int n_stft = (int) (n_wav / hop) + 1; + std::vector sstft((size_t) n_stft * 18); + for (int fr = 0; fr < n_stft; fr++) { + const int64_t c0 = (int64_t) fr * hop - n_fft / 2; // centered, reflect padded + for (int k = 0; k < n_bins; k++) { + double re = 0.0, im = 0.0; + for (int i = 0; i < n_fft; i++) { + int64_t idx = c0 + i; + if (idx < 0) idx = -idx; + if (idx >= n_wav) idx = 2 * (n_wav - 1) - idx; + const double v = (double) src[(size_t) idx] * win[(size_t) i]; + const double a = 2.0 * M_PI * k * i / n_fft; + re += v * cos(a); + im -= v * sin(a); + } + sstft[(size_t) fr * 18 + k ] = (float) re; + sstft[(size_t) fr * 18 + 9 + k] = (float) im; + } + } + + // vocoder graph on mel + source stft + std::vector spec; + { + mtmd_subctx_encode_params vp = *params; + vp.gen_process = MTMD_SUBCTX_GEN_PROCESS_TTS_VOCODE; + vp.mel_in = &mel_data; + vp.sstft_in = &sstft; + vp.vocode_n_mel = n_mel_out; + vp.vocode_n_stft = n_stft; + vp.out_audio = nullptr; + vp.out_spec = &spec; + if (!encode(&vp)) { + LOG_ERR("%s: vocoder stage failed\n", __func__); + return false; + } + } + + // istft: mag = clipped exp, phase = sin, hann overlap-add + const int n_frames_out = (int) (spec.size() / 18); + const int64_t n_out = (int64_t) (n_frames_out - 1) * hop; + std::vector acc((size_t) n_out + n_fft, 0.0); + std::vector wsum((size_t) n_out + n_fft, 0.0); + for (int fr = 0; fr < n_frames_out; fr++) { + double frame[16]; + for (int i = 0; i < n_fft; i++) { + double v = 0.0; + for (int k = 0; k < n_bins; k++) { + const double mag = fmin(exp((double) spec[(size_t) fr * 18 + k]), 1e2); + const double ph = sin((double) spec[(size_t) fr * 18 + 9 + k]); + const double re = mag * cos(ph), im = mag * sin(ph); + const double a = 2.0 * M_PI * k * i / n_fft; + const double w = (k == 0 || k == n_fft / 2) ? 1.0 : 2.0; + v += w * (re * cos(a) - im * sin(a)); + } + frame[i] = v / n_fft; + } + const int64_t o = (int64_t) fr * hop; + for (int i = 0; i < n_fft; i++) { + acc [(size_t) (o + i)] += frame[i] * win[(size_t) i]; + wsum[(size_t) (o + i)] += (double) win[(size_t) i] * win[(size_t) i]; + } + } + GGML_ASSERT(params->out_audio); + auto & out_audio = *params->out_audio; + out_audio.resize((size_t) std::min(n_out - n_fft / 2, n_wav)); + for (size_t i = 0; i < out_audio.size(); i++) { + const size_t j = i + n_fft / 2; // drop the centering pad + const double v = wsum[j] > 1e-11 ? acc[j] / wsum[j] : 0.0; + out_audio[(size_t) i] = (float) fmax(-0.99, fmin(0.99, v)); + } + + // trim_fade of the reference: silence 20 ms then fade in 20 ms, hides the artifact of the prompt boundary + const size_t n_trim = 24000 / 50; + for (size_t i = 0; i < 2 * n_trim && i < out_audio.size(); i++) { + const double g = i < n_trim ? 0.0 + : (cos(M_PI * (1.0 - (double) (i - n_trim) / n_trim)) + 1.0) / 2.0; + out_audio[(size_t) i] *= (float) g; + } + return true; + } + + // optional outputs: a pipeline yields codes or feats, and not all have an eos head + if (params->out_codes != nullptr) { + ggml_tensor * codes = ggml_graph_get_tensor(gf, "out_codes"); + if (codes != nullptr) { + auto & out_codes = *params->out_codes; + out_codes.resize(ggml_nelements(codes)); + ggml_backend_tensor_get(codes, out_codes.data(), 0, ggml_nbytes(codes)); + } + } + if (params->out_feats != nullptr) { + ggml_tensor * feats = ggml_graph_get_tensor(gf, "out_feats"); + if (feats != nullptr) { + auto & out_feats = *params->out_feats; + out_feats.resize(ggml_nelements(feats)); + ggml_backend_tensor_get(feats, out_feats.data(), 0, ggml_nbytes(feats)); + } + } + if (params->out_is_eos != nullptr) { + ggml_tensor * eos = ggml_graph_get_tensor(gf, "out_eos_score"); + if (eos != nullptr) { + GGML_ASSERT(ggml_nelements(eos) == 1); + float score = 0.0f; + ggml_backend_tensor_get(eos, &score, 0, sizeof(float)); + *params->out_is_eos = score > hparams.gen_eos_threshold; + } + } + if (params->out_audio != nullptr) { + ggml_tensor * audio = ggml_graph_get_tensor(gf, "out_audio"); + if (audio == nullptr) { + GGML_ABORT("out_audio requested but graph has no \"out_audio\" tensor"); + } + auto & out_audio = *params->out_audio; + out_audio.resize(ggml_nelements(audio)); + ggml_backend_tensor_get(audio, out_audio.data(), 0, ggml_nbytes(audio)); + + if (model.proj_type == PROJECTOR_TYPE_SOPRANO) { + std::vector spectrum = std::move(out_audio); + mtmd_graph_soprano::decode_spectrum(model, spectrum, out_audio); + } + + // drop the tail audio that comes from the code-0 rear padding, drama3 has no padding + if (model.proj_type != PROJECTOR_TYPE_DRAMA3_GEN) { + const int64_t n_codes = params->codes ? model.gen_code_head_w->ne[2] + 1 : 0; + const int64_t n_frames_w = hparams.wav_tfm_swa; + const int64_t n_frames = params->codes ? (int64_t) params->codes->size() / n_codes : n_frames_w; + if (n_frames < n_frames_w) { + const size_t hop = out_audio.size() / n_frames_w; + out_audio.resize((size_t) n_frames * hop); + } + } + } + if (params->enc_state != nullptr && (model.proj_type == PROJECTOR_TYPE_LFM3A || model.proj_type == PROJECTOR_TYPE_GEMMA4A)) { + const auto & slots = params->enc_state->slots; + size_t i = 0; + for (const auto & slot : list_enc_state_slots(hparams, model)) { + ggml_tensor * t = ggml_graph_get_tensor(gf, ("state_out_" + slot.name).c_str()); + GGML_ASSERT(t != nullptr); + copy_gen_state_out(t, slots[i++]); + } + } else if (params->enc_state != nullptr && model.proj_type == PROJECTOR_TYPE_NEMOTRON_TRANSCRIBE) { + mtmd_subctx_enc_state * st = params->enc_state; + ggml_tensor * chunk = ggml_graph_get_tensor(gf, "spk_chunk"); + ggml_tensor * preds = ggml_graph_get_tensor(gf, "spk_preds"); + GGML_ASSERT(chunk != nullptr && preds != nullptr); + st->spk_chunk.resize(ggml_nelements(chunk)); + st->spk_preds.resize(ggml_nelements(preds)); + ggml_backend_tensor_get(chunk, st->spk_chunk.data(), 0, ggml_nbytes(chunk)); + ggml_backend_tensor_get(preds, st->spk_preds.data(), 0, ggml_nbytes(preds)); + } else if (params->enc_state != nullptr) { + mtmd_subctx_enc_state * st = params->enc_state; + ggml_tensor * tail = ggml_graph_get_tensor(gf, "out_stem_tail"); + GGML_ASSERT(tail != nullptr); + st->stem_tail.resize(ggml_nelements(tail)); + ggml_backend_tensor_get(tail, st->stem_tail.data(), 0, ggml_nbytes(tail)); + + const int n_new = (imgs.entries[0].nx() - 2) / 2; + for (int i = 0; i < n_new; i++) { + st->slot_pos[(st->n_past + i) % st->n_ctx] = st->n_past + i; + } + st->n_past += n_new; + } + + if (params->gen_process == MTMD_SUBCTX_GEN_PROCESS_GEN_WAV) { + const auto & slots = params->gen_state->slots; + size_t i = 0; + for (const auto & slot : list_gen_state_slots(hparams, model)) { + if (slot.in_place) { + i++; + continue; + } + ggml_tensor * t = ggml_graph_get_tensor(gf, ("state_out_" + slot.name).c_str()); + if (t == nullptr) { + GGML_ABORT("graph has no \"state_out_%s\" tensor", slot.name.c_str()); + } + GGML_ASSERT(i < slots.size()); + copy_gen_state_out(t, slots[i++]); + } + } + + // + // Debug: dump final embeddings if MTMD_DEBUG_EMBEDDINGS is set + // + + if (debug_output_embeddings && embeddings != nullptr) { + const int64_t n_embd = embeddings->ne[0]; + const int64_t n_tokens = embeddings->ne[1]; + std::vector emb_data(ggml_nelements(embeddings)); + ggml_backend_tensor_get(embeddings, emb_data.data(), 0, ggml_nbytes(embeddings)); + + LOG_INF("\n=== MTMD_DEBUG_EMBEDDINGS ===\n"); + LOG_INF("Shape: [%lld, %lld]\n", (long long)n_embd, (long long)n_tokens); + + // TEMP debugging (parity validation), will be removed before merge + // when the env var holds a path, dump the raw data: [int32 n_tokens][int32 n_embd][f32 data] + const char * dump_path = std::getenv("MTMD_DEBUG_EMBEDDINGS"); + if (dump_path && strcmp(dump_path, "1") != 0) { + FILE * f = fopen(dump_path, "wb"); + if (f) { + const int32_t hdr[2] = { (int32_t)n_tokens, (int32_t)n_embd }; + fwrite(hdr, sizeof(hdr), 1, f); + fwrite(emb_data.data(), sizeof(float), emb_data.size(), f); + fclose(f); + } + } + + // Print first few values of first token + LOG_INF("Token 0 (first 16 values): "); + for (int i = 0; i < std::min((int64_t)16, n_embd); i++) { + LOG_INF("%.6f ", emb_data[i]); + } + LOG_INF("\n"); + + // Print last few values of first token + if (n_embd > 16) { + LOG_INF("Token 0 (last 16 values): "); + for (int64_t i = n_embd - 16; i < n_embd; i++) { + LOG_INF("%.6f ", emb_data[i]); + } + LOG_INF("\n"); + } + + // Compute and print statistics + float sum = 0.0f, sum_sq = 0.0f, min_val = emb_data[0], max_val = emb_data[0]; + for (size_t i = 0; i < emb_data.size(); i++) { + sum += emb_data[i]; + sum_sq += emb_data[i] * emb_data[i]; + min_val = std::min(min_val, emb_data[i]); + max_val = std::max(max_val, emb_data[i]); + } + float mean = sum / emb_data.size(); + float variance = (sum_sq / emb_data.size()) - (mean * mean); + LOG_INF("Stats: mean=%.6f, std=%.6f, min=%.6f, max=%.6f, sum=%.6f\n", + mean, sqrtf(variance), min_val, max_val, sum); + LOG_INF("=== END MTMD_DEBUG_EMBEDDINGS ===\n\n"); + } + + return true; +} diff --git a/tools/mtmd/mtmd-subcontext.h b/tools/mtmd/mtmd-subcontext.h new file mode 100644 index 000000000000..7e6d8232a23b --- /dev/null +++ b/tools/mtmd/mtmd-subcontext.h @@ -0,0 +1,204 @@ +#pragma once + +#include "ggml.h" +#include "ggml-backend.h" +#include "ggml-cpp.h" +#include "mtmd-impl.h" +#include "mtmd-model.h" + +#include +#include + +#include +#include +#include +#include + +// !!! Internal header, to be used by mtmd only !!! + +// runs the graphs of one mtmd_model on the shared backends and scheduler + +enum mtmd_flash_attn_type { + MTMD_FLASH_ATTN_TYPE_AUTO = -1, + MTMD_FLASH_ATTN_TYPE_DISABLED = 0, + MTMD_FLASH_ATTN_TYPE_ENABLED = 1, +}; + +struct mtmd_subctx_params { + enum mtmd_flash_attn_type flash_attn_type; + bool warmup; +}; + +// the device the weights and the graphs are placed on, nullptr means CPU +ggml_backend_dev_t mtmd_select_device(bool use_gpu, ggml_backend_dev_t device); + +// shared by the subcontexts, not thread-safe +struct mtmd_backends { + ggml_backend_t backend = nullptr; // backend_cpu if no accelerator + ggml_backend_t backend_cpu = nullptr; + + // for ggml_backend_sched_new(), backend_cpu last + std::vector ptrs; + std::vector bufts; + + std::shared_ptr threadpool; // can be nullptr + + // its buffers fit the largest graph reserved by the subcontexts + ggml_backend_sched_ptr sched; + + // device nullptr means CPU + mtmd_backends(ggml_backend_dev_t device, int n_threads, int max_nodes, + ggml_backend_sched_eval_callback cb_eval, void * cb_eval_user_data); + ~mtmd_backends(); + + std::map get_mem_compute() const; + + mtmd_backends(const mtmd_backends &) = delete; + mtmd_backends & operator=(const mtmd_backends &) = delete; +}; + +// compute the log-mel of n_frames frames, the waveform must already be padded and window must hold one whole frame +// out has the layout [n_frames, n_mel]; must not run at the same time as another pass +struct mtmd_audio_mel_config; +bool mtmd_audio_log_mel(const mtmd_audio_mel_config & config, const float * samples, const float * window, int64_t n_frames, float * out); + +enum mtmd_subctx_gen_process { + MTMD_SUBCTX_GEN_PROCESS_GEN_UNKNOWN, + MTMD_SUBCTX_GEN_PROCESS_GEN_CODE, // h_state to codes + MTMD_SUBCTX_GEN_PROCESS_GEN_WAV, // codes to raw PCM audio + MTMD_SUBCTX_GEN_PROCESS_TTS, // full utterance of codes to raw PCM audio + MTMD_SUBCTX_GEN_PROCESS_TTS_VOCODE, // internal: mel + source stft to istft input + MTMD_SUBCTX_GEN_PROCESS_TOKENIZE, // raw PCM audio to semantic speech tokens +}; + +// persistent state of a causal audio encoder, for streaming input (one per stream) +// it holds the sliding window of past keys/values, so each encode() call only takes the new frames +struct mtmd_subctx_enc_state { + int32_t n_ctx = 0; // ring slots: attention window + frames per call + int32_t n_past = 0; // absolute position of the next frame + + std::vector slot_pos; // absolute position held by each slot, -1 if empty + std::vector stem_tail; // last conv stem output frame, empty on a cold start + + ggml_context_ptr ctx_data; + ggml_backend_buffer_ptr buf; + std::vector k; // per layer [n_embd_kv, n_ctx] + std::vector v; + + // lfm3a: same order as list_lfm3a_state_slots(), F32 [ne0, ne1], updated in place + std::vector slots; + + // nemotron transcribe: the window's normalized log-mel, cache and FIFO rows in; its speaker rows and predictions out + std::vector mel_norm; + std::vector spk_prev; + std::vector spk_chunk, spk_preds; +}; + +// per-sequence state of audio generation: RNG and the slots kept between GEN_WAV calls +struct mtmd_subctx_gen_state { + std::mt19937 rng; + + ggml_context_ptr ctx_data; + ggml_backend_buffer_ptr buf; + std::vector slots; // same order as list_gen_state_slots(), F32 [ne0, ne1] + + // clears the slots and keeps the RNG stream + void reset() { + if (buf) { + ggml_backend_buffer_clear(buf.get(), 0); + } + } +}; + +struct mtmd_subctx_encode_params { + int n_threads = 1; + const mtmd_image_f32_batch * imgs = nullptr; + std::vector * out_embd = nullptr; + + // for audio gen, imgs has exactly one entry: hidden state from backbone (GEN_CODE) or unused (GEN_WAV) + mtmd_subctx_gen_process gen_process = MTMD_SUBCTX_GEN_PROCESS_GEN_UNKNOWN; + + // GEN_CODE: out_embd receives the embd to feed back to the backbone + int32_t code0 = 0; // semantic code sampled by the backbone + int32_t top_k = 50; + float top_p = 1.0f; + std::vector * out_codes = nullptr; // this frame's 16 sampled codes + std::vector * out_feats = nullptr; // continuous counterpart of out_codes + float temp = 0.0f; // sampling temperature, noise scale for flow-matching decoders + bool * out_is_eos = nullptr; + + // TOKENIZE: speech embedding rows of the produced codes + std::vector * out_code_embd = nullptr; + + // GEN_WAV + const std::vector * codes = nullptr; // this frame's 16 RVQ codes + const std::vector * feats = nullptr; // continuous counterpart of codes + std::vector * out_audio = nullptr; // decoded PCM samples, F32 + const mtmd_gen_ref * ref = nullptr; // TTS reference conditioning of the voice + // TTS_VOCODE internal stage inputs + const std::vector * mel_in = nullptr; + const std::vector * sstft_in = nullptr; + int vocode_n_mel = 0; + int vocode_n_stft = 0; + std::vector * out_spec = nullptr; + + // audio gen: state of one sequence, encode() updates it in place + struct mtmd_subctx_gen_state * gen_state = nullptr; + + // streaming audio input, updated in place by encode() + struct mtmd_subctx_enc_state * enc_state = nullptr; +}; + +struct mtmd_subcontext { + const mtmd_model & model; + mtmd_backends & backends; + + std::vector buf_compute_meta; + + int max_nodes = 8192; + mtmd_flash_attn_type flash_attn_type = MTMD_FLASH_ATTN_TYPE_AUTO; + bool is_allocated = false; + + bool debug_output_embeddings = false; + + bool support_batch = false; + + // model and backends must outlive the subcontext + mtmd_subcontext(const mtmd_model & model, mtmd_backends & backends, const mtmd_subctx_params & params); + + bool encode(mtmd_subctx_encode_params * params); + + // reserve the compute buffers for the given batch + void warmup(const mtmd_image_f32_batch & batch); + + // TODO: remove encode_image() and always use batched version + bool encode_image(int n_threads, const mtmd_image_f32 * img, std::vector & out_vec); + bool encode_batch(int n_threads, const mtmd_image_f32_batch * imgs, std::vector & out_batch_embd); + + // n_frames_max is the maximum number of encoder frames per encode() call, free the result with delete + mtmd_subctx_enc_state * enc_state_init(int32_t n_frames_max); + + // seed UINT32_MAX means random, free the result with delete + mtmd_subctx_gen_state * gen_state_init(uint32_t seed); + +private: + struct support_info_op { + ggml_tensor * op; + + // true if the op runs on the accelerated backend + bool is_accel = true; + }; + + struct support_info_graph { + // whether the backend supports flash attention + bool fattn = true; + ggml_tensor * fattn_op = nullptr; // for debugging + + std::vector ops; + }; + + mtmd_image_f32_batch get_dummy_batch() const; + + // only initialize backend buffers, but do not allocate them yet + support_info_graph reserve_compute_meta(const mtmd_image_f32_batch & batch); +}; diff --git a/tools/mtmd/mtmd.cpp b/tools/mtmd/mtmd.cpp index d4ea2385258c..9b45d3de312a 100644 --- a/tools/mtmd/mtmd.cpp +++ b/tools/mtmd/mtmd.cpp @@ -1,5 +1,6 @@ -#include "clip.h" -#include "clip-impl.h" +#include "mtmd-subcontext.h" +#include "models/models.h" +#include "mtmd-impl.h" #include "mtmd.h" #include "mtmd-internal.h" #include "mtmd-audio.h" @@ -27,9 +28,25 @@ #include #include #include +#include #include #include +std::string mtmd_format(const char * fmt, ...) { + va_list ap; + va_list ap2; + va_start(ap, fmt); + va_copy(ap2, ap); + int size = vsnprintf(NULL, 0, fmt, ap); + GGML_ASSERT(size >= 0 && size < INT_MAX); // NOLINT + std::vector buf(size + 1); + int size2 = vsnprintf(buf.data(), size + 1, fmt, ap2); + GGML_ASSERT(size2 == size); + va_end(ap2); + va_end(ap); + return std::string(buf.data(), size); +} + // remember to bump this if the serialization format changes #define MTMD_SERIALIZATION_VERSION 4 @@ -106,10 +123,10 @@ std::string mtmd_serialization::read() { return str; } -// only mtmd.cpp needs these, so they're implemented here rather than in clip-impl.h -void clip_image_f32::serialize(mtmd_serialization & ser) const { +// only mtmd.cpp needs these, so they're implemented here rather than in mtmd-impl.h +void mtmd_image_f32::serialize(mtmd_serialization & ser) const { // remember to bump MTMD_SERIALIZATION_VERSION if this is changed - // note: buf is intentionally NOT serialized; the loaded clip_image_f32 will always be a placeholder + // note: buf is intentionally NOT serialized; the loaded mtmd_image_f32 will always be a placeholder ser.write(add_viewsep); ser.write(add_newline); ser.write(lead_pad); @@ -121,16 +138,16 @@ void clip_image_f32::serialize(mtmd_serialization & ser) const { ser.write((int32_t)nx_); ser.write((int32_t)ny_); } -void clip_image_f32::deserialize(mtmd_serialization & ser) { +void mtmd_image_f32::deserialize(mtmd_serialization & ser) { add_viewsep = ser.read(); add_newline = ser.read(); lead_pad = ser.read(); if (ser.version >= 3) { const int32_t suffix_raw = ser.read(); - if (suffix_raw < 0 || suffix_raw >= CLIP_SUFFIX_COUNT) { + if (suffix_raw < 0 || suffix_raw >= MTMD_SUFFIX_COUNT) { throw std::runtime_error("invalid suffix type"); } - suffix_type = (clip_suffix_type)suffix_raw; + suffix_type = (mtmd_suffix_type)suffix_raw; anyres.grid_x = ser.read(); anyres.grid_y = ser.read(); anyres.orig_nx = ser.read(); @@ -141,7 +158,7 @@ void clip_image_f32::deserialize(mtmd_serialization & ser) { buf.clear(); // always a placeholder after loading } -void clip_image_f32_batch::serialize(mtmd_serialization & ser) const { +void mtmd_image_f32_batch::serialize(mtmd_serialization & ser) const { // remember to bump MTMD_SERIALIZATION_VERSION if this is changed ser.write(is_audio); ser.write(entries.size()); @@ -149,7 +166,7 @@ void clip_image_f32_batch::serialize(mtmd_serialization & ser) const { entry.serialize(ser); } } -void clip_image_f32_batch::deserialize(mtmd_serialization & ser) { +void mtmd_image_f32_batch::deserialize(mtmd_serialization & ser) { is_audio = ser.read(); uint64_t n = ser.read(); constexpr size_t min_entry_bytes = sizeof(uint8_t) * 2 + sizeof(int32_t) * 2; @@ -159,7 +176,7 @@ void clip_image_f32_batch::deserialize(mtmd_serialization & ser) { entries.clear(); entries.reserve(n); for (uint64_t i = 0; i < n; i++) { - clip_image_f32 entry; + mtmd_image_f32 entry; entry.deserialize(ser); entries.push_back(std::move(entry)); } @@ -176,6 +193,7 @@ struct mtmd_bitmap { std::string id; // optional user-defined id, for ex: can be set to image hash, useful for KV cache tracking bool is_audio = false; // true if the bitmap is audio bool is_voice = false; + std::shared_ptr gen_ref; // gen-side reference of a predefined voice, if any bool mergeable = false; // [QWEN_VIDEO] set only on frames of the same video bool is_video = false; // see mtmd_bitmap_set_is_video() @@ -201,9 +219,14 @@ struct mtmd_bitmap { } } - static mtmd_bitmap * init_voice(const std::string & name, const float * rows, uint32_t n_embd, uint32_t n_rows) { + static mtmd_bitmap * init_voice(const std::string & name, const mtmd_gen_voice & voice, uint32_t n_embd) { + const float * rows = voice.rows.data(); + const uint32_t n_rows = voice.rows.size() / n_embd; auto * bitmap = new mtmd_bitmap(nullptr, n_embd, n_rows); bitmap->is_voice = true; + if (!voice.ref.tokens.empty()) { + bitmap->gen_ref = std::make_shared(voice.ref); + } bitmap->id = name; bitmap->data.assign((const unsigned char *) rows, (const unsigned char *) (rows + (size_t) n_embd * n_rows)); return bitmap; @@ -262,7 +285,7 @@ struct mtmd_image_tokens { uint32_t n = 0; for (size_t k = 0; k < batch_f32.entries.size(); ++k) { const auto [gw, gh] = canvas_tile_grid(k); - n += gw * gh + (uint32_t) clip_suffix_rows(batch_f32.entries[k].suffix_type).size(); + n += gw * gh + (uint32_t) mtmd_suffix_rows(batch_f32.entries[k].suffix_type).size(); } return n; } @@ -278,7 +301,7 @@ struct mtmd_image_tokens { } return nx * ny * nz; } - clip_image_f32_batch batch_f32; // preprocessed image patches + mtmd_image_f32_batch batch_f32; // preprocessed image patches std::string id; // optional user-defined ID, useful for KV cache tracking // true if one of entries in batch_f32 is a placeholder @@ -342,25 +365,36 @@ struct mtmd_image_tokens { }; using mtmd_image_tokens_ptr = std::unique_ptr; +// the mel of a clip is computed by its first encode, its chunks share the result +using mtmd_audio_source_ptr = std::shared_ptr; + // an audio track split across chunks (e.g. the soundtrack of a video): it is encoded once, as one clip, and every // chunk takes its rows of it struct mtmd_audio_track { - clip_image_f32_batch batch_f32; // the mel of the whole track + mtmd_image_f32_batch batch_f32; // the mel of the whole track + mtmd_audio_source_ptr source; uint32_t n_tokens = 0; std::vector embd; // filled by the first encode + std::shared_ptr gen_ref; // gen-side reference of a predefined voice, if any }; struct mtmd_audio_tokens { uint32_t n_tokens = 0; // number of tokens - clip_image_f32_batch batch_f32; // preprocessed image patches + mtmd_image_f32_batch batch_f32; // preprocessed image patches std::string id; // optional user-defined ID, useful for KV cache tracking + mtmd_audio_source_ptr source; // if set, batch_f32 has the shape of the mel of chunk source_idx + uint32_t source_idx = 0; + // a slice of a track: the rows [row_begin, row_begin + n_tokens) of it, batch_f32 is unused std::shared_ptr track; uint32_t row_begin = 0; // true if one of entries in batch_f32 is a placeholder bool is_placeholder() const { + if (source || (track && track->source)) { + return false; + } if (track && track->batch_f32.entries.empty()) { return track->embd.empty(); // a voice: its rows are not serialized } @@ -377,6 +411,8 @@ struct mtmd_audio_tokens { n_tokens, batch_f32.clone(), id, + source, + source_idx, track, row_begin, }; @@ -406,6 +442,7 @@ struct mtmd_audio_tokens { track->n_tokens = ser.read(); track->batch_f32.deserialize(ser); } + source.reset(); } }; using mtmd_audio_tokens_ptr = std::unique_ptr; @@ -534,13 +571,13 @@ const char * mtmd_default_marker() { return "<__media__>"; } -static clip_flash_attn_type mtmd_get_clip_flash_attn_type(enum llama_flash_attn_type flash_attn_type) { +static mtmd_flash_attn_type mtmd_to_flash_attn_type(enum llama_flash_attn_type flash_attn_type) { switch (flash_attn_type) { - case LLAMA_FLASH_ATTN_TYPE_AUTO: return CLIP_FLASH_ATTN_TYPE_AUTO; - case LLAMA_FLASH_ATTN_TYPE_DISABLED: return CLIP_FLASH_ATTN_TYPE_DISABLED; - case LLAMA_FLASH_ATTN_TYPE_ENABLED: return CLIP_FLASH_ATTN_TYPE_ENABLED; + case LLAMA_FLASH_ATTN_TYPE_AUTO: return MTMD_FLASH_ATTN_TYPE_AUTO; + case LLAMA_FLASH_ATTN_TYPE_DISABLED: return MTMD_FLASH_ATTN_TYPE_DISABLED; + case LLAMA_FLASH_ATTN_TYPE_ENABLED: return MTMD_FLASH_ATTN_TYPE_ENABLED; } - return CLIP_FLASH_ATTN_TYPE_AUTO; + return MTMD_FLASH_ATTN_TYPE_AUTO; } mtmd_context_params mtmd_context_params_default() { @@ -564,13 +601,28 @@ mtmd_context_params mtmd_context_params_default() { return params; } +// chatterbox multilingual has a perceiver in its conditioning encoder (speaker encoding context) +static bool cbx_is_mtl(const mtmd_model & model) { + return model.cbx_read_tensor("a.cenc.perceiver.pre_attention_query", nullptr, 0) > 0; +} + struct mtmd_context { - struct clip_ctx * ctx_v; // vision - struct clip_ctx * ctx_a; // audio + // must outlive the subcontexts below + mtmd_models models; + std::unique_ptr backends; // shared by the subcontexts + + // held by every public function that runs a graph, the internals never lock + std::mutex compute_mutex; + + std::unique_ptr ctx_v; // vision + std::unique_ptr ctx_a; // audio std::vector out_embd; // image embedding vector // generation context - struct clip_ctx * ctx_gen_a; // audio + std::unique_ptr ctx_gen_a; // audio + + // gen-side reference data of the last encoded chunk (chatterbox), see mtmd_gen_state_set_ref() + mtmd_gen_ref gen_ref; bool print_timings; int n_threads; @@ -603,7 +655,10 @@ struct mtmd_context { bool ov_img_first = false; // MiniCPM-V 4.6/4.7 prepends an N tag before - bool use_image_id = false; + bool has_prompt_img_id = false; + + // Kimi-K3.1 writes the original image size after img_beg + bool has_prompt_img_size = false; // string template for slice image delimiters with row/col (idefics3) std::string sli_img_start_tmpl; @@ -651,39 +706,68 @@ struct mtmd_context { pos_type = MTMD_POS_TYPE_MROPE; } break; default: - throw std::runtime_error(string_format("unsupported decoder rope type: %d\n", decoder_rope_type)); + throw std::runtime_error(mtmd_format("unsupported decoder rope type: %d\n", decoder_rope_type)); } } - clip_context_params ctx_clip_params { - /* use_gpu */ ctx_params.use_gpu, - /* device */ ctx_params.device, - /* flash_attn_type */ mtmd_get_clip_flash_attn_type(ctx_params.flash_attn_type), - /* image_min_tokens */ ctx_params.image_min_tokens, - /* image_max_tokens */ ctx_params.image_max_tokens, - /* warmup */ ctx_params.warmup, - /* cb_eval */ ctx_params.cb_eval, - /* cb_eval_user_data */ ctx_params.cb_eval_user_data, - /* no_alloc */ no_alloc, - /* progress_callback */ ctx_params.progress_callback, - /* progress_callback_user_data */ ctx_params.progress_callback_user_data, - }; + ggml_backend_dev_t device = mtmd_select_device(ctx_params.use_gpu, ctx_params.device); + + mtmd_model_params model_params; + model_params.image_min_tokens = ctx_params.image_min_tokens; + model_params.image_max_tokens = ctx_params.image_max_tokens; + model_params.no_alloc = no_alloc; + model_params.progress_callback = ctx_params.progress_callback; + model_params.progress_callback_user_data = ctx_params.progress_callback_user_data; + + ggml_backend_buffer_type_t buft = device + ? ggml_backend_dev_buffer_type(device) + : ggml_backend_dev_buffer_type(ggml_backend_dev_by_type(GGML_BACKEND_DEVICE_TYPE_CPU)); + models = mtmd_model_load(mmproj_fname, model_params, buft); - auto res = clip_init(mmproj_fname, ctx_clip_params); - ctx_v = res.ctx_v; - ctx_a = res.ctx_a; - ctx_gen_a = res.ctx_gen_a; // soprano and pocket-tts without voice cloning have no encoder - if (!ctx_v && !ctx_a && !ctx_gen_a) { - throw std::runtime_error(string_format("Failed to load CLIP model from %s\n", mmproj_fname)); + if (!models.v && !models.a && !models.gen_a) { + throw std::runtime_error(mtmd_format("Failed to load mmproj model from %s\n", mmproj_fname)); + } + + int max_nodes = 0; + for (const auto * m : { models.v.get(), models.a.get(), models.gen_a.get() }) { + if (m) { + max_nodes = std::max(max_nodes, m->max_nodes); + } + } + + // no_alloc only measures memory, nothing is computed + backends = std::make_unique(device, no_alloc ? 1 : ctx_params.n_threads, max_nodes, + ctx_params.cb_eval, ctx_params.cb_eval_user_data); + + // like llama_context::sched_reserve(), each warmup reserves its graph on the shared scheduler + mtmd_subctx_params sctx_params { + /* flash_attn_type */ mtmd_to_flash_attn_type(ctx_params.flash_attn_type), + /* warmup */ ctx_params.warmup, + }; + if (models.v) { + ctx_v = std::make_unique(*models.v, *backends, sctx_params); + } + if (models.a) { + ctx_a = std::make_unique(*models.a, *backends, sctx_params); + } + if (models.gen_a) { + ctx_gen_a = std::make_unique(*models.gen_a, *backends, sctx_params); + } + for (size_t i = 0; i < backends->ptrs.size(); ++i) { + const size_t size = ggml_backend_sched_get_buffer_size(backends->sched.get(), backends->ptrs[i]); + if (size > 1) { + LOG_INF("%s: %10s compute buffer size = %8.2f MiB\n", __func__, + ggml_backend_buft_name(backends->bufts[i]), size / 1024.0 / 1024.0); + } } // if both vision and audio mmproj are present, we need to validate their n_embd if (ctx_v && ctx_a) { - int n_embd_v = clip_n_mmproj_embd(ctx_v); - int n_embd_a = clip_n_mmproj_embd(ctx_a); + int n_embd_v = ctx_v->model.n_mmproj_embd(); + int n_embd_a = ctx_a->model.n_mmproj_embd(); if (n_embd_v != n_embd_a) { - throw std::runtime_error(string_format( + throw std::runtime_error(mtmd_format( "mismatch between vision and audio mmproj (n_embd_v = %d, n_embd_a = %d)\n", n_embd_v, n_embd_a)); } @@ -692,17 +776,17 @@ struct mtmd_context { // since we already validate n_embd of vision and audio mmproj, // we can safely assume that they are the same // a gen-only mmproj is validated below - int n_embd_clip = ctx_v || ctx_a ? clip_n_mmproj_embd(ctx_v ? ctx_v : ctx_a) : n_embd_text; - if (n_embd_text > 0 && n_embd_text != n_embd_clip) { - throw std::runtime_error(string_format( + int n_embd_mmproj = ctx_v || ctx_a ? (ctx_v ? ctx_v : ctx_a)->model.n_mmproj_embd() : n_embd_text; + if (n_embd_text > 0 && n_embd_text != n_embd_mmproj) { + throw std::runtime_error(mtmd_format( "mismatch between text model (n_embd = %d) and mmproj (n_embd = %d)\n" "hint: you may be using wrong mmproj\n", - n_embd_text, n_embd_clip)); + n_embd_text, n_embd_mmproj)); } if (ctx_gen_a) { - int n_embd_gen = clip_n_mmproj_embd(ctx_gen_a); + int n_embd_gen = ctx_gen_a->model.n_mmproj_embd(); if (n_embd_text > 0 && n_embd_text != n_embd_gen) { - throw std::runtime_error(string_format( + throw std::runtime_error(mtmd_format( "mismatch between text model (n_embd = %d) and gen-audio mmproj (n_embd = %d)\n" "hint: you may be using wrong mmproj\n", n_embd_text, n_embd_gen)); @@ -720,7 +804,7 @@ struct mtmd_context { GGML_ASSERT(ctx_v != nullptr); image_preproc.reset(); - projector_type proj = clip_get_projector_type(ctx_v); + projector_type proj = ctx_v->model.proj_type; switch (proj) { case PROJECTOR_TYPE_MLP: @@ -731,16 +815,16 @@ struct mtmd_context { case PROJECTOR_TYPE_JANUS_PRO: case PROJECTOR_TYPE_GLM_EDGE: { - bool has_pinpoints = !clip_get_hparams(ctx_v)->image_res_candidates.empty(); + bool has_pinpoints = !ctx_v->model.hparams.image_res_candidates.empty(); if (has_pinpoints) { - image_preproc = std::make_unique(ctx_v); + image_preproc = std::make_unique(models.v.get()); } else { - image_preproc = std::make_unique(ctx_v); + image_preproc = std::make_unique(models.v.get()); } } break; case PROJECTOR_TYPE_MINICPMV: { - int minicpmv_version = clip_get_hparams(ctx_v)->minicpmv_version; + int minicpmv_version = ctx_v->model.hparams.minicpmv_version; if (minicpmv_version == 2) { // minicpmv 2.5 format: // (overview) (slice) (slice) \n ... @@ -767,9 +851,9 @@ struct mtmd_context { ov_img_first = true; } else if (minicpmv_version != 0) { - throw std::runtime_error(string_format("unsupported minicpmv version: %d\n", minicpmv_version)); + throw std::runtime_error(mtmd_format("unsupported minicpmv version: %d\n", minicpmv_version)); } - image_preproc = std::make_unique(ctx_v); + image_preproc = std::make_unique(models.v.get()); } break; case PROJECTOR_TYPE_MINICPMV4_6: case PROJECTOR_TYPE_MINICPMV4_7: @@ -783,42 +867,49 @@ struct mtmd_context { tok_row_end = {lookup_token("\n")}; tok_row_end_trail = false; // no trailing end-of-row token ov_img_first = true; - use_image_id = true; - if (clip_get_projector_type(ctx_v) == PROJECTOR_TYPE_MINICPM_OCR_1_0) { - image_preproc = std::make_unique(ctx_v); + has_prompt_img_id = true; + if (ctx_v->model.proj_type == PROJECTOR_TYPE_MINICPM_OCR_1_0) { + image_preproc = std::make_unique(models.v.get()); } else { - image_preproc = std::make_unique(ctx_v); + image_preproc = std::make_unique(models.v.get()); } } break; + case PROJECTOR_TYPE_QWEN3VL: + { + // <|vision_start|> ... (image embeddings) ... <|vision_end|> + img_beg = "<|vision_start|>"; + img_end = "<|vision_end|>"; + image_preproc = std::make_unique(models.v.get()); + } break; case PROJECTOR_TYPE_QWEN2VL: case PROJECTOR_TYPE_QWEN25VL: - case PROJECTOR_TYPE_QWEN3VL: case PROJECTOR_TYPE_LING3VL: case PROJECTOR_TYPE_MIMOVL: + case PROJECTOR_TYPE_MAGEVL: { // <|vision_start|> ... (image embeddings) ... <|vision_end|> img_beg = "<|vision_start|>"; img_end = "<|vision_end|>"; - image_preproc = std::make_unique(ctx_v); + image_preproc = std::make_unique(models.v.get()); } break; case PROJECTOR_TYPE_MINIMAX_M3: { // ]<]start of image[>[ ... (image embeddings) ... ]<]end of image[>[ img_beg = "]<]start of image[>["; img_end = "]<]end of image[>["; - image_preproc = std::make_unique(ctx_v); + image_preproc = std::make_unique(models.v.get()); } break; case PROJECTOR_TYPE_SENSENOVA_U1: { img_beg = ""; img_end = ""; - image_preproc = std::make_unique(ctx_v); + image_preproc = std::make_unique(models.v.get()); } break; case PROJECTOR_TYPE_MUSE_GLIMMER: { img_beg = "<|image_start|>"; img_end = "<|image_end|>"; - image_preproc = std::make_unique(ctx_v); + image_preproc = std::make_unique(models.v.get()); } break; case PROJECTOR_TYPE_LFM3VL: { @@ -827,14 +918,14 @@ struct mtmd_context { img_end = "<|vision_end|>"; vid_beg = img_beg; vid_end = img_end; - image_preproc = std::make_unique(ctx_v); + image_preproc = std::make_unique(models.v.get()); } break; case PROJECTOR_TYPE_YOUTUVL: { // <|vision_start|> ... (image embeddings) ... <|vision_end|> img_beg = "<|vision_start|>"; img_end = "<|vision_end|>"; - image_preproc = std::make_unique(ctx_v); + image_preproc = std::make_unique(models.v.get()); } break; case PROJECTOR_TYPE_YASA2: { @@ -847,7 +938,7 @@ struct mtmd_context { // images. This will be implemented in a future PR (dispatch on has_pinpoints // - see LDP/COGVLM branch above) and emit image_grid_pinpoints in the conversion // script. - image_preproc = std::make_unique(ctx_v); + image_preproc = std::make_unique(models.v.get()); } break; case PROJECTOR_TYPE_GEMMA3: case PROJECTOR_TYPE_GEMMA3NV: @@ -855,7 +946,7 @@ struct mtmd_context { // ... (image embeddings) ... img_beg = ""; img_end = ""; - image_preproc = std::make_unique(ctx_v); + image_preproc = std::make_unique(models.v.get()); } break; case PROJECTOR_TYPE_IDEFICS3: { @@ -865,22 +956,22 @@ struct mtmd_context { tok_ov_img_end = {lookup_token("")}; tok_row_end = {lookup_token("\n")}; sli_img_start_tmpl = ""; - image_preproc = std::make_unique(ctx_v); + image_preproc = std::make_unique(models.v.get()); } break; case PROJECTOR_TYPE_PIXTRAL: { // https://github.com/huggingface/transformers/blob/1cd110c6cb6a6237614130c470e9a902dbc1a4bd/docs/source/en/model_doc/pixtral.md img_end = "[IMG_END]"; - if (clip_get_hparams(ctx_v)->image_longest_edge > 0) { - image_preproc = std::make_unique(ctx_v); + if (ctx_v->model.hparams.image_longest_edge > 0) { + image_preproc = std::make_unique(models.v.get()); } else { - image_preproc = std::make_unique(ctx_v); + image_preproc = std::make_unique(models.v.get()); } } break; case PROJECTOR_TYPE_PHI4: { // Phi-4 uses media marker insertion only. Keep image boundary text empty. - image_preproc = std::make_unique(ctx_v); + image_preproc = std::make_unique(models.v.get()); } break; case PROJECTOR_TYPE_LLAMA4: { @@ -889,7 +980,7 @@ struct mtmd_context { img_end = "<|image_end|>"; LOG_WRN("%s: llama 4 vision is known to have degraded quality:\n" " https://github.com/ggml-org/llama.cpp/pull/13282\n", __func__); - image_preproc = std::make_unique(ctx_v); + image_preproc = std::make_unique(models.v.get()); ov_img_first = false; } break; case PROJECTOR_TYPE_STEP3VL: @@ -906,14 +997,14 @@ struct mtmd_context { tok_row_end = {lookup_token("")}; tok_row_end_trail = false; ov_img_first = false; // patches first, overview last - image_preproc = std::make_unique(ctx_v); + image_preproc = std::make_unique(models.v.get()); } break; case PROJECTOR_TYPE_INTERNVL: { // ... (image embeddings) ... img_beg = ""; img_end = ""; - image_preproc = std::make_unique(ctx_v); + image_preproc = std::make_unique(models.v.get()); ov_img_first = false; } break; case PROJECTOR_TYPE_COHERE2V: @@ -924,14 +1015,14 @@ struct mtmd_context { tok_sli_img_end = {lookup_token("<|IMG_LINE_BREAK|>")}; tok_ov_img_end = tok_sli_img_end; ov_img_first = false; - image_preproc = std::make_unique(ctx_v); + image_preproc = std::make_unique(models.v.get()); } break; case PROJECTOR_TYPE_KIMIVL: { // <|media_start|> ... (image embeddings) ... <|media_end|> img_beg = "<|media_start|>"; img_end = "<|media_end|>"; - image_preproc = std::make_unique(ctx_v); + image_preproc = std::make_unique(models.v.get()); } break; case PROJECTOR_TYPE_KIMIK25: { @@ -941,29 +1032,38 @@ struct mtmd_context { // <|begin_of_image|> ... (image embeddings) ... <|end_of_image|> img_beg = "<|begin_of_image|>"; img_end = "<|end_of_image|>"; + image_preproc = std::make_unique(models.v.get()); } else { // <|media_begin|> ... (image embeddings) ... <|media_end|> img_beg = "<|media_begin|>"; img_end = "<|media_end|>"; + image_preproc = std::make_unique(models.v.get()); } - image_preproc = std::make_unique(ctx_v); + } break; + case PROJECTOR_TYPE_KIMIK31: + { + // <|media_begin|>image WxH<|media_content|> ... (image embeddings) ... <|media_end|> + img_beg = "<|media_begin|>"; + img_end = "<|media_end|>"; + has_prompt_img_size = true; + image_preproc = std::make_unique(models.v.get()); } break; case PROJECTOR_TYPE_LIGHTONOCR: { // <|im_start|> ... (image embeddings) ... <|im_end|> img_beg = "<|im_start|>"; img_end = "<|im_end|>"; - image_preproc = std::make_unique(ctx_v); + image_preproc = std::make_unique(models.v.get()); } break; case PROJECTOR_TYPE_DEEPSEEK4V: { // no vocab tokens are added; the start/end/newline markers are learned embeddings emitted by the encoder - image_preproc = std::make_unique(ctx_v); + image_preproc = std::make_unique(models.v.get()); } break; case PROJECTOR_TYPE_DEEPSEEK41V: { // as deepseek4v: markers come from the encoder, the block is decoded causally - image_preproc = std::make_unique(ctx_v); + image_preproc = std::make_unique(models.v.get()); } break; case PROJECTOR_TYPE_DOTS_OCR: case PROJECTOR_TYPE_DOTS3NOTE_V: @@ -971,17 +1071,17 @@ struct mtmd_context { // <|img|> ... (image embeddings) ... <|endofimg|> img_beg = "<|img|>"; img_end = "<|endofimg|>"; - image_preproc = std::make_unique(ctx_v); + image_preproc = std::make_unique(models.v.get()); } break; case PROJECTOR_TYPE_NEMOTRON_V2_VL: { - image_preproc = std::make_unique(ctx_v); + image_preproc = std::make_unique(models.v.get()); } break; case PROJECTOR_TYPE_NEMOTRON_OMNI: { img_beg = ""; img_end = ""; - image_preproc = std::make_unique(ctx_v); + image_preproc = std::make_unique(models.v.get()); } break; case PROJECTOR_TYPE_LFM2: { @@ -998,33 +1098,33 @@ struct mtmd_context { sli_img_start_tmpl = "<|img_row_%d_col_%d|>"; tok_ov_img_start = {lookup_token("<|img_thumbnail|>")}; ov_img_first = false; - image_preproc = std::make_unique(ctx_v); + image_preproc = std::make_unique(models.v.get()); } break; case PROJECTOR_TYPE_D1OMNI_V: { // same tiles and thumbnail as lfm2, without separator tokens - image_preproc = std::make_unique(ctx_v); + image_preproc = std::make_unique(models.v.get()); } break; case PROJECTOR_TYPE_GLM4V: { // <|begin_of_image|> ... (image embeddings) ... <|end_of_image|> img_beg = "<|begin_of_image|>"; img_end = "<|end_of_image|>"; - image_preproc = std::make_unique(ctx_v); + image_preproc = std::make_unique(models.v.get()); } break; case PROJECTOR_TYPE_GLM5V: { // <|begin_of_image|> ... (image embeddings) ... <|end_of_image|> img_beg = "<|begin_of_image|>"; img_end = "<|end_of_image|>"; - image_preproc = std::make_unique(ctx_v); + image_preproc = std::make_unique(models.v.get()); } break; case PROJECTOR_TYPE_PADDLEOCR: { // <|IMAGE_START|> ... (image embeddings) ... <|IMAGE_END|> img_beg = "<|IMAGE_START|>"; img_end = "<|IMAGE_END|>"; - image_preproc = std::make_unique(ctx_v); + image_preproc = std::make_unique(models.v.get()); } break; case PROJECTOR_TYPE_GEMMA4V: case PROJECTOR_TYPE_GEMMA4UV: @@ -1032,13 +1132,13 @@ struct mtmd_context { // <|image> ... (image embeddings) ... img_beg = "<|image>"; img_end = ""; - image_preproc = std::make_unique(ctx_v); + image_preproc = std::make_unique(models.v.get()); } break; case PROJECTOR_TYPE_DEEPSEEKOCR: case PROJECTOR_TYPE_DEEPSEEKOCR2: { img_end = "\n"; // prevent empty batch on llama-server - image_preproc = std::make_unique(ctx_v); + image_preproc = std::make_unique(models.v.get()); ov_img_first = false; } break; case PROJECTOR_TYPE_HUNYUANVL: @@ -1046,24 +1146,24 @@ struct mtmd_context { // note: these use fullwidth | (U+FF5C) and ▁ (U+2581) to match the tokenizer vocabulary img_beg = "<|hy_place▁holder▁no▁100|>"; img_end = "<|hy_place▁holder▁no▁101|>"; - image_preproc = std::make_unique(ctx_v); + image_preproc = std::make_unique(models.v.get()); } break; case PROJECTOR_TYPE_EXAONE4_5: { // ... (image embeddings) ... img_beg = ""; img_end = ""; - image_preproc = std::make_unique(ctx_v); + image_preproc = std::make_unique(models.v.get()); } break; case PROJECTOR_TYPE_GRANITE4_VISION: { // ... (image embeddings) \n ... img_beg = ""; img_end = "\n"; - image_preproc = std::make_unique(ctx_v); + image_preproc = std::make_unique(models.v.get()); } break; default: - throw std::runtime_error(string_format("%s: unexpected vision projector type %d\n", __func__, proj)); + throw std::runtime_error(mtmd_format("%s: unexpected vision projector type %d\n", __func__, proj)); } GGML_ASSERT(image_preproc != nullptr); @@ -1073,7 +1173,7 @@ struct mtmd_context { GGML_ASSERT(ctx_a != nullptr); audio_preproc.reset(); - projector_type proj = clip_get_projector_type(ctx_a); + projector_type proj = ctx_a->model.proj_type; LOG_WRN("%s: audio input is in experimental stage and may have reduced quality:\n" " https://github.com/ggml-org/llama.cpp/discussions/13759\n", __func__); @@ -1086,31 +1186,31 @@ struct mtmd_context { // <|audio_bos|> ... (embeddings) ... <|audio_eos|> aud_beg = "<|audio_bos|>"; aud_end = "<|audio_eos|>"; - audio_preproc = std::make_unique(ctx_a); + audio_preproc = std::make_unique(models.a.get()); } break; case PROJECTOR_TYPE_MUSE_SPARK_AUDIO: { aud_beg = "<|soft_audio_start|>"; aud_end = "<|soft_audio_end|>"; - audio_preproc = std::make_unique(ctx_a); + audio_preproc = std::make_unique(models.a.get()); } break; case PROJECTOR_TYPE_QWEN3A: { aud_beg = "<|audio_start|>"; aud_end = "<|audio_end|>"; - audio_preproc = std::make_unique(ctx_a); + audio_preproc = std::make_unique(models.a.get()); } break; case PROJECTOR_TYPE_VOXTRAL: { // [BEGIN_AUDIO] ... (embeddings) ... aud_beg = "[BEGIN_AUDIO]"; - audio_preproc = std::make_unique(ctx_a); + audio_preproc = std::make_unique(models.a.get()); } break; case PROJECTOR_TYPE_MUSIC_FLAMINGO: { // ... (embeddings) ... aud_beg = ""; - audio_preproc = std::make_unique(ctx_a); + audio_preproc = std::make_unique(models.a.get()); } break; case PROJECTOR_TYPE_VOXTRAL_RT: { @@ -1120,15 +1220,15 @@ struct mtmd_context { case PROJECTOR_TYPE_GLMA: case PROJECTOR_TYPE_MERALION: { - audio_preproc = std::make_unique(ctx_a); + audio_preproc = std::make_unique(models.a.get()); } break; case PROJECTOR_TYPE_LFM2A: { - audio_preproc = std::make_unique(ctx_a); + audio_preproc = std::make_unique(models.a.get()); } break; case PROJECTOR_TYPE_D1OMNI_A: { - audio_preproc = std::make_unique(ctx_a); + audio_preproc = std::make_unique(models.a.get()); } break; case PROJECTOR_TYPE_LFM3A: { @@ -1137,75 +1237,85 @@ struct mtmd_context { aud_beg = img_beg; aud_end = img_end; } - audio_preproc = std::make_unique(ctx_a); + audio_preproc = std::make_unique(models.a.get()); } break; case PROJECTOR_TYPE_GRANITE_SPEECH: { - audio_preproc = std::make_unique(ctx_a); + audio_preproc = std::make_unique(models.a.get()); } break; case PROJECTOR_TYPE_GEMMA4A: { aud_beg = "<|audio>"; aud_end = ""; - audio_preproc = std::make_unique(ctx_a); + audio_preproc = std::make_unique(models.a.get()); } break; case PROJECTOR_TYPE_PARAKEET: { - audio_preproc = std::make_unique(ctx_a); + audio_preproc = std::make_unique(models.a.get()); + } break; + case PROJECTOR_TYPE_NEMOTRON_TRANSCRIBE: + { + audio_preproc = std::make_unique(models.a.get()); } break; case PROJECTOR_TYPE_GEMMA4UA: { aud_beg = "<|audio>"; aud_end = ""; - audio_preproc = std::make_unique(ctx_a); + audio_preproc = std::make_unique(models.a.get()); } break; case PROJECTOR_TYPE_DOTS3NOTE_A: { // <|audio_comp_start|> ... (embeddings) ... <|audio_comp_end|> aud_beg = "<|audio_comp_start|>"; aud_end = "<|audio_comp_end|>"; - audio_preproc = std::make_unique(ctx_a); + audio_preproc = std::make_unique(models.a.get()); } break; case PROJECTOR_TYPE_MIMO_AUDIO: { aud_beg = "<|mimo_audio_start|>"; aud_end = "<|mimo_audio_end|>"; - audio_preproc = std::make_unique(ctx_a); + audio_preproc = std::make_unique(models.a.get()); } break; case PROJECTOR_TYPE_QWEN3TTS_SPKENC: { - audio_preproc = std::make_unique(ctx_a); + audio_preproc = std::make_unique(models.a.get()); } break; case PROJECTOR_TYPE_POCKETTTS_SPKENC: { - audio_preproc = std::make_unique(ctx_a); + audio_preproc = std::make_unique(models.a.get()); + } break; + case PROJECTOR_TYPE_CHATTERBOX_SPKENC: + { + audio_preproc = std::make_unique(models.a.get(), cbx_is_mtl(*models.a)); } break; default: - throw std::runtime_error(string_format("%s: unexpected audio projector type %d\n", __func__, proj)); + throw std::runtime_error(mtmd_format("%s: unexpected audio projector type %d\n", __func__, proj)); } // initialize audio preprocessor if (audio_preproc) { + audio_preproc->mel_config.n_threads = n_threads; + audio_preproc->mel_config.sched = backends->sched.get(); audio_preproc->initialize(); } } - // get clip ctx based on chunk type - clip_ctx * get_clip_ctx(const mtmd_input_chunk * chunk) const { + // get the subcontext based on chunk type + mtmd_subcontext * get_subctx(const mtmd_input_chunk * chunk) const { if (chunk->type == MTMD_INPUT_CHUNK_TYPE_IMAGE) { - return ctx_v; + return ctx_v.get(); } else if (chunk->type == MTMD_INPUT_CHUNK_TYPE_AUDIO) { - return ctx_a; + return ctx_a.get(); } GGML_ABORT("unknown chunk type"); } projector_type proj_type_v() const { - return ctx_v ? clip_get_projector_type(ctx_v) : PROJECTOR_TYPE_UNKNOWN; + return ctx_v ? ctx_v->model.proj_type : PROJECTOR_TYPE_UNKNOWN; } projector_type proj_type_a() const { - return ctx_a ? clip_get_projector_type(ctx_a) : PROJECTOR_TYPE_UNKNOWN; + return ctx_a ? ctx_a->model.proj_type : PROJECTOR_TYPE_UNKNOWN; } // [LFM3_OMNI] the audio of a video is its soundtrack, see MTMD_VIDEO_TEMPLATE_LFM3_OMNI @@ -1215,20 +1325,14 @@ struct mtmd_context { int64_t n_embd_out() const { if (ctx_v) { - return clip_n_mmproj_embd(ctx_v); + return ctx_v->model.n_mmproj_embd(); } else if (ctx_a) { - return clip_n_mmproj_embd(ctx_a); + return ctx_a->model.n_mmproj_embd(); } else { - throw std::runtime_error("no CLIP model loaded"); + throw std::runtime_error("no mmproj model loaded"); } } - ~mtmd_context() { - clip_free(ctx_a); - clip_free(ctx_v); - clip_free(ctx_gen_a); - } - private: llama_token lookup_token(const std::string & token_text) { if (vocab == nullptr) { @@ -1347,7 +1451,7 @@ struct mtmd_tokenizer { for (const auto & part : parts_str) { if (part == ctx->media_marker) { if (i_bm >= bitmaps.size()) { - throw std::runtime_error(string_format("number of media markers in text (%zu) exceeds number of bitmaps (%zu)", i_bm + 1, bitmaps.size())); + throw std::runtime_error(mtmd_format("number of media markers in text (%zu) exceeds number of bitmaps (%zu)", i_bm + 1, bitmaps.size())); } parts.push_back({"", bitmaps[i_bm++]}); } else { @@ -1362,7 +1466,7 @@ struct mtmd_tokenizer { } } if (n_markers != bitmaps.size()) { - throw std::runtime_error(string_format("number of media markers in text (%zu) does not match number of bitmaps (%zu)", n_markers, bitmaps.size())); + throw std::runtime_error(mtmd_format("number of media markers in text (%zu) does not match number of bitmaps (%zu)", n_markers, bitmaps.size())); } expand_lazy_bitmaps(); @@ -1407,7 +1511,7 @@ struct mtmd_tokenizer { &out_bm, &out_str); if (out_bm && out_str) { - throw std::runtime_error(string_format("lazy callback cannot return both bitmap and text")); + throw std::runtime_error(mtmd_format("lazy callback cannot return both bitmap and text")); } if (res == 0) { // OK, append the returned chunk; lazy part is not yet added @@ -1431,7 +1535,7 @@ struct mtmd_tokenizer { break; } else if (res == -2) { // error - throw std::runtime_error(string_format("lazy callback returned error")); + throw std::runtime_error(mtmd_format("lazy callback returned error")); } } if (!st.pieces.empty()) { @@ -1453,7 +1557,7 @@ struct mtmd_tokenizer { // [QWEN_VIDEO] handle frame merging for models that support it (i.e. qwen-vl) int n_merge_frames = 1; if (ctx->ctx_v) { - n_merge_frames = clip_model_n_temporal_merge(ctx->ctx_v); + n_merge_frames = ctx->ctx_v->model.n_temporal_merge(); GGML_ASSERT(n_merge_frames <= 2 && "we only support merging maximum 2 images for now; open an issue if this model supports merging more"); } @@ -1564,6 +1668,9 @@ struct mtmd_tokenizer { if (!ctx->img_beg.empty()) { add_text(ctx->img_beg, true); // add image begin token } + if (ctx->has_prompt_img_size) { + add_text(mtmd_format("image %ux%u<|media_content|>", bitmaps[0]->nx, bitmaps[0]->ny), true); + } // TODO @ngxson : this is quite hacky because preprocessor only support batch with one single element, that need to be fixed in the future (e.g. by changing the preprocessor interface always take single input) @@ -1579,8 +1686,8 @@ struct mtmd_tokenizer { return 2; } - // convert mtmd_bitmap to clip_image_u8 - clip_image_u8 img_u8; + // convert mtmd_bitmap to mtmd_image_u8 + mtmd_image_u8 img_u8; img_u8.set_size( {(int)bmp->nx, (int)bmp->ny}, bmp->is_placeholder()); @@ -1618,11 +1725,11 @@ struct mtmd_tokenizer { if (has_tiling_grid && ctx->proj_type_v() == PROJECTOR_TYPE_MINICPMV4_7) { GGML_ASSERT(bitmaps.size() == 1); - if (ctx->use_image_id) { + if (ctx->has_prompt_img_id) { add_text("" + std::to_string(n_images_added) + "", true); } add_text(ctx->tok_ov_img_start); - // the separators after are appended by clip, see add_canvas_chunk() + // the separators after are appended by the graph, see add_canvas_chunk() add_canvas_chunk(std::move(preproc_out), bitmaps[0]->id); } else if (has_tiling_grid) { // [QWEN_VIDEO] we do not support "frame merging" for llama-uhd style, so no batching for now @@ -1643,7 +1750,7 @@ struct mtmd_tokenizer { // add overview image (first) if (ctx->ov_img_first) { - if (ctx->use_image_id) { + if (ctx->has_prompt_img_id) { add_text("" + std::to_string(n_images_added) + "", true); } add_text(ctx->tok_ov_img_start); @@ -1672,7 +1779,7 @@ struct mtmd_tokenizer { auto & curr_chunk = chunks[y * n_col + x]; auto & curr_batch = curr_chunk.tokens_image->batch_f32; if (curr_batch.entries.size() != 1) { - throw std::runtime_error(string_format("%s: expect 1 image in batch_f32", __func__)); + throw std::runtime_error(mtmd_format("%s: expect 1 image in batch_f32", __func__)); } LOG_DBG("%s: adding slice image at row %d col %d\n", __func__, y, x); @@ -1717,8 +1824,8 @@ struct mtmd_tokenizer { size_t n_tokens = 0; for (auto & e : preproc_out.entries) { - n_tokens += clip_n_output_tokens(ctx->ctx_v, &e); - if (clip_model_n_temporal_merge(ctx->ctx_v) == 2) { + n_tokens += ctx->ctx_v->model.n_output_tokens(&e); + if (ctx->ctx_v->model.n_temporal_merge() == 2) { // [QWEN_VIDEO] pair input is merged to the same embd, so only count as one image break; } @@ -1727,12 +1834,12 @@ struct mtmd_tokenizer { mtmd_image_tokens_ptr image_tokens(new mtmd_image_tokens); // [QWEN_VIDEO] improve this in the future - image_tokens->n_temporal_merge = clip_model_n_temporal_merge(ctx->ctx_v); + image_tokens->n_temporal_merge = ctx->ctx_v->model.n_temporal_merge(); if (mtmd_decode_use_mrope(ctx)) { // for Qwen2VL, we need this information for M-RoPE decoding positions - image_tokens->nx = clip_n_output_tokens_x(ctx->ctx_v, &preproc_out.entries[0]); - image_tokens->ny = clip_n_output_tokens_y(ctx->ctx_v, &preproc_out.entries[0]); + image_tokens->nx = ctx->ctx_v->model.n_output_tokens_x(&preproc_out.entries[0]); + image_tokens->ny = ctx->ctx_v->model.n_output_tokens_y(&preproc_out.entries[0]); } else { // other models, we only need the total number of tokens image_tokens->nx = n_tokens; @@ -1753,7 +1860,7 @@ struct mtmd_tokenizer { GGML_ASSERT(n_tokens == (size_t)image_tokens->n_tokens()); } - clip_image_f32_batch batch_f32; + mtmd_image_f32_batch batch_f32; batch_f32.is_audio = false; batch_f32.is_video = bitmaps[0]->is_video_frame(); batch_f32.entries = std::move(preproc_out.entries); @@ -1810,7 +1917,7 @@ struct mtmd_tokenizer { GGML_ASSERT(ctx->audio_preproc != nullptr); // preprocess audio - std::vector mel_spec_chunks; + auto source = std::make_shared(); { std::vector dummy; const float * samples = nullptr; @@ -1834,30 +1941,45 @@ struct mtmd_tokenizer { LOG_ERR("%s: this model takes audio input only via mtmd_audio_stream\n", __func__); return 2; } - bool ok = ctx->audio_preproc->preprocess(samples, n_samples, mel_spec_chunks); - if (!ok) { + *source = ctx->audio_preproc->preprocess(samples, n_samples); + if (source->entries.empty()) { LOG_ERR("Unable to preprocess audio\n"); return 2; } } + // one chunk for the five entries of a chatterbox clip, one token per talker conditioning row + if (ctx->ctx_a->model.proj_type == PROJECTOR_TYPE_CHATTERBOX_SPKENC) { + GGML_ASSERT(source->entries.size() == 5); + // mtl: spkr, perceiver query rows, emotion; turbo: spkr, t3 speech tokens + const size_t n_query = ctx->ctx_a->model.cbx_read_tensor("a.cenc.perceiver.pre_attention_query", nullptr, 0) / ctx->n_embd_out(); + const size_t n_tokens = cbx_is_mtl(ctx->ctx_a->model) ? 2 + n_query : 1 + (size_t) source->entries[2].nx() / 4; + + mtmd_image_f32_batch batch_f32; + batch_f32.is_audio = true; + batch_f32.entries = std::move(source->entries); + + mtmd_audio_tokens_ptr audio_tokens(new mtmd_audio_tokens); + audio_tokens->n_tokens = (uint32_t) n_tokens; + audio_tokens->batch_f32 = std::move(batch_f32); + audio_tokens->id = bitmap->id; + + mtmd_input_chunk chunk{ + MTMD_INPUT_CHUNK_TYPE_AUDIO, + {}, // text tokens + nullptr, // image tokens + std::move(audio_tokens), + }; + cur.entries.emplace_back(std::move(chunk)); + } else // consider each mel_spec as a separate audio chunk // TODO: maybe support batching, but this may come with memory cost - for (auto & mel_spec : mel_spec_chunks) { - const bool is_placeholder = mel_spec.data.empty(); - - // Validate dimensions fit in clip_image_size (int) - GGML_ASSERT(mel_spec.n_len <= INT32_MAX && mel_spec.n_len >= 0); - GGML_ASSERT(mel_spec.n_mel <= INT32_MAX && mel_spec.n_mel >= 0); - clip_image_f32 mel_f32; - mel_f32.set_size( - {(int)mel_spec.n_len, (int)mel_spec.n_mel}, - is_placeholder, /* is_audio */ true); - mel_f32.cpy_buf(mel_spec.data); + for (size_t i = 0; i < source->entries.size(); i++) { + mtmd_image_f32 mel_f32 = source->entries[i]; - size_t n_tokens = clip_n_output_tokens(ctx->ctx_a, &mel_f32); + size_t n_tokens = ctx->ctx_a->model.n_output_tokens(&mel_f32); - clip_image_f32_batch batch_f32; + mtmd_image_f32_batch batch_f32; batch_f32.is_audio = true; batch_f32.entries.push_back(std::move(mel_f32)); @@ -1865,6 +1987,10 @@ struct mtmd_tokenizer { audio_tokens->n_tokens = n_tokens; audio_tokens->batch_f32 = std::move(batch_f32); audio_tokens->id = bitmap->id; // optional + if (source->needs_compute()) { + audio_tokens->source = source; + audio_tokens->source_idx = (uint32_t) i; + } LOG_DBG("audio_tokens->n_tokens = %d\n", audio_tokens->n_tokens); @@ -1894,6 +2020,7 @@ struct mtmd_tokenizer { track->n_tokens = bitmap.ny; const auto & buf = bitmap.get_ro_buf(); track->embd.assign((const float *) buf.data(), (const float *) (buf.data() + buf.size())); + track->gen_ref = bitmap.gen_ref; mtmd_audio_tokens_ptr audio_tokens(new mtmd_audio_tokens); audio_tokens->n_tokens = bitmap.ny; @@ -1921,22 +2048,27 @@ struct mtmd_tokenizer { samples.insert(samples.end(), data, data + piece->nx); } } - std::vector mels; - if (!ctx->audio_preproc || !ctx->audio_preproc->preprocess(samples.data(), samples.size(), mels) || mels.size() != 1) { + if (!ctx->audio_preproc) { + LOG_ERR("%s: unable to preprocess the soundtrack\n", __func__); + return 2; + } + auto source = std::make_shared(ctx->audio_preproc->preprocess(samples.data(), samples.size())); + if (source->entries.size() != 1) { LOG_ERR("%s: unable to preprocess the soundtrack\n", __func__); return 2; } - clip_image_f32 mel_f32; - mel_f32.set_size({(int) mels[0].n_len, (int) mels[0].n_mel}, mels[0].data.empty(), /* is_audio */ true); - mel_f32.cpy_buf(mels[0].data); + mtmd_image_f32 mel_f32 = source->entries[0]; st.track = std::make_shared(); - st.track->n_tokens = clip_n_output_tokens(ctx->ctx_a, &mel_f32); + st.track->n_tokens = ctx->ctx_a->model.n_output_tokens(&mel_f32); + if (source->needs_compute()) { + st.track->source = source; + } st.track->batch_f32.is_audio = true; st.track->batch_f32.entries.push_back(std::move(mel_f32)); } // rows of [begin, end): round(sample / samples per row), the last piece takes the rest of the track - const double samples_per_row = 8.0 * clip_get_hparams(ctx->ctx_a)->audio_hop_len; // one row per 8 mel frames + const double samples_per_row = 8.0 * ctx->ctx_a->model.hparams.audio_hop_len; // one row per 8 mel frames const uint32_t n_tokens = st.track->n_tokens; st.n_samples += bitmap.nx; const uint32_t begin = st.n_assigned; @@ -1961,7 +2093,7 @@ struct mtmd_tokenizer { return 0; } - // MiniCPM-V 4.7: the overview and all slices go in one chunk, clip appends the separators after each tile: + // MiniCPM-V 4.7: the overview and all slices go in one chunk, the graph appends the separators after each tile: // [ov] [S00] [S01] \n [S10] [S11] void add_canvas_chunk(mtmd_image_preproc_out && preproc_out, const std::string & id) { const int n_col = preproc_out.grid_x; @@ -1971,9 +2103,9 @@ struct mtmd_tokenizer { GGML_ASSERT((int) slices.size() == n_col * n_row); auto & ov = preproc_out.overview; - ov.suffix_type = CLIP_SUFFIX_MINICPMV_OV; + ov.suffix_type = MTMD_SUFFIX_MINICPMV_OV; if (!slices.empty()) { - ov.suffix_type = CLIP_SUFFIX_MINICPMV_OV_SLICE; + ov.suffix_type = MTMD_SUFFIX_MINICPMV_OV_SLICE; ov.anyres.grid_x = n_col; ov.anyres.grid_y = n_row; } @@ -1981,11 +2113,11 @@ struct mtmd_tokenizer { for (int x = 0; x < n_col; x++) { auto & suffix = slices[y * n_col + x].suffix_type; if (y == n_row - 1 && x == n_col - 1) { - suffix = CLIP_SUFFIX_MINICPMV_LAST; + suffix = MTMD_SUFFIX_MINICPMV_LAST; } else if (x == n_col - 1) { - suffix = CLIP_SUFFIX_MINICPMV_ROW_END; + suffix = MTMD_SUFFIX_MINICPMV_ROW_END; } else { - suffix = CLIP_SUFFIX_MINICPMV_SLICE; + suffix = MTMD_SUFFIX_MINICPMV_SLICE; } } } @@ -1999,12 +2131,12 @@ struct mtmd_tokenizer { entries.push_back(std::move(slice)); } // token grid of the last entry, the grids of the other entries are scaled from it - image_tokens->nx = clip_n_output_tokens_x(ctx->ctx_v, &entries.back()); - image_tokens->ny = clip_n_output_tokens_y(ctx->ctx_v, &entries.back()); + image_tokens->nx = ctx->ctx_v->model.n_output_tokens_x(&entries.back()); + image_tokens->ny = ctx->ctx_v->model.n_output_tokens_y(&entries.back()); size_t n_tokens = 0; for (const auto & entry : entries) { - n_tokens += clip_n_output_tokens(ctx->ctx_v, &entry); + n_tokens += ctx->ctx_v->model.n_output_tokens(&entry); } GGML_ASSERT(n_tokens == image_tokens->n_tokens()); @@ -2020,9 +2152,9 @@ struct mtmd_tokenizer { std::vector split_batch_to_chunk(mtmd_image_preproc_out && preproc_out, const std::string & id) { std::vector chunks; - auto process_chunk = [&](clip_image_f32 && img) { + auto process_chunk = [&](mtmd_image_f32 && img) { mtmd_image_tokens_ptr image_tokens(new mtmd_image_tokens); - image_tokens->nx = clip_n_output_tokens(ctx->ctx_v, &img); + image_tokens->nx = ctx->ctx_v->model.n_output_tokens(&img); image_tokens->ny = 1; image_tokens->batch_f32.entries.push_back(std::move(img)); image_tokens->id = id; @@ -2041,14 +2173,14 @@ struct mtmd_tokenizer { // overview image first auto & overview = preproc_out.overview; if (overview.nx() == 0 || overview.ny() == 0) { - throw std::runtime_error(string_format("%s: invalid overview image for llava-uhd style preprocessing\n", __func__)); + throw std::runtime_error(mtmd_format("%s: invalid overview image for llava-uhd style preprocessing\n", __func__)); } process_chunk(std::move(preproc_out.overview)); // then, process slices for (auto & entry : preproc_out.entries) { if (entry.nx() == 0 || entry.ny() == 0) { - throw std::runtime_error(string_format("%s: invalid image slice for llava-uhd style preprocessing\n", __func__)); + throw std::runtime_error(mtmd_format("%s: invalid image slice for llava-uhd style preprocessing\n", __func__)); } process_chunk(std::move(entry)); } @@ -2143,8 +2275,8 @@ int32_t mtmd_tokenize_from_parts(const mtmd_context * ctx, } static int32_t mtmd_encode_impl(mtmd_context * ctx, const mtmd_image_tokens * image_tokens, std::vector & out_embd) { - clip_ctx * ctx_clip = ctx->ctx_v; - if (!ctx_clip) { + mtmd_subcontext * sctx = ctx->ctx_v.get(); + if (!sctx) { LOG_ERR("%s: this API does not support non-vision input, please use mtmd_encode_chunk instead\n", __func__); return 1; } @@ -2162,10 +2294,10 @@ static int32_t mtmd_encode_impl(mtmd_context * ctx, const mtmd_image_tokens * im // the tiles differ in size, encode them one by one size_t offset = 0; for (const auto & entry : image_tokens->batch_f32.entries) { - clip_image_f32_batch one; + mtmd_image_f32_batch one; one.entries.push_back(entry); - std::vector embd((size_t) n_embd_out * clip_n_output_tokens(ctx_clip, &entry)); - if (!clip_image_batch_encode(ctx_clip, ctx->n_threads, &one, embd)) { + std::vector embd((size_t) n_embd_out * sctx->model.n_output_tokens(&entry)); + if (!sctx->encode_batch(ctx->n_threads, &one, embd)) { return 1; } GGML_ASSERT(offset + embd.size() <= out_embd.size()); @@ -2176,16 +2308,30 @@ static int32_t mtmd_encode_impl(mtmd_context * ctx, const mtmd_image_tokens * im return 0; } - bool ok = clip_image_batch_encode( - ctx_clip, - ctx->n_threads, + bool ok = sctx->encode_batch(ctx->n_threads, &image_tokens->batch_f32, out_embd); return ok ? 0 : 1; } +static int32_t cbx_ref_encode(mtmd_context * ctx, const mtmd_audio_tokens * audio_tokens, std::vector & out_embd); + +static bool mtmd_audio_batch_encode(mtmd_context * ctx, const mtmd_image_f32_batch & batch_f32, mtmd_audio_preproc_out * source, uint32_t idx, std::vector & out_embd) { + mtmd_image_f32_batch batch = batch_f32.clone(); + if (source) { + if (!ctx->audio_preproc->compute(*source)) { + LOG_ERR("%s: unable to compute the mel\n", __func__); + return false; + } + GGML_ASSERT(idx < source->entries.size()); + batch.entries[0] = source->entries[idx]; + } + return ctx->ctx_a->encode_batch(ctx->n_threads, &batch, out_embd); +} + static int32_t mtmd_encode_chunk_impl(mtmd_context * ctx, const mtmd_input_chunk * chunk, std::vector & out_embd) { + ctx->gen_ref = {}; // see mtmd_gen_state_set_ref() if (chunk->type == MTMD_INPUT_CHUNK_TYPE_TEXT) { LOG_WRN("mtmd_encode_chunk has no effect for text chunks\n"); return 0; @@ -2221,7 +2367,7 @@ static int32_t mtmd_encode_chunk_impl(mtmd_context * ctx, const mtmd_input_chunk if (track) { if (track->embd.empty()) { track->embd.resize((size_t) track->n_tokens * ctx->n_embd_out()); - if (!clip_image_batch_encode(ctx->ctx_a, ctx->n_threads, &track->batch_f32, track->embd)) { + if (!mtmd_audio_batch_encode(ctx, track->batch_f32, track->source.get(), 0, track->embd)) { track->embd.clear(); return 1; } @@ -2230,14 +2376,16 @@ static int32_t mtmd_encode_chunk_impl(mtmd_context * ctx, const mtmd_input_chunk const size_t off = (size_t) chunk->tokens_audio->row_begin * n_mmproj_embd; out_embd.assign(track->embd.begin() + off, track->embd.begin() + off + (size_t) chunk->tokens_audio->n_tokens * n_mmproj_embd); + if (track->gen_ref) { + ctx->gen_ref = *track->gen_ref; + } return 0; } + if (ctx->ctx_a->model.proj_type == PROJECTOR_TYPE_CHATTERBOX_SPKENC) { + return cbx_ref_encode(ctx, chunk->tokens_audio.get(), out_embd); + } out_embd.resize((size_t)chunk->tokens_audio->n_tokens * ctx->n_embd_out()); - bool ok = clip_image_batch_encode( - ctx->ctx_a, - ctx->n_threads, - &chunk->tokens_audio->batch_f32, - out_embd); + bool ok = mtmd_audio_batch_encode(ctx, chunk->tokens_audio->batch_f32, chunk->tokens_audio->source.get(), chunk->tokens_audio->source_idx, out_embd); return ok ? 0 : 1; } @@ -2247,6 +2395,7 @@ static int32_t mtmd_encode_chunk_impl(mtmd_context * ctx, const mtmd_input_chunk int32_t mtmd_encode_chunk(mtmd_context * ctx, const mtmd_input_chunk * chunk) { // this is the non-batching version + std::lock_guard lock(ctx->compute_mutex); try { return mtmd_encode_chunk_impl(ctx, chunk, ctx->out_embd); } catch (const std::exception & e) { @@ -2256,6 +2405,7 @@ int32_t mtmd_encode_chunk(mtmd_context * ctx, const mtmd_input_chunk * chunk) { } int32_t mtmd_encode(mtmd_context * ctx, const mtmd_image_tokens * image_tokens) { + std::lock_guard lock(ctx->compute_mutex); try { return mtmd_encode_impl(ctx, image_tokens, ctx->out_embd); } catch (const std::exception & e) { @@ -2279,9 +2429,9 @@ mtmd_gen_audio_info mtmd_gen_audio_get_info(const mtmd_context * ctx) { info.type = MTMD_GEN_AUDIO_TYPE_NONE; return info; } - info.model_variant = clip_get_hparams(ctx->ctx_gen_a)->gen_model_variant.c_str(); - info.n_voices = (int32_t) clip_gen_get_voices(ctx->ctx_gen_a).size(); - switch (clip_get_projector_type(ctx->ctx_gen_a)) { + info.model_variant = ctx->ctx_gen_a->model.hparams.gen_model_variant.c_str(); + info.n_voices = (int32_t) ctx->ctx_gen_a->model.gen_voices.size(); + switch (ctx->ctx_gen_a->model.proj_type) { case PROJECTOR_TYPE_QWEN3TTS_GEN: info.type = MTMD_GEN_AUDIO_TYPE_QWEN3TTS; info.sample_rate = 24000; @@ -2298,6 +2448,14 @@ mtmd_gen_audio_info mtmd_gen_audio_get_info(const mtmd_context * ctx) { info.type = MTMD_GEN_AUDIO_TYPE_LFM3A; info.sample_rate = 24000; break; + case PROJECTOR_TYPE_CHATTERBOX: + info.type = MTMD_GEN_AUDIO_TYPE_CHATTERBOX; + info.sample_rate = 24000; + break; + case PROJECTOR_TYPE_DRAMA3_GEN: + info.type = MTMD_GEN_AUDIO_TYPE_DRAMA3; + info.sample_rate = 44100; + break; default: info.type = MTMD_GEN_AUDIO_TYPE_NONE; break; @@ -2305,11 +2463,360 @@ mtmd_gen_audio_info mtmd_gen_audio_get_info(const mtmd_context * ctx) { return info; } +size_t mtmd_gen_audio_read_tensor(mtmd_context * ctx, const char * name, float * out, size_t n_max) { + if (!ctx->ctx_gen_a) { + return 0; + } + return ctx->ctx_gen_a->model.cbx_read_tensor(name, out, n_max); +} + +// raw 192-dim CAMPPlus x-vector of the fbank entry +static bool cbx_ref_xvec(mtmd_context * ctx, const mtmd_image_f32 & fbank, std::vector & xvec) { + mtmd_image_f32 mel_img; + mel_img.set_size(fbank.get_size(), false, true); + mel_img.cpy_buf(fbank.get_ro_buf()); + mtmd_image_f32_batch batch; + batch.is_audio = true; + batch.entries.push_back(std::move(mel_img)); + xvec.resize(192); + if (!ctx->ctx_a->encode_batch(ctx->n_threads, &batch, xvec)) { + LOG_ERR("%s: speaker encoder failed\n", __func__); + return false; + } + return true; +} + +// s3 speech tokens of a log-mel entry, rows optionally receives their speech embedding rows +static int32_t cbx_ref_tokenize(mtmd_context * ctx, const mtmd_image_f32 & mel, + std::vector & codes, std::vector * rows) { + mtmd_image_f32 mel_img; + mel_img.set_size(mel.get_size(), false, true); + mel_img.cpy_buf(mel.get_ro_buf()); + + mtmd_image_f32_batch batch; + batch.is_audio = true; + batch.entries.push_back(std::move(mel_img)); + + std::vector out_codes; + std::vector out_embd; + + mtmd_subctx_encode_params params; + params.imgs = &batch; + params.n_threads = ctx->n_threads; + params.gen_process = MTMD_SUBCTX_GEN_PROCESS_TOKENIZE; + params.out_codes = &out_codes; + params.out_code_embd = rows ? &out_embd : nullptr; + + if (!ctx->ctx_gen_a->encode(¶ms)) { + LOG_ERR("%s: encode failed (tokenize)\n", __func__); + return 1; + } + + codes = std::move(out_codes); + if (rows) { + *rows = std::move(out_embd); + } + return 0; +} + +// talker conditioning rows: the speaker row, then on mtl the perceiver block and the emotion row +static int32_t cbx_ref_cond(mtmd_context * ctx, const mtmd_image_f32 & ve_mel, + const std::vector & pse, std::vector & out_rows) { + mtmd_subcontext * sctx = ctx->ctx_a.get(); + // n_expect 0 accepts any non-empty size + auto read_t = [&](const char * name, std::vector & v, size_t n_expect) -> bool { + const size_t n = sctx->model.cbx_read_tensor(name, nullptr, 0); + if (n == 0 || (n_expect > 0 && n != n_expect)) { + return false; + } + v.resize(n); + return sctx->model.cbx_read_tensor(name, v.data(), n) == n; + }; + + // voice encoder (embeds_from_wavs): 3-layer lstm per 160-frame partial, the normalized partial embeddings are averaged + const std::vector & mel = ve_mel.get_ro_buf(); + const int n_mel = 40; + const int n_partial = 160; + const int n_h = 256; + const int step = (int) lround((16000.0 / 1.3) / n_partial); // reference rate 1.3 + if (ve_mel.nx() < n_partial || mel.size() < (size_t) ve_mel.nx() * n_mel) { + LOG_ERR("%s: voice encoder mel too short\n", __func__); + return 1; + } + const int n_wins = (ve_mel.nx() - n_partial) / step + 1; + + std::vector w_ih[3], w_hh[3], b_ih[3], b_hh[3]; + std::vector w_proj, b_proj; + for (int l = 0; l < 3; l++) { + const std::string s = std::to_string(l); + const size_t n_in = l == 0 ? n_mel : n_h; + if (!read_t(("a.ve.lstm.weight_ih_l" + s).c_str(), w_ih[l], 4 * n_h * n_in) || + !read_t(("a.ve.lstm.weight_hh_l" + s).c_str(), w_hh[l], 4 * n_h * n_h) || + !read_t(("a.ve.lstm.bias_ih_l" + s).c_str(), b_ih[l], 4 * n_h) || + !read_t(("a.ve.lstm.bias_hh_l" + s).c_str(), b_hh[l], 4 * n_h)) { + LOG_ERR("%s: model has no voice encoder\n", __func__); + return 1; + } + } + if (!read_t("a.ve.proj.weight", w_proj, n_h * n_h) || !read_t("a.ve.proj.bias", b_proj, n_h)) { + LOG_ERR("%s: model has no voice encoder projection\n", __func__); + return 1; + } + + std::vector ve(n_h, 0.0f); + std::vector h((size_t) 3 * n_h), c((size_t) 3 * n_h), x(n_h), g((size_t) 4 * n_h); + for (int p = 0; p < n_wins; p++) { + std::fill(h.begin(), h.end(), 0.0f); + std::fill(c.begin(), c.end(), 0.0f); + for (int t = 0; t < n_partial; t++) { + const float * in = mel.data() + (size_t) (p * step + t) * n_mel; + int n_in = n_mel; + for (int l = 0; l < 3; l++) { + float * hl = h.data() + (size_t) l * n_h; + float * cl = c.data() + (size_t) l * n_h; + for (int j = 0; j < 4 * n_h; j++) { + double acc = b_ih[l][(size_t) j] + b_hh[l][(size_t) j]; + const float * wi = w_ih[l].data() + (size_t) j * n_in; + for (int i = 0; i < n_in; i++) { + acc += (double) wi[i] * in[i]; + } + const float * wh = w_hh[l].data() + (size_t) j * n_h; + for (int i = 0; i < n_h; i++) { + acc += (double) wh[i] * hl[i]; + } + g[(size_t) j] = (float) acc; + } + // torch gate order: input, forget, cell, output + for (int i = 0; i < n_h; i++) { + const float gi = 1.0f / (1.0f + expf(-g[(size_t) i])); + const float gf = 1.0f / (1.0f + expf(-g[(size_t) i + n_h])); + const float gc = tanhf(g[(size_t) i + 2 * n_h]); + const float go = 1.0f / (1.0f + expf(-g[(size_t) i + 3 * n_h])); + cl[i] = gf * cl[i] + gi * gc; + x[(size_t) i] = go * tanhf(cl[i]); + } + memcpy(hl, x.data(), (size_t) n_h * sizeof(float)); + in = hl; + n_in = n_h; + } + } + // projected, relu'd, normalized partial embedding + std::vector e(n_h); + double norm = 0.0; + for (int o = 0; o < n_h; o++) { + double acc = b_proj[(size_t) o]; + const float * w = w_proj.data() + (size_t) o * n_h; + const float * hl = h.data() + (size_t) 2 * n_h; + for (int i = 0; i < n_h; i++) { + acc += (double) w[i] * hl[i]; + } + e[(size_t) o] = (float) std::max(acc, 0.0); + norm += (double) e[(size_t) o] * e[(size_t) o]; + } + norm = sqrt(norm); + for (int o = 0; o < n_h; o++) { + ve[(size_t) o] += (float) (e[(size_t) o] / norm); + } + } + double norm = 0.0; + for (float v : ve) { + norm += (double) v * v; + } + norm = sqrt(norm); + for (float & v : ve) { + v = (float) (v / norm); + } + + // speaker projection row + const int n_e = (int) ctx->n_embd_out(); + std::vector w_spkr, b_spkr; + if (!read_t("a.cenc.spkr_enc.weight", w_spkr, (size_t) n_e * n_h) || !read_t("a.cenc.spkr_enc.bias", b_spkr, n_e)) { + LOG_ERR("%s: model has no speaker conditioning projection\n", __func__); + return 1; + } + std::vector rows((size_t) n_e); + for (int o = 0; o < n_e; o++) { + double acc = b_spkr[(size_t) o]; + const float * w = w_spkr.data() + (size_t) o * n_h; + for (int i = 0; i < n_h; i++) { + acc += (double) w[i] * ve[(size_t) i]; + } + rows[(size_t) o] = (float) acc; + } + + // the perceiver runs its shared attention block as cross then self attention + std::vector query; + if (read_t("a.cenc.perceiver.pre_attention_query", query, 0)) { + const int n_head = 4; + const size_t n_ee = (size_t) n_e * n_e; + if (pse.empty() || pse.size() % n_e != 0 || query.size() % n_e != 0 || n_e % n_head != 0) { + LOG_ERR("%s: reference speech embeddings required for the perceiver\n", __func__); + return 1; + } + std::vector ln_w, ln_b, wq, bq, wk, bk, wv, bv, wo, bo, emo; + if (!read_t("a.cenc.perceiver.attn.norm.weight", ln_w, n_e) || !read_t("a.cenc.perceiver.attn.norm.bias", ln_b, n_e) || + !read_t("a.cenc.perceiver.attn.to_q.weight", wq, n_ee) || !read_t("a.cenc.perceiver.attn.to_q.bias", bq, n_e) || + !read_t("a.cenc.perceiver.attn.to_k.weight", wk, n_ee) || !read_t("a.cenc.perceiver.attn.to_k.bias", bk, n_e) || + !read_t("a.cenc.perceiver.attn.to_v.weight", wv, n_ee) || !read_t("a.cenc.perceiver.attn.to_v.bias", bv, n_e) || + !read_t("a.cenc.perceiver.attn.proj_out.weight", wo, n_ee) || !read_t("a.cenc.perceiver.attn.proj_out.bias", bo, n_e) || + !read_t("a.cenc.emotion_adv_fc.weight", emo, n_e)) { + LOG_ERR("%s: model has an incomplete perceiver\n", __func__); + return 1; + } + const int d_head = n_e / n_head; + + auto layer_norm = [&](const float * in, float * out) { + double mean = 0.0, var = 0.0; + for (int i = 0; i < n_e; i++) { + mean += in[i]; + } + mean /= n_e; + for (int i = 0; i < n_e; i++) { + var += (in[i] - mean) * (in[i] - mean); + } + const double sd = sqrt(var / n_e + 1e-5); + for (int i = 0; i < n_e; i++) { + out[i] = (float) ((in[i] - mean) / sd * ln_w[(size_t) i] + ln_b[(size_t) i]); + } + }; + auto linear = [&](const std::vector & w, const std::vector & b, + const std::vector & in, int n_rows, std::vector & out) { + out.resize((size_t) n_rows * n_e); + for (int r = 0; r < n_rows; r++) { + for (int o = 0; o < n_e; o++) { + double acc = b[(size_t) o]; + const float * wr = w.data() + (size_t) o * n_e; + const float * ir = in.data() + (size_t) r * n_e; + for (int i = 0; i < n_e; i++) { + acc += (double) wr[i] * ir[i]; + } + out[(size_t) r * n_e + o] = (float) acc; + } + } + }; + auto attn_block = [&](const std::vector & x1, int n1, + const std::vector & x2, int n2, std::vector & out) { + std::vector nx1((size_t) n1 * n_e), nx2((size_t) n2 * n_e); + for (int r = 0; r < n1; r++) { + layer_norm(x1.data() + (size_t) r * n_e, nx1.data() + (size_t) r * n_e); + } + for (int r = 0; r < n2; r++) { + layer_norm(x2.data() + (size_t) r * n_e, nx2.data() + (size_t) r * n_e); + } + std::vector q, k, v; + linear(wq, bq, nx1, n1, q); + linear(wk, bk, nx2, n2, k); + linear(wv, bv, nx2, n2, v); + + std::vector ctxt((size_t) n1 * n_e); + std::vector sc((size_t) n2); + for (int hd = 0; hd < n_head; hd++) { + const int off = hd * d_head; + for (int t = 0; t < n1; t++) { + double mx = -1e30; + for (int s = 0; s < n2; s++) { + double acc = 0.0; + for (int i = 0; i < d_head; i++) { + acc += (double) q[(size_t) t * n_e + off + i] * k[(size_t) s * n_e + off + i]; + } + sc[(size_t) s] = acc / sqrt((double) d_head); + mx = std::max(mx, sc[(size_t) s]); + } + double sum = 0.0; + for (int s = 0; s < n2; s++) { + sc[(size_t) s] = exp(sc[(size_t) s] - mx); + sum += sc[(size_t) s]; + } + for (int i = 0; i < d_head; i++) { + double acc = 0.0; + for (int s = 0; s < n2; s++) { + acc += sc[(size_t) s] * v[(size_t) s * n_e + off + i]; + } + ctxt[(size_t) t * n_e + off + i] = (float) (acc / sum); + } + } + } + linear(wo, bo, ctxt, n1, out); + for (size_t i = 0; i < out.size(); i++) { + out[i] += x1[i]; + } + }; + + const int n_q = (int) (query.size() / n_e); + std::vector pre, p32; + attn_block(query, n_q, pse, (int) (pse.size() / (size_t) n_e), pre); + attn_block(pre, n_q, pre, n_q, p32); + + rows.insert(rows.end(), p32.begin(), p32.end()); + const float exaggeration = 0.5f; // reference default + for (int i = 0; i < n_e; i++) { + rows.push_back(emo[(size_t) i] * exaggeration); + } + } + + out_rows = std::move(rows); + return 0; +} + +// talker conditioning rows go to out_embd, the flow decoder reference to ctx->gen_ref +static int32_t cbx_ref_encode(mtmd_context * ctx, const mtmd_audio_tokens * audio_tokens, std::vector & out_embd) { + if (!ctx->ctx_gen_a) { + LOG_ERR("%s: reference encoding needs the audio generation context\n", __func__); + return 1; + } + const auto & entries = audio_tokens->batch_f32.entries; + GGML_ASSERT(entries.size() == 5); + + std::vector xvec; + if (!cbx_ref_xvec(ctx, entries[0], xvec)) { + return 1; + } + + std::vector flow_codes, t3_codes; + std::vector t3_rows; + if (cbx_ref_tokenize(ctx, entries[1], flow_codes, nullptr) != 0 || + cbx_ref_tokenize(ctx, entries[2], t3_codes, &t3_rows) != 0) { + return 1; + } + + const size_t n_feat = (size_t) entries[3].nx() * 80; + if ((size_t) entries[3].nx() != 2 * flow_codes.size()) { + LOG_ERR("%s: reference mel length %d does not match %zu codes\n", + __func__, entries[3].nx(), flow_codes.size()); + return 1; + } + + // [spkr, perceiver block] on mtl, [spkr, reference speech rows] on turbo + const bool mtl = cbx_is_mtl(ctx->ctx_a->model); + std::vector pse; + if (mtl) { + pse = std::move(t3_rows); + } + std::vector rows; + if (cbx_ref_cond(ctx, entries[4], pse, rows) != 0) { + return 1; + } + if (!mtl) { + rows.insert(rows.end(), t3_rows.begin(), t3_rows.end()); + } + if (rows.size() != (size_t) audio_tokens->n_tokens * (size_t) ctx->n_embd_out()) { + LOG_ERR("%s: unexpected number of conditioning rows\n", __func__); + return 1; + } + + ctx->gen_ref.tokens = std::move(flow_codes); + ctx->gen_ref.feat.assign(entries[3].get_ro_buf().begin(), entries[3].get_ro_buf().begin() + n_feat); + ctx->gen_ref.spk = std::move(xvec); + + out_embd = std::move(rows); + return 0; +} + const char * mtmd_gen_audio_voice_name(const mtmd_context * ctx, int32_t idx) { if (!ctx->ctx_gen_a || idx < 0) { return nullptr; } - const auto & voices = clip_gen_get_voices(ctx->ctx_gen_a); + const auto & voices = ctx->ctx_gen_a->model.gen_voices; return idx < (int32_t) voices.size() ? std::next(voices.begin(), idx)->first.c_str() : nullptr; } @@ -2318,7 +2825,7 @@ mtmd_bitmap * mtmd_gen_audio_get_voice(const mtmd_context * ctx, const char * na return nullptr; } // the name is matched case-insensitively, as the lfm3a "tts" chat template lowers it - const auto & voices = clip_gen_get_voices(ctx->ctx_gen_a); + const auto & voices = ctx->ctx_gen_a->model.gen_voices; const auto it = std::find_if(voices.begin(), voices.end(), [name](const auto & voice) { return std::equal(voice.first.begin(), voice.first.end(), name, name + strlen(name), [](unsigned char a, unsigned char b) { return std::tolower(a) == std::tolower(b); }); @@ -2326,8 +2833,7 @@ mtmd_bitmap * mtmd_gen_audio_get_voice(const mtmd_context * ctx, const char * na if (it == voices.end()) { return nullptr; } - const uint32_t n_embd = clip_n_mmproj_embd(ctx->ctx_gen_a); - return mtmd_bitmap::init_voice(it->first, it->second.data(), n_embd, it->second.size() / n_embd); + return mtmd_bitmap::init_voice(it->first, it->second, ctx->ctx_gen_a->model.n_mmproj_embd()); } mtmd_gen_inp mtmd_gen_inp_default(const mtmd_context * ctx) { @@ -2337,7 +2843,7 @@ mtmd_gen_inp mtmd_gen_inp_default(const mtmd_context * ctx) { return inp; } - switch (clip_get_projector_type(ctx->ctx_gen_a)) { + switch (ctx->ctx_gen_a->model.proj_type) { case PROJECTOR_TYPE_QWEN3TTS_GEN: // https://huggingface.co/Qwen/Qwen3-TTS-12Hz-1.7B-Base/blob/main/generation_config.json inp.top_k = 50; @@ -2348,15 +2854,21 @@ mtmd_gen_inp mtmd_gen_inp_default(const mtmd_context * ctx) { // https://github.com/kyutai-labs/pocket-tts/blob/main/pocket_tts/default_parameters.py inp.top_k = 50; inp.top_p = 1.0f; - inp.temp = 0.7f; + inp.temp = 0.3f; break; case PROJECTOR_TYPE_SOPRANO: inp.type = MTMD_GEN_PROCESS_TYPE_GEN_WAV; break; + case PROJECTOR_TYPE_DRAMA3_GEN: + // codebook heads, same as the reference (sglang-omni fishaudio_drama3) + inp.top_k = 30; + inp.top_p = 0.9f; + inp.temp = 1.0f; + break; case PROJECTOR_TYPE_LFM3A_GEN: { // MaskGIT sampling runs in the helper, with the settings of the checkpoint - const auto * hparams = clip_get_hparams(ctx->ctx_gen_a); + const auto * hparams = &ctx->ctx_gen_a->model.hparams; inp.top_k = hparams->gen_top_k; inp.top_p = 1.0f; inp.temp = hparams->gen_temp; @@ -2370,19 +2882,22 @@ mtmd_gen_inp mtmd_gen_inp_default(const mtmd_context * ctx) { } struct mtmd_gen_state { - clip_ctx * ctx_clip = nullptr; - clip_gen_state * clip_state = nullptr; + mtmd_subcontext * sctx = nullptr; + mtmd_subctx_gen_state * gen_state = nullptr; std::vector out_codes; // this frame's 16 sampled codes (GEN_CODE) std::vector out_feats; // this frame's continuous features, if any (GEN_CODE) std::vector out_embd; // next-step hidden state fed back to backbone (GEN_CODE) std::vector out_audio; // decoded PCM samples for the current frame (GEN_WAV) + // voice reference for GEN_WAV, from a speaker clip or a predefined voice (chatterbox) + mtmd_gen_ref ref; + // lfm3a: the codec graph yields STFT frames, the ISTFT runs on the host std::unique_ptr istft; ~mtmd_gen_state() { - clip_gen_state_free(clip_state); + delete gen_state; } }; @@ -2398,13 +2913,13 @@ mtmd_gen_state * mtmd_gen_state_init(mtmd_context * ctx, mtmd_gen_state_params p return nullptr; } auto st = std::make_unique(); - st->ctx_clip = ctx->ctx_gen_a; - st->clip_state = clip_gen_state_init(ctx->ctx_gen_a, params.seed); - if (!st->clip_state) { + st->sctx = ctx->ctx_gen_a.get(); + st->gen_state = ctx->ctx_gen_a->gen_state_init(params.seed); + if (!st->gen_state) { return nullptr; } - if (clip_get_projector_type(ctx->ctx_gen_a) == PROJECTOR_TYPE_LFM3A_GEN) { - const auto * hparams = clip_get_hparams(ctx->ctx_gen_a); + if (ctx->ctx_gen_a->model.proj_type == PROJECTOR_TYPE_LFM3A_GEN) { + const auto * hparams = &ctx->ctx_gen_a->model.hparams; st->istft = std::make_unique(hparams->audio_n_fft, hparams->audio_hop_len); } return st.release(); @@ -2415,17 +2930,17 @@ void mtmd_gen_state_free(mtmd_gen_state * state) { } void mtmd_gen_state_reset(mtmd_gen_state * state) { - clip_gen_state_reset(state->clip_state); + state->gen_state->reset(); if (state->istft) { - const auto * hparams = clip_get_hparams(state->ctx_clip); + const auto * hparams = &state->sctx->model.hparams; state->istft = std::make_unique(hparams->audio_n_fft, hparams->audio_hop_len); } } // lfm3a GEN_WAV: ISTFT of the codec output, n_codes == 0 flushes and resets the state static int32_t mtmd_gen_wav_lfm3a(mtmd_context * ctx, mtmd_gen_state * state, const mtmd_gen_inp * inp, mtmd_gen_out * out) { - clip_ctx * ctx_clip = ctx->ctx_gen_a; - const auto & hparams = *clip_get_hparams(ctx_clip); + mtmd_subcontext * sctx = ctx->ctx_gen_a.get(); + const auto & hparams = sctx->model.hparams; const int n_bins = hparams.audio_n_fft / 2 + 1; const int n_trim = (hparams.audio_n_fft - hparams.audio_hop_len) / 2; @@ -2443,23 +2958,23 @@ static int32_t mtmd_gen_wav_lfm3a(mtmd_context * ctx, mtmd_gen_state * state, co std::vector feats; // no hidden-state input, the batch entry is an unused placeholder - clip_image_f32 dummy; + mtmd_image_f32 dummy; dummy.set_size({1, 1}, false, true); dummy.cpy_buf(std::vector(1, 0.0f)); - clip_image_f32_batch batch; + mtmd_image_f32_batch batch; batch.is_audio = true; batch.entries.push_back(std::move(dummy)); - clip_encode_params params; + mtmd_subctx_encode_params params; params.imgs = &batch; params.n_threads = ctx->n_threads; - params.gen_process = CLIP_GEN_PROCESS_GEN_WAV; + params.gen_process = MTMD_SUBCTX_GEN_PROCESS_GEN_WAV; params.codes = &in_codes; params.out_feats = &feats; - params.gen_state = state->clip_state; - if (!clip_encode(ctx_clip, ¶ms)) { - LOG_ERR("%s: clip_encode failed (codec)\n", __func__); + params.gen_state = state->gen_state; + if (!sctx->encode(¶ms)) { + LOG_ERR("%s: encode failed (codec)\n", __func__); return 1; } @@ -2484,36 +2999,49 @@ static int32_t mtmd_gen_wav_lfm3a(mtmd_context * ctx, mtmd_gen_state * state, co return 0; } +int32_t mtmd_gen_state_set_ref(mtmd_context * ctx, mtmd_gen_state * state) { + if (!state || state->sctx != ctx->ctx_gen_a.get()) { + LOG_ERR("%s: invalid gen state\n", __func__); + return 1; + } + if (ctx->gen_ref.tokens.empty()) { + LOG_ERR("%s: the last encode produced no reference data\n", __func__); + return 1; + } + state->ref = ctx->gen_ref; + return 0; +} + static int32_t mtmd_gen_audio_process_impl(mtmd_context * ctx, mtmd_gen_state * state, const mtmd_gen_inp * inp, mtmd_gen_out * out) { - clip_ctx * ctx_clip = ctx->ctx_gen_a; - if (!ctx_clip) { + mtmd_subcontext * sctx = ctx->ctx_gen_a.get(); + if (!sctx) { LOG_ERR("%s: model does not support audio generation\n", __func__); return 1; } - if (!state || state->ctx_clip != ctx_clip) { + if (!state || state->sctx != sctx) { LOG_ERR("%s: invalid gen state\n", __func__); return 1; } *out = {}; - if (clip_get_projector_type(ctx_clip) == PROJECTOR_TYPE_SOPRANO) { - const size_t n_embd = (size_t) clip_n_mmproj_embd(ctx_clip); + if (sctx->model.proj_type == PROJECTOR_TYPE_SOPRANO) { + const size_t n_embd = (size_t) sctx->model.n_mmproj_embd(); if (inp->type != MTMD_GEN_PROCESS_TYPE_GEN_WAV || !inp->feats || inp->n_feats < 2 * n_embd || inp->n_feats % n_embd != 0 || inp->codes) { LOG_ERR("%s: soprano requires at least 2 frames of features, without codes\n", __func__); return 1; } } - if (inp->type == MTMD_GEN_PROCESS_TYPE_GEN_WAV && clip_get_projector_type(ctx_clip) == PROJECTOR_TYPE_LFM3A_GEN) { + if (inp->type == MTMD_GEN_PROCESS_TYPE_GEN_WAV && sctx->model.proj_type == PROJECTOR_TYPE_LFM3A_GEN) { return mtmd_gen_wav_lfm3a(ctx, state, inp, out); } if (inp->type == MTMD_GEN_PROCESS_TYPE_GEN_CODE) { - const size_t n_embd = (size_t) clip_n_mmproj_embd(ctx_clip); + const size_t n_embd = (size_t) sctx->model.n_mmproj_embd(); // the hidden state may be omitted when the graph does not use it - clip_image_f32 hidden_state; + mtmd_image_f32 hidden_state; hidden_state.set_size({(int) n_embd, 1}, false, true); hidden_state.cpy_buf(inp->embd ? std::vector(inp->embd, inp->embd + n_embd) : std::vector(n_embd, 0.0f)); @@ -2522,7 +3050,7 @@ static int32_t mtmd_gen_audio_process_impl(mtmd_context * ctx, mtmd_gen_state * in_codes.assign(inp->codes, inp->codes + inp->n_codes); } - clip_image_f32_batch batch; + mtmd_image_f32_batch batch; batch.is_audio = true; batch.entries.push_back(std::move(hidden_state)); @@ -2531,10 +3059,10 @@ static int32_t mtmd_gen_audio_process_impl(mtmd_context * ctx, mtmd_gen_state * std::vector out_feats; bool is_eos = false; - clip_encode_params params; + mtmd_subctx_encode_params params; params.imgs = &batch; params.n_threads = ctx->n_threads; - params.gen_process = CLIP_GEN_PROCESS_GEN_CODE; + params.gen_process = MTMD_SUBCTX_GEN_PROCESS_GEN_CODE; params.out_embd = &out_embd; params.out_codes = &out_codes; params.out_feats = &out_feats; @@ -2544,10 +3072,10 @@ static int32_t mtmd_gen_audio_process_impl(mtmd_context * ctx, mtmd_gen_state * params.temp = inp->temp; params.codes = inp->codes ? &in_codes : nullptr; params.out_is_eos = &is_eos; - params.gen_state = state->clip_state; + params.gen_state = state->gen_state; - if (!clip_encode(ctx_clip, ¶ms)) { - LOG_ERR("%s: clip_encode failed (gen_code)\n", __func__); + if (!sctx->encode(¶ms)) { + LOG_ERR("%s: encode failed (gen_code)\n", __func__); return 1; } @@ -2565,6 +3093,44 @@ static int32_t mtmd_gen_audio_process_impl(mtmd_context * ctx, mtmd_gen_state * } // MTMD_GEN_PROCESS_TYPE_GEN_WAV + if (sctx->model.proj_type == PROJECTOR_TYPE_CHATTERBOX) { + if (!inp->codes || inp->n_codes == 0) { + LOG_ERR("%s: codes required for gen_wav\n", __func__); + return 1; + } + if (state->ref.tokens.empty()) { + LOG_ERR("%s: gen_wav needs a voice reference, see mtmd_gen_state_set_ref()\n", __func__); + return 1; + } + std::vector in_codes(inp->codes, inp->codes + inp->n_codes); + + // the batch entry is unused, present to satisfy the encode interface + mtmd_image_f32 dummy; + dummy.set_size({1, 1}, false, true); + dummy.cpy_buf(std::vector(1, 0.0f)); + mtmd_image_f32_batch batch; + batch.is_audio = true; + batch.entries.push_back(std::move(dummy)); + + mtmd_subctx_encode_params params; + params.imgs = &batch; + params.n_threads = ctx->n_threads; + params.gen_process = MTMD_SUBCTX_GEN_PROCESS_TTS; + params.codes = &in_codes; + params.out_audio = &state->out_audio; + params.ref = &state->ref; + params.gen_state = state->gen_state; + + if (!sctx->encode(¶ms)) { + LOG_ERR("%s: encode failed (gen_wav)\n", __func__); + return 1; + } + + out->audio = state->out_audio.data(); + out->n_samples = state->out_audio.size(); + return 0; + } + const bool has_codes = inp->codes && inp->n_codes > 0; const bool has_feats = inp->feats && inp->n_feats > 0; if (has_codes == has_feats) { @@ -2581,25 +3147,25 @@ static int32_t mtmd_gen_audio_process_impl(mtmd_context * ctx, mtmd_gen_state * // gen_wav has no hidden-state input, the batch entry is an unused placeholder // TODO @ngxson : some models in the future may require hidden-state input, need to update this code later - clip_image_f32 dummy; + mtmd_image_f32 dummy; dummy.set_size({1, 1}, false, true); dummy.cpy_buf(std::vector(1, 0.0f)); - clip_image_f32_batch batch; + mtmd_image_f32_batch batch; batch.is_audio = true; batch.entries.push_back(std::move(dummy)); - clip_encode_params params; + mtmd_subctx_encode_params params; params.imgs = &batch; params.n_threads = ctx->n_threads; - params.gen_process = CLIP_GEN_PROCESS_GEN_WAV; + params.gen_process = MTMD_SUBCTX_GEN_PROCESS_GEN_WAV; params.codes = has_codes ? &in_codes : nullptr; params.feats = has_feats ? &in_feats : nullptr; params.out_audio = &state->out_audio; - params.gen_state = state->clip_state; + params.gen_state = state->gen_state; - if (!clip_encode(ctx_clip, ¶ms)) { - LOG_ERR("%s: clip_encode failed (code2wav)\n", __func__); + if (!sctx->encode(¶ms)) { + LOG_ERR("%s: encode failed (code2wav)\n", __func__); return 1; } @@ -2610,6 +3176,7 @@ static int32_t mtmd_gen_audio_process_impl(mtmd_context * ctx, mtmd_gen_state * } int32_t mtmd_gen_audio_process(mtmd_context * ctx, mtmd_gen_state * state, const struct mtmd_gen_inp * inp, struct mtmd_gen_out * out) { + std::lock_guard lock(ctx->compute_mutex); try { return mtmd_gen_audio_process_impl(ctx, state, inp, out); } catch (const std::exception & e) { @@ -2634,7 +3201,7 @@ int32_t mtmd_batch_add_chunk(mtmd_batch * batch, const mtmd_input_chunk * chunk) return 1; } - auto * ctx = batch->ctx->get_clip_ctx(chunk); + auto * ctx = batch->ctx->get_subctx(chunk); if (!ctx) { LOG_ERR("%s: model does not support input chunk type %d\n", __func__, (int)chunk->type); return 1; @@ -2646,7 +3213,7 @@ int32_t mtmd_batch_add_chunk(mtmd_batch * batch, const mtmd_input_chunk * chunk) return 0; } - if (!clip_support_batch(ctx)) { + if (!ctx->support_batch) { // if no batching support, batch can only have one single chunk return 2; // "batch too large" error code } @@ -2718,6 +3285,7 @@ static int32_t mtmd_batch_encode_impl(mtmd_batch * batch) { } int32_t mtmd_batch_encode(mtmd_batch * batch) { + std::lock_guard lock(batch->ctx->compute_mutex); try { return mtmd_batch_encode_impl(batch); } catch (const std::exception & e) { @@ -2756,7 +3324,7 @@ bool mtmd_decode_use_non_causal(const mtmd_context * ctx, const mtmd_input_chunk { // E2B (n_embd = 1536) and E4B (n_embd = 2560) always use causal // note: use mmproj n_embd, because text model may not be provided (e.g. mtmd_get_memory_usage) - const int n_embd = clip_n_mmproj_embd(ctx->ctx_v); + const int n_embd = ctx->ctx_v->model.n_mmproj_embd(); return n_embd != 1536 && n_embd != 2560; } case PROJECTOR_TYPE_GEMMA4UV: @@ -2785,39 +3353,54 @@ int mtmd_get_audio_sample_rate(const mtmd_context * ctx) { if (!ctx->ctx_a) { return -1; } - return clip_get_hparams(ctx->ctx_a)->audio_sample_rate; + return ctx->ctx_a->model.hparams.audio_sample_rate; } -mtmd_video_config mtmd_get_video_config(const mtmd_context * ctx, - int32_t src_width, int32_t src_height, - int32_t src_n_frames, float src_fps) { - mtmd_video_config cfg{MTMD_VIDEO_TEMPLATE_DEFAULT, 0.0f, 0, 0, 0}; - if (!ctx->ctx_v || clip_get_projector_type(ctx->ctx_v) != PROJECTOR_TYPE_LFM3VL) { +mtmd_video_config mtmd_video_config_init(mtmd_video_template tmpl, const mtmd_image_preprocessor * preproc, + int32_t src_width, int32_t src_height, + int32_t src_n_frames, float src_fps) { + mtmd_video_config cfg{tmpl, 0.0f, 0, 0, 0}; + if (tmpl == MTMD_VIDEO_TEMPLATE_DEFAULT) { return cfg; } - // matches Lfm3VlVideoProcessor: 2 fps sampling with a [4, 768] frame clamp + // matches Qwen3VLVideoProcessor and Lfm3VlVideoProcessor: 2 fps sampling with a [4, 768] frame clamp // LFM3-Omni samples at most 384 frames and hears the soundtrack - const bool omni = ctx->video_has_soundtrack(); - cfg.tmpl = omni ? MTMD_VIDEO_TEMPLATE_LFM3_OMNI : MTMD_VIDEO_TEMPLATE_LFM3; - cfg.fps = 2.0f; + cfg.fps = 2.0f; if (src_n_frames <= 0 || src_fps <= 0.0f) { return cfg; } int32_t n = (int32_t)((double) src_n_frames / src_fps * cfg.fps); - n = std::clamp(n, 4, omni ? 384 : 768); + n = std::clamp(n, 4, tmpl == MTMD_VIDEO_TEMPLATE_LFM3_OMNI ? 384 : 768); n = std::min(n, src_n_frames); cfg.n_frames = n; - clip_image_size size; - if (ctx->image_preproc && ctx->image_preproc->get_video_frame_size(n, src_width, src_height, size)) { + mtmd_image_size size; + if (preproc && preproc->get_video_frame_size(n, src_width, src_height, size)) { cfg.frame_width = size.width; cfg.frame_height = size.height; } return cfg; } +mtmd_video_config mtmd_get_video_config(const mtmd_context * ctx, + int32_t src_width, int32_t src_height, + int32_t src_n_frames, float src_fps) { + mtmd_video_template tmpl = MTMD_VIDEO_TEMPLATE_DEFAULT; + switch (ctx->ctx_v ? ctx->ctx_v->model.proj_type : PROJECTOR_TYPE_UNKNOWN) { + case PROJECTOR_TYPE_LFM3VL: + tmpl = ctx->video_has_soundtrack() ? MTMD_VIDEO_TEMPLATE_LFM3_OMNI : MTMD_VIDEO_TEMPLATE_LFM3; + break; + case PROJECTOR_TYPE_QWEN3VL: + tmpl = MTMD_VIDEO_TEMPLATE_QWEN3VL; + break; + default: + break; + } + return mtmd_video_config_init(tmpl, ctx->image_preproc.get(), src_width, src_height, src_n_frames, src_fps); +} + const char * mtmd_get_marker(const mtmd_context * ctx) { return ctx->media_marker.c_str(); } @@ -3131,7 +3714,7 @@ static mtmd_decoder_pos mtmd_canvas_decoder_pos(const mtmd_image_tokens * image_ } i -= gw * gh; - const size_t n_sep = clip_suffix_rows(entries[k].suffix_type).size(); + const size_t n_sep = mtmd_suffix_rows(entries[k].suffix_type).size(); if (i < n_sep) { const uint32_t p = t + 1 + i; pos.t = p; @@ -3233,7 +3816,7 @@ llama_pos mtmd_image_tokens_get_n_pos(const mtmd_image_tokens * image_tokens) { // one position per tile, plus one per separator llama_pos n_pos = 0; for (const auto & entry : image_tokens->batch_f32.entries) { - n_pos += 1 + (llama_pos) clip_suffix_rows(entry.suffix_type).size(); + n_pos += 1 + (llama_pos) mtmd_suffix_rows(entry.suffix_type).size(); } return n_pos; } @@ -3277,14 +3860,16 @@ mtmd_input_chunks * mtmd_test_create_input_chunks() { return chunks; } +struct mtmd_logger_state g_logger_state = {mtmd_log_callback_default, NULL}; + void mtmd_log_set(ggml_log_callback log_callback, void * user_data) { - g_logger_state.log_callback = log_callback ? log_callback : clip_log_callback_default; + g_logger_state.log_callback = log_callback ? log_callback : mtmd_log_callback_default; g_logger_state.log_callback_user_data = user_data; } struct mtmd_caps mtmd_get_cap_from_file(const char * fname) { try { - auto tmp = clip_get_cap(fname); + auto tmp = mtmd_model_get_caps(fname); mtmd_caps cap; cap.inp_audio = tmp.has_audio; cap.inp_vision = tmp.has_vision; @@ -3300,14 +3885,12 @@ struct mtmd_caps mtmd_get_cap_from_file(const char * fname) { // Debugging API (NOT intended for public use) // -static void mtmd_debug_encode_impl(mtmd_context * ctx, clip_ctx * ctx_clip, clip_image_f32 & image) { - clip_set_debug_output_embeddings(ctx_clip, true); - int n_mmproj_embd = clip_n_mmproj_embd(ctx_clip); - int n_tokens = clip_n_output_tokens(ctx_clip, &image); +static void mtmd_debug_encode_impl(mtmd_context * ctx, mtmd_subcontext * sctx, mtmd_image_f32 & image) { + sctx->debug_output_embeddings = true; + int n_mmproj_embd = sctx->model.n_mmproj_embd(); + int n_tokens = sctx->model.n_output_tokens(&image); std::vector embd_output(n_tokens * n_mmproj_embd, 0.0f); - bool ok = clip_image_encode( - ctx_clip, - ctx->n_threads, + bool ok = sctx->encode_image(ctx->n_threads, &image, embd_output); if (!ok) { @@ -3316,6 +3899,7 @@ static void mtmd_debug_encode_impl(mtmd_context * ctx, clip_ctx * ctx_clip, clip } void mtmd_debug_encode_image(mtmd_context * ctx, const std::vector> & image) { + std::lock_guard lock(ctx->compute_mutex); if (!ctx->ctx_v) { LOG_ERR("%s: model does not support vision input\n", __func__); return; @@ -3326,19 +3910,20 @@ void mtmd_debug_encode_image(mtmd_context * ctx, const std::vectorctx_v, inp_image); + mtmd_debug_encode_impl(ctx, ctx->ctx_v.get(), inp_image); } void mtmd_debug_encode_audio(mtmd_context * ctx, const std::vector & input) { + std::lock_guard lock(ctx->compute_mutex); if (!ctx->ctx_a) { LOG_ERR("%s: model does not support audio input\n", __func__); return; } - int n_mel = clip_get_hparams(ctx->ctx_a)->n_mel_bins; + int n_mel = ctx->ctx_a->model.hparams.n_mel_bins; const int audio_nx = (int)input.size(); std::vector audio_buf(audio_nx * n_mel); for (int i = 0; i < audio_nx; i++) { @@ -3346,11 +3931,11 @@ void mtmd_debug_encode_audio(mtmd_context * ctx, const std::vector & inpu audio_buf[j * audio_nx + i] = input[i]; } } - clip_image_f32 inp_audio; + mtmd_image_f32 inp_audio; inp_audio.set_size({audio_nx, n_mel}, false, true); inp_audio.cpy_buf(audio_buf); LOG_INF("%s: created input audio with nx=%d, ny=%d\n", __func__, audio_nx, n_mel); - mtmd_debug_encode_impl(ctx, ctx->ctx_a, inp_audio); + mtmd_debug_encode_impl(ctx, ctx->ctx_a.get(), inp_audio); } void mtmd_debug_preprocess_image(mtmd_context * ctx, const std::vector & rgb_values, int nx, int ny) { @@ -3358,13 +3943,13 @@ void mtmd_debug_preprocess_image(mtmd_context * ctx, const std::vector LOG_ERR("%s: model does not support vision input\n", __func__); return; } - clip_image_u8 img_u8; + mtmd_image_u8 img_u8; img_u8.set_size({nx, ny}, false); img_u8.cpy_buf(rgb_values); GGML_ASSERT(ctx->image_preproc != nullptr); mtmd_image_preproc_out preproc_out = ctx->image_preproc->preprocess(img_u8); - clip_image_f32_batch batch_f32; + mtmd_image_f32_batch batch_f32; batch_f32.is_audio = false; for (auto & entry : preproc_out.entries) { batch_f32.entries.push_back(std::move(entry)); @@ -3378,25 +3963,26 @@ void mtmd_debug_preprocess_image(mtmd_context * ctx, const std::vector } void mtmd_debug_preprocess_audio(mtmd_context * ctx, const std::vector & samples) { + std::lock_guard lock(ctx->compute_mutex); if (!ctx->ctx_a || !ctx->audio_preproc) { LOG_ERR("%s: model does not support audio input\n", __func__); return; } - std::vector mel_spec_chunks; - bool ok = ctx->audio_preproc->preprocess(samples.data(), samples.size(), mel_spec_chunks); - if (!ok) { + mtmd_audio_preproc_out out = ctx->audio_preproc->preprocess(samples.data(), samples.size()); + if (out.entries.empty() || !ctx->audio_preproc->compute(out)) { LOG_ERR("%s: failed to preprocess audio\n", __func__); return; } - LOG_INF("%s: preprocessed audio to %zu mel spec chunks\n", __func__, mel_spec_chunks.size()); - for (size_t i = 0; i < mel_spec_chunks.size(); i++) { - LOG_INF("%s: mel spec chunk %zu has n_len=%d, n_mel=%d\n", __func__, i, mel_spec_chunks[i].n_len, mel_spec_chunks[i].n_mel); + LOG_INF("%s: preprocessed audio to %zu mel spec chunks\n", __func__, out.entries.size()); + for (size_t i = 0; i < out.entries.size(); i++) { + const auto & mel = out.entries[i]; + const auto & data = mel.get_ro_buf(); + LOG_INF("%s: mel spec chunk %zu has n_len=%d, n_mel=%d\n", __func__, i, mel.nx(), mel.ny()); // dump mel entries: data is stored as [n_mel][n_len] (mel-major) - const auto & mel = mel_spec_chunks[i]; - for (int m = 0; m < mel.n_mel; m++) { - for (int t = 0; t < mel.n_len; t++) { - LOG_INF("mel[%zu][m=%d][t=%d] = %f\n", i, m, t, mel.data[m * mel.n_len + t]); + for (int m = 0; m < mel.ny(); m++) { + for (int t = 0; t < mel.nx(); t++) { + LOG_INF("mel[%zu][m=%d][t=%d] = %f\n", i, m, t, data[(size_t) m * mel.nx() + t]); } } } @@ -3418,24 +4004,18 @@ mtmd_memory_usage mtmd_get_memory_usage(const char * mmproj_fname, mtmd_log_set(stub_log_callback, nullptr); // suppress logging ctx.reset(new mtmd_context(mmproj_fname, nullptr, ctx_params, true)); mtmd_log_set(saved_log_callback, saved_log_user_data); // restore log callback - std::map total_mem; - auto merge = [&](const struct clip_ctx * c) { - for (auto & [dev, size] : clip_get_mem_usage(c)) { - total_mem[dev] += size; + // compute buffers are shared, weights are per model + std::map total_mem = ctx->backends->get_mem_compute(); + for (const auto * m : { ctx->models.v.get(), ctx->models.a.get(), ctx->models.gen_a.get() }) { + if (m) { + for (auto & [dev, size] : m->mem_weights) { + total_mem[dev] += size; + } } - }; - if (ctx->ctx_v) { - merge(ctx->ctx_v); - } - if (ctx->ctx_a) { - merge(ctx->ctx_a); - } - if (ctx->ctx_gen_a) { - merge(ctx->ctx_gen_a); } mtmd_memory_usage res; res.backend_mem_usage = std::move(total_mem); - res.image_max_tokens = ctx->ctx_v ? clip_get_image_max_tokens(ctx->ctx_v) : -1; + res.image_max_tokens = ctx->ctx_v ? ctx->ctx_v->model.image_max_tokens() : -1; res.use_non_causal = ctx->ctx_v ? mtmd_decode_use_non_causal(ctx.get(), nullptr) : false; return res; } catch (const std::exception & e) { @@ -3452,7 +4032,7 @@ mtmd_memory_usage mtmd_get_memory_usage(const char * mmproj_fname, // the model-independent part of a stream, the model-specific part is in the subclasses below struct mtmd_audio_stream { mtmd_context * ctx; - clip_ctx * ctx_clip; + mtmd_subcontext * sctx; // model constants int n_mel; @@ -3496,10 +4076,10 @@ struct mtmd_audio_stream { // Voxtral Realtime: a row is an audio embedding plus the embedding of the previous token struct mtmd_audio_stream_voxtral_rt : mtmd_audio_stream { - clip_enc_state * enc_state = nullptr; + mtmd_subctx_enc_state * enc_state = nullptr; int n_fft; - int n_frames_max; // encoder frames per clip_encode() call + int n_frames_max; // encoder frames per encode() call int n_right_pad; // silence tokens added by flush() float log_mel_max; // global_log_mel_max in params.json @@ -3511,7 +4091,7 @@ struct mtmd_audio_stream_voxtral_rt : mtmd_audio_stream { std::vector tok_embd; ~mtmd_audio_stream_voxtral_rt() override { - clip_enc_state_free(enc_state); + delete enc_state; } bool init(const mtmd_audio_stream_params & params) override; @@ -3524,7 +4104,7 @@ struct mtmd_audio_stream_voxtral_rt : mtmd_audio_stream { // training samples are max 30 s: restart the sequence at a silent chunk after 15 s, always after 30 s; the encoder keeps its state struct mtmd_audio_stream_lfm3a : mtmd_audio_stream { int win; // samples per mel frame - clip_enc_state * enc_state = nullptr; + mtmd_subctx_enc_state * enc_state = nullptr; // live ASR static constexpr int64_t chunk_rows = 12; @@ -3554,7 +4134,7 @@ struct mtmd_audio_stream_lfm3a : mtmd_audio_stream { bool end_done = false; ~mtmd_audio_stream_lfm3a() override { - clip_enc_state_free(enc_state); + delete enc_state; } bool init(const mtmd_audio_stream_params & params) override; @@ -3565,12 +4145,40 @@ struct mtmd_audio_stream_lfm3a : mtmd_audio_stream { int32_t step_live(llama_token last_token); }; +// Gemma 4: the encoder is causal +struct mtmd_audio_stream_gemma4a : mtmd_audio_stream { + mtmd_subctx_enc_state * enc_state = nullptr; + + int n_fft; + int win; + int pad_left; + int64_t seg_len; + int64_t seg_start = 0; // first sample of the current segment, n_frames counts its frames + + std::vector beg_rows; + std::vector end_rows; + bool beg_done = false; + bool end_done = false; + + ~mtmd_audio_stream_gemma4a() override { + delete enc_state; + } + + bool init(const mtmd_audio_stream_params & params) override; + int32_t flush(bool end) override; + int32_t step(llama_token last_token) override; + + int64_t n_seg_frames(int64_t n, bool complete) const; + bool compute_mel(int64_t n_seg, bool complete); + bool encode(int n); +}; + bool mtmd_support_audio_stream(const mtmd_context * ctx) { if (!ctx->ctx_a || !ctx->text_model) { return false; } - const projector_type proj = clip_get_projector_type(ctx->ctx_a); - return proj == PROJECTOR_TYPE_VOXTRAL_RT || proj == PROJECTOR_TYPE_LFM3A; + const projector_type proj = ctx->ctx_a->model.proj_type; + return proj == PROJECTOR_TYPE_VOXTRAL_RT || proj == PROJECTOR_TYPE_LFM3A || proj == PROJECTOR_TYPE_GEMMA4A; } mtmd_audio_stream_params mtmd_audio_stream_params_default(void) { @@ -3586,19 +4194,20 @@ mtmd_audio_stream * mtmd_audio_stream_init(mtmd_context * ctx, mtmd_audio_stream LOG_ERR("%s: the model does not support streaming audio input\n", __func__); return nullptr; } - const clip_hparams * hparams = clip_get_hparams(ctx->ctx_a); + const mtmd_hparams * hparams = &ctx->ctx_a->model.hparams; std::unique_ptr s; - switch (clip_get_projector_type(ctx->ctx_a)) { + switch (ctx->ctx_a->model.proj_type) { case PROJECTOR_TYPE_VOXTRAL_RT: s = std::make_unique(); break; case PROJECTOR_TYPE_LFM3A: s = std::make_unique(); break; + case PROJECTOR_TYPE_GEMMA4A: s = std::make_unique(); break; default: GGML_ABORT("unsupported projector"); } s->ctx = ctx; - s->ctx_clip = ctx->ctx_a; + s->sctx = ctx->ctx_a.get(); s->n_mel = hparams->n_mel_bins; s->hop = hparams->audio_hop_len; - s->n_embd = clip_n_mmproj_embd(ctx->ctx_a); + s->n_embd = ctx->ctx_a->model.n_mmproj_embd(); if (s->n_embd != ctx->n_embd_text) { LOG_ERR("%s: mmproj embedding size %d does not match the text model (%d)\n", __func__, s->n_embd, ctx->n_embd_text); return nullptr; @@ -3626,10 +4235,12 @@ int32_t mtmd_audio_stream_flush(mtmd_audio_stream * s, bool end) { if (s->ended) { return 1; } + std::lock_guard lock(s->ctx->compute_mutex); return s->flush(end); } int32_t mtmd_audio_stream_step(mtmd_audio_stream * s, llama_token last_token) { + std::lock_guard lock(s->ctx->compute_mutex); return s->step(last_token); } @@ -3642,12 +4253,29 @@ bool mtmd_audio_stream_is_seq_start(const mtmd_audio_stream * s) { } +// token embeddings of aud_beg / aud_end +static bool mtmd_audio_stream_marker_rows(const mtmd_context * ctx, const std::string & text, std::vector & rows) { + rows.clear(); + if (text.empty()) { + return true; + } + const llama_vocab * vocab = llama_model_get_vocab(ctx->text_model); + std::vector tokens(text.size() + 4); + const int32_t n = llama_tokenize(vocab, text.c_str(), (int32_t) text.size(), tokens.data(), (int32_t) tokens.size(), false, true); + if (n <= 0) { + LOG_ERR("%s: cannot tokenize the audio marker %s\n", __func__, text.c_str()); + return false; + } + rows.resize((size_t) n * ctx->n_embd_text); + return llama_model_get_tok_embd_rows(ctx->text_model, tokens.data(), n, rows.data()) != 0; +} + // // Voxtral Realtime // bool mtmd_audio_stream_voxtral_rt::init(const mtmd_audio_stream_params & params) { - const clip_hparams * hparams = clip_get_hparams(ctx_clip); + const mtmd_hparams * hparams = &sctx->model.hparams; const llama_vocab * vocab = ctx->vocab; n_fft = hparams->audio_n_fft; @@ -3661,7 +4289,7 @@ bool mtmd_audio_stream_voxtral_rt::init(const mtmd_audio_stream_params & params) // the encoder works in whole decoder rows int n_step = (int) ((int64_t) params.step_ms * hparams->audio_sample_rate / 1000 / hop); n_step = std::max(n_step, frames_per_tok); - n_step = CLIP_ALIGN(n_step, frames_per_tok); + n_step = MTMD_ALIGN(n_step, frames_per_tok); n_step_mel = std::min(n_step, 2 * n_frames_max); if (params.n_prompt > 0) { @@ -3690,7 +4318,7 @@ bool mtmd_audio_stream_voxtral_rt::init(const mtmd_audio_stream_params & params) cache.fill_hann_window(hparams->audio_window_len, true); cache.fill_mel_filterbank_matrix(n_mel, n_fft, hparams->audio_sample_rate); - enc_state = clip_enc_state_init(ctx_clip, n_frames_max); + enc_state = sctx->enc_state_init(n_frames_max); if (!enc_state) { return false; } @@ -3774,20 +4402,20 @@ static bool mtmd_audio_stream_encode(mtmd_audio_stream_voxtral_rt * s, bool forc } } - clip_image_f32 img; + mtmd_image_f32 img; img.set_size({n_in, s->n_mel}, false, /* is_audio */ true); img.cpy_buf(data); - clip_image_f32_batch batch; + mtmd_image_f32_batch batch; batch.is_audio = true; batch.entries.push_back(std::move(img)); std::vector res((size_t) s->n_embd * (n / s->frames_per_tok)); - clip_encode_params params; + mtmd_subctx_encode_params params; params.n_threads = s->ctx->n_threads; params.imgs = &batch; params.out_embd = &res; params.enc_state = s->enc_state; - if (!clip_encode(s->ctx_clip, ¶ms)) { + if (!s->sctx->encode(¶ms)) { LOG_ERR("%s: failed to encode audio\n", __func__); return false; } @@ -3866,7 +4494,7 @@ int32_t mtmd_audio_stream_voxtral_rt::step(llama_token last_token) { // bool mtmd_audio_stream_lfm3a::init(const mtmd_audio_stream_params & params) { - const clip_hparams * hparams = clip_get_hparams(ctx_clip); + const mtmd_hparams * hparams = &sctx->model.hparams; // one row per 8 mel frames (three stride-2 convs), mel frames are not centered win = hparams->audio_window_len; @@ -3874,7 +4502,7 @@ bool mtmd_audio_stream_lfm3a::init(const mtmd_audio_stream_params & params) { samples_per_tok = frames_per_tok * hop; const int n_step = (int) ((int64_t) params.step_ms * hparams->audio_sample_rate / 1000 / hop); - n_step_mel = CLIP_ALIGN(std::max(n_step, frames_per_tok), frames_per_tok); + n_step_mel = MTMD_ALIGN(std::max(n_step, frames_per_tok), frames_per_tok); if (params.n_prompt > 0) { // live ASR, a chunk per encoder call @@ -3897,26 +4525,13 @@ bool mtmd_audio_stream_lfm3a::init(const mtmd_audio_stream_params & params) { max_seq_rows = std::lround(30.0 * hparams->audio_sample_rate / samples_per_tok); } - enc_state = clip_enc_state_init(ctx_clip, n_step_mel); + enc_state = sctx->enc_state_init(n_step_mel); if (!enc_state) { return false; } // the markers the tokenizer puts around an audio clip go around the stream as well - const llama_vocab * vocab = llama_model_get_vocab(ctx->text_model); - auto marker_rows = [&](const std::string & text, std::vector & rows) { - if (text.empty()) { - return true; - } - std::vector tokens(text.size() + 4); - const int32_t n = llama_tokenize(vocab, text.c_str(), (int32_t) text.size(), tokens.data(), (int32_t) tokens.size(), false, true); - if (n <= 0) { - return false; - } - rows.resize((size_t) n * n_embd); - return llama_model_get_tok_embd_rows(ctx->text_model, tokens.data(), n, rows.data()) != 0; - }; - return marker_rows(ctx->aud_beg, beg_rows) && marker_rows(ctx->aud_end, end_rows); + return mtmd_audio_stream_marker_rows(ctx, ctx->aud_beg, beg_rows) && mtmd_audio_stream_marker_rows(ctx, ctx->aud_end, end_rows); } // end: pad the tail like mtmd_audio_preprocessor_lfm3a, so the rows match encoding the clip at once @@ -3947,16 +4562,17 @@ static bool mtmd_audio_stream_compute_mel(mtmd_audio_stream_lfm3a * s) { } const int64_t start = s->n_frames * s->hop - s->buf_pos; const size_t len = (size_t) (s->win + (n_new - 1) * s->hop); - std::vector mel_chunks; - if (!s->ctx->audio_preproc->preprocess(s->buf.data() + start, len, mel_chunks) || mel_chunks.size() != 1) { + mtmd_audio_preproc_out out = s->ctx->audio_preproc->preprocess(s->buf.data() + start, len); + if (out.entries.size() != 1 || !s->ctx->audio_preproc->compute(out)) { LOG_ERR("%s: unable to preprocess audio\n", __func__); return false; } - const auto & mel = mel_chunks[0]; - GGML_ASSERT(mel.n_len == n_new && mel.n_mel == s->n_mel); + const auto & mel = out.entries[0]; + const auto & data = mel.get_ro_buf(); + GGML_ASSERT(mel.nx() == n_new && mel.ny() == s->n_mel); for (int64_t t = 0; t < n_new; t++) { for (int m = 0; m < s->n_mel; m++) { - s->mel.push_back(mel.data[(size_t) m * mel.n_len + t]); + s->mel.push_back(data[(size_t) m * mel.nx() + t]); } } s->n_frames = n_total; @@ -3988,22 +4604,22 @@ static bool mtmd_audio_stream_encode(mtmd_audio_stream_lfm3a * s, bool flush) { data[(size_t) m * n + t] = s->mel[(size_t) t * s->n_mel + m]; } } - clip_image_f32 img; + mtmd_image_f32 img; img.set_size({n, s->n_mel}, false, /* is_audio */ true); img.cpy_buf(data); - const int n_rows = clip_n_output_tokens(s->ctx_clip, &img); + const int n_rows = s->sctx->model.n_output_tokens(&img); - clip_image_f32_batch batch; + mtmd_image_f32_batch batch; batch.is_audio = true; batch.entries.push_back(std::move(img)); std::vector res((size_t) s->n_embd * n_rows); - clip_encode_params params; + mtmd_subctx_encode_params params; params.n_threads = s->ctx->n_threads; params.imgs = &batch; params.out_embd = &res; params.enc_state = s->enc_state; - if (!clip_encode(s->ctx_clip, ¶ms)) { + if (!s->sctx->encode(¶ms)) { LOG_ERR("%s: failed to encode audio\n", __func__); return false; } @@ -4137,6 +4753,192 @@ int32_t mtmd_audio_stream_lfm3a::step_live(llama_token last_token) { return (int32_t) (out.size() / n_embd); } +// +// Gemma 4 +// + +bool mtmd_audio_stream_gemma4a::init(const mtmd_audio_stream_params & params) { + const mtmd_hparams * hparams = &sctx->model.hparams; + + if (params.n_prompt > 0) { + LOG_ERR("%s: Gemma 4 takes no prompt, its audio is a chat turn\n", __func__); + return false; + } + if (!ctx->audio_preproc || !ctx->audio_preproc->mel_config.cache) { + LOG_ERR("%s: the audio preprocessor is not initialized\n", __func__); + return false; + } + + // one row per 4 mel frames (two stride-2 convs) + n_fft = hparams->audio_n_fft; + win = hparams->audio_window_len; + pad_left = win / 2; + seg_len = (int64_t) mtmd_audio_preprocessor_gemma4a::segment_seconds * hparams->audio_sample_rate; + frames_per_tok = 4; + samples_per_tok = frames_per_tok * hop; + + // whole attention blocks, so the rows match the offline path + const int n_step = (int) ((int64_t) params.step_ms * hparams->audio_sample_rate / 1000 / hop); + n_step_mel = MTMD_ALIGN(std::max(n_step, frames_per_tok), (int) mtmd_graph_gemma4a::chunk_size * frames_per_tok); + + enc_state = sctx->enc_state_init(n_step_mel); + if (!enc_state) { + return false; + } + + return mtmd_audio_stream_marker_rows(ctx, ctx->aud_beg, beg_rows) && mtmd_audio_stream_marker_rows(ctx, ctx->aud_end, end_rows); +} + +// mel frames of a segment with n samples +// complete: same count as mtmd_audio_preprocessor_gemma4a, otherwise: only frames with a full window +int64_t mtmd_audio_stream_gemma4a::n_seg_frames(int64_t n, bool complete) const { + if (complete) { + return std::max(0, (n + pad_left - (win + 1)) / hop + 1); + } + return n + pad_left >= win + 1 ? (n + pad_left - (win + 1)) / hop + 1 : 0; +} + +// frame k reads samples [k * hop - pad_left, k * hop - pad_left + n_fft) of the segment, zero outside of it +// the window is zero past win, so the samples after the segment end do not count +bool mtmd_audio_stream_gemma4a::compute_mel(int64_t n_seg, bool complete) { + const int64_t n_total = n_seg_frames(n_seg, complete); + const int64_t n_new = n_total - n_frames; + if (n_new <= 0) { + return true; + } + + const int64_t first = seg_start + n_frames * hop - pad_left; + std::vector samples((size_t) ((n_new - 1) * hop + n_fft), 0.0f); + for (size_t j = 0; j < samples.size(); j++) { + const int64_t idx = first + (int64_t) j; + if (idx >= seg_start && idx < seg_start + n_seg) { + GGML_ASSERT(idx >= buf_pos && idx - buf_pos < (int64_t) buf.size()); + samples[j] = buf[idx - buf_pos]; + } + } + + const mtmd_audio_mel_config & cfg = ctx->audio_preproc->mel_config; + std::vector frames((size_t) n_new * n_mel); + if (!mtmd_audio_log_mel(cfg, samples.data(), cfg.cache->hann_window.data(), n_new, frames.data())) { + LOG_ERR("%s: unable to compute the mel spectrogram\n", __func__); + return false; + } + for (int64_t t = 0; t < n_new; t++) { + for (int m = 0; m < n_mel; m++) { + mel.push_back(frames[(size_t) m * n_new + t]); + } + } + n_frames = n_total; + + // drop the samples no frame will need again + const int64_t keep_from = std::max(seg_start, seg_start + n_frames * hop - pad_left); + if (keep_from > buf_pos) { + buf.erase(buf.begin(), buf.begin() + (keep_from - buf_pos)); + buf_pos = keep_from; + } + return true; +} + +bool mtmd_audio_stream_gemma4a::encode(int n) { + std::vector data((size_t) n_mel * n); + for (int m = 0; m < n_mel; m++) { + for (int t = 0; t < n; t++) { + data[(size_t) m * n + t] = mel[(size_t) t * n_mel + m]; + } + } + mtmd_image_f32 img; + img.set_size({n, n_mel}, false, /* is_audio */ true); + img.cpy_buf(data); + const int n_out = sctx->model.n_output_tokens(&img); + + mtmd_image_f32_batch batch; + batch.is_audio = true; + batch.entries.push_back(std::move(img)); + + std::vector res((size_t) n_embd * n_out); + mtmd_subctx_encode_params params; + params.n_threads = ctx->n_threads; + params.imgs = &batch; + params.out_embd = &res; + params.enc_state = enc_state; + if (!sctx->encode(¶ms)) { + LOG_ERR("%s: failed to encode audio\n", __func__); + return false; + } + embd.insert(embd.end(), res.begin(), res.end()); + mel.erase(mel.begin(), mel.begin() + (size_t) n * n_mel); + return true; +} + +// no padding here, step() pads a segment once it is complete +int32_t mtmd_audio_stream_gemma4a::flush(bool end) { + const int64_t n_full = n_samples / seg_len; + const int64_t n_rest = n_samples % seg_len; + const int64_t rows_full = (n_seg_frames(seg_len, true) + frames_per_tok - 1) / frames_per_tok; + const int64_t rows_rest = end ? (n_seg_frames(n_rest, true) + frames_per_tok - 1) / frames_per_tok + : n_seg_frames(n_rest, false) / frames_per_tok; + n_rows_flushed = n_full * rows_full + rows_rest; + pending_flush = true; + ended = end; + return 0; +} + +int32_t mtmd_audio_stream_gemma4a::step(llama_token /* last_token */) { + const bool flush = pending_flush; + pending_flush = false; + + // whole rows only, except at the end of a segment: a partial row puts its zero padding into the state + while (true) { + const int64_t seg_end = seg_start + seg_len; + const int64_t n_seg = std::min(n_samples, seg_end) - seg_start; + const bool complete = n_samples >= seg_end || ended; + if (!compute_mel(n_seg, complete)) { + return -2; + } + const int n_avail = (int) (mel.size() / n_mel); + int n = n_avail; + if (!complete) { + n -= n % frames_per_tok; + if (!flush) { + n -= n % n_step_mel; + } + } + if (n > 0 && !encode(n)) { + return -2; + } + if (n_samples < seg_end) { + break; + } + // next segment starts from a zero state + seg_start = seg_end; + n_frames = 0; + ggml_backend_buffer_clear(enc_state->buf.get(), 0); + } + + // rows carry no text, all of them go out at once and last_token is not used + const int64_t n_avail = (int64_t) (embd.size() / n_embd); + out.clear(); + if (n_avail > 0 || ended) { + if (!beg_done) { + beg_done = true; + out = beg_rows; + } + out.insert(out.end(), embd.begin(), embd.end()); + embd.clear(); + if (ended && !end_done) { + end_done = true; + out.insert(out.end(), end_rows.begin(), end_rows.end()); + } + } + const int64_t n_out = (int64_t) (out.size() / n_embd); + if (n_out == 0) { + return ended ? -1 : 0; + } + n_rows += n_out; + n_rows_done += n_avail; // the markers are not audio rows + return (int32_t) n_out; +} + int64_t mtmd_audio_stream_get_n_rows(const mtmd_audio_stream * s) { return s->n_rows_done; } diff --git a/tools/mtmd/mtmd.h b/tools/mtmd/mtmd.h index 4f69a644d857..e14f345a657b 100644 --- a/tools/mtmd/mtmd.h +++ b/tools/mtmd/mtmd.h @@ -63,6 +63,7 @@ enum mtmd_video_template { MTMD_VIDEO_TEMPLATE_DEFAULT, // "Video:" prefix, then a "[XmY.YYs]" timestamp every N ms MTMD_VIDEO_TEMPLATE_LFM3, // "" before every group of 2 frames MTMD_VIDEO_TEMPLATE_LFM3_OMNI, // same as LFM3, then the soundtrack of each group, as an audio bitmap after its frames + MTMD_VIDEO_TEMPLATE_QWEN3VL, // "" before every group of 2 frames, no markers around the whole video }; // opaque types @@ -352,6 +353,7 @@ DEPRECATED(MTMD_API int32_t mtmd_encode(mtmd_context * ctx, const mtmd_image_tok "use mtmd_encode_chunk() instead"); // text chunk will be ignored silently, only media chunk will be encoded +// the output is stored in ctx: do not call it from several threads at once // returns 0 on success // returns 1 on generic error MTMD_API int32_t mtmd_encode_chunk(mtmd_context * ctx, @@ -376,6 +378,7 @@ MTMD_API void mtmd_batch_free(mtmd_batch * batch); // returns 3 if it cannot be batched with the existing chunks in the batch MTMD_API int32_t mtmd_batch_add_chunk(mtmd_batch * batch, const mtmd_input_chunk * chunk); +// thread-safe, the output is stored in the batch // returns 0 on success // returns 1 on generic error MTMD_API int32_t mtmd_batch_encode(mtmd_batch * batch); @@ -430,11 +433,13 @@ MTMD_API int32_t mtmd_audio_stream_push(mtmd_audio_stream * stream, const float // pad the stream so that all pending rows can be produced // end = false: the stream stays open (use it on a pause, to get a partial transcript) // end = true: no more push() is accepted +// thread-safe across streams // returns 0 on success, 1 if the stream has ended MTMD_API int32_t mtmd_audio_stream_flush(mtmd_audio_stream * stream, bool end); // produce the next decoder rows // last_token is the token sampled from the previous row, LLAMA_TOKEN_NULL before the first sample +// thread-safe across streams // return value: // > 0 number of rows ready, read them with mtmd_audio_stream_get_output_embd() // 0 more audio is needed @@ -465,6 +470,8 @@ enum mtmd_gen_audio_type { MTMD_GEN_AUDIO_TYPE_POCKETTTS, MTMD_GEN_AUDIO_TYPE_SOPRANO, MTMD_GEN_AUDIO_TYPE_LFM3A, + MTMD_GEN_AUDIO_TYPE_CHATTERBOX, + MTMD_GEN_AUDIO_TYPE_DRAMA3, }; struct mtmd_gen_audio_info { @@ -476,6 +483,8 @@ struct mtmd_gen_audio_info { MTMD_API struct mtmd_gen_audio_info mtmd_gen_audio_get_info(const mtmd_context * ctx); +// read a named tensor of the gen audio model as F32, returns the element count (0 if not found), out may be null to query the size +MTMD_API size_t mtmd_gen_audio_read_tensor(mtmd_context * ctx, const char * name, float * out, size_t n_max); // name of the idx-th predefined voice, nullptr when idx is out of range MTMD_API const char * mtmd_gen_audio_voice_name(const mtmd_context * ctx, int32_t idx); @@ -500,7 +509,7 @@ struct mtmd_gen_inp { float * embd; // the hidden state from backbone, must have n_text_embd elements int32_t top_k; float top_p; - float temp; // sampling temperature, or noise scale for flow-matching decoders + float temp; // sampling temperature, or noise scale for flow-matching decoders (0 = no noise) float penalty_repeat; // repetition penalty of the sampled codes, <= 1 is off int32_t penalty_last_n; // frames the repetition penalty looks back on @@ -543,11 +552,16 @@ MTMD_API void mtmd_gen_state_free(mtmd_gen_state * state); // clear the decoder state (cold start), keep the RNG stream MTMD_API void mtmd_gen_state_reset(mtmd_gen_state * state); +// copy the gen-side reference data of the last mtmd_encode_chunk() call into the state (speaker clip or predefined voice) +// returns non-zero if the last encode produced no such data +MTMD_API int32_t mtmd_gen_state_set_ref(mtmd_context * ctx, mtmd_gen_state * state); + // defaults tuned for the loaded pipeline, callers override only what they care about MTMD_API struct mtmd_gen_inp mtmd_gen_inp_default(const mtmd_context * ctx); // run one step and update the state in place // the caller drives the backbone and collects the audio frames +// thread-safe across gen states MTMD_API int32_t mtmd_gen_audio_process(mtmd_context * ctx, mtmd_gen_state * state, const struct mtmd_gen_inp * inp, diff --git a/tools/quantize/quantize.cpp b/tools/quantize/quantize.cpp index 38950036cd82..0debcfddaffa 100644 --- a/tools/quantize/quantize.cpp +++ b/tools/quantize/quantize.cpp @@ -510,7 +510,8 @@ int llama_quantize(int argc, char ** argv) { llama_model_kv_override kvo; std::strcpy(kvo.key, LLM_KV_QUANTIZE_IMATRIX_FILE); kvo.tag = LLAMA_KV_OVERRIDE_TYPE_STR; - strncpy(kvo.val_str, imatrix_file.c_str(), 127); + const std::string fname_imatrix = std::filesystem::path(imatrix_file).filename().string(); + strncpy(kvo.val_str, fname_imatrix.c_str(), 127); kvo.val_str[127] = '\0'; kv_overrides.emplace_back(std::move(kvo)); } @@ -519,7 +520,8 @@ int llama_quantize(int argc, char ** argv) { // TODO: list multiple datasets when there are more than one std::strcpy(kvo.key, LLM_KV_QUANTIZE_IMATRIX_DATASET); kvo.tag = LLAMA_KV_OVERRIDE_TYPE_STR; - strncpy(kvo.val_str, imatrix_datasets[0].c_str(), 127); + const std::string fname_dataset = std::filesystem::path(imatrix_datasets[0]).filename().string(); + strncpy(kvo.val_str, fname_dataset.c_str(), 127); kvo.val_str[127] = '\0'; kv_overrides.emplace_back(std::move(kvo)); } diff --git a/tools/rpc/CMakeLists.txt b/tools/rpc/CMakeLists.txt index 2891c7d034cf..20c1f087eeec 100644 --- a/tools/rpc/CMakeLists.txt +++ b/tools/rpc/CMakeLists.txt @@ -13,6 +13,11 @@ if (LLAMA_BUILD_TESTS AND UNIX AND NOT GGML_BACKEND_DL) $ $) set_property(TEST test-rpc-multi-server PROPERTY LABELS main) + add_executable(test-rpc-unreachable ${PROJECT_SOURCE_DIR}/tests/test-rpc-unreachable.cpp) + target_link_libraries(test-rpc-unreachable PRIVATE ggml ggml-rpc) + target_include_directories(test-rpc-unreachable PRIVATE ${PROJECT_SOURCE_DIR}/ggml/src) + add_test(NAME test-rpc-unreachable COMMAND test-rpc-unreachable) + set_property(TEST test-rpc-unreachable PROPERTY LABELS main) endif() if(LLAMA_TOOLS_INSTALL) diff --git a/tools/rpc/rpc-server.cpp b/tools/rpc/rpc-server.cpp index 7cb25947d681..8aeb15db53a1 100644 --- a/tools/rpc/rpc-server.cpp +++ b/tools/rpc/rpc-server.cpp @@ -10,7 +10,6 @@ #endif #include #include -#include #include #include #include @@ -24,25 +23,16 @@ #endif -// common_get_path_from_env() is adapted to avoid utf8_to_wstring -static std::filesystem::path common_get_path_from_env(const std::string & name) { -#ifdef _WIN32 - std::wstring wname; - for (const char * p = name.c_str(); *p; ++p) { - wname.push_back((wchar_t)*p); - } - const wchar_t * wvalue = _wgetenv(wname.c_str()); - return wvalue ? std::filesystem::path(wvalue) : std::filesystem::path(); -#else +// NOTE: this is copied from common.cpp to avoid linking with libcommon +static std::string common_get_env(const std::string & name) { const char * value = std::getenv(name.c_str()); - return value ? std::filesystem::path(value) : std::filesystem::path(); -#endif + return value == nullptr ? "" : value; } // NOTE: this is copied from common.cpp to avoid linking with libcommon #if !defined(_WIN32) static std::filesystem::path get_home_directory() { - std::filesystem::path home = common_get_path_from_env("HOME"); + std::filesystem::path home = common_get_env("HOME"); if (!home.empty()) { return home; } @@ -56,19 +46,19 @@ static std::filesystem::path get_home_directory() { // NOTE: this is copied from common.cpp to avoid linking with libcommon static std::filesystem::path fs_get_cache_directory() { - std::filesystem::path cache_directory = common_get_path_from_env("LLAMA_CACHE"); + std::filesystem::path cache_directory = common_get_env("LLAMA_CACHE"); if (!cache_directory.empty()) { return cache_directory; } #if defined(_WIN32) - cache_directory = common_get_path_from_env("LOCALAPPDATA"); + cache_directory = common_get_env("LOCALAPPDATA"); if (cache_directory.empty()) { throw std::runtime_error("Failed to find %LOCALAPPDATA% directory"); } #elif defined(__APPLE__) cache_directory = get_home_directory() / "Library/Caches"; #else - cache_directory = common_get_path_from_env("XDG_CACHE_HOME"); + cache_directory = common_get_env("XDG_CACHE_HOME"); if (cache_directory.empty()) { cache_directory = get_home_directory() / ".cache"; } diff --git a/tools/server/README-dev.md b/tools/server/README-dev.md index 17b39f0aaaba..85fb3f78eae5 100644 --- a/tools/server/README-dev.md +++ b/tools/server/README-dev.md @@ -201,7 +201,7 @@ Invoke a tool call, request body is a JSON object with: Headers: - `x-tool-cwd`: optional; if set, use as the CWD for tool; this is not part of tool's params because it's meant to be set by the runtime, not the LLM itself -- `x-tool-runtime`: optional; if set, run the tool inside this isolate instead of on the host. Either `docker-container:` or `podman-container:`, using an already-running container, or `ssh:`, running the tool on a remote host +- `x-tool-runtime`: optional; if set, run the tool inside this isolate instead of on the host. Either `docker-container:` or `podman-container:`, using an already-running container, or `ssh:`, running the tool on a remote host. Refused with 403 when the server is started with `--tools-runtime`, so clients cannot move tools out of that sandbox Returns JSON object. There are two response formats (MCP tools use the same two formats: their result content is concatenated into `plain_text_response`, and RPC or tool errors are surfaced as the `error` string): @@ -239,6 +239,8 @@ That requires `JSON.stringify` when formatted to message content: } ``` +When the client disconnects during an MCP tool call, or the call exceeds `timeout_ms`, `llama-server` sends `notifications/cancelled` with the request id to the MCP server and returns an error. + Set `stream: true` in the request body to stream a tool's output as it runs, instead of waiting for it to finish. Only certain tools accept this (for ex. `exec_shell_command`); returns 404 if tool doesn't support it. diff --git a/tools/server/README.md b/tools/server/README.md index afa448b8a736..27d1417518f4 100644 --- a/tools/server/README.md +++ b/tools/server/README.md @@ -68,9 +68,8 @@ For the full list of features, please refer to [server's changelog](https://gith | `-kvo, --kv-offload, -nkvo, --no-kv-offload` | whether to enable KV cache offloading (default: enabled)
(env: LLAMA_ARG_KV_OFFLOAD) | | `--repack, -nr, --no-repack` | whether to enable weight repacking (default: enabled)
(env: LLAMA_ARG_REPACK) | | `--no-host` | bypass host buffer allowing extra buffers to be used
(env: LLAMA_ARG_NO_HOST) | -| `-ctk, --cache-type-k TYPE` | KV cache data type for K
allowed values: f32, f16, bf16, q8_0, q4_0, q4_1, iq4_nl, q5_0, q5_1
(default: f16)
(env: LLAMA_ARG_CACHE_TYPE_K) | -| `-ctv, --cache-type-v TYPE` | KV cache data type for V
allowed values: f32, f16, bf16, q8_0, q4_0, q4_1, iq4_nl, q5_0, q5_1
(default: f16)
(env: LLAMA_ARG_CACHE_TYPE_V) | -| `-dt, --defrag-thold N` | KV cache defragmentation threshold (DEPRECATED)
(env: LLAMA_ARG_DEFRAG_THOLD) | +| `-ctk, --cache-type-k TYPE` | KV cache data type for K
allowed values: f32, f16, bf16, f8_e4m3, q8_0, q4_0, q4_1, iq4_nl, q5_0, q5_1
(default: f16)
(env: LLAMA_ARG_CACHE_TYPE_K) | +| `-ctv, --cache-type-v TYPE` | KV cache data type for V
allowed values: f32, f16, bf16, f8_e4m3, q8_0, q4_0, q4_1, iq4_nl, q5_0, q5_1
(default: f16)
(env: LLAMA_ARG_CACHE_TYPE_V) | | `--rpc SERVERS` | comma-separated list of RPC servers (host:port)
(env: LLAMA_ARG_RPC) | | `-lm, --load-mode MODE` | model loading mode (default: auto)
- auto: mmap, unless a device does not support it
- none: no special loading mode
- mmap: memory-map model (if mmap disabled, slower load but may reduce pageouts if not using mlock)
- mlock: force system to keep model in RAM rather than swapping or compressing
- mmap+mlock: mmap + force system to keep model in RAM rather than swapping or compressing
- dio: use DirectIO if available

(env: LLAMA_ARG_LOAD_MODE) | | `-lzm, --lazy-mode MODE` | on-demand reading of certain tensors, for example per-layer embeddings (default: auto)
- on: read the rows of such tensors from disk on demand instead of keeping them resident (requires mmap)
- auto: on, but only for tensors larger than 4 GiB
- off: always keep them resident
(env: LLAMA_ARG_LAZY_MODE) | @@ -81,6 +80,7 @@ For the full list of features, please refer to [server's changelog](https://gith | `-cmoe, --cpu-moe` | keep all Mixture of Experts (MoE) weights in the CPU
(env: LLAMA_ARG_CPU_MOE) | | `-ncmoe, --n-cpu-moe N` | keep the Mixture of Experts (MoE) weights of the first N layers in the CPU
(env: LLAMA_ARG_N_CPU_MOE) | | `--moe-cache-mib N` | GPU cache size in MiB for the MoE experts kept in the CPU. with multiple GPUs, it is split among them like the layers (--tensor-split) (default: 0, disabled)
(env: LLAMA_ARG_MOE_CACHE_MIB) | +| `--moe-cache-cpu-share F` | share of the MoE cache misses that the CPU computes from the host experts instead of uploading them to the GPU once the cache is full, from 0 to 1 (default: 0.00)
(env: LLAMA_ARG_MOE_CACHE_CPU_SHARE) | | `-ncffn, --n-cpu-ffn N` | keep the dense FFN weights of the first N layers in the CPU
(dense models; for MoE expert weights use --n-cpu-moe)
(env: LLAMA_ARG_N_CPU_FFN) | | `-ngl, --gpu-layers, --n-gpu-layers N` | max. number of layers to store in VRAM, either an exact number, 'auto', or 'all' (default: auto)
(env: LLAMA_ARG_N_GPU_LAYERS) | | `-sm, --split-mode {none,layer,row,tensor}` | how to split the model across multiple GPUs, one of:
- none: use one GPU only
- layer (default): split layers and KV across GPUs (pipelined)
- row: split weight across GPUs by rows (parallelized)
- tensor: split weights and KV across GPUs (parallelized, EXPERIMENTAL)
(env: LLAMA_ARG_SPLIT_MODE) | @@ -98,6 +98,7 @@ For the full list of features, please refer to [server's changelog](https://gith | `--control-vector-scaled FNAME:SCALE,...` | add a control vector with user defined scaling SCALE
note: use comma-separated values (format: FNAME:SCALE,...) | | `--control-vector-layer-range START END` | layer range to apply the control vector(s) to, start and end inclusive | | `-m, --model FNAME` | model path to load
(env: LLAMA_ARG_MODEL) | +| `--ple, --shard FNAME,...` | extra GGUF shards for the model, comma separated; may be given more than once
paths may live anywhere, including on different drives; --ple is an alias for --shard
(env: LLAMA_ARG_SHARD) | | `-mu, --model-url MODEL_URL` | model download url (default: unused)
(env: LLAMA_ARG_MODEL_URL) | | `-dr, --docker-repo [/][:quant]` | Docker Hub model repository. repo is optional, default to ai/. quant is optional, default to :latest.
example: gemma3
(default: unused)
(env: LLAMA_ARG_DOCKER_REPO) | | `-hf, -hfr, --hf-repo /[:quant]` | Hugging Face model repository; quant is optional, case-insensitive, default to Q4_K_M, or falls back to the first file in the repo if Q4_K_M doesn't exist.
mmproj is also downloaded automatically if available. to disable, add --no-mmproj
example: ggml-org/GLM-4.7-Flash-GGUF:Q4_K_M
(default: unused)
(env: LLAMA_ARG_HF_REPO) | @@ -112,8 +113,8 @@ For the full list of features, please refer to [server's changelog](https://gith | `-lv, --verbosity, --log-verbosity N` | Set the verbosity threshold. Messages with a higher verbosity will be ignored. Values:
- 0: generic output
- 1: error
- 2: warning
- 3: info
- 4: trace (more info)
- 5: debug
(default: 3)

(env: LLAMA_ARG_LOG_VERBOSITY) | | `--log-prefix, --no-log-prefix` | Enable prefix in log messages
(env: LLAMA_ARG_LOG_PREFIX) | | `--log-timestamps, --no-log-timestamps` | Enable timestamps in log messages
(env: LLAMA_ARG_LOG_TIMESTAMPS) | -| `--spec-draft-type-k, -ctkd, --cache-type-k-draft TYPE` | KV cache data type for K for the draft model
allowed values: f32, f16, bf16, q8_0, q4_0, q4_1, iq4_nl, q5_0, q5_1
(default: f16)
(env: LLAMA_ARG_SPEC_DRAFT_CACHE_TYPE_K) | -| `--spec-draft-type-v, -ctvd, --cache-type-v-draft TYPE` | KV cache data type for V for the draft model
allowed values: f32, f16, bf16, q8_0, q4_0, q4_1, iq4_nl, q5_0, q5_1
(default: f16)
(env: LLAMA_ARG_SPEC_DRAFT_CACHE_TYPE_V) | +| `--spec-draft-type-k, -ctkd, --cache-type-k-draft TYPE` | KV cache data type for K for the draft model
allowed values: f32, f16, bf16, f8_e4m3, q8_0, q4_0, q4_1, iq4_nl, q5_0, q5_1
(default: f16)
(env: LLAMA_ARG_SPEC_DRAFT_CACHE_TYPE_K) | +| `--spec-draft-type-v, -ctvd, --cache-type-v-draft TYPE` | KV cache data type for V for the draft model
allowed values: f32, f16, bf16, f8_e4m3, q8_0, q4_0, q4_1, iq4_nl, q5_0, q5_1
(default: f16)
(env: LLAMA_ARG_SPEC_DRAFT_CACHE_TYPE_V) | ### Sampling params @@ -194,16 +195,17 @@ For the full list of features, please refer to [server's changelog](https://gith | `--port PORT` | port to listen (default: 9931)
(env: LLAMA_ARG_PORT) | | `--reuse-port` | allow multiple sockets to bind to the same port (default: disabled)
(env: LLAMA_ARG_REUSE_PORT) | | `--path PATH` | path to serve static files from (default: )
(env: LLAMA_ARG_STATIC_PATH) | -| `--cors-origins ORIGINS` | comma-separated list of allowed origins for CORS (default: *)
if set to special value 'localhost', reflect the Origin header only if it is localhost
(env: LLAMA_ARG_CORS_ORIGINS) | +| `--cors-origins ORIGINS` | comma-separated list of allowed origins for CORS (default: *)
if set to special value 'localhost', only allow same-origin requests (e.g. the built-in UI)
(env: LLAMA_ARG_CORS_ORIGINS) | | `--cors-methods METHODS` | comma-separated list of allowed methods for CORS (default: GET, POST, DELETE, OPTIONS)
(env: LLAMA_ARG_CORS_METHODS) | | `--cors-headers HEADERS` | comma-separated list of allowed headers for CORS (default: *)
(env: LLAMA_ARG_CORS_HEADERS) | | `--cors-credentials, --no-cors-credentials` | whether to allow credentials for CORS (default: enabled)
note: if this is enabled and --cors-origins is set to * (default), the Origin header will be echoed back, and credentials will always be allowed
(env: LLAMA_ARG_CORS_CREDENTIALS) | | `--api-prefix PREFIX` | prefix path the server serves from, without the trailing slash (default: )
(env: LLAMA_ARG_API_PREFIX) | | `--ui-config, --webui-config JSON` | JSON that provides default UI settings (overrides UI defaults)
(env: LLAMA_ARG_UI_CONFIG) | | `--ui-config-file, --webui-config-file PATH` | JSON file that provides default UI settings (overrides UI defaults)
(env: LLAMA_ARG_UI_CONFIG_FILE) | -| `--ui-mcp-proxy, --webui-mcp-proxy, --no-ui-mcp-proxy, --no-webui-mcp-proxy` | experimental: whether to enable MCP CORS proxy - do not enable in untrusted environments (default: disabled)
(env: LLAMA_ARG_UI_MCP_PROXY) | +| `--ui-mcp-proxy, --webui-mcp-proxy, --no-ui-mcp-proxy, --no-webui-mcp-proxy` | experimental: whether to enable MCP CORS proxy, for the Web UI to reach public MCP servers without CORS headers; local and private addresses are refused - do not enable in untrusted environments (default: disabled)
note: for security reasons, this will limit --cors-origins to localhost by default
(env: LLAMA_ARG_UI_MCP_PROXY) | | `--tools TOOL1,TOOL2,...` | experimental: whether to enable built-in tools for AI agents - do not enable in untrusted environments (default: no tools)
specify "all" to enable all tools
available tools: read_file, file_glob_search, grep_search, exec_shell_command, write_file, edit_file, get_info
note: for security reasons, this will limit --cors-origins to localhost by default
(env: LLAMA_ARG_TOOLS) | | `--tools-runtime OPTION` | experimental: run tools in a separate runtime environment (default: none, use host environment)
available options:
'docker:', 'podman:': spin up a new container and reuse it for all invocations, clean up on server exit
'docker-container:', 'podman-container:': use an existing container by ID, won't stop on server exit
'ssh:': run tools on a remote POSIX host over SSH, key-based auth and a trusted host key are required

(env: LLAMA_ARG_TOOLS_RUNTIME) | +| `--allow-unauthenticated-tools` | allow server tools, MCP servers or MCP proxy on a non-loopback address without an API key - only for trusted networks (default: disabled)
(env: LLAMA_ARG_ALLOW_UNAUTHENTICATED_TOOLS) | | `--mcp-servers-config PATH` | experimental: path to JSON file with MCP server definitions (Cursor-compatible format) - do not enable in untrusted environments (default: none)
note: for security reasons, this will limit --cors-origins to localhost by default
(env: LLAMA_ARG_MCP_SERVERS_CONFIG) | | `--mcp-servers-json JSON` | experimental: inline JSON with MCP server definitions (Cursor-compatible format) - do not enable in untrusted environments (default: none)
note: for security reasons, this will limit --cors-origins to localhost by default
(env: LLAMA_ARG_MCP_SERVERS_JSON) | | `-ag, --agent, -no-ag, --no-agent` | whether to enable CORS proxy and all built-in tools - do not enable in untrusted environments (default: disabled)
note: for security reasons, this will limit --cors-origins to localhost by default
(env: LLAMA_ARG_AGENT) | @@ -227,6 +229,7 @@ For the full list of features, please refer to [server's changelog](https://gith | `--slots, --no-slots` | expose slots monitoring endpoint (default: enabled)
(env: LLAMA_ARG_ENDPOINT_SLOTS) | | `--slot-save-path PATH` | path to save slot kv cache (default: disabled)
(env: LLAMA_ARG_SLOT_SAVE_PATH) | | `--media-path PATH` | directory for loading local media files; files can be accessed via file:// URLs using relative paths (default: disabled) | +| `--media-allow-private-urls` | allow http(s) media URLs (e.g. image_url) that point to local or private addresses (default: disabled, only public addresses)
(env: LLAMA_ARG_MEDIA_ALLOW_PRIVATE_URLS) | | `--models-dir PATH` | directory containing models for the router server (default: disabled)
(env: LLAMA_ARG_MODELS_DIR) | | `--models-preset PATH` | path to INI file containing model presets for the router server (default: disabled)
(env: LLAMA_ARG_MODELS_PRESET) | | `--models-max N` | for router server, maximum number of models to load simultaneously (default: 4, 0 = unlimited)
(env: LLAMA_ARG_MODELS_MAX) | @@ -238,7 +241,7 @@ For the full list of features, please refer to [server's changelog](https://gith | `--reasoning-effort LEVEL` | reasoning effort level given to the chat template: 'default' to keep the template default,
or a level such as 'minimal', 'low', 'medium', 'high', 'xhigh' or 'max' (default: default)
(env: LLAMA_ARG_REASONING_EFFORT) | | `--reasoning-budget N` | token budget for thinking: -1 for unrestricted, 0 for immediate end, N>0 for token budget (default: -1)
(env: LLAMA_ARG_THINK_BUDGET) | | `--reasoning-budget-message MESSAGE` | message injected before the end-of-thinking tag when reasoning budget is exhausted (default: none)
(env: LLAMA_ARG_THINK_BUDGET_MESSAGE) | -| `--reasoning-preserve, --no-reasoning-preserve` | preserve reasoning trace in the full history, not just the last assistant message (default: template default)
compatible with certain templates having 'supports_preserve_reasoning' capability
example: https://docs.z.ai/guides/capabilities/thinking-mode#preserved-thinking
(env: LLAMA_ARG_REASONING_PRESERVE) | +| `--reasoning-preserve, --no-reasoning-preserve` | preserve reasoning trace in the full history, not just the last assistant message (default: enabled)
compatible with certain templates having 'supports_preserve_reasoning' capability
example: https://docs.z.ai/guides/capabilities/thinking-mode#preserved-thinking
(env: LLAMA_ARG_REASONING_PRESERVE) | | `--chat-template JINJA_TEMPLATE` | set custom jinja chat template (default: template taken from model's metadata)
if suffix/prefix are specified, template will be disabled
only commonly used templates are accepted (unless --jinja is set before this flag):
list of built-in templates:
bailing, bailing-think, bailing2, chatglm3, chatglm4, chatml, command-r, deepseek, deepseek-ocr, deepseek2, deepseek3, exaone-moe, exaone3, exaone4, falcon3, gemma, gigachat, glmedge, gpt-oss, granite, granite-4.0, granite-4.1, grok-2, hunyuan-dense, hunyuan-moe, hunyuan-vl, kimi-k2, llama2, llama2-sys, llama2-sys-bos, llama2-sys-strip, llama3, llama4, megrez, minicpm, mistral-v1, mistral-v3, mistral-v3-tekken, mistral-v7, mistral-v7-tekken, monarch, openchat, orion, pangu-embedded, phi3, phi4, rwkv-world, seed_oss, smolvlm, solar-open, vicuna, vicuna-orca, yandex, zephyr
(env: LLAMA_ARG_CHAT_TEMPLATE) | | `--chat-template-file JINJA_TEMPLATE_FILE` | set custom jinja chat template file (default: template taken from model's metadata)
if suffix/prefix are specified, template will be disabled
only commonly used templates are accepted (unless --jinja is set before this flag):
list of built-in templates:
bailing, bailing-think, bailing2, chatglm3, chatglm4, chatml, command-r, deepseek, deepseek-ocr, deepseek2, deepseek3, exaone-moe, exaone3, exaone4, falcon3, gemma, gigachat, glmedge, gpt-oss, granite, granite-4.0, granite-4.1, grok-2, hunyuan-dense, hunyuan-moe, hunyuan-vl, kimi-k2, llama2, llama2-sys, llama2-sys-bos, llama2-sys-strip, llama3, llama4, megrez, minicpm, mistral-v1, mistral-v3, mistral-v3-tekken, mistral-v7, mistral-v7-tekken, monarch, openchat, orion, pangu-embedded, phi3, phi4, rwkv-world, seed_oss, smolvlm, solar-open, vicuna, vicuna-orca, yandex, zephyr
(env: LLAMA_ARG_CHAT_TEMPLATE_FILE) | | `--skip-chat-parsing, --no-skip-chat-parsing` | force a pure content parser, even if a Jinja template is specified; model will output everything in the content section, including any reasoning and/or tool calls (default: disabled)
(env: LLAMA_ARG_SKIP_CHAT_PARSING) | @@ -292,6 +295,8 @@ For the full list of features, please refer to [server's changelog](https://gith | `--spec-ngram-size-n N` | the argument has been removed. use the respective --spec-ngram-*-size-n or --spec-ngram-mod-n-match | | `--spec-ngram-size-m N` | the argument has been removed. use the respective --spec-ngram-*-size-m | | `--spec-ngram-min-hits N` | the argument has been removed. use the respective --spec-ngram-*-min-hits | +| `--tts-lang FNAME` | language (ISO 639-1) for audio generation
see tts/README.md for per-model usage notes | +| `--tts-voice VOICE` | voice for audio generation (model-dependent)
see tts/README.md for per-model usage notes | | `--embd-gemma-default` | use default EmbeddingGemma model (note: can download weights from the internet) | | `--fim-qwen-1.5b-default` | use default Qwen 2.5 Coder 1.5B (note: can download weights from the internet) | | `--fim-qwen-3b-default` | use default Qwen 2.5 Coder 3B (note: can download weights from the internet) | @@ -388,7 +393,7 @@ Every server is spawned once at startup to list its tools, then stopped, and res The child process runs with the same privileges as the server, so only declare commands you trust. As with `--tools`, `--cors-origins` then defaults to `localhost`. -Note: `--ui-mcp-proxy` is unrelated, it only lets the Web UI reach remote MCP servers from the browser. +Note: `--ui-mcp-proxy` is unrelated, it only lets the Web UI reach public MCP servers that do not send CORS headers. It refuses local and private addresses (loopback, LAN, link-local); connect to those from the Web UI directly. Any server written against the [MCP specification](https://modelcontextprotocol.io) works as is, whether it uses an official SDK or not: the transport is one JSON-RPC message per line on stdio, so a script wrapping an existing program is a valid server too. @@ -861,7 +866,9 @@ Returns raw audio bytes (`audio/wav` by default) rather than JSON. For more info `speaker_ref_b64`: Base64-encoded reference audio to clone the speaker's voice. Optional. Alternatively, upload the reference audio as a `speaker_ref` file field via `multipart/form-data` (see example below) - if both are provided, the uploaded file takes precedence. Rejected when the model has no voice encoder (e.g. Pocket TTS without voice cloning). -`top_k`, `top_p`: Sampling params for the acoustic code predictor. Optional, model-dependent defaults apply. +`top_k`, `top_p`: Sampling params for both the backbone sampler and the acoustic code predictor. Optional, `0` or absent keeps the server `--top-k`/`--top-p` for the backbone and the model defaults for the code predictor. + +`temperature`: Temperature for the backbone sampler and the audio frames, for models that support it. `0` is greedy sampling. Optional, absent keeps the server `--temp` for the backbone and the model defaults for the audio. `repeat_penalty`: Repetition penalty applied to the backbone sampler over the whole generation (default `1.05`). Without this, the backbone can loop and re-generate the same utterance. @@ -869,7 +876,7 @@ Returns raw audio bytes (`audio/wav` by default) rather than JSON. For more info `n_predict`: Max number of audio frames to generate. Defaults to `512`; generation normally stops earlier once the model emits an end-of-speech token. -`response_format`: `wav` (default) or `pcm` (raw `float32` little-endian mono samples, no header; the response `Content-Type` carries the sample rate, e.g. `audio/pcm;rate=24000;encoding=float;bits=32`). +`response_format`: `wav` (default), `pcm` (raw `float32` little-endian mono samples, no header; the response `Content-Type` carries the sample rate, e.g. `audio/pcm;rate=24000;encoding=float;bits=32`) or `pcm16` (same, as signed 16-bit samples, e.g. `audio/pcm;rate=24000;encoding=signed-int;bits=16`). `stream`: If `true`, the response is streamed as audio becomes available instead of waiting for the full generation to finish. WAV streaming writes an RFC-noncompliant header with an unknown (`0xFFFFFFFF`) size field, since the final length isn't known up front; most players and decoders (ffmpeg, VLC, ...) handle this by reading until EOF. @@ -1443,7 +1450,7 @@ Given a ChatML-formatted json description in `messages`, it returns the predicte See [OpenAI Chat Completions API documentation](https://platform.openai.com/docs/api-reference/chat). llama.cpp `/completion`-specific features such as `mirostat` are also supported. -The `response_format` parameter supports both plain JSON output (e.g. `{"type": "json_object"}`) and schema-constrained JSON (e.g. `{"type": "json_object", "schema": {"type": "string", "minLength": 10, "maxLength": 100}}` or `{"type": "json_schema", "schema": {"properties": { "name": { "title": "Name", "type": "string" }, "date": { "title": "Date", "type": "string" }, "participants": { "items": {"type: "string" }, "title": "Participants", "type": "string" } } } }`), similar to other OpenAI-inspired API providers. +The `response_format` parameter supports both plain JSON output (e.g. `{"type": "json_object"}`) and schema-constrained JSON (e.g. `{"type": "json_object", "schema": {"type": "string", "minLength": 10, "maxLength": 100}}` or `{"type": "json_schema", "schema": {"properties": { "name": { "title": "Name", "type": "string" }, "date": { "title": "Date", "type": "string" }, "participants": { "items": {"type: "string" }, "title": "Participants", "type": "string" } } } }`), similar to other OpenAI-inspired API providers. The OpenAI wrapper form `{"type": "json_schema", "json_schema": {"name": ..., "strict": ..., "schema": {...}}}` is also accepted. `chat_template_kwargs`: Allows sending additional parameters to the json templating system. For example: `{"enable_thinking": false}` @@ -1593,11 +1600,11 @@ Models that write the text while they listen use this endpoint: Voxtral Realtime Live ASR models can also hold a conversation, so they transcribe live only in sessions opened with `/v1/realtime?intent=transcription`, as OpenAI transcription sessions are; other sessions are a [realtime conversation](#realtime-conversation). -Audio must be PCM16, mono, at the model sample rate. The sample rate is returned in `session.created` as `audio.input.format.rate`. Each `input_audio_buffer.append` carries a base64-encoded chunk of PCM16 samples. +Audio must be PCM16, mono. By default it is at the model sample rate, returned in `session.created` as `audio.input.format.rate`. A client can send another rate (8000 to 192000 Hz) by setting `audio.input.format.rate` with `session.update`, the server then resamples it. Each `input_audio_buffer.append` carries a base64-encoded chunk of PCM16 samples. *Client events:* -- `session.update`: accepted for compatibility, the server replies with `session.updated` +- `session.update`: only `audio.input.format.rate` is used, the server replies with `session.updated` - `input_audio_buffer.append`: `audio` holds base64 PCM16 samples - `input_audio_buffer.commit`: flushes the audio received so far, so the words still held back by the model's delay are produced. The server replies with `input_audio_buffer.committed` and starts a new item. With `"final": true`, the input ends instead and the last item is completed once the stream drains. - `input_audio_buffer.clear`: accepted for compatibility, audio already sent cannot be discarded @@ -1616,7 +1623,7 @@ Note that words held back by the model's delay at the time of a commit are attri #### Realtime conversation -Models that answer after the audio, like LFM3-Audio and LFM3-Omni, hold a conversation on the same endpoint (with an `asr` template, unless the session is opened with `?intent=transcription`). The audio is prefilled as it arrives, so the response starts right after the last chunk. Audio input is PCM16 mono at the model sample rate (`audio.input.format.rate` of `session.created`), audio output is PCM16 mono at `audio.output.format.rate`. +Models that answer after the audio, like LFM3-Audio and LFM3-Omni, hold a conversation on the same endpoint (with an `asr` template, unless the session is opened with `?intent=transcription`). The audio is prefilled as it arrives, so the response starts right after the last chunk. Audio input is PCM16 mono, by default at the model sample rate (`audio.input.format.rate` of `session.created`); another rate set with `session.update` is resampled by the server. Audio output is PCM16 mono at `audio.output.format.rate`. `session.type` selects the session, it can be changed with `session.update` before the first audio or message: @@ -1625,7 +1632,7 @@ Models that answer after the audio, like LFM3-Audio and LFM3-Omni, hold a conver *Client events:* -- `session.update`: `type`, `instructions` (the system prompt, before the first audio or message), `output_modalities`, `max_output_tokens` (an integer or `"inf"`, the default), `tools` (functions with `name`, `description` and `parameters`, before the first audio or message) and `tool_choice` (`"auto"`, the default, `"none"` or `"required"`) +- `session.update`: `type`, `audio.input.format.rate` (8000 to 192000 Hz), `instructions` (the system prompt, before the first audio or message), `output_modalities`, `max_output_tokens` (an integer or `"inf"`, the default), `tools` (functions with `name`, `description` and `parameters`, before the first audio or message) and `tool_choice` (`"auto"`, the default, `"none"` or `"required"`) - `input_audio_buffer.append`: `audio` holds base64 PCM16 samples - `input_audio_buffer.commit`: ends the audio of the user turn, in a transcription session the transcript follows - `conversation.item.create`: a user message with `input_text` content, or a `function_call_output` with the `call_id` and `output` of a tool call of the last response @@ -1649,6 +1656,33 @@ Tools go into the system prompt with the chat template, and the tool calls are p See [examples/realtime_client.py](../../examples/realtime_client.py) for a client with the `openai` library. +### POST `/v1/audio/speech`: OpenAI-compatible Speech API + +Generates speech from text, for models that can generate audio. The request is converted to a [`/tts`](#post-tts-generate-speech-audio-from-text) request, so the `/tts` options such as `lang`, `seed` or `speaker_ref_b64` can also be used. + +*Options:* + +`input`: The text to speak. + +`voice`: One of the `voices` listed by `GET /props`, as a name or as `{"id": "..."}`. OpenAI voice names such as `alloy` are not mapped: a model with predefined voices rejects them. + +`response_format`: `wav` or `pcm` (raw signed 16-bit little-endian mono samples, no header). Defaults to `wav`, unlike the `mp3` of the OpenAI API. Other formats are rejected. The sample rate is the one of the model and is not resampled to the 24 kHz of the OpenAI API, the response `Content-Type` carries it. + +`stream_format`: `audio` streams the response as it becomes available, see `stream` in `/tts` for the WAV header it writes. `sse` is rejected. The response is not streamed when omitted. + +`speed`: Only `1.0` is accepted. + +`instructions`: Ignored. + +*Examples:* + +```shell +curl http://127.0.0.1:8080/v1/audio/speech \ + -H "Content-Type: application/json" \ + -d '{"model": "tts", "input": "Hello, this is a test."}' \ + -o output.wav +``` + ### POST `/v1/responses`: OpenAI-compatible Responses API *Options:* @@ -2476,7 +2510,7 @@ To pause a download, send an event to `/models/unload`: it keeps its partial fil Download procedure: - Send POST request to `/models` - Subscribe to `/models/sse` for updates -- On downloading completed, you will receive either `download_finished` or `download_failed` event +- On downloading completed, you will receive either `download_finished` or `download_failed` event. A download that stopped because a client paused or cancelled it fails with `{"stopped": true}` as its data - Call GET `/models` to trigger model list update. If the download success, you should see the new model in the list Payload: diff --git a/tools/server/server-chat.cpp b/tools/server/server-chat.cpp index a7431800b40b..7b080d17a1e5 100644 --- a/tools/server/server-chat.cpp +++ b/tools/server/server-chat.cpp @@ -311,6 +311,34 @@ json server_chat_convert_responses_to_chatcmpl(const json & response_body) { chatcmpl_body.erase("reasoning"); } + if (response_body.contains("text")) { + const json & text = response_body.at("text"); + if (!text.is_object()) { + throw std::invalid_argument("'text' must be an object"); + } + + chatcmpl_body.erase("text"); + if (text.contains("format")) { + const json & format = text.at("format"); + if (!format.is_object()) { + throw std::invalid_argument("'text.format' must be an object"); + } + + const std::string type = json_value(format, "type", std::string()); + if (type == "json_schema") { + chatcmpl_body["response_format"] = { + {"type", "json_schema"}, + {"json_schema", format}, + }; + chatcmpl_body["response_format"]["json_schema"].erase("type"); + } else if (type == "json_object" || type == "text") { + chatcmpl_body["response_format"] = format; + } else if (!type.empty()) { + throw std::invalid_argument("'text.format.type' must be one of 'text', 'json_object', or 'json_schema'"); + } + } + } + // llama.cpp params (e.g. seed, min_p) go under extra_params and are merged into the converted body if (response_body.contains("extra_params")) { chatcmpl_body.erase("extra_params"); @@ -737,6 +765,42 @@ json convert_transcriptions_to_chatcmpl( return chatcmpl_body; } +json convert_speech_to_tts(const json & inp_body) { + json tts_body = inp_body; // copy all fields + + // a custom voice is given as {"id": "..."} + if (inp_body.contains("voice") && inp_body.at("voice").is_object()) { + tts_body["voice"] = json_value(inp_body.at("voice"), "id", std::string()); + } + + // the default of the OpenAI API is mp3, we have no encoder for it + const std::string response_format = json_value(inp_body, "response_format", std::string("wav")); + if (response_format == "pcm") { + tts_body["response_format"] = "pcm16"; // "pcm" is float32 in /tts + } else if (response_format == "wav") { + tts_body["response_format"] = "wav"; + } else { + throw std::invalid_argument("Only 'wav' and 'pcm' response_format are supported for speech"); + } + + const std::string stream_format = json_value(inp_body, "stream_format", std::string()); + if (stream_format == "audio") { + tts_body["stream"] = true; + } else if (!stream_format.empty()) { + throw std::invalid_argument("Only 'audio' stream_format is supported for speech"); + } + + if (json_value(inp_body, "speed", 1.0) != 1.0) { + throw std::invalid_argument("'speed' is not supported for speech"); + } + + if (!json_value(inp_body, "instructions", std::string()).empty()) { + SRV_WRN("%s", "'instructions' is not supported for speech and is ignored\n"); + } + + return tts_body; +} + llama_tokens server_chat_get_asr_prompt(const llama_model * model) { const char * src = llama_model_chat_template(model, "asr"); if (src == nullptr) { diff --git a/tools/server/server-chat.h b/tools/server/server-chat.h index b4b6f1ec6582..cba4a073c3d8 100644 --- a/tools/server/server-chat.h +++ b/tools/server/server-chat.h @@ -24,6 +24,9 @@ json convert_transcriptions_to_chatcmpl( const std::map & in_files, std::vector & out_files); +// convert OpenAI speech API format to the /tts format +json convert_speech_to_tts(const json & body); + // prompt of the "asr" chat template, live ASR models (e.g. LFM3-Audio live ASR) transcribe with it // empty if the model has no such template, throws on a template error llama_tokens server_chat_get_asr_prompt(const llama_model * model); diff --git a/tools/server/server-common.cpp b/tools/server/server-common.cpp index 311eb4f12fb4..0013c09cd0a2 100644 --- a/tools/server/server-common.cpp +++ b/tools/server/server-common.cpp @@ -9,6 +9,10 @@ #include "server-common.h" +#include + +#include +#include #include #include #include @@ -200,6 +204,19 @@ const char * get_media_marker() { return marker.c_str(); } +// +// audio utils +// + +std::string pcm_f32_to_s16(const float * samples, size_t n) { + std::string bytes(n * sizeof(int16_t), '\0'); + for (size_t i = 0; i < n; i++) { + const int16_t v = (int16_t) std::lround(std::max(-1.0f, std::min(1.0f, samples[i])) * 32767.0f); + memcpy(&bytes[2 * i], &v, sizeof(v)); + } + return bytes; +} + // // model output modalities // @@ -1200,10 +1217,55 @@ json oaicompat_completion_params_parse(const json & body) { // - file:// for local files (only allowed if media_path is set) // - data: for base64 encoded data with uri scheme (e.g. data:image/png;base64,...) // - raw base64 encoded data +static bool ipv4_is_public(const unsigned char * b) { + return !(b[0] == 0 || b[0] == 10 || b[0] == 127 || b[0] >= 224 // this network, private, loopback, multicast, reserved + || (b[0] == 100 && (b[1] & 0xc0) == 64) // 100.64.0.0/10 CGNAT + || (b[0] == 169 && b[1] == 254) // link-local, cloud metadata + || (b[0] == 172 && (b[1] & 0xf0) == 16) // 172.16.0.0/12 + || (b[0] == 192 && b[1] == 168) // 192.168.0.0/16 + || (b[0] == 192 && b[1] == 0 && b[2] == 0) // 192.0.0.0/24 + || (b[0] == 198 && (b[1] & 0xfe) == 18)); // 198.18.0.0/15 +} + +bool server_addr_is_public(const std::string & addr) { + unsigned char b[16]; + if (inet_pton(AF_INET, addr.c_str(), b) == 1) { + return ipv4_is_public(b); + } + if (inet_pton(AF_INET6, addr.c_str(), b) != 1) { + return false; // e.g. scoped address "fe80::1%eth0" + } + static const unsigned char v4_mapped[12] = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0xff, 0xff}; + static const unsigned char nat64[12] = {0, 0x64, 0xff, 0x9b, 0, 0, 0, 0, 0, 0, 0, 0}; + static const unsigned char zero[12] = {}; + if (memcmp(b, v4_mapped, 12) == 0 || memcmp(b, nat64, 12) == 0) { + return ipv4_is_public(b + 12); + } + return memcmp(b, zero, 12) != 0 // ::/96: unspecified, loopback, IPv4-compatible + && (b[0] & 0xfe) != 0xfc // fc00::/7 unique local + && !(b[0] == 0xfe && (b[1] & 0x80) == 0x80) // fe80::/10 link-local, fec0::/10 site-local + && b[0] != 0xff; // multicast +} + +std::string server_resolve_public_addr(const std::string & host, bool allow_private) { + std::vector addrs; + httplib::hosted_at(host, addrs); + if (addrs.empty()) { + throw std::invalid_argument("cannot resolve host: " + host); + } + for (const auto & addr : addrs) { + if (!allow_private && !server_addr_is_public(addr)) { + throw std::invalid_argument("host is not a public address: " + host); + } + } + return addrs[0]; +} + void handle_media( std::vector & out_files, const std::string & url, - const std::string & media_path) { + const std::string & media_path, + bool allow_private_urls) { if (!media_path.empty()) { // should already be enforced by arg.cpp, but checking just in case GGML_ASSERT(media_path.back() == DIRECTORY_SEPARATOR); @@ -1215,6 +1277,9 @@ void handle_media( common_remote_params params; params.max_size = 1024 * 1024 * 10; // 10MB params.timeout = 10; // seconds + params.resolve_host = [allow_private_urls](const std::string & host) { + return server_resolve_public_addr(host, allow_private_urls); + }; SRV_INF("downloading image from '%s'\n", url.c_str()); auto res = common_remote_get_content(url, params); if (200 <= res.first && res.first < 300) { @@ -1281,7 +1346,7 @@ static void oaicompat_content_load_media(json & content, const server_chat_param json image_url = json_value(p, "image_url", json::object()); std::string url = json_value(image_url, "url", std::string()); - handle_media(out_files, url, opt.media_path); + handle_media(out_files, url, opt.media_path, opt.media_allow_private_urls); p["type"] = "media_marker"; p["text"] = get_media_marker(); @@ -1296,7 +1361,7 @@ static void oaicompat_content_load_media(json & content, const server_chat_param json input_audio = json_value(p, "input_audio", json::object()); std::string url = json_value(input_audio, "data", json_value(input_audio, "url", std::string())); - handle_media(out_files, url, opt.media_path); + handle_media(out_files, url, opt.media_path, opt.media_allow_private_urls); p["type"] = "media_marker"; p["text"] = get_media_marker(); @@ -1311,7 +1376,7 @@ static void oaicompat_content_load_media(json & content, const server_chat_param json input_video = json_value(p, type, json::object()); std::string url = json_value(input_video, "data", json_value(input_video, "url", std::string())); - handle_media(out_files, url, opt.media_path); + handle_media(out_files, url, opt.media_path, opt.media_allow_private_urls); p["type"] = "media_marker"; p["text"] = get_media_marker(); @@ -1390,10 +1455,14 @@ json oaicompat_chat_params_parse( json_schema = json_value(response_format, "schema", json::object()); } } else if (response_type == "json_schema") { - auto schema_wrapper = json_value(response_format, "json_schema", json::object()); - json_schema = json_value(schema_wrapper, "schema", json::object()); + if (response_format.contains("json_schema")) { + auto schema_wrapper = json_value(response_format, "json_schema", json::object()); + json_schema = json_value(schema_wrapper, "schema", json::object()); + } else { + json_schema = json_value(response_format, "schema", json::object()); + } } else if (!response_type.empty() && response_type != "text") { - throw std::invalid_argument("response_format type must be one of \"text\" or \"json_object\", but got: " + response_type); + throw std::invalid_argument("response_format type must be one of \"text\", \"json_object\" or \"json_schema\", but got: " + response_type); } } diff --git a/tools/server/server-common.h b/tools/server/server-common.h index 01bf12e61ed2..05622849a1e7 100644 --- a/tools/server/server-common.h +++ b/tools/server/server-common.h @@ -110,6 +110,13 @@ std::string gen_tool_call_id(); // get a random marker; note: each time the server restarts, the marker will be different const char * get_media_marker(); +// +// audio utils +// + +// float samples to signed 16-bit PCM bytes, clamped to [-1, 1] +std::string pcm_f32_to_s16(const float * samples, size_t n); + // // model output modalities // @@ -292,11 +299,19 @@ llama_tokens tokenize_mixed(const llama_vocab * vocab, const json & json_prompt, // if validate_utf8(text) == text.size(), then the whole text is valid utf8 size_t validate_utf8(const std::string& text); +// true if the address is publicly routable +bool server_addr_is_public(const std::string & addr); + +// resolve host to an address to connect to; throws if not resolvable or not public (unless allow_private) +std::string server_resolve_public_addr(const std::string & host, bool allow_private); + // load a media file from an URL (http, file, data) or from raw base64 data +// http(s) URLs must resolve to public addresses unless allow_private_urls void handle_media( std::vector & out_files, const std::string & url, - const std::string & media_path); + const std::string & media_path, + bool allow_private_urls = false); // process mtmd prompt, return the server_tokens containing both text tokens and media chunks // if is_placeholder is true, the media chunk will be treated as placeholder for counting tokens; the output tokens are not usable for actual inference (e.g. for submitting a task to server_queue) @@ -308,7 +323,7 @@ server_tokens process_mtmd_prompt( bool is_placeholder = false); // decode one media url (http/file/data-uri) into out_files; throws on failure -void handle_media(std::vector & out_files, const std::string & url, const std::string & media_path); +void handle_media(std::vector & out_files, const std::string & url, const std::string & media_path, bool allow_private_urls); /** * break the input "prompt" object into multiple prompt if needed, then tokenize them @@ -358,6 +373,7 @@ struct server_chat_params { int reasoning_budget = -1; std::string reasoning_budget_message; std::string media_path; + bool media_allow_private_urls = false; bool force_pure_content = false; }; diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp index 48f1e93a21b3..a6059ab38050 100644 --- a/tools/server/server-context.cpp +++ b/tools/server/server-context.cpp @@ -358,9 +358,12 @@ struct server_slot { return false; } - llama_state_seq_get_data_ext(ctx_tgt, cur->data.main.data(), cur_size_tgt, id, LLAMA_STATE_SEQ_FLAGS_NONE); - if (ctx_dft) { - llama_state_seq_get_data_ext(ctx_dft, cur->data.drft.data(), cur_size_dft, id, LLAMA_STATE_SEQ_FLAGS_NONE); + const size_t n_tgt = llama_state_seq_get_data_ext(ctx_tgt, cur->data.main.data(), cur_size_tgt, id, LLAMA_STATE_SEQ_FLAGS_NONE); + const size_t n_dft = ctx_dft ? llama_state_seq_get_data_ext(ctx_dft, cur->data.drft.data(), cur_size_dft, id, LLAMA_STATE_SEQ_FLAGS_NONE) : 0; + if (n_tgt != cur_size_tgt || n_dft != cur_size_dft) { + SLT_ERR(*this, "%s", "failed to save complete prompt state\n"); + prompt_cache.states.pop_back(); + return false; } return true; @@ -1736,6 +1739,7 @@ struct server_context_impl { /* reasoning_budget */ params_base.sampling.reasoning_budget_tokens, /* reasoning_budget_msg */ params_base.sampling.reasoning_budget_message, /* media_path */ params_base.media_path, + /* media_allow_private */ params_base.media_allow_private_urls, /* force_pure_content */ params_base.force_pure_content_parser }; @@ -2731,21 +2735,22 @@ struct server_context_impl { for (auto it = slot.prompt.checkpoints.begin(); slot.prompt.checkpoints.size() + 1 >= (size_t) params_base.n_ctx_checkpoints && it != slot.prompt.checkpoints.end(); ) { - if (it->id_task != id_task && last >= 0 && it->n_tokens <= last + params_base.checkpoint_min_step) { + const auto & cur = **it; + if (cur.id_task != id_task && last >= 0 && cur.n_tokens <= last + params_base.checkpoint_min_step) { SLT_TRC(slot, "erasing context checkpoint too close to an earlier one (pos_min = %d, pos_max = %d, n_tokens = %" PRId64 ", size = %.3f MiB)\n", - it->pos_min, it->pos_max, it->n_tokens, (float) it->size() / 1024 / 1024); + cur.pos_min, cur.pos_max, cur.n_tokens, (float) cur.size() / 1024 / 1024); it = slot.prompt.checkpoints.erase(it); continue; } - last = it->n_tokens; + last = cur.n_tokens; ++it; } while (slot.prompt.checkpoints.size() >= (size_t) params_base.n_ctx_checkpoints) { // make room for the new checkpoint, if needed - const auto & cur = slot.prompt.checkpoints.front(); + const auto & cur = *slot.prompt.checkpoints.front(); SLT_WRN(slot, "erasing old context checkpoint (pos_min = %d, pos_max = %d, n_tokens = %" PRId64 ", size = %.3f MiB)\n", cur.pos_min, cur.pos_max, cur.n_tokens, (float) cur.size() / 1024 / 1024); @@ -2757,8 +2762,8 @@ struct server_context_impl { { const int64_t n_tokens_new = slot.prompt.n_tokens() - n_tokens_cur; for (auto it = slot.prompt.checkpoints.begin(); it != slot.prompt.checkpoints.end(); ) { - if (it->n_tokens == n_tokens_new) { - SLT_TRC(slot, "superseding context checkpoint at n_tokens = %" PRId64 "\n", it->n_tokens); + if ((*it)->n_tokens == n_tokens_new) { + SLT_TRC(slot, "superseding context checkpoint at n_tokens = %" PRId64 "\n", (*it)->n_tokens); it = slot.prompt.checkpoints.erase(it); } else { ++it; @@ -2766,7 +2771,8 @@ struct server_context_impl { } } - auto & cur = slot.prompt.checkpoints.emplace_back(); + auto ckpt = std::make_shared(); + auto & cur = *ckpt; cur.id_task = id_task; @@ -2780,6 +2786,8 @@ struct server_context_impl { // stash the draft's speculative state with the checkpoint common_speculative_get_state(spec.get(), slot.id, cur.data_spec); + slot.prompt.checkpoints.push_back(std::move(ckpt)); + SLT_TRC(slot, "created context checkpoint %d of %d (pos_min = %d, pos_max = %d, n_tokens = %" PRId64 ", size = %.3f MiB)\n", (int) slot.prompt.checkpoints.size(), params_base.n_ctx_checkpoints, cur.pos_min, @@ -2799,7 +2807,8 @@ struct server_context_impl { return true; } - static bool ckpt_read_buf(std::ifstream & ifs, std::vector & buf, size_t n_avail, size_t & n_read) { + template + static bool ckpt_read_buf(std::ifstream & ifs, Buf & buf, size_t n_avail, size_t & n_read) { uint64_t n = 0; // check the size against the bytes left in the file before allocating, the size field may be corrupted if (!ckpt_read(ifs, &n, sizeof(n), n_read) || n > n_avail - n_read) { @@ -2814,7 +2823,8 @@ struct server_context_impl { n_written += size; } - static void ckpt_write_buf(std::ofstream & ofs, const std::vector & buf, size_t & n_written) { + template + static void ckpt_write_buf(std::ofstream & ofs, const Buf & buf, size_t & n_written) { const uint64_t n = buf.size(); ckpt_write(ofs, &n, sizeof(n), n_written); if (n > 0) { @@ -2828,7 +2838,7 @@ struct server_context_impl { if (slot.prompt.checkpoints.empty()) { return true; } - std::ofstream ofs(std::filesystem::u8path(filepath), std::ios::binary | std::ios::app); + std::ofstream ofs(filepath, std::ios::binary | std::ios::app); if (!ofs) { SLT_WRN(slot, "failed to append context checkpoints to '%s'\n", filepath.c_str()); return false; @@ -2839,7 +2849,8 @@ struct server_context_impl { ckpt_write(ofs, &magic, sizeof(magic), n_written); ckpt_write(ofs, &version, sizeof(version), n_written); ckpt_write(ofs, &count, sizeof(count), n_written); - for (const auto & cur : slot.prompt.checkpoints) { + for (const auto & ckpt : slot.prompt.checkpoints) { + const auto & cur = *ckpt; ckpt_write(ofs, &cur.n_tokens, sizeof(cur.n_tokens), n_written); ckpt_write(ofs, &cur.pos_min, sizeof(cur.pos_min), n_written); ckpt_write(ofs, &cur.pos_max, sizeof(cur.pos_max), n_written); @@ -2859,7 +2870,7 @@ struct server_context_impl { // returns the number of bytes consumed, 0 if there is no usable appendix size_t load_slot_checkpoints(const std::string & filepath, size_t offset, server_slot & slot) const { - std::ifstream ifs(std::filesystem::u8path(filepath), std::ios::binary | std::ios::ate); + std::ifstream ifs(filepath, std::ios::binary | std::ios::ate); const size_t file_size = ifs ? (size_t) ifs.tellg() : 0; if (!ifs || file_size < offset || !ifs.seekg(offset)) { return 0; @@ -2877,9 +2888,10 @@ struct server_context_impl { SLT_WRN(slot, "invalid context checkpoint appendix in '%s' - ignored\n", filepath.c_str()); return 0; } - std::list checkpoints; + std::list> checkpoints; for (uint32_t i = 0; i < count; ++i) { - common_prompt_checkpoint cur; + auto ckpt = std::make_shared(); + auto & cur = *ckpt; cur.id_task = -1; // not created by a task - marks a checkpoint restored from a slot file if (!ckpt_read(ifs, &cur.n_tokens, sizeof(cur.n_tokens), n_read) || !ckpt_read(ifs, &cur.pos_min, sizeof(cur.pos_min), n_read) || @@ -2895,23 +2907,23 @@ struct server_context_impl { SLT_WRN(slot, "invalid context checkpoint appendix in '%s' - ignored\n", filepath.c_str()); return 0; } - checkpoints.push_back(std::move(cur)); + checkpoints.push_back(std::move(ckpt)); if (checkpoints.size() > (size_t) params_base.n_ctx_checkpoints) { checkpoints.pop_front(); } } // the slot file does not check the draft context - test-load one draft checkpoint, drop the draft data if it does not fit - if (ctx_dft != nullptr && !checkpoints.empty() && !checkpoints.back().data_dft.empty()) { - const bool ok = checkpoints.back().load_dft(ctx_dft, slot.id, LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY); + if (ctx_dft != nullptr && !checkpoints.empty() && !checkpoints.back()->data_dft.empty()) { + const bool ok = checkpoints.back()->load_dft(ctx_dft, slot.id, LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY); llama_memory_seq_rm(llama_get_memory(ctx_dft), slot.id, -1, -1); if (!ok) { SLT_WRN(slot, "draft context checkpoint data in '%s' does not match the draft context - dropped\n", filepath.c_str()); for (auto & cur : checkpoints) { - cur.clear_dft(); + cur->clear_dft(); } } } - slot.prompt.checkpoints = std::move(checkpoints); + slot.prompt.checkpoints.assign(checkpoints.begin(), checkpoints.end()); SLT_INF(slot, "restored %zu context checkpoint(s) from '%s'\n", slot.prompt.checkpoints.size(), filepath.c_str()); return n_read; } @@ -2934,7 +2946,7 @@ struct server_context_impl { if (data_dft.empty() && data_spec.empty()) { return true; } - std::ofstream ofs(std::filesystem::u8path(filepath), std::ios::binary | std::ios::app); + std::ofstream ofs(filepath, std::ios::binary | std::ios::app); if (!ofs) { SLT_WRN(slot, "failed to append the draft state to '%s'\n", filepath.c_str()); return false; @@ -2960,7 +2972,7 @@ struct server_context_impl { if (ctx_dft != nullptr) { llama_memory_seq_rm(llama_get_memory(ctx_dft), slot.id, -1, -1); } - std::ifstream ifs(std::filesystem::u8path(filepath), std::ios::binary | std::ios::ate); + std::ifstream ifs(filepath, std::ios::binary | std::ios::ate); const size_t file_size = ifs ? (size_t) ifs.tellg() : 0; if (!ifs || file_size < offset || !ifs.seekg(offset)) { return 0; @@ -3413,13 +3425,19 @@ struct server_context_impl { } void iterate(std::vector & slots, std::function callback) { - for (auto & slot : slots) { - try { - callback(slot); - } catch (const std::exception & e) { - SLT_ERR(slot, "got exception: %s\n", e.what()); - send_error(slot, std::string("got exception: ") + e.what(), ERROR_TYPE_SERVER); - slot.release(); + for (int pass = 0; pass < 2; ++pass) { + for (auto & slot : slots) { + const bool is_child = slot.is_processing() && slot.task->is_child(); + if (is_child != (pass == 1)) { + continue; + } + try { + callback(slot); + } catch (const std::exception & e) { + SLT_ERR(slot, "got exception: %s\n", e.what()); + send_error(slot, std::string("got exception: ") + e.what(), ERROR_TYPE_SERVER); + slot.release(); + } } } } @@ -3996,6 +4014,7 @@ struct server_context_impl { // check if this is a child slot if (slot.state == SLOT_STATE_WAIT_OTHER) { SLT_DBG(slot, "%s", "waiting for parent slot to complete\n"); + SLT_DBG(slot, "%s", "__TEST_TAG_WAIT_PARENT_SLOT__\n"); return; } @@ -4254,7 +4273,8 @@ struct server_context_impl { const auto it = std::find_if( slot.prompt.checkpoints.rbegin(), slot.prompt.checkpoints.rend(), - [&](const auto & cur) { + [&](const auto & ckpt) { + const auto & cur = *ckpt; // guarantee that a checkpoint will result in at least one token being processed [TAG_PROMPT_LOGITS] SLT_TRC(slot, "checking checkpoint with [%d, %d] against %d...\n", cur.pos_min, cur.pos_max, pos_min_thold); // workaround for [TAG_CHECKPOINTS_FIX_POS_MIN] @@ -4268,10 +4288,12 @@ struct server_context_impl { bool do_reset = it == slot.prompt.checkpoints.rend(); if (!do_reset) { + const auto & cur = **it; + // restore the context checkpoint - if (!it->load_tgt(ctx_tgt, slot.id, LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY) || - !it->load_dft(ctx_dft, slot.id, LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY)) { - if (it->id_task != -1) { + if (!cur.load_tgt(ctx_tgt, slot.id, LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY) || + !cur.load_dft(ctx_dft, slot.id, LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY)) { + if (cur.id_task != -1) { GGML_ABORT("failed to restore context checkpoint\n"); } // restored from a slot file, not guaranteed to load - fall back to full prompt re-processing @@ -4281,12 +4303,14 @@ struct server_context_impl { } if (!do_reset) { + const auto & cur = **it; + // restore the draft's speculative state - common_speculative_set_state(spec.get(), slot.id, it->data_spec); + common_speculative_set_state(spec.get(), slot.id, cur.data_spec); - pos_next = std::min(pos_next, std::max(it->pos_min + 1, it->pos_max)); - n_past = std::min(slot.prompt.tokens.size_up_to_pos(pos_next), (size_t) it->n_tokens); - SLT_TRC(slot, "restored context checkpoint (pos_min = %d, pos_max = %d, n_tokens = %" PRId64 ", n_past = %d, size = %.3f MiB)\n", it->pos_min, it->pos_max, it->n_tokens, n_past, (float) it->size() / 1024 / 1024); + pos_next = std::min(pos_next, std::max(cur.pos_min + 1, cur.pos_max)); + n_past = std::min(slot.prompt.tokens.size_up_to_pos(pos_next), (size_t) cur.n_tokens); + SLT_TRC(slot, "restored context checkpoint (pos_min = %d, pos_max = %d, n_tokens = %" PRId64 ", n_past = %d, size = %.3f MiB)\n", cur.pos_min, cur.pos_max, cur.n_tokens, n_past, (float) cur.size() / 1024 / 1024); } if (do_reset) { @@ -4301,7 +4325,7 @@ struct server_context_impl { { // erase any checkpoints with pos_max > pos_next, or that cover tokens not shared with the new prompt for (auto it = slot.prompt.checkpoints.begin(); it != slot.prompt.checkpoints.end();) { - const auto & cur = *it; + const auto & cur = **it; if (cur.pos_max > pos_next || cur.n_tokens > n_prefix) { SLT_TRC(slot, "erased invalidated context checkpoint (pos_min = %d, pos_max = %d, n_tokens = %" PRId64 ", n_swa = %d, pos_next = %d, size = %.3f MiB)\n", cur.pos_min, cur.pos_max, cur.n_tokens, n_swa, pos_next, (float) cur.size() / 1024 / 1024); it = slot.prompt.checkpoints.erase(it); @@ -4504,7 +4528,7 @@ struct server_context_impl { const auto pos = slot.prompt.n_tokens(); const auto & checkpoints = slot.prompt.checkpoints; - if (pos == last_user_pos || checkpoints.empty() || pos > checkpoints.back().n_tokens + params_base.checkpoint_min_step) { + if (pos == last_user_pos || checkpoints.empty() || pos > checkpoints.back()->n_tokens + params_base.checkpoint_min_step) { break; } } @@ -4578,7 +4602,7 @@ struct server_context_impl { do_checkpoint = do_checkpoint && ( slot.prompt.checkpoints.empty() || is_last_user_message || near_prompt_end || - n_tokens_start > slot.prompt.checkpoints.back().n_tokens + params_base.checkpoint_min_step); + n_tokens_start > slot.prompt.checkpoints.back()->n_tokens + params_base.checkpoint_min_step); SLT_DBG(slot, "main/do_checkpoint = %s, pos_min = %d, pos_max = %d\n", do_checkpoint ? "yes" : "no", pos_min, pos_max); // note: we create the checkpoint before calling llama_decode(), so the current batch is not @@ -5279,12 +5303,19 @@ server_context_meta server_context::get_meta() const { // may have bypass_sleep = true if the task does not use ctx_server struct server_res_generator : server_res_spipe { server_response_reader rd; + server_queue * queue_awake = nullptr; // set while this response keeps the server awake server_res_generator(server_queue & queue_tasks, server_response & queue_results, int sleep_idle_seconds, bool bypass_sleep = false) : rd(queue_tasks, queue_results, HTTP_POLLING_SECONDS) { // fast path in case sleeping is disabled bypass_sleep |= sleep_idle_seconds < 0; if (!bypass_sleep) { queue_tasks.wait_until_no_sleep(); + queue_awake = &queue_tasks; + } + } + ~server_res_generator() { + if (queue_awake) { + queue_awake->release_inflight(); } } void ok(const json & response_data) { @@ -5721,6 +5752,25 @@ json server_routes::get_model_info() const { return get_res_model_info(*meta); } +// body of /tts and /v1/audio/speech; multipart form fields arrive as strings, the numeric and boolean params get their types back +static json parse_tts_body(const server_http_req & req) { + json body = json::parse(req.body); + for (const char * k : { "top_k", "n_predict", "max_tokens", "seed" }) { + if (body.contains(k) && body[k].is_string()) { + body[k] = std::stoll(body[k].get()); + } + } + for (const char * k : { "top_p", "temperature", "repeat_penalty", "speed" }) { + if (body.contains(k) && body[k].is_string()) { + body[k] = std::stof(body[k].get()); + } + } + if (body.contains("stream") && body["stream"].is_string()) { + body["stream"] = body["stream"] == "true"; + } + return body; +} + void server_routes::init_routes() { // IMPORTANT: all lambda functions must start with create_response() // this is to ensure that the server_res_generator can handle sleeping case correctly @@ -6144,7 +6194,7 @@ void server_routes::init_routes() { // live transcription: Voxtral Realtime (no chat), or an "asr" model in a session opened with ?intent=transcription const bool live_asr = !ctx_server.asr_prompt.empty() && req.get_param("intent") == "transcription"; if (ctx_server.mtmd_cap.audio_stream && ctx_server.mtmd_cap.chat && !live_asr) { - server_realtime_conversation({queue_tasks, queue_results, *meta, params.sampling, ctx_server.mctx, ctx_server.mtmd_cap, ctx_server.vocab}, conn); + server_realtime_conversation({queue_tasks, queue_results, *meta, params, ctx_server.mctx, ctx_server.mtmd_cap, ctx_server.vocab}, conn); return; } @@ -6155,12 +6205,13 @@ void server_routes::init_routes() { } const std::string session_id = "sess_" + random_string(); - const json session_config = json { + const int model_rate = mtmd_get_audio_sample_rate(ctx_server.mctx); + json session_config = json { {"id", session_id}, {"object", "realtime.transcription_session"}, {"type", "transcription"}, {"audio", {{"input", { - {"format", {{"type", "audio/pcm"}, {"rate", mtmd_get_audio_sample_rate(ctx_server.mctx)}}}, + {"format", {{"type", "audio/pcm"}, {"rate", model_rate}}}, {"transcription", {{"model", meta->model_name}}}, {"turn_detection", nullptr}, {"noise_reduction", nullptr}, @@ -6225,6 +6276,8 @@ void server_routes::init_routes() { // client events are read on this thread, the transcript is sent from the thread below std::atomic input_ended{false}; std::thread reader([&]() { + int input_rate = model_rate; + mtmd_helper::resampler_ptr resampler; // input_rate -> model_rate std::string msg; while (conn.read(msg) != 0) { json ev; @@ -6236,6 +6289,21 @@ void server_routes::init_routes() { } const std::string type = json_value(ev, "type", std::string()); if (type == "session.update" || type == "transcription_session.update") { + const json session = json_value(ev, "session", json::object()); + const json audio = session.is_object() ? session.value("audio", json::object()) : json::object(); + const json in = audio.is_object() ? audio.value("input", json::object()) : json::object(); + if (in.is_object() && in.contains("format") && in["format"].is_object()) { + const int rate = json_value(in["format"], "rate", input_rate); + if (rate < 8000 || rate > 192000) { + send_error_event("session.update: the input audio rate must be between 8000 and 192000 Hz", "invalid_request_error"); + continue; + } + if (rate != input_rate) { + input_rate = rate; + resampler.reset(); + session_config["audio"]["input"]["format"]["rate"] = rate; + } + } send_event(json {{"type", "session.updated"}, {"session", session_config}}); } else if (type == "input_audio_buffer.append") { std::string pcm; @@ -6251,6 +6319,25 @@ void server_routes::init_routes() { memcpy(&v, pcm.data() + 2 * i, sizeof(v)); samples[i] = v / 32768.0f; } + if (!resampler) { + resampler.reset(mtmd_helper_resampler_init(input_rate, model_rate)); + if (!resampler) { + send_error_event("failed to initialize the audio resampler", "server_error"); + break; + } + } + const float * out = nullptr; + const int64_t n = mtmd_helper_resampler_process(resampler.get(), samples.data(), samples.size(), &out); + if (n < 0) { + send_error_event("failed to resample the audio", "server_error"); + break; + } + if (n == 0) { + continue; + } + if (out != samples.data()) { + samples.assign(out, out + n); + } if (!post_control(SERVER_CONTROL_ACTION_AUDIO_APPEND, std::move(samples))) { break; } @@ -6444,196 +6531,14 @@ void server_routes::init_routes() { }; this->post_tts = [this](const server_http_req & req) { - auto res = create_response(); - res->set_req(&req); // will also set spipe if needed - - if (!ctx_server.has_cap_tts()) { - res->error(format_error_response("this server does not support audio generation", ERROR_TYPE_NOT_SUPPORTED)); - return res; - } - - const auto info = mtmd_gen_audio_get_info(ctx_server.mctx); - - json body = json::parse(req.body); - - // multipart form fields arrive as strings, correct the types of the numeric and boolean params - for (const char * k : { "top_k", "n_predict", "max_tokens", "seed" }) { - if (body.contains(k) && body[k].is_string()) { - body[k] = std::stoll(body[k].get()); - } - } - for (const char * k : { "top_p", "repeat_penalty" }) { - if (body.contains(k) && body[k].is_string()) { - body[k] = std::stof(body[k].get()); - } - } - if (body.contains("stream") && body["stream"].is_string()) { - body["stream"] = body["stream"] == "true"; - } - - std::string prompt = json_value(body, "input", json_value(body, "prompt", std::string())); - if (prompt.empty()) { - res->error(format_error_response("\"input\" must be a non-empty string", ERROR_TYPE_INVALID_REQUEST)); - return res; - } - - const std::string response_format = json_value(body, "response_format", std::string("wav")); - - server_task task(SERVER_TASK_TYPE_TTS); - server_schema::eval_tts_schema(params, task, body); - const bool stream = task.params.stream; - // a model with a "tts" chat template (e.g. LFM3-Audio) speaks the prompt rendered from it - const std::string lang = json_value(body, "lang", std::string()); - const std::string voice = json_value(body, "voice", std::string()); - const std::string tts_prompt = common_chat_get_tts_prompt(meta->chat_params.tmpls.get(), prompt, lang, voice); - // a model with predefined voices only speaks with one of them - mtmd::bitmap_ptr voice_ref; - if (!voice.empty() && info.n_voices > 0) { - voice_ref.reset(mtmd_gen_audio_get_voice(ctx_server.mctx, voice.c_str())); - if (!voice_ref) { - res->error(format_error_response("unknown voice \"" + voice + "\", see \"voices\" in GET /props", ERROR_TYPE_INVALID_REQUEST)); - return res; - } - } - task.tts_inp.set_prompt(tts_prompt.empty() ? prompt : tts_prompt); - task.tts_inp.set_lang(lang); - task.tts_inp.data.stream = stream; - task.tts_inp.data.out_type = response_format == "pcm" - ? MTMD_HELPER_GEN_AUDIO_OUTTYPE_PCM - : MTMD_HELPER_GEN_AUDIO_OUTTYPE_WAV; - // note: -1 is clamped to 0 by the sampler, it will disable the penalty - task.params.sampling.penalty_last_n = task.params.n_predict > 0 ? task.params.n_predict : 512; - task.params.sampling.seed = task.tts_inp.data.seed; // same seed for the codec/vocoder RNG - if (task.tts_inp.data.top_k > 0) { - task.params.sampling.top_k = task.tts_inp.data.top_k; - } - if (task.tts_inp.data.top_p > 0) { - task.params.sampling.top_p = task.tts_inp.data.top_p; - } - - // speaker reference: either an uploaded form file ("speaker_ref") or a base64 JSON field ("speaker_ref_b64") - const unsigned char * speaker_ref_data = nullptr; - size_t speaker_ref_len = 0; - std::string speaker_ref_b64_decoded; - - auto speaker_ref_file = req.files.find("speaker_ref"); - if (speaker_ref_file != req.files.end()) { - speaker_ref_data = speaker_ref_file->second.data.data(); - speaker_ref_len = speaker_ref_file->second.data.size(); - } else { - std::string speaker_ref_b64 = json_value(body, "speaker_ref_b64", std::string()); - if (!speaker_ref_b64.empty()) { - try { - speaker_ref_b64_decoded = base64::decode(speaker_ref_b64); - } catch (const std::exception &) { - res->error(format_error_response("\"speaker_ref_b64\" is not valid base64", ERROR_TYPE_INVALID_REQUEST)); - return res; - } - speaker_ref_data = (const unsigned char *) speaker_ref_b64_decoded.data(); - speaker_ref_len = speaker_ref_b64_decoded.size(); - } - } - - if (speaker_ref_len > 0) { - if (voice_ref) { - res->error(format_error_response("\"voice\" and \"speaker_ref\" cannot be used together", ERROR_TYPE_INVALID_REQUEST)); - return res; - } - if (!mtmd_support_audio(ctx_server.mctx)) { - res->error(format_error_response("this model has no voice encoder, \"speaker_ref\" cannot be used", ERROR_TYPE_INVALID_REQUEST)); - return res; - } - auto wrapper = mtmd_helper_bitmap_init_from_buf(ctx_server.mctx, speaker_ref_data, speaker_ref_len, false, ctx_server.init_opt); - if (!wrapper.bitmap) { - res->error(format_error_response("failed to decode \"speaker_ref\"", ERROR_TYPE_INVALID_REQUEST)); - return res; - } - task.tts_inp.set_speaker_ref(mtmd::bitmap_ptr(wrapper.bitmap)); - } else if (voice_ref) { - task.tts_inp.set_speaker_ref(std::move(voice_ref)); - } else if (info.n_voices > 0) { - // without a voice prompt the model produces near silence: the first voice is the default - const char * first = mtmd_gen_audio_voice_name(ctx_server.mctx, 0); - SRV_WRN("no voice given, speaking with \"%s\"\n", first); - task.tts_inp.set_speaker_ref(mtmd::bitmap_ptr(mtmd_gen_audio_get_voice(ctx_server.mctx, first))); - } else { - SRV_WRN("%s", "no speaker reference provided, the model may behave randomly\n"); - } - - auto & rd = res->rd; - task.id = rd.get_new_id(); - rd.post_task(std::move(task)); - - // raw float32 LE mono samples; audio/L16 would imply 16-bit big-endian (RFC 2586) - const std::string content_type = response_format == "pcm" - ? "audio/pcm;rate=" + std::to_string(info.sample_rate) + ";encoding=float;bits=32" - : "audio/wav"; - - if (!stream) { - auto result = rd.next(req.should_stop); - if (!result) { - GGML_ASSERT(req.should_stop()); - return res; // connection is closed - } - if (result->is_error()) { - res->error(result->to_json()); - return res; - } - auto * tts_res = dynamic_cast(result.get()); - GGML_ASSERT(tts_res != nullptr); - res->status = 200; - res->content_type = content_type; - res->data = std::move(tts_res->audio); - return res; - } else { - auto first_result = rd.next(req.should_stop); - if (!first_result) { - GGML_ASSERT(req.should_stop()); - return res; // connection is closed - } - if (first_result->is_error()) { - res->error(first_result->to_json()); - return res; - } - auto * first_tts_res = dynamic_cast(first_result.get()); - GGML_ASSERT(first_tts_res != nullptr); - - res->status = 200; - res->content_type = content_type; - res->data = std::move(first_tts_res->audio); - bool is_done = first_tts_res->final; - - res->set_next([res_this = res.get(), is_done](std::string & output) mutable -> bool { - // flush buffered audio before is_done - the first result can already be final - if (!res_this->data.empty()) { - output = std::move(res_this->data); - res_this->data.clear(); - return true; - } - if (is_done) { - return false; - } - if (res_this->should_stop()) { - return false; - } - - server_response_reader & rd = res_this->rd; - if (!rd.has_next()) { - return false; - } - auto result = rd.next([&res_this]() { return res_this->should_stop(); }); - if (!result || result->is_error()) { - return false; - } - auto * tts_res = dynamic_cast(result.get()); - GGML_ASSERT(tts_res != nullptr); - output = std::move(tts_res->audio); - is_done = tts_res->final; - return true; - }); - } + return handle_tts_impl(req, parse_tts_body(req)); + }; - return res; + this->post_speech_oai = [this](const server_http_req & req) { + json body = convert_speech_to_tts(parse_tts_body(req)); + SRV_DBG("%s\n", "Request converted: OpenAI Speech -> TTS"); + SRV_DBG("converted request: %s\n", body.dump().c_str()); + return handle_tts_impl(req, std::move(body)); }; this->post_systemone = [this](const server_http_req & req) { @@ -7047,7 +6952,7 @@ std::unique_ptr server_routes::handle_rerank_impl(const se auto add_image = [&](const std::string & url) { // marker is appended by format_prompt_rerank, one per decoded file try { - handle_media(out_files, url, params.media_path); + handle_media(out_files, url, params.media_path, params.media_allow_private_urls); } catch (const std::runtime_error & e) { throw std::invalid_argument(e.what()); } @@ -7152,6 +7057,195 @@ std::unique_ptr server_routes::handle_rerank_impl(const se return res; } +std::unique_ptr server_routes::handle_tts_impl(const server_http_req & req, json body) { + auto res = create_response(); + res->set_req(&req); // will also set spipe if needed + + if (!ctx_server.has_cap_tts()) { + res->error(format_error_response("this server does not support audio generation", ERROR_TYPE_NOT_SUPPORTED)); + return res; + } + + const auto info = mtmd_gen_audio_get_info(ctx_server.mctx); + + std::string prompt = json_value(body, "input", json_value(body, "prompt", std::string())); + if (prompt.empty()) { + res->error(format_error_response("\"input\" must be a non-empty string", ERROR_TYPE_INVALID_REQUEST)); + return res; + } + + const std::string response_format = json_value(body, "response_format", std::string("wav")); + const bool pcm16 = response_format == "pcm16"; + + server_task task(SERVER_TASK_TYPE_TTS); + server_schema::eval_tts_schema(params, task, body); + const bool stream = task.params.stream; + // a model with a "tts" chat template (e.g. LFM3-Audio) speaks the prompt rendered from it + const std::string lang = json_value(body, "lang", params.tts_lang); + // speaker_ref in the request overrides the default voice + const bool has_speaker_ref = req.files.count("speaker_ref") > 0 || !json_value(body, "speaker_ref_b64", std::string()).empty(); + const std::string voice = json_value(body, "voice", has_speaker_ref ? std::string() : params.tts_voice); + const std::string tts_prompt = common_chat_get_tts_prompt(meta->chat_params.tmpls.get(), prompt, lang, voice); + // a model with predefined voices only speaks with one of them + mtmd::bitmap_ptr voice_ref; + if (!voice.empty() && info.n_voices > 0) { + voice_ref.reset(mtmd_gen_audio_get_voice(ctx_server.mctx, voice.c_str())); + if (!voice_ref) { + res->error(format_error_response("unknown voice \"" + voice + "\", see \"voices\" in GET /props", ERROR_TYPE_INVALID_REQUEST)); + return res; + } + } + task.tts_inp.set_prompt(tts_prompt.empty() ? prompt : tts_prompt); + task.tts_inp.set_lang(lang); + task.tts_inp.data.stream = stream; + task.tts_inp.data.out_type = response_format == "pcm" || pcm16 + ? MTMD_HELPER_GEN_AUDIO_OUTTYPE_PCM + : MTMD_HELPER_GEN_AUDIO_OUTTYPE_WAV; + // note: -1 is clamped to 0 by the sampler, it will disable the penalty + task.params.sampling.penalty_last_n = task.params.n_predict > 0 ? task.params.n_predict : 512; + task.params.sampling.seed = task.tts_inp.data.seed; // same seed for the codec/vocoder RNG + if (task.tts_inp.data.top_k > 0) { + task.params.sampling.top_k = task.tts_inp.data.top_k; + } + if (task.tts_inp.data.top_p > 0) { + task.params.sampling.top_p = task.tts_inp.data.top_p; + } + if (task.tts_inp.data.temp >= 0) { + task.params.sampling.temp = task.tts_inp.data.temp; + } + + // speaker reference: either an uploaded form file ("speaker_ref") or a base64 JSON field ("speaker_ref_b64") + const unsigned char * speaker_ref_data = nullptr; + size_t speaker_ref_len = 0; + std::string speaker_ref_b64_decoded; + + auto speaker_ref_file = req.files.find("speaker_ref"); + if (speaker_ref_file != req.files.end()) { + speaker_ref_data = speaker_ref_file->second.data.data(); + speaker_ref_len = speaker_ref_file->second.data.size(); + } else { + std::string speaker_ref_b64 = json_value(body, "speaker_ref_b64", std::string()); + if (!speaker_ref_b64.empty()) { + try { + speaker_ref_b64_decoded = base64::decode(speaker_ref_b64); + } catch (const std::exception &) { + res->error(format_error_response("\"speaker_ref_b64\" is not valid base64", ERROR_TYPE_INVALID_REQUEST)); + return res; + } + speaker_ref_data = (const unsigned char *) speaker_ref_b64_decoded.data(); + speaker_ref_len = speaker_ref_b64_decoded.size(); + } + } + + if (speaker_ref_len > 0) { + if (voice_ref) { + res->error(format_error_response("\"voice\" and \"speaker_ref\" cannot be used together", ERROR_TYPE_INVALID_REQUEST)); + return res; + } + if (!mtmd_support_audio(ctx_server.mctx)) { + res->error(format_error_response("this model has no voice encoder, \"speaker_ref\" cannot be used", ERROR_TYPE_INVALID_REQUEST)); + return res; + } + auto wrapper = mtmd_helper_bitmap_init_from_buf(ctx_server.mctx, speaker_ref_data, speaker_ref_len, false, ctx_server.init_opt); + if (!wrapper.bitmap) { + res->error(format_error_response("failed to decode \"speaker_ref\"", ERROR_TYPE_INVALID_REQUEST)); + return res; + } + task.tts_inp.set_speaker_ref(mtmd::bitmap_ptr(wrapper.bitmap)); + } else if (voice_ref) { + task.tts_inp.set_speaker_ref(std::move(voice_ref)); + } else if (info.n_voices > 0) { + // without a voice prompt the model produces near silence: the first voice is the default + const char * first = mtmd_gen_audio_voice_name(ctx_server.mctx, 0); + SRV_WRN("no voice given, speaking with \"%s\"\n", first); + task.tts_inp.set_speaker_ref(mtmd::bitmap_ptr(mtmd_gen_audio_get_voice(ctx_server.mctx, first))); + } else { + SRV_WRN("%s", "no speaker reference provided, the model may behave randomly\n"); + } + + auto & rd = res->rd; + task.id = rd.get_new_id(); + rd.post_task(std::move(task)); + + // raw LE mono samples; audio/L16 would imply 16-bit big-endian (RFC 2586) + const std::string pcm_type = "audio/pcm;rate=" + std::to_string(info.sample_rate); + const std::string content_type = pcm16 ? pcm_type + ";encoding=signed-int;bits=16" + : response_format == "pcm" ? pcm_type + ";encoding=float;bits=32" + : "audio/wav"; + + // the pipeline outputs float32 samples + auto take_audio = [pcm16](server_task_result_tts * tts_res) { + const std::string & audio = tts_res->audio; + return pcm16 ? pcm_f32_to_s16((const float *) audio.data(), audio.size() / sizeof(float)) : std::move(tts_res->audio); + }; + + if (!stream) { + auto result = rd.next(req.should_stop); + if (!result) { + GGML_ASSERT(req.should_stop()); + return res; // connection is closed + } + if (result->is_error()) { + res->error(result->to_json()); + return res; + } + auto * tts_res = dynamic_cast(result.get()); + GGML_ASSERT(tts_res != nullptr); + res->status = 200; + res->content_type = content_type; + res->data = take_audio(tts_res); + return res; + } else { + auto first_result = rd.next(req.should_stop); + if (!first_result) { + GGML_ASSERT(req.should_stop()); + return res; // connection is closed + } + if (first_result->is_error()) { + res->error(first_result->to_json()); + return res; + } + auto * first_tts_res = dynamic_cast(first_result.get()); + GGML_ASSERT(first_tts_res != nullptr); + + res->status = 200; + res->content_type = content_type; + res->data = take_audio(first_tts_res); + bool is_done = first_tts_res->final; + + res->set_next([res_this = res.get(), is_done, take_audio](std::string & output) mutable -> bool { + // flush buffered audio before is_done - the first result can already be final + if (!res_this->data.empty()) { + output = std::move(res_this->data); + res_this->data.clear(); + return true; + } + if (is_done) { + return false; + } + if (res_this->should_stop()) { + return false; + } + + server_response_reader & rd = res_this->rd; + if (!rd.has_next()) { + return false; + } + auto result = rd.next([&res_this]() { return res_this->should_stop(); }); + if (!result || result->is_error()) { + return false; + } + auto * tts_res = dynamic_cast(result.get()); + GGML_ASSERT(tts_res != nullptr); + output = take_audio(tts_res); + is_done = tts_res->final; + return true; + }); + } + + return res; +} + std::unique_ptr server_routes::handle_count_tokens(const server_http_req & req, task_response_type res_type) { auto res = create_response(); std::vector files; diff --git a/tools/server/server-context.h b/tools/server/server-context.h index 05c538ea1445..4b145cb8a13e 100644 --- a/tools/server/server-context.h +++ b/tools/server/server-context.h @@ -134,6 +134,7 @@ struct server_routes { server_http_context::handler_t post_embeddings_oai; server_http_context::handler_t post_rerank; server_http_context::handler_t post_tts; + server_http_context::handler_t post_speech_oai; server_http_context::handler_t post_systemone; server_http_context::handler_t get_lora_adapters; server_http_context::handler_t post_lora_adapters; @@ -154,6 +155,7 @@ struct server_routes { std::unique_ptr handle_slots_erase(const server_http_req &, int id_slot); std::unique_ptr handle_embeddings_impl(const server_http_req & req, task_response_type res_type); std::unique_ptr handle_rerank_impl(const server_http_req & req); + std::unique_ptr handle_tts_impl(const server_http_req & req, json body); std::unique_ptr handle_count_tokens(const server_http_req & req, task_response_type res_type); // using unique_ptr to allow late initialization of const diff --git a/tools/server/server-cors-proxy.h b/tools/server/server-cors-proxy.h index 3600a33ec647..41d00740416f 100644 --- a/tools/server/server-cors-proxy.h +++ b/tools/server/server-cors-proxy.h @@ -11,6 +11,7 @@ #include #include "server-http.h" +#include "server-common.h" static std::string proxy_header_to_lower(std::string header) { std::transform(header.begin(), header.end(), header.begin(), [](unsigned char c) { @@ -68,9 +69,20 @@ static server_http_res_ptr proxy_request(const server_http_req & req, std::strin req.files, req.should_stop, 600, // timeout_read (default to 10 minutes) - 600 // timeout_write (default to 10 minutes) + 600, // timeout_write (default to 10 minutes) + // test-only escape hatch + server_resolve_public_addr(parsed_url.host, !common_get_env("LLAMA_SERVER_DEBUG_PROXY_ALLOW_PRIVATE").empty()) ); + // route redirects back through the proxy, so each hop is checked + if (proxy->status >= 300 && proxy->status < 400) { + for (auto & [key, value] : proxy->headers) { + if (proxy_header_to_lower(key) == "location") { + value = "?url=" + httplib::encode_uri_component(common_http_resolve_location(target_url, value)); + } + } + } + return proxy; } diff --git a/tools/server/server-http.cpp b/tools/server/server-http.cpp index cf147c7e70f4..5bdece55b6f3 100644 --- a/tools/server/server-http.cpp +++ b/tools/server/server-http.cpp @@ -6,6 +6,9 @@ #include +#include +#include +#include #include #include #include @@ -71,14 +74,97 @@ static void log_server_request(const httplib::Request & req, const httplib::Resp SRV_DBG("response: %s\n", res.body.c_str()); } -// returns true if the Origin header value's host is localhost / 127.0.0.1 / ::1 (any port) -static bool origin_is_localhost(const std::string & origin) { - try { - const std::string host = common_http_parse_url(origin).host; - return host == "localhost" || host == "127.0.0.1" || host == "::1"; - } catch (const std::exception &) { +static std::string ascii_lower(std::string s) { + std::transform(s.begin(), s.end(), s.begin(), [](unsigned char c) { return std::tolower(c); }); + return s; +} + +// true if the text is a dotted IPv4 address in 127.0.0.0/8 +static bool ipv4_is_loopback(const std::string & s) { + unsigned a, b, c, d; + char tail; + return std::sscanf(s.c_str(), "%u.%u.%u.%u%c", &a, &b, &c, &d, &tail) == 4 && a == 127 && b < 256 && c < 256 && d < 256; +} + +// true if the address is loopback: 127.0.0.0/8, ::1 or an IPv4 mapped 127.0.0.0/8 +static bool addr_is_loopback(const std::string & addr) { + static const std::string mapped = "::ffff:"; + if (addr.compare(0, mapped.size(), mapped) == 0) { + return ipv4_is_loopback(addr.substr(mapped.size())); + } + return addr == "::1" || ipv4_is_loopback(addr); +} + +// the name of a Host header value, lowercase, without port and IPv6 brackets +static std::string host_name(const std::string & host) { + if (!host.empty() && host[0] == '[') { + const size_t end = host.find(']'); + return end == std::string::npos ? "" : ascii_lower(host.substr(1, end - 1)); + } + return ascii_lower(host.substr(0, host.find(':'))); +} + +// DNS rebinding guard: a page whose own name resolves to 127.0.0.1 reaches a loopback +// connection with that name in Host, so Host names loopback or the host of an allowed origin +static bool request_host_is_allowed(const std::string & cors_origins, const httplib::Request & req) { + if (!addr_is_loopback(req.local_addr)) { + return true; + } + const std::string host = req.get_header_value("Host"); + const std::string name = host_name(host); + // the unspecified addresses only reach this machine, e.g. curl http://0.0.0.0:8080 + if (name == "localhost" || name == "0.0.0.0" || name == "::" || addr_is_loopback(name)) { + return true; + } + if (cors_origins == "localhost") { return false; } + for (const auto & entry : string_split(cors_origins, ',')) { + const std::string allowed = string_strip(entry); + const size_t sep = allowed.find("://"); + if (sep != std::string::npos && ascii_lower(allowed.substr(sep + 3)) == ascii_lower(host)) { + return true; + } + } + return false; +} + +// CSRF guard when CORS is not "*" +// no ACAO header only hides the response, browsers still send simple requests (text/plain POST, no-cors GET), so reject them here +static bool request_origin_is_allowed(const std::string & cors_origins, const httplib::Request & req, bool is_frontend_path) { + if (cors_origins == "*") { + return true; + } + + if (!request_host_is_allowed(cors_origins, req)) { + return false; + } + + const std::string origin = req.get_header_value("Origin"); + if (!origin.empty()) { + // same-origin, e.g. UI opened via LAN address + const std::string host = req.get_header_value("Host"); + const size_t sep = origin.find("://"); + if (!host.empty() && sep != std::string::npos && ascii_lower(origin.substr(sep + 3)) == ascii_lower(host)) { + return true; + } + if (cors_origins == "localhost") { + return false; // same-origin only, other localhost ports can be untrusted apps + } + for (const auto & allowed : string_split(cors_origins, ',')) { + if (string_strip(allowed) == origin) { + return true; + } + } + return false; + } + + // no Origin: non-browser client or browser no-cors GET; if cross-site, only allow navigation to the UI + const std::string site = req.get_header_value("Sec-Fetch-Site"); + if (site == "cross-site" || site == "same-site") { + return req.method == "GET" && req.get_header_value("Sec-Fetch-Mode") == "navigate" && is_frontend_path; + } + return true; } // For Google Cloud Platform deployment compatibility @@ -328,19 +414,31 @@ bool server_http_context::init_listener(const common_params & params) { return true; }; + // a router child has no API key; the router strips the browser headers, so only a browser sends them here + const bool managed_by_router = std::getenv(SERVER_ENV_ROUTER_PORT) != nullptr; + // register server middlewares - srv->set_pre_routing_handler([¶ms, middleware_validate_api_key, middleware_server_state](const httplib::Request & req, httplib::Response & res) { + srv->set_pre_routing_handler([¶ms, managed_by_router, middleware_validate_api_key, middleware_server_state](const httplib::Request & req, httplib::Response & res) { + if (managed_by_router && (req.has_header("Origin") || req.has_header("Sec-Fetch-Site") || req.has_header("Sec-Fetch-Mode"))) { + SRV_WRN("rejected browser request on router child: %s %s\n", req.method.c_str(), req.path.c_str()); + res.status = 403; + res.set_content( + safe_json_to_str(json { + {"error", { + {"message", "Browser requests must go through the router"}, + {"type", "permission_error"}, + {"code", 403} + }} + }), + "application/json; charset=utf-8" + ); + return httplib::Server::HandlerResponse::Handled; + } if (params.cors_credentials && params.cors_origins == "*") { // special case: echo back the Origin header to allow any origin to access the server with credentials res.set_header("Access-Control-Allow-Origin", req.get_header_value("Origin")); } else if (params.cors_origins == "localhost") { - // special case: only reflect the Origin header if it is a localhost origin - std::string origin = req.get_header_value("Origin"); - if (!origin.empty() && origin_is_localhost(origin)) { - res.set_header("Access-Control-Allow-Origin", origin); - } else if (!origin.empty()) { - SRV_WRN("(CORS) skip non-localhost origin: %s\n", origin.c_str()); - } + // special case: same-origin only, no CORS header needed } else { res.set_header("Access-Control-Allow-Origin", params.cors_origins); } @@ -352,6 +450,23 @@ bool server_http_context::init_listener(const common_params & params) { res.set_content("", "text/html"); // blank response, no data return httplib::Server::HandlerResponse::Handled; // skip further processing } + if (!request_origin_is_allowed(params.cors_origins, req, frontend_paths.count(req.path) > 0)) { + SRV_WRN("(CORS) rejected cross-origin request: %s %s, host = '%s', origin = '%s', sec-fetch-site = '%s'\n", + req.method.c_str(), req.path.c_str(), req.get_header_value("Host").c_str(), + req.get_header_value("Origin").c_str(), req.get_header_value("Sec-Fetch-Site").c_str()); + res.status = 403; + res.set_content( + safe_json_to_str(json { + {"error", { + {"message", "Cross-origin request not allowed (see --cors-origins)"}, + {"type", "permission_error"}, + {"code", 403} + }} + }), + "application/json; charset=utf-8" + ); + return httplib::Server::HandlerResponse::Handled; + } if (!middleware_server_state(req, res)) { return httplib::Server::HandlerResponse::Handled; } diff --git a/tools/server/server-mcp.cpp b/tools/server/server-mcp.cpp index 65e39179dfac..08929ba54fe1 100644 --- a/tools/server/server-mcp.cpp +++ b/tools/server/server-mcp.cpp @@ -244,14 +244,30 @@ json server_mcp_transport::send_rpc(const json & request, const std::function= deadline) { + notify_cancelled(request, "request timed out"); return {{"error", {{"code", -32603}, {"message", "request timed out"}}}}; } return {{"error", {{"code", -32603}, {"message", "transport closed"}}}}; } +// MCP cancellation: a notification that lets the server abandon work nobody waits for anymore +// (e.g. a slow tool call after the client dropped the connection). the spec forbids cancelling initialize. +void server_mcp_transport::notify_cancelled(const json & request, const std::string & reason) { + if (!request.contains("id") || request.value("method", "") == "initialize") { + return; + } + json notif = { + {"jsonrpc", "2.0"}, + {"method", "notifications/cancelled"}, + {"params", {{"requestId", request.at("id")}, {"reason", reason}}}, + }; + to_server.write(notif.dump()); +} + bool server_mcp_transport::ensure_init(const std::function & should_stop) { if (initialized) { return true; @@ -348,23 +364,22 @@ struct server_mcp_stdio::process_handle { FILE * err = nullptr; // child stderr }; -#if defined(_WIN32) -// config strings are UTF-8 (from JSON) and subprocess.h converts them with CP_UTF8, so inputs must be UTF-8, not the active code page -static std::string wide_to_utf8(const wchar_t * s, int len /* -1 for NUL-terminated */) { - return wstring_to_utf8(len == -1 ? std::wstring(s) : std::wstring(s, s + len)); -} -#endif - static std::string mcp_resolve_command(const std::string & command) { #if defined(_WIN32) - // For Windows: make sure we handle ".exe" correctly, as well as UTF-8 - std::wstring wcmd = utf8_to_wstring(command); - wchar_t buf[MAX_PATH * 4]; - const DWORD cap = (DWORD) (sizeof(buf) / sizeof(buf[0])); - - auto search = [&](const wchar_t * ext) -> std::string { - DWORD n = SearchPathW(NULL, wcmd.c_str(), ext, cap, buf, NULL); - return (n > 0 && n < cap) ? wide_to_utf8(buf, (int) n) : std::string(); + // the application manifest makes SearchPathA use UTF-8 + auto search = [&](const char * ext) -> std::string { + std::string result; + for (;;) { + DWORD n = SearchPathA(NULL, command.c_str(), ext, result.size(), result.data(), NULL); + if (n == 0) { + return {}; + } + if (n < result.size()) { + result.resize(n); + return result; + } + result.resize((size_t) n + 1); + } }; std::string found = search(NULL); // exact path / already-extensioned / .exe on PATH @@ -372,26 +387,20 @@ static std::string mcp_resolve_command(const std::string & command) { return found; } - std::wstring pathext; - DWORD need = GetEnvironmentVariableW(L"PATHEXT", NULL, 0); - if (need > 0) { - pathext.resize(need); - DWORD got = GetEnvironmentVariableW(L"PATHEXT", &pathext[0], need); - pathext.resize(got); - } + std::string pathext = common_get_env("PATHEXT"); if (pathext.empty()) { - pathext = L".COM;.EXE;.BAT;.CMD"; + pathext = ".COM;.EXE;.BAT;.CMD"; } for (size_t start = 0; start <= pathext.size();) { - size_t sep = pathext.find(L';', start); - std::wstring ext = pathext.substr(start, sep == std::wstring::npos ? std::wstring::npos : sep - start); + size_t sep = pathext.find(';', start); + std::string ext = pathext.substr(start, sep == std::string::npos ? std::string::npos : sep - start); if (!ext.empty()) { found = search(ext.c_str()); if (!found.empty()) { return found; } } - if (sep == std::wstring::npos) { + if (sep == std::string::npos) { break; } start = sep + 1; @@ -408,7 +417,7 @@ static std::vector mcp_parent_env() { LPWCH block = GetEnvironmentStringsW(); if (block) { for (LPWCH e = block; *e; e += wcslen(e) + 1) { - env.emplace_back(wide_to_utf8(e, -1)); + env.emplace_back(wstring_to_utf8(e)); } FreeEnvironmentStringsW(block); } @@ -689,7 +698,7 @@ void server_mcp::start(const common_params & params) { } }; if (!params.mcp_servers_config.empty()) { - std::ifstream f(std::filesystem::u8path(params.mcp_servers_config), std::ios::in); + std::ifstream f(params.mcp_servers_config, std::ios::in); if (!f) { throw std::runtime_error("failed to open MCP config file: " + params.mcp_servers_config); } diff --git a/tools/server/server-mcp.h b/tools/server/server-mcp.h index c7f33a3f797d..2b4eb89201f4 100644 --- a/tools/server/server-mcp.h +++ b/tools/server/server-mcp.h @@ -82,9 +82,10 @@ struct server_mcp_transport { std::string last_error; std::vector tools; - // both assume rpc_mutex is already held by the public caller + // all assume rpc_mutex is already held by the public caller bool ensure_init(const std::function & should_stop); // initialize handshake, once json send_rpc(const json & request, const std::function & should_stop); // returns the reply or an {"error": ...} + void notify_cancelled(const json & request, const std::string & reason); // notifications/cancelled for an abandoned request }; // diff --git a/tools/server/server-models.cpp b/tools/server/server-models.cpp index a18488715328..5b7d34501c23 100644 --- a/tools/server/server-models.cpp +++ b/tools/server/server-models.cpp @@ -265,6 +265,23 @@ struct server_monitor { // delete). distinct from params.timeout_read/write which only applies to the generation proxy static constexpr int STREAM_LOOKUP_TIMEOUT_MS = 250; +// router already checked the origin; child sees a loopback Host, so do not forward origin headers +static bool is_browser_origin_header(const std::string & key) { + std::string lower = key; + std::transform(lower.begin(), lower.end(), lower.begin(), ::tolower); + return lower == "origin" || lower.compare(0, 10, "sec-fetch-") == 0; +} + +static std::map child_request_headers(const std::map & headers) { + std::map out; + for (const auto & [key, value] : headers) { + if (!is_browser_origin_header(key)) { + out.emplace(key, value); + } + } + return out; +} + static void unset_reserved_args(common_preset & preset, bool unset_model_args) { preset.unset_option("LLAMA_ARG_SSL_KEY_FILE"); preset.unset_option("LLAMA_ARG_SSL_CERT_FILE"); @@ -279,6 +296,7 @@ static void unset_reserved_args(common_preset & preset, bool unset_model_args) { preset.unset_option("LLAMA_ARG_CONNECT_CODE"); if (unset_model_args) { preset.unset_option("LLAMA_ARG_MODEL"); + preset.unset_option("LLAMA_ARG_SHARD"); preset.unset_option("LLAMA_ARG_MMPROJ"); preset.unset_option("LLAMA_ARG_ALIAS"); preset.unset_option("LLAMA_ARG_HF_REPO"); @@ -597,7 +615,7 @@ void server_unfinished_downloads::save() { data["downloads"].push_back({ {"model", d.model}, {"sidecars", d.sidecars}, {"partials", partials} }); } try { - fs_write_atomic(std::filesystem::u8path(unfinished_downloads_path()), data.dump()); + fs_write_atomic(unfinished_downloads_path(), data.dump()); } catch (const std::exception & e) { LOG_WRN("failed to record unfinished downloads: %s\n", e.what()); } @@ -614,7 +632,7 @@ void server_unfinished_downloads::delete_partials(const std::vectorsecond.meta.status == SERVER_MODEL_STATUS_DOWNLOADING) { SRV_INF("cancelling download for model name=%s\n", name.c_str()); + it->second.download_stop_requested = true; it->second.request_exit(); // for convenience, we wait the status change here wait(lk, name, [](const server_model_meta & new_meta) { @@ -1370,6 +1391,7 @@ void server_models::unload_all() { for (auto & [name, inst] : mapping) { if (inst.meta.status == SERVER_MODEL_STATUS_DOWNLOADING) { SRV_INF("cancelling download for model name=%s\n", name.c_str()); + inst.download_stop_requested = true; inst.request_exit(); } else if (inst.meta.is_running()) { SRV_INF("stopping model instance name=%s\n", name.c_str()); @@ -1465,6 +1487,7 @@ void server_models::update_status(const std::string & name, const update_status_ void server_models::update_download_progress(const std::string & name, const common_download_progress & progress, bool done, bool ok) { json curr; + bool stopped = false; { std::lock_guard lk(mutex); auto it = mapping.find(name); @@ -1473,6 +1496,7 @@ void server_models::update_download_progress(const std::string & name, const com // mark the instance to be erased on next load_models() call it->second.meta.status = SERVER_MODEL_STATUS_DOWNLOADED; need_reload = true; + stopped = it->second.download_stop_requested; // a download that failed keeps its tag: its partial files are still // there and re-posting the tag resumes them @@ -1494,7 +1518,13 @@ void server_models::update_download_progress(const std::string & name, const com } if (done) { cv.notify_all(); // notify in case unload() is waiting for download to be cancelled - notify_sse(ok ? "download_finished" : "download_failed", name, {}); + // a stop a client asked for (pause or cancel) is no error: say so, so that every + // client, not only the one that asked, can tell it from a failed download + if (ok) { + notify_sse("download_finished", name, {}); + } else { + notify_sse("download_failed", name, stopped ? json{{"stopped", true}} : json{}); + } } else { notify_sse("download_progress", name, curr); } @@ -1517,6 +1547,7 @@ bool server_models::remove(const std::string & name) { if (it->second.meta.status == SERVER_MODEL_STATUS_DOWNLOADING) { // cancel in-flight download SRV_INF("cancelling download for model name=%s\n", name.c_str()); + it->second.download_stop_requested = true; it->second.request_exit(); } else if (it->second.meta.is_running()) { // stop running instance @@ -1702,7 +1733,7 @@ server_http_res_ptr server_models::proxy_request(const server_http_req & req, co // sleep after req_count++, so the model counts as busy while we wait here std::this_thread::sleep_for(std::chrono::seconds(2)); } - SRV_INF("proxying request to model %s on port %d\n", name.c_str(), meta->port); + SRV_TRC("proxying request to model %s on port %d\n", name.c_str(), meta->port); std::string proxy_path = req.path; if (!req.query_string.empty()) { proxy_path += '?' + req.query_string; @@ -1713,7 +1744,7 @@ server_http_res_ptr server_models::proxy_request(const server_http_req & req, co CHILD_ADDR, meta->port, proxy_path, - req.headers, + child_request_headers(req.headers), req.body, req.files, // a detached request belongs to a replay session @@ -1757,7 +1788,7 @@ void server_models::proxy_ws(const server_http_req & req, server_http_context::w mapping[name].meta.last_used = ggml_time_ms(); mapping[name].req_count++; } - SRV_INF("proxying websocket to model %s on port %d\n", name.c_str(), meta->port); + SRV_TRC("proxying websocket to model %s on port %d\n", name.c_str(), meta->port); std::string url = std::string("ws://") + CHILD_ADDR + ":" + std::to_string(meta->port) + req.path; if (!req.query_string.empty()) { @@ -1768,7 +1799,8 @@ void server_models::proxy_ws(const server_http_req & req, server_http_context::w for (const auto & [key, value] : req.headers) { std::string lower = key; std::transform(lower.begin(), lower.end(), lower.begin(), ::tolower); - if (lower == "host" || lower == "connection" || lower == "upgrade" || lower.compare(0, 14, "sec-websocket-") == 0) { + if (lower == "host" || lower == "connection" || lower == "upgrade" || lower.compare(0, 14, "sec-websocket-") == 0 + || is_browser_origin_header(key)) { continue; } headers.emplace(key, value); @@ -2409,7 +2441,7 @@ void server_models_routes::init_routes() { CHILD_ADDR, owner->port, child_path, - req.headers, + child_request_headers(req.headers), req.body, req.files, req.should_stop, @@ -2624,7 +2656,8 @@ server_http_proxy::server_http_proxy( const std::map & files, const std::function should_stop, int32_t timeout_read, - int32_t timeout_write + int32_t timeout_write, + const std::string & resolved_addr ) { // shared between reader and writer threads auto cli = std::make_shared(host, port); @@ -2639,7 +2672,11 @@ server_http_proxy::server_http_proxy( } // setup Client - cli->set_follow_location(true); + if (!resolved_addr.empty()) { + // pin the checked address; a redirect could lead to an unchecked one + cli->set_direct_hostname_addr_map({{host, resolved_addr}}); + } + cli->set_follow_location(resolved_addr.empty()); cli->set_connection_timeout(timeout_read, 0); // use --timeout value instead of hardcoded 5 s cli->set_write_timeout(timeout_read, 0); // reversed for cli (client) vs srv (server) cli->set_read_timeout(timeout_write, 0); diff --git a/tools/server/server-models.h b/tools/server/server-models.h index 0db948773c6e..e18ae3a31f43 100644 --- a/tools/server/server-models.h +++ b/tools/server/server-models.h @@ -170,6 +170,7 @@ struct server_models { std::shared_ptr subproc; // shared with the monitor thread server_model_meta meta; int req_count = 0; // number of active proxy requests + bool download_stop_requested = false; // a client asked the download to stop (pause or cancel) // ask the child to exit (it handles the command on its stdin, see server_role::setup) void request_exit() const; @@ -443,7 +444,8 @@ struct server_http_proxy : server_http_res { const std::map & files, const std::function should_stop, int32_t timeout_read, - int32_t timeout_write + int32_t timeout_write, + const std::string & resolved_addr = "" // if set: connect to this address only, do not follow redirects ); ~server_http_proxy() { if (cleanup_pipes) { diff --git a/tools/server/server-queue.cpp b/tools/server/server-queue.cpp index c4ce90721b94..75daebb0d299 100644 --- a/tools/server/server-queue.cpp +++ b/tools/server/server-queue.cpp @@ -114,9 +114,7 @@ void server_queue::pop_deferred_task(int id_slot) { void server_queue::wait_until_no_sleep() { std::unique_lock lock(mutex_tasks); - if (!sleeping) { - return; - } else { + if (sleeping) { if (!req_stop_sleeping) { QUE_DBG("%s", "requesting to stop sleeping\n"); req_stop_sleeping = true; @@ -127,6 +125,13 @@ void server_queue::wait_until_no_sleep() { return !sleeping; }); } + n_inflight++; +} + +void server_queue::release_inflight() { + std::unique_lock lock(mutex_tasks); + GGML_ASSERT(n_inflight > 0); + n_inflight--; } void server_queue::terminate() { @@ -289,7 +294,7 @@ void server_queue::start_loop(int64_t idle_sleep_ms) { constexpr auto max_wait_time = std::chrono::seconds(1); auto should_sleep = [&]() -> bool { // caller must hold mutex_tasks - if (idle_sleep_ms < 0) { + if (idle_sleep_ms < 0 || n_inflight > 0) { return false; } int64_t now = ggml_time_ms(); diff --git a/tools/server/server-queue.h b/tools/server/server-queue.h index 692255e4f6e7..f9d66be64064 100644 --- a/tools/server/server-queue.h +++ b/tools/server/server-queue.h @@ -19,6 +19,7 @@ struct server_queue { bool sleeping = false; bool req_stop_sleeping = false; int64_t time_last_task = 0; + int n_inflight = 0; // requests that keep the server awake, see wait_until_no_sleep() // queues std::deque queue_tasks; @@ -67,8 +68,12 @@ struct server_queue { // if sleeping, request exiting sleep state and wait until it is done // returns immediately if not sleeping + // the server then stays awake until the caller calls release_inflight() void wait_until_no_sleep(); + // let the server sleep again once the caller no longer uses the server state + void release_inflight(); + bool is_sleeping() { std::unique_lock lock(mutex_tasks); return sleeping; diff --git a/tools/server/server-realtime.cpp b/tools/server/server-realtime.cpp index 76faaba3cfc3..8ed3dd535f11 100644 --- a/tools/server/server-realtime.cpp +++ b/tools/server/server-realtime.cpp @@ -7,6 +7,7 @@ #include "base64.hpp" #include "chat.h" #include "common.h" +#include "json-schema-to-grammar.h" #include "mtmd.h" #include "mtmd-helper.h" @@ -75,7 +76,12 @@ bool parse_max_output_tokens(const json & j, int32_t & n) { // realtime tools have the format of the Responses API bool parse_tools(const json & j, std::vector & tools, std::string & err) { try { - tools = common_chat_tools_parse_oaicompat(server_chat_convert_responses_tools(j)); + auto parsed = common_chat_tools_parse_oaicompat(server_chat_convert_responses_tools(j)); + // reject the schemas that would fail later, when the chat template builds the tool call grammar + for (const auto & tool : parsed) { + json_schema_to_grammar(json::parse(tool.parameters)); + } + tools = std::move(parsed); } catch (const std::exception & e) { err = std::string("tools: ") + e.what(); return false; @@ -124,6 +130,8 @@ struct realtime_conversation { std::string type = "realtime"; std::string instructions; std::string language; // transcription + std::string out_language; + std::string out_voice; mtmd_helper_gen_audio_out_mode out_mode = MTMD_HELPER_GEN_AUDIO_OUT_AUTO; int32_t max_output_tokens = -1; // "inf" int input_rate = 16000; @@ -150,6 +158,7 @@ struct realtime_conversation { bool tool_results = false; // tool results went in after the last response, the next one answers them int n_turns = 0; std::string next_header; // text that goes in before the next user turn + mtmd_helper::resampler_ptr resampler; // input_rate -> model rate // the item or response the results belong to, shared with the result loop std::mutex mutex; @@ -169,7 +178,8 @@ struct realtime_conversation { std::atomic closed{false}; - realtime_conversation(const server_realtime_deps & deps, server_http_context::ws_conn & conn) : deps(deps), conn(conn) {} + realtime_conversation(const server_realtime_deps & deps, server_http_context::ws_conn & conn) + : deps(deps), conn(conn), out_language(deps.params.tts_lang), out_voice(deps.params.tts_voice) {} // // events @@ -200,7 +210,11 @@ struct realtime_conversation { } json audio = {{"input", audio_in}}; if (deps.cap.audio_gen) { - audio["output"] = {{"format", {{"type", "audio/pcm"}, {"rate", output_rate}}}}; + audio["output"] = { + {"format", {{"type", "audio/pcm"}, {"rate", output_rate}}}, + {"voice", out_voice.empty() ? json(nullptr) : json(out_voice)}, + {"language", out_language.empty() ? json(nullptr) : json(out_language)}, + }; } return { {"id", id}, @@ -238,7 +252,7 @@ struct realtime_conversation { // sampling of a response: the server defaults, with the tool call grammar of chat common_params_sampling response_sampling(const common_chat_session * chat) const { - common_params_sampling sampling = deps.sampling; + common_params_sampling sampling = deps.params.sampling; if (chat) { chat->apply_sampling(sampling); } @@ -356,7 +370,10 @@ struct realtime_conversation { memcpy(&v, pcm.data() + 2 * i, sizeof(v)); samples[i] = v / 32768.0f; } - if (samples.empty() || !ensure_turn()) { + if (samples.empty()) { + return; + } + if (!resample(samples) || samples.empty() || !ensure_turn()) { return; } if (control(SERVER_CONTROL_ACTION_AUDIO_APPEND, {}, std::move(samples))) { @@ -364,6 +381,26 @@ struct realtime_conversation { } } + bool resample(std::vector & samples) { + if (!resampler) { + resampler.reset(mtmd_helper_resampler_init(input_rate, mtmd_get_audio_sample_rate(deps.mctx))); + if (!resampler) { + send_error("failed to initialize the audio resampler", "server_error"); + return false; + } + } + const float * out = nullptr; + const int64_t n = mtmd_helper_resampler_process(resampler.get(), samples.data(), samples.size(), &out); + if (n < 0) { + send_error("failed to resample the audio", "server_error"); + return false; + } + if (out != samples.data()) { + samples.assign(out, out + n); + } + return true; + } + // ends the audio of the open turn, it becomes a user item bool commit() { if (!has_audio) { @@ -514,6 +551,8 @@ struct realtime_conversation { return; } response.out_mode = mode; + response.lang = out_language; + response.voice = out_voice; // with tools, the output goes through the tool call parser of the chat template and its grammar std::optional chat; @@ -563,6 +602,10 @@ struct realtime_conversation { void on_session_update(const json & ev) { const json session = ev.value("session", json::object()); + if (!session.is_object()) { + send_error("session.update: session must be an object"); + return; + } if (session.contains("type") && session["type"].is_string() && session["type"] != type) { const std::string t = session["type"]; if (t != "realtime" && t != "transcription") { @@ -613,12 +656,41 @@ struct realtime_conversation { } const json audio = session.value("audio", json::object()); const json in = audio.value("input", json::object()); - if (in.contains("format") && in["format"].is_object() && json_value(in["format"], "rate", input_rate) != input_rate) { - send_error("session.update: the input audio must be PCM16 at " + std::to_string(input_rate) + " Hz"); + if (in.contains("format") && in["format"].is_object()) { + const int rate = json_value(in["format"], "rate", input_rate); + if (rate < 8000 || rate > 192000) { + send_error("session.update: the input audio rate must be between 8000 and 192000 Hz"); + return; + } + if (rate != input_rate) { + input_rate = rate; + resampler.reset(); + } } if (in.contains("transcription") && in["transcription"].is_object()) { language = json_value(in["transcription"], "language", language); } + const json out = audio.value("output", json::object()); + if (out.contains("voice") || out.contains("language")) { + std::string voice = out_voice; + if (out.contains("voice")) { + // string or {"id": name} + const json & v = out["voice"]; + voice = v.is_object() ? json_value(v, "id", std::string()) : v.is_string() ? v.get() : std::string(); + } + const std::string lang = json_value(out, "language", out_language); + const auto & voices = deps.meta.tts_voices; + if (!voice.empty() && !voices.empty() && std::find(voices.begin(), voices.end(), voice) == voices.end()) { + send_error("session.update: unknown voice \"" + voice + "\", see \"voices\" in GET /props"); + return; + } + if (started && (voice != out_voice || lang != out_language)) { + send_error("session.update: the output voice and language cannot change after the first audio or message"); + return; + } + out_voice = voice; + out_language = lang; + } if (!started && !configure()) { send_error("the chat template does not fit the realtime turn format", "server_error"); return; @@ -632,6 +704,11 @@ struct realtime_conversation { if (type == "transcription") { const auto preset = common_chat_get_asr_prompt(deps.meta.chat_params.tmpls.get(), language); instructions = preset.system; + } else if (deps.cap.audio_gen) { + const std::string speech = common_chat_get_speech_system_prompt(deps.meta.chat_params.tmpls.get(), out_language, out_voice); + if (!speech.empty()) { + instructions = instructions.empty() ? speech : speech + "\n" + instructions; + } } const bool ok = derive_turn_text(); instructions = saved; @@ -672,12 +749,7 @@ struct realtime_conversation { // static std::string encode_pcm16(const std::vector & samples) { - std::string bytes(samples.size() * sizeof(int16_t), '\0'); - for (size_t i = 0; i < samples.size(); i++) { - const int16_t v = (int16_t) std::lround(std::max(-1.0f, std::min(1.0f, samples[i])) * 32767.0f); - memcpy(&bytes[2 * i], &v, sizeof(v)); - } - return base64::encode(bytes); + return base64::encode(pcm_f32_to_s16(samples.data(), samples.size())); } // the message item of the response is added with its first delta @@ -735,10 +807,10 @@ struct realtime_conversation { void on_result(const server_task_result_realtime & res) { std::lock_guard lock(mutex); if (!transcribe_item_id.empty()) { - if (!res.text.empty()) { - transcript += res.text; + if (!res.content.empty()) { + transcript += res.content.text; send_event({{"type", "conversation.item.input_audio_transcription.delta"}, {"item_id", transcribe_item_id}, - {"content_index", 0}, {"delta", res.text}}); + {"content_index", 0}, {"delta", res.content.text}}); } if (res.response_done) { send_event({{"type", "conversation.item.input_audio_transcription.completed"}, {"item_id", transcribe_item_id}, @@ -751,12 +823,12 @@ struct realtime_conversation { // with tools, the text goes through the parser: content and tool calls std::vector diffs; if (parse) { - if (!res.text.empty() || res.response_done) { - parse->update_chat_msg(common_chat_input(res.text), !res.response_done, diffs, true); + if (!res.content.empty() || res.response_done) { + parse->update_chat_msg(res.content, !res.response_done, diffs, true); } - } else if (!res.text.empty()) { + } else if (!res.content.empty()) { diffs.emplace_back(); - diffs.back().content_delta = res.text; + diffs.back().content_delta = res.content.text; } for (const auto & diff : diffs) { if (!diff.content_delta.empty()) { @@ -832,7 +904,7 @@ struct realtime_conversation { { server_task task(SERVER_TASK_TYPE_REALTIME); task.id = rd.get_new_id(); - task.params.sampling = deps.sampling; + task.params.sampling = deps.params.sampling; task.params.stream = true; task.params.oaicompat_cmpl_id = id; task.params.oaicompat_model = deps.meta.model_name; @@ -855,7 +927,11 @@ struct realtime_conversation { std::thread reader([this]() { std::string msg; while (conn.read(msg) != 0) { - handle(msg); + try { + handle(msg); + } catch (const std::exception & e) { + send_error(e.what(), "server_error"); + } } closed = true; conn.close(); @@ -871,7 +947,12 @@ struct realtime_conversation { break; } if (auto * res = dynamic_cast(result.get())) { - on_result(*res); + try { + on_result(*res); + } catch (const std::exception & e) { + send_error(e.what(), "server_error"); + break; + } } } @@ -919,11 +1000,11 @@ bool server_realtime_slot::fail(const std::string & msg, enum error_type type) { return false; } -void server_realtime_slot::send_result(std::string text, std::vector audio, bool response_done) { +void server_realtime_slot::send_result(common_chat_input content, std::vector audio, bool response_done) { auto res = std::make_unique(); res->id = id_task; res->index = index; - res->text = std::move(text); + res->content = std::move(content); res->audio = std::move(audio); res->response_done = response_done; res->cancelled = response_done && cancelled; @@ -1006,6 +1087,15 @@ bool server_realtime_slot::response_start(const server_realtime_op & op) { inp.data.stream = true; inp.data.out_type = MTMD_HELPER_GEN_AUDIO_OUTTYPE_PCM; inp.data.out_mode = op.response.out_mode; + inp.set_lang(op.response.lang); + // predefined voice -> speaker ref, LFM3 uses the system prompt instead + if (!op.response.voice.empty() && mtmd_gen_audio_get_info(mctx).n_voices > 0) { + mtmd::bitmap_ptr voice(mtmd_gen_audio_get_voice(mctx, op.response.voice.c_str())); + if (!voice) { + return fail("unknown voice \"" + op.response.voice + "\"", ERROR_TYPE_INVALID_REQUEST); + } + inp.set_speaker_ref(std::move(voice)); + } if (gen.set_input(inp.get()) != 0) { return fail("failed to start the response"); } @@ -1046,8 +1136,8 @@ std::string server_realtime_slot::piece(llama_token tok) const { // one text token or audio frame of the response being generated bool server_realtime_slot::response_step() { - bool stop = false; - std::string text; + bool stop = false; + common_chat_input content; std::vector audio; if (gen.valid()) { @@ -1056,7 +1146,7 @@ bool server_realtime_slot::response_step() { return fail("response generation failed"); } if (sampled != LLAMA_TOKEN_NULL && !stop) { - text = piece(sampled); + content.append(piece(sampled), sampled); } if (h_next) { h_state.assign(h_next, h_next + llama_model_n_embd(model)); @@ -1064,7 +1154,7 @@ bool server_realtime_slot::response_step() { } else { stop = llama_vocab_is_eog(vocab, sampled); if (!stop) { - text = piece(sampled); + content.append(piece(sampled), sampled); if (!decode(&sampled, nullptr, 1, true)) { return false; } @@ -1091,8 +1181,8 @@ bool server_realtime_slot::response_step() { audio.assign((const float *) data, (const float *) data + data_len / sizeof(float)); } } - if (!text.empty() || !audio.empty()) { - send_result(std::move(text), std::move(audio)); + if (!content.empty() || !audio.empty()) { + send_result(std::move(content), std::move(audio)); } if (done) { if (gen.valid()) { @@ -1107,7 +1197,7 @@ bool server_realtime_slot::response_step() { } bool server_realtime_slot::update() { - llama_set_embeddings(ctx, true); + llama_set_embeddings(ctx, gen.valid()); if (responding) { return response_step(); diff --git a/tools/server/server-realtime.h b/tools/server/server-realtime.h index dd871261bb1d..93c864df22d8 100644 --- a/tools/server/server-realtime.h +++ b/tools/server/server-realtime.h @@ -15,7 +15,7 @@ #include struct server_context_meta; -struct common_params_sampling; +struct common_params; struct mtmd_context; struct mtmd_helper_cap; struct llama_vocab; @@ -24,7 +24,7 @@ struct server_realtime_deps { server_queue & queue_tasks; server_response & queue_results; const server_context_meta & meta; - const common_params_sampling & sampling; + const common_params & params; mtmd_context * mctx; const mtmd_helper_cap & cap; const llama_vocab * vocab; @@ -87,7 +87,7 @@ struct server_realtime_slot { std::set preserved; // special tokens of the response that go out as text (tool calls) bool fail(const std::string & msg, enum error_type type = ERROR_TYPE_SERVER); - void send_result(std::string text, std::vector audio, bool response_done = false); + void send_result(common_chat_input content, std::vector audio, bool response_done = false); bool decode(const llama_token * tokens, const float * rows, int32_t n, bool logits_last); bool audio_rows(); void sample_next(); diff --git a/tools/server/server-schema.cpp b/tools/server/server-schema.cpp index 0a14a9620961..76d634d589de 100644 --- a/tools/server/server-schema.cpp +++ b/tools/server/server-schema.cpp @@ -542,11 +542,15 @@ std::vector> make_tts_schema(server_task & task) { add((new field_num("top_k", task.tts_inp.data.top_k)) ->set_limits(0, INT32_MAX) - ->set_desc("Top-k for the acoustic code predictor, 0 to use the model default")); + ->set_desc("Top-k for the backbone sampler and the acoustic code predictor, 0 to use the defaults")); add((new field_num("top_p", task.tts_inp.data.top_p)) ->set_limits(0.0f, 1.0f) - ->set_desc("Top-p for the acoustic code predictor, 0.0 to use the model default")); + ->set_desc("Top-p for the backbone sampler and the acoustic code predictor, 0.0 to use the defaults")); + + add((new field_num("temperature", task.tts_inp.data.temp)) + ->set_limits(0.0f, std::numeric_limits::infinity()) + ->set_desc("Temperature for the backbone sampler and the audio frames, for models that support it, 0.0 = greedy")); add((new field_num("seed", task.tts_inp.data.seed)) ->set_desc("RNG seed for the backbone sampler and the codec/vocoder (-1 = random)")); @@ -573,6 +577,7 @@ void eval_tts_schema(const common_params & params_base, server_task & task, cons task.params.n_predict = -1; task.tts_inp.data.top_k = 0; task.tts_inp.data.top_p = 0.0f; + task.tts_inp.data.temp = -1.0f; // not given: --temp for the backbone, model default for the audio task.tts_inp.data.seed = params_base.sampling.seed; field_eval_context ctx(task.params); diff --git a/tools/server/server-task.cpp b/tools/server/server-task.cpp index 1a0d434a1f31..be3bc539a9cd 100644 --- a/tools/server/server-task.cpp +++ b/tools/server/server-task.cpp @@ -1513,7 +1513,7 @@ json server_task_result_rerank::to_json() { // json server_task_result_realtime::to_json() { return json { - {"text", text}, + {"text", content.text}, {"n_samples", audio.size()}, {"response_done", response_done}, {"cancelled", cancelled}, @@ -1767,7 +1767,7 @@ server_prompt_cache_state * server_prompt_cache::alloc(const server_prompt & pro // calculate checkpoints size to see if it will fit with the prompt size_t checkpoints_size = 0; for (const auto & ckpt : prompt.checkpoints) { - checkpoints_size += ckpt.size(); + checkpoints_size += ckpt->size(); } const size_t state_size_new = state_size_tgt + state_size_dft + checkpoints_size; @@ -1802,8 +1802,8 @@ server_prompt_cache_state * server_prompt_cache::alloc(const server_prompt & pro } } - std::vector state_data_tgt; - std::vector state_data_dft; + common_state_data state_data_tgt; + common_state_data state_data_dft; // check if we can allocate enough memory for the new state try { @@ -1836,7 +1836,7 @@ server_prompt_cache_state * server_prompt_cache::alloc(const server_prompt & pro } bool server_prompt_cache::load(server_prompt & prompt, const server_tokens & tokens_new, llama_context * ctx_tgt, llama_context * ctx_dft, int32_t id_slot) { - const int lcp_best = prompt.tokens.get_common_prefix(tokens_new); + int lcp_best = prompt.tokens.get_common_prefix(tokens_new); float f_keep_best = prompt.tokens.size() > 0 ? float(lcp_best) / prompt.tokens.size() : -1.0f; // empty slot: any cache entry wins float f_sim_best = float(lcp_best) / tokens_new.size(); @@ -1862,6 +1862,7 @@ bool server_prompt_cache::load(server_prompt & prompt, const server_tokens & tok if (f_keep_best < f_keep_cur && f_sim_best < f_sim_cur) { f_keep_best = f_keep_cur; f_sim_best = f_sim_cur; + lcp_best = lcp_cur; it_best = it; } @@ -1904,6 +1905,13 @@ bool server_prompt_cache::load(server_prompt & prompt, const server_tokens & tok } } + SRV_DBG("%s", "__TEST_TAG_PROMPT_CACHE_RESTORE__\n"); + + // the whole cached prompt is a prefix of the new one + if (lcp_best == (int) it_best->prompt.tokens.size()) { + SRV_DBG("%s", "__TEST_TAG_PROMPT_CACHE_FULL_RESTORE__\n"); + } + prompt = std::move(it_best->prompt); states.erase(it_best); diff --git a/tools/server/server-task.h b/tools/server/server-task.h index 21b649ac429a..6d550d104dc0 100644 --- a/tools/server/server-task.h +++ b/tools/server/server-task.h @@ -70,6 +70,8 @@ struct server_control_response { mtmd_helper_gen_audio_out_mode out_mode = MTMD_HELPER_GEN_AUDIO_OUT_AUTO; common_params_sampling sampling; // of the text tokens: tool call grammar and preserved tokens int32_t n_predict = -1; // text tokens and audio frames, -1: until the context is full + std::string lang; // empty: model default + std::string voice; // empty: model default }; struct task_params { @@ -555,7 +557,7 @@ struct server_task_result_embd : server_task_result { // one event of a realtime conversation: text and audio of the response being generated, or its end struct server_task_result_realtime : server_task_result { - std::string text; + common_chat_input content; // the text with its token std::vector audio; // PCM F32 at the sample rate of mtmd_gen_audio_get_info() bool response_done = false; bool cancelled = false; @@ -686,7 +688,8 @@ struct server_task_result_apply_lora : server_task_result { struct server_prompt { server_tokens tokens; - std::list checkpoints; + // a checkpoint does not change after it is created, so the prompt cache shares it with the slot instead of a copy + std::list> checkpoints; void clear() { tokens.clear(); @@ -706,8 +709,8 @@ struct server_prompt { }; struct server_prompt_data { - std::vector main; - std::vector drft; + common_state_data main; + common_state_data drft; size_t size() const { return main.size() + drft.size(); @@ -722,7 +725,7 @@ struct server_prompt_cache_state { size_t res = data.size(); for (const auto & ckpt : prompt.checkpoints) { - res += ckpt.size(); + res += ckpt->size(); } return res; diff --git a/tools/server/server-tools.cpp b/tools/server/server-tools.cpp index ad6dd1d781ab..c92f1e9f88bb 100644 --- a/tools/server/server-tools.cpp +++ b/tools/server/server-tools.cpp @@ -112,29 +112,14 @@ enum class list_kind { all, // both }; -// a narrow path uses the active code page on Windows, so every crossing between -// a std::string (always UTF-8 here) and fs::path is converted explicitly -static fs::path path_from_utf8(const std::string & s) { - return fs::u8path(s); -} - -// '/' separators on every platform: Windows accepts them, the web UI needs them -static std::string path_to_utf8(const fs::path & p) { - const auto s = p.generic_u8string(); - return std::string(s.begin(), s.end()); -} - // home directory, read once at first use (getenv is not thread safe against setenv) static const std::string & home_dir() { static const std::string home = [] { #ifdef _WIN32 - // the narrow getenv would return the profile path in the active code page - const wchar_t * w = _wgetenv(L"HOME"); - if (w == nullptr) w = _wgetenv(L"USERPROFILE"); - return w ? path_to_utf8(fs::path(w)) : std::string(); + const std::string h = common_get_env("HOME"); + return h.empty() ? common_get_env("USERPROFILE") : h; #else - const char * h = getenv("HOME"); - return h ? std::string(h) : std::string(); + return common_get_env("HOME"); #endif }(); return home; @@ -321,7 +306,7 @@ class tools_io_basic : public tools_io { std::string resolve(const std::string & path) const override { const std::string p = expand_home(path); - fs::path full = path_from_utf8(p); + fs::path full = p; if (!full.is_absolute()) { if (cwd.empty()) { std::error_code ec; @@ -329,7 +314,7 @@ class tools_io_basic : public tools_io { if (ec) return p; full = cur / full; } else { - full = path_from_utf8(cwd) / full; + full = fs::path(cwd) / full; } } @@ -339,27 +324,27 @@ class tools_io_basic : public tools_io { if (!full.has_filename() && full != full.root_path()) { full = full.parent_path(); } - return path_to_utf8(full); + return full.generic_string(); } bool is_directory(const std::string & path) const override { std::error_code ec; - return fs::is_directory(path_from_utf8(resolve(path)), ec) && !ec; + return fs::is_directory(resolve(path), ec) && !ec; } bool is_regular_file(const std::string & path) const override { std::error_code ec; - return fs::is_regular_file(path_from_utf8(resolve(path)), ec) && !ec; + return fs::is_regular_file(resolve(path), ec) && !ec; } bool file_size(const std::string & path, uintmax_t & out_size) const override { std::error_code ec; - out_size = fs::file_size(path_from_utf8(resolve(path)), ec); + out_size = fs::file_size(resolve(path), ec); return !ec; } bool read_file(const std::string & path, std::string & out) const override { - std::ifstream f(path_from_utf8(resolve(path)), std::ios::binary); + std::ifstream f(resolve(path), std::ios::binary); if (!f) return false; std::ostringstream ss; ss << f.rdbuf(); @@ -369,7 +354,7 @@ class tools_io_basic : public tools_io { bool write_file(const std::string & path, const std::string & content) const override { std::error_code ec; - fs::path fpath = path_from_utf8(resolve(path)); + fs::path fpath = resolve(path); if (fpath.has_parent_path()) { fs::create_directories(fpath.parent_path(), ec); if (ec) return false; @@ -405,7 +390,7 @@ class tools_io_basic : public tools_io { if (line.empty()) continue; std::replace(line.begin(), line.end(), '\\', '/'); if (max_depth > 0 && entry_depth(line) > max_depth) continue; - if (is_regular_file(path_to_utf8(path_from_utf8(base) / path_from_utf8(line)))) { + if (is_regular_file((fs::path(base) / line).generic_string())) { out.entries.push_back({line, false}); } } @@ -475,7 +460,7 @@ class tools_io_basic : public tools_io { std::vector result; std::vector> stack; - stack.emplace_back(path_from_utf8(base), fs::path(), 0); + stack.emplace_back(base, fs::path(), 0); while (!stack.empty()) { if (std::chrono::steady_clock::now() >= deadline) { @@ -510,16 +495,16 @@ class tools_io_basic : public tools_io { if (tec) continue; if (is_dir) { if (kind == list_kind::dirs || kind == list_kind::all) { - result.push_back({path_to_utf8(rel_dir / fname), true}); + result.push_back({(rel_dir / fname).generic_string(), true}); } // junk directories stay selectable but are never walked: they can be enormous - if (junk_dir_names().count(get_effective_name(path_to_utf8(fname))) > 0) continue; + if (junk_dir_names().count(get_effective_name(fname.generic_string())) > 0) continue; if (!is_link(entry) && (max_depth == 0 || depth + 1 < max_depth)) { stack.emplace_back(entry.path(), rel_dir / fname, depth + 1); } } else if (entry.is_regular_file(tec)) { if (kind == list_kind::files || kind == list_kind::all) { - result.push_back({path_to_utf8(rel_dir / fname), false}); + result.push_back({(rel_dir / fname).generic_string(), false}); } } } @@ -867,7 +852,7 @@ static std::unique_ptr make_tools_io(const json & params) { // no '/' in pattern -> match basename at any depth; else match full relative path static bool path_glob_match(const std::string & pattern, const std::string & rel_path) { if (pattern.find('/') == std::string::npos) { - return glob_match(pattern, path_to_utf8(path_from_utf8(rel_path).filename())); + return glob_match(pattern, fs::path(rel_path).filename().generic_string()); } if (pattern == "**" || pattern.rfind("**/", 0) == 0 || pattern.rfind('/', 0) == 0) { return glob_match(pattern, rel_path); @@ -911,7 +896,7 @@ struct server_tool_read_file : server_tool { }; } - json invoke(json params, server_tool::stream *) const override { + json invoke(json params, server_tool::stream *, const std::function &) const override { std::string path = params.at("path").get(); int start_line = json_value(params, "start_line", 1); int end_line = json_value(params, "end_line", -1); // -1 = no limit @@ -1028,7 +1013,7 @@ struct server_tool_file_glob_search : server_tool { }; } - json invoke(json params, server_tool::stream *) const override { + json invoke(json params, server_tool::stream *, const std::function &) const override { auto io = make_tools_io(params); const std::string path = params.at("path").get(); @@ -1143,7 +1128,7 @@ struct server_tool_grep_search : server_tool { }; } - json invoke(json params, server_tool::stream *) const override { + json invoke(json params, server_tool::stream *, const std::function &) const override { std::string path = params.at("path").get(); std::string pat_str = params.at("pattern").get(); std::string include = json_value(params, "include", std::string("**")); @@ -1190,7 +1175,7 @@ struct server_tool_grep_search : server_tool { for (const auto & entry : listing.entries) { if (!path_glob_match(include, entry.rel)) continue; if (!exclude.empty() && path_glob_match(exclude, entry.rel)) continue; - files.emplace_back(path_to_utf8(path_from_utf8(abs_path) / path_from_utf8(entry.rel)), entry.rel); + files.emplace_back((fs::path(abs_path) / entry.rel).generic_string(), entry.rel); } } else { return {{"error", "path does not exist: " + path}}; @@ -1286,7 +1271,7 @@ struct server_tool_exec_shell_command : server_tool { }; } - json invoke(json params, server_tool::stream * st) const override { + json invoke(json params, server_tool::stream * st, const std::function &) const override { std::string command = params.at("command").get(); int timeout = json_value(params, "timeout", 10); size_t max_output = (size_t) json_value(params, "max_output_size", (int) SERVER_TOOL_EXEC_SHELL_COMMAND_MAX_OUTPUT_SIZE); @@ -1363,7 +1348,7 @@ struct server_tool_write_file : server_tool { }; } - json invoke(json params, server_tool::stream *) const override { + json invoke(json params, server_tool::stream *, const std::function &) const override { std::string path = params.at("path").get(); std::string content = params.at("content").get(); @@ -1420,7 +1405,7 @@ struct server_tool_edit_file : server_tool { }; } - json invoke(json params, server_tool::stream *) const override { + json invoke(json params, server_tool::stream *, const std::function &) const override { std::string path = params.at("path").get(); const json & edits_json = params.at("edits"); @@ -1736,7 +1721,7 @@ struct server_tool_get_info : server_tool { }; } - json invoke(json params, server_tool::stream *) const override { + json invoke(json params, server_tool::stream *, const std::function &) const override { auto io = make_tools_io(params); // inside an isolate, we always use the linux command @@ -1757,7 +1742,7 @@ struct server_tool_get_info : server_tool { if (cwd.empty()) { if (json_value(params, "runtime", std::string()).empty()) { std::error_code ec; - cwd = path_to_utf8(fs::current_path(ec)); + cwd = fs::current_path(ec).generic_string(); } else { auto pwd = io->run({"pwd"}, SERVER_TOOL_GET_INFO_MAX_OUTPUT, SERVER_TOOL_GET_INFO_TIMEOUT); cwd = pwd.exit_code == 0 && !pwd.timed_out ? string_strip(pwd.output) : "unknown"; @@ -1850,8 +1835,8 @@ struct server_mcp_tool : server_tool { }; } - json invoke(json params, server_tool::stream *) const override { - return mcp_mgr.call_tool(server_name, tool_name, params); + json invoke(json params, server_tool::stream *, const std::function & should_stop) const override { + return mcp_mgr.call_tool(server_name, tool_name, params, should_stop); } }; @@ -1925,7 +1910,7 @@ struct server_tools_container_runtime : server_tools_runtime { "llama-tools-runtime-cid-%zu.tmp", std::hash{}(std::this_thread::get_id())); fs::remove(cidfile, ec); - std::vector args = {bin, "run", "--rm", "-i", "--cidfile", path_to_utf8(cidfile), image, "sh"}; + std::vector args = {bin, "run", "--rm", "-i", "--cidfile", cidfile.generic_string(), image, "sh"}; int options = subprocess_option_no_window | subprocess_option_inherit_environment | subprocess_option_search_user_path; @@ -2108,15 +2093,20 @@ void server_tools::setup(const std::vector & enabled_tools, } // accept x-tool-runtime header to route tool I/O through an isolate, e.g. "docker-container:"; - // falls back to the --tools-runtime isolate, if configured + // --tools-runtime wins: it is the sandbox, the client must not move tools out of it if (params.contains("runtime")) { params.erase("runtime"); } auto runtime_header = get_header(req.headers, "x-tool-runtime"); - if (!runtime_header.empty()) { - params["runtime"] = runtime_header; - } else if (runtime) { + if (runtime) { + if (!runtime_header.empty()) { + res->status = 403; + res->data = safe_json_to_str(format_error_response("x-tool-runtime is not allowed when --tools-runtime is set", ERROR_TYPE_PERMISSION)); + return res; + } params["runtime"] = runtime->spec(); + } else if (!runtime_header.empty()) { + params["runtime"] = runtime_header; } // x-resp-type header is only used by read_file for now @@ -2143,7 +2133,7 @@ void server_tools::setup(const std::vector & enabled_tools, auto done = std::make_unique(); try { - tool.invoke(params, &st); + tool.invoke(params, &st, req.should_stop); } catch (const std::exception & e) { done->error_msg = e.what(); } catch (...) { @@ -2168,7 +2158,7 @@ void server_tools::setup(const std::vector & enabled_tools, return true; }; } else { - json result = tool.invoke(params, nullptr); + json result = tool.invoke(params, nullptr, req.should_stop); res->status = 200; res->data = safe_json_to_str(result); } diff --git a/tools/server/server-tools.h b/tools/server/server-tools.h index e7332f2e5743..5a63ee973bef 100644 --- a/tools/server/server-tools.h +++ b/tools/server/server-tools.h @@ -26,7 +26,8 @@ struct server_tool { std::function alive; void push(const std::string & chunk); }; - virtual json invoke(json params, stream * st = nullptr) const = 0; + // should_stop: true once the caller stopped waiting, e.g. the client dropped the connection; a null should_stop means "never stop" + virtual json invoke(json params, stream * st = nullptr, const std::function & should_stop = nullptr) const = 0; json to_json() const; }; diff --git a/tools/server/server.cpp b/tools/server/server.cpp index 8b1d426506fd..ca3eb2dbbe0a 100644 --- a/tools/server/server.cpp +++ b/tools/server/server.cpp @@ -13,11 +13,14 @@ #include "llama.h" #include "log.h" +#include #include #include #include +#include #include #include // for std::thread::hardware_concurrency +#include #if defined(_WIN32) #include @@ -186,6 +189,8 @@ static void server_register_routes(server_http_context & ctx_http, server_routes ctx_http.post("/v1/audio/transcriptions", ex_wrapper(routes.post_transcriptions_oai)); ctx_http.post("/audio/transcriptions", ex_wrapper(routes.post_transcriptions_oai)); ctx_http.post("/tts", ex_wrapper(routes.post_tts)); + ctx_http.post("/v1/audio/speech", ex_wrapper(routes.post_speech_oai)); + ctx_http.post("/audio/speech", ex_wrapper(routes.post_speech_oai)); ctx_http.ws ("/v1/realtime", routes.ws_realtime); ctx_http.post("/v1/messages", ex_wrapper(routes.post_anthropic_messages)); // anthropic messages API ctx_http.post("/infill", ex_wrapper(routes.post_infill)); @@ -254,6 +259,17 @@ static bool server_setup_features( SRV_WRN("%s", "security: no API key is set and CORS allows all origins (see https://github.com/ggml-org/llama.cpp/pull/25655)\n"); } + // tools can run shell commands: do not expose them to the network without auth + const bool tools_enabled = !params.server_tools.empty() || !params.mcp_servers_config.empty() || !params.mcp_servers_json.empty() || params.ui_mcp_proxy; + const bool loopback_only = std::all_of(params.hostnames.begin(), params.hostnames.end(), [](const std::string & h) { + return string_ends_with(h, ".sock") || h == "localhost" || h == "::1" || h == "[::1]" || string_starts_with(h, "127."); + }); + if (tools_enabled && !loopback_only && params.api_keys.empty() && !params.server_tools_allow_unauth) { + SRV_ERR("%s", "security: server tools, MCP servers or MCP proxy are enabled on a non-loopback address without an API key\n"); + SRV_ERR("%s", "set --api-key, bind to 127.0.0.1, or pass --allow-unauthenticated-tools on a trusted network\n"); + return false; + } + std::vector warn_names; if (role_type == SERVER_ROLE_ROUTER) { warn_names.push_back("router mode"); @@ -352,7 +368,10 @@ static bool server_on_listening(server_http_context & ctx_http, const common_par struct server_download_state : public common_download_callback { server_role * role; std::function should_stop; - std::atomic last_progress_time{0}; // multiple files downloading in different threads + // files download in parallel threads: throttle each file on its own, a shared + // timer lets a small, fast file (e.g. a draft) starve the updates of the others + std::mutex progress_mutex; + std::unordered_map last_progress_time; bool is_ok = false; server_download_state(server_role * r) : role(r) {} @@ -382,11 +401,17 @@ struct server_download_state : public common_download_callback { } void on_update(const common_download_progress & p) override { int64_t now = ggml_time_ms(); - // throttle progress updates to avoid flooding logs - if (now - last_progress_time.load(std::memory_order_relaxed) >= 100) { - on_progress(p); - last_progress_time.store(now, std::memory_order_relaxed); + // throttle progress updates to avoid flooding logs; the first update of a file + // always passes, it carries the file's total size + { + std::lock_guard lk(progress_mutex); + auto it = last_progress_time.find(p.url); + if (it != last_progress_time.end() && now - it->second < 100) { + return; + } + last_progress_time[p.url] = now; } + on_progress(p); } void on_done(const common_download_progress & p, bool) override { on_progress(p); @@ -504,6 +529,7 @@ static int server_run_router(common_params & params, int argc, char ** argv, con routes.post_responses_oai = models_routes->proxy_post; routes.post_transcriptions_oai = models_routes->proxy_post; routes.post_tts = models_routes->proxy_post; + routes.post_speech_oai = models_routes->proxy_post; routes.ws_realtime = models_routes->proxy_ws; routes.post_anthropic_messages = models_routes->proxy_post; routes.post_anthropic_count_tokens = models_routes->proxy_post; diff --git a/tools/server/tests/README.md b/tools/server/tests/README.md index f566b43644b1..d0f8b7938997 100644 --- a/tools/server/tests/README.md +++ b/tools/server/tests/README.md @@ -39,6 +39,47 @@ To run slow tests (will download many models, make sure to set `LLAMA_CACHE` if SLOW_TESTS=1 ./tests.sh ``` +The speculative-decoding method tests are opt-in and marked `heavy`. The compact suite covers MTP, +DFlash, DSpark, and EAGLE-3 with five pinned files totaling 6.271 GiB: + +```shell +SPEC_METHODS=compact SPEC_MODEL_DIR=/path/to/model-cache PYTEST_WORKERS=1 ./tests.sh -s -v unit/test_spec_methods.py +``` + +- MTP target: `Qwen3.5-4B-MTP-Q4_K_M.gguf`, 2.904 GiB. +- Shared DFlash/DSpark/EAGLE-3 target: `Qwen3-4B-Q4_K_M.gguf`, 2.326 GiB. +- DFlash draft: `dflash-Qwen3-4B-Q4_K_M.gguf`, 294.065 MiB. +- DSpark draft: `dspark-Qwen3-4B-Q4_K_M.gguf`, 544.453 MiB. +- EAGLE-3 draft: `eagle3-Qwen3-4B-Q8_0.gguf`, 227.075 MiB. + +All files come from `ggml-org/spec-for-testing-GGUF`, pinned to revision +`a0736c8e98d3fe580352dacffa3fbd5e9c22a437` in `unit/test_spec_methods.py`. +The consolidated repository retains the licenses of its source models; review its model card and +the source model cards before use. + +Use `SPEC_METHODS=nightly` for DFlash2 and LiLiCorr, or a comma-separated subset such as +`SPEC_METHODS=mtp,dflash`. The nightly pair requires 41.422 GiB and remains separate because no +verified compact DFlash2 GGUF or compact LiLiCorr checkpoint is available. `SPEC_METHODS=all` runs both +tiers and downloads about 47.693 GiB. + +The `Server (speculative decoding)` workflow runs only by explicit manual dispatch on the dedicated +DGX Spark runner. It does not use the GitHub Actions cache. Set the `SPEC_METHOD_MODEL_DIR` repository +variable to the runner's persistent model directory when its mount path is confirmed. Heavyweight +DFlash2 and LiLiCorr measurements belong in the internal nightly performance/quality runs. + +The acceptance floors use one-sided regression checks: improvements do not fail. Three identical +runs on the GB10 runner against the PR merged with current master produced MTP `0.6640 / 2.9766`, +DFlash `0.2004 / 2.3687`, DSpark `0.3367 / 3.3246`, and EAGLE-3 `0.2606 / 1.7757` for acceptance +rate and length. The compact floors subtract six percentage points and 0.2 accepted tokens per +verification step. The six-point rate margin is wider than the largest observed cross-environment +shift of 0.0506. Warm compact runs completed in about 40 seconds. The DFlash2 and LiLiCorr nightly +floors retain their earlier three-point rate and 0.2 length margins. + +DFlash's token-acceptance rate is lower because the denominator counts every token proposed in each +seven-token draft block. It accepted 219 of 1,093 proposed tokens over 160 verification steps, +equivalent to 1.369 accepted draft tokens per step and a mean acceptance length of 2.3687. The floor +is intended to detect regressions from that deterministic baseline, not to define a quality target. + To run with stdout/stderr display in real time (verbose output, but useful for debugging): ```shell @@ -76,7 +117,7 @@ Example using `gdb`: $ gdb --args ../../../build/bin/llama-server \ --host 127.0.0.1 --port 8080 \ --temp 0.8 --seed 42 \ - --hf-repo ggml-org/models --hf-file tinyllamas/stories260K.gguf \ + --hf-repo ggml-org-dev/stories260K --hf-file stories260K.gguf \ --batch-size 32 --no-slots --alias tinyllama-2 --ctx-size 512 \ --parallel 2 --n-predict 64 ``` diff --git a/tools/server/tests/fixtures/mcp_slow_server.py b/tools/server/tests/fixtures/mcp_slow_server.py index 7f8e67835acc..879d48ffe92b 100644 --- a/tools/server/tests/fixtures/mcp_slow_server.py +++ b/tools/server/tests/fixtures/mcp_slow_server.py @@ -1,6 +1,7 @@ #!/usr/bin/env python3 """ MCP server that sleeps before responding, for timeout testing. +With --log, it appends every message it reads from stdin to that file, for cancellation testing. """ import json import sys @@ -70,6 +71,7 @@ def handle_tools_call(params, req_id): def main(): parser = argparse.ArgumentParser() parser.add_argument("--delay", type=float, default=5.0, help="Delay in seconds for sleep tool") + parser.add_argument("--log", help="File to append every message read from stdin to") args = parser.parse_args() # Override the sleep duration @@ -102,6 +104,9 @@ def handle_tools_call(params, req_id): line = line.strip() if not line: continue + if args.log: + with open(args.log, "a") as log: + log.write(line + "\n") try: request = json.loads(line) except json.JSONDecodeError: diff --git a/tools/server/tests/pytest.ini b/tools/server/tests/pytest.ini index 6df308df74d5..42d1ab9cce51 100644 --- a/tools/server/tests/pytest.ini +++ b/tools/server/tests/pytest.ini @@ -1,4 +1,5 @@ [pytest] markers = + heavy: requires a dedicated runner and large model downloads slow: marks tests as slow (deselect with '-m "not slow"') serial diff --git a/tools/server/tests/unit/test_basic.py b/tools/server/tests/unit/test_basic.py index b9e9f84f6383..423a7c326cfa 100644 --- a/tools/server/tests/unit/test_basic.py +++ b/tools/server/tests/unit/test_basic.py @@ -98,8 +98,8 @@ def test_server_slots(): def test_load_split_model(): global server server.offline = False - server.model_hf_repo = "ggml-org/models" - server.model_hf_file = "tinyllamas/split/stories15M-q8_0-00001-of-00003.gguf" + server.model_hf_repo = "ggml-org-dev/stories15M" + server.model_hf_file = "split/stories15M-q8_0-00001-of-00003.gguf" server.model_alias = "tinyllama-split" server.start() res = server.make_request("POST", "/completion", data={ diff --git a/tools/server/tests/unit/test_chat_completion.py b/tools/server/tests/unit/test_chat_completion.py index afbcef030664..380e0d1f25ba 100644 --- a/tools/server/tests/unit/test_chat_completion.py +++ b/tools/server/tests/unit/test_chat_completion.py @@ -238,6 +238,10 @@ def test_apply_chat_template(): ({"type": "json_object", "schema": {"const": "42"}}, 6, "\"42\""), ({"type": "json_object", "schema": {"items": [{"type": "integer"}]}}, 10, "[ -3000 ]"), ({"type": "json_schema", "json_schema": {"schema": {"const": "foooooo"}}}, 10, "\"foooooo\""), + ({"type": "json_schema", "schema": {"const": "foooooo"}}, 10, "\"foooooo\""), + # the json_schema wrapper takes precedence over a top-level schema, even when its schema is {} + ({"type": "json_schema", "json_schema": {"schema": {"const": "foooooo"}}, "schema": {"const": "42"}}, 10, "\"foooooo\""), + ({"type": "json_schema", "json_schema": {"schema": {}}, "schema": {"type": "hiccup"}}, 10, ".+"), ({"type": "json_object"}, 10, "(\\{|John)+"), ({"type": "sound"}, 0, None), # invalid response format (expected to fail) diff --git a/tools/server/tests/unit/test_compat_anthropic.py b/tools/server/tests/unit/test_compat_anthropic.py index 13d0774452eb..220417745dab 100644 --- a/tools/server/tests/unit/test_compat_anthropic.py +++ b/tools/server/tests/unit/test_compat_anthropic.py @@ -11,7 +11,7 @@ def get_test_image_base64() -> str: """Get a test image in base64 format""" # Use the same test image as test_vision_api.py - IMG_URL = "https://huggingface.co/ggml-org/tinygemma3-GGUF/resolve/main/test/11_truck.png" + IMG_URL = "https://huggingface.co/ggml-org-dev/tinygemma3-GGUF/resolve/main/test/11_truck.png" response = requests.get(IMG_URL) response.raise_for_status() return base64.b64encode(response.content).decode("utf-8") diff --git a/tools/server/tests/unit/test_compat_oai_speech.py b/tools/server/tests/unit/test_compat_oai_speech.py new file mode 100644 index 000000000000..888942f02ba9 --- /dev/null +++ b/tools/server/tests/unit/test_compat_oai_speech.py @@ -0,0 +1,137 @@ +import pytest +import requests +import struct +from utils import * + +server: ServerProcess + +BODY = { + "model": "tts", + "input": "Hello there.", + "voice": "female", + "seed": 42, + "n_predict": 8, +} + + +@pytest.fixture(autouse=True) +def create_server(): + global server + server = ServerPreset.tinylfm3omni() + + +def post_audio(path: str, **extra) -> requests.Response: + return requests.post(server.make_url(path), json={**BODY, **extra}, timeout=DEFAULT_REQUEST_TIMEOUT) + + +def assert_wav(res: requests.Response, data_size: int | None = None): + assert res.status_code == 200 + assert res.headers["Content-Type"] == "audio/wav" + assert res.content[:4] == b"RIFF" and res.content[8:12] == b"WAVE" + assert struct.unpack(" pos + 8 + + +@pytest.mark.parametrize("path", ["/v1/audio/speech", "/audio/speech"]) +def test_speech_default_format_is_wav(path: str): + server.start() + assert_wav(post_audio(path)) + + +def test_speech_wav_same_as_tts(): + server.start() + res = post_audio("/v1/audio/speech", response_format="wav") + assert_wav(res) + assert res.content == post_audio("/tts").content + + +def test_speech_pcm_is_signed_16_bit(): + server.start() + res = post_audio("/v1/audio/speech", response_format="pcm") + assert res.status_code == 200 + assert res.headers["Content-Type"] == "audio/pcm;rate=24000;encoding=signed-int;bits=16" + + # /tts keeps the float samples + ref = post_audio("/tts", response_format="pcm") + assert ref.headers["Content-Type"] == "audio/pcm;rate=24000;encoding=float;bits=32" + + samples = struct.unpack(f"<{len(res.content) // 2}h", res.content) + floats = struct.unpack(f"<{len(ref.content) // 4}f", ref.content) + assert len(samples) == len(floats) > 0 + assert any(v != 0 for v in samples) + for s, f in zip(samples, floats): + assert abs(s - max(-1.0, min(1.0, f)) * 32767) <= 1 + + +def test_speech_stream_pcm(): + server.start() + res = post_audio("/v1/audio/speech", response_format="pcm", stream_format="audio") + assert res.status_code == 200 + assert res.headers.get("Transfer-Encoding") == "chunked" + assert res.content == post_audio("/v1/audio/speech", response_format="pcm").content + + +def test_speech_stream_wav_has_unknown_size(): + server.start() + res = post_audio("/v1/audio/speech", stream_format="audio") + assert_wav(res, data_size=0xFFFFFFFF) + + +def test_speech_voice(): + server.start() + female = post_audio("/v1/audio/speech", voice="female") + male = post_audio("/v1/audio/speech", voice="male") + assert_wav(female) + assert_wav(male) + assert female.content != male.content + assert post_audio("/v1/audio/speech", voice={"id": "male"}).content == male.content + + +def test_speech_multipart(): + server.start() + fields = {k: (None, str(v)) for k, v in BODY.items()} + res = requests.post(server.make_url("/v1/audio/speech"), files=fields, timeout=DEFAULT_REQUEST_TIMEOUT) + assert_wav(res) + assert res.content == post_audio("/v1/audio/speech").content + + +@pytest.mark.parametrize("extra", [ + {"speed": 1.0}, + {"instructions": "Speak slowly."}, +]) +def test_speech_accepted_params(extra: dict): + server.start() + assert_wav(post_audio("/v1/audio/speech", **extra)) + + +@pytest.mark.parametrize("extra,message", [ + ({"response_format": "mp3"}, "response_format"), + ({"response_format": "opus"}, "response_format"), + ({"stream_format": "sse"}, "stream_format"), + ({"speed": 1.5}, "speed"), + ({"voice": "alloy"}, "unknown voice"), + ({"input": ""}, "input"), +]) +def test_speech_invalid_request(extra: dict, message: str): + server.start() + res = post_audio("/v1/audio/speech", **extra) + assert res.status_code == 400 + assert message in res.json()["error"]["message"] + + +def test_speech_no_tts_model(): + global server + server = ServerPreset.tinyllama2() + server.start() + assert post_audio("/v1/audio/speech").status_code == 501 # ERROR_TYPE_NOT_SUPPORTED + + +def test_speech_router(): + global server + model = server.model_hf_repo + server = ServerPreset.router() + server.start() + assert_wav(post_audio("/v1/audio/speech", model=model)) diff --git a/tools/server/tests/unit/test_lora.py b/tools/server/tests/unit/test_lora.py index 00b2f245f60f..e21d9048bf58 100644 --- a/tools/server/tests/unit/test_lora.py +++ b/tools/server/tests/unit/test_lora.py @@ -3,7 +3,7 @@ server = ServerPreset.stories15m_moe() -LORA_FILE_URL = "https://huggingface.co/ggml-org/stories15M_MOE/resolve/main/moe_shakespeare15M.gguf" +LORA_FILE_URL = "https://huggingface.co/ggml-org-dev/stories15M_MOE/resolve/main/moe_shakespeare15M.gguf" @pytest.fixture(autouse=True) def create_server(): diff --git a/tools/server/tests/unit/test_mcp_servers.py b/tools/server/tests/unit/test_mcp_servers.py index 877b732a5349..7a9f0bea57b3 100644 --- a/tools/server/tests/unit/test_mcp_servers.py +++ b/tools/server/tests/unit/test_mcp_servers.py @@ -9,6 +9,7 @@ 4. Misconfigured MCP servers do not crash the server 5. Multiple MCP servers can be configured simultaneously 6. Warmup populates the tool list at startup +7. The MCP server receives notifications/cancelled when the client drops or the call times out """ import json import os @@ -319,29 +320,60 @@ def call_tool(): server.stop() -def test_mcp_tool_timeout(): - """Tool call should timeout if MCP server is too slow.""" - global server +def _slow_server_config(log_path: str, timeout_ms: int) -> str: MCP_SLOW_SERVER = os.path.join(FIXTURES_DIR, "mcp_slow_server.py") - mcp_json = _mcp_config_json({ + return _mcp_config_json({ "slow": { "command": sys.executable, - "args": [MCP_SLOW_SERVER, "--delay", "5"], - "timeout_ms": 500 + "args": [MCP_SLOW_SERVER, "--log", log_path], + "timeout_ms": timeout_ms, } }) - server = _start_server_with_mcp(mcp_json) + + +def _messages_read_by_mcp_server(log_path: str) -> list: + """Parses the fixture's log, skipping a last line that the fixture is still writing.""" + with open(log_path) as f: + return [json.loads(line) for line in f if line.endswith("\n")] + + +def _wait_for_cancellation(log_path: str, limit_seconds: float) -> dict: + """Returns the notifications/cancelled message once the MCP server has read it.""" + deadline = time.time() + limit_seconds + while time.time() < deadline: + for message in _messages_read_by_mcp_server(log_path): + if message.get("method") == "notifications/cancelled": + return message + time.sleep(0.1) + raise AssertionError(f"the MCP server read no notifications/cancelled: {_messages_read_by_mcp_server(log_path)}") + + +def _tools_call_id(log_path: str): + """Returns the id of the one tools/call request the MCP server has read.""" + ids = [m["id"] for m in _messages_read_by_mcp_server(log_path) if m.get("method") == "tools/call"] + assert len(ids) == 1, ids + return ids[0] + + +def test_mcp_tool_timeout(): + """A tool call that exceeds timeout_ms returns an error and sends notifications/cancelled to the MCP server.""" + global server + with tempfile.NamedTemporaryFile(suffix=".jsonl", delete=False) as f: + log_path = f.name + server = _start_server_with_mcp(_slow_server_config(log_path, timeout_ms=500)) try: res = server.make_request("POST", "/tools", data={ "tool": "slow_sleep", - "params": {"seconds": 5} + "params": {"seconds": 1} }) assert res.status_code == 200, res.body - body = res.body - assert "error" in body, body + assert "error" in res.body, res.body + cancellation = _wait_for_cancellation(log_path, limit_seconds=10) + assert cancellation["params"]["requestId"] == _tools_call_id(log_path) finally: server.stop() + os.unlink(log_path) def test_mcp_warmup_partial_failure(): @@ -711,3 +743,23 @@ def test_mcp_grandchild_holding_pipes_does_not_deadlock(): assert elapsed < 3, f"server shutdown took {elapsed:.1f}s (expected < 3s) — teardown likely deadlocked" finally: server.stop() + + +def test_mcp_dropped_client_sends_cancellation(): + """Dropping the connection during a tool call sends notifications/cancelled for that request to the MCP server.""" + global server + with tempfile.NamedTemporaryFile(suffix=".jsonl", delete=False) as f: + log_path = f.name + server = _start_server_with_mcp(_slow_server_config(log_path, timeout_ms=30000)) + + try: + with pytest.raises(requests.exceptions.ReadTimeout): + server.make_request("POST", "/tools", data={ + "tool": "slow_sleep", + "params": {"seconds": 1.5} + }, timeout=0.5) + cancellation = _wait_for_cancellation(log_path, limit_seconds=10) + assert cancellation["params"]["requestId"] == _tools_call_id(log_path) + finally: + server.stop() + os.unlink(log_path) diff --git a/tools/server/tests/unit/test_proxy.py b/tools/server/tests/unit/test_proxy.py index cb439b7a4560..58fd8c824826 100644 --- a/tools/server/tests/unit/test_proxy.py +++ b/tools/server/tests/unit/test_proxy.py @@ -35,6 +35,7 @@ def test_mcp_proxy(): def test_mcp_proxy_custom_port(): global server server.ui_mcp_proxy = True + server.debug_proxy_allow_private = True server.start() # try getting the server's models API via the proxy @@ -60,6 +61,7 @@ def log_message(self, format, *args): try: global server server.ui_mcp_proxy = True + server.debug_proxy_allow_private = True server.start() res = server.make_request("POST", f"/cors-proxy?url=http://127.0.0.1:{target.server_port}/", data={}) @@ -69,3 +71,62 @@ def log_message(self, format, *args): target.shutdown() target.server_close() + + +@pytest.mark.parametrize("target", [ + "http://127.0.0.1:{port}/models", + "http://localhost:{port}/models", + "http://[::1]:{port}/models", + "http://[::ffff:127.0.0.1]:{port}/models", + "http://10.0.0.1/", + "http://172.16.0.1/", + "http://192.168.1.1/", + "http://100.64.0.1/", + "http://169.254.169.254/latest/meta-data/", # cloud metadata + "http://0.0.0.0:{port}/models", +]) +def test_mcp_proxy_refuses_non_public_targets(target: str): + global server + server.ui_mcp_proxy = True + server.start() + target = target.format(port=server.server_port) + res = server.make_request("GET", f"/cors-proxy?url={target}") + assert res.status_code == 400 + + +def test_mcp_proxy_redirect_goes_through_proxy(): + class RedirectHandler(BaseHTTPRequestHandler): + def do_GET(self): + if self.path == "/start": + self.send_response(302) + self.send_header("Location", "/next") + self.end_headers() + else: + self.send_response(200) + self.end_headers() + self.wfile.write(b"final") + + def log_message(self, format, *args): + pass + + target = ThreadingHTTPServer(("127.0.0.1", 0), RedirectHandler) + target_thread = threading.Thread(target=target.serve_forever, daemon=True) + target_thread.start() + + try: + global server + server.ui_mcp_proxy = True + server.debug_proxy_allow_private = True + server.start() + + url = server.make_url(f"/cors-proxy?url=http://127.0.0.1:{target.server_port}/start") + # redirect points back to the proxy + res = requests.get(url, allow_redirects=False) + assert res.status_code == 302 + assert res.headers["Location"] == f"?url=http%3A%2F%2F127.0.0.1%3A{target.server_port}%2Fnext" + res = requests.get(url) + assert res.status_code == 200 + assert res.text == "final" + finally: + target.shutdown() + target.server_close() diff --git a/tools/server/tests/unit/test_router.py b/tools/server/tests/unit/test_router.py index 98f6aab9e7c5..e4e62086b0ed 100644 --- a/tools/server/tests/unit/test_router.py +++ b/tools/server/tests/unit/test_router.py @@ -27,7 +27,7 @@ def test_router_props(): @pytest.mark.parametrize( "model,success", [ - ("ggml-org/tinygemma3-GGUF:Q8_0", True), + ("ggml-org-dev/tinygemma3-GGUF:Q8_0", True), ("non-existent/model", False), ] ) @@ -109,7 +109,7 @@ def _load_model_and_wait( def test_router_unload_model(): global server server.start() - model_id = "ggml-org/tinygemma3-GGUF:Q8_0" + model_id = "ggml-org-dev/tinygemma3-GGUF:Q8_0" _load_model_and_wait(model_id) @@ -125,9 +125,9 @@ def test_router_models_max_evicts_lru(): server.start() candidate_models = [ - "ggml-org/tinygemma3-GGUF:Q8_0", - "ggml-org/test-model-stories260K:F32", - "ggml-org/test-model-stories260K-infill:F32", + "ggml-org-dev/tinygemma3-GGUF:Q8_0", + "ggml-org-dev/test-model-stories260K:F32", + "ggml-org-dev/test-model-stories260K-infill:F32", ] # Load only the first 2 models to fill the cache @@ -150,9 +150,9 @@ def test_router_models_max_evicts_lru(): # server_lru_sched tests (relying on LLAMA_SERVER_DEBUG_FAKE_TIMING) -MODEL_A = "ggml-org/tinygemma3-GGUF:Q8_0" -MODEL_B = "ggml-org/test-model-stories260K:F32" -MODEL_C = "ggml-org/test-model-stories260K-infill:F32" +MODEL_A = "ggml-org-dev/tinygemma3-GGUF:Q8_0" +MODEL_B = "ggml-org-dev/test-model-stories260K:F32" +MODEL_C = "ggml-org-dev/test-model-stories260K-infill:F32" def _tokenize(model_id: str, timeout: float | None = DEFAULT_REQUEST_TIMEOUT) -> ServerResponse: @@ -388,7 +388,7 @@ def test_router_no_models_autoload(): global server server.no_models_autoload = True server.start() - model_id = "ggml-org/tinygemma3-GGUF:Q8_0" + model_id = "ggml-org-dev/tinygemma3-GGUF:Q8_0" res = server.make_request( "POST", @@ -422,7 +422,7 @@ def test_router_api_key_required(): server.api_key = "sk-router-secret" server.start() - model_id = "ggml-org/tinygemma3-GGUF:Q8_0" + model_id = "ggml-org-dev/tinygemma3-GGUF:Q8_0" auth_headers = {"Authorization": f"Bearer {server.api_key}"} res = server.make_request( @@ -463,10 +463,10 @@ def test_router_reload_models(): with open(preset_path, "w") as f: f.write( "[model-reload-a]\n" - "hf-repo = ggml-org/test-model-stories260K\n" + "hf-repo = ggml-org-dev/test-model-stories260K\n" "\n" "[model-reload-b]\n" - "hf-repo = ggml-org/test-model-stories260K-infill\n" + "hf-repo = ggml-org-dev/test-model-stories260K-infill\n" ) server.models_preset = preset_path @@ -480,10 +480,10 @@ def test_router_reload_models(): with open(preset_path, "w") as f: f.write( "[model-reload-b]\n" - "hf-repo = ggml-org/test-model-stories260K-infill\n" + "hf-repo = ggml-org-dev/test-model-stories260K-infill\n" "\n" "[model-reload-c]\n" - "hf-repo = ggml-org/test-model-stories260K\n" + "hf-repo = ggml-org-dev/test-model-stories260K\n" ) try: @@ -500,14 +500,14 @@ def test_router_dedup_cache_models(): global server preset_path = os.path.join(TMP_DIR, "test_dedup.ini") - main_cache_id = "ggml-org/test-model-stories260K:F32" - draft_cache_id = "ggml-org/test-model-stories260K-infill:F32" + main_cache_id = "ggml-org-dev/test-model-stories260K:F32" + draft_cache_id = "ggml-org-dev/test-model-stories260K-infill:F32" with open(preset_path, "w") as f: f.write( "[model-dedup]\n" - "hf-repo = ggml-org/test-model-stories260K\n" - "spec-draft-hf = ggml-org/test-model-stories260K-infill\n" + "hf-repo = ggml-org-dev/test-model-stories260K\n" + "spec-draft-hf = ggml-org-dev/test-model-stories260K-infill\n" "dedup-cache-models = 1\n" ) @@ -520,7 +520,7 @@ def test_router_dedup_cache_models(): assert main_cache_id not in ids, "main cache model should be hidden by dedup" assert draft_cache_id not in ids, "draft cache model should be hidden by dedup" # other cache models are unaffected - assert "ggml-org/tinygemma3-GGUF:Q8_0" in ids + assert "ggml-org-dev/tinygemma3-GGUF:Q8_0" in ids # the hidden model is only hidden from the listing, it can still be used res = server.make_request("POST", "/tokenize", data={"model": main_cache_id, "content": "hello"}) @@ -530,8 +530,8 @@ def test_router_dedup_cache_models(): with open(preset_path, "w") as f: f.write( "[model-dedup]\n" - "hf-repo = ggml-org/test-model-stories260K\n" - "spec-draft-hf = ggml-org/test-model-stories260K-infill\n" + "hf-repo = ggml-org-dev/test-model-stories260K\n" + "spec-draft-hf = ggml-org-dev/test-model-stories260K-infill\n" ) ids = _get_model_ids(is_reload=True) assert main_cache_id in ids @@ -544,8 +544,8 @@ def test_router_dedup_cache_models(): "dedup-cache-models = 1\n" "\n" "[model-dedup]\n" - "hf-repo = ggml-org/test-model-stories260K\n" - "spec-draft-hf = ggml-org/test-model-stories260K-infill\n" + "hf-repo = ggml-org-dev/test-model-stories260K\n" + "spec-draft-hf = ggml-org-dev/test-model-stories260K-infill\n" ) ids = _get_model_ids(is_reload=True) assert "model-dedup" in ids @@ -557,7 +557,7 @@ def test_router_dedup_cache_models(): def test_router_remote_preset(): global server - server.model_hf_repo = "ggml-org/test-preset-ci" + server.model_hf_repo = "ggml-org-dev/test-preset-ci" server.model_hf_file = None server.offline = False server.start() @@ -574,7 +574,7 @@ def test_router_remote_preset(): _load_model_and_wait(model_id) -MODEL_DOWNLOAD_ID = "ggml-org/test-model-router-download:F16" +MODEL_DOWNLOAD_ID = "ggml-org-dev/test-model-router-download:F16" MODEL_DOWNLOAD_TIMEOUT = 30 @@ -843,7 +843,7 @@ def test_router_preset_model_sidecars(): preset_path = os.path.join(TMP_DIR, "test_preset_sidecars.ini") with open(preset_path, "w") as f: - f.write("[preset-sidecars]\nhf-repo = ggml-org/tinygemma3-GGUF:Q8_0\n") + f.write("[preset-sidecars]\nhf-repo = ggml-org-dev/tinygemma3-GGUF:Q8_0\n") server.models_preset = preset_path server.start() @@ -929,3 +929,66 @@ def test_router_download_drops_stale_partials(): _wait_for_model_status(MODEL_DOWNLOAD_ID, {"downloaded", "unloaded"}, timeout=MODEL_DOWNLOAD_TIMEOUT) server.make_request("DELETE", f"/models?model={MODEL_DOWNLOAD_ID}") + + +def test_router_restricted_cors_same_origin_reaches_child(): + # same-origin request via LAN address must not be refused by the child + global server + lan_ip = get_lan_ip() + if lan_ip is None: + pytest.skip("no non-loopback interface") # ty: ignore[too-many-positional-arguments] + server.cors_origins = "localhost" + server.server_host = "0.0.0.0" + server.start() + body = { + "model": "ggml-org-dev/tinygemma3-GGUF:Q8_0", + "max_tokens": 4, + "messages": [{"role": "user", "content": "hello"}], + } + res = requests.post(server.make_url("/chat/completions", host=lan_ip), json=body, headers={ + "Origin": f"http://{lan_ip}:{server.server_port}", + "Sec-Fetch-Site": "same-origin", + }) + assert res.status_code == 200, res.text + + res = requests.post(server.make_url("/chat/completions", host="127.0.0.1"), json=body, headers={"Origin": "http://evil.com"}) + assert res.status_code == 403 + + +def test_router_child_refuses_browser_requests(tmp_path): + import re + global server + log_path = tmp_path / "router.log" + server.log_path = str(log_path) + server.start() + model_id = "ggml-org-dev/tinygemma3-GGUF:Q8_0" + _load_model_and_wait(model_id) + + pattern = rf"spawning server instance with name={re.escape(model_id)} on port (\d+)" + match = None + for _ in range(50): + match = re.search(pattern, log_path.read_text()) + if match: + break + time.sleep(0.1) + assert match, "child port not found in router log" + child_url = f"http://127.0.0.1:{match.group(1)}" + + # non-browser clients can reach the child directly + assert requests.get(f"{child_url}/health").status_code == 200 + + for headers in [ + {"Origin": "http://evil.com"}, + {"Origin": child_url}, + {"Sec-Fetch-Site": "cross-site", "Sec-Fetch-Mode": "no-cors"}, + {"Sec-Fetch-Site": "same-origin"}, + ]: + assert requests.get(f"{child_url}/health", headers=headers).status_code == 403, headers + + # browser requests through the router still reach the child + res = requests.post(server.make_url("/chat/completions"), json={ + "model": model_id, + "max_tokens": 4, + "messages": [{"role": "user", "content": "hello"}], + }, headers={"Origin": f"http://127.0.0.1:{server.server_port}", "Sec-Fetch-Site": "same-origin"}) + assert res.status_code == 200, res.text diff --git a/tools/server/tests/unit/test_security.py b/tools/server/tests/unit/test_security.py index 36fc439f9b73..ceec582acf0d 100644 --- a/tools/server/tests/unit/test_security.py +++ b/tools/server/tests/unit/test_security.py @@ -110,12 +110,12 @@ def test_cors_options(origin: str, cors_header: str, cors_header_value: str): @pytest.mark.parametrize("origin", [ "http://localhost", "http://localhost:8080", - "http://127.0.0.1", "http://127.0.0.1:3000", - "http://[::1]", "http://[::1]:3000", + "http://evil.com", + "http://localhost.evil.com", ]) -def test_cors_origins_localhost_reflects(origin: str): +def test_cors_origins_localhost_no_cross_origin_header(origin: str): global server server = ServerPreset.router() server.cors_origins = "localhost" @@ -126,49 +126,172 @@ def test_cors_origins_localhost_reflects(origin: str): "Access-Control-Request-Headers": "Authorization", }) assert res.status_code == 200 - assert res.headers["Access-Control-Allow-Origin"] == origin + assert "Access-Control-Allow-Origin" not in res.headers -@pytest.mark.parametrize("origin", [ - "http://web.mydomain.fr", - "http://evil.com", - "http://notlocalhost", - "http://localhost.evil.com", -]) -def test_cors_origins_localhost_rejects(origin: str): +@pytest.mark.parametrize("feature", ["server_tools", "ui_mcp_proxy"]) +def test_cors_origins_defaults_to_localhost(feature: str): global server server = ServerPreset.router() + if feature == "server_tools": + server.server_tools = "all" + else: + server.ui_mcp_proxy = True + server.start() + res = server.make_request("GET", "/props", headers={"Origin": f"http://127.0.0.1:{server.server_port}"}) + assert res.status_code == 200 + res = server.make_request("GET", "/props", headers={"Origin": "http://localhost:5173"}) + assert res.status_code == 403 + + +def _raw_post(path: str, body: str, headers: dict) -> requests.Response: + # text/plain: "simple" request, no CORS preflight + return requests.post(server.make_url(path), data=body, headers={"Content-Type": "text/plain", **headers}) + + +@pytest.mark.parametrize("headers,expected_status", [ + ({}, 200), # non-browser client + ({"Origin": "http://127.0.0.1:{port}"}, 200), # same-origin + ({"Origin": "http://localhost:3000"}, 403), # other localhost app + ({"Origin": "http://evil.com"}, 403), # cross-origin simple request + ({"Origin": "http://localhost.evil.com"}, 403), + ({"Origin": "null"}, 403), # sandboxed iframe / file:// + ({"Sec-Fetch-Site": "cross-site", "Sec-Fetch-Mode": "no-cors"}, 403), +]) +def test_cors_origins_localhost_blocks_simple_requests(headers: dict, expected_status: int): + global server + server.api_key = None server.cors_origins = "localhost" server.start() - res = server.make_request("OPTIONS", "/completions", headers={ - "Origin": origin, - "Access-Control-Request-Method": "POST", - "Access-Control-Request-Headers": "Authorization", + headers = {k: v.format(port=server.server_port) for k, v in headers.items()} + res = _raw_post("/tokenize", '{"content": "hello"}', headers) + assert res.status_code == expected_status + + +def test_cors_origins_localhost_allows_same_origin(): + global server + lan_ip = get_lan_ip() + if lan_ip is None: + pytest.skip("no non-loopback interface") # ty: ignore[too-many-positional-arguments] + server.api_key = None + server.cors_origins = "localhost" + server.server_host = "0.0.0.0" + server.start() + # UI opened via LAN address: Origin matches Host + url = server.make_url("/tokenize", host=lan_ip) + res = requests.post(url, data='{"content": "hello"}', headers={ + "Content-Type": "text/plain", + "Origin": f"http://{lan_ip}:{server.server_port}", }) assert res.status_code == 200 - assert "Access-Control-Allow-Origin" not in res.headers -def test_cors_origins_defaults_to_localhost_with_tools_enabled(): +@pytest.mark.parametrize("host,headers,expected_status", [ + ("localhost", {}, 200), + ("127.0.0.1", {}, 200), + ("127.0.0.2", {}, 200), + ("[::1]", {}, 200), + ("0.0.0.0", {}, 200), + ("evil.com", {"Origin": "http://evil.com:{port}"}, 403), # rebinding, Origin matches Host + ("evil.com", {"Sec-Fetch-Site": "same-origin"}, 403), # rebinding, same-origin GET has no Origin + ("127.0.0.1.evil.com", {}, 403), + ("localhost.evil.com", {}, 403), +]) +def test_cors_origins_localhost_blocks_dns_rebinding(host: str, headers: dict, expected_status: int): global server - server = ServerPreset.router() - server.server_tools = "all" + server.api_key = None + server.cors_origins = "localhost" server.start() - res = server.make_request("OPTIONS", "/completions", headers={ - "Origin": "http://localhost:8080", - "Access-Control-Request-Method": "POST", - "Access-Control-Request-Headers": "Authorization", - }) + port = server.server_port + headers = {k: v.format(port=port) for k, v in headers.items()} + res = _raw_post("/tokenize", '{"content": "hello"}', {"Host": f"{host}:{port}", **headers}) + assert res.status_code == expected_status + + +def test_cors_origins_explicit_list_allows_its_hosts(): + global server + server.api_key = None + server.cors_origins = "https://ia.example.com" + server.start() + # reverse proxy that keeps the Host of the public name + for host, expected_status in [ + ("ia.example.com", 200), + ("evil.com", 403), + ]: + res = _raw_post("/tokenize", '{"content": "hello"}', {"Host": host, "Origin": f"https://{host}"}) + assert res.status_code == expected_status, host + + +def test_cors_origins_explicit_list_blocks_simple_requests(): + global server + server.api_key = None + server.cors_origins = "https://a.example, https://b.example" + server.start() + for origin, expected_status in [ + ("https://a.example", 200), + ("https://b.example", 200), + ("https://c.example", 403), + ]: + res = _raw_post("/tokenize", '{"content": "hello"}', {"Origin": origin}) + assert res.status_code == expected_status, origin + + +def test_cors_origins_wildcard_keeps_allowing_all(): + global server + server.api_key = None + server.cors_origins = "*" + server.start() + res = _raw_post("/tokenize", '{"content": "hello"}', {"Origin": "http://evil.com"}) assert res.status_code == 200 - assert res.headers["Access-Control-Allow-Origin"] == "http://localhost:8080" - res = server.make_request("OPTIONS", "/completions", headers={ - "Origin": "http://evil.com", - "Access-Control-Request-Method": "POST", - "Access-Control-Request-Headers": "Authorization", - }) + +def test_cors_cross_site_navigation_to_ui_allowed(): + global server + server.api_key = None + server.cors_origins = "localhost" + server.start() + nav = {"Sec-Fetch-Site": "cross-site", "Sec-Fetch-Mode": "navigate"} + # UI may not be built in tests, only check it is not refused + assert requests.get(server.make_url("/"), headers=nav).status_code != 403 + assert requests.get(server.make_url("/props"), headers=nav).status_code == 403 + + +def test_tools_drive_by_exec_is_blocked(): + global server + server.api_key = None + server.server_tools = "exec_shell_command" # implies --cors-origins localhost + server.start() + body = '{"tool": "exec_shell_command", "params": {"command": "echo csrf-guard-test"}}' + res = _raw_post("/tools", body, {"Origin": "http://evil.com"}) + assert res.status_code == 403 + assert "csrf-guard-test" not in res.text + + res = _raw_post("/tools", body, {"Origin": "http://localhost:5173"}) + assert res.status_code == 403 + + res = _raw_post("/tools", body, {"Origin": f"http://127.0.0.1:{server.server_port}"}) assert res.status_code == 200 - assert "Access-Control-Allow-Origin" not in res.headers + assert "csrf-guard-test" in res.text + + +@pytest.mark.parametrize("server_host,api_key,allow_unauth,should_start", [ + ("127.0.0.1", None, False, True), + ("0.0.0.0", None, False, False), + ("0.0.0.0", TEST_API_KEY, False, True), + ("0.0.0.0", None, True, True), +]) +def test_tools_refuse_unauthenticated_network_bind(server_host: str, api_key: str | None, allow_unauth: bool, should_start: bool): + global server + server = ServerPreset.router() + server.server_tools = "all" + server.server_host = server_host + server.api_key = api_key + server.allow_unauthenticated_tools = allow_unauth + if should_start: + server.start() + else: + with pytest.raises(RuntimeError): + server.start() def test_cors_proxy_only_forwards_explicit_proxy_headers(): @@ -191,6 +314,7 @@ def log_message(self, format, *args): server = ServerPreset.tinyllama2() server.api_key = TEST_API_KEY server.ui_mcp_proxy = True + server.debug_proxy_allow_private = True server.start() res = server.make_request("GET", f"/cors-proxy?url=http://127.0.0.1:{target.server_port}/capture", headers={ diff --git a/tools/server/tests/unit/test_slot_save.py b/tools/server/tests/unit/test_slot_save.py index 488b63701d1b..70d06c2a98cc 100644 --- a/tools/server/tests/unit/test_slot_save.py +++ b/tools/server/tests/unit/test_slot_save.py @@ -217,8 +217,8 @@ def test_slot_erase(): # Erase remains gated on the slot's content. # -IMG_URL_CAT = "https://huggingface.co/ggml-org/tinygemma3-GGUF/resolve/main/test/91_cat.png" -IMG_URL_TRUCK = "https://huggingface.co/ggml-org/tinygemma3-GGUF/resolve/main/test/11_truck.png" +IMG_URL_CAT = "https://huggingface.co/ggml-org-dev/tinygemma3-GGUF/resolve/main/test/91_cat.png" +IMG_URL_TRUCK = "https://huggingface.co/ggml-org-dev/tinygemma3-GGUF/resolve/main/test/11_truck.png" def _get_img_base64(url: str) -> str: @@ -785,7 +785,7 @@ def test_slot_restore_damaged_checkpoint_appendix(swa_server, damage): @pytest.mark.parametrize("ctkd_restore", ["f16", "q8_0"]) def test_slot_restore_checkpoints_draft_kv_type_change(swa_server, ctkd_restore): server = swa_server - server.model_draft_hf_repo = "ggml-org/tinygemma3-GGUF:Q8_0" # same file as the target, already in the HF cache + server.model_draft_hf_repo = "ggml-org-dev/tinygemma3-GGUF:Q8_0" # same file as the target, already in the HF cache server.spec_type = "draft-simple" server.ctkd = "f16" server.start() diff --git a/tools/server/tests/unit/test_spec_methods.py b/tools/server/tests/unit/test_spec_methods.py new file mode 100644 index 000000000000..eb8e879b77ab --- /dev/null +++ b/tools/server/tests/unit/test_spec_methods.py @@ -0,0 +1,213 @@ +import os +import re +from dataclasses import dataclass + +import pytest +from utils import ServerProcess, download_file + +# Runs each speculative method greedily and checks that it still drafts and accepts as expected. +# accept rate = accepted / drafted, accept len = 1 + accepted / verification steps (from /metrics). +# DFlash2 and LiLiCorr use --spec-type draft-dflash; the draft GGUF metadata selects the variant. +# Needs real models and a GPU. Use SPEC_METHODS=compact, nightly, all, or a comma-separated method list. + +HF = "https://huggingface.co" +MODEL_REPO = "ggml-org/spec-for-testing-GGUF" +MODEL_REVISION = "a0736c8e98d3fe580352dacffa3fbd5e9c22a437" + + +@dataclass(frozen=True) +class SpecCase: + name: str + spec_type: str + target: str + draft: str | None # None when the heads are part of the target (MTP) + n_max: int + min_accept_rate: float + min_accept_len: float + + +CASES = [ + # Compact floors are the minimum of three identical GB10 calibration runs + # against current master minus six acceptance-rate percentage points and + # 0.2 accepted tokens per verification step. + SpecCase( + name="mtp", + spec_type="draft-mtp", + target="Qwen3.5-4B-MTP-Q4_K_M.gguf", + draft=None, + n_max=3, + min_accept_rate=0.6040, + min_accept_len=2.7766, + ), + SpecCase( + name="dflash", + spec_type="draft-dflash", + target="Qwen3-4B-Q4_K_M.gguf", + draft="dflash-Qwen3-4B-Q4_K_M.gguf", + n_max=7, + min_accept_rate=0.1404, + min_accept_len=2.1687, + ), + SpecCase( + name="dspark", + spec_type="draft-dspark", + target="Qwen3-4B-Q4_K_M.gguf", + draft="dspark-Qwen3-4B-Q4_K_M.gguf", + n_max=7, + min_accept_rate=0.2767, + min_accept_len=3.1246, + ), + SpecCase( + name="eagle3", + spec_type="draft-eagle3", + target="Qwen3-4B-Q4_K_M.gguf", + draft="eagle3-Qwen3-4B-Q8_0.gguf", + n_max=3, + min_accept_rate=0.2006, + min_accept_len=1.5757, + ), + # Nightly floors retain the earlier three-point rate and 0.2 length margins. + SpecCase( + name="dflash2", + spec_type="draft-dflash", + target="Qwen3.8-27B-Q4_K_M.gguf", + draft="dflash2-Qwen3.8-27B-Q8_0.gguf", + n_max=7, + min_accept_rate=0.5180, + min_accept_len=4.5414, + ), + SpecCase( + name="lilicorr", + spec_type="draft-dflash", + target="NVIDIA-Nemotron-3.5-Bolt-74B-A9B-24GB.gguf", + draft="lilicorr-NVIDIA-Nemotron-3.5-Bolt-74B-A9B-24GB-NVFP4.gguf", + n_max=7, + min_accept_rate=0.6399, + min_accept_len=5.4444, + ), +] + +COMPACT_METHODS = {"mtp", "dflash", "dspark", "eagle3"} +NIGHTLY_METHODS = {"dflash2", "lilicorr"} + +PROMPTS = [ + "Write a Python function that returns the n-th Fibonacci number using memoization.\n\n```python\n", + "The three main differences between TCP and UDP are:\n1.", + "Question: What is the capital of France, and what river flows through it?\nAnswer:", +] + +N_PREDICT = 128 + + +def model_path(file: str) -> str: + """Download a pinned GGUF from the consolidated test repository.""" + model_dir = os.environ.get("SPEC_MODEL_DIR", "./tmp") + os.makedirs(model_dir, exist_ok=True) + return download_file( + f"{HF}/{MODEL_REPO}/resolve/{MODEL_REVISION}/{file}", + os.path.join(model_dir, file), + ) + + +def make_server(case: SpecCase) -> ServerProcess: + server = ServerProcess() + server.model_hf_repo = None + server.model_hf_file = None + server.model_file = model_path(case.target) + server.n_ctx = 1024 + server.n_slots = 1 + server.n_gpu_layer = int(os.environ.get("N_GPU_LAYERS", 99)) + server.fa = "on" + server.temperature = 0.0 + server.server_metrics = True + server.spec_type = case.spec_type + server.spec_draft_n_gpu_layer = server.n_gpu_layer + if case.draft: + server.model_draft = model_path(case.draft) + if case.n_max: + server.spec_draft_n_max = case.n_max + return server + + +def generate(server: ServerProcess) -> list[list[int]]: + outputs = [] + for prompt in PROMPTS: + res = server.make_request( + "POST", + "/completion", + data={ + "prompt": prompt, + "temperature": 0.0, + "top_k": 1, + "seed": 42, + "n_predict": N_PREDICT, + "cache_prompt": False, + "return_tokens": True, + }, + ) + assert res.status_code == 200 + outputs.append(res.body["tokens"]) + return outputs + + +def spec_metrics(server: ServerProcess) -> dict[str, float]: + # Use exact cumulative counters: the equivalent server log values are rounded + # and emitted separately for each request. + res = server.make_request("GET", "/metrics") + assert res.status_code == 200 + values = {} + for name in ("draft_tokens", "accepted_tokens", "drafts"): + m = re.search( + rf"^llamacpp:spec_decode_num_{name}_total (\S+)$", res.body, re.MULTILINE + ) + assert m is not None, f"missing spec_decode_num_{name}_total in /metrics" + values[name] = float(m.group(1)) + return values + + +def selected_cases(): + wanted = os.environ.get("SPEC_METHODS") + if not wanted: + return [] + aliases = { + "all": {case.name for case in CASES}, + "compact": COMPACT_METHODS, + "nightly": NIGHTLY_METHODS, + } + selected = aliases.get( + wanted, {name.strip() for name in wanted.split(",") if name.strip()} + ) + unknown = selected - {c.name for c in CASES} + if unknown: + raise ValueError(f"unknown SPEC_METHODS: {', '.join(sorted(unknown))}") + return [case for case in CASES if case.name in selected] + + +@pytest.mark.heavy +@pytest.mark.slow +@pytest.mark.parametrize("case", selected_cases(), ids=lambda c: c.name) +def test_spec_method(case: SpecCase): + server = make_server(case) + try: + server.start(timeout_seconds=1800) + outputs = generate(server) + m = spec_metrics(server) + finally: + server.stop() + + accept_rate = m["accepted_tokens"] / m["draft_tokens"] if m["draft_tokens"] else 0.0 + accept_len = 1.0 + m["accepted_tokens"] / m["drafts"] if m["drafts"] else 1.0 + print( + f"SPEC_RESULT {case.name}: drafted={m['draft_tokens']:.0f} accepted={m['accepted_tokens']:.0f} " + f"steps={m['drafts']:.0f} accept_rate={accept_rate:.4f} accept_len={accept_len:.4f}" + ) + + assert all(outputs), f"{case.name}: at least one request returned no tokens" + assert m["draft_tokens"] > 0, f"{case.name}: no tokens were drafted" + assert m["drafts"] > 0, f"{case.name}: no speculative verification steps ran" + assert ( + accept_rate >= case.min_accept_rate + ), f"{case.name}: accept rate {accept_rate:.3f} < {case.min_accept_rate}" + assert ( + accept_len >= case.min_accept_len + ), f"{case.name}: accept length {accept_len:.3f} < {case.min_accept_len}" diff --git a/tools/server/tests/unit/test_speculative.py b/tools/server/tests/unit/test_speculative.py index 0d41b1173af0..10597fdb0fb7 100644 --- a/tools/server/tests/unit/test_speculative.py +++ b/tools/server/tests/unit/test_speculative.py @@ -5,7 +5,7 @@ server = ServerPreset.stories15m_moe() -MODEL_DRAFT_FILE_URL = "https://huggingface.co/ggml-org/tiny-llamas/resolve/main/stories15M-q4_0.gguf" +MODEL_DRAFT_FILE_URL = "https://huggingface.co/ggml-org-dev/tiny-llamas/resolve/main/stories15M-q4_0.gguf" def create_server(): global server diff --git a/tools/server/tests/unit/test_stream.py b/tools/server/tests/unit/test_stream.py index a1ef55567bc7..741086316133 100644 --- a/tools/server/tests/unit/test_stream.py +++ b/tools/server/tests/unit/test_stream.py @@ -10,7 +10,7 @@ # a model name with slashes exercises the query string routing of the stream routes: the id # cannot travel as a path param because the decoded slash would split it before capture -MODEL = "ggml-org/tinygemma3-GGUF:Q8_0" +MODEL = "ggml-org-dev/tinygemma3-GGUF:Q8_0" STREAM_ID = f"conv-stream-test::{MODEL}" QS = "conv_id=" + quote(STREAM_ID, safe="") diff --git a/tools/server/tests/unit/test_systemone.py b/tools/server/tests/unit/test_systemone.py index 67e128e11e88..0c94a06d16ad 100644 --- a/tools/server/tests/unit/test_systemone.py +++ b/tools/server/tests/unit/test_systemone.py @@ -122,10 +122,13 @@ def test_systemone_invalid_request(data: dict): assert "error" in res.body -def test_systemone_shared_prompt(): +def test_systemone_shared_prompt(tmp_path): global server server = ServerPreset.tinyopenjev() + server.n_slots = len(TEST_QUESTIONS) server.server_metrics = True + server.debug = True + server.log_path = str(tmp_path / "server.log") server.start() res = server.make_request("POST", "/v1/systemone", data={ "state": TEST_STATE, @@ -139,8 +142,19 @@ def test_systemone_shared_prompt(): assert n_cached % 2 == 0 assert n_processed + n_cached == res.body["usage"]["input_tokens"] - # with one slot the prompt cannot be shared, the answers must be the same + # the next request takes an empty slot as parent, so child slot 0 has a lower id than the parent + res_next = server.make_request("POST", "/v1/systemone", data={ + "state": "Unrelated follow-up case: " * 10, + "questions": TEST_QUESTIONS, + }) + assert res_next.status_code == 200 + + # in each request, each child waits for the parent during 1 batch only server.stop() + with open(server.log_path) as f: + assert f.read().count("__TEST_TAG_WAIT_PARENT_SLOT__") == 2 * (len(TEST_QUESTIONS) - 1) + + # with one slot the prompt cannot be shared, the answers must be the same server = ServerPreset.tinyopenjev() server.n_slots = 1 server.start() @@ -181,7 +195,7 @@ def test_systemone_prompt_cache(cache_ram, tmp_path): log = f.read() # the prompt cache is opt-in for decision models, and restores their states once enabled assert ("__TEST_TAG_CACHE_IDLE_SLOTS_ENABLED__" in log) == (cache_ram is not None) - assert ("found better prompt" in log) == (cache_ram is not None) + assert ("__TEST_TAG_PROMPT_CACHE_RESTORE__" in log) == (cache_ram is not None) def test_systemone_prompt_cache_child_slots(tmp_path): @@ -210,7 +224,7 @@ def decide(state, questions): decide(TEST_STATE, TEST_QUESTIONS) with open(server.log_path) as f: - n_restored = f.read().count("found better prompt with f_keep = 1.000") + n_restored = f.read().count("__TEST_TAG_PROMPT_CACHE_FULL_RESTORE__") # every overwritten state comes back whole from the prompt cache for state in states: @@ -219,7 +233,7 @@ def decide(state, questions): server.stop() with open(server.log_path) as f: - assert f.read().count("found better prompt with f_keep = 1.000") == n_restored + len(states) + assert f.read().count("__TEST_TAG_PROMPT_CACHE_FULL_RESTORE__") == n_restored + len(states) def test_systemone_images(): diff --git a/tools/server/tests/unit/test_tools_builtin.py b/tools/server/tests/unit/test_tools_builtin.py index d4ebd28d10e5..b3af7360443b 100755 --- a/tools/server/tests/unit/test_tools_builtin.py +++ b/tools/server/tests/unit/test_tools_builtin.py @@ -206,6 +206,21 @@ def test_tools_builtin_runtime_header(container_engine: str, container_id: str): assert "hello container" in exec_res["plain_text_response"] +def test_tools_builtin_runtime_header_refused_with_configured_runtime(container_engine: str, container_id: str): + global server + server.server_tools_runtime = f"{container_engine}-container:{container_id}" + server.start() + + res = call_tool("exec_shell_command", {"command": "echo sandboxed"}) + assert "sandboxed" in res["plain_text_response"] + + for runtime in ["ssh:user@internal-host", f"{container_engine}-container:other", server.server_tools_runtime]: + res = server.make_request("POST", "/tools", + data={"tool": "exec_shell_command", "params": {"command": "echo hi"}}, + headers={"x-tool-runtime": runtime}) + assert res.status_code == 403, res.body + + def test_tools_builtin_runtime_header_unknown_scheme(): global server server.start() diff --git a/tools/server/tests/unit/test_vision_api.py b/tools/server/tests/unit/test_vision_api.py index dd04e2e2b4c2..cad45ee9b5b4 100644 --- a/tools/server/tests/unit/test_vision_api.py +++ b/tools/server/tests/unit/test_vision_api.py @@ -6,8 +6,8 @@ server: ServerProcess def get_img_url(id: str) -> str: - IMG_URL_0 = "https://huggingface.co/ggml-org/tinygemma3-GGUF/resolve/main/test/11_truck.png" - IMG_URL_1 = "https://huggingface.co/ggml-org/tinygemma3-GGUF/resolve/main/test/91_cat.png" + IMG_URL_0 = "https://huggingface.co/ggml-org-dev/tinygemma3-GGUF/resolve/main/test/11_truck.png" + IMG_URL_1 = "https://huggingface.co/ggml-org-dev/tinygemma3-GGUF/resolve/main/test/91_cat.png" if id == "IMG_URL_0": return IMG_URL_0 elif id == "IMG_URL_1": @@ -204,3 +204,41 @@ def test_vision_embeddings_oai_content(): # same prompt and image in both formats assert data[0]["embedding"] == data[1]["embedding"] assert data[0]["embedding"] != data[2]["embedding"] + + +@pytest.mark.parametrize("allow_private", [False, True]) +def test_vision_image_url_private_address(allow_private: bool): + from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer + import threading + img = requests.get(get_img_url("IMG_URL_0")).content + + class ImageHandler(BaseHTTPRequestHandler): + def do_GET(self): + self.send_response(200) + self.send_header("Content-Type", "image/png") + self.end_headers() + self.wfile.write(img) + + def log_message(self, format, *args): + pass + + target = ThreadingHTTPServer(("127.0.0.1", 0), ImageHandler) + threading.Thread(target=target.serve_forever, daemon=True).start() + try: + global server + server.media_allow_private_urls = allow_private + server.start() + res = server.make_request("POST", "/chat/completions", data={ + "temperature": 0.0, + "top_k": 1, + "messages": [ + {"role": "user", "content": [ + {"type": "text", "text": "What is this:\n"}, + {"type": "image_url", "image_url": {"url": f"http://127.0.0.1:{target.server_port}/img.png"}}, + ]}, + ], + }) + assert res.status_code == (200 if allow_private else 400) + finally: + target.shutdown() + target.server_close() diff --git a/tools/server/tests/utils.py b/tools/server/tests/utils.py index 0ca49192a477..71470c2b612e 100644 --- a/tools/server/tests/utils.py +++ b/tools/server/tests/utils.py @@ -52,8 +52,8 @@ class ServerProcess: debug: bool = False server_port: int = 8080 server_host: str = "127.0.0.1" - model_hf_repo: str | None = "ggml-org/models" - model_hf_file: str | None = "tinyllamas/stories260K.gguf" + model_hf_repo: str | None = "ggml-org-dev/stories260K" + model_hf_file: str | None = "stories260K.gguf" model_alias: str = "tinyllama-2" temperature: float = 0.8 seed: int = 42 @@ -103,6 +103,7 @@ class ServerProcess: spec_draft_n_min: int | None = None spec_draft_n_max: int | None = None spec_draft_n_ubatch: int | None = None + spec_draft_n_gpu_layer: int | None = None spec_synth_len: float | None = None spec_synth_rates: List[float] | None = None no_ui: bool | None = None @@ -121,6 +122,9 @@ class ServerProcess: no_cache_idle_slots: bool = False log_path: str | None = None ui_mcp_proxy: bool = False + debug_proxy_allow_private: bool = False # test-only: lets the CORS proxy reach local servers + media_allow_private_urls: bool = False + allow_unauthenticated_tools: bool = False backend_sampling: bool = False gcp_compat: bool = False server_tools: str | None = None @@ -147,6 +151,8 @@ def start(self, timeout_seconds: int = DEFAULT_HTTP_TIMEOUT) -> None: **os.environ, "LLAMA_SERVER_DEBUG_FAKE_TIMING": "1", } + if self.debug_proxy_allow_private: + env["LLAMA_SERVER_DEBUG_PROXY_ALLOW_PRIVATE"] = "1" if self.llama_cache is not None: env["LLAMA_CACHE"] = self.llama_cache elif "LLAMA_CACHE" not in os.environ: @@ -262,6 +268,8 @@ def start(self, timeout_seconds: int = DEFAULT_HTTP_TIMEOUT) -> None: server_args.extend(["--spec-draft-n-min", self.spec_draft_n_min]) if self.spec_draft_n_ubatch: server_args.extend(["--spec-draft-n-ubatch", self.spec_draft_n_ubatch]) + if self.spec_draft_n_gpu_layer: + server_args.extend(["--spec-draft-ngl", self.spec_draft_n_gpu_layer]) if self.spec_synth_len is not None: server_args.extend(["--spec-synth-len", self.spec_synth_len]) if self.spec_synth_rates is not None: @@ -299,6 +307,10 @@ def start(self, timeout_seconds: int = DEFAULT_HTTP_TIMEOUT) -> None: server_args.append("--no-cache-idle-slots") if self.ui_mcp_proxy: server_args.append("--ui-mcp-proxy") + if self.media_allow_private_urls: + server_args.append("--media-allow-private-urls") + if self.allow_unauthenticated_tools: + server_args.append("--allow-unauthenticated-tools") if self.server_tools: server_args.extend(["--tools", self.server_tools]) if self.server_tools_runtime: @@ -385,6 +397,8 @@ def stop(self) -> None: def make_url(self, path: str, host: str | None = None) -> str: if host is None: host = self.server_host.split(",")[0].strip() + # a wildcard bind cannot be connected to on Windows + host = {"0.0.0.0": "127.0.0.1", "::": "[::1]"}.get(host, host) return f"http://{host}:{self.server_port}{path}" def make_request( @@ -561,7 +575,7 @@ def load_all() -> None: def tinyllama2() -> ServerProcess: server = ServerProcess() server.offline = True # will be downloaded by load_all() - server.model_hf_repo = "ggml-org/test-model-stories260K" + server.model_hf_repo = "ggml-org-dev/test-model-stories260K" server.model_hf_file = None server.model_alias = "tinyllama-2" server.n_ctx = 512 @@ -575,8 +589,8 @@ def tinyllama2() -> ServerProcess: def bert_bge_small() -> ServerProcess: server = ServerProcess() server.offline = True # will be downloaded by load_all() - server.model_hf_repo = "ggml-org/models" - server.model_hf_file = "bert-bge-small/ggml-model-f16.gguf" + server.model_hf_repo = "ggml-org-dev/bert-bge-small-GGUF" + server.model_hf_file = "ggml-model-f16.gguf" server.model_alias = "bert-bge-small" server.n_ctx = 512 server.n_batch = 128 @@ -590,8 +604,8 @@ def bert_bge_small() -> ServerProcess: def bert_bge_small_with_fa() -> ServerProcess: server = ServerProcess() server.offline = True # will be downloaded by load_all() - server.model_hf_repo = "ggml-org/models" - server.model_hf_file = "bert-bge-small/ggml-model-f16.gguf" + server.model_hf_repo = "ggml-org-dev/bert-bge-small-GGUF" + server.model_hf_file = "ggml-model-f16.gguf" server.model_alias = "bert-bge-small" server.n_ctx = 1024 server.n_batch = 300 @@ -606,7 +620,7 @@ def bert_bge_small_with_fa() -> ServerProcess: def tinyllama_infill() -> ServerProcess: server = ServerProcess() server.offline = True # will be downloaded by load_all() - server.model_hf_repo = "ggml-org/test-model-stories260K-infill" + server.model_hf_repo = "ggml-org-dev/test-model-stories260K-infill" server.model_hf_file = None server.model_alias = "tinyllama-infill" server.n_ctx = 2048 @@ -621,7 +635,7 @@ def tinyllama_infill() -> ServerProcess: def stories15m_moe() -> ServerProcess: server = ServerProcess() server.offline = True # will be downloaded by load_all() - server.model_hf_repo = "ggml-org/stories15M_MOE" + server.model_hf_repo = "ggml-org-dev/stories15M_MOE" server.model_hf_file = "stories15M_MOE-F16.gguf" server.model_alias = "stories15m-moe" server.n_ctx = 2048 @@ -636,8 +650,8 @@ def stories15m_moe() -> ServerProcess: def jina_reranker_tiny() -> ServerProcess: server = ServerProcess() server.offline = True # will be downloaded by load_all() - server.model_hf_repo = "ggml-org/models" - server.model_hf_file = "jina-reranker-v1-tiny-en/ggml-model-f16.gguf" + server.model_hf_repo = "ggml-org-dev/jina-reranker-v1-tiny-en-GGUF" + server.model_hf_file = "ggml-model-f16.gguf" server.model_alias = "jina-reranker" server.n_ctx = 512 server.n_batch = 512 @@ -656,7 +670,7 @@ def tinylaya() -> ServerProcess: server.model_file = local_model server.model_hf_repo = None else: - server.model_hf_repo = "ggml-org/tinylaya-for-testing-gguf" + server.model_hf_repo = "ggml-org-dev/tinylaya-for-testing-gguf" server.n_ctx = 1024 server.n_batch = 512 server.n_ubatch = 512 @@ -670,7 +684,7 @@ def tinyopenjev() -> ServerProcess: server.offline = True # will be downloaded by load_all() # mmproj is already provided by HF registry API server.model_hf_file = None - server.model_hf_repo = "ggml-org/tinyopenjev-for-testing-gguf:Q8_0" + server.model_hf_repo = "ggml-org-dev/tinyopenjev-for-testing-gguf:Q8_0" server.n_ctx = 4096 server.n_batch = 512 server.n_slots = 4 @@ -683,7 +697,7 @@ def tinygemma3() -> ServerProcess: server.offline = True # will be downloaded by load_all() # mmproj is already provided by HF registry API server.model_hf_file = None - server.model_hf_repo = "ggml-org/tinygemma3-GGUF:Q8_0" + server.model_hf_repo = "ggml-org-dev/tinygemma3-GGUF:Q8_0" server.model_alias = "tinygemma3" server.n_ctx = 1024 server.n_batch = 512 @@ -692,6 +706,20 @@ def tinygemma3() -> ServerProcess: server.seed = 42 return server + @staticmethod + def tinylfm3omni() -> ServerProcess: + server = ServerProcess() + server.offline = True # will be downloaded by load_all() + # mmproj is already provided by HF registry API + server.model_hf_file = None + server.model_hf_repo = "ggml-org-dev/tinyLFM3-Omni-for-testing-gguf:Q2_K" + server.model_alias = "tinylfm3omni" + server.n_ctx = 1024 + server.n_batch = 512 + server.n_slots = 1 + server.seed = 42 + return server + @staticmethod def router() -> ServerProcess: server = ServerProcess() @@ -777,5 +805,17 @@ def download_file(url: str, output_file_path: str | None = None) -> str: return output_file +def get_lan_ip() -> str | None: + # address of the interface that routes outside, no packet is sent + import socket + try: + with socket.socket(socket.AF_INET, socket.SOCK_DGRAM) as s: + s.connect(("192.0.2.1", 9)) + ip = s.getsockname()[0] + except OSError: + return None + return None if ip.startswith("127.") else ip + + def is_slow_test_allowed(): return os.environ.get("SLOW_TESTS") == "1" or os.environ.get("SLOW_TESTS") == "ON" diff --git a/tools/tts/README.md b/tools/tts/README.md index 9bec176245fc..39d773d85685 100644 --- a/tools/tts/README.md +++ b/tools/tts/README.md @@ -15,7 +15,7 @@ llama-tts -hf ggml-org/Qwen3-TTS-12Hz-1.7B-Base-GGUF -p "Hello world" --output o Note: it's highly recommended to always provide a speaker reference voice (via `--tts-speaker-file`); otherwise, the model's performance may be degraded. Common params: -- Sampling params such as `--top-k`, `--top-p`, `--temp`, etc. +- Sampling params such as `--top-k`, `--top-p`, `--temp`, etc. `--temp` also sets the temperature of the audio frames for models that support it, without it they use the model default - `-n ` limits the output length, e.g. `-n 500`. Note that how many milliseconds each frame represents varies by model - Core inference params such as `-ngl`, `-b`, `-ub`, etc. @@ -121,3 +121,67 @@ The mmproj holds the audio encoder (for ASR), the depthformer and the codec deco python convert_hf_to_gguf.py path/to/lfm3-audio --outfile lfm3-audio.gguf python convert_hf_to_gguf.py path/to/lfm3-audio --mmproj --outfile mmproj-lfm3-audio.gguf ``` + +## Chatterbox + +Two variants are supported: [Chatterbox Turbo](https://huggingface.co/ResembleAI/chatterbox-turbo) (English) and [Chatterbox Multilingual V3](https://huggingface.co/ResembleAI/chatterbox) (23 languages). + +Available params: +- `--tts-speaker-file` should point to a speaker reference audio file (wav, mp3). Optional, the predefined voice is used without it +- `--tts-voice` can be `default`, the voice shipped with the model (`conds.pt`) +- `--tts-lang` sets the language of the multilingual variant, for example `fr` (default: none). It is not used by Turbo + +Example usage: + +```sh +llama-tts -m chatterbox-mtl.gguf \ + -mm mmproj-chatterbox-mtl.gguf \ + -p "Bonjour tout le monde" \ + --tts-lang fr \ + --tts-speaker-file speaker.mp3 \ + --output out.wav +``` + +Note: the multilingual variant runs classifier-free guidance with a second sequence. `llama-tts` sets this up by itself, but `llama-server` does not support it yet. + +**Note for GGUF conversion:** + +The upstream repositories ship raw checkpoints without `config.json`. Download the files below into a directory, then convert it as usual (the variant is detected from the talker file): +- Turbo: `t3_turbo_v1.safetensors`, `s3gen_meanflow.safetensors`, `ve.safetensors`, `conds.pt` and the GPT-2 tokenizer files (`vocab.json`, `merges.txt`, `added_tokens.json`, `special_tokens_map.json`, `tokenizer_config.json`) +- Multilingual V3: `t3_mtl23ls_v3.safetensors`, `s3gen_v3.safetensors`, `ve.safetensors`, `conds.pt`, `mtl_tokenizer.json` + +```sh +python convert_hf_to_gguf.py path/to/chatterbox --outfile chatterbox.gguf +python convert_hf_to_gguf.py path/to/chatterbox --mmproj --outfile mmproj-chatterbox.gguf +``` + +## Drama 3 + +[Fish Audio Drama 3](https://huggingface.co/fishaudio/drama-3): a Qwen3.5-MoE backbone (35B, 3B active) samples each frame's main token, a codebook tower samples the 9 other codes, and the Fish DAC decodes them to 44.1 kHz audio. + +Available params: +- Speaker tags `<|speaker:0|>` to `<|speaker:15|>` in the text give each character its own voice, a text without a tag starts with `<|speaker:0|>` +- The language is the one of the text, `--tts-lang` is not used +- Note: `--tts-speaker-file` is not supported yet (voice cloning) +- Note: `--top-k` and `--top-p` are not used, frames are sampled with the settings of the reference (main token: temperature 1.0, top-p 0.9, repetition-aware redraw over the last 16 frames; other codes: temperature 1.0, top-k 30, top-p 0.9) +- Each frame is 2048 samples (about 46 ms), generation stops at the end-of-speech token +- The codebook tower keeps the whole sequence in its KV cache, the prompt and the generated frames are limited to 4096 positions + +Example usage: + +```sh +llama-tts -m drama3.gguf \ + -mm mmproj-drama3.gguf \ + -p "<|speaker:0|> Did you remember to buy the bread? <|speaker:1|> Of course I did, it is on the kitchen table." \ + -n 1000 \ + --output out.wav +``` + +**Note for GGUF conversion:** + +The routed experts of the checkpoint are FP8, they are dequantized at conversion. The mmproj holds the codebook tower and the decoder half of `codec.pth`, it is best kept in BF16: + +```sh +python convert_hf_to_gguf.py path/to/drama-3 --outtype q8_0 --outfile drama3.gguf +python convert_hf_to_gguf.py path/to/drama-3 --mmproj --outtype bf16 --outfile mmproj-drama3.gguf +``` diff --git a/tools/tts/tts.cpp b/tools/tts/tts.cpp index 8578f8869151..5f5c2b973f04 100644 --- a/tools/tts/tts.cpp +++ b/tools/tts/tts.cpp @@ -151,6 +151,10 @@ int main(int argc, char ** argv) { inp.top_k = params.sampling.top_k; inp.top_p = params.sampling.top_p; inp.seed = params.sampling.seed; + // without --temp the audio frames keep the model default, not the CLI default + if (params.sampling.user_sampling_config & common_params_sampling_config::COMMON_PARAMS_SAMPLING_CONFIG_TEMP) { + inp.temp = params.sampling.temp; + } // // stage 1: process prompt via backbone model, generate semantic representation diff --git a/tools/ui/.env.example b/tools/ui/.env.example index a1e6c54915ba..a561a68a6560 100644 --- a/tools/ui/.env.example +++ b/tools/ui/.env.example @@ -1,3 +1,21 @@ VITE_PUBLIC_APP_NAME='llama-ui' VITE_DEBUG='true' VITE_PUBLIC_SERVER_ORIGIN='http://localhost:8033' + +# Demo videos (npm run demo:pr / demo:docs), see demo/README.md +# Where demo scripts and videos are stored; unset keeps videos in demo/videos +# DEMO_STORAGE='hf://buckets//' +# DEMO_STORAGE='file:///path/to/dir' +# Public base URL of DEMO_STORAGE, used to link videos in PR descriptions +# DEMO_STORAGE_PUBLIC_URL='https://...' +# HF dataset for demo conversations (llama.app traces): `demo:traces` pushes the +# built-in ones there, and demos can import any .jsonl from it +# DEMO_TRACES_DATASET='/' +# DEMO_GITHUB_REPO='ggml-org/llama.cpp' +# DEMO_SERVER_ORIGIN='http://localhost:9931' +# DEMO_LLAMA_SERVER='../../build/bin/llama-server' +# DEMO_COLOR_SCHEME='light' +# Device pixel ratio demos render and record at (1440x900 viewport) +# DEMO_SCALE='2' +# Longer side of the GIF preview, in pixels; it shrinks further to stay under 5 MB +# DEMO_PREVIEW_WIDTH='800' diff --git a/tools/ui/.npmrc b/tools/ui/.npmrc index 0a690322a452..fd58f2418d44 100644 --- a/tools/ui/.npmrc +++ b/tools/ui/.npmrc @@ -1,3 +1,6 @@ engine-strict=true ignore-scripts=true min-release-age=7 +# The single-bundle production build needs more than the default heap on CI runners; +# npm passes this to every script it runs, on every platform +node-options=--max-old-space-size=6144 diff --git a/tools/ui/README.md b/tools/ui/README.md index c6ec54721c2a..2a8a4d817143 100644 --- a/tools/ui/README.md +++ b/tools/ui/README.md @@ -684,6 +684,19 @@ llama-server serves the UI with `Cross-Origin-Embedder-Policy: require-corp` and - **MCP server icons and web-search result favicons** have no CORS guarantee, so they go through the server's `/cors-proxy` when the user started llama-server with `--ui-mcp-proxy`; without it they fall back to the bundled glyph - **Per-row Hub lookups** wait until the row is near the viewport (the `nearViewport` action), so a long model list does not fire one request per row on mount +### 10. Nested Surfaces + +The sidebar stacks four surfaces, outermost first: + +| Layer | Example | Corners | +| ----- | ---------------------------------- | ---------------------------------- | +| Page | the body background | none | +| Panel | the sidebar | `rounded-2xl` | +| Group | the sidebar's footer, inset in it | `rounded-(--surface-group-radius)` | +| Row | Models, MCP or Settings in a group | `rounded-(--surface-row-radius)` | + +A row keeps its corners concentric with its group's: its radius is the group's less the padding between their edges, so the curves run parallel instead of pinching at the corners. `app.css` holds the three tokens (`--surface-group-radius`, `--surface-group-padding` and the derived `--surface-row-radius`); change the group's radius or padding there and the rows follow. A group pads its rows with `p-(--surface-group-padding)`. `SidebarNavigationActions` rounds every row from the token, so the actions at the top of the sidebar (New chat, Search) match the footer's rows; a phone sizes its rows for a thumb instead. A selected row is raised and edged (`bg-background`, `shadow-sm`, `ring-1`), where a hovered one is only tinted, so the two never look alike. The group's hairline border is left out of the sum. `tests/client/nested-surfaces.svelte.test.ts` checks the rule on the rendered footer. + --- ## Testing @@ -710,6 +723,8 @@ npm run test:server # Server-side unit tests npm run test:ui # Storybook visual tests ``` +Locally, the browser tests (e2e, client and Storybook) run in your installed Google Chrome with its sandbox on, since Chrome updates itself and Playwright's bundled Chromium lags behind security fixes. CI (`CI` set) uses the bundled Chromium from `npx playwright install`. + ### Storybook Development ```bash diff --git a/tools/ui/demo/.gitignore b/tools/ui/demo/.gitignore new file mode 100644 index 000000000000..5a02216c4546 --- /dev/null +++ b/tools/ui/demo/.gitignore @@ -0,0 +1,5 @@ +.runs/ +test-results/ +videos/ +*.demo.ts +/traces/ diff --git a/tools/ui/demo/README.md b/tools/ui/demo/README.md new file mode 100644 index 000000000000..70617780da67 --- /dev/null +++ b/tools/ui/demo/README.md @@ -0,0 +1,133 @@ +# Demo videos + +Scripted Playwright walkthroughs of the UI, exported as mp4 with an animated GIF preview, for pull requests and documentation. The harness lives here; the demo scripts don't: + +- **PR demos** live in the PR description, as a `ts demo` code block. +- **Docs demos** live in the storage target, under `scripts/docs/`. + +``` +demo/ +├── cli.ts demo:pr, demo:docs and demo:traces +├── storage.ts storage targets (hf://, file://) +├── helpers.ts `test` with a `demo` fixture: cursor, captions, human-like timing +├── recorder.ts screencast recording and mp4 export +├── fixtures/ fixture MCP servers, seeded settings and demo conversations +└── videos/ output (git-ignored) +``` + +## Setup + +### Tools + +| Tool | Used for | +| ---------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Node 22.18+ | runs `demo/cli.ts` directly (TypeScript type stripping) | +| `npx playwright install chromium` | the browser demos record in, with its sandbox on | +| `ffmpeg` | mp4 encoding and the GIF preview | +| `gh`, logged in | reads the PR description and writes the `## Demo` section, so you need edit rights on the PR | +| `hf`, logged in with a write token (`hf auth login`) | uploads to the bucket and the traces dataset | +| `llama-server` | `build/bin/llama-server`, or one already running at `DEMO_SERVER_ORIGIN` (default `:9931`). It runs in router mode, so demos can pick any model in the local cache; scripts that send messages need at least one. | + +`gh` is only needed for `demo:pr`, and `hf` only with the bucket or dataset below. Without them, the video stays in `demo/videos/`. + +Videos are recorded through Chromium's screencast at `DEMO_SCALE` (2x by default, so 2880x1800) and 60 fps, starting once the page has settled. + +### Hugging Face bucket (videos, previews, scripts) + +The bucket must be public, since GitHub loads the preview from it: + +```bash +hf buckets create /llama-ui-demos +``` + +Then set it in `tools/ui/.env` (see `.env.example`): + +```bash +DEMO_STORAGE='hf://buckets//llama-ui-demos' +DEMO_STORAGE_PUBLIC_URL='https://huggingface.co/buckets//llama-ui-demos/resolve' +``` + +Layout: `videos/pr//.{mp4,gif,demo.ts}`, `videos/docs/.{mp4,gif}` and `scripts/docs/*.demo.ts` (the docs scripts). Without `DEMO_STORAGE_PUBLIC_URL`, videos are uploaded but not linked in the PR. `DEMO_STORAGE='file:///some/dir'` stores them in a local folder instead. + +### Hugging Face dataset (conversations, optional) + +Only needed for demos that import conversations other than the built-in ones, or to publish the built-in ones with `npm run demo:traces`. It can be private: it is read with your `hf` login. + +```bash +hf repos create /llama-ui-demo-traces --repo-type dataset --private +``` + +```bash +DEMO_TRACES_DATASET='/llama-ui-demo-traces' +``` + +### Other storage + +Only Hugging Face buckets and `file://` folders are supported for now. S3, R2 or similar would be one more class in `storage.ts` (`upload` and `downloadDir`) plus the matching `DEMO_STORAGE_PUBLIC_URL`. + +## Recording a pull request + +Add the script to the PR description: + +````md +
+Demo script + +```ts demo +test('demo', async ({ demo, page }) => { + await demo.click(page.getByRole('button', { exact: true, name: 'Settings' })); + await demo.caption('What this shows'); +}); +``` + +
+```` + +Then run: + +```bash +npm run demo:pr -- 31303 +``` + +A PR by someone else runs its code and demo script on your machine, with your `gh` and `hf` logins, so `demo:pr` refuses it until you've read its diff and demo block and pass the head commit you read: `npm run demo:pr -- 31303 --trust `. If the PR gets new commits, it refuses again. + +It checks out the PR head in a temporary worktree, records the script against it, uploads `videos/pr//.{mp4,gif,demo.ts}` and adds or updates a `## Demo` section in the PR description with the preview linking to the video. + +A large PR can show its changes in several shorter videos: each `test()` in the block is recorded as its own video (`-1.mp4`, `-2.mp4`, ...), and the `## Demo` section shows their previews in a table, each headed by its test's title: up to four portrait (phone) videos to a row, or two landscape ones. Helpers and `test.use` at the top of the block are shared by all of them. + +````ts +test('Header and safe areas', async ({ demo, page }) => { + // ... +}); + +test('Sidebar gestures', async ({ demo, page }) => { + // ... +}); +``` GitHub has no API for attachments, so for an inline player drag `demo/videos/pr-.mp4` into the description instead. + +## Recording docs + +```bash +npm run demo:docs +```` + +Records every `scripts/docs/*.demo.ts` from the storage target and uploads `videos/docs/.{mp4,gif}`. + +## Writing a demo + +Scripts get `test` and `expect` from `helpers.ts` without importing them. `demo.click`, `demo.hover` and `demo.type` glide a visible cursor to the target before acting, with timing varied like a person's (seeded per demo, so re-takes match); `demo.scroll` turns the mouse wheel over a target; `demo.zoom` pinches in on a control or a point to show a detail too small to read in the frame, holds, and pinches back out (with the fingertips drawn on a touch device); `demo.caption` shows a caption at the top of the frame. Use `test.use({ settings, startPath })` to seed other settings or start on another route. + +## Conversations + +Demos that need existing chats import them with `test.use({ traces: ['long-chat'] })`, through Settings > Import/Export before the video starts. Traces use the export format (llama.app JSONL), so the Hub shows them as Llama traces: + +- `fixtures/traces.ts` generates the built-in ones: `long-chat` (36 tall exchanges) and `agentic-debugging` (long reasoning, shell output and code blocks). +- Any other name is downloaded as `.jsonl` from `DEMO_TRACES_DATASET`, e.g. an exported conversation uploaded there. + +`npm run demo:traces` writes the built-in traces to `demo/traces/` and pushes them to `DEMO_TRACES_DATASET`. + +## Trying a script locally + +To try a script locally, put it in `demo/` as `.demo.ts`, importing from `./helpers.ts`, and run `npm run demo:record -- demo/`. `npm run demo:codegen` opens Playwright codegen against `:5180` for a first draft. + +The config starts, or reuses when already running, the fixture MCP servers on `:8765`, `llama-server` on `:9931` (router mode, `--tools all`) and the UI's vite dev server. diff --git a/tools/ui/demo/cli.ts b/tools/ui/demo/cli.ts new file mode 100644 index 000000000000..a7835013b8c4 --- /dev/null +++ b/tools/ui/demo/cli.ts @@ -0,0 +1,411 @@ +/** + * Records demo videos for pull requests and docs. + * + * node demo/cli.ts pr record the ```ts demo block from a PR description, a video per test + * (--trust to record someone else's PR at that head commit) + * node demo/cli.ts docs record every docs/*.demo.ts script in DEMO_STORAGE + * node demo/cli.ts traces write the built-in traces, and push them to DEMO_TRACES_DATASET + * + * Settings come from tools/ui/.env (see .env.example). + */ + +import { DEMO_TRACES, generateTrace } from './fixtures/traces.ts'; +import { type DemoStorage, getDemoStorage, getPublicUrl } from './storage.ts'; +import { execFileSync } from 'node:child_process'; +import { createHash } from 'node:crypto'; +import { + existsSync, + mkdirSync, + readdirSync, + readFileSync, + rmSync, + statSync, + symlinkSync, + writeFileSync +} from 'node:fs'; +import { createServer } from 'node:net'; +import { tmpdir } from 'node:os'; +import { basename, join, relative, resolve } from 'node:path'; + +const DEMO_DIR = import.meta.dirname; +const UI_DIR = resolve(DEMO_DIR, '..'); +const RUNS_DIR = join(DEMO_DIR, '.runs'); +const VIDEOS_DIR = join(DEMO_DIR, 'videos'); +const TRACES_DIR = join(DEMO_DIR, 'traces'); +const DEMO_BLOCK = /```(?:ts|typescript)[ \t]+demo[ \t]*\r?\n([\s\S]*?)```/; +const HELPERS_IMPORT = /^import\s[^;]*from\s+['"][^'"]*helpers(?:\.ts)?['"];?\s*$/gm; +const SECTION_START = ''; +const SECTION_END = ''; +/** GitHub's image proxy refuses larger images, so a bigger preview shows as a broken image. */ +const PREVIEW_MAX_BYTES = 5 * 1024 * 1024; +const envFile = join(UI_DIR, '.env'); + +if (existsSync(envFile)) process.loadEnvFile(envFile); + +const REPO = process.env.DEMO_GITHUB_REPO ?? 'ggml-org/llama.cpp'; + +function run( + cmd: string, + args: string[], + opts: { cwd?: string; env?: NodeJS.ProcessEnv } = {} +): string { + return execFileSync(cmd, args, { + cwd: opts.cwd ?? UI_DIR, + encoding: 'utf8', + env: opts.env + }).trim(); +} + +/** Writes a demo script into .runs/, importing the harness helpers. */ +function writeDemoScript(name: string, source: string): string { + const file = join(RUNS_DIR, `${name}.demo.ts`); + const helpers = relative(RUNS_DIR, join(DEMO_DIR, 'helpers.ts')); + const body = source.replace(HELPERS_IMPORT, '').trim(); + + mkdirSync(RUNS_DIR, { recursive: true }); + writeFileSync(file, `import { expect, test } from './${helpers}';\n\n${body}\n`); + + return file; +} + +async function freePort(): Promise { + return new Promise((done, fail) => { + const server = createServer(); + + server.once('error', fail); + server.listen(0, () => { + const { port } = server.address() as { port: number }; + + server.close(() => done(port)); + }); + }); +} + +/** A video recorded from one `test()` of a script. */ +interface Video { + mp4: string; + title: string; +} + +/** Records the given scripts; returns a video for each of their tests, in order. */ +async function record(scripts: string[], uiDir = UI_DIR): Promise { + const port = await freePort(); + const manifests = scripts.map((s) => join(VIDEOS_DIR, `${basename(s, '.demo.ts')}.videos.jsonl`)); + + for (const manifest of manifests) rmSync(manifest, { force: true }); + + execFileSync( + 'npx', + [ + 'playwright', + 'test', + '--config=demo/playwright.config.ts', + ...scripts.map((s) => relative(UI_DIR, s)) + ], + { + cwd: UI_DIR, + env: { ...process.env, DEMO_UI_DIR: uiDir, DEMO_UI_PORT: String(port) }, + stdio: 'inherit' + } + ); + + // helpers.ts lists each video it encodes in the script's manifest + return manifests.flatMap((manifest) => + readFileSync(manifest, 'utf8') + .trim() + .split('\n') + .map((line) => JSON.parse(line) as Video) + ); +} + +/** + * Animated GIF preview that GitHub renders inline, its longer side DEMO_PREVIEW_WIDTH (so a + * phone's portrait video is that tall), shrunk until it fits under PREVIEW_MAX_BYTES. + */ +function makePreview(mp4: string): string { + const gif = mp4.replace(/\.mp4$/, '.gif'); + + for (let side = Number(process.env.DEMO_PREVIEW_WIDTH ?? 800); ; side = Math.round(side * 0.8)) { + execFileSync('ffmpeg', [ + '-y', + '-loglevel', + 'error', + '-i', + mp4, + '-vf', + `fps=10,scale='if(gt(iw,ih),${side},-2)':'if(gt(iw,ih),-2,${side})':flags=lanczos,split[a][b];[a]palettegen=stats_mode=diff[p];[b][p]paletteuse=dither=bayer:bayer_scale=5:diff_mode=rectangle`, + '-loop', + '0', + gif + ]); + + if (statSync(gif).size <= PREVIEW_MAX_BYTES || side < 320) return gif; + } +} + +function upload(storage: DemoStorage, files: Record): void { + for (const [key, file] of Object.entries(files)) { + storage.upload(file, key); + console.log(`uploaded ${key}`); + } +} + +/** + * Public URL with a content hash, so a re-recorded file at the same key gets a new + * URL: GitHub's image proxy caches previews by URL. + */ +function versionedUrl(key: string, file: string): null | string { + const url = getPublicUrl(key); + const version = createHash('sha1').update(readFileSync(file)).digest('hex').slice(0, 10); + + return url && `${url}?v=${version}`; +} + +/** Whether a video is taller than wide, like a phone's. */ +function isPortrait(mp4: string): boolean { + const [width, height] = run('ffprobe', [ + '-v', + 'error', + '-select_streams', + 'v:0', + '-show_entries', + 'stream=width,height', + '-of', + 'csv=p=0', + mp4 + ]) + .split(',') + .map(Number); + + return height > width; +} + +/** + * One preview, or a table of them when the script has several tests, each headed by its + * test's title: up to four portrait videos to a row, or two landscape ones, so each stays + * large enough to read. + */ +function demoSection( + links: { gif: string; mp4: string; title: string }[], + sha: string, + columns: number +): string { + const preview = ({ gif, mp4, title }: (typeof links)[number]) => + `[![${title.replace(/[[\]|]/g, '')}](${gif})](${mp4})`; + const cell = (text: string) => text.replace(/\|/g, '\\|'); + const rows = Array.from({ length: Math.ceil(links.length / columns) }, (_, i) => + links.slice(i * columns, (i + 1) * columns) + ); + const pad = (cells: string[]) => [...cells, ...Array(columns - cells.length).fill(' ')]; + const body = + links.length === 1 + ? [preview({ ...links[0], title: 'Demo' })] + : rows.flatMap((row, i) => [ + // a table has one header row, so later rows title their videos in bold + `| ${pad(row.map((l) => (i === 0 ? cell(l.title) : `**${cell(l.title)}**`))).join(' | ')} |`, + ...(i === 0 ? [`|${Array(columns).fill('---').join('|')}|`] : []), + `| ${pad(row.map(preview)).join(' | ')} |` + ]); + const note = + links.length === 1 ? 'the preview for the full video' : 'a preview for its full video'; + + return [ + SECTION_START, + ...body, + '', + `Recorded at ${sha.slice(0, 10)}; click ${note}.`, + SECTION_END + ].join('\n'); +} + +/** Replaces the managed demo section, or adds a `## Demo` section before the details. */ +function withDemoSection(body: string, section: string): string { + const start = body.indexOf(SECTION_START); + const end = body.indexOf(SECTION_END); + + if (start !== -1 && end > start) { + return body.slice(0, start) + section + body.slice(end + SECTION_END.length); + } + + const block = `## Demo\n\n${section}\n\n`; + const anchor = body.search(/^## (Additional information|Requirements)\b/m); + + return anchor === -1 + ? `${body.trimEnd()}\n\n${block}` + : body.slice(0, anchor) + block + body.slice(anchor); +} + +async function recordPr(number: string, trustedSha?: string): Promise { + const pr = JSON.parse( + run('gh', ['pr', 'view', number, '--repo', REPO, '--json', 'author,body,headRefOid']) + ) as { author: { login: string }; body: string; headRefOid: string }; + const source = DEMO_BLOCK.exec(pr.body)?.[1]; + + if (!source) throw new Error(`PR #${number} has no \`\`\`ts demo block in its description`); + + const sha = pr.headRefOid; + + // the PR's code and demo script run on this machine, with the gh and hf logins, + // so someone else's PR only runs at a head commit you've read and passed to --trust + if (pr.author.login !== run('gh', ['api', 'user', '--jq', '.login'])) { + if (!trustedSha) { + throw new Error( + `PR #${number} is by ${pr.author.login}, and recording it runs its code here: ` + + `read its diff and demo block, then run again with --trust ${sha.slice(0, 10)}` + ); + } + + if (trustedSha.length < 7 || !sha.startsWith(trustedSha)) { + throw new Error(`PR #${number} head is ${sha.slice(0, 10)}, not the trusted ${trustedSha}`); + } + } + + const worktree = join(tmpdir(), `llama-ui-demo-pr-${number}`); + + run('git', ['fetch', '--no-tags', `https://github.com/${REPO}.git`, `pull/${number}/head`]); + + if (run('git', ['rev-parse', 'FETCH_HEAD']) !== sha) { + throw new Error(`PR #${number} head moved while fetching, run again`); + } + + rmSync(worktree, { force: true, recursive: true }); + run('git', ['worktree', 'prune']); + run('git', ['worktree', 'add', '--detach', worktree, sha]); + symlinkSync(join(UI_DIR, 'node_modules'), join(worktree, 'tools/ui/node_modules')); + + const name = `pr-${number}`; + const script = writeDemoScript(name, source); + + try { + const videos = (await record([script], join(worktree, 'tools/ui'))).map((video) => ({ + ...video, + gif: makePreview(video.mp4) + })); + const storage = getDemoStorage(); + + for (const { gif, mp4 } of videos) console.log(`\nvideo: ${mp4}\npreview: ${gif}`); + + if (!storage) { + console.log('DEMO_STORAGE is not set: drag the mp4 into the PR description.'); + + return; + } + + const prefix = `videos/pr/${number}/${sha.slice(0, 10)}`; + // a script with several tests gets -1.mp4, -2.mp4 and so on + const keys = videos.map((_, i) => (videos.length === 1 ? prefix : `${prefix}-${i + 1}`)); + + upload(storage, { [`${prefix}.demo.ts`]: script }); + videos.forEach(({ gif, mp4 }, i) => { + upload(storage, { [`${keys[i]}.gif`]: gif, [`${keys[i]}.mp4`]: mp4 }); + }); + + if (!getPublicUrl(prefix)) { + console.log('DEMO_STORAGE_PUBLIC_URL is not set: not linking the video in the PR.'); + + return; + } + + const links = videos.map(({ gif, mp4, title }, i) => ({ + gif: versionedUrl(`${keys[i]}.gif`, gif) ?? '', + mp4: versionedUrl(`${keys[i]}.mp4`, mp4) ?? '', + title + })); + + execFileSync('gh', ['pr', 'edit', number, '--repo', REPO, '--body-file', '-'], { + cwd: UI_DIR, + input: withDemoSection( + pr.body, + demoSection(links, sha, Math.min(links.length, isPortrait(videos[0].mp4) ? 4 : 2)) + ), + stdio: ['pipe', 'inherit', 'inherit'] + }); + console.log(`linked the demo in https://github.com/${REPO}/pull/${number}`); + } finally { + rmSync(script, { force: true }); + run('git', ['worktree', 'remove', '--force', worktree]); + } +} + +async function recordDocs(): Promise { + const storage = getDemoStorage(); + + if (!storage) throw new Error('DEMO_STORAGE is not set: docs demo scripts are read from it'); + + const downloaded = join(RUNS_DIR, 'docs-source'); + + rmSync(downloaded, { force: true, recursive: true }); + + if (!storage.downloadDir('scripts/docs', downloaded)) { + throw new Error('no scripts/docs/*.demo.ts in DEMO_STORAGE'); + } + + const scripts = readdirSync(downloaded) + .filter((f) => f.endsWith('.demo.ts')) + .map((f) => + writeDemoScript(basename(f, '.demo.ts'), readFileSync(join(downloaded, f), 'utf8')) + ); + + try { + const videos = await record(scripts); + + for (const { mp4 } of videos) { + const name = basename(mp4, '.mp4'); + + upload(storage, { + [`videos/docs/${name}.gif`]: makePreview(mp4), + [`videos/docs/${name}.mp4`]: mp4 + }); + } + } finally { + rmSync(RUNS_DIR, { force: true, recursive: true }); + } +} + +/** Writes the built-in traces to demo/traces/ and uploads them to the traces dataset. */ +function writeTraces(): void { + rmSync(TRACES_DIR, { force: true, recursive: true }); + mkdirSync(TRACES_DIR, { recursive: true }); + + for (const name of Object.keys(DEMO_TRACES)) { + writeFileSync(join(TRACES_DIR, `${name}.jsonl`), `${generateTrace(name)}\n`); + console.log(`wrote demo/traces/${name}.jsonl`); + } + + const dataset = process.env.DEMO_TRACES_DATASET; + + if (!dataset) { + console.log('DEMO_TRACES_DATASET is not set: not uploading.'); + + return; + } + + execFileSync( + 'hf', + [ + 'upload', + dataset, + TRACES_DIR, + '.', + '--repo-type', + 'dataset', + '--commit-message', + 'Update demo traces' + ], + { stdio: 'inherit' } + ); +} + +const [command, arg, ...flags] = process.argv.slice(2); +const trustIndex = flags.indexOf('--trust'); + +if (command === 'pr' && arg) { + await recordPr(arg, trustIndex === -1 ? undefined : flags[trustIndex + 1]); +} else if (command === 'docs') { + await recordDocs(); +} else if (command === 'traces') { + writeTraces(); +} else { + console.error('usage: node demo/cli.ts pr [--trust ] | docs | traces'); + process.exit(1); +} diff --git a/tools/ui/demo/fixtures/demo-config.ts b/tools/ui/demo/fixtures/demo-config.ts new file mode 100644 index 000000000000..d4a272f6cdd4 --- /dev/null +++ b/tools/ui/demo/fixtures/demo-config.ts @@ -0,0 +1,49 @@ +/** Shared fixture data for demo recordings (used by both Node and the browser seed). */ + +export const DEMO_MCP_PORT = 8765; + +export interface DemoMcpServer { + id: string; + name: string; + title: string; + tools: { name: string; description: string; result: string }[]; +} + +export const DEMO_MCP_SERVERS: DemoMcpServer[] = [ + { + id: 'weather', + name: 'weather', + title: 'Weather', + tools: [ + { + description: 'Get the weather forecast for a city', + name: 'get_forecast', + result: 'Sunny, 22°C' + }, + { + description: 'List active weather alerts for a region', + name: 'get_alerts', + result: 'No active alerts' + } + ] + }, + { + id: 'notes', + name: 'notes', + title: 'Notes', + tools: [ + { description: 'Search notes by keyword', name: 'search_notes', result: 'No notes found' }, + { description: 'Create a new note', name: 'create_note', result: 'Note created' }, + { description: 'List recent notes', name: 'list_notes', result: 'No notes yet' } + ] + } +]; + +/** `mcpServers` setting value pointing the UI at the fixture servers. */ +export function demoMcpServerSettings(): object[] { + return DEMO_MCP_SERVERS.map((server) => ({ + enabled: true, + id: `demo-${server.id}`, + url: `http://localhost:${DEMO_MCP_PORT}/${server.id}/mcp` + })); +} diff --git a/tools/ui/demo/fixtures/mcp-servers.ts b/tools/ui/demo/fixtures/mcp-servers.ts new file mode 100644 index 000000000000..15529e4a314e --- /dev/null +++ b/tools/ui/demo/fixtures/mcp-servers.ts @@ -0,0 +1,91 @@ +/** + * Fixture MCP servers for demo recordings. + * + * Serves a few stateless Streamable HTTP MCP servers with canned tools, so + * demos show MCP groups without depending on real third-party servers. + * + * Usage: node demo/fixtures/mcp-servers.ts + */ + +import { DEMO_MCP_PORT, DEMO_MCP_SERVERS, type DemoMcpServer } from './demo-config.ts'; +import { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js'; +import { StreamableHTTPServerTransport } from '@modelcontextprotocol/sdk/server/streamableHttp.js'; +import { createServer, type IncomingMessage, type ServerResponse } from 'node:http'; +import { z } from 'zod'; + +function buildServer(fixture: DemoMcpServer): McpServer { + const server = new McpServer({ name: fixture.name, title: fixture.title, version: '1.0.0' }); + + for (const tool of fixture.tools) { + server.registerTool( + tool.name, + { description: tool.description, inputSchema: { query: z.string().optional() } }, + async () => ({ content: [{ text: tool.result, type: 'text' }] }) + ); + } + + return server; +} + +function setCorsHeaders(res: ServerResponse): void { + res.setHeader('Access-Control-Allow-Origin', '*'); + res.setHeader('Access-Control-Allow-Headers', '*'); + res.setHeader('Access-Control-Allow-Methods', 'GET, POST, DELETE, OPTIONS'); + res.setHeader('Access-Control-Expose-Headers', 'mcp-session-id, mcp-protocol-version'); + // the UI runs with COEP require-corp + res.setHeader('Cross-Origin-Resource-Policy', 'cross-origin'); +} + +async function readBody(req: IncomingMessage): Promise { + const chunks: Buffer[] = []; + + for await (const chunk of req) chunks.push(chunk as Buffer); + + const raw = Buffer.concat(chunks).toString(); + + return raw ? JSON.parse(raw) : undefined; +} + +const httpServer = createServer(async (req, res) => { + setCorsHeaders(res); + + if (req.method === 'OPTIONS') { + res.writeHead(204).end(); + + return; + } + + const path = new URL(req.url ?? '/', 'http://localhost').pathname; + const fixture = DEMO_MCP_SERVERS.find((s) => path === `/${s.id}/mcp`); + + if (!fixture) { + res.writeHead(path === '/health' ? 200 : 404).end(); + + return; + } + + if (req.method !== 'POST') { + // stateless: no standalone SSE stream or session teardown + res.writeHead(405).end(); + + return; + } + + const server = buildServer(fixture); + const transport = new StreamableHTTPServerTransport({ + enableJsonResponse: true, + sessionIdGenerator: undefined + }); + + res.on('close', () => { + transport.close(); + server.close(); + }); + + await server.connect(transport); + await transport.handleRequest(req, res, await readBody(req)); +}); + +httpServer.listen(DEMO_MCP_PORT, () => { + console.log(`demo MCP servers listening on http://localhost:${DEMO_MCP_PORT}`); +}); diff --git a/tools/ui/demo/fixtures/traces.ts b/tools/ui/demo/fixtures/traces.ts new file mode 100644 index 000000000000..cc7ed8979a95 --- /dev/null +++ b/tools/ui/demo/fixtures/traces.ts @@ -0,0 +1,312 @@ +/** + * Synthetic conversations for demos, in the UI's export format (llama.app JSONL + * traces): a `session` record followed by one `message` record per message. + * Deterministic, so a trace and the demos recorded from it don't drift. Messages + * carry no model, so the UI doesn't flag one the server doesn't have. + */ + +import { createHash } from 'node:crypto'; + +const HARNESS = 'llama.app'; +const START = Date.UTC(2026, 9, 1, 9, 0, 0); + +interface ToolCall { + function: { arguments: string; name: string }; + id: string; + type: 'function'; +} + +interface TraceMessage { + children: string[]; + content: string; + convId: string; + id: string; + parent: null | string; + reasoningContent?: string; + role: 'assistant' | 'system' | 'tool' | 'user'; + timestamp: number; + toolCallId?: string; + toolCalls?: ToolCall[]; + type: 'root' | 'text'; +} + +/** Builds one linear conversation, message by message. */ +class Trace { + private calls = 0; + private readonly id: string; + private readonly messages: TraceMessage[] = []; + private readonly name: string; + + assistant(content: string, reasoning?: string): this { + return this.push({ content, reasoningContent: reasoning, role: 'assistant' }); + } + + constructor(name: string) { + this.name = name; + this.id = uuid(name, 'conv'); + this.push({ content: '', role: 'system', type: 'root' }); + } + + toJsonl(): string { + const last = this.messages.at(-1)!; + const session = { + currNode: last.id, + harness: HARNESS, + id: this.id, + lastModified: last.timestamp, + name: this.name, + type: 'session' + }; + + return [session, ...this.messages.map((message) => ({ message, type: 'message' }))] + .map((record) => JSON.stringify(record)) + .join('\n'); + } + + /** An assistant turn calling one tool, followed by the tool's result. */ + tool(name: string, args: Record, result: string, reasoning?: string): this { + const id = uuid(this.name, `call-${this.calls++}`).replaceAll('-', ''); + + this.push({ + content: '', + reasoningContent: reasoning, + role: 'assistant', + toolCalls: [{ function: { arguments: JSON.stringify(args), name }, id, type: 'function' }] + }); + + return this.push({ content: result, role: 'tool', toolCallId: id }); + } + + user(content: string): this { + return this.push({ content, role: 'user' }); + } + + private push(message: Partial & Pick): this { + const parent = this.messages.at(-1); + const index = this.messages.length; + const entry: TraceMessage = { + children: [], + convId: this.id, + id: uuid(this.name, `msg-${index}`), + parent: parent?.id ?? null, + timestamp: START + index * 20_000, + type: 'text', + ...message + }; + + parent?.children.push(entry.id); + this.messages.push(entry); + + return this; + } +} + +function uuid(...parts: string[]): string { + const hex = createHash('sha1').update(parts.join(':')).digest('hex'); + + return `${hex.slice(0, 8)}-${hex.slice(8, 12)}-4${hex.slice(13, 16)}-a${hex.slice(17, 20)}-${hex.slice(20, 32)}`; +} + +const STEPS = [ + ['set up the project', 'cargo new forecast && cd forecast', 'Cargo.toml', 'toml'], + ['parse command line arguments', 'cargo add clap --features derive', 'src/cli.rs', 'rust'], + ['call the weather API', 'cargo add reqwest --features json,blocking', 'src/api.rs', 'rust'], + ['deserialize the response', 'cargo add serde --features derive', 'src/model.rs', 'rust'], + ['cache responses on disk', 'cargo add directories', 'src/cache.rs', 'rust'], + ['print a table', 'cargo add comfy-table', 'src/table.rs', 'rust'], + ['handle errors', 'cargo add anyhow', 'src/main.rs', 'rust'], + ['add colors', 'cargo add owo-colors', 'src/style.rs', 'rust'], + ['read a config file', 'cargo add toml', 'src/config.rs', 'rust'], + ['write tests', 'cargo add --dev assert_cmd predicates', 'tests/cli.rs', 'rust'], + ['set up CI', 'mkdir -p .github/workflows', '.github/workflows/ci.yml', 'yaml'], + ['publish a release', 'cargo install cargo-dist', 'dist-workspace.toml', 'toml'] +] as const; + +function codeFor(step: number, file: string, lang: string): string { + const lines = + lang === 'rust' + ? [ + `// ${file}`, + 'use anyhow::{Context, Result};', + '', + `pub fn step_${step}(input: &str) -> Result> {`, + ' let mut out = Vec::new();', + ' for (i, line) in input.lines().enumerate() {', + ' let trimmed = line.trim();', + ' if trimmed.is_empty() {', + ' continue;', + ' }', + ' out.push(format!("{i:>3}: {trimmed}"));', + ' }', + ` Ok(out).context("step ${step} failed")`, + '}' + ] + : lang === 'yaml' + ? [ + `# ${file}`, + 'on: [push, pull_request]', + 'jobs:', + ' test:', + ' runs-on: ubuntu-latest' + ] + : [`# ${file}`, `name = "forecast"`, `version = "0.${step}.0"`, 'edition = "2024"']; + + return `\`\`\`${lang}\n${lines.join('\n')}\n\`\`\``; +} + +/** A long chat with tall messages, for scrolling and landing demos. */ +function longChat(): Trace { + const trace = new Trace('Building a weather CLI in Rust'); + + for (let i = 0; i < 36; i++) { + const [goal, command, file, lang] = STEPS[i % STEPS.length]; + const round = Math.floor(i / STEPS.length); + const ask = + round === 0 ? `How do I ${goal}?` : `Can we revisit how we ${goal}? (pass ${round + 1})`; + + trace.user(ask); + trace.assistant( + [ + `To ${goal}, start from a clean working tree so the change stays easy to review.`, + '', + '```sh', + command, + '```', + '', + `Then add \`${file}\`:`, + '', + codeFor(i + 1, file, lang), + '', + 'A few things to keep in mind:', + '', + `- Keep \`${file}\` focused on one job, so the next step only touches what it needs.`, + '- Run `cargo clippy` after each step; it catches most mistakes early.', + '- Commit once it builds, so you can always go back to a working version.', + '', + i % 3 === 0 + ? `Next we can look at how to ${STEPS[(i + 1) % STEPS.length][0]}.` + : 'Let me know if anything here does not build.' + ].join('\n'), + `The user wants to ${goal}. Give the command, the file and a short checklist.` + ); + } + + return trace; +} + +function testLog(failing: boolean): string { + const lines = ['running 48 tests']; + + for (let i = 1; i <= 48; i++) { + const ok = !failing || (i !== 17 && i !== 31); + + lines.push( + `test cache::tests::entry_${String(i).padStart(2, '0')} ... ${ok ? 'ok' : 'FAILED'}` + ); + } + + if (failing) { + lines.push( + '', + 'failures:', + '', + '---- cache::tests::entry_17 stdout ----', + "thread 'cache::tests::entry_17' panicked at src/cache.rs:88:9:", + 'assertion `left == right` failed: entry expired too early', + ' left: Some(1759312800)', + ' right: Some(1759316400)', + '', + '---- cache::tests::entry_31 stdout ----', + "thread 'cache::tests::entry_31' panicked at src/cache.rs:88:9:", + 'assertion `left == right` failed: entry expired too early', + '', + 'test result: FAILED. 46 passed; 2 failed; 0 ignored; 0 measured; 0 filtered out' + ); + } else { + lines.push('', 'test result: ok. 48 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out'); + } + + return `${lines.join('\n')}\n\n[exit code: ${failing ? 101 : 0}]`; +} + +function reasoning(topic: string, paragraphs: number): string { + const out: string[] = []; + + for (let i = 0; i < paragraphs; i++) { + out.push( + `${i === 0 ? `Looking at ${topic}.` : 'Another angle:'} The cache stores an expiry timestamp per entry, ` + + 'and the failing tests are the ones created right at an hour boundary. If the expiry is computed ' + + 'from local time while the check uses UTC, entries created near the boundary would expire an hour ' + + `early. Step ${i + 1}: confirm by reading how \`expires_at\` is built, then compare with \`is_fresh\`.` + ); + } + + return out.join('\n\n'); +} + +/** An agentic session with long reasoning, shell output and code, for nested scroller demos. */ +function agenticDebugging(): Trace { + const trace = new Trace('Fix the flaky cache tests'); + const fixed = Array.from({ length: 60 }, (_, i) => + i === 0 + ? 'use chrono::{DateTime, Duration, Utc};' + : i === 30 + ? ' let expires_at = Utc::now() + self.ttl; // was Local::now()' + : ` // cache.rs line ${i + 1}: entry handling stays the same` + ).join('\n'); + + trace.user('The cache tests fail on CI every few runs. Can you find out why and fix it?'); + + for (let round = 0; round < 3; round++) { + trace.tool( + 'exec_shell_command', + { command: 'cargo test cache -- --test-threads=1' }, + testLog(true), + reasoning('the failing cache tests', 4 + round) + ); + trace.tool( + 'exec_shell_command', + { command: "grep -n 'expires_at\\|is_fresh' src/cache.rs" }, + Array.from( + { length: 30 }, + (_, i) => `${80 + i}: let expires_at = Local::now() + self.ttl; // entry ${i}` + ).join('\n') + '\n\n[exit code: 0]', + reasoning('where the expiry is computed', 3) + ); + trace.assistant( + [ + `Round ${round + 1}: the expiry uses local time while the freshness check uses UTC.`, + '', + '```rust', + fixed, + '```', + '', + 'I switched the expiry to `Utc::now()` so both sides use the same clock.' + ].join('\n'), + reasoning('the fix', 5) + ); + trace.user(round < 2 ? 'Still flaky on CI, can you check again?' : 'Run the tests once more.'); + } + + trace.tool( + 'exec_shell_command', + { command: 'cargo test cache' }, + testLog(false), + reasoning('the final test run', 2) + ); + trace.assistant( + 'All 48 cache tests pass now, including the two that failed at the hour boundary.' + ); + + return trace; +} + +/** Built-in traces, by name. */ +export const DEMO_TRACES: Record Trace> = { + 'agentic-debugging': agenticDebugging, + 'long-chat': longChat +}; + +export function generateTrace(name: string): null | string { + return DEMO_TRACES[name]?.().toJsonl() ?? null; +} diff --git a/tools/ui/demo/helpers.ts b/tools/ui/demo/helpers.ts new file mode 100644 index 000000000000..1c0ad7734f4e --- /dev/null +++ b/tools/ui/demo/helpers.ts @@ -0,0 +1,511 @@ +import { demoMcpServerSettings } from './fixtures/demo-config.ts'; +import { generateTrace } from './fixtures/traces.ts'; +import { ScreencastRecorder } from './recorder.ts'; +import { type CDPSession, type Locator, type Page, test as base } from '@playwright/test'; +import { execFileSync } from 'node:child_process'; +import { appendFileSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs'; +import { basename, join, resolve } from 'node:path'; + +const VIDEOS_DIR = resolve(import.meta.dirname, 'videos'); +/** How long a demo waits for the app to be ready before it starts recording. */ +const APP_READY_TIMEOUT_MS = 90000; +/** How far below the viewport's top a pinch has to start for Chromium to zoom at all. */ +const ZOOM_TOP_INSET_PX = 64; + +/** + * Draws a fake cursor and click ripples, since recorded videos don't include + * the OS pointer, and a fingertip per touch, so a pinch shows as two fingers. + * Runs in the page via addInitScript. + */ +function installCursorOverlay(): void { + const mount = () => { + const style = document.createElement('style'); + + style.textContent = ` + #demo-cursor { position: fixed; top: 0; left: 0; z-index: 2147483647; pointer-events: none; + width: 22px; height: 22px; margin: -3px 0 0 -3px; transition: transform 80ms ease-out; } + #demo-cursor.down { transform: scale(0.85); } + .demo-ripple { position: fixed; z-index: 2147483646; pointer-events: none; width: 36px; height: 36px; + margin: -18px 0 0 -18px; border-radius: 50%; background: rgba(59, 130, 246, 0.35); + animation: demo-ripple 450ms ease-out forwards; } + @keyframes demo-ripple { from { transform: scale(0.3); opacity: 1; } to { transform: scale(1.4); opacity: 0; } } + #demo-caption { position: fixed; left: 50%; top: 72px; z-index: 2147483647; pointer-events: none; + transform: translate(-50%, -8px); opacity: 0; transition: opacity 200ms ease, transform 200ms ease; + max-width: 80%; padding: 10px 18px; border-radius: 10px; background: rgba(17, 17, 17, 0.88); + color: #fff; font: 500 18px/1.4 system-ui, sans-serif; text-align: center; } + #demo-caption.visible { opacity: 1; transform: translate(-50%, 0); } + .demo-finger { position: fixed; z-index: 2147483646; pointer-events: none; width: 40px; height: 40px; + margin: -20px 0 0 -20px; border-radius: 50%; background: rgba(59, 130, 246, 0.3); + border: 2px solid rgba(59, 130, 246, 0.55); transition: opacity 200ms ease; } + `; + document.head.appendChild(style); + + const cursor = document.createElement('div'); + + cursor.id = 'demo-cursor'; + cursor.innerHTML = ``; + document.documentElement.appendChild(cursor); + + const caption = document.createElement('div'); + + caption.id = 'demo-caption'; + document.documentElement.appendChild(caption); + + document.addEventListener( + 'mousemove', + (e) => (cursor.style.translate = `${e.clientX}px ${e.clientY}px`), + true + ); + document.addEventListener( + 'mousedown', + (e) => { + cursor.classList.add('down'); + const ripple = document.createElement('div'); + + ripple.className = 'demo-ripple'; + ripple.style.left = `${e.clientX}px`; + ripple.style.top = `${e.clientY}px`; + document.documentElement.appendChild(ripple); + setTimeout(() => ripple.remove(), 500); + }, + true + ); + document.addEventListener('mouseup', () => cursor.classList.remove('down'), true); + + const fingers: HTMLDivElement[] = []; + const placeFingers = (e: TouchEvent) => { + Array.from(e.touches).forEach((touch, i) => { + const finger = (fingers[i] ??= Object.assign(document.createElement('div'), { + className: 'demo-finger' + })); + + finger.style.left = `${touch.clientX}px`; + finger.style.top = `${touch.clientY}px`; + finger.style.opacity = '1'; + document.documentElement.appendChild(finger); + }); + fingers.slice(e.touches.length).forEach((finger) => (finger.style.opacity = '0')); + }; + + for (const type of ['touchstart', 'touchmove', 'touchend', 'touchcancel']) { + document.addEventListener(type, placeFingers as EventListener, { + capture: true, + passive: true + }); + } + }; + + if (document.readyState === 'loading') { + document.addEventListener('DOMContentLoaded', mount); + } else { + mount(); + } +} + +/** Seeds UI settings once per demo, so later reloads keep what the demo changed. */ +function seedSettings(config: Record): void { + const key = 'LlamaUi.config'; + + if (localStorage.getItem(key) === null) { + localStorage.setItem(key, JSON.stringify(config)); + } +} + +/** Seeded randomness, so a demo moves the same way on every recording. */ +function createRandom(seed: string): () => number { + let state = [...seed].reduce((h, c) => Math.imul(h ^ c.charCodeAt(0), 2654435761), 1); + + return () => { + state = (state + 0x6d2b79f5) | 0; + + let t = Math.imul(state ^ (state >>> 15), 1 | state); + + t = (t + Math.imul(t ^ (t >>> 7), 61 | t)) ^ t; + + return ((t ^ (t >>> 14)) >>> 0) / 4294967296; + }; +} + +const easeInOut = (t: number) => (t < 0.5 ? 4 * t * t * t : 1 - (-2 * t + 2) ** 3 / 2); + +/** + * Drives the page like a person: the cursor glides along a slight arc with + * Fitts-like durations, settles briefly before pressing, and typing and pauses + * vary a little. + */ +export class Demo { + private cdp: CDPSession | undefined; + private readonly page: Page; + private readonly random: () => number; + private x = 0; + private y = 0; + + /** Shows a caption at the top of the frame for `ms`, clear of the chat form. */ + async caption(text: string, ms = 2200): Promise { + await this.page.evaluate((t) => { + const el = document.getElementById('demo-caption'); + + if (!el) return; + + el.textContent = t; + el.classList.add('visible'); + }, text); + await this.page.waitForTimeout(ms); + await this.page.evaluate(() => + document.getElementById('demo-caption')?.classList.remove('visible') + ); + await this.page.waitForTimeout(250); + } + + async click(target: Locator, ms = 400): Promise { + await this.moveTo(target); + await this.page.waitForTimeout(this.between(80, 160)); + await this.page.mouse.down(); + await this.page.waitForTimeout(this.between(50, 90)); + await this.page.mouse.up(); + await this.pause(ms); + } + + constructor(page: Page, seed: string) { + this.page = page; + this.random = createRandom(seed); + + const viewport = page.viewportSize(); + + this.x = (viewport?.width ?? 0) / 2; + this.y = (viewport?.height ?? 0) / 2; + } + + async hover(target: Locator, ms = 350): Promise { + await this.moveTo(target); + await this.pause(ms); + } + + /** Glides the cursor to a point near the center of `target`. */ + async moveTo(target: Locator): Promise { + await target.scrollIntoViewIfNeeded(); + const box = await target.boundingBox(); + + if (!box) throw new Error(`demo: ${target} has no bounding box`); + + const x = box.x + box.width * this.between(0.4, 0.6); + const y = box.y + box.height * this.between(0.4, 0.6); + const dx = x - this.x; + const dy = y - this.y; + const distance = Math.hypot(dx, dy); + const duration = 110 + 55 * Math.log2(distance / 40 + 1); + // bow the path a little to one side, like a wrist turning + const bow = distance * this.between(-0.08, 0.08); + const fromX = this.x; + const fromY = this.y; + const started = Date.now(); + + for (;;) { + const t = Math.min((Date.now() - started) / duration, 1); + const p = easeInOut(t); + const arc = Math.sin(Math.PI * p) * bow; + + this.x = fromX + dx * p - (dy / (distance || 1)) * arc; + this.y = fromY + dy * p + (dx / (distance || 1)) * arc; + await this.page.mouse.move(this.x, this.y); + + if (t === 1) break; + + await this.page.waitForTimeout(8); + } + } + + /** Waits about `ms`, varied by up to 15% so pauses don't feel metronomic. */ + async pause(ms = 800): Promise { + await this.page.waitForTimeout(ms * this.between(0.85, 1.15)); + } + + async press(key: string, ms = 600): Promise { + await this.page.keyboard.press(key); + await this.pause(ms); + } + + /** Scrolls the wheel over `target` by `dy` pixels (negative scrolls up), in small steps like a trackpad. */ + async scroll(target: Locator, dy: number, ms = 500): Promise { + await this.moveTo(target); + + const step = Math.sign(dy) * 40; + + for (let done = 0; Math.abs(done) < Math.abs(dy); done += step) { + await this.page.mouse.wheel(0, step); + await this.page.waitForTimeout(16); + } + + await this.pause(ms); + } + + /** Types quickly with uneven key timing, a little slower after spaces and punctuation. */ + async type(target: Locator, text: string, ms = 500): Promise { + await this.click(target, 200); + + for (const char of text) { + await this.page.keyboard.type(char); + await this.page.waitForTimeout( + this.between(15, 45) + (/[\s.,!?]/.test(char) ? this.between(20, 60) : 0) + ); + } + + await this.pause(ms); + } + + /** + * Pinches in on `target`, a control or a point in the viewport, holds about `ms`, and pinches + * back out to 1x: a two-finger zoom onto a detail too small to read in the frame. On a touch + * device the fingertips show. + */ + async zoom(target: Locator | { x: number; y: number }, scale = 2.5, ms = 1400): Promise { + const point = 'x' in target ? target : await this.center(target); + // a pinch anchored in the viewport's top band does not zoom at all + const at = { x: point.x, y: Math.max(point.y, ZOOM_TOP_INSET_PX) }; + const viewport = this.page.viewportSize() ?? { height: 0, width: 0 }; + const middle = { x: viewport.width / 2, y: viewport.height / 2 }; + + await this.pinch(at, scale, 1400); + + // One near the bottom edge overshoots its ratio, and a pinch there again misses too. A small + // pinch from the middle evens it out, drawing the detail toward the middle, so it stays in view + const reached = await this.pageScale(); + + if (Math.abs(reached / scale - 1) > 0.03) await this.pinch(middle, scale / reached, 1400); + + await this.liftFingers(); + await this.pause(ms); + + // back to exactly 1x, which the next gesture needs, in one quick pinch from the middle of + // the viewport, and another for what is left when one lands short + for (let tries = 0; ; tries++) { + const current = await this.pageScale(); + + if (current <= 1.001) break; + + if (tries === 4) throw new Error(`demo: zoom stuck at ${current.toFixed(3)}x`); + + await this.pinch(middle, 1 / current, 1800); + } + + await this.liftFingers(); + await this.pause(200); + } + + private between(min: number, max: number): number { + return min + (max - min) * this.random(); + } + + private async center(target: Locator): Promise<{ x: number; y: number }> { + const box = await target.boundingBox(); + + if (!box) throw new Error(`demo: ${target} has no bounding box`); + + return { x: box.x + box.width / 2, y: box.y + box.height / 2 }; + } + + /** A pinch that reaches the screen's edge can end without its last touchend reaching the page. */ + private async liftFingers(): Promise { + await this.page.evaluate(() => + document + .querySelectorAll('.demo-finger') + .forEach((finger) => (finger.style.opacity = '0')) + ); + } + + private pageScale(): Promise { + return this.page.evaluate(() => window.visualViewport?.scale ?? 1); + } + + /** + * A two-finger pinch by `factor` around `at`, in viewport pixels. While zoomed in, the gesture + * reads its position in page-scale units, so it is divided by the scale the page is at. + */ + private async pinch(at: { x: number; y: number }, factor: number, speed: number): Promise { + const cdp = await this.session(); + const current = await this.pageScale(); + + await cdp.send('Input.synthesizePinchGesture', { + gestureSourceType: 'touch', + relativeSpeed: speed, + scaleFactor: factor, + x: at.x / current, + y: at.y / current + }); + } + + private async session(): Promise { + this.cdp ??= await this.page.context().newCDPSession(this.page); + + return this.cdp; + } +} + +interface DemoOptions { + /** Path the demo starts from. */ + startPath: string; + /** Extra UI settings merged into the seeded config. */ + settings: Record; + /** + * Conversations imported before the demo starts: built-in traces from + * fixtures/traces.ts, or `.jsonl` files from DEMO_TRACES_DATASET. + */ + traces: string[]; +} + +/** Reads a trace by name, generating built-in ones and downloading the rest. */ +function loadTrace(name: string, dir: string): string { + const generated = generateTrace(name); + + if (generated) return generated; + + const dataset = process.env.DEMO_TRACES_DATASET; + + if (!dataset) + throw new Error(`demo: no built-in trace "${name}" and DEMO_TRACES_DATASET is not set`); + + execFileSync( + 'hf', + ['download', dataset, `${name}.jsonl`, '--repo-type', 'dataset', '--local-dir', dir], + { stdio: 'inherit' } + ); + + return readFileSync(join(dir, `${name}.jsonl`), 'utf8'); +} + +/** Imports conversations through Settings > Import/Export, like a user would. */ +async function importTraces(page: Page, file: string): Promise { + // the expanded sidebar adds the shortcut to the name + const settings = page.getByRole('button', { name: /^Settings(\s|$)/ }).first(); + const reachable = (target: Locator) => + target.click({ timeout: 2000, trial: true }).then( + () => true, + () => false + ); + // a collapsed sidebar can sit under the page: on a phone the page's menu key slides it aside, + // on a desktop its logo expands it + const keys = [ + page.locator('xpath=//button[@aria-label="Expand navigation"][not(ancestor::aside)]'), + page.locator('aside button[aria-label="Expand navigation"]') + ]; + + // a few rounds, as a cold dev server reloads the page once it has optimized its dependencies, + // which can land right after a key opened the sidebar and close it again + for (let round = 0; round < 3 && !(await reachable(settings)); round++) { + for (const key of keys) { + if (await reachable(settings)) break; + + if ((await key.count()) > 0 && (await reachable(key.first()))) await key.first().click(); + } + } + + await settings.click(); + + // a dialog on a desktop or tablet, a page of its own on a phone + const settingsView = page + .getByRole('dialog') + .or(page.getByRole('heading', { exact: true, level: 1, name: 'Settings' })); + + await settingsView.first().waitFor(); + + const scope = (await page.getByRole('dialog').count()) > 0 ? page.getByRole('dialog') : page; + + await scope.getByText('Import/Export', { exact: true }).filter({ visible: true }).click(); + + const chooser = page.waitForEvent('filechooser'); + + await scope.getByRole('button', { name: 'Import conversations' }).click(); + await (await chooser).setFiles(file); + await page.getByRole('button', { name: /^Import \(\d+\)$/ }).click(); + await page.getByRole('heading', { name: /^Imported \d+ conversation/ }).waitFor(); + + // the app remembers the sidebar the import expanded; a demo starts with it collapsed, as on a + // first visit + await page.evaluate(() => { + for (const key of Object.keys(localStorage)) { + if (key.endsWith('.sidebarExpanded')) localStorage.removeItem(key); + } + }); +} + +export const test = base.extend({ + demo: async ({ page }, use, testInfo) => { + await use(new Demo(page, `${testInfo.file}:${testInfo.title}`)); + }, + page: async ({ page, settings, startPath, traces }, use, testInfo) => { + await page.addInitScript(installCursorOverlay); + await page.addInitScript(seedSettings, { + mcpServers: JSON.stringify(demoMcpServerSettings()), + ...settings + }); + + if (traces.length > 0) { + const dir = testInfo.outputPath('traces'); + const file = join(dir, 'import.jsonl'); + + mkdirSync(dir, { recursive: true }); + writeFileSync(file, traces.map((name) => loadTrace(name, dir)).join('\n')); + await page.goto('/'); + await page + .getByRole('textbox', { name: 'Type a message...' }) + .waitFor({ timeout: APP_READY_TIMEOUT_MS }); + await importTraces(page, file); + } + + await page.goto(startPath); + + // a hash route after the import doesn't reload, which the conversations list needs + if (traces.length > 0) await page.reload(); + + // a fresh checkout's dev server optimizes its dependencies on the first load and reloads the + // page, which can take well past an action's timeout; the recording has not started yet + await page + .getByRole('textbox', { name: 'Type a message...' }) + .waitFor({ timeout: APP_READY_TIMEOUT_MS }); + + const viewport = page.viewportSize(); + + if (viewport) await page.mouse.move(viewport.width / 2, viewport.height / 2); + + // let MCP connections and fonts settle before the recording starts + await page.waitForTimeout(1500); + + const recorder = new ScreencastRecorder(testInfo.outputPath('frames')); + + await recorder.start(page); + await use(page); + await recorder.stop(); + await page.close(); + + if (testInfo.status !== testInfo.expectedStatus) return; + + const file = basename(testInfo.file).replace(/\.demo\.ts$/, ''); + const name = testInfo.title === 'demo' ? file : `${file}--${slugify(testInfo.title)}`; + const mp4 = resolve(VIDEOS_DIR, `${name}.mp4`); + + mkdirSync(VIDEOS_DIR, { recursive: true }); + recorder.encode(mp4); + // the CLI reads which videos a script made, in order, with their titles + appendFileSync( + resolve(VIDEOS_DIR, `${file}.videos.jsonl`), + `${JSON.stringify({ mp4, title: testInfo.title })}\n` + ); + console.log(`demo video: ${mp4}`); + }, + + settings: [{}, { option: true }], + + startPath: ['/', { option: true }], + + traces: [[], { option: true }] +}); + +export { expect } from '@playwright/test'; + +function slugify(text: string): string { + return text + .toLowerCase() + .replace(/[^a-z0-9]+/g, '-') + .replace(/^-|-$/g, ''); +} diff --git a/tools/ui/demo/playwright.config.ts b/tools/ui/demo/playwright.config.ts new file mode 100644 index 000000000000..2ac9c4c4bd24 --- /dev/null +++ b/tools/ui/demo/playwright.config.ts @@ -0,0 +1,87 @@ +import { DEMO_MCP_PORT } from './fixtures/demo-config.ts'; +import { defineConfig, devices } from '@playwright/test'; +import { existsSync } from 'node:fs'; +import { resolve } from 'node:path'; + +const ENV_FILE = resolve(import.meta.dirname, '../.env'); + +if (existsSync(ENV_FILE)) process.loadEnvFile(ENV_FILE); + +/** + * Demo video recordings. + * + * Env (also read from tools/ui/.env): + * DEMO_UI_DIR tools/ui checkout to record (default: this one), e.g. a PR worktree + * DEMO_UI_PORT port for that checkout's vite dev server (default: 5180) + * DEMO_SERVER_ORIGIN llama-server the UI proxies to (default: http://localhost:9931) + * DEMO_LLAMA_SERVER llama-server binary started when none runs there (default: build/bin) + * DEMO_COLOR_SCHEME light | dark (default: light) + */ + +// Playwright clears outputDir when a run starts, so concurrent recordings each get +// their own; workers inherit the env, so they resolve the same directory +process.env.DEMO_RUN_ID ??= String(process.pid); + +const UI_DIR = resolve(process.env.DEMO_UI_DIR ?? resolve(import.meta.dirname, '..')); +const UI_PORT = Number(process.env.DEMO_UI_PORT ?? 5180); +const SERVER_ORIGIN = process.env.DEMO_SERVER_ORIGIN ?? 'http://localhost:9931'; +const SERVER_PORT = Number(new URL(SERVER_ORIGIN).port || 80); +const LLAMA_SERVER = + process.env.DEMO_LLAMA_SERVER ?? resolve(import.meta.dirname, '../../../build/bin/llama-server'); + +export const VIEWPORT = { height: 900, width: 1440 }; +/** Device pixel ratio the demos render and record at. */ +const SCALE = Number(process.env.DEMO_SCALE ?? 2); + +export default defineConfig({ + expect: { + timeout: 10000 + }, + fullyParallel: false, + outputDir: `test-results/${process.env.DEMO_RUN_ID}`, + projects: [ + { + name: 'chromium', + use: { ...devices['Desktop Chrome'], deviceScaleFactor: SCALE, viewport: VIEWPORT } + } + ], + reporter: 'line', + retries: 0, + testDir: '.', + testMatch: ['**/*.demo.ts'], + timeout: 120000, + use: { + actionTimeout: 10000, + baseURL: `http://localhost:${UI_PORT}`, + colorScheme: process.env.DEMO_COLOR_SCHEME === 'dark' ? 'dark' : 'light', + // headless Chromium screencasts at 1x unless the scale is forced at launch; the sandbox + // is on since a PR demo runs that PR's code (Playwright turns it off by default). Not the + // installed Chrome: its headless mode screencasts a window that ignores the viewport + launchOptions: { args: [`--force-device-scale-factor=${SCALE}`], chromiumSandbox: true }, + // recorded through the screencast in recorder.ts, at the device pixel ratio + video: 'off', + viewport: VIEWPORT + }, + webServer: [ + { + command: `node ${resolve(import.meta.dirname, 'fixtures/mcp-servers.ts')}`, + reuseExistingServer: true, + url: `http://localhost:${DEMO_MCP_PORT}/health` + }, + { + command: `${LLAMA_SERVER} --host 127.0.0.1 --port ${SERVER_PORT} --tools all`, + reuseExistingServer: true, + timeout: 60000, + url: `${SERVER_ORIGIN}/health` + }, + { + command: `npx vite dev --port ${UI_PORT} --strictPort`, + cwd: UI_DIR, + env: { VITE_PUBLIC_SERVER_ORIGIN: SERVER_ORIGIN }, + reuseExistingServer: true, + timeout: 120000, + url: `http://localhost:${UI_PORT}` + } + ], + workers: 1 +}); diff --git a/tools/ui/demo/recorder.ts b/tools/ui/demo/recorder.ts new file mode 100644 index 000000000000..3d1faa9a4765 --- /dev/null +++ b/tools/ui/demo/recorder.ts @@ -0,0 +1,109 @@ +/** + * Records a page through Chromium's screencast instead of Playwright's video, + * which encodes VP8 at about 1 Mbit/s at 1x scale. Frames arrive at the page's + * device pixel ratio, each with its timestamp, and are encoded once into an + * H.264 mp4 (what GitHub accepts in PR descriptions). + */ + +import type { CDPSession, Page } from '@playwright/test'; +import { execFileSync } from 'node:child_process'; +import { mkdirSync, rmSync, writeFileSync } from 'node:fs'; +import { writeFile } from 'node:fs/promises'; +import { join } from 'node:path'; + +const FPS = 60; +const JPEG_QUALITY = 92; + +interface Frame { + file: string; + /** Seconds, on the same clock as Date.now(). */ + time: number; +} + +export class ScreencastRecorder { + private readonly dir: string; + private readonly frames: Frame[] = []; + private readonly pending: Promise[] = []; + private session: CDPSession | undefined; + private stoppedAt = 0; + + constructor(dir: string) { + this.dir = dir; + } + + /** Encodes the frames, each held until the next one arrived (the screencast only sends changes). */ + encode(mp4: string): void { + if (this.frames.length === 0) throw new Error('demo: the screencast captured no frames'); + + const list = this.frames.flatMap((frame, i) => { + const end = this.frames[i + 1]?.time ?? this.stoppedAt; + + return [`file '${frame.file}'`, `duration ${Math.max(end - frame.time, 1 / FPS).toFixed(4)}`]; + }); + const listFile = join(this.dir, 'frames.txt'); + + // the concat demuxer ignores the last entry's duration unless the file is repeated + writeFileSync(listFile, [...list, `file '${this.frames.at(-1)!.file}'`, ''].join('\n')); + execFileSync( + 'ffmpeg', + [ + '-y', + '-loglevel', + 'error', + '-f', + 'concat', + '-safe', + '0', + '-i', + listFile, + '-vf', + `fps=${FPS},scale=trunc(iw/2)*2:trunc(ih/2)*2`, + '-c:v', + 'libx264', + '-preset', + 'slow', + '-crf', + '18', + '-pix_fmt', + 'yuv420p', + '-movflags', + '+faststart', + mp4 + ], + { stdio: 'inherit' } + ); + rmSync(this.dir, { force: true, recursive: true }); + } + + async start(page: Page): Promise { + const size = page.viewportSize(); + const scale = await page.evaluate(() => window.devicePixelRatio); + + rmSync(this.dir, { force: true, recursive: true }); + mkdirSync(this.dir, { recursive: true }); + + this.session = await page.context().newCDPSession(page); + this.session.on('Page.screencastFrame', ({ data, metadata, sessionId }) => { + const file = join(this.dir, `${String(this.frames.length).padStart(6, '0')}.jpg`); + + // acknowledged first: Chromium sends the next frame only after the ack + this.session?.send('Page.screencastFrameAck', { sessionId }).catch(() => {}); + this.frames.push({ file, time: metadata.timestamp ?? Date.now() / 1000 }); + this.pending.push(writeFile(file, Buffer.from(data, 'base64'))); + }); + await this.session.send('Page.startScreencast', { + everyNthFrame: 1, + format: 'jpeg', + maxHeight: size ? size.height * scale : undefined, + maxWidth: size ? size.width * scale : undefined, + quality: JPEG_QUALITY + }); + } + + async stop(): Promise { + this.stoppedAt = Date.now() / 1000; + await this.session?.send('Page.stopScreencast').catch(() => {}); + await this.session?.detach().catch(() => {}); + await Promise.all(this.pending); + } +} diff --git a/tools/ui/demo/storage.ts b/tools/ui/demo/storage.ts new file mode 100644 index 000000000000..def006d7b8df --- /dev/null +++ b/tools/ui/demo/storage.ts @@ -0,0 +1,92 @@ +/** + * Storage targets for demo scripts and videos, picked by the scheme of the + * DEMO_STORAGE URI: + * + * hf://buckets//[/prefix] Hugging Face bucket, via the `hf` CLI login + * file:///some/dir local or mounted directory + */ + +import { execFileSync } from 'node:child_process'; +import { copyFileSync, cpSync, existsSync, mkdirSync } from 'node:fs'; +import { dirname, join } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +export interface DemoStorage { + /** Copies every file under remote `dir` into local `target`; returns false when it has none. */ + downloadDir(dir: string, target: string): boolean; + /** Copies local `file` to remote `key`. */ + upload(file: string, key: string): void; +} + +function joinKey(base: string, key: string): string { + return `${base.replace(/\/+$/, '')}/${key}`; +} + +// plain fields: the CLI runs under Node's type stripping, which has no parameter properties +class HfStorage implements DemoStorage { + private readonly base: string; + + constructor(base: string) { + this.base = base; + } + + downloadDir(dir: string, target: string): boolean { + mkdirSync(target, { recursive: true }); + + try { + execFileSync('hf', ['buckets', 'sync', joinKey(this.base, dir), target], { + stdio: 'inherit' + }); + } catch { + return false; + } + + return true; + } + + upload(file: string, key: string): void { + execFileSync('hf', ['buckets', 'cp', file, joinKey(this.base, key)], { stdio: 'inherit' }); + } +} + +class FileStorage implements DemoStorage { + private readonly root: string; + + constructor(root: string) { + this.root = root; + } + + downloadDir(dir: string, target: string): boolean { + const source = join(this.root, dir); + + if (!existsSync(source)) return false; + + cpSync(source, target, { recursive: true }); + + return true; + } + + upload(file: string, key: string): void { + const target = join(this.root, key); + + mkdirSync(dirname(target), { recursive: true }); + copyFileSync(file, target); + } +} + +export function getDemoStorage(uri = process.env.DEMO_STORAGE): DemoStorage | null { + if (!uri) return null; + + if (uri.startsWith('hf://')) return new HfStorage(uri); + + if (uri.startsWith('file://')) return new FileStorage(fileURLToPath(uri)); + + throw new Error(`DEMO_STORAGE: unsupported scheme in "${uri}" (use hf:// or file://)`); +} + +/** Public URL of a stored key, when DEMO_STORAGE_PUBLIC_URL is set. */ +export function getPublicUrl(key: string): string | null { + const base = process.env.DEMO_STORAGE_PUBLIC_URL; + + return base ? joinKey(base, key) : null; +} diff --git a/tools/ui/package-lock.json b/tools/ui/package-lock.json index 4fc68726b5bf..fb2b68eaa822 100644 --- a/tools/ui/package-lock.json +++ b/tools/ui/package-lock.json @@ -9,9 +9,15 @@ "version": "1.0.0", "devDependencies": { "@chromatic-com/storybook": "5.5.1", + "@codemirror/commands": "6.11.1", + "@codemirror/lang-markdown": "6.5.2", + "@codemirror/language": "6.13.1", + "@codemirror/state": "6.7.6", + "@codemirror/view": "6.43.15", "@eslint/compat": "1.4.1", "@eslint/js": "9.39.2", "@internationalized/date": "3.12.2", + "@lezer/highlight": "1.2.5", "@lucide/svelte": "1.46.0", "@modelcontextprotocol/sdk": "1.32.1", "@playwright/test": "1.56.1", @@ -47,6 +53,7 @@ "globals": "16.5.0", "highlight.js": "11.11.1", "http-server": "14.1.1", + "llama-brand": "https://codeload.github.com/ggml-org/llama-brand/tar.gz/f05540643c1ca2a18d732ed6491c839686da9a3a", "mdast": "3.0.0", "mdsvex": "0.12.7", "mermaid": "11.17.2", @@ -1799,6 +1806,154 @@ "storybook": "^0.0.0-0 || ^10.1.0 || ^10.1.0-0 || ^10.2.0-0 || ^10.3.0-0 || ^10.4.0-0 || ^10.5.0-0 || ^10.6.0-0 || ^11.0.0-0" } }, + "node_modules/@codemirror/autocomplete": { + "version": "6.20.4", + "resolved": "https://registry.npmjs.org/@codemirror/autocomplete/-/autocomplete-6.20.4.tgz", + "integrity": "sha512-yF8RhOf2SXlg00+qtupzLhrglcas61boqaONiNcbPIebHCpJocpqTBA6dyrtUrQO47hqyTdeob/3XnYilT7TaQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@codemirror/language": "^6.0.0", + "@codemirror/state": "^6.0.0", + "@codemirror/view": "^6.17.0", + "@lezer/common": "^1.0.0" + } + }, + "node_modules/@codemirror/commands": { + "version": "6.11.1", + "resolved": "https://registry.npmjs.org/@codemirror/commands/-/commands-6.11.1.tgz", + "integrity": "sha512-O/4hG3SC1YwcmQ0d2UVNDs+AsaNWd1iHVxbTeEBuqH+6bExAiPK3iS/BvpY6rZGURALv4ZD3sIgcCmRvw3ehBg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@codemirror/language": "^6.0.0", + "@codemirror/state": "^6.7.0", + "@codemirror/view": "^6.27.0", + "@lezer/common": "^1.1.0" + } + }, + "node_modules/@codemirror/lang-css": { + "version": "6.3.1", + "resolved": "https://registry.npmjs.org/@codemirror/lang-css/-/lang-css-6.3.1.tgz", + "integrity": "sha512-kr5fwBGiGtmz6l0LSJIbno9QrifNMUusivHbnA1H6Dmqy4HZFte3UAICix1VuKo0lMPKQr2rqB+0BkKi/S3Ejg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@codemirror/autocomplete": "^6.0.0", + "@codemirror/language": "^6.0.0", + "@codemirror/state": "^6.0.0", + "@lezer/common": "^1.0.2", + "@lezer/css": "^1.1.7" + } + }, + "node_modules/@codemirror/lang-html": { + "version": "6.4.12", + "resolved": "https://registry.npmjs.org/@codemirror/lang-html/-/lang-html-6.4.12.tgz", + "integrity": "sha512-pw2ReWKUqSkbvh76RAT4NYxiogRu+PWkR2ukAwO9uOgrm8uipkzjtKKtNpyeAQwHOqxEeSvAXZ6vr3AfyB9y/w==", + "dev": true, + "license": "MIT", + "dependencies": { + "@codemirror/autocomplete": "^6.0.0", + "@codemirror/lang-css": "^6.0.0", + "@codemirror/lang-javascript": "^6.0.0", + "@codemirror/language": "^6.4.0", + "@codemirror/state": "^6.0.0", + "@codemirror/view": "^6.17.0", + "@lezer/common": "^1.0.0", + "@lezer/css": "^1.1.0", + "@lezer/html": "^1.3.12" + } + }, + "node_modules/@codemirror/lang-javascript": { + "version": "6.2.5", + "resolved": "https://registry.npmjs.org/@codemirror/lang-javascript/-/lang-javascript-6.2.5.tgz", + "integrity": "sha512-zD4e5mS+50htS7F+TYjBPsiIFGanfVqg4HyUz6WNFikgOPf2BgKlx+TQedI1w6n/IqRBVBbBWmGFdLB/7uxO4A==", + "dev": true, + "license": "MIT", + "dependencies": { + "@codemirror/autocomplete": "^6.0.0", + "@codemirror/language": "^6.6.0", + "@codemirror/lint": "^6.0.0", + "@codemirror/state": "^6.0.0", + "@codemirror/view": "^6.17.0", + "@lezer/common": "^1.0.0", + "@lezer/javascript": "^1.0.0" + } + }, + "node_modules/@codemirror/lang-markdown": { + "version": "6.5.2", + "resolved": "https://registry.npmjs.org/@codemirror/lang-markdown/-/lang-markdown-6.5.2.tgz", + "integrity": "sha512-AwBOdkWYuA//WcM0xO5PfHPUcmz/O2i5o0Nsg1U69SII/loCJlFI1Romd9xp2HYb1kYJRGZotyqRghuHH5n8Kw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@codemirror/autocomplete": "^6.7.1", + "@codemirror/lang-html": "^6.0.0", + "@codemirror/language": "^6.3.0", + "@codemirror/state": "^6.0.0", + "@codemirror/view": "^6.0.0", + "@lezer/common": "^1.2.1", + "@lezer/markdown": "^1.0.0" + } + }, + "node_modules/@codemirror/language": { + "version": "6.13.1", + "resolved": "https://registry.npmjs.org/@codemirror/language/-/language-6.13.1.tgz", + "integrity": "sha512-zfvU2r3TyylQLMj/te+5CnQ0rSqZi8B1oTCBbQkagcLW4Xc6imab6qN+pEJhsuNOLaJqiRyBXp9UhWA1DhVvDw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@codemirror/state": "^6.0.0", + "@codemirror/streamparser": "^6.0.0", + "@codemirror/view": "^6.23.0", + "@lezer/common": "^1.5.0", + "@lezer/highlight": "^1.0.0", + "@lezer/lr": "^1.0.0", + "style-mod": "^4.0.0" + } + }, + "node_modules/@codemirror/lint": { + "version": "6.9.7", + "resolved": "https://registry.npmjs.org/@codemirror/lint/-/lint-6.9.7.tgz", + "integrity": "sha512-28/+iWLYxKxsvGYhSYL7zaCZqLz5+FFFDq9tVsvGv9kv8RY4fFAchJ5WX9M3YrrRlTIsECjsXPqeNgnSmNP2dg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@codemirror/state": "^6.0.0", + "@codemirror/view": "^6.42.0", + "crelt": "^1.0.5" + } + }, + "node_modules/@codemirror/state": { + "version": "6.7.6", + "resolved": "https://registry.npmjs.org/@codemirror/state/-/state-6.7.6.tgz", + "integrity": "sha512-kAz+AncRtKuIknedxT1bq4XwXv4UowhbkHU1myPrtVb/jZtImWuV5BXzv5vK6i3kYACsdiZiQKFQQ5Mq7elW8w==", + "dev": true, + "license": "MIT", + "dependencies": { + "@marijn/find-cluster-break": "^1.0.0" + } + }, + "node_modules/@codemirror/streamparser": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/@codemirror/streamparser/-/streamparser-6.0.0.tgz", + "integrity": "sha512-JTFPF6xrfjt0mnfGPgzcLiMwYmvIyNaoizp8BYzv8FE8KrT6GBGDZ190rtt+O7Wke3gG/MFHyiY9YGNeimof4w==", + "dev": true, + "license": "MIT" + }, + "node_modules/@codemirror/view": { + "version": "6.43.15", + "resolved": "https://registry.npmjs.org/@codemirror/view/-/view-6.43.15.tgz", + "integrity": "sha512-HHsfpqFeHyXLs983pSRe38Xajs0vFWZQ2BuX0m7Ki8w3jZ5iD8cuR3lBz2/5/X53Dhf32SBHKh6rtSTB6tKeGw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@codemirror/state": "^6.7.0", + "crelt": "^1.0.6", + "style-mod": "^4.1.0", + "w3c-keyname": "^2.2.4" + } + }, "node_modules/@emnapi/core": { "version": "1.9.2", "resolved": "https://registry.npmjs.org/@emnapi/core/-/core-1.9.2.tgz", @@ -3273,6 +3428,80 @@ "@jridgewell/sourcemap-codec": "^1.4.14" } }, + "node_modules/@lezer/common": { + "version": "1.5.3", + "resolved": "https://registry.npmjs.org/@lezer/common/-/common-1.5.3.tgz", + "integrity": "sha512-H0iErY4e43LpXbYDyBci5W4v/RwTgGV3YzYOlJPiGZ8RY8w51kVE+xLn1tg3Z6UmSfyETvtsUK3r7YsgDQEI7Q==", + "dev": true, + "license": "MIT" + }, + "node_modules/@lezer/css": { + "version": "1.3.9", + "resolved": "https://registry.npmjs.org/@lezer/css/-/css-1.3.9.tgz", + "integrity": "sha512-ZmzgvTOBdJWGIsY/ItAfN7UW8DV802oZG+jeG6E508SwEzz5YhuAajptpWPICvIyqYAD2+VjMdOS40GgGkyEOQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@lezer/common": "^1.2.0", + "@lezer/highlight": "^1.0.0", + "@lezer/lr": "^1.3.0" + } + }, + "node_modules/@lezer/highlight": { + "version": "1.2.5", + "resolved": "https://registry.npmjs.org/@lezer/highlight/-/highlight-1.2.5.tgz", + "integrity": "sha512-O1GMVKgtf5YspFaRzpmqkVgMtIk2HG9uMAbH6wdAIdNh3TVzCsch/+cRi+6l2//UGDKe6NB9BW3cvuR5ezMQ0w==", + "dev": true, + "license": "MIT", + "dependencies": { + "@lezer/common": "^1.3.0" + } + }, + "node_modules/@lezer/html": { + "version": "1.3.13", + "resolved": "https://registry.npmjs.org/@lezer/html/-/html-1.3.13.tgz", + "integrity": "sha512-oI7n6NJml729m7pjm9lvLvmXbdoMoi2f+1pwSDJkl9d68zGr7a9Btz8NdHTGQZtW2DA25ybeuv/SyDb9D5tseg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@lezer/common": "^1.2.0", + "@lezer/highlight": "^1.0.0", + "@lezer/lr": "^1.0.0" + } + }, + "node_modules/@lezer/javascript": { + "version": "1.5.6", + "resolved": "https://registry.npmjs.org/@lezer/javascript/-/javascript-1.5.6.tgz", + "integrity": "sha512-sEvOqpl4vJQca0NugSQK02vQPbc5KrvoojnYGoqqFgFO0MtcYhwGiHKVEB+LuXYJnfyXo4QPA8ztbrEa4Fw5NA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@lezer/common": "^1.2.0", + "@lezer/highlight": "^1.1.3", + "@lezer/lr": "^1.3.0" + } + }, + "node_modules/@lezer/lr": { + "version": "1.4.11", + "resolved": "https://registry.npmjs.org/@lezer/lr/-/lr-1.4.11.tgz", + "integrity": "sha512-UwgZcl1R66H2hDzBrA7kWyYmX6s8p5TQ+e/t7CuvgCtBEPxbEPaHYHu+pJBvTvDotEJpuQkJjbXuHiw5Cixxgw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@lezer/common": "^1.0.0" + } + }, + "node_modules/@lezer/markdown": { + "version": "1.8.1", + "resolved": "https://registry.npmjs.org/@lezer/markdown/-/markdown-1.8.1.tgz", + "integrity": "sha512-M5YmEcKWO4mJ45Y1hECwqyu8cLv25EiXOK554PVOvDOWOP0olwelHHQUWr9sjF+d1bENVhy0nfJ8E57XZUTiXg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@lezer/common": "^1.5.0", + "@lezer/highlight": "^1.0.0" + } + }, "node_modules/@lucide/svelte": { "version": "1.46.0", "resolved": "https://registry.npmjs.org/@lucide/svelte/-/svelte-1.46.0.tgz", @@ -3283,6 +3512,13 @@ "svelte": "^5" } }, + "node_modules/@marijn/find-cluster-break": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/@marijn/find-cluster-break/-/find-cluster-break-1.0.4.tgz", + "integrity": "sha512-Wy0V7+SGUjnF9/TkiM1hKVDPj7jKXduPNboMVtHTA8dySMURWqfg/JZ9E2Sq8JgSJmkl7k7Qe9FLeMSrSraWmQ==", + "dev": true, + "license": "MIT" + }, "node_modules/@mdx-js/react": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/@mdx-js/react/-/react-3.1.1.tgz", @@ -8029,6 +8265,13 @@ "layout-base": "^1.0.0" } }, + "node_modules/crelt": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/crelt/-/crelt-1.0.7.tgz", + "integrity": "sha512-aK6BbWfhf4U/wCcLHKPJl/xa6VkVstRaPywWtMKGwuOLc/wZTyQYuoxgvZnNsBvv7Kg3YTBQYYBCggcviQczuA==", + "dev": true, + "license": "MIT" + }, "node_modules/cross-spawn": { "version": "7.0.6", "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz", @@ -12084,6 +12327,13 @@ "node": ">=10" } }, + "node_modules/llama-brand": { + "version": "0.1.0", + "resolved": "https://codeload.github.com/ggml-org/llama-brand/tar.gz/f05540643c1ca2a18d732ed6491c839686da9a3a", + "integrity": "sha512-kRDXUIUdfcDt7w4hh3JrFTpqxf2i7R0PHVLGnwUdr4L8EMqE32mEECoXUz1uoUlFaQ+h6QaVHvJWS5y98AoaYA==", + "dev": true, + "license": "CC-BY-NC-4.0" + }, "node_modules/locate-character": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/locate-character/-/locate-character-3.0.0.tgz", @@ -15787,6 +16037,13 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/style-mod": { + "version": "4.1.4", + "resolved": "https://registry.npmjs.org/style-mod/-/style-mod-4.1.4.tgz", + "integrity": "sha512-XXWIQt633/EpAFx8aZDOTjBzrCaGmhvEQlQo6MVPfa2OzO2cWo+4hV9h+6UkHYlXGfy+ODXKUdP7Pthmcu5ATw==", + "dev": true, + "license": "MIT" + }, "node_modules/style-to-object": { "version": "1.0.14", "resolved": "https://registry.npmjs.org/style-to-object/-/style-to-object-1.0.14.tgz", @@ -17511,6 +17768,13 @@ "node": ">=18" } }, + "node_modules/w3c-keyname": { + "version": "2.2.8", + "resolved": "https://registry.npmjs.org/w3c-keyname/-/w3c-keyname-2.2.8.tgz", + "integrity": "sha512-dpojBhNsCNN7T82Tm7k26A6G9ML3NkhDsnw9n/eoxSRlVBB4CEtIQ/KTCLI2Fwf3ataSXRhYFkQi3SlnFwPvPQ==", + "dev": true, + "license": "MIT" + }, "node_modules/web-namespaces": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/web-namespaces/-/web-namespaces-2.0.1.tgz", diff --git a/tools/ui/package.json b/tools/ui/package.json index e87d40aa0279..cc3785727280 100644 --- a/tools/ui/package.json +++ b/tools/ui/package.json @@ -18,6 +18,11 @@ "test": "npm run test:ui -- --run && npm run test:client -- --run && npm run test:unit -- --run && npm run test:e2e", "test:e2e": "playwright test", "test:e2e:pwa": "playwright test tests/e2e/pwa.e2e.ts", + "demo:codegen": "playwright codegen --target=playwright-test --viewport-size=1440,900 --output=demo/recording.demo.ts http://localhost:5180/", + "demo:docs": "node demo/cli.ts docs", + "demo:pr": "node demo/cli.ts pr", + "demo:record": "playwright test --config=demo/playwright.config.ts", + "demo:traces": "node demo/cli.ts traces", "test:client": "vitest --project=client", "test:unit": "vitest --project=unit", "test:unit:pwa": "vitest --project=unit --run tests/unit/pwa.spec.ts", @@ -29,9 +34,15 @@ }, "devDependencies": { "@chromatic-com/storybook": "5.5.1", + "@codemirror/commands": "6.11.1", + "@codemirror/lang-markdown": "6.5.2", + "@codemirror/language": "6.13.1", + "@codemirror/state": "6.7.6", + "@codemirror/view": "6.43.15", "@eslint/compat": "1.4.1", "@eslint/js": "9.39.2", "@internationalized/date": "3.12.2", + "@lezer/highlight": "1.2.5", "@lucide/svelte": "1.46.0", "@modelcontextprotocol/sdk": "1.32.1", "@playwright/test": "1.56.1", @@ -67,6 +78,7 @@ "globals": "16.5.0", "highlight.js": "11.11.1", "http-server": "14.1.1", + "llama-brand": "https://codeload.github.com/ggml-org/llama-brand/tar.gz/f05540643c1ca2a18d732ed6491c839686da9a3a", "mdast": "3.0.0", "mdsvex": "0.12.7", "mermaid": "11.17.2", diff --git a/tools/ui/playwright.config.ts b/tools/ui/playwright.config.ts index 057ed416df59..2b6dde6cc4c1 100644 --- a/tools/ui/playwright.config.ts +++ b/tools/ui/playwright.config.ts @@ -9,7 +9,12 @@ export default defineConfig({ projects: [ { name: 'chromium', - use: { ...devices['Desktop Chrome'] } + use: { + ...devices['Desktop Chrome'], + // locally, the installed (auto-updated) Chrome with its sandbox on: the bundled + // Chromium lags behind security fixes and Playwright disables its sandbox by default + ...(process.env.CI ? {} : { channel: 'chrome', launchOptions: { chromiumSandbox: true } }) + } } ], reporter: 'line', diff --git a/tools/ui/pwa-assets-dark.config.ts b/tools/ui/pwa-assets-dark.config.ts index 4d8114ee76df..7a1128f35bb0 100644 --- a/tools/ui/pwa-assets-dark.config.ts +++ b/tools/ui/pwa-assets-dark.config.ts @@ -1,10 +1,8 @@ -import { writeThemeFavicons } from './scripts/favicon-colorize'; -import { FAVICON_COLORS, PWA_ASSET_GENERATOR } from './src/lib/constants/pwa.constants'; +import { writeBrandFavicons } from './scripts/brand-favicons'; +import { PWA_ASSET_GENERATOR } from './src/lib/constants/pwa.constants'; import { defineConfig } from '@vite-pwa/assets-generator/config'; -writeThemeFavicons(FAVICON_COLORS.LIGHT, FAVICON_COLORS.DARK, { - padding: PWA_ASSET_GENERATOR.FAVICON_PADDING -}); +writeBrandFavicons(); export default defineConfig({ headLinkOptions: { diff --git a/tools/ui/pwa-assets.config.ts b/tools/ui/pwa-assets.config.ts index f9f8662a20af..bd0bc0024f5b 100644 --- a/tools/ui/pwa-assets.config.ts +++ b/tools/ui/pwa-assets.config.ts @@ -1,6 +1,5 @@ -import { writeThemeFavicons } from './scripts/favicon-colorize'; +import { writeBrandFavicons } from './scripts/brand-favicons'; import { - FAVICON_COLORS, PWA_ASSET_GENERATOR, PWA_GENERATOR_DEVICES, THEME_COLORS @@ -14,9 +13,7 @@ import { import { readFileSync } from 'node:fs'; import { resolve } from 'node:path'; -writeThemeFavicons(FAVICON_COLORS.LIGHT, FAVICON_COLORS.DARK, { - padding: PWA_ASSET_GENERATOR.FAVICON_PADDING -}); +writeBrandFavicons(); export default defineConfig({ headLinkOptions: { @@ -26,7 +23,7 @@ export default defineConfig({ preset: combinePresetAndAppleSplashScreens( { ...minimal2023Preset, - // tiny margin so favicon.ico / pwa-*.png breathe inside the canvas + // llama-brand's favicons carry their own margin transparent: { ...minimal2023Preset.transparent, padding: PWA_ASSET_GENERATOR.FAVICON_PADDING diff --git a/tools/ui/scripts/brand-favicons.ts b/tools/ui/scripts/brand-favicons.ts new file mode 100644 index 000000000000..87a0585161ba --- /dev/null +++ b/tools/ui/scripts/brand-favicons.ts @@ -0,0 +1,27 @@ +import { FAVICON_PATHS } from '../src/lib/constants/pwa.constants'; +import { copyFileSync, mkdirSync } from 'node:fs'; +import { createRequire } from 'node:module'; +import { dirname, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const HERE = dirname(fileURLToPath(import.meta.url)); +const DEFAULT_OUT_DIR = resolve(HERE, '..', 'static'); +const require = createRequire(import.meta.url); + +/** The web favicons llama-brand draws for the UI, optically centred, keyed by the file they become. */ +export const BRAND_FAVICONS = { + [FAVICON_PATHS.SVG_DARK]: 'llama-brand/logo/svg/app/web/favicon-dark.svg', + [FAVICON_PATHS.SVG_LIGHT]: 'llama-brand/logo/svg/app/web/favicon.svg' +} as const; + +/** + * Copy llama-brand's light and dark favicons into the static directory, where + * they are served as they are and feed the PWA asset generator. + */ +export function writeBrandFavicons(outDir: string = DEFAULT_OUT_DIR): void { + mkdirSync(outDir, { recursive: true }); + + for (const [fileName, specifier] of Object.entries(BRAND_FAVICONS)) { + copyFileSync(require.resolve(specifier), resolve(outDir, fileName)); + } +} diff --git a/tools/ui/scripts/favicon-colorize.ts b/tools/ui/scripts/favicon-colorize.ts deleted file mode 100644 index 54a951296a1f..000000000000 --- a/tools/ui/scripts/favicon-colorize.ts +++ /dev/null @@ -1,113 +0,0 @@ -import { mkdirSync, readFileSync, writeFileSync } from 'node:fs'; -import { dirname, resolve } from 'node:path'; -import { fileURLToPath } from 'node:url'; - -const HERE = dirname(fileURLToPath(import.meta.url)); -const PROJECT_ROOT = resolve(HERE, '..'); -const DEFAULT_LOGO = resolve(PROJECT_ROOT, 'src/lib/assets/logo.svg'); -const DEFAULT_OUT_DIR = resolve(PROJECT_ROOT, 'static'); -const DEFAULT_OUT_LIGHT = resolve(DEFAULT_OUT_DIR, 'favicon.svg'); -const DEFAULT_OUT_DARK = resolve(DEFAULT_OUT_DIR, 'favicon-dark.svg'); -const CURRENT_COLOR = 'currentColor'; - -export interface ColorizedFavicon { - light: string; - dark: string; -} - -export interface WriteThemeFaviconsOptions { - sourcePath?: string; - lightOutPath?: string; - darkOutPath?: string; - /** - * Fraction of the icon (0..1) to leave as an even margin on each side. - * Applied by wrapping the inner content in a `` so the - * source `src/lib/assets/logo.svg` is not modified. Pass 0 to disable. - */ - padding?: number; -} - -/** - * Replace every `currentColor` occurrence in the SVG with the given color. - * Pure: no filesystem access, so it is straightforward to unit-test. - */ -export function colorizeFaviconSvg( - svg: string, - lightColor: string, - darkColor: string -): ColorizedFavicon { - return { - dark: svg.replaceAll(CURRENT_COLOR, darkColor), - light: svg.replaceAll(CURRENT_COLOR, lightColor) - }; -} - -/** - * Shrink the inner SVG content uniformly and re-center it so `padding` (a - * 0..1 fraction) is reserved as equal margin on each side. Returns the input - * unchanged for non-positive padding, missing/invalid `viewBox`, or unexpected - * markup so the caller always gets a renderable SVG. - */ -export function padFaviconSvg(svg: string, padding: number): string { - if (!(padding > 0) || padding >= 1) return svg; - - const viewBoxMatch = svg.match(/viewBox\s*=\s*["']([^"']+)["']/i); - - if (!viewBoxMatch) return svg; - - const parts = viewBoxMatch[1] - .trim() - .split(/[\s,]+/) - .map(Number); - - if (parts.length !== 4 || parts.some((n) => !Number.isFinite(n))) return svg; - - const [, , width, height] = parts; - - if (width <= 0 || height <= 0) return svg; - - const scale = 1 - padding; - const translateX = (padding * width) / 2; - const translateY = (padding * height) / 2; - const openTagStart = svg.search(/', openTagStart); - - if (openTagEnd === -1) return svg; - - const closeStart = svg.lastIndexOf('`; - - return `${openTag}${group}${inner}${closeTag}`; -} - -/** - * Read `src/lib/assets/logo.svg`, colorize it for both themes, and write - * the results to the static directory so the PWA asset generator can consume - * them. Paths can be overridden for tests. - */ -export function writeThemeFavicons( - lightColor: string, - darkColor: string, - { - darkOutPath = DEFAULT_OUT_DARK, - lightOutPath = DEFAULT_OUT_LIGHT, - padding = 0, - sourcePath = DEFAULT_LOGO - }: WriteThemeFaviconsOptions = {} -): void { - const source = readFileSync(sourcePath, 'utf-8'); - const { dark, light } = colorizeFaviconSvg(source, lightColor, darkColor); - - mkdirSync(dirname(lightOutPath), { recursive: true }); - writeFileSync(lightOutPath, padFaviconSvg(light, padding)); - writeFileSync(darkOutPath, padFaviconSvg(dark, padding)); -} diff --git a/tools/ui/src/app.css b/tools/ui/src/app.css index bf1a071a9b82..7a92c0ba0f4f 100644 --- a/tools/ui/src/app.css +++ b/tools/ui/src/app.css @@ -1,8 +1,11 @@ @import 'tailwindcss'; @source '.'; +/* stories may use classes the app does not, so tailwind scans them too */ +@source '../tests/stories'; @plugin '@tailwindcss/forms'; @plugin '@tailwindcss/typography'; @import 'tw-animate-css'; +@import './lib/styles/safe-area.css'; @custom-variant dark (&:is(.dark *)); @@ -35,6 +38,15 @@ :root { --radius: 0.625rem; + /* the page runs under the status bar and over the home indicator + (viewport-fit=cover); what is pinned to a screen edge keeps clear of them + with the pt-safe / pb-safe utilities (see lib/styles/safe-area.css) */ + --safe-area-top: env(safe-area-inset-top, 0px); + --safe-area-bottom: env(safe-area-inset-bottom, 0px); + /* where the phone's corner controls (menu toggle, context gauge, sidebar logo) sit */ + --corner-control-top: calc(1rem + var(--safe-area-top)); + /* where a phone page's title starts: a row below the corner controls */ + --page-top: calc(var(--corner-control-top) + 3.75rem); /* expanded desktop sidebar width, overridden once the user drags its edge */ --sidebar-width: 18rem; --background: oklch(1 0 0); @@ -70,6 +82,20 @@ --sidebar-ring: oklch(0.708 0 0); --code-background: oklch(0.985 0 0); --code-foreground: oklch(0.145 0 0); + /* phone sidebar width, which the page slides aside to reveal */ + --sidebar-reveal: min(80dvw, 20rem); + /* the desktop sidebar's muted/60 over the page, made opaque so the slid page's corners can match it */ + --sidebar-reveal-tint: color-mix(in oklab, var(--muted) 60%, var(--background)); + /* how far the phone sidebar is open, 0 to 1, stepped every frame by useSidebarReveal */ + --sidebar-reveal-progress: 0; + /* the same on a log curve: the sidebar's contents hold on until the page nearly covers them */ + --sidebar-reveal-ease: calc(log(1 + 9 * var(--sidebar-reveal-progress)) / log(10)); + /* the page background, blending into the sidebar tint as the phone sidebar opens */ + --sidebar-reveal-bg: color-mix( + in oklab, + var(--sidebar-reveal-tint) calc(var(--sidebar-reveal-progress) * 100%), + var(--background) + ); --font-mono: ui-monospace, SFMono-Regular, 'SF Mono', Monaco, 'Cascadia Code', 'Roboto Mono', Consolas, 'Liberation Mono', Menlo, monospace; @@ -78,9 +104,42 @@ --chat-form-area-height: 8rem; --chat-form-area-offset: 2rem; --chat-form-padding-top: 6rem; + + /* Nested surfaces, outermost first: the page, the sidebar on it, a group inset in the + sidebar (its footer) and a row in the group. A row keeps its corners concentric with the + group's: its radius is the group's less the padding between their edges. The sidebar's + actions above the footer share the row radius, so every action row rounds alike. See "Nested + surfaces" in the README; tests/client/nested-surfaces.svelte.test.ts holds the rule */ + --surface-group-radius: 1rem; + --surface-group-padding: 0.25rem; + --surface-row-radius: calc(var(--surface-group-radius) - var(--surface-group-padding)); + + /* the chat form card holding one line of text, as it rests on a new chat */ + --chat-form-collapsed-height: 6.375rem; + /* the empty chat form card, its text and actions in one row: a new chat centers its top edge */ + --chat-form-resting-height: 3.375rem; + /* the chat form's corners while its text and actions share one row */ + --chat-form-single-row-radius: 1.25rem; + /* the card's translucent fill over the page, made opaque: the phone sheet, and the page and Safari's bars around it */ + --chat-form-sheet-fill: color-mix(in oklab, var(--muted) 60%, var(--background)); --max-message-height: max(24rem, min(80dvh, calc(100dvh - var(--chat-form-area-height) - 12rem))); } +/* iOS 26 gives an installed app with viewport-fit=cover and a translucent status bar a + viewport short of that bar (WebKit bug 301108), but only while the page is shorter than + the screen. A page that reaches past it gets the full screen, after which this is 100%, + capped at the screen so it never scrolls; elsewhere the inset is 0 and it is 100% too. */ +@media (display-mode: standalone) { + /* the installed app's status bar already sets them apart from the screen's edge */ + :root { + --corner-control-top: calc(0.25rem + var(--safe-area-top)); + } + + html { + min-height: min(100lvh, calc(100% + var(--safe-area-top))); + } +} + @media (min-width: 640px) { :root { --chat-form-area-height: 24rem; @@ -90,6 +149,7 @@ } .dark { + --chat-form-sheet-fill: color-mix(in oklab, var(--muted) 75%, var(--background)); --background: oklch(0.16 0 0); --foreground: oklch(0.985 0 0); --card: oklch(0.205 0 0); @@ -169,11 +229,21 @@ } body { - @apply bg-background text-foreground antialiased; + @apply bg-(--sidebar-reveal-bg) text-foreground antialiased; scrollbar-width: thin; scrollbar-gutter: stable; } + /* While the chat form is a full-screen sheet on a phone, the page around it takes the + sheet's fill: iOS paints the installed app's status bar and home indicator areas, and + Safari tints its bars, from this background */ + @media (width < 48rem) { + html[data-chat-form-sheet], + html[data-chat-form-sheet] body { + background-color: var(--chat-form-sheet-fill); + } + } + /* no elastic bounce past the ends of the page: it drags the sticky and fixed bars (tab bar, chat form) along and they jump back on release */ html, @@ -181,6 +251,30 @@ overscroll-behavior: none; } + /* no gray flash over a tapped control, which has its own press state, and + no text inflation when a phone turns to landscape */ + html { + -webkit-tap-highlight-color: transparent; + -webkit-text-size-adjust: 100%; + text-size-adjust: 100%; + } + + /* a tap clicks at once rather than waiting to see if it is a double-tap zoom */ + button, + a, + [role='button'] { + touch-action: manipulation; + } + + /* holding a control does not select its label or pop the link callout; + links stay selectable, as in a message they are content */ + button, + [role='button'] { + -webkit-touch-callout: none; + -webkit-user-select: none; + user-select: none; + } + /* same for dialogs, whose panels and lists scroll on their own with sticky headers and footers. Scroll containers only: Firefox treats an overflow-hidden box (truncated text) as one, and with `none` it would @@ -198,7 +292,7 @@ } *:hover { - scrollbar-color: hsl(var(--muted-foreground) / 0.3) transparent; + scrollbar-color: color-mix(in oklch, var(--muted-foreground) 30%, transparent) transparent; } *::-webkit-scrollbar { @@ -217,11 +311,11 @@ } *:hover::-webkit-scrollbar-thumb { - background: hsl(var(--muted-foreground) / 0.3); + background: color-mix(in oklch, var(--muted-foreground) 30%, transparent); } *::-webkit-scrollbar-thumb:hover { - background: hsl(var(--muted-foreground) / 0.5); + background: color-mix(in oklch, var(--muted-foreground) 50%, transparent); } :where(code, pre, kbd, samp) { @@ -251,6 +345,125 @@ ); } + /* The same fade at the top of a phone's screen: solid behind the status bar and the row of + corner controls under it (the menu key, the chat's header), fading out below them, so the + page scrolls away under them instead of into them. It rides on the page as it slides aside + for the sidebar and stays as it is: the rounded frame over the slid page clips it. */ + .status-bar-fade { + /* down to where a page's content starts (--page-top): solid to the middle of the corner + row, then a long, even fade over its lower half and on to the content */ + height: var(--page-top); + background: linear-gradient( + to bottom, + var(--background) calc(var(--corner-control-top) + 1.25rem), + color-mix(in srgb, var(--background) 55%, transparent) + calc(var(--corner-control-top) + 2.5rem), + transparent 100% + ); + } + + /* iOS Safari colours the screen's top, above the page, from a solid fixed element at the + page's top edge (theme-color no longer), and shows the page through it otherwise. The + fade above is a gradient, which it does not take, so this strip under it carries the + page's colour there, as the installed app has. Safari finds it by hit-testing the top + edge, so it takes taps (pointer-events: none would hide it) and is thicker than the + 10px Safari ignores; what it covers sits under the fade's solid part anyway */ + .status-bar-edge { + height: 0.75rem; + background-color: var(--background); + } + + /* The bottom edge's counterpart, which Safari takes for its bottom toolbar the same way: + without it, it finds the chat form there and takes the form's colour */ + .screen-bottom-edge { + height: 0.75rem; + background-color: var(--background); + } + + /* The same fade at the bottom of a phone's screen: solid behind the home indicator and + fading out above it, so the page ends softly at the screen's edge and under the chat + form instead of running out under it. It stays through the slide as the top one does */ + .screen-bottom-fade { + height: calc(var(--safe-area-bottom) + 3rem); + background: linear-gradient( + to top, + var(--background) var(--safe-area-bottom), + color-mix(in srgb, var(--background) 60%, transparent) calc(var(--safe-area-bottom) + 1.5rem), + transparent 100% + ); + } + + /* The sidebar's "Show more" row, stuck to its bottom: an opaque copy of the + panel's translucent muted tint, and a fade above it, like the one under + the tab bar, so the rows scrolling beneath end softly */ + .sidebar-show-more { + --sidebar-surface: color-mix(in oklab, var(--muted) 60%, var(--background)); + background: var(--sidebar-surface); + + &::before { + content: ''; + position: absolute; + inset-inline: 0; + bottom: 100%; + height: 1rem; + pointer-events: none; + background: linear-gradient( + to top, + var(--sidebar-surface) 0%, + color-mix(in srgb, var(--sidebar-surface) 60%, transparent) 50%, + transparent 100% + ); + } + } + + /* An opaque copy of the sidebar panel's translucent tint, under what moves + over the other sections while dragged */ + .sidebar-drag-surface, + .sidebar-drag-overlay { + background: color-mix(in oklab, var(--muted) 60%, var(--background)); + } + + /* the copy of a chat row that follows the pointer, raised above the page */ + .sidebar-drag-overlay { + box-shadow: + 0 10px 30px -10px rgb(0 0 0 / 0.35), + 0 0 0 1px color-mix(in srgb, var(--foreground) 12%, transparent); + } + + /* While something is dragged, what it passes over takes no hover: only the + sections themselves stay targets, for the drop */ + body[data-sidebar-dragging] [data-sidebar-section] * { + pointer-events: none; + } + + .sidebar-drag-overlay { + position: fixed; + z-index: 1000; + border-radius: 0.5rem; + pointer-events: none; + } + + /* the phone sidebar's contents zoom out and fade as the page slides back over them */ + .sidebar-reveal-content { + opacity: var(--sidebar-reveal-ease); + scale: calc(0.98 + 0.02 * var(--sidebar-reveal-ease)); + } + + /* the slid page's fixed bars, held where they sit on screen, see useSidebarReveal */ + .sidebar-reveal-pin-top { + translate: 0 var(--sidebar-reveal-pin-top, 0px); + } + + .sidebar-reveal-pin-bottom { + translate: 0 calc(-1 * var(--sidebar-reveal-pin-bottom, 0px)); + } + + @media (prefers-reduced-motion: reduce) { + .sidebar-reveal-content { + scale: none; + } + } + .shimmer-text { background: linear-gradient( 90deg, diff --git a/tools/ui/src/app.html b/tools/ui/src/app.html index ef2787ad1bb4..65a9d92886e9 100644 --- a/tools/ui/src/app.html +++ b/tools/ui/src/app.html @@ -12,7 +12,7 @@ %sveltekit.head% diff --git a/tools/ui/src/lib/assets/logo.svg b/tools/ui/src/lib/assets/logo.svg deleted file mode 100644 index 05424790afea..000000000000 --- a/tools/ui/src/lib/assets/logo.svg +++ /dev/null @@ -1,7 +0,0 @@ - - - - - - - diff --git a/tools/ui/src/lib/components/app/actions/GlassKey.svelte b/tools/ui/src/lib/components/app/actions/GlassKey.svelte new file mode 100644 index 000000000000..b2ccffe6a16b --- /dev/null +++ b/tools/ui/src/lib/components/app/actions/GlassKey.svelte @@ -0,0 +1,67 @@ + + +{#snippet key(props: Record = {})} + +{/snippet} + + +{#if tooltip && !deviceStore.isMobile} + + + {#snippet child({ props })} + {@render key(props)} + {/snippet} + + + +

{label}

+
+
+{:else} + {@render key()} +{/if} diff --git a/tools/ui/src/lib/components/app/actions/index.ts b/tools/ui/src/lib/components/app/actions/index.ts index 4bb2a58d6672..73ab769135b9 100644 --- a/tools/ui/src/lib/components/app/actions/index.ts +++ b/tools/ui/src/lib/components/app/actions/index.ts @@ -11,3 +11,6 @@ export { default as ActionIcon } from './ActionIcon.svelte'; /** Copy-to-clipboard icon button with clipboard logic. */ export { default as ActionIconCopyToClipboard } from './ActionIconCopyToClipboard.svelte'; + +/** A round matte glass key: a phone's corner keys and the key back to a chat's bottom. */ +export { default as GlassKey } from './GlassKey.svelte'; diff --git a/tools/ui/src/lib/components/app/chat/ChatAttachments/ChatAttachmentsList/ChatAttachmentsListItem/ChatAttachmentsListItem.svelte b/tools/ui/src/lib/components/app/chat/ChatAttachments/ChatAttachmentsList/ChatAttachmentsListItem/ChatAttachmentsListItem.svelte index 754856d60b86..b62cbc2dbae8 100644 --- a/tools/ui/src/lib/components/app/chat/ChatAttachments/ChatAttachmentsList/ChatAttachmentsListItem/ChatAttachmentsListItem.svelte +++ b/tools/ui/src/lib/components/app/chat/ChatAttachments/ChatAttachmentsList/ChatAttachmentsListItem/ChatAttachmentsListItem.svelte @@ -106,18 +106,22 @@ {readonly} width={imageWidth} /> -{:else if isPdfFile(item.attachment, item.uploadedFile)} - 0 ? `PDF, ${pageCount} ${pageCount === 1 ? 'page' : 'pages'}` : 'PDF'} name={item.name} onRemove={onFileRemove} onclick={() => onPreview?.(item)} + preview={item.preview} {readonly} - size={item.size} - textContent={item.textContent} - uploadedFile={item.uploadedFile} + width={imageWidth} /> {:else} void; onRemove?: (id: string) => void; name: string; @@ -20,6 +21,7 @@ height = 'h-16', id, imageClass = '', + label, name, onclick, onRemove, @@ -49,6 +51,14 @@ {@render image()} {/if} + {#if label} + + {label} + + {/if} + {#if !readonly}
(PdfViewMode.PAGES); let pdfImages = $state([]); let pdfImagesLoading = $state(false); + // a failure stays on screen: the effect does not retry while an error is set let pdfImagesError = $state(null); + let pdfText = $state(null); + let pdfTextLoading = $state(false); + let pdfTextError = $state(null); let language = $derived(getLanguageFromFilename(displayName)); + // older PDFs sent as images stored a placeholder in `content`, so their text is extracted again + let hasStoredText = $derived( + currentItem?.attachment?.type === AttachmentType.PDF + ? !( + currentItem.attachment.processedAsImages && + PDF_AS_IMAGES_LEGACY_CONTENT_REGEX.test(currentItem.attachment.content) + ) + : !!displayTextContent + ); + + let isLoading = $derived(pdfImagesLoading || pdfTextLoading); + + function getPdfFile(): File | null { + if (currentItem?.uploadedFile?.file) { + return currentItem.uploadedFile.file; + } + + if ( + currentItem?.attachment && + 'base64Data' in currentItem.attachment && + currentItem.attachment.base64Data + ) { + const byteArray = Uint8Array.from(atob(currentItem.attachment.base64Data), (char) => + char.charCodeAt(0) + ); + + return new File([byteArray], displayName, { type: MimeTypeApplication.PDF }); + } + + return null; + } + async function loadPdfImages() { if (pdfImages.length > 0 || pdfImagesLoading || !currentItem) return; + if ( + currentItem.attachment?.type === AttachmentType.PDF && + currentItem.attachment.images?.length + ) { + pdfImages = currentItem.attachment.images; + + return; + } + + if (pdfImagesError) return; + pdfImagesLoading = true; - pdfImagesError = null; try { - let file: File | null = null; - - if (currentItem.uploadedFile?.file) { - file = currentItem.uploadedFile.file; - } else if (currentItem.attachment) { - // Check if we have pre-processed images - if ( - 'images' in currentItem.attachment && - currentItem.attachment.images && - Array.isArray(currentItem.attachment.images) && - currentItem.attachment.images.length > 0 - ) { - pdfImages = currentItem.attachment.images; - - return; - } - - // Convert base64 back to File for processing - if ('base64Data' in currentItem.attachment && currentItem.attachment.base64Data) { - const base64Data = currentItem.attachment.base64Data; - const byteCharacters = atob(base64Data); - const byteNumbers = new Array(byteCharacters.length); - - for (let i = 0; i < byteCharacters.length; i++) { - byteNumbers[i] = byteCharacters.charCodeAt(i); - } - const byteArray = new Uint8Array(byteNumbers); - - file = new File([byteArray], displayName, { type: 'application/pdf' }); - } - } + const file = getPdfFile(); - if (file) { - pdfImages = await convertPDFToImage(file); - } else { + if (!file) { throw new Error('No PDF file available for conversion'); } + + pdfImages = await convertPDFToImage(file); } catch (error) { pdfImagesError = error instanceof Error ? error.message : 'Failed to load PDF images'; } finally { @@ -78,26 +94,62 @@ } } + async function loadPdfText() { + if (pdfText !== null || pdfTextLoading || !currentItem) return; + + if (hasStoredText) { + pdfText = displayTextContent ?? ''; + + return; + } + + if (pdfTextError) return; + + pdfTextLoading = true; + + try { + const file = getPdfFile(); + + if (!file) { + throw new Error('No PDF file available for text extraction'); + } + + pdfText = await convertPDFToText(file); + } catch (error) { + pdfTextError = error instanceof Error ? error.message : 'Failed to extract PDF text'; + } finally { + pdfTextLoading = false; + } + } + $effect(() => { if (pdfViewMode === PdfViewMode.PAGES) { loadPdfImages(); + } else { + loadPdfText(); } });
+ {#if chatFormActions.onWorkingDirectoryClick} + + {/if} + {#if toolsPanel.totalToolCount > 0} (toolsExpanded = open)} open={toolsExpanded}> diff --git a/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionAdd/ChatFormActionAddDropdown.svelte b/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionAdd/ChatFormActionAddDropdown.svelte index 2cae6e8caeba..f746a73cb72b 100644 --- a/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionAdd/ChatFormActionAddDropdown.svelte +++ b/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionAdd/ChatFormActionAddDropdown.svelte @@ -1,5 +1,5 @@
@@ -29,24 +39,34 @@ - {#if !hasAudioModality} - -

Current model does not support audio

+ {#if tooltip} + +

{tooltip}

{/if} diff --git a/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActions.svelte b/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActions.svelte index c51ac72a1326..23b63aa58f03 100644 --- a/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActions.svelte +++ b/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActions.svelte @@ -1,6 +1,5 @@
{#if showAddButton} -
+
{/if}
- {#if hasProcessedTokens} + + {#if showContextGauge.current && !deviceStore.isMobile} {/if} @@ -205,7 +191,14 @@ /> {:else if shouldShowRecordButton} - + {:else} { @@ -43,7 +52,7 @@
diff --git a/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormContextGauge/ContextGaugeDial.svelte b/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormContextGauge/ContextGaugeDial.svelte index ca520862d459..f9f2346af1bc 100644 --- a/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormContextGauge/ContextGaugeDial.svelte +++ b/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormContextGauge/ContextGaugeDial.svelte @@ -1,44 +1,49 @@ - + diff --git a/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormContextGauge/context-gauge.ts b/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormContextGauge/context-gauge.ts index e0a7f74780ec..552d0c4b9d13 100644 --- a/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormContextGauge/context-gauge.ts +++ b/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormContextGauge/context-gauge.ts @@ -1,4 +1,5 @@ -import { ColorLevel } from '$lib/enums'; +import { ColorLevel, MessageRole } from '$lib/enums'; +import type { ApiProcessingState, DatabaseMessage } from '$lib/types'; const WARNING_THRESHOLD = 80; const CRITICAL_THRESHOLD = 95; @@ -38,3 +39,42 @@ export function colorLevelBgClass(level: ColorLevel): string { return 'bg-muted'; } } + +/** + * Whether the conversation has spent any tokens yet, so the gauge has something to show: the + * timings its replies recorded, or the live state of a reply still being generated. + */ +export function hasProcessedTokens( + messages: DatabaseMessage[], + liveState: ApiProcessingState | null +): boolean { + let totalHistoricalTokens = 0; + + for (const m of messages) { + if (m.role !== MessageRole.ASSISTANT) continue; + + const timings = m.timings; + + if (!timings) continue; + + const agenticLlm = timings.agentic?.llm; + + if (agenticLlm?.prompt_n != null || agenticLlm?.predicted_n != null) { + totalHistoricalTokens += (agenticLlm?.prompt_n ?? 0) + (agenticLlm?.predicted_n ?? 0); + } else { + totalHistoricalTokens += (timings.prompt_n ?? 0) + (timings.predicted_n ?? 0); + } + } + + if (totalHistoricalTokens > 0) return true; + + if (!liveState) return false; + + const livePromptTokens = Math.max( + liveState.promptTokens ?? 0, + liveState.promptProgress?.processed ?? 0 + ); + const liveOutputTokens = liveState.outputTokensUsed ?? 0; + + return livePromptTokens > 0 || liveOutputTokens > 0; +} diff --git a/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormCurrentWorkingDirectory/ChatFormCurrentWorkingDirectory.svelte b/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormCurrentWorkingDirectory/ChatFormCurrentWorkingDirectory.svelte index ecd16984678e..59020eec2d96 100644 --- a/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormCurrentWorkingDirectory/ChatFormCurrentWorkingDirectory.svelte +++ b/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormCurrentWorkingDirectory/ChatFormCurrentWorkingDirectory.svelte @@ -1,8 +1,9 @@ - - - - Open working directory picker - - - event.preventDefault()} - onOpenAutoFocus={(event) => event.preventDefault()} - onkeydown={handleKeydown} - preventScroll={false} - side="top" - sideOffset={12} - > -
- + +{/if} + + +{#snippet picker(inDrawer: boolean)} +
+ + + {#if !fileSearchEnabled} +
{searchUnavailableMessage}
+ {:else if query.trim() && (search.isSearching || queryResults.length > 0 || searchError)} + nav.setHover(index)} + rawQuery={query} + results={queryResults} /> - - {#if !fileSearchEnabled} -
{searchUnavailableMessage}
- {:else if query.trim() && (search.isSearching || queryResults.length > 0 || searchError)} - nav.setHover(index)} - rawQuery={query} - results={queryResults} - /> - {/if} - - {#if pickerSupported && fileSearchEnabled} - + {/if} + + {#if directory} + + {/if} + + {#if homeBase && fileSearchEnabled} + + + + Searching in: + + {abbreviateHome(searchScope, homeBase)} + + {/if} +
+{/snippet} + + +{#if deviceStore.isMobile} + + + + Working directory + + Choose the directory the chat works in - - - Browse - - {/if} - - {#if homeBase && fileSearchEnabled} - - - - Searching in: - - {abbreviateHome(searchScope, homeBase)} - - {/if} -
-
- + + +
+ {@render picker(true)} +
+ + +{:else} + + + + event.preventDefault()} + onOpenAutoFocus={(event) => event.preventDefault()} + onkeydown={handleKeydown} + preventScroll={false} + side="top" + sideOffset={12} + > + {@render picker(false)} + + +{/if} diff --git a/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormInput/ChatFormInput.svelte b/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormInput/ChatFormInput.svelte index 4a2cc386af81..540a344dac7c 100644 --- a/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormInput/ChatFormInput.svelte +++ b/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormInput/ChatFormInput.svelte @@ -1,78 +1,234 @@ -{#if useRichInput} - + -{:else} - -{/if} +
+ + diff --git a/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormInput/ChatFormInputBasic.svelte b/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormInput/ChatFormInputBasic.svelte deleted file mode 100644 index 0cd4516d87c3..000000000000 --- a/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormInput/ChatFormInputBasic.svelte +++ /dev/null @@ -1,82 +0,0 @@ - - -
- -
diff --git a/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormInput/ChatFormInputRich.svelte b/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormInput/ChatFormInputRich.svelte deleted file mode 100644 index 70251ea0cbcc..000000000000 --- a/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormInput/ChatFormInputRich.svelte +++ /dev/null @@ -1,867 +0,0 @@ - - -
-
-
- - diff --git a/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormInput/mention-badge-widget.ts b/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormInput/mention-badge-widget.ts new file mode 100644 index 000000000000..5f32fa8d26ce --- /dev/null +++ b/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormInput/mention-badge-widget.ts @@ -0,0 +1,104 @@ +// File mention badges for the chat form input: in the block being edited, each +// `[name](file://path)` link shows as the same chip the rendered markdown draws. +// The link stays in the source untouched; the badge is an atomic stand-in for it. + +import type { EditorState, Range } from '@codemirror/state'; +import { Decoration, WidgetType } from '@codemirror/view'; +import { codeRanges } from '$lib/components/app/content/MarkdownEditor/markdown-syntax'; +import { SETTINGS_KEYS } from '$lib/constants'; +import { settingsStore } from '$lib/stores/settings/index.svelte'; +import { toolsStore } from '$lib/stores/tools.svelte'; +import { + decodeFileLinkPath, + fileMentionLinkRe, + getMentionBadgeIconPaths, + getMentionBadgeLabel, + MENTION_BADGE_CLASSNAME, + MENTION_BADGE_ICON_CLASSNAME, + MENTION_BADGE_SVG_ATTRIBUTES +} from '$lib/utils'; + +const MENTION_RE = new RegExp(fileMentionLinkRe().source, 'g'); + +class MentionBadgeWidget extends WidgetType { + readonly decodedPath: string; + readonly label: string; + + constructor( + readonly name: string, + readonly path: string + ) { + super(); + this.decodedPath = decodeFileLinkPath(path); + this.label = getMentionBadgeLabel( + name, + this.decodedPath, + settingsStore.getConfig(SETTINGS_KEYS.SHOW_FULL_PATH_IN_MENTIONS), + toolsStore.serverHome + ); + } + + // the label follows the mention settings, so a changed setting redraws the badge + eq(other: MentionBadgeWidget) { + return other.path === this.path && other.label === this.label; + } + + toDOM() { + const badge = document.createElement('span'); + + badge.title = this.decodedPath; + badge.className = `${MENTION_BADGE_CLASSNAME} align-middle`; + + const svg = document.createElementNS(MENTION_BADGE_SVG_ATTRIBUTES['xmlns'], 'svg'); + + for (const [attr, value] of Object.entries(MENTION_BADGE_SVG_ATTRIBUTES)) { + svg.setAttribute(attr, value); + } + + for (const cls of MENTION_BADGE_ICON_CLASSNAME.split(/\s+/).filter(Boolean)) { + svg.classList.add(cls); + } + + for (const d of getMentionBadgeIconPaths(this.path)) { + const path = document.createElementNS(MENTION_BADGE_SVG_ATTRIBUTES['xmlns'], 'path'); + + path.setAttribute('d', d); + svg.appendChild(path); + } + + const label = document.createElement('span'); + + label.classList.add('shrink-0', 'truncate'); + label.textContent = this.label; + + badge.append(svg, label); + + return badge; + } +} + +/** + * Badge widgets for the mention links between `from` and `to`. Code is literal: a + * mention link in a code block or in inline code stays text. + */ +export function fileMentionWidgets( + state: EditorState, + from: number, + to: number +): Range[] { + const code = codeRanges(state, from, to); + const widgets: Range[] = []; + + for (const match of state.doc.sliceString(from, to).matchAll(MENTION_RE)) { + const start = from + match.index; + const end = start + match[0].length; + + if (code.some(([codeFrom, codeTo]) => start < codeTo && end > codeFrom)) continue; + + const widget = new MentionBadgeWidget(match[1], match[2]); + + widgets.push(Decoration.replace({ widget }).range(start, end)); + } + + return widgets; +} diff --git a/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormPickers/ChatFormPicker/ChatFormPickerPopover.svelte b/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormPickers/ChatFormPicker/ChatFormPickerPopover.svelte index 2d91ceb5400c..f2ac4aba2988 100644 --- a/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormPickers/ChatFormPicker/ChatFormPickerPopover.svelte +++ b/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormPickers/ChatFormPicker/ChatFormPickerPopover.svelte @@ -4,6 +4,8 @@ interface Props { class?: string; + /** What the popover floats by, instead of the box its trigger fills. */ + customAnchor?: HTMLElement | null; isOpen?: boolean; srLabel?: string; onClose?: () => void; @@ -14,6 +16,7 @@ let { children, class: className = '', + customAnchor = null, isOpen = $bindable(false), onClose, onKeydown, @@ -40,6 +43,7 @@ event.preventDefault()} onkeydown={onKeydown} preventScroll={false} diff --git a/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormPickers/ChatFormPickerCommand.svelte b/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormPickers/ChatFormPickerCommand.svelte index e3ff9d780a41..6aa3c5e45f7b 100644 --- a/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormPickers/ChatFormPickerCommand.svelte +++ b/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormPickers/ChatFormPickerCommand.svelte @@ -15,6 +15,7 @@ */ interface Props { class?: string; + customAnchor?: HTMLElement | null; isOpen: boolean; query: string; commands: ChatFormCommand[]; @@ -22,7 +23,15 @@ onSelect: (command: ChatFormCommand) => void; } - let { class: className = '', commands, isOpen, onClose, onSelect, query }: Props = $props(); + let { + class: className = '', + commands, + customAnchor = null, + isOpen, + onClose, + onSelect, + query + }: Props = $props(); const trimmedQuery = $derived((query ?? '').trim().toLowerCase()); @@ -96,6 +105,7 @@ void; @@ -30,6 +31,7 @@ let { class: className = '', + customAnchor = null, isOpen = false, onClose, onPromptLoadComplete, @@ -358,6 +360,7 @@ void; onCommandSelect?: (command: ChatFormCommand) => void; @@ -35,12 +36,12 @@ } let { + anchor, commandQuery, commands = [], isCommandPickerOpen, isMentionPickerOpen, isPromptPickerOpen, - mentionAnchor, mentionQuery, onCommandPickerClose, onCommandSelect, @@ -80,6 +81,7 @@ {})} onSelect={onCommandSelect ?? (() => {})} @@ -88,6 +90,7 @@ {})} onOpened={onMentionOpened} diff --git a/tools/ui/src/lib/components/app/chat/ChatForm/chat-form-collapsed-height.ts b/tools/ui/src/lib/components/app/chat/ChatForm/chat-form-collapsed-height.ts new file mode 100644 index 000000000000..738152bfcb75 --- /dev/null +++ b/tools/ui/src/lib/components/app/chat/ChatForm/chat-form-collapsed-height.ts @@ -0,0 +1,89 @@ +import { CHAT_FORM_RESIZE_MS } from '$lib/constants'; + +export interface CollapsedHeightHoldOptions { + getCard: () => HTMLElement | undefined; + /** The editor's box, which sets its minimum and maximum height. */ + getEditorBox: () => HTMLElement | undefined; + /** The element holding the editor's text. */ + getContent: () => HTMLElement | undefined; + getValue: () => string; +} + +/** + * Holds the collapsed card's real height in `--chat-form-collapsed-height` from expanding + * until the collapse has settled, so a quick toggle never measures a card mid-transition. + */ +export function createCollapsedHeightHold(options: CollapsedHeightHoldOptions) { + let hold: { + card: number; + content: number; + editorMax: number; + editorMin: number; + text: string; + } | null = null; + let releaseTimer: ReturnType | undefined; + + const contentHeight = () => options.getContent()?.offsetHeight ?? 0; + + function setHeight(px: number | null) { + const style = options.getCard()?.style; + + if (px === null) style?.removeProperty('--chat-form-collapsed-height'); + else style?.setProperty('--chat-form-collapsed-height', `${px}px`); + } + + /** Before the form expands. */ + function start() { + const card = options.getCard(); + const box = options.getEditorBox(); + + clearTimeout(releaseTimer); + + if (hold || !card || !box) return; + + const editor = getComputedStyle(box); + + hold = { + card: card.getBoundingClientRect().height, + content: contentHeight(), + editorMax: parseFloat(editor.maxHeight) || Infinity, + editorMin: parseFloat(editor.minHeight) || 0, + text: options.getValue() + }; + + // in place with no transition of its own, so the one the switch starts begins here + card.style.transition = 'none'; + setHeight(hold.card); + void getComputedStyle(card).minHeight; + card.style.transition = ''; + } + + /** Before the form collapses: the height it settles at, held until the transition is over. */ + function settle() { + if (!hold) return; + + // the held height, unless the text changed meanwhile: then the height the card will + // have with it, its editor clamped as when collapsed + const held = hold; + const editorBox = (content: number) => + Math.max(held.editorMin, Math.min(content, held.editorMax)); + + setHeight( + options.getValue() === held.text + ? held.card + : held.card - editorBox(held.content) + editorBox(contentHeight()) + ); + + clearTimeout(releaseTimer); + releaseTimer = setTimeout(() => { + hold = null; + setHeight(null); + }, CHAT_FORM_RESIZE_MS); + } + + function destroy() { + clearTimeout(releaseTimer); + } + + return { destroy, settle, start }; +} diff --git a/tools/ui/src/lib/components/app/chat/ChatForm/chat-form-expand.ts b/tools/ui/src/lib/components/app/chat/ChatForm/chat-form-expand.ts new file mode 100644 index 000000000000..2ddd3a7cbdd8 --- /dev/null +++ b/tools/ui/src/lib/components/app/chat/ChatForm/chat-form-expand.ts @@ -0,0 +1,63 @@ +import { hasMarkdownStructure } from '$lib/components/app/content/MarkdownEditor/markdown-structure'; +import { CHAT_FORM_EXPAND_MIN_LINES } from '$lib/constants'; + +/** The collapsed card's `rounded-4xl` on a phone. */ +const COLLAPSED_CARD_RADIUS = '2rem'; + +export type SheetBox = { height: number; left: number; right: number; top: number }; +/** Where the card and its actions are on screen, and the card's top padding. */ +export type SheetRects = { actionsTop: number; card: SheetBox; padTop: number }; +export type SheetFrame = { actions: Keyframe; card: Keyframe }; + +/** + * Should a message expand the chat form on its own? Once it has several lines, or a + * markdown block in its finished lines: the line still being typed does not count, as + * a lone `-`, `#` or `1.` already parses as a block. + */ +export function shouldExpandChatForm(text: string): boolean { + const finishedLines = text.slice(0, text.lastIndexOf('\n') + 1); + + return ( + text.split('\n').length >= CHAT_FORM_EXPAND_MIN_LINES || hasMarkdownStructure(finishedLines) + ); +} + +/** The laid-out phone sheet as it is: full screen, square. */ +export const SHEET_AT_REST: SheetFrame = { + actions: { top: '0px' }, + card: { borderRadius: '0px', clipPath: 'inset(0 0 0 0 round 0px)', top: '0px' } +}; + +/** + * The laid-out `sheet` drawn as the collapsed card at `card`: moved there, clipped to its + * edges, its actions held at their place. Offsets, not transforms: WebKit composites + * transforms apart from the clip, and the two drift. + */ +export function sheetFrameAt(sheet: SheetRects, card: SheetRects): SheetFrame { + const rise = Math.max(0, sheet.padTop - card.padTop); + const dy = card.card.top - sheet.card.top - rise; + const clip = [ + rise, + sheet.card.right - card.card.right, + sheet.card.height - rise - card.card.height, + card.card.left - sheet.card.left + ].map((px) => `${Math.max(0, px)}px`); + + return { + actions: { top: `${card.actionsTop - sheet.actionsTop - dy}px` }, + card: { + borderRadius: COLLAPSED_CARD_RADIUS, + clipPath: `inset(${clip.join(' ')} round ${COLLAPSED_CARD_RADIUS})`, + top: `${dy}px` + } + }; +} + +/** `rects` moved down the screen by `dy`. */ +export function shiftSheetRects(rects: SheetRects, dy: number): SheetRects { + return { + ...rects, + actionsTop: rects.actionsTop + dy, + card: { ...rects.card, top: rects.card.top + dy } + }; +} diff --git a/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessage.svelte b/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessage.svelte index 0c32882ef60e..bed8e6dbaf01 100644 --- a/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessage.svelte +++ b/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessage.svelte @@ -112,6 +112,7 @@ let editedUploadedFiles = $state([]); let isEditing = $state(false); let showDeleteDialog = $state(false); + let showForkDialog = $state(false); let shouldBranchAfterEdit = $state(false); let textareaElement: HTMLTextAreaElement | undefined = $state(); @@ -188,9 +189,15 @@ navigateToSibling: handleNavigateToSibling, requestDelete: handleDelete, setShowDeleteDialog: handleShowDeleteDialogChange, + setShowForkDialog: (show: boolean) => { + showForkDialog = show; + }, get showDeleteDialog() { return showDeleteDialog; }, + get showForkDialog() { + return showForkDialog; + }, get siblingInfo() { return siblingInfo; } @@ -335,7 +342,7 @@ function focusMainChatForm() { if (deviceStore.isMobile) return; - document.querySelector('.chat-screen-form-wrapper textarea')?.focus(); + document.querySelector('.chat-screen-form-wrapper .cm-content')?.focus(); } async function handleSaveEdit() { diff --git a/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageAssistant/ChatMessageAssistant.svelte b/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageAssistant/ChatMessageAssistant.svelte index ff23acf3cf3d..d443b633af68 100644 --- a/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageAssistant/ChatMessageAssistant.svelte +++ b/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageAssistant/ChatMessageAssistant.svelte @@ -6,6 +6,7 @@ ChatMessageAssistantProcessingInfo, ChatMessageAssistantRawOutput, ChatMessageAssistantStatistics, + ChatMessageContextMenu, ChatMessageEditForm } from '$lib/components/app'; import { getChatMessageEditContext } from '$lib/contexts'; @@ -173,16 +174,23 @@ {#if editCtx.isEditing} {:else} - {#if showRawOutput} - - {:else} - - {/if} + + {#if showRawOutput} + + {:else} + + {/if} + {/if} {#if showProcessingInfoBottom} diff --git a/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageSystem/ChatMessageSystem.svelte b/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageSystem/ChatMessageSystem.svelte index 0c5d8fd7c094..3a7022227a4d 100644 --- a/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageSystem/ChatMessageSystem.svelte +++ b/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageSystem/ChatMessageSystem.svelte @@ -84,7 +84,7 @@