From 391fac16460f15233a7740550d858ac96df3419d Mon Sep 17 00:00:00 2001 From: Oliver Simons Date: Mon, 14 Sep 2026 19:07:53 +0200 Subject: [PATCH 01/23] ci : add ubuntu-cuda builds to release (#28186) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * release : add ubuntu-cuda build job (12.8/13.3, x64+arm64) * Add GCC 14 for CUDA arm64 builds in CI * Eplicit bash * Install git for CCCL fetch * Install git before we clone/checkout * Match CI names for WIndows * Whitelist llama.cpp repo to git * Use $GITHUB_WORKSPACE * Also ship dependent libs on Ubuntu Need NCCL additionally as it's pre-built available on Linux * Avoid duplicate files in packaged cudart * Copy NCCL license * Install CURL to fetch NCCL license * Update .github/workflows/release.yml Co-authored-by: Sigbjørn Skjæret * Remove NCCL until licensing has been confirmed --------- Co-authored-by: Sigbjørn Skjæret --- .github/workflows/release.yml | 143 ++++++++++++++++++++++++++++++++++ 1 file changed, 143 insertions(+) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 91dcbe48b..be36b0432 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -310,6 +310,145 @@ jobs: with: key: release-${{ matrix.os }}-vulkan + ubuntu-cuda: + name: ubuntu-cuda (${{ matrix.label }}, ${{ matrix.build }}) + needs: [check-release, ui-build] + if: ${{ needs.check-release.outputs.should_release == 'true' }} + + strategy: + matrix: + include: + # label = short version used in artifact names / release body + # cuda = full container image tag + - build: 'x64' + os: ubuntu-24.04 + cuda: '12.8.2' + label: '12.8' + defines: '-DGGML_CUDA_CUB_3DOT2=ON' + - build: 'x64' + os: ubuntu-24.04 + cuda: '13.3.1' + label: '13.3' + defines: '' + - build: 'arm64' + os: ubuntu-24.04-arm + cuda: '13.3.1' + label: '13.3' + defines: '' + + runs-on: ${{ matrix.os }} + container: nvidia/cuda:${{ matrix.cuda }}-devel-ubuntu24.04 + + permissions: + actions: write + + steps: + # the container has no git; install it before checkout so that a real git + # repository is created (the get-tag-name action and the build both need it) + - name: Install git + run: | + apt-get update + apt-get install -y --no-install-recommends git + + - name: Clone + id: checkout + uses: actions/checkout@v6 + with: + fetch-depth: 0 + + # checkout runs as the host user; in-container steps run as root, so git + # refuses to touch a repo it does not own. Mark the workspace as safe. + # use the env var: the github.workspace context holds the HOST path, + # GITHUB_WORKSPACE the container path + - name: Git safe directory + run: git config --global --add safe.directory "$GITHUB_WORKSPACE" + + - name: Download UI build + uses: actions/download-artifact@v7 + with: + name: llama-ui.zip + path: tools/ui/dist + + - name: Dependencies + id: depends + # container jobs default to sh (dash); need bash for the [[ ]] below + shell: bash + run: | + apt-get update + apt-get install -y --no-install-recommends build-essential cmake ninja-build libssl-dev jq python3-venv + # the container ships GCC 13, which does not know the 'sme' march + # feature used by the armv9.2 CPU variant of GGML_CPU_ALL_VARIANTS + if [[ "${{ matrix.build }}" == "arm64" ]]; then + apt-get install -y --no-install-recommends gcc-14 g++-14 + echo "CC=gcc-14" >> "$GITHUB_ENV" + echo "CXX=g++-14" >> "$GITHUB_ENV" + fi + + - name: ccache + uses: ggml-org/ccache-action@v1.2.24 + with: + key: release-ubuntu-${{ matrix.os }}-cuda-${{ matrix.label }}-${{ matrix.build }} + evict-old-files: 1d + max-size: "1G" + + - name: Build + id: cmake_build + # no CMAKE_CUDA_ARCHITECTURES: use the broad default arch set from + # ggml/src/ggml-cuda/CMakeLists.txt so the release binary covers many GPUs + run: | + cmake -B build \ + -DCMAKE_INSTALL_RPATH='$ORIGIN' \ + -DCMAKE_BUILD_WITH_INSTALL_RPATH=ON \ + -DGGML_BACKEND_DL=ON \ + -DGGML_NATIVE=OFF \ + -DGGML_CPU_ALL_VARIANTS=ON \ + -DGGML_CUDA=ON \ + -DGGML_CUDA_NCCL=OFF \ + ${{ env.CMAKE_ARGS }} ${{ matrix.defines }} + cmake --build build --config Release -j $(nproc) + + - name: Determine tag name + id: tag + uses: ./.github/actions/get-tag-name + + - name: Pack artifacts + id: pack_artifacts + run: | + cp LICENSE ./build/bin/ + tar -czvf llama-${{ steps.tag.outputs.name }}-bin-ubuntu-cuda-${{ matrix.label }}-${{ matrix.build }}.tar.gz --transform "s,^\.,llama-${{ steps.tag.outputs.name }}," -C ./build/bin . + + - name: Upload artifacts + uses: actions/upload-artifact@v6 + with: + path: llama-${{ steps.tag.outputs.name }}-bin-ubuntu-cuda-${{ matrix.label }}-${{ matrix.build }}.tar.gz + name: llama-bin-ubuntu-cuda-${{ matrix.label }}-${{ matrix.build }}.tar.gz + + # ship the CUDA runtime libraries the backend links against, mirroring + # the windows-cuda cudart zip - extract next to the binaries ($ORIGIN rpath) + - name: Pack CUDA runtime + id: pack_cuda_runtime + run: | + major="${{ matrix.label }}" + major="${major%%.*}" + mkdir -p ./cudart + # cp -L dereferences the SONAME symlinks into plain files, so the + # tarball holds exactly 3 files with no versioned duplicates + cp -L /usr/local/cuda/lib64/libcudart.so.${major} ./cudart/ + cp -L /usr/local/cuda/lib64/libcublas.so.${major} ./cudart/ + cp -L /usr/local/cuda/lib64/libcublasLt.so.${major} ./cudart/ + tar -czvf cudart-llama-${{ steps.tag.outputs.name }}-bin-ubuntu-cuda-${{ matrix.label }}-${{ matrix.build }}.tar.gz --transform "s,^\.,cudart-llama-${{ steps.tag.outputs.name }}-bin-ubuntu-cuda-${{ matrix.label }}-${{ matrix.build }}," -C ./cudart . + + - name: Upload CUDA runtime + uses: actions/upload-artifact@v6 + with: + path: cudart-llama-${{ steps.tag.outputs.name }}-bin-ubuntu-cuda-${{ matrix.label }}-${{ matrix.build }}.tar.gz + name: cudart-llama-bin-ubuntu-cuda-${{ matrix.label }}-${{ matrix.build }}.tar.gz + + - name: ccache-clear + uses: ./.github/actions/ccache-clear + with: + key: release-ubuntu-${{ matrix.os }}-cuda-${{ matrix.label }}-${{ matrix.build }} + android-arm64: needs: [check-release, ui-build] if: ${{ needs.check-release.outputs.should_release == 'true' }} @@ -1572,6 +1711,7 @@ jobs: - ubuntu-24-rocm - ubuntu-cpu - ubuntu-vulkan + - ubuntu-cuda - ubuntu-24-openvino - ubuntu-24-sycl - android-arm64 @@ -1703,6 +1843,9 @@ jobs: - [Ubuntu s390x (CPU)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-s390x.tar.gz) - [Ubuntu x64 (Vulkan)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-vulkan-x64.tar.gz) - [Ubuntu arm64 (Vulkan)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-vulkan-arm64.tar.gz) + - [Ubuntu x64 (CUDA 12)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-cuda-12.8-x64.tar.gz) - [CUDA 12.8 libraries](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/cudart-llama-${{ steps.tag.outputs.name }}-bin-ubuntu-cuda-12.8-x64.tar.gz) + - [Ubuntu x64 (CUDA 13)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-cuda-13.3-x64.tar.gz) - [CUDA 13.3 libraries](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/cudart-llama-${{ steps.tag.outputs.name }}-bin-ubuntu-cuda-13.3-x64.tar.gz) + - [Ubuntu arm64 (CUDA 13)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-cuda-13.3-arm64.tar.gz) - [CUDA 13.3 libraries](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/cudart-llama-${{ steps.tag.outputs.name }}-bin-ubuntu-cuda-13.3-arm64.tar.gz) - [Ubuntu x64 (ROCm 10.0)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-rocm-10.0-x64.tar.gz) - [Ubuntu x64 (OpenVINO)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-openvino-${{ needs.ubuntu-24-openvino.outputs.openvino_version }}-x64.tar.gz) - [Ubuntu x64 (SYCL FP32)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-sycl-fp32-x64.tar.gz) From bfdc32183d57f1e35bacf35c47d6311e2028bbbc Mon Sep 17 00:00:00 2001 From: uvos Date: Mon, 14 Sep 2026 20:26:22 +0200 Subject: [PATCH 02/23] HIP: fattn-mma: use fp32 accumulation on MFMA devices (#28576) use fp32 accumulators in fattn-mma on CDNA --- ggml/src/ggml-cuda/fattn-mma-f16.cuh | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/ggml/src/ggml-cuda/fattn-mma-f16.cuh b/ggml/src/ggml-cuda/fattn-mma-f16.cuh index 578f6cf79..a29065577 100644 --- a/ggml/src/ggml-cuda/fattn-mma-f16.cuh +++ b/ggml/src/ggml-cuda/fattn-mma-f16.cuh @@ -181,7 +181,7 @@ static constexpr __host__ __device__ fattn_mma_config ggml_cuda_fattn_mma_get_co GGML_CUDA_FATTN_MMA_CONFIG_CASE( 64, 64, 8, 128, 1, 64, 32, 32, 32, 1, true); GGML_CUDA_FATTN_MMA_CONFIG_CASE( 64, 64, 16, 256, 2, 64, 32, 32, 32, 1, true); GGML_CUDA_FATTN_MMA_CONFIG_CASE( 64, 64, 32, 256, 2, 64, 32, 32, 32, 1, true); - GGML_CUDA_FATTN_MMA_CONFIG_CASE( 64, 64, 64, 256, 4, 64, 32, 32, 32, 1, true); + GGML_CUDA_FATTN_MMA_CONFIG_CASE( 64, 64, 64, 256, 3, 64, 32, 32, 32, 1, true); GGML_CUDA_FATTN_MMA_CONFIG_CASE( 80, 80, 8, 256, 2, 64, 40, 40, 40, 1, true); GGML_CUDA_FATTN_MMA_CONFIG_CASE( 80, 80, 16, 256, 2, 64, 40, 40, 40, 1, true); @@ -1141,7 +1141,7 @@ template struct mma_tile_sizes { using T_C_KQ = tile<16, 16, float>; // column-major using T_A_VKQ = tile<16, 8, half2>; // row-major using T_B_VKQ = tile<16, 8, half2>; // column-major - using T_C_VKQ = tile<16, 8, half2>; // column-major + using T_C_VKQ = tile<16, 16, float>; // column-major }; #else // Volta template struct mma_tile_sizes { @@ -1227,7 +1227,9 @@ static __device__ __forceinline__ void flash_attn_ext_f16_process_tile( T_C_VKQ VKQ_C[cols_per_warp == 8 ? DV/T_C_VKQ::I : DV/(2*T_C_VKQ::J)]; #elif defined(AMD_WMMA_AVAILABLE) && defined(RDNA3) T_C_VKQ VKQ_C[DV % 32 != 0 ? DV/T_C_VKQ::J : DV/(2*T_C_VKQ::J)]; -#elif defined(AMD_WMMA_AVAILABLE) || defined(AMD_MFMA_AVAILABLE) +#elif defined(AMD_MFMA_AVAILABLE) + T_C_VKQ VKQ_C[ DV/T_C_VKQ::J]; +#elif defined(AMD_WMMA_AVAILABLE) T_C_VKQ VKQ_C[ DV/(2*T_C_VKQ::J)]; #else // Volta T_C_VKQ VKQ_C[ DV/(2*T_C_VKQ::J)]; From 96ffdc41ceb055e1c2d3d96667ae6d9f0ccb710b Mon Sep 17 00:00:00 2001 From: uvos Date: Mon, 14 Sep 2026 21:25:22 +0200 Subject: [PATCH 03/23] CI: hip-quality-check: ignore spill added in bfdc32183d57f1e35bacf35c47d6311e2028bbbc (#28909) the kernel spills 5 registers but is still faster than before the change --- scripts/hip/gcn-cdna-vgpr-check.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/hip/gcn-cdna-vgpr-check.py b/scripts/hip/gcn-cdna-vgpr-check.py index 40fb78941..f660735f4 100644 --- a/scripts/hip/gcn-cdna-vgpr-check.py +++ b/scripts/hip/gcn-cdna-vgpr-check.py @@ -64,6 +64,7 @@ def main(): '_ZL12rwkv_wkv_f32ILi128EEviiiiPKfS1_S1_S1_S1_S1_Pf', '_ZL9mul_mat_qIL9ggml_type10ELi64ELb1EEvPKcPKiS4_S4_PfS5_PKf15HIP_vector_typeIjLj3EEiiiiiS9_S9_iiiS9_S9_iiiS9_', '_ZL9mul_mat_qIL9ggml_type42ELi128ELb1EEvPKcPKiS4_S4_PfS5_PKf15HIP_vector_typeIjLj3EEiiiiiS9_S9_iiiS9_S9_iiiS9_', + '_ZL18flash_attn_ext_f16ILi576ELi512ELi2ELi32ELb0ELb1ELb0EEvPKcS1_S1_S1_S1_PKiPfP15HIP_vector_typeIfLj2EEffffjfiS5_IjLj3EEiiiiiiiiiiiliiliiiiil' } functions = parse_log_file(log_file) From 7cf1c54a96d4e950ffa614b94babf762803a8de7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sigbj=C3=B8rn=20Skj=C3=A6ret?= Date: Mon, 14 Sep 2026 22:21:39 +0200 Subject: [PATCH 04/23] ci : reuse build tag name when used instead of safe one (#28911) --- .github/actions/get-tag-name/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/get-tag-name/action.yml b/.github/actions/get-tag-name/action.yml index 7ace23b2a..46acce582 100644 --- a/.github/actions/get-tag-name/action.yml +++ b/.github/actions/get-tag-name/action.yml @@ -14,7 +14,7 @@ runs: run: | BUILD_NUMBER="$(git rev-list --count HEAD)" SHORT_HASH="$(git rev-parse --short=7 HEAD)" - if [[ "${{ env.BRANCH_NAME }}" == "master" ]]; then + if [[ "${{ env.BRANCH_NAME }}" == "master" || "${{ env.BRANCH_NAME }}" == "b${BUILD_NUMBER}" ]]; then echo "name=b${BUILD_NUMBER}" >> $GITHUB_OUTPUT else SAFE_NAME=$(echo "${{ env.BRANCH_NAME }}" | tr '/' '-') From 1bc7a5af0d14b1fb72f266abbd1237b394187115 Mon Sep 17 00:00:00 2001 From: Abhiram <78226909+geckguy@users.noreply.github.com> Date: Tue, 15 Sep 2026 04:41:26 +0530 Subject: [PATCH 05/23] webui: stop re-probing disabled /tools endpoint on every message (#28646) When /tools returns 403 (server started without tools), the web UI refetched the tool list before every chat message, since the guard treated an empty tool list as "not yet fetched". Each retry returned 403 and could trip fail2ban. Skip the refetch once the store flags the endpoint as disabled, and detect that state via the response status code instead of string- matching the error message. The tools panel keeps probing on open so the UI recovers once the server is restarted with tools enabled. Fixes #28299 --- tools/ui/src/lib/hooks/use-tools-panel.svelte.ts | 2 +- tools/ui/src/lib/stores/agentic/index.svelte.ts | 10 ++++++++-- tools/ui/src/lib/stores/tools.svelte.ts | 9 +++------ 3 files changed, 12 insertions(+), 9 deletions(-) diff --git a/tools/ui/src/lib/hooks/use-tools-panel.svelte.ts b/tools/ui/src/lib/hooks/use-tools-panel.svelte.ts index 21deed32d..cb361aad8 100644 --- a/tools/ui/src/lib/hooks/use-tools-panel.svelte.ts +++ b/tools/ui/src/lib/hooks/use-tools-panel.svelte.ts @@ -47,7 +47,7 @@ export function useToolsPanel(): UseToolsPanelReturn { if (toolsStore.toolGroups.length > 0) return null; - // Tools endpoint is unreachable (404) — server started without --tools + // Tools endpoint unreachable (403) — server started without tools if (toolsStore.isToolsEndpointUnreachable) { return `To enable Server Tools you need to run llama-server with ${CLI_FLAGS.TOOLS} all or ${CLI_FLAGS.TOOLS} flag. To see MCP Tools you need to add / enable MCP Server(s).`; } diff --git a/tools/ui/src/lib/stores/agentic/index.svelte.ts b/tools/ui/src/lib/stores/agentic/index.svelte.ts index 50db1be0c..121ee7739 100644 --- a/tools/ui/src/lib/stores/agentic/index.svelte.ts +++ b/tools/ui/src/lib/stores/agentic/index.svelte.ts @@ -315,8 +315,14 @@ class AgenticStore { // Clear any pending permissions/continue requests for this conversation when starting a new flow this.gates.clear(conversationId); - // Ensure server tools are fetched before checking if agentic is enabled - if (toolsStore.serverTools.length === 0 && !toolsStore.loading) { + // Ensure server tools are fetched before checking if agentic is enabled. + // A disabled /tools endpoint stays disabled for the life of the server, + // so the tools panel is the only place that probes it again. + if ( + toolsStore.serverTools.length === 0 && + !toolsStore.loading && + !toolsStore.isToolsEndpointUnreachable + ) { await toolsStore.fetchServerTools(); } diff --git a/tools/ui/src/lib/stores/tools.svelte.ts b/tools/ui/src/lib/stores/tools.svelte.ts index db05e3cd5..1d4133408 100644 --- a/tools/ui/src/lib/stores/tools.svelte.ts +++ b/tools/ui/src/lib/stores/tools.svelte.ts @@ -32,7 +32,7 @@ import { mcpStore } from '$lib/stores/mcp/index.svelte'; import { modelsStore } from '$lib/stores/models/index.svelte'; import { settingsStore } from '$lib/stores/settings/index.svelte'; import type { OpenAIToolDefinition, ToolEntry, ToolGroup } from '$lib/types'; -import { buildSandboxToolDefinition } from '$lib/utils'; +import { ApiError, buildSandboxToolDefinition } from '$lib/utils'; import { SvelteMap, SvelteSet } from 'svelte/reactivity'; /** Stable selection identity for a tool, shared by the disabled set and the permission store */ @@ -246,13 +246,10 @@ class ToolsStore { toolInfos.filter((info) => info.uses_cwd).map((info) => info.tool) ); } catch (err) { - const errorMessage = err instanceof Error ? err.message : String(err); - - this._error = errorMessage; + this._error = err instanceof Error ? err.message : String(err); // 403 from /tools means the server was started without --tools - // TODO: check status code instead of relying on message - if (errorMessage.includes('this feature is disabled')) { + if (err instanceof ApiError && err.status === 403) { this._toolsEndpointUnreachable = true; console.info('[ToolsStore] Server tools are disabled on the server'); } else { From 69eb250670f471586fcec69caacd3c014aefb185 Mon Sep 17 00:00:00 2001 From: Daniel Bevenius Date: Tue, 15 Sep 2026 05:26:09 +0200 Subject: [PATCH 06/23] cmake : use PROJECT_SOURCE_DIR instead of CMAKE_SOURCE_DIR (#28771) This commit updates cmake to use PROJECT_SOURCE_DIR instead of CMAKE_SOURCE_DIR for paths in function calls. The motivation for this is that when using add_subdirectory, CMAKE_SOURCE_DIR is fixed to the top-level projects source directory, that is the caller of add_subdirectory and not the llama.cpp root which means that common/common.h header will not be resolved. Refs: https://github.com/ggml-org/llama.cpp/pull/28091#issuecomment-5636106377 --- .ecrc | 2 +- app/CMakeLists.txt | 2 +- examples/eval-callback/CMakeLists.txt | 2 +- examples/test-cmake/.gitignore | 1 + examples/test-cmake/CMakeLists.txt | 19 ++++++++++++++----- examples/test-cmake/README.md | 19 ++++++++++++++----- examples/test-cmake/build.sh | 17 ++++++++++++++--- examples/test-cmake/test-cmake.cpp | 4 ++++ tests/CMakeLists.txt | 2 +- tools/server/CMakeLists.txt | 4 ++-- tools/tuning/CMakeLists.txt | 2 +- 11 files changed, 54 insertions(+), 20 deletions(-) diff --git a/.ecrc b/.ecrc index c68877ec2..0338e4faa 100644 --- a/.ecrc +++ b/.ecrc @@ -1,5 +1,5 @@ { - "Exclude": ["^\\.gitmodules$", "stb_image\\.h"], + "Exclude": ["^\\.gitmodules$", "stb_image\\.h", "examples/test-cmake/build/", "examples/test-cmake/build-subdir/"], "Disable": { "IndentSize": true } diff --git a/app/CMakeLists.txt b/app/CMakeLists.txt index 3450ff490..0b044228a 100644 --- a/app/CMakeLists.txt +++ b/app/CMakeLists.txt @@ -16,7 +16,7 @@ target_link_libraries(${TARGET} PRIVATE target_compile_features(${TARGET} PRIVATE cxx_std_17) # Automatically add all files from the 'licenses' directory -file(GLOB EXTRA_LICENSES "${CMAKE_SOURCE_DIR}/licenses/LICENSE-*") +file(GLOB EXTRA_LICENSES "${PROJECT_SOURCE_DIR}/licenses/LICENSE-*") foreach(FILE_PATH ${EXTRA_LICENSES}) get_filename_component(FILE_NAME "${FILE_PATH}" NAME) diff --git a/examples/eval-callback/CMakeLists.txt b/examples/eval-callback/CMakeLists.txt index 63fbe59dc..96e1e1b35 100644 --- a/examples/eval-callback/CMakeLists.txt +++ b/examples/eval-callback/CMakeLists.txt @@ -18,7 +18,7 @@ if(LLAMA_BUILD_TESTS) -DDEST=${MODEL_DEST} -DNAME=${MODEL_NAME} -DHASH=${MODEL_HASH} - -P ${CMAKE_SOURCE_DIR}/cmake/download-models.cmake + -P ${PROJECT_SOURCE_DIR}/cmake/download-models.cmake ) set_tests_properties(${TEST_TARGET}-download-model PROPERTIES FIXTURES_SETUP ${TEST_TARGET}-download-model) add_test(NAME ${TEST_TARGET} COMMAND llama-eval-callback -m "${MODEL_DEST}" --prompt hello --seed 42 -ngl 0) diff --git a/examples/test-cmake/.gitignore b/examples/test-cmake/.gitignore index 0ddff317a..b630ddb7d 100644 --- a/examples/test-cmake/.gitignore +++ b/examples/test-cmake/.gitignore @@ -1,3 +1,4 @@ llama-build-install install build +build-subdir diff --git a/examples/test-cmake/CMakeLists.txt b/examples/test-cmake/CMakeLists.txt index ed5cb1f3c..6ceb3359e 100644 --- a/examples/test-cmake/CMakeLists.txt +++ b/examples/test-cmake/CMakeLists.txt @@ -3,11 +3,20 @@ project(llama-simple) set(CMAKE_CXX_STANDARD 17) -find_package(llama 0.1.0 REQUIRED) +option(LLAMA_TEST_USE_SUBDIR "Use add_subdirectory instead of find_package" OFF) + +if(LLAMA_TEST_USE_SUBDIR) + add_subdirectory(../../ llama.cpp) +else() + find_package(llama 0.1.0 REQUIRED) +endif() add_executable(test-cmake test-cmake.cpp) target_link_libraries(test-cmake PRIVATE llama) -target_compile_definitions(test-cmake PRIVATE - LLAMA_BUILD_NUMBER=${LLAMA_BUILD_NUMBER} - LLAMA_BUILD_COMMIT="${LLAMA_BUILD_COMMIT}" -) + +if(DEFINED LLAMA_BUILD_NUMBER) + target_compile_definitions(test-cmake PRIVATE + LLAMA_BUILD_NUMBER=${LLAMA_BUILD_NUMBER} + LLAMA_BUILD_COMMIT="${LLAMA_BUILD_COMMIT}" + ) +endif() diff --git a/examples/test-cmake/README.md b/examples/test-cmake/README.md index 2f6a2fcfe..03895abfb 100644 --- a/examples/test-cmake/README.md +++ b/examples/test-cmake/README.md @@ -5,17 +5,18 @@ enable troubleshooting issues and exploration. The idea is that this can be used after making changes to llama.cpp installation cmake configuration and then verify it locally. -### Usage -The following will configure, build, and install llama.cpp +### find_package +The following will configure, build, and install llama.cpp, and the build a +project that uses find_package to use the installation. Configuring/build/install: ```console ./build-install.sh ``` The above command will create a directory named `install` in the current directory -which will have the follwing files in its lib directory: +which will have the following files in its lib directory: ```console -(venv) $ ls install/lib/ +$ ls install/lib/ cmake libggml.so libllama-common.so.0 libllama.so.0.1.0 llama.cpp libggml-base.so libggml.so.0 libllama-common.so.0.1.0 libmtmd.so pkgconfig libggml-base.so.0 libggml.so.0.19.0 libllama.so libmtmd.so.0 @@ -24,7 +25,7 @@ libggml-base.so.0.19.0 libllama-common.so libllama.so.0 libmtmd.so Build/run this project using the installation created above: ```console -(venv) $ ./build.sh +$ ./build.sh -- Configuring done (0.0s) -- Generating done (0.0s) -- Build files have been written to: /path/to/llama.cpp/examples/test-cmake/build @@ -34,3 +35,11 @@ Build/run this project using the installation created above: load_backend: loaded CPU backend from /path/to/llama.cpp/examples/test-cmake/install/lib/llama.cpp/libggml-cpu-alderlake.so [test-cmake] Backend initialized. ``` + +### add_subdirectory +The following will use add_subdirectory to include llama.cpp in a cmake project +and is intended to simulate projects that build llama.cpp in this way. + +```console +$ USE_SUBDIR=ON ./build.sh +``` diff --git a/examples/test-cmake/build.sh b/examples/test-cmake/build.sh index a212732b8..869a64160 100755 --- a/examples/test-cmake/build.sh +++ b/examples/test-cmake/build.sh @@ -2,6 +2,17 @@ set -e -cmake -S . -B build -DCMAKE_PREFIX_PATH="${PWD}/install" -cmake --build build -LD_LIBRARY_PATH="${PWD}/install/lib/llama.cpp:${PWD}/install/lib${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" ./build/test-cmake +if [ "${USE_SUBDIR:-OFF}" = "ON" ]; then + BUILD_DIR="build-subdir" + CMAKE_ARGS="-DLLAMA_TEST_USE_SUBDIR=ON -DLLAMA_BUILD_COMMON=ON -DLLAMA_BUILD_TOOLS=ON -DLLAMA_BUILD_SERVER=ON-DLLAMA_BUILD_TESTS=ON" + LIB_PATH="${PWD}/${BUILD_DIR}/bin" +else + BUILD_DIR="build" + CMAKE_ARGS="-DCMAKE_PREFIX_PATH=${PWD}/install" + LIB_PATH="${PWD}/install/lib/llama.cpp" +fi + +cmake --fresh -S . -B "${BUILD_DIR}" ${CMAKE_ARGS} +cmake --build "${BUILD_DIR}" -j 8 + +LD_LIBRARY_PATH="${LIB_PATH}:${PWD}/install/lib${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" "./${BUILD_DIR}/test-cmake" diff --git a/examples/test-cmake/test-cmake.cpp b/examples/test-cmake/test-cmake.cpp index dc1a9ae60..fea27c7e8 100644 --- a/examples/test-cmake/test-cmake.cpp +++ b/examples/test-cmake/test-cmake.cpp @@ -2,8 +2,12 @@ #include int main(void) { +#ifdef LLAMA_BUILD_NUMBER printf("[test-cmake] llama.cpp version: %s, build: %d (%s)\n", llama_version(), LLAMA_BUILD_NUMBER, LLAMA_BUILD_COMMIT); +#else + printf("[test-cmake] llama.cpp version: %s\n", llama_version()); +#endif printf("[test-cmake] ggml version: %s, commit: %s\n", ggml_version(), ggml_commit()); printf("[test-cmake] Initializing backend...\n"); llama_backend_init(); diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index cca90ef30..a398344c8 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -289,7 +289,7 @@ add_test(NAME test-download-model COMMAND ${CMAKE_COMMAND} -DDEST=${MODEL_DEST} -DNAME=${MODEL_NAME} -DHASH=${MODEL_HASH} - -P ${CMAKE_SOURCE_DIR}/cmake/download-models.cmake + -P ${PROJECT_SOURCE_DIR}/cmake/download-models.cmake ) set_tests_properties(test-download-model PROPERTIES FIXTURES_SETUP test-download-model) diff --git a/tools/server/CMakeLists.txt b/tools/server/CMakeLists.txt index 43c245633..4adaaceef 100644 --- a/tools/server/CMakeLists.txt +++ b/tools/server/CMakeLists.txt @@ -30,7 +30,7 @@ if (BUILD_SHARED_LIBS) endif() target_include_directories(${TARGET} PRIVATE ../mtmd) -target_include_directories(${TARGET} PRIVATE ${CMAKE_SOURCE_DIR}) +target_include_directories(${TARGET} PRIVATE ${PROJECT_SOURCE_DIR}) target_link_libraries(${TARGET} PUBLIC llama-common mtmd ${CMAKE_THREAD_LIBS_INIT}) # llama-server-impl: server logic, reusable by app @@ -47,7 +47,7 @@ add_library(${TARGET} set_target_properties(${TARGET} PROPERTIES WINDOWS_EXPORT_ALL_SYMBOLS ON) target_include_directories(${TARGET} PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}) -target_include_directories(${TARGET} PRIVATE ../mtmd ${CMAKE_SOURCE_DIR}) +target_include_directories(${TARGET} PRIVATE ../mtmd ${PROJECT_SOURCE_DIR}) target_link_libraries(${TARGET} PUBLIC server-context llama-ui cpp-httplib ${CMAKE_THREAD_LIBS_INIT}) add_dependencies(${TARGET} llama-ui-assets) diff --git a/tools/tuning/CMakeLists.txt b/tools/tuning/CMakeLists.txt index 39ff00180..f07983882 100644 --- a/tools/tuning/CMakeLists.txt +++ b/tools/tuning/CMakeLists.txt @@ -3,7 +3,7 @@ set(TARGET ggml-metal-tuning) add_executable(${TARGET} main.cpp bench.cpp fa-vec.cpp) target_link_libraries(${TARGET} PRIVATE ggml ${CMAKE_THREAD_LIBS_INIT}) target_compile_features(${TARGET} PRIVATE cxx_std_17) -target_include_directories(${TARGET} PRIVATE ${CMAKE_SOURCE_DIR}/ggml/src/ggml-metal) +target_include_directories(${TARGET} PRIVATE ${PROJECT_SOURCE_DIR}/ggml/src/ggml-metal) if(LLAMA_TOOLS_INSTALL) install(TARGETS ${TARGET} RUNTIME) From 4c9233c034fc450dcf34c7c0988aebe6da5cdf1a Mon Sep 17 00:00:00 2001 From: Aman Karki Date: Tue, 15 Sep 2026 09:12:21 +0530 Subject: [PATCH 07/23] cuda : enable i16 and i32 for DUP (#28897) * cuda : enable i16 and i32 for DUP * docs : update ops table for DUP on CUDA --- docs/ops.md | 2 +- docs/ops/CUDA.csv | 8 ++++---- ggml/src/ggml-cuda/cpy.cu | 8 ++++++++ ggml/src/ggml-cuda/ggml-cuda.cu | 5 +---- 4 files changed, 14 insertions(+), 9 deletions(-) diff --git a/docs/ops.md b/docs/ops.md index cc8d25382..ceb5d46cf 100644 --- a/docs/ops.md +++ b/docs/ops.md @@ -44,7 +44,7 @@ Legend: | DSV4_HC_COMB | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | | DSV4_HC_POST | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | | DSV4_HC_PRE | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | -| DUP | ❌ | ✅ | ✅ | 🟡 | ❌ | ❌ | 🟡 | 🟡 | ✅ | ✅ | ❌ | ❌ | ❌ | +| DUP | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ | 🟡 | 🟡 | ✅ | ✅ | ❌ | ❌ | ❌ | | ELU | ❌ | ✅ | ✅ | 🟡 | 🟡 | ❌ | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ | | EXP | ❌ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ | | EXPM1 | ❌ | ❌ | ✅ | 🟡 | 🟡 | ❌ | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ | diff --git a/docs/ops/CUDA.csv b/docs/ops/CUDA.csv index 22c84dd14..570bdf75e 100644 --- a/docs/ops/CUDA.csv +++ b/docs/ops/CUDA.csv @@ -5000,14 +5000,14 @@ "CUDA0","REPEAT_BACK","type=f32,ne=[8,6,4,2],nr=[1,1,1,2],v=1","support","1","yes","CUDA" "CUDA0","DUP","type=f32,ne=[10,10,20,1]","support","1","yes","CUDA" "CUDA0","DUP","type=f16,ne=[10,10,20,1]","support","1","yes","CUDA" -"CUDA0","DUP","type=i32,ne=[10,10,20,1]","support","0","no","CUDA" -"CUDA0","DUP","type=i16,ne=[10,10,20,1]","support","0","no","CUDA" +"CUDA0","DUP","type=i32,ne=[10,10,20,1]","support","1","yes","CUDA" +"CUDA0","DUP","type=i16,ne=[10,10,20,1]","support","1","yes","CUDA" "CUDA0","DUP","type=f32,ne=[10,10,5,1],permute=[0,2,1,3]","support","1","yes","CUDA" "CUDA0","DUP","type=f16,ne=[10,10,5,1],permute=[0,2,1,3]","support","1","yes","CUDA" "CUDA0","DUP","type=f32,ne=[10,10,5,1],permute=[1,0,2,3]","support","1","yes","CUDA" "CUDA0","DUP","type=f16,ne=[10,10,5,1],permute=[1,0,2,3]","support","1","yes","CUDA" -"CUDA0","DUP","type=i16,ne=[10,8,3,1],permute=[0,2,1,3]","support","0","no","CUDA" -"CUDA0","DUP","type=i16,ne=[10,8,3,1],permute=[1,2,0,3]","support","0","no","CUDA" +"CUDA0","DUP","type=i16,ne=[10,8,3,1],permute=[0,2,1,3]","support","1","yes","CUDA" +"CUDA0","DUP","type=i16,ne=[10,8,3,1],permute=[1,2,0,3]","support","1","yes","CUDA" "CUDA0","SET","type_src=f32,type_dst=f32,ne=[6,5,4,3],dim=1","support","1","yes","CUDA" "CUDA0","SET","type_src=f32,type_dst=f32,ne=[6,5,4,3],dim=2","support","1","yes","CUDA" "CUDA0","SET","type_src=f32,type_dst=f32,ne=[6,5,4,3],dim=3","support","1","yes","CUDA" diff --git a/ggml/src/ggml-cuda/cpy.cu b/ggml/src/ggml-cuda/cpy.cu index fd7ffc0bc..7a9984585 100644 --- a/ggml/src/ggml-cuda/cpy.cu +++ b/ggml/src/ggml-cuda/cpy.cu @@ -589,6 +589,14 @@ void ggml_cuda_cpy(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, gg ggml_cpy_scalar_cuda (src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream); } + } else if (src0->type == GGML_TYPE_I16 && src1->type == GGML_TYPE_I16) { + if (can_be_transposed) { + ggml_cpy_scalar_cuda + (src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream); + } else { + ggml_cpy_scalar_cuda + (src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream); + } } else if (src0->type == GGML_TYPE_F32 && src1->type == GGML_TYPE_I32) { if (contiguous_srcs) { ggml_cpy_scalar_contiguous_cuda diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu index 790553888..43003245c 100644 --- a/ggml/src/ggml-cuda/ggml-cuda.cu +++ b/ggml/src/ggml-cuda/ggml-cuda.cu @@ -5298,10 +5298,7 @@ static bool ggml_backend_cuda_device_supports_op(ggml_backend_dev_t dev, const g return false; } break; case GGML_OP_DUP: - { - ggml_type src0_type = op->src[0]->type; - return src0_type != GGML_TYPE_I32 && src0_type != GGML_TYPE_I16; - } break; + return true; case GGML_OP_ARGMAX: case GGML_OP_COUNT_EQUAL: { From 987498f4592a76897863cf53711dce38380c082b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sigbj=C3=B8rn=20Skj=C3=A6ret?= Date: Tue, 15 Sep 2026 09:04:35 +0200 Subject: [PATCH 08/23] ci : fix android release (#28936) --- .github/workflows/release.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index be36b0432..76b855c45 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -484,6 +484,7 @@ jobs: uses: android-actions/setup-android@40fd30fb8d7440372e1316f5d1809ec01dcd3699 # v4.0.1 with: log-accepted-android-sdk-licenses: false + packages: 'platform-tools' - name: Install NDK run: | From 0ecb159c9e93056a4742afe4195d05a2912b1746 Mon Sep 17 00:00:00 2001 From: shivamkumard-ctrl Date: Tue, 15 Sep 2026 13:34:17 +0530 Subject: [PATCH 09/23] ci: Bump CUDA Windows x64 builds to 13.4.1 (#28930) --- .github/actions/windows-setup-cuda/action.yml | 54 +++++++++---------- .github/workflows/build-cuda-windows.yml | 2 +- .github/workflows/release.yml | 4 +- 3 files changed, 30 insertions(+), 30 deletions(-) diff --git a/.github/actions/windows-setup-cuda/action.yml b/.github/actions/windows-setup-cuda/action.yml index 2048740a2..e67b6321e 100644 --- a/.github/actions/windows-setup-cuda/action.yml +++ b/.github/actions/windows-setup-cuda/action.yml @@ -100,36 +100,36 @@ runs: echo "CUDA_PATH=C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.1" | Out-File -FilePath $env:GITHUB_ENV -Append -Encoding utf8 echo "CUDA_PATH_V13_1=C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.1" | Out-File -FilePath $env:GITHUB_ENV -Append -Encoding utf8 - - name: Install Cuda Toolkit 13.3 - if: ${{ inputs.cuda_version == '13.3' }} + - name: Install Cuda Toolkit 13.4 for x64 + if: ${{ inputs.cuda_version == '13.4' && inputs.cuda_arch == 'x64' }} shell: pwsh run: | - mkdir -p "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3" + mkdir -p "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" choco install unzip -y - curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_crt/windows-x86_64/cuda_crt-windows-x86_64-13.3.33-archive.zip" - curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_cudart/windows-x86_64/cuda_cudart-windows-x86_64-13.3.29-archive.zip" - curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_nvcc/windows-x86_64/cuda_nvcc-windows-x86_64-13.3.33-archive.zip" - curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_nvrtc/windows-x86_64/cuda_nvrtc-windows-x86_64-13.3.33-archive.zip" - curl -O "https://developer.download.nvidia.com/compute/cuda/redist/libcublas/windows-x86_64/libcublas-windows-x86_64-13.5.1.27-archive.zip" - curl -O "https://developer.download.nvidia.com/compute/cuda/redist/libnvvm/windows-x86_64/libnvvm-windows-x86_64-13.3.33-archive.zip" - curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_nvtx/windows-x86_64/cuda_nvtx-windows-x86_64-13.3.29-archive.zip" - curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_profiler_api/windows-x86_64/cuda_profiler_api-windows-x86_64-13.3.27-archive.zip" - curl -O "https://developer.download.nvidia.com/compute/cuda/redist/visual_studio_integration/windows-x86_64/visual_studio_integration-windows-x86_64-13.3.27-archive.zip" - curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cccl/windows-x86_64/cccl-windows-x86_64-13.3.3.3.1-archive.zip" - unzip '*.zip' -d "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3" - xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3\cuda_crt-windows-x86_64-13.3.33-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3" /E /I /H /Y - xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3\cuda_cudart-windows-x86_64-13.3.29-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3" /E /I /H /Y - xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3\cuda_nvcc-windows-x86_64-13.3.33-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3" /E /I /H /Y - xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3\cuda_nvrtc-windows-x86_64-13.3.33-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3" /E /I /H /Y - xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3\libcublas-windows-x86_64-13.5.1.27-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3" /E /I /H /Y - xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3\libnvvm-windows-x86_64-13.3.33-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3" /E /I /H /Y - xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3\cuda_nvtx-windows-x86_64-13.3.29-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3" /E /I /H /Y - xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3\cuda_profiler_api-windows-x86_64-13.3.27-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3" /E /I /H /Y - xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3\visual_studio_integration-windows-x86_64-13.3.27-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3" /E /I /H /Y - xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3\cccl-windows-x86_64-13.3.3.3.1-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3" /E /I /H /Y - echo "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3\bin" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append - echo "CUDA_PATH=C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3" | Out-File -FilePath $env:GITHUB_ENV -Append -Encoding utf8 - echo "CUDA_PATH_V13_3=C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3" | Out-File -FilePath $env:GITHUB_ENV -Append -Encoding utf8 + curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_crt/windows-x86_64/cuda_crt-windows-x86_64-13.4.59-archive.zip" + curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_cudart/windows-x86_64/cuda_cudart-windows-x86_64-13.4.49-archive.zip" + curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_nvcc/windows-x86_64/cuda_nvcc-windows-x86_64-13.4.59-archive.zip" + curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_nvrtc/windows-x86_64/cuda_nvrtc-windows-x86_64-13.4.59-archive.zip" + curl -O "https://developer.download.nvidia.com/compute/cuda/redist/libcublas/windows-x86_64/libcublas-windows-x86_64-13.7.0.27-archive.zip" + curl -O "https://developer.download.nvidia.com/compute/cuda/redist/libnvvm/windows-x86_64/libnvvm-windows-x86_64-13.4.59-archive.zip" + curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_nvtx/windows-x86_64/cuda_nvtx-windows-x86_64-13.4.49-archive.zip" + curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_profiler_api/windows-x86_64/cuda_profiler_api-windows-x86_64-13.4.49-archive.zip" + curl -O "https://developer.download.nvidia.com/compute/cuda/redist/visual_studio_integration/windows-x86_64/visual_studio_integration-windows-x86_64-13.4.49-archive.zip" + curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cccl/windows-x86_64/cccl-windows-x86_64-13.3.4.2.1-archive.zip" + unzip '*.zip' -d "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" + xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4\cuda_crt-windows-x86_64-13.4.59-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" /E /I /H /Y + xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4\cuda_cudart-windows-x86_64-13.4.49-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" /E /I /H /Y + xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4\cuda_nvcc-windows-x86_64-13.4.59-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" /E /I /H /Y + xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4\cuda_nvrtc-windows-x86_64-13.4.59-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" /E /I /H /Y + xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4\libcublas-windows-x86_64-13.7.0.27-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" /E /I /H /Y + xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4\libnvvm-windows-x86_64-13.4.59-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" /E /I /H /Y + xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4\cuda_nvtx-windows-x86_64-13.4.49-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" /E /I /H /Y + xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4\cuda_profiler_api-windows-x86_64-13.4.49-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" /E /I /H /Y + xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4\visual_studio_integration-windows-x86_64-13.4.49-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" /E /I /H /Y + xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4\cccl-windows-x86_64-13.3.4.2.1-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" /E /I /H /Y + echo "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4\bin" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append + echo "CUDA_PATH=C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" | Out-File -FilePath $env:GITHUB_ENV -Append -Encoding utf8 + echo "CUDA_PATH_V13_4=C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" | Out-File -FilePath $env:GITHUB_ENV -Append -Encoding utf8 - name: Install Cuda Toolkit 13.4 for ARM64 if: ${{ inputs.cuda_version == '13.4' && inputs.cuda_arch == 'arm64' }} diff --git a/.github/workflows/build-cuda-windows.yml b/.github/workflows/build-cuda-windows.yml index 416724ec7..e08553e6c 100644 --- a/.github/workflows/build-cuda-windows.yml +++ b/.github/workflows/build-cuda-windows.yml @@ -34,7 +34,7 @@ jobs: - cuda: '12.4' arch: x64 defines: '-DGGML_CUDA_CUB_3DOT2=ON' - - cuda: '13.3' + - cuda: '13.4' arch: x64 defines: '' - cuda: '13.4' diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 76b855c45..8389f017b 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1127,7 +1127,7 @@ jobs: - cuda: '12.4' arch: x64 defines: '-DGGML_CUDA_CUB_3DOT2=ON' - - cuda: '13.3' + - cuda: '13.4' arch: x64 defines: '' - cuda: '13.4' @@ -1860,7 +1860,7 @@ jobs: - [Windows arm64 (CPU)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-win-cpu-arm64.zip) - [Windows arm64 (OpenCL Adreno)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-win-opencl-adreno-arm64.zip) - [Windows x64 (CUDA 12)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-win-cuda-12.4-x64.zip) - [CUDA 12.4 DLLs](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/cudart-llama-bin-win-cuda-12.4-x64.zip) - - [Windows x64 (CUDA 13)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-win-cuda-13.3-x64.zip) - [CUDA 13.3 DLLs](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/cudart-llama-bin-win-cuda-13.3-x64.zip) + - [Windows x64 (CUDA 13)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-win-cuda-13.4-x64.zip) - [CUDA 13.4 DLLs](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/cudart-llama-bin-win-cuda-13.4-x64.zip) - [Windows arm64 (CUDA 13)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-win-cuda-13.4-arm64.zip) - [CUDA 13.4 DLLs](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/cudart-llama-bin-win-cuda-13.4-arm64.zip) - [Windows x64 (Vulkan)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-win-vulkan-x64.zip) - [Windows x64 (OpenVINO)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-win-openvino-${{ needs.windows-openvino.outputs.openvino_version }}-x64.zip) From 1e7bcf3da4b2741868d152fa47976fb2501c85e3 Mon Sep 17 00:00:00 2001 From: Yanzhao Wang Date: Tue, 15 Sep 2026 01:10:40 -0700 Subject: [PATCH 10/23] metal : add FA kernels for HSK=96, HSV=64 (MiniCPM3) (#28599) * metal : add FA kernels for HSK=96, HSV=64 (MiniCPM3) MiniCPM3 sets attention.key_length to 96 and does not set attention.value_length, which defaults to n_embd / n_head = 64. Metal had no (96, 64) instantiation, so -fa auto aborted on the missing kernel_flash_attn_ext_vec_f16_dk96_dv64. Instantiate the tile kernel at (96, 64) for every K/V type that already has (96, 96), and the vec kernel for the NE=4 configurations. Of the NE values the vec dispatch considers, only NE=4 works here, because NL = 32/NE has to divide both DK/4 = 24 and DV/4 = 16. * tests : avoid redundant FA vec slice coverage --- ggml/src/ggml-metal/ggml-metal-ops.cpp | 1 + ggml/src/ggml-metal/ggml-metal-tuning.cpp | 3 +++ ggml/src/ggml-metal/kernels/fa.metal | 31 +++++++++++++++++++++++ tests/test-backend-ops.cpp | 3 ++- 4 files changed, 37 insertions(+), 1 deletion(-) diff --git a/ggml/src/ggml-metal/ggml-metal-ops.cpp b/ggml/src/ggml-metal/ggml-metal-ops.cpp index b4e87cb2c..da0040a0c 100644 --- a/ggml/src/ggml-metal/ggml-metal-ops.cpp +++ b/ggml/src/ggml-metal/ggml-metal-ops.cpp @@ -2923,6 +2923,7 @@ static int ggml_metal_op_flash_attn_ext_n_kv_max_sparse(const ggml_tensor * op) const bool dk_dv_ok = (dk == 32 && dv == 32) || (dk == 64 && dv == 64) || (dk == 96 && dv == 96) || + (dk == 96 && dv == 64) || (dk == 128 && dv == 128) || (dk == 192 && dv == 128) || (dk == 192 && dv == 192) || diff --git a/ggml/src/ggml-metal/ggml-metal-tuning.cpp b/ggml/src/ggml-metal/ggml-metal-tuning.cpp index 2323269c4..a1d28638a 100644 --- a/ggml/src/ggml-metal/ggml-metal-tuning.cpp +++ b/ggml/src/ggml-metal/ggml-metal-tuning.cpp @@ -34,6 +34,9 @@ int fa_vec_baseline_ne(int dk, int dv) { if (dk == 96 && dv == 96) { return 4; } + if (dk == 96 && dv == 64) { + return 4; + } if (dk == 128 && dv == 128) { return 1; } diff --git a/ggml/src/ggml-metal/kernels/fa.metal b/ggml/src/ggml-metal/kernels/fa.metal index d0e928d73..71e6e373e 100644 --- a/ggml/src/ggml-metal/kernels/fa.metal +++ b/ggml/src/ggml-metal/kernels/fa.metal @@ -930,6 +930,7 @@ template [[host_name("kernel_flash_attn_ext_f32_dk64_dv64" )]] kernel flash_at template [[host_name("kernel_flash_attn_ext_f32_dk72_dv72" )]] kernel flash_attn_ext_t kernel_flash_attn_ext; template [[host_name("kernel_flash_attn_ext_f32_dk80_dv80" )]] kernel flash_attn_ext_t kernel_flash_attn_ext; template [[host_name("kernel_flash_attn_ext_f32_dk96_dv96" )]] kernel flash_attn_ext_t kernel_flash_attn_ext; +template [[host_name("kernel_flash_attn_ext_f32_dk96_dv64" )]] kernel flash_attn_ext_t kernel_flash_attn_ext; template [[host_name("kernel_flash_attn_ext_f32_dk112_dv112")]] kernel flash_attn_ext_t kernel_flash_attn_ext; template [[host_name("kernel_flash_attn_ext_f32_dk128_dv128")]] kernel flash_attn_ext_t kernel_flash_attn_ext; template [[host_name("kernel_flash_attn_ext_f32_dk192_dv192")]] kernel flash_attn_ext_t kernel_flash_attn_ext; @@ -946,6 +947,7 @@ template [[host_name("kernel_flash_attn_ext_f16_dk64_dv64" )]] kernel flash_at template [[host_name("kernel_flash_attn_ext_f16_dk72_dv72" )]] kernel flash_attn_ext_t kernel_flash_attn_ext; template [[host_name("kernel_flash_attn_ext_f16_dk80_dv80" )]] kernel flash_attn_ext_t kernel_flash_attn_ext; template [[host_name("kernel_flash_attn_ext_f16_dk96_dv96" )]] kernel flash_attn_ext_t kernel_flash_attn_ext; +template [[host_name("kernel_flash_attn_ext_f16_dk96_dv64" )]] kernel flash_attn_ext_t kernel_flash_attn_ext; template [[host_name("kernel_flash_attn_ext_f16_dk112_dv112")]] kernel flash_attn_ext_t kernel_flash_attn_ext; template [[host_name("kernel_flash_attn_ext_f16_dk128_dv128")]] kernel flash_attn_ext_t kernel_flash_attn_ext; template [[host_name("kernel_flash_attn_ext_f16_dk192_dv192")]] kernel flash_attn_ext_t kernel_flash_attn_ext; @@ -963,6 +965,7 @@ template [[host_name("kernel_flash_attn_ext_bf16_dk64_dv64" )]] kernel flash_at template [[host_name("kernel_flash_attn_ext_bf16_dk72_dv72" )]] kernel flash_attn_ext_t kernel_flash_attn_ext; template [[host_name("kernel_flash_attn_ext_bf16_dk80_dv80" )]] kernel flash_attn_ext_t kernel_flash_attn_ext; template [[host_name("kernel_flash_attn_ext_bf16_dk96_dv96" )]] kernel flash_attn_ext_t kernel_flash_attn_ext; +template [[host_name("kernel_flash_attn_ext_bf16_dk96_dv64" )]] kernel flash_attn_ext_t kernel_flash_attn_ext; template [[host_name("kernel_flash_attn_ext_bf16_dk112_dv112")]] kernel flash_attn_ext_t kernel_flash_attn_ext; template [[host_name("kernel_flash_attn_ext_bf16_dk128_dv128")]] kernel flash_attn_ext_t kernel_flash_attn_ext; template [[host_name("kernel_flash_attn_ext_bf16_dk192_dv192")]] kernel flash_attn_ext_t kernel_flash_attn_ext; @@ -980,6 +983,7 @@ template [[host_name("kernel_flash_attn_ext_q4_0_dk64_dv64" )]] kernel flash_at template [[host_name("kernel_flash_attn_ext_q4_0_dk72_dv72" )]] kernel flash_attn_ext_t kernel_flash_attn_ext; template [[host_name("kernel_flash_attn_ext_q4_0_dk80_dv80" )]] kernel flash_attn_ext_t kernel_flash_attn_ext; template [[host_name("kernel_flash_attn_ext_q4_0_dk96_dv96" )]] kernel flash_attn_ext_t kernel_flash_attn_ext; +template [[host_name("kernel_flash_attn_ext_q4_0_dk96_dv64" )]] kernel flash_attn_ext_t kernel_flash_attn_ext; template [[host_name("kernel_flash_attn_ext_q4_0_dk112_dv112")]] kernel flash_attn_ext_t kernel_flash_attn_ext; template [[host_name("kernel_flash_attn_ext_q4_0_dk128_dv128")]] kernel flash_attn_ext_t kernel_flash_attn_ext; template [[host_name("kernel_flash_attn_ext_q4_0_dk192_dv192")]] kernel flash_attn_ext_t kernel_flash_attn_ext; @@ -996,6 +1000,7 @@ template [[host_name("kernel_flash_attn_ext_q4_1_dk64_dv64" )]] kernel flash_at template [[host_name("kernel_flash_attn_ext_q4_1_dk72_dv72" )]] kernel flash_attn_ext_t kernel_flash_attn_ext; template [[host_name("kernel_flash_attn_ext_q4_1_dk80_dv80" )]] kernel flash_attn_ext_t kernel_flash_attn_ext; template [[host_name("kernel_flash_attn_ext_q4_1_dk96_dv96" )]] kernel flash_attn_ext_t kernel_flash_attn_ext; +template [[host_name("kernel_flash_attn_ext_q4_1_dk96_dv64" )]] kernel flash_attn_ext_t kernel_flash_attn_ext; template [[host_name("kernel_flash_attn_ext_q4_1_dk112_dv112")]] kernel flash_attn_ext_t kernel_flash_attn_ext; template [[host_name("kernel_flash_attn_ext_q4_1_dk128_dv128")]] kernel flash_attn_ext_t kernel_flash_attn_ext; template [[host_name("kernel_flash_attn_ext_q4_1_dk192_dv192")]] kernel flash_attn_ext_t kernel_flash_attn_ext; @@ -1012,6 +1017,7 @@ template [[host_name("kernel_flash_attn_ext_q5_0_dk64_dv64" )]] kernel flash_at template [[host_name("kernel_flash_attn_ext_q5_0_dk72_dv72" )]] kernel flash_attn_ext_t kernel_flash_attn_ext; template [[host_name("kernel_flash_attn_ext_q5_0_dk80_dv80" )]] kernel flash_attn_ext_t kernel_flash_attn_ext; template [[host_name("kernel_flash_attn_ext_q5_0_dk96_dv96" )]] kernel flash_attn_ext_t kernel_flash_attn_ext; +template [[host_name("kernel_flash_attn_ext_q5_0_dk96_dv64" )]] kernel flash_attn_ext_t kernel_flash_attn_ext; template [[host_name("kernel_flash_attn_ext_q5_0_dk112_dv112")]] kernel flash_attn_ext_t kernel_flash_attn_ext; template [[host_name("kernel_flash_attn_ext_q5_0_dk128_dv128")]] kernel flash_attn_ext_t kernel_flash_attn_ext; template [[host_name("kernel_flash_attn_ext_q5_0_dk192_dv192")]] kernel flash_attn_ext_t kernel_flash_attn_ext; @@ -1028,6 +1034,7 @@ template [[host_name("kernel_flash_attn_ext_q5_1_dk64_dv64" )]] kernel flash_at template [[host_name("kernel_flash_attn_ext_q5_1_dk72_dv72" )]] kernel flash_attn_ext_t kernel_flash_attn_ext; template [[host_name("kernel_flash_attn_ext_q5_1_dk80_dv80" )]] kernel flash_attn_ext_t kernel_flash_attn_ext; template [[host_name("kernel_flash_attn_ext_q5_1_dk96_dv96" )]] kernel flash_attn_ext_t kernel_flash_attn_ext; +template [[host_name("kernel_flash_attn_ext_q5_1_dk96_dv64" )]] kernel flash_attn_ext_t kernel_flash_attn_ext; template [[host_name("kernel_flash_attn_ext_q5_1_dk112_dv112")]] kernel flash_attn_ext_t kernel_flash_attn_ext; template [[host_name("kernel_flash_attn_ext_q5_1_dk128_dv128")]] kernel flash_attn_ext_t kernel_flash_attn_ext; template [[host_name("kernel_flash_attn_ext_q5_1_dk192_dv192")]] kernel flash_attn_ext_t kernel_flash_attn_ext; @@ -1044,6 +1051,7 @@ template [[host_name("kernel_flash_attn_ext_q8_0_dk64_dv64" )]] kernel flash_at template [[host_name("kernel_flash_attn_ext_q8_0_dk72_dv72" )]] kernel flash_attn_ext_t kernel_flash_attn_ext; template [[host_name("kernel_flash_attn_ext_q8_0_dk80_dv80" )]] kernel flash_attn_ext_t kernel_flash_attn_ext; template [[host_name("kernel_flash_attn_ext_q8_0_dk96_dv96" )]] kernel flash_attn_ext_t kernel_flash_attn_ext; +template [[host_name("kernel_flash_attn_ext_q8_0_dk96_dv64" )]] kernel flash_attn_ext_t kernel_flash_attn_ext; template [[host_name("kernel_flash_attn_ext_q8_0_dk112_dv112")]] kernel flash_attn_ext_t kernel_flash_attn_ext; template [[host_name("kernel_flash_attn_ext_q8_0_dk128_dv128")]] kernel flash_attn_ext_t kernel_flash_attn_ext; template [[host_name("kernel_flash_attn_ext_q8_0_dk192_dv192")]] kernel flash_attn_ext_t kernel_flash_attn_ext; @@ -1905,6 +1913,29 @@ template [[host_name("kernel_flash_attn_ext_vec_q8_0_dk96_dv96")]] kernel flas template [[host_name("kernel_flash_attn_ext_vec_q8_0_dk96_dv96_q2_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; template [[host_name("kernel_flash_attn_ext_vec_q8_0_dk96_dv96_q4_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_f32_dk96_dv64")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_f16_dk96_dv64")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_f16_dk96_dv64_q2_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_f16_dk96_dv64_q4_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +#if defined(GGML_METAL_HAS_BF16) +template [[host_name("kernel_flash_attn_ext_vec_bf16_dk96_dv64")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +#endif +template [[host_name("kernel_flash_attn_ext_vec_q4_0_dk96_dv64")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_0_dk96_dv64_q2_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_0_dk96_dv64_q4_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_1_dk96_dv64")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_1_dk96_dv64_q2_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_1_dk96_dv64_q4_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_0_dk96_dv64")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_0_dk96_dv64_q2_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_0_dk96_dv64_q4_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_1_dk96_dv64")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_1_dk96_dv64_q2_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_1_dk96_dv64_q4_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q8_0_dk96_dv64")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q8_0_dk96_dv64_q2_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q8_0_dk96_dv64_q4_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; + template [[host_name("kernel_flash_attn_ext_vec_f32_dk128_dv128")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; template [[host_name("kernel_flash_attn_ext_vec_f16_dk128_dv128")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; template [[host_name("kernel_flash_attn_ext_vec_f16_dk128_dv128_q1_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp index f650e0123..4f2665497 100644 --- a/tests/test-backend-ops.cpp +++ b/tests/test-backend-ops.cpp @@ -10602,7 +10602,8 @@ static std::vector> make_test_cases_eval() { for (int hsk : { 40, 64, 72, 80, 96, 128, 192, 256, 320, 512, 576 }) { for (int hsv : { 40, 64, 72, 80, 96, 128, 192, 256, 512 }) { - if (hsk != 192 && hsk != 320 && hsk != 576 && hsk != hsv) continue; + if (hsk != 96 && hsk != 192 && hsk != 320 && hsk != 576 && hsk != hsv) continue; + if (hsk == 96 && (hsv != 64 && hsv != 96)) continue; // MiniCPM3 if (hsk == 192 && (hsv != 128 && hsv != 192)) continue; if (hsk == 576 && hsv != 512) continue; // DeepSeek MLA if (hsk == 320 && hsv != 256) continue; // Mistral4 MLA From 1af6c65de09e88af221f5dbc127fa13fba96f145 Mon Sep 17 00:00:00 2001 From: Aaron Teo Date: Tue, 15 Sep 2026 17:23:16 +0800 Subject: [PATCH 11/23] ci: bump kleidiai runners from 22.04 to 24.04 (#28885) * ci: bump kleidiai runners from 22.04 to 24.04 Signed-off-by: Aaron Teo * ci: promote warnings to hard errors for ci Signed-off-by: Aaron Teo --------- Signed-off-by: Aaron Teo --- .github/workflows/build-self-hosted.yml | 4 ++-- .github/workflows/server-self-hosted.yml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build-self-hosted.yml b/.github/workflows/build-self-hosted.yml index 2e05988bf..1337a0ed5 100644 --- a/.github/workflows/build-self-hosted.yml +++ b/.github/workflows/build-self-hosted.yml @@ -361,7 +361,7 @@ jobs: LLAMA_ARG_THREADS=$(nproc) GG_BUILD_HIGH_PERF=1 GG_BUILD_EXTRA_TESTS_0=1 bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp cpu-arm64-high-perf-graviton4: - runs-on: ah-ubuntu_22_04-c8g_8x + runs-on: ah-ubuntu_24_04-c8g_8x steps: - name: Clone @@ -404,7 +404,7 @@ jobs: bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp cpu-arm64-graviton4-kleidiai: - runs-on: ah-ubuntu_22_04-c8g_8x + runs-on: ah-ubuntu_24_04-c8g_8x steps: - name: Clone diff --git a/.github/workflows/server-self-hosted.yml b/.github/workflows/server-self-hosted.yml index de30d1a74..8dc4637c4 100644 --- a/.github/workflows/server-self-hosted.yml +++ b/.github/workflows/server-self-hosted.yml @@ -192,7 +192,7 @@ jobs: PYTEST_WORKERS=1 ./tests.sh server-kleidiai: - runs-on: ah-ubuntu_22_04-c8g_8x + runs-on: ah-ubuntu_24_04-c8g_8x steps: - name: Clone @@ -232,7 +232,7 @@ jobs: - name: Build id: cmake_build run: | - cmake -B build -DGGML_SCHED_NO_REALLOC=ON -DGGML_CPU_KLEIDIAI=ON + cmake -B build -DGGML_SCHED_NO_REALLOC=ON -DGGML_CPU_KLEIDIAI=ON -DLLAMA_FATAL_WARNINGS=ON cmake --build build --config Release -j $(nproc) --target llama-server - name: Python setup From 6ec1a7e956cfd5dfc111b6d3fa8e7d2c219106db Mon Sep 17 00:00:00 2001 From: lhez Date: Tue, 15 Sep 2026 02:29:02 -0700 Subject: [PATCH 12/23] opencl: add generic ssm_scan (#28881) * opencl: add generic ssm_scan * opencl: fix whitespace --- ggml/src/ggml-opencl/ggml-opencl.cpp | 238 ++++++++++++++--------- ggml/src/ggml-opencl/kernels/ssm_scan.cl | 130 +++++++++++++ 2 files changed, 281 insertions(+), 87 deletions(-) diff --git a/ggml/src/ggml-opencl/ggml-opencl.cpp b/ggml/src/ggml-opencl/ggml-opencl.cpp index 39c592e88..5b99f5d00 100644 --- a/ggml/src/ggml-opencl/ggml-opencl.cpp +++ b/ggml/src/ggml-opencl/ggml-opencl.cpp @@ -982,6 +982,7 @@ struct ggml_backend_opencl_context { // [size_idx][kda][tgpp] where size_idx: 0=S_V=16, 1=32, 2=64, 3=128; kda: 0 or 1. // tgpp 0 = TG variant (COLS_PER_LANE_GROUP=1), tgpp 1 = prefill variant (COLS_PER_LANE_GROUP=4). cl_kernel kernel_gated_delta_net_f32[4][2][2] = {}; + cl_kernel kernel_ssm_scan_f32 = nullptr; cl_kernel kernel_ssm_scan_f32_mamba2_d128 = nullptr; cl_kernel kernel_ssm_scan_f32_mamba2_d256 = nullptr; @@ -3457,7 +3458,7 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) { GGML_LOG_CONT("."); } - // ssm_scan (Mamba-2 fused per-token recurrent step; d_state in {128, 256}) + // ssm_scan { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { @@ -3469,8 +3470,34 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) { cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); + CL_CHECK((backend_ctx->kernel_ssm_scan_f32 = clCreateKernel(prog, "kernel_ssm_scan_f32", &err), err)); CL_CHECK((backend_ctx->kernel_ssm_scan_f32_mamba2_d128 = clCreateKernel(prog, "kernel_ssm_scan_f32_mamba2_d128", &err), err)); CL_CHECK((backend_ctx->kernel_ssm_scan_f32_mamba2_d256 = clCreateKernel(prog, "kernel_ssm_scan_f32_mamba2_d256", &err), err)); + + cl_kernel * kernels[] = { + &backend_ctx->kernel_ssm_scan_f32_mamba2_d128, + &backend_ctx->kernel_ssm_scan_f32_mamba2_d256 + }; + + // specialized kernels use subgroups and assume subgroup size is 64, + // if device does not support subgroups or subgroup size is not 64, + // release these kernels + for (int i = 0; i < 2; ++i) { + size_t subgroup_size = 0; +#if CL_TARGET_OPENCL_VERSION >= 210 + const size_t local_work_size[] = { 64, 1 }; + const cl_int subgroup_err = clGetKernelSubGroupInfo(*kernels[i], backend_ctx->device, CL_KERNEL_MAX_SUB_GROUP_SIZE_FOR_NDRANGE, + sizeof(local_work_size), local_work_size, sizeof(subgroup_size), &subgroup_size, nullptr); + if (subgroup_err != CL_SUCCESS) { + subgroup_size = 0; + } +#endif + // The specialized kernels reduce over one 64-lane subgroup. + if (subgroup_size != 64) { + CL_CHECK(clReleaseKernel(*kernels[i])); + *kernels[i] = nullptr; + } + } CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); } @@ -8734,22 +8761,16 @@ static bool ggml_opencl_supports_op(ggml_backend_dev_t dev, const struct ggml_te case GGML_OP_SSM_CONV: return (op->src[0]->type == GGML_TYPE_F32 && op->src[1]->type == GGML_TYPE_F32 && op->type == GGML_TYPE_F32); case GGML_OP_SSM_SCAN: { - // Mamba-2 fused per-token scan. Requires src3->ne[0] == 1 (scalar - // A per head); d_state in {128, 256}; all sources f32. Falls back - // to CPU otherwise (incl. Mamba-1 element-wise A). - for (int i = 0; i < 6; ++i) { - if (op->src[i]->type != GGML_TYPE_F32) { + if (op->type != GGML_TYPE_F32 || op->src[0]->type != GGML_TYPE_F32 || + op->src[1]->type != GGML_TYPE_F32 || op->src[2]->type != GGML_TYPE_F32 || + op->src[3]->type != GGML_TYPE_F32 || op->src[4]->type != GGML_TYPE_F32 || + op->src[5]->type != GGML_TYPE_F32 || op->src[6]->type != GGML_TYPE_I32) { return false; } + + const int64_t d_state = op->src[0]->ne[0]; + return d_state >= 1 && d_state <= 256 && (d_state & (d_state - 1)) == 0; } - if (op->type != GGML_TYPE_F32) { - return false; - } - const int K = ggml_get_op_params_i32(op, 0); - const int d_state = (int) op->src[0]->ne[0]; - const bool is_mamba2 = (op->src[3]->ne[0] == 1); - return is_mamba2 && (d_state == 128 || d_state == 256) && (K == 1); - } case GGML_OP_GATED_DELTA_NET: { // Match the Vulkan backend: only F32 -> F32, S_v in {16, 32, 64, 128}. @@ -14043,81 +14064,109 @@ static void ggml_cl_mean(ggml_backend_t backend, const ggml_tensor * src0, const } static void ggml_cl_ssm_scan(ggml_backend_t backend, ggml_tensor * dst) { - const ggml_tensor * src0 = dst->src[0]; // s - const ggml_tensor * src1 = dst->src[1]; // x - const ggml_tensor * src2 = dst->src[2]; // dt - const ggml_tensor * src3 = dst->src[3]; // A - const ggml_tensor * src4 = dst->src[4]; // B - const ggml_tensor * src5 = dst->src[5]; // C - const ggml_tensor * src6 = dst->src[6]; // ids - - GGML_ASSERT(src0 && src1 && src2 && src3 && src4 && src5 && src6 && dst); + GGML_ASSERT(dst); + GGML_ASSERT(dst->extra); + GGML_ASSERT(dst->src[0]); + GGML_ASSERT(dst->src[0]->extra); + GGML_ASSERT(dst->src[1]); + GGML_ASSERT(dst->src[1]->extra); + GGML_ASSERT(dst->src[2]); + GGML_ASSERT(dst->src[2]->extra); + GGML_ASSERT(dst->src[3]); + GGML_ASSERT(dst->src[3]->extra); + GGML_ASSERT(dst->src[4]); + GGML_ASSERT(dst->src[4]->extra); + GGML_ASSERT(dst->src[5]); + GGML_ASSERT(dst->src[5]->extra); + GGML_ASSERT(dst->src[6]); + GGML_ASSERT(dst->src[6]->extra); ggml_backend_opencl_context * backend_ctx = (ggml_backend_opencl_context *) backend->context; - ggml_tensor_extra_cl * e0 = (ggml_tensor_extra_cl *) src0->extra; - ggml_tensor_extra_cl * e1 = (ggml_tensor_extra_cl *) src1->extra; - ggml_tensor_extra_cl * e2 = (ggml_tensor_extra_cl *) src2->extra; - ggml_tensor_extra_cl * e3 = (ggml_tensor_extra_cl *) src3->extra; - ggml_tensor_extra_cl * e4 = (ggml_tensor_extra_cl *) src4->extra; - ggml_tensor_extra_cl * e5 = (ggml_tensor_extra_cl *) src5->extra; - ggml_tensor_extra_cl * e6 = (ggml_tensor_extra_cl *) src6->extra; - ggml_tensor_extra_cl * ed = (ggml_tensor_extra_cl *) dst->extra; + ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *) dst->src[0]->extra; + ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *) dst->src[1]->extra; + ggml_tensor_extra_cl * extra2 = (ggml_tensor_extra_cl *) dst->src[2]->extra; + ggml_tensor_extra_cl * extra3 = (ggml_tensor_extra_cl *) dst->src[3]->extra; + ggml_tensor_extra_cl * extra4 = (ggml_tensor_extra_cl *) dst->src[4]->extra; + ggml_tensor_extra_cl * extra5 = (ggml_tensor_extra_cl *) dst->src[5]->extra; + ggml_tensor_extra_cl * extra6 = (ggml_tensor_extra_cl *) dst->src[6]->extra; + ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *) dst->extra; - cl_ulong o0 = e0->offset + src0->view_offs; - cl_ulong o1 = e1->offset + src1->view_offs; - cl_ulong o2 = e2->offset + src2->view_offs; - cl_ulong o3 = e3->offset + src3->view_offs; - cl_ulong o4 = e4->offset + src4->view_offs; - cl_ulong o5 = e5->offset + src5->view_offs; - cl_ulong o6 = e6->offset + src6->view_offs; - cl_ulong od = ed->offset + dst->view_offs; + const cl_ulong offset0 = extra0->offset + dst->src[0]->view_offs; + const cl_ulong offset1 = extra1->offset + dst->src[1]->view_offs; + const cl_ulong offset2 = extra2->offset + dst->src[2]->view_offs; + const cl_ulong offset3 = extra3->offset + dst->src[3]->view_offs; + const cl_ulong offset4 = extra4->offset + dst->src[4]->view_offs; + const cl_ulong offset5 = extra5->offset + dst->src[5]->view_offs; + const cl_ulong offset6 = extra6->offset + dst->src[6]->view_offs; + const cl_ulong offsetd = extrad->offset + dst->view_offs; - const int d_state = (int) src0->ne[0]; - const int head_dim = (int) src0->ne[1]; - const int n_head = (int) src1->ne[1]; - const int n_group = (int) src4->ne[1]; - const int n_tokens = (int) src1->ne[2]; - const int n_seqs = (int) src1->ne[3]; + const ggml_tensor * s = dst->src[0]; + const ggml_tensor * x = dst->src[1]; + const ggml_tensor * dt = dst->src[2]; + const ggml_tensor * A = dst->src[3]; + const ggml_tensor * B = dst->src[4]; + const ggml_tensor * C = dst->src[5]; - // Mirror CPU ref: s_off = ggml_nelements(src1) * sizeof(float) - const cl_ulong s_off_bytes = (cl_ulong) ggml_nelements(src1) * sizeof(float); + const cl_ulong s_nb1 = s->nb[1]; + const cl_ulong s_nb2 = s->nb[2]; + const cl_ulong s_nb3 = s->nb[3]; + const cl_ulong x_nb1 = x->nb[1]; + const cl_ulong x_nb2 = x->nb[2]; + const cl_ulong x_nb3 = x->nb[3]; + const cl_ulong dt_nb1 = dt->nb[1]; + const cl_ulong dt_nb2 = dt->nb[2]; + const cl_ulong A_nb1 = A->nb[1]; + const cl_ulong B_nb1 = B->nb[1]; + const cl_ulong B_nb2 = B->nb[2]; + const cl_ulong B_nb3 = B->nb[3]; + const cl_ulong C_nb1 = C->nb[1]; + const cl_ulong C_nb2 = C->nb[2]; + const cl_ulong C_nb3 = C->nb[3]; - cl_kernel kernel = (d_state == 128) - ? backend_ctx->kernel_ssm_scan_f32_mamba2_d128 - : backend_ctx->kernel_ssm_scan_f32_mamba2_d256; - GGML_ASSERT(kernel != nullptr); + const cl_uint A_ne0 = A->ne[0]; + const cl_uint d_state = s->ne[0]; + const cl_int head_dim = x->ne[0]; + const cl_int n_head = x->ne[1]; + const cl_int n_group = B->ne[1]; + const cl_int n_tokens = x->ne[2]; + const cl_uint n_seqs = x->ne[3]; + const cl_uint K = ggml_get_op_params_i32(dst, 0); + const cl_ulong s_off_bytes = (cl_ulong) ggml_nelements(x) * sizeof(float); - cl_ulong s0_nb2 = src0->nb[2]; - cl_ulong s0_nb3 = src0->nb[3]; - cl_ulong x_nb2 = src1->nb[2]; - cl_ulong x_nb3 = src1->nb[3]; - cl_ulong dt_nb1 = src2->nb[1]; - cl_ulong dt_nb2 = src2->nb[2]; - cl_ulong A_nb1 = src3->nb[1]; - cl_ulong B_nb2 = src4->nb[2]; - cl_ulong B_nb3 = src4->nb[3]; - cl_ulong C_nb2 = src5->nb[2]; - cl_ulong C_nb3 = src5->nb[3]; + cl_kernel kernel = backend_ctx->kernel_ssm_scan_f32; + size_t nth = d_state; + if (A_ne0 == 1 && K == 1) { + cl_kernel kernel_mamba2 = nullptr; + if (d_state == 128) { + kernel_mamba2 = backend_ctx->kernel_ssm_scan_f32_mamba2_d128; + } else if (d_state == 256) { + kernel_mamba2 = backend_ctx->kernel_ssm_scan_f32_mamba2_d256; + } + if (kernel_mamba2 != nullptr) { + kernel = kernel_mamba2; + nth = 64; + } + } - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &e0->data_device)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &o0)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &e1->data_device)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &o1)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &e2->data_device)); - CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &o2)); - CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_mem), &e3->data_device)); - CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &o3)); - CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_mem), &e4->data_device)); - CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &o4)); - CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_mem), &e5->data_device)); - CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ulong), &o5)); - CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_mem), &e6->data_device)); - CL_CHECK(clSetKernelArg(kernel, 13, sizeof(cl_ulong), &o6)); - CL_CHECK(clSetKernelArg(kernel, 14, sizeof(cl_mem), &ed->data_device)); - CL_CHECK(clSetKernelArg(kernel, 15, sizeof(cl_ulong), &od)); - CL_CHECK(clSetKernelArg(kernel, 16, sizeof(cl_ulong), &s0_nb2)); - CL_CHECK(clSetKernelArg(kernel, 17, sizeof(cl_ulong), &s0_nb3)); + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extra2->data_device)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offset2)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_mem), &extra3->data_device)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &offset3)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_mem), &extra4->data_device)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &offset4)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_mem), &extra5->data_device)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ulong), &offset5)); + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_mem), &extra6->data_device)); + CL_CHECK(clSetKernelArg(kernel, 13, sizeof(cl_ulong), &offset6)); + CL_CHECK(clSetKernelArg(kernel, 14, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 15, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 16, sizeof(cl_ulong), &s_nb2)); + CL_CHECK(clSetKernelArg(kernel, 17, sizeof(cl_ulong), &s_nb3)); CL_CHECK(clSetKernelArg(kernel, 18, sizeof(cl_ulong), &x_nb2)); CL_CHECK(clSetKernelArg(kernel, 19, sizeof(cl_ulong), &x_nb3)); CL_CHECK(clSetKernelArg(kernel, 20, sizeof(cl_ulong), &dt_nb1)); @@ -14128,15 +14177,30 @@ static void ggml_cl_ssm_scan(ggml_backend_t backend, ggml_tensor * dst) { CL_CHECK(clSetKernelArg(kernel, 25, sizeof(cl_ulong), &C_nb2)); CL_CHECK(clSetKernelArg(kernel, 26, sizeof(cl_ulong), &C_nb3)); CL_CHECK(clSetKernelArg(kernel, 27, sizeof(cl_ulong), &s_off_bytes)); - CL_CHECK(clSetKernelArg(kernel, 28, sizeof(int), &head_dim)); - CL_CHECK(clSetKernelArg(kernel, 29, sizeof(int), &n_head)); - CL_CHECK(clSetKernelArg(kernel, 30, sizeof(int), &n_group)); - CL_CHECK(clSetKernelArg(kernel, 31, sizeof(int), &n_tokens)); + CL_CHECK(clSetKernelArg(kernel, 28, sizeof(cl_int), &head_dim)); + CL_CHECK(clSetKernelArg(kernel, 29, sizeof(cl_int), &n_head)); + CL_CHECK(clSetKernelArg(kernel, 30, sizeof(cl_int), &n_group)); + CL_CHECK(clSetKernelArg(kernel, 31, sizeof(cl_int), &n_tokens)); - size_t global_work_size[] = { (size_t)n_head * head_dim * 64, (size_t)n_seqs, 1 }; - size_t local_work_size[] = { 64, 1, 1 }; + if (kernel == backend_ctx->kernel_ssm_scan_f32) { + CL_CHECK(clSetKernelArg(kernel, 32, sizeof(cl_ulong), &s_nb1)); + CL_CHECK(clSetKernelArg(kernel, 33, sizeof(cl_ulong), &x_nb1)); + CL_CHECK(clSetKernelArg(kernel, 34, sizeof(cl_ulong), &B_nb1)); + CL_CHECK(clSetKernelArg(kernel, 35, sizeof(cl_ulong), &C_nb1)); + CL_CHECK(clSetKernelArg(kernel, 36, sizeof(cl_uint), &A_ne0)); + CL_CHECK(clSetKernelArg(kernel, 37, sizeof(cl_uint), &d_state)); + CL_CHECK(clSetKernelArg(kernel, 38, sizeof(cl_uint), &n_seqs)); + CL_CHECK(clSetKernelArg(kernel, 39, sizeof(cl_uint), &K)); + CL_CHECK(clSetKernelArg(kernel, 40, d_state * sizeof(float), nullptr)); + } - backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); + size_t global_work_size[] = { + (size_t) head_dim * (size_t) n_head * nth, + (size_t) n_seqs, + }; + size_t local_work_size[] = { nth, 1 }; + + backend_ctx->enqueue_ndrange_kernel(kernel, 2, global_work_size, local_work_size, dst); } static void ggml_cl_ssm_conv(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { diff --git a/ggml/src/ggml-opencl/kernels/ssm_scan.cl b/ggml/src/ggml-opencl/kernels/ssm_scan.cl index 37698d123..1889b74cd 100644 --- a/ggml/src/ggml-opencl/kernels/ssm_scan.cl +++ b/ggml/src/ggml-opencl/kernels/ssm_scan.cl @@ -214,3 +214,133 @@ kernel void kernel_ssm_scan_f32_mamba2_d256( s_warp[tid + 128] = state2; s_warp[tid + 192] = state3; } + +kernel void kernel_ssm_scan_f32( + global const char * s_buf, + ulong s_off, + global const char * x_buf, + ulong x_off, + global const char * dt_buf, + ulong dt_off, + global const char * A_buf, + ulong A_off, + global const char * B_buf, + ulong B_off, + global const char * C_buf, + ulong C_off, + global const char * ids_buf, + ulong ids_off, + global char * dst_buf, + ulong dst_off, + ulong s_nb2, + ulong s_nb3, + ulong x_nb2, + ulong x_nb3, + ulong dt_nb1, + ulong dt_nb2, + ulong A_nb1, + ulong B_nb2, + ulong B_nb3, + ulong C_nb2, + ulong C_nb3, + ulong state_off, + int head_dim, + int n_head, + int n_group, + int n_tokens, + ulong s_nb1, + ulong x_nb1, + ulong B_nb1, + ulong C_nb1, + uint A_ne0, + uint d_state, + uint n_seqs, + uint K, + local float * reduce +) { + global const char * s_data = s_buf + s_off; + global const char * x_data = x_buf + x_off; + global const char * dt_data = dt_buf + dt_off; + global const char * A_data = A_buf + A_off; + global const char * B_data = B_buf + B_off; + global const char * C_data = C_buf + C_off; + global const int * ids_data = (global const int *) (ids_buf + ids_off); + global float * dst = (global float *) (dst_buf + dst_off); + const uint y_elems = state_off / sizeof(float); + + const uint tid = get_local_id(0); + const uint inner_idx = get_group_id(0); + const uint seq_idx = get_group_id(1); + const uint head_idx = inner_idx / head_dim; + const uint dim_idx = inner_idx - head_idx * head_dim; + const uint group_idx = head_idx / (n_head / n_group); + const uint state_slot = (uint) ids_data[seq_idx]; + + const ulong s_idx = (ulong) state_slot * s_nb3 + + (ulong) head_idx * s_nb2 + + (ulong) dim_idx * s_nb1 + + (ulong) tid * sizeof(float); + float state = *((global const float *) (s_data + s_idx)); + + const ulong A_idx = (ulong) head_idx * A_nb1 + + (ulong) (tid % A_ne0) * sizeof(float); + const float A_value = *((global const float *) (A_data + A_idx)); + + for (int token_idx = 0; token_idx < n_tokens; ++token_idx) { + const ulong x_idx = (ulong) head_idx * x_nb1 + + (ulong) token_idx * x_nb2 + + (ulong) seq_idx * x_nb3 + + (ulong) dim_idx * sizeof(float); + const ulong dt_idx = (ulong) token_idx * dt_nb1 + + (ulong) seq_idx * dt_nb2 + + (ulong) head_idx * sizeof(float); + const ulong B_idx = (ulong) group_idx * B_nb1 + + (ulong) token_idx * B_nb2 + + (ulong) seq_idx * B_nb3 + + (ulong) tid * sizeof(float); + const ulong C_idx = (ulong) group_idx * C_nb1 + + (ulong) token_idx * C_nb2 + + (ulong) seq_idx * C_nb3 + + (ulong) tid * sizeof(float); + + const float x_value = *((global const float *) (x_data + x_idx)); + const float dt_value = *((global const float *) (dt_data + dt_idx)); + const float B_value = *((global const float *) (B_data + B_idx)); + const float C_value = *((global const float *) (C_data + C_idx)); + const float dt_soft_plus = dt_value > 20.0f ? dt_value : log(1.0f + exp(dt_value)); + const float dA = exp(dt_soft_plus * A_value); + const float x_dt = x_value * dt_soft_plus; + + state = mad(state, dA, B_value * x_dt); + reduce[tid] = state * C_value; + barrier(CLK_LOCAL_MEM_FENCE); + + for (uint stride = d_state / 2; stride > 0; stride >>= 1) { + if (tid < stride) { + reduce[tid] += reduce[tid + stride]; + } + barrier(CLK_LOCAL_MEM_FENCE); + } + + if (tid == 0) { + const uint y_idx = dim_idx + head_idx * head_dim + + token_idx * n_head * head_dim + + seq_idx * n_tokens * n_head * head_dim; + dst[y_idx] = reduce[0]; + } + + const uint snapshot_slot = n_tokens - 1 - token_idx; + if (snapshot_slot > 0 && snapshot_slot < K) { + const uint snapshot_idx = y_elems + tid + dim_idx * d_state + + head_idx * d_state * head_dim + + (snapshot_slot * n_seqs + seq_idx) * d_state * head_dim * n_head; + dst[snapshot_idx] = state; + } + barrier(CLK_LOCAL_MEM_FENCE); + } + + const uint state_idx = y_elems + tid + dim_idx * d_state + + head_idx * d_state * head_dim + + seq_idx * d_state * head_dim * n_head; + dst[state_idx] = state; +} From 77d554b26d88c96a3dbc0e685233312f064c93ee Mon Sep 17 00:00:00 2001 From: Zijun Yu Date: Tue, 15 Sep 2026 17:29:19 +0800 Subject: [PATCH 13/23] OpenVINO: optimize stateful decode and GPU MoE inference (#28638) * exclude GPU/NPU failing POOL_2D case * Fix pool case * ggml-openvino: fix stateful decode for Gemma-4 per-layer-type head sizes * ggml-openvino: fix MSVC narrowing error in permute * ggml-openvino: classify sliding-window layers structurally on interleaved-SWA models * ggml-openvino: add GGML_OPENVINO_REQUANT_KQUANT to select a 4-bit requant target * ggml-openvino: add GGML_OPENVINO_SPILL_DIR to spill weight buffers to disk * Stateful Performance: Added pass::KVStateSeqAxis to change KV layout * ggml-openvino: fix stateful decode past the sliding-window size Assisted-by: Claude Sonnet * ggml-openvino: refuse stateful decode that cannot resume from the KV state The stateful path seeds its KV state from ggml's cache when the decode position is ahead of what the state holds. That only works when ggml's cache is a plain prefix, where cell i holds position i. A sliding-window layer keeps just the last n_swa positions and drops the rest, so past the window cell i no longer holds position i and the seeded state is wrong. Slicing the state to the decode position also had no bounds check, so a position past the end surfaced as a bare ov::Exception from the ROI constructor (llama_decode ret = -3, with no reason given at default verbosity). Refuse both cases with a clear message instead, and refuse on the compile path too, where a new model starts with an empty state and so can only serve a sequence from its beginning. Reproducible with llama-bench -d, which restores a saved sequence state rather than recomputing the depth prefill. Assisted-by: Claude Opus 5 * ggml-openvino: use the per-layer KV head count for the stateful KV state The stateful path reinterprets ggml's KV buffer [1, 1, seq, n_heads_kv * head_size] as [1, seq, n_heads_kv, head_size]. The head size is already taken from the tensor's own combined dim, because gemma-4 varies it per layer type, but the head count still came from a model-level scalar that compute_llm_params() overwrites per attention node, so it ended up holding whatever the last layer said. gemma-4 varies the head count per layer too: 12B has 8 x 256 sliding layers and 1 x 512 full layers, 31B has 16 x 256 and 4 x 512. So 40 of 12B's 48 layers were split as 1 x 2048 instead of 8 x 256, and attention read the state with the wrong head split - both models decoded garbage on CPU and GPU. E2B is unaffected, its head count is 1 everywhere. Record the count per layer instead and look it up by the cache_k_l leaf name. Key it by layer, not by layer type: the sliding/full classification comes from cache extents, which tie at a small -c, while the head count does not. The stateful state trim now derives its sequence axis per state for the same reason, since pass::KVStateSeqAxis matches per state on the head count. Assisted-by: Claude Opus 5 * ggml-openvino: apply the KV state relayout to any KV head count pass::KVStateSeqAxis was limited to states with a single KV head, where moving the sequence axis from dim 1 to dim 2 is a pure metadata change. The limit was also based on a measurement showing no gain for a multi-head model, but that was taken at depth 0, which is the one depth where this change does nothing. With several heads the pass does more than move metadata: it drops the reader side transpose of the whole accumulated state, which the graph otherwise redoes every token at a cost that grows with the context length, and replaces it with a transpose of the single new row. Measured on GPU, tg128, alternating arms: gemma-4-12B 6.27 -> 9.11 t/s at depth 8192 (stateless is 7.69, so stateful now wins at depth instead of losing), Llama-3.2-1B 47.8 -> 59.6 t/s. Both are within noise at depth 0, which is why the earlier check saw nothing. The state refill needs the rows copied rather than reinterpreted now: ggml stores [seq][n_heads_kv * head_size], and a relayout state with several heads is a different element order. Without that, a refill would seed wrong data - it is reachable today through llama-bench -d. Assisted-by: Claude Opus 5 * ggml-openvino : support ggml_rope_set_offset and simplify op support gating * add more cpy cases * reject BF16 cpy on NPU * Remove mul_mat_id fallback, gate large mul_mat_id only for mxfp4 * ggml-openvino: fuse the MoE expert block into MOECompressed on GPU * ggml-openvino: skip GPU MUL_MAT_ID for unbound expert tensors * ggml-openvino: requantize grouped 8-bit MoE experts on GPU * Enable special strided CPY for conv state writeback * openvino: support cacheless encoder models on NPU Packed QKV views used by mmBERT were rejected by the ROPE support check. This split Q/K RoPE onto CPU, prevented cacheless attention detection, and sent fragmented encoder graphs through the decoder-oriented NPUW path. Accept packed QKV RoPE views, detect cacheless attention from its mask, and run these models as a single full-sequence prefill without NPUW or a decode graph. Also provide static mask, output index, and mean-pooling shapes and inputs. * openvino: optimize norm and RoPE translation Replace the decomposed mean/variance normalization graph with an opset6 MVN operation. This preserves the GGML epsilon placement while allowing OpenVINO plugins to compile normalization as one operation with fewer intermediate tensors. Cache RoPE sine and cosine outputs in the graph-wide tensor map. Build the cache key from all RoPE parameters and the optional frequency-factor input so compatible Q/K and layer nodes share one subgraph without mixing different RoPE configurations. Expose NodeContext::put_shared() to publish translator-created outputs for graph-level reuse. * ggml-openvino : simplify op translators and enable IMROPE/NEOX RoPE fusion * remove unnecessary include and clean up PAD * fix mulmat bug * use ov::as_type_ptr instead of std::dynamic_pointer_cast * ggml-openvino: fix mixed-dtype ADD/SWIGLU_CLAMP, gate unsupported ROPE/SOFTPLUS cases - translate_add: upcast mismatched operand types (e.g. f16/f32 in fused ADD_ADD) to f32, add, then cast once to the output type. opset1::Add requires matching input types and downcasting first lost precision. - translate_glu_swiglu_clamp: same fix, f16 Swish/Clamp rounding was drifting past the test tolerance. - supports_op: reject ROPE with ne[3] > 1 (multi-sequence) since the cos/sin tables only cover one sequence, and SOFTPLUS on GPU since the OpenVINO GPU kernel overflows to inf for large inputs (CPU is fine). - ci/run.sh: serialize test-backend-ops on OpenVINO GPU; running two workers concurrently crashes the GPU plugin (CL_OUT_OF_RESOURCES). * openvino: share compiled models with per-context inference state; fix thread-safety * ggml-openvino: gate MoE expert-sum ReduceSum shortcut past 8 experts The ReduceSum shortcut for the MoE expert-plane-sum ADD chain drifts past the 1e-7 test tolerance for >8 experts (f32 accumulation order vs CPU reference), intermittently, like the existing Q4_K/Q5_K NMSE case. Expose is_moe_expert_sum_add() so supports_op can gate on expert count and fall back to CPU for just that reduction op. * ggml-openvino: gate degenerate m=1,n=1 MUL_MAT on GPU CI hit ERR=1.8e-3 (> 5e-4 tolerance) for a scalar-output f32 dot product (m=1,n=1,k=2048); didn't reproduce locally in 8 tries, so likely an internal fp16 accumulation path the GPU plugin picks for this tiny shape. m=1 output dim doesn't occur in real model weights, so gate it. * ggml-openvino: make SoftPlus decomposition opt-in native Assisted-by: Codex --------- Co-authored-by: Mostafa Faheem Co-authored-by: Mustafa Cavus Co-authored-by: zhaixuejun1993 Co-authored-by: ravi9 --- ci/run.sh | 5 + docs/backend/OPENVINO.md | 3 + ggml/src/ggml-openvino/ggml-decoder.cpp | 266 +++++++-- ggml/src/ggml-openvino/ggml-decoder.h | 76 ++- .../src/ggml-openvino/ggml-openvino-extra.cpp | 88 +++ ggml/src/ggml-openvino/ggml-openvino-extra.h | 5 +- ggml/src/ggml-openvino/ggml-openvino.cpp | 187 +++++-- ggml/src/ggml-openvino/ggml-quants.cpp | 72 ++- ggml/src/ggml-openvino/ggml-quants.h | 10 + ggml/src/ggml-openvino/openvino/frontend.cpp | 3 +- .../src/ggml-openvino/openvino/node_context.h | 4 + ggml/src/ggml-openvino/openvino/op/add.cpp | 16 +- ggml/src/ggml-openvino/openvino/op/diag.cpp | 33 +- ggml/src/ggml-openvino/openvino/op/div.cpp | 18 +- .../openvino/op/glu_geglu_quick.cpp | 6 +- .../ggml-openvino/openvino/op/glu_swiglu.cpp | 27 +- .../openvino/op/moe_compressed.hpp | 90 +++ .../ggml-openvino/openvino/op/mul_mat_id.cpp | 60 +- ggml/src/ggml-openvino/openvino/op/mulmat.cpp | 16 +- ggml/src/ggml-openvino/openvino/op/norm.cpp | 35 +- ggml/src/ggml-openvino/openvino/op/pad.cpp | 4 +- .../src/ggml-openvino/openvino/op/permute.cpp | 16 +- ggml/src/ggml-openvino/openvino/op/rope.cpp | 277 ++++----- .../ggml-openvino/openvino/op/set_rows.cpp | 3 +- .../ggml-openvino/openvino/op/transpose.cpp | 2 - .../ggml-openvino/openvino/op/unary_silu.cpp | 27 - .../openvino/op/unary_softplus.cpp | 6 + ggml/src/ggml-openvino/openvino/op_table.cpp | 4 +- ggml/src/ggml-openvino/openvino/op_table.h | 1 - .../openvino/pass/fuse_moe_compressed.cpp | 273 +++++++++ .../openvino/pass/fuse_moe_compressed.h | 19 + .../openvino/pass/kv_state_seq_axis.cpp | 114 ++++ .../openvino/pass/kv_state_seq_axis.h | 24 + .../openvino/pass/squeeze_matmul.cpp | 3 +- .../openvino/translate_session.cpp | 85 ++- ggml/src/ggml-openvino/utils.cpp | 529 ++++++++++++++---- ggml/src/ggml-openvino/utils.h | 44 +- 37 files changed, 1860 insertions(+), 591 deletions(-) create mode 100644 ggml/src/ggml-openvino/openvino/op/moe_compressed.hpp delete mode 100644 ggml/src/ggml-openvino/openvino/op/unary_silu.cpp create mode 100644 ggml/src/ggml-openvino/openvino/pass/fuse_moe_compressed.cpp create mode 100644 ggml/src/ggml-openvino/openvino/pass/fuse_moe_compressed.h create mode 100644 ggml/src/ggml-openvino/openvino/pass/kv_state_seq_axis.cpp create mode 100644 ggml/src/ggml-openvino/openvino/pass/kv_state_seq_axis.h diff --git a/ci/run.sh b/ci/run.sh index a9f92a065..0595fac5a 100755 --- a/ci/run.sh +++ b/ci/run.sh @@ -669,6 +669,11 @@ function gg_run_test_backend_ops { args_extra="" fi + # TODO: OpenVINO GPU plugin crashes (CL_OUT_OF_RESOURCES) with 2 concurrent workers on GPU. + if [ ! -z "${GG_BUILD_OPENVINO}" ] && [ "${GGML_OPENVINO_DEVICE:-}" = "GPU" ]; then + args_extra="" + fi + # TODO: reduce the test-backend-ops timeout to 1800s if [ ! -z ${GG_BUILD_HIGH_PERF} ]; then (time timeout 3600 ./bin/test-backend-ops ${args_extra} -b CPU) 2>&1 | tee -a $OUT/${ci}-test-backend-ops.log diff --git a/docs/backend/OPENVINO.md b/docs/backend/OPENVINO.md index 9b43807d3..c1e39c5bf 100644 --- a/docs/backend/OPENVINO.md +++ b/docs/backend/OPENVINO.md @@ -719,10 +719,13 @@ Boolean flags follow a uniform convention: set to a **positive integer** (e.g. ` | `GGML_OPENVINO_STATEFUL_EXECUTION`| Boolean | `0` | Enable stateful KV cache for better performance. Recommended on CPU, GPU. | | `GGML_OPENVINO_DISABLE_CACHE` | Boolean | `0` | Disable the in-process compiled-model / decoder cache (cache is on by default). Set to `1` to disable. | | `GGML_OPENVINO_DISABLE_KV_SLICE` | Boolean | `0` | Disable the KV-cache input-tensor slicing optimization (slicing is on by default on CPU/GPU). Set to `1` to disable. | +| `GGML_OPENVINO_DISABLE_KV_STATE_RELAYOUT` | Boolean | `0` | Disable the stateful KV-state sequence-axis relayout (relayout is on by default). It moves the KV state sequence axis from dim 1 to dim 2, so the GPU plugin can append new tokens in place instead of copying the whole state every token, and the reader side no longer transposes the whole accumulated state. Set to `1` to disable. | | `GGML_OPENVINO_MANUAL_GQA_ATTN` | Boolean | device-based | Tri-state. When **unset**, manual GQA attention is enabled by default on `GPU` and disabled on other devices. Set to a positive integer to force-enable, or `0` to force-disable. | | `GGML_OPENVINO_MEMORY_OPTIMIZE` | Boolean | `0` | Umbrella switch for compile-time memory reductions. Enables `GGML_OPENVINO_REDUCE_COMPILE_MEM` and, on GPU, `GGML_OPENVINO_RELEASE_WEIGHTS` unless those fine-grained variables are explicitly set. | | `GGML_OPENVINO_REDUCE_COMPILE_MEM`| Boolean | inherits from `GGML_OPENVINO_MEMORY_OPTIMIZE` | Reduce compile-time host memory use by streaming weight requantization and avoiding extra weight-node materialization where possible. Set explicitly to override the umbrella switch. | | `GGML_OPENVINO_RELEASE_WEIGHTS` | Boolean | inherits from `GGML_OPENVINO_MEMORY_OPTIMIZE` on GPU | GPU-only. Release host weight buffers after the compiled model cache can reuse the device/plugin copy. Requires stable graph shapes; dynamic workloads that need recompilation should leave this disabled. | +| `GGML_OPENVINO_SPILL_DIR` | String | `not set` | Directory for a disk-backed weight buffer. When set, the repacked weight buffer is mapped from an unlinked file on this path instead of anonymous memory, so its pages are reclaimable under memory pressure instead of staying pinned, cutting the load-time host memory peak. Must point at real storage; a tmpfs mount (e.g. `/tmp` on many systems) backs it with RAM and makes the peak worse. | +| `GGML_OPENVINO_REQUANT_KQUANT` | String | `not set` | Requantize Q6_K/Q5_K weights (and matching MoE expert weights) to a 4-bit target instead of the default Q8_0_C, trading accuracy for less memory traffic. One of `q4_sym128` (Q6_K/Q5_K only), `q4_sym128_all` (Q4_K too, drops its per-group zero point), `q4_asym64_all` (Q6_K/Q5_K/Q4_K, keeps a real zero point at group 64), or `native` (no requantization). | | `GGML_OPENVINO_PROFILING` | Boolean | `0` | Enable execution-time profiling. | | `GGML_OPENVINO_DUMP_CGRAPH` | Boolean | `0` | Dump the GGML compute graph to `cgraph_ov.txt`. | | `GGML_OPENVINO_DUMP_IR` | Boolean | `0` | Serialize OpenVINO IR files with timestamps. | diff --git a/ggml/src/ggml-openvino/ggml-decoder.cpp b/ggml/src/ggml-openvino/ggml-decoder.cpp index 006e005cb..0b99834aa 100644 --- a/ggml/src/ggml-openvino/ggml-decoder.cpp +++ b/ggml/src/ggml-openvino/ggml-decoder.cpp @@ -117,16 +117,7 @@ bool is_same_shape(const ggml_tensor * a, const ggml_tensor * b) { bool is_conv_states_all_tensor(const ggml_tensor * tensor) { return tensor != nullptr && strncmp(tensor->name, "conv_states_all", strlen("conv_states_all")) == 0; } - -// CPY writing the tail of conv_input (the concat of the previous conv state and the new tokens) -// back into a slot block of the recurrent state cache. Detected structurally because the rollback -// variant (cparams.n_rs_seq > 0) emits one such CPY per snapshot slot without naming them. -bool is_conv_state_writeback(const ggml_tensor * node) { - return node->op == GGML_OP_CPY && node->view_src != nullptr && GgmlOvDecoder::is_kvcache(node->view_src, nullptr) && - node->src[0] != nullptr && node->src[0]->op == GGML_OP_VIEW && node->src[0]->src[0] != nullptr && - node->src[0]->src[0]->op == GGML_OP_CONCAT && node->src[1] != nullptr && node->src[1]->op == GGML_OP_VIEW && - node->src[1]->view_src == node->view_src; -} +} // namespace // MoE expert aggregation (build_moe_ffn in llama-graph.cpp): each expert plane is // `ggml_view_2d(experts, n_embd, n_tokens, experts->nb[2], i*experts->nb[1])` and the planes @@ -174,20 +165,31 @@ bool is_moe_expert_sum_add(const ggml_tensor * node) { return base != nullptr && base->ne[1] > 1 && plane_indices.size() == static_cast(base->ne[1]); } -} // namespace -static std::string get_tensor_ov_name(const ggml_cgraph * cgraph, const ggml_tensor * tensor) { +std::string GgmlOvDecoder::get_tensor_name(const ggml_cgraph * cgraph, const ggml_tensor * tensor) { if (tensor == nullptr) { return ""; } - const size_t hash_pos = ggml_hash_find(&cgraph->visited_hash_set, tensor); - if (((tensor->flags & GGML_TENSOR_FLAG_COMPUTE) || GgmlOvDecoder::is_kvcache(tensor, nullptr)) && - hash_pos != GGML_HASHSET_FULL && ggml_bitset_get(cgraph->visited_hash_set.used, hash_pos)) { - return std::string(tensor->name) + "#" + std::to_string(hash_pos); + if ((tensor->flags & GGML_TENSOR_FLAG_COMPUTE) || is_kvcache(tensor, nullptr)) { + // Hash-table slots depend on tensor addresses and differ between contexts. + // Graph ordinals disambiguate duplicate names while keeping compiled-model + // ports identical for equivalent graphs in different contexts. + const auto * node = std::find(cgraph->nodes, cgraph->nodes + cgraph->n_nodes, tensor); + if (node != cgraph->nodes + cgraph->n_nodes) { + return std::string(tensor->name) + "#n" + std::to_string(node - cgraph->nodes); + } + const auto * leaf = std::find(cgraph->leafs, cgraph->leafs + cgraph->n_leafs, tensor); + if (leaf != cgraph->leafs + cgraph->n_leafs) { + return std::string(tensor->name) + "#l" + std::to_string(leaf - cgraph->leafs); + } } return tensor->name; } +static std::string get_tensor_ov_name(const ggml_cgraph * cgraph, const ggml_tensor * tensor) { + return GgmlOvDecoder::get_tensor_name(cgraph, tensor); +} + static std::string get_tensor_graph_input_ov_name(const GgmlOvDecoder * decoder, const ggml_cgraph * cgraph, const ggml_tensor * tensor, @@ -198,8 +200,20 @@ static std::string get_tensor_graph_input_ov_name(const GgmlOvDecoder * decoder, if (GgmlOvDecoder::is_inp_emb(tensor, op)) { return "embd"; } - if (decoder->is_stateful() && GgmlOvDecoder::is_inp_mask(tensor, op)) { - return std::string(tensor->name).find("swa") == std::string::npos ? "self_kq_mask" : "self_kq_mask_swa"; + if (GgmlOvDecoder::is_inp_mask(tensor, op)) { + // Give the two attention masks distinct OV parameter names. build_attn_inp_kq_mask() + // names the full-attention mask and the sliding-window mask identically, so keying a + // parameter off the name alone makes the second mask overwrite the first and both + // attention types read one parameter. Tell them apart by tensor identity, using the + // SWA classification computed in compute_llm_params(). An empty swa_layers set means + // there is only one mask in play and the plain name is correct. + const bool is_swa = decoder->is_swa_mask(tensor); + if (decoder->is_stateful()) { + return is_swa ? "self_kq_mask_swa" : "self_kq_mask"; + } + if (is_swa) { + return get_tensor_ov_name(cgraph, tensor) + "_swa"; + } } return get_tensor_ov_name(cgraph, tensor); } @@ -318,9 +332,7 @@ int GgmlOvDecoder::compute_op_case(const ggml_tensor * node) const { break; } case GGML_OP_MUL_MAT: { - if (node->src[0]->op == GGML_OP_VIEW && node->src[1]->op == GGML_OP_VIEW) { - op_case = 3; - } else if (node->src[1]->op == GGML_OP_SOFT_MAX) { + if (node->src[1]->op == GGML_OP_SOFT_MAX) { // In the case of `-fa off`, softmax is used, v_trans=true, the dynamic dim is ne[0] for cache_v op_case = 2; } @@ -441,7 +453,7 @@ int GgmlOvDecoder::compute_op_case(const ggml_tensor * node) const { if (node->src[0]->op == GGML_OP_VIEW) { if (node->src[0]->src[0]->op == GGML_OP_GATED_DELTA_NET) { op_case = 1; - } else if (is_conv_state_writeback(node)) { + } else if (GgmlOvDecoder::is_conv_state_writeback(node)) { op_case = 2; break; } else if (is_conv_states_all_tensor(node->view_src) && node->src[1] != nullptr && @@ -532,6 +544,40 @@ std::optional extract_layer_from_name(const std::string & name) { return layer; } +// Recover the sliding window width from ggml's own SWA mask. llama.cpp never passes n_swa to a +// backend, but fill_mask() writes it into the mask: a query row keeps exactly the cells inside +// its window, so the widest row counts min(pos + 1, n_swa) unmasked cells. Counting rather than +// looking for a contiguous band is what makes this work on the KV-cache mask, where columns are +// physical cache cells in arbitrary order, not positions. +// Assumes LLAMA_SWA_TYPE_STANDARD, the only type the caller reconstructs. +static int get_swa_window_from_mask(const ggml_tensor * mask) { + if (mask->data == nullptr || !ggml_backend_buffer_is_host(mask->buffer)) { + return -1; + } + if (mask->type != GGML_TYPE_F16 && mask->type != GGML_TYPE_F32) { + return -1; + } + + const int64_t n_kv = mask->ne[0]; + const int64_t n_tokens = mask->ne[1]; + int64_t window = 0; + + for (int64_t r = 0; r < n_tokens; r++) { + int64_t kept = 0; + for (int64_t c = 0; c < n_kv; c++) { + const size_t i = (size_t) r * n_kv + c; + const float v = mask->type == GGML_TYPE_F16 ? ggml_fp16_to_fp32(((const ggml_fp16_t *) mask->data)[i]) : + ((const float *) mask->data)[i]; + if (v > -INFINITY) { + kept++; + } + } + window = std::max(window, kept); + } + + return window > 0 ? (int) window : -1; +} + std::pair GgmlOvDecoder::compute_llm_params(ggml_cgraph * cgraph, bool is_static) { ModelParams model_params; ComputeParams compute_params; @@ -597,6 +643,97 @@ std::pair GgmlOvDecoder::compute_llm_params(ggml_cgr return -1; }; + // Resolve the attention mask an attention node consumes, mirroring the src layout that + // get_attention_pattern_case() classifies. Used by the SWA pre-pass below. + auto get_attention_op_mask = [&get_attention_pattern_case](const ggml_tensor * node) -> const ggml_tensor * { + switch (get_attention_pattern_case(node)) { + case 0: + case 1: + return node->src[3]; + case 2: + case 3: + return node->src[1]; + default: + return nullptr; + } + }; + + // Pre-pass: classify sliding-window vs full-attention layers. + // + // An interleaved-SWA model keeps two KV caches and two attention masks, and hands each layer + // whichever pair matches its attention type. The mask tensor does not say which is which: both + // are named "attn_inp_kq_mask" by build_attn_inp_kq_mask(), and both carry the same n_kv because + // llama_kv_cache::get_n_kv() pads occupancy up to a common multiple. + // + // The KV cache does say. Each cache allocates cache_k_l once at load time with its own cell + // count: the windowed cache is sized from the window + // (PAD(min(size_base, n_swa*(unified ? n_seq_max : 1) + n_ubatch), 256), see + // llama_kv_cache_iswa), the full-attention one spans the whole context. Read the LEAF buffer + // behind the VIEW rather than the VIEW itself: the leaf extent is a constant per layer, known + // from the first graph onwards, while the view grows with context depth and would invert the + // comparison at shallow depth. + // + // Layers whose leaf is smaller than the largest leaf are the windowed ones. When every layer + // reports the same extent there is no distinction to draw -- either the model has no windowed + // layers, or the window is at least as large as the context so the two caches coincide, in + // which case a windowed layer and a full-attention one compute the same thing. + // + // Getting this wrong is silent and severe: with the windowed layers classified as + // full-attention, permute's KV slicing uses attention_size instead of attention_size_swa. The + // two agree while the context is shorter than the window, then diverge, and the mask add fails + // shape inference ("Failed to broadcast-merge input shapes") partway into a long prompt. + { + std::map layer_extent; // layer -> leaf cache_k cell count + std::map layer_mask; // layer -> mask it consumes + int64_t max_extent = 0; + + for (int i = 0; i < cgraph->n_nodes; i++) { + const ggml_tensor * mask = get_attention_op_mask(cgraph->nodes[i]); + if (mask == nullptr) { + continue; + } + const ggml_tensor * cache_k_permute = nullptr; + switch (get_attention_pattern_case(cgraph->nodes[i])) { + case 0: cache_k_permute = cgraph->nodes[i]->src[1]; break; + case 1: cache_k_permute = cgraph->nodes[i]->src[1]->src[0]; break; + case 2: cache_k_permute = cgraph->nodes[i]->src[0]->src[0]; break; + default: cache_k_permute = cgraph->nodes[i]->src[0]->src[0]->src[0]; break; + } + const ggml_tensor * cache_k_view = cache_k_permute->src[0]; + if (cache_k_view->op != GGML_OP_VIEW) { + continue; + } + const ggml_tensor * leaf = cache_k_view->src[0]; + auto layer = extract_layer_from_name(leaf->name); + if (!layer.has_value()) { + continue; + } + layer_extent[layer.value()] = leaf->ne[1]; + layer_mask[layer.value()] = mask; + max_extent = std::max(max_extent, leaf->ne[1]); + } + + for (const auto & [layer, extent] : layer_extent) { + if (extent < max_extent) { + model_params.swa_layers.push_back(layer); + if (model_params.swa_mask == nullptr) { + model_params.swa_mask = layer_mask[layer]; + } + } + } + std::sort(model_params.swa_layers.begin(), model_params.swa_layers.end()); + + if (ggml_openvino_getenv_int("GGML_OPENVINO_LOG_SWA_LAYERS")) { + std::string per_layer; + for (const auto & [layer, extent] : layer_extent) { + per_layer += " " + std::to_string(layer) + ":" + std::to_string(extent) + + (extent < max_extent ? "(swa)" : ""); + } + GGML_LOG_WARN("ov-swa: attn_layers=%zu max_extent=%ld swa_layers=%zu |%s\n", layer_extent.size(), + (long) max_extent, model_params.swa_layers.size(), per_layer.c_str()); + } + } + bool rope_seen = false; for (int i = 0; i < cgraph->n_nodes; i++) { auto * node = cgraph->nodes[i]; @@ -654,11 +791,14 @@ std::pair GgmlOvDecoder::compute_llm_params(ggml_cgr ggml_tensor * cache_k = cache_k_view->src[0]; int layer = extract_layer_from_name(cache_k->name).value(); - std::string mask_name(mask->name); + // Classified by the pre-pass above, which groups layers by mask tensor identity. The + // mask NAME cannot be used: build_attn_inp_kq_mask() gives both masks the same name. + const bool layer_is_swa = std::find(model_params.swa_layers.begin(), model_params.swa_layers.end(), + layer) != model_params.swa_layers.end(); model_params.kv_buffer_ctx_id = ggml_backend_openvino_buffer_get_ctx_id(cache_k->buffer); - if (mask_name.find("swa") != std::string::npos) { - model_params.swa_layers.push_back(layer); + model_params.n_heads_kv_per_layer[layer] = cache_k_permute->ne[2]; + if (layer_is_swa) { model_params.ctx_per_seq_swa = cache_k->ne[1]; } else { model_params.ctx_per_seq = cache_k->ne[1]; @@ -671,8 +811,9 @@ std::pair GgmlOvDecoder::compute_llm_params(ggml_cgr memcpy(&offset, cache_k_view->op_params, sizeof(size_t)); compute_params.seq_active_start = offset / seq_size; - if (mask_name.find("swa") != std::string::npos) { + if (layer_is_swa) { compute_params.attention_size_swa = mask->ne[0]; + compute_params.swa_window = get_swa_window_from_mask(mask); } else { compute_params.attention_size = mask->ne[0]; } @@ -708,11 +849,11 @@ std::pair GgmlOvDecoder::compute_llm_params(ggml_cgr // mixed SWA/non-SWA layers with different n_dims or freq_base), we cannot // share a single precomputed rope_sin/rope_cos. Track divergence so the // translator falls back to per-op make_sin_cos in that case. - static_assert(sizeof(model_params.rope_params) == sizeof(int32_t) * 15, "rope_params size"); + static_assert(sizeof(model_params.rope_params) == sizeof(int32_t) * 16, "rope_params size"); if (!rope_seen) { - memcpy(model_params.rope_params, node->op_params, sizeof(int32_t) * 15); + memcpy(model_params.rope_params, node->op_params, sizeof(int32_t) * 16); rope_seen = true; - } else if (memcmp(model_params.rope_params, node->op_params, sizeof(int32_t) * 15) != 0) { + } else if (memcmp(model_params.rope_params, node->op_params, sizeof(int32_t) * 16) != 0) { model_params.mixed_rope_params = true; } } @@ -752,8 +893,41 @@ std::pair GgmlOvDecoder::compute_llm_params(ggml_cgr } } } + if (model_params.n_heads_kv == -1) { + for (int i = 0; i < cgraph->n_nodes; i++) { + const auto * node = cgraph->nodes[i]; + const ggml_tensor * mask = nullptr; + if (node->op == GGML_OP_SOFT_MAX) { + mask = node->src[1]; + } else if (node->op == GGML_OP_FLASH_ATTN_EXT) { + mask = node->src[3]; + } else { + continue; + } + if (mask == nullptr || mask->op != GGML_OP_NONE || !(mask->flags & GGML_TENSOR_FLAG_INPUT) || + node->src[0] == nullptr) { + continue; + } + model_params.is_cacheless_attn = true; + model_params.n_seq = 1; + model_params.ctx_per_seq = mask->ne[0]; + compute_params.input_len = node->src[0]->ne[1]; + compute_params.token_len_per_seq = compute_params.input_len; + break; + } + } + auto * output_tensor = cgraph->nodes[cgraph->n_nodes - 1]; compute_params.output_len = output_tensor->ne[1]; + if (model_params.is_cacheless_attn) { + for (int i = 0; i < cgraph->n_nodes; i++) { + const auto * node = cgraph->nodes[i]; + if (node->op == GGML_OP_GET_ROWS && is_output_idx(node->src[1], node)) { + compute_params.output_len = node->src[1]->ne[0]; + break; + } + } + } // for NPU, output_len is always 1 except for llama-perplexity if (is_static && compute_params.output_len == 0) { compute_params.output_len = 1; @@ -790,6 +964,10 @@ ov::PartialShape GgmlOvDecoder::get_graph_input_shape(const ggml_tensor * op, // output index input_shape = ov::PartialShape{1, 1, 1, m_is_static ? m_compute_params.output_len : -1}; + } else if (is_inp_mean(input, op)) { + input_shape = m_is_static ? ov::PartialShape{1, 1, input->ne[1], m_prefill_chunk_size} : + ov::PartialShape{1, 1, -1, -1}; + } else if (is_inp_mask(input, op)) { // mask if (m_is_static) { @@ -814,11 +992,19 @@ ov::PartialShape GgmlOvDecoder::get_graph_input_shape(const ggml_tensor * op, if (is_stateful() && !is_flat_kv) { // Convert stateless KV cache layout [1, 1, seq, n_heads_kv * head_size] // to stateful layout [1, seq, n_heads_kv, head_size]. + // NOTE: Gemma4 uses per-layer-type KV shapes, so no single scalar describes every + // layer. E2B varies only the head size (sliding 256, full 512); 12B also varies the + // head COUNT (sliding 8 x 256, full 1 x 512). Take the head count for this tensor's + // own layer type and derive the head size from its own combined dim, so both layer + // types get the correct split. Using the model-level count split 12B's sliding + // states as 1 x 2048 and decoded garbage. assert(input_shape.size() == 4 && input_shape[0] == 1 && input_shape[1] == 1 && - input_shape[2].is_dynamic() && - input_shape[3] == (m_model_params.n_heads_kv * m_model_params.head_size)); - input_shape = {input_shape[0], ov::Dimension::dynamic(), m_model_params.n_heads_kv, - m_model_params.head_size}; + input_shape[2].is_dynamic() && input_shape[3].is_static()); + const int n_heads_kv = get_n_heads_kv_for_tensor(input); + assert(n_heads_kv > 0 && input_shape[3].get_length() % n_heads_kv == 0); + const int64_t combined_dim = input_shape[3].get_length(); // n_heads_kv * head_size + const int64_t head_size = combined_dim / n_heads_kv; + input_shape = {input_shape[0], ov::Dimension::dynamic(), n_heads_kv, head_size}; } } else if (is_kv_idx(input, op)) { @@ -840,8 +1026,14 @@ ov::PartialShape GgmlOvDecoder::get_graph_input_shape(const ggml_tensor * op, if (op->op == GGML_OP_SOFT_MAX && op->src[1] != nullptr && op->src[1]->op == GGML_OP_NONE && op->src[1]->flags & GGML_TENSOR_FLAG_INPUT && op->src[1] == input) { // for softmax input mask, the shape is [1, 1, seq_active, seq_active], where seq_active is determined by the input active sequence length instead of the kv cache sequence length - input_shape[2] = -1; - input_shape[3] = -1; + if (m_is_static) { + const int64_t seq_active = m_is_prefill ? m_prefill_chunk_size : 1; + input_shape[2] = seq_active; + input_shape[3] = seq_active; + } else { + input_shape[2] = -1; + input_shape[3] = -1; + } } return input_shape; } @@ -894,6 +1086,10 @@ void GgmlOvDecoder::add_extra_inputs() { if (m_compute_params.attention_size_swa != -1) { create_1d_input("attention_size_swa", m_compute_params.attention_size_swa); } + // only the stateful SWA mask consumes this + if (is_stateful() && m_compute_params.swa_window != -1) { + create_1d_input("swa_window", m_compute_params.swa_window); + } create_1d_input("n_seq_active", m_compute_params.n_seq_active); create_1d_input("seq_active_start", m_compute_params.seq_active_start); create_1d_input("seq_active_end", m_compute_params.seq_active_start + m_compute_params.n_seq_active); diff --git a/ggml/src/ggml-openvino/ggml-decoder.h b/ggml/src/ggml-openvino/ggml-decoder.h index 74cb73850..7f9d45a48 100644 --- a/ggml/src/ggml-openvino/ggml-decoder.h +++ b/ggml/src/ggml-openvino/ggml-decoder.h @@ -21,18 +21,28 @@ struct ModelParams { int ctx_per_seq_swa = -1; int n_seq = 1; int n_heads_kv = -1; + // Per-layer KV head count. gemma-4 12B interleaves 8 x 256 sliding layers with 1 x 512 + // full-attention layers, so no single scalar describes every layer. Keyed by layer, not by + // layer TYPE, because the SWA classification depends on the context size (extents tie at a + // small -c) while the head count does not. + std::map n_heads_kv_per_layer; int head_size = -1; int state_size = -1; // for SSM molels, eg qwen35 - int32_t rope_params[15]; + int32_t rope_params[16]; bool mixed_rope_params = false; + bool is_cacheless_attn = false; std::vector swa_layers; + // The sliding-window mask tensor, identified in compute_llm_params() by grouping attention + // layers on the mask they consume. Only used to tell the two masks apart when naming OV + // parameters -- both carry the same tensor name. Null when the graph has a single mask. + const ggml_tensor * swa_mask = nullptr; std::vector kv_names; size_t kv_buffer_ctx_id = 0; bool same_rope_params(const ModelParams & other) const { return mixed_rope_params == other.mixed_rope_params && - memcmp(rope_params, other.rope_params, sizeof(int32_t) * 15) == 0; + memcmp(rope_params, other.rope_params, sizeof(int32_t) * 16) == 0; } bool can_reuse_dynamically(const ModelParams & other) const { return same_rope_params(other); } @@ -48,6 +58,11 @@ struct ComputeParams { int attention_size = -1; int attention_size_swa = -1; int attention_size_static = -1; // encoder/cross-attn KV fill level (whisper) + // Sliding window width, read back from the band of ggml's own SWA mask. ggml never passes + // n_swa down to a backend, but fill_mask() bakes it into the mask contents, so the widest + // unmasked row recovers it. Shorter than n_swa while the sequence is still short, which is + // harmless: every causal pair is inside the window then anyway. + int swa_window = -1; int input_len = -1; int token_len_per_seq = -1; int past_kv_len = -1; @@ -96,8 +111,15 @@ struct ComputeParams { // models use a fixed end-anchored offset in the translator. }; +// defined below; declared here because GgmlOvDecoder uses it inline +std::optional extract_layer_from_name(const std::string & name); + +// detects the MoE expert-plane-sum ADD chain (see definition); used by supports_op too +bool is_moe_expert_sum_add(const ggml_tensor * node); + class GgmlOvDecoder : public ov::frontend::ggml::GgmlDecoder { public: + static std::string get_tensor_name(const ggml_cgraph * cgraph, const ggml_tensor * tensor); struct NodeInfo { ggml_tensor * node; std::string node_name; @@ -250,6 +272,21 @@ public: m_model_params.swa_layers.end(); } + // KV head count for one layer. Sliding and full layers can differ (gemma-4 12B), so callers + // that reinterpret a KV buffer must use this and not the model-level n_heads_kv. + int get_n_heads_kv_for_layer(int layer) const { + auto it = m_model_params.n_heads_kv_per_layer.find(layer); + return it != m_model_params.n_heads_kv_per_layer.end() ? it->second : m_model_params.n_heads_kv; + } + + // Same, for a KV cache tensor: its layer comes from the leaf name (cache_k_l). + int get_n_heads_kv_for_tensor(const ggml_tensor * kv_tensor) const { + if (auto layer = extract_layer_from_name(std::string(kv_tensor->name)); layer.has_value()) { + return get_n_heads_kv_for_layer(layer.value()); + } + return m_model_params.n_heads_kv; + } + int get_past_kv_len() const { return m_compute_params.past_kv_len; } int get_input_len() const { return m_compute_params.input_len; } @@ -340,6 +377,12 @@ public: (op->op == GGML_OP_SOFT_MAX && tensor == op->src[1]); } + inline static bool is_inp_mean(const ggml_tensor * tensor, const ggml_tensor * op) { + return op->op == GGML_OP_MUL_MAT && tensor == op->src[1] && tensor->op == GGML_OP_NONE && + (tensor->flags & GGML_TENSOR_FLAG_INPUT) && tensor->type == GGML_TYPE_F32 && + op->src[0] != nullptr && op->src[0]->op != GGML_OP_NONE; + } + inline static bool is_rope_freqs_weight(const ggml_tensor * tensor, const ggml_tensor * op) { return op->op == GGML_OP_ROPE && tensor == op->src[2]; } @@ -353,10 +396,21 @@ public: (op != nullptr && op->op == GGML_OP_SET_ROWS && op->src[2] == tensor); } + inline static bool is_conv_state_writeback(const ggml_tensor * node) { + return node->op == GGML_OP_CPY && node->view_src != nullptr && is_kvcache(node->view_src, nullptr) && + node->src[0] != nullptr && node->src[0]->op == GGML_OP_VIEW && node->src[0]->src[0] != nullptr && + node->src[0]->src[0]->op == GGML_OP_CONCAT && node->src[1] != nullptr && + node->src[1]->op == GGML_OP_VIEW && node->src[1]->view_src == node->view_src; + } + inline static bool is_kv_idx(const ggml_tensor * tensor, const ggml_tensor * op) { return op->op == GGML_OP_SET_ROWS && op->src[1] == tensor; } + bool is_swa_mask(const ggml_tensor * tensor) const { + return m_model_params.swa_mask != nullptr && tensor == m_model_params.swa_mask; + } + inline static bool is_output_idx(const ggml_tensor * tensor, const ggml_tensor * op) { return op->op == GGML_OP_GET_ROWS && tensor == op->src[1] && op->src[0]->op != GGML_OP_NONE && op->src[1]->op == GGML_OP_NONE; @@ -375,8 +429,22 @@ public: if (is_inp_emb(tensor, op)) { return "embd"; } - if (is_stateful() && is_inp_mask(tensor, op)) { - return std::string(tensor->name).find("swa") == std::string::npos ? "self_kq_mask" : "self_kq_mask_swa"; + if (is_inp_mask(tensor, op)) { + // Give the two attention masks distinct OV parameter names. + // + // An interleaved-SWA model builds one full-attention mask and one sliding-window mask, + // but build_attn_inp_kq_mask() names them identically, so keying a parameter off + // tensor->name alone makes the second mask OVERWRITE the first in m_model_inputs: both + // attention types then read a single parameter, and the windowed layers silently run + // against an unbanded mask. Disambiguate using the SWA layer set computed in + // compute_llm_params(), which classifies by mask tensor identity rather than by name. + // + // When no SWA layer was found there is only one mask in play, so the plain name is + // correct and no _swa parameter is created. + if (m_model_params.swa_layers.empty()) { + return "self_kq_mask"; + } + return is_swa_mask(tensor) ? "self_kq_mask_swa" : "self_kq_mask"; } return tensor->name; } diff --git a/ggml/src/ggml-openvino/ggml-openvino-extra.cpp b/ggml/src/ggml-openvino/ggml-openvino-extra.cpp index 36dfa4d94..52e1a297c 100644 --- a/ggml/src/ggml-openvino/ggml-openvino-extra.cpp +++ b/ggml/src/ggml-openvino/ggml-openvino-extra.cpp @@ -31,6 +31,7 @@ void ggml_openvino_device_config::init() { // String values (use ggml_openvino_getenv_str) "GGML_OPENVINO_DEVICE", "GGML_OPENVINO_CACHE_DIR", + "GGML_OPENVINO_SPILL_DIR", "GGML_OPENVINO_DEBUG_NODE", "GGML_OPENVINO_COMPILED_MODEL_CACHE_DIR", "GGML_OPENVINO_NPU_COMPILE_CONFIG", @@ -56,6 +57,11 @@ void ggml_openvino_device_config::init() { "GGML_OPENVINO_RELEASE_WEIGHTS", "GGML_OPENVINO_REDUCE_COMPILE_MEM", "GGML_OPENVINO_LOG_UNSUPPORTED_OPS", + "GGML_OPENVINO_LOG_SWA_LAYERS", + "GGML_OPENVINO_NATIVE_SOFTPLUS", + "GGML_OPENVINO_DISABLE_REMOTE_OUTPUTS", + "GGML_OPENVINO_REQUANT_KQUANT", + "GGML_OPENVINO_DISABLE_KV_STATE_RELAYOUT", }; for (const char * const & env_var : env_var_names) { @@ -263,9 +269,81 @@ std::optional ggml_openvino_get_requant_type(const ggml_tensor * if (ggml_openvino_is_npu()) { return ExtraQuantType::Q4_0_128; } + // By default Q6_K/Q5_K are requantized to Q8_0_C, which *inflates* 6- and 5-bit weights to 8 + // while the rest of the model stays at 4 bits, and Q4_K keeps its native group-32 layout + // (an f16 scale plus an f16 zero point per 32 weights = 0.125 B/weight of metadata). + // Decode of a large model is bandwidth-bound, so both cost throughput. + // + // GGML_OPENVINO_REQUANT_KQUANT selects a 4-bit target instead. Names are + // q4_[_all]: says whether a per-group zero point is kept, + // is the group size, and the _all suffix sends Q4_K down the same path (without it only + // Q6_K/Q5_K are touched): + // q4_sym128 Q6_K/Q5_K -> Q4_0_128 (u4, group 128, symmetric) + // q4_sym128_all and Q4_K too -- drops Q4_K's per-32 zero point, which costs some accuracy + // q4_asym64_all Q6_K/Q5_K and Q4_K -> Q4_1_64 (u4, group 64, asymmetric) -- most of the + // metadata saving while keeping a real zero point + // native no requantization at all (keep Q6_K/Q5_K as they are) + // + // The asymmetric target is only offered in its _all form: leaving Q4_K at its native group 32 + // while Q6_K/Q5_K move to group 64 gives the Q/K/V projections different group counts, and the + // GPU plugin's FullyConnectedHorizontalFusion concatenates their scale constants, which then + // fails shape inference. Requantizing all three keeps the group size uniform. + const char * rq = ggml_openvino_getenv_str("GGML_OPENVINO_REQUANT_KQUANT"); + auto is_opt = [rq](const char * name) { + return rq && strcmp(rq, name) == 0; + }; + const bool sym128 = is_opt("q4_sym128"); + const bool sym128_all = is_opt("q4_sym128_all"); + const bool asym64_all = is_opt("q4_asym64_all"); + + if (tensor->type == GGML_TYPE_Q4_K) { + if (sym128_all) { + return ExtraQuantType::Q4_0_128; + } + if (asym64_all) { + return ExtraQuantType::Q4_1_64; + } + } + // MoE expert weights (3D, ne[2] = n_expert) stored as Q5_1/Q8_0 are the expert-side + // equivalent of Q6_K/Q5_K: kept at 8 bits by default while the rest of the model is at 4 + // (gemma-4 26B-A4B keeps its down projection there). Send them to 4 bits under the same + // option, at group 64 rather than 128: the down expert has k=704, which 64 divides + // (704/64 = 11) and 128 does not. + if (tensor->ne[2] > 1 && (tensor->type == GGML_TYPE_Q5_1 || tensor->type == GGML_TYPE_Q8_0)) { + if (sym128 || sym128_all) { + return ExtraQuantType::Q4_0_64; + } + if (asym64_all) { + return ExtraQuantType::Q4_1_64; + } + // TODO: temporary workaround for a known OpenVINO GPU-plugin bug -- remove once the + // plugin computes grouped 8-bit GatherMatmulCompressed correctly. This costs accuracy + // (5/8-bit -> 4-bit) on any model it applies to, so it must not outlive the bug. + // + // On GPU these would otherwise stay in their native *grouped 8-bit* layout, which the GPU + // plugin's GatherMatmulCompressed computes incorrectly -- gemma-4 26B-A4B (whose down + // projection is Q5_1) produces garbage, while the same graph is correct on CPU. It is + // specific to grouped 8 bit: the gate/up experts are grouped u4 *with* a zero point and + // are fine, and Qwen3.5 / granite are fine because their Q5_K/Q6_K down projections + // already requantize to per-channel Q8_0_C (grouped=0). Sending these to grouped 4 bit + // avoids the broken layout and restores correct output. + // Opt out with GGML_OPENVINO_REQUANT_KQUANT=native. + if (ggml_openvino_get_device_name() == "GPU" && !is_opt("native")) { + return ExtraQuantType::Q4_0_64; + } + } switch (tensor->type) { case GGML_TYPE_Q6_K: case GGML_TYPE_Q5_K: + if (sym128 || sym128_all) { + return ExtraQuantType::Q4_0_128; + } + if (asym64_all) { + return ExtraQuantType::Q4_1_64; + } + if (is_opt("native")) { + return std::nullopt; + } return ExtraQuantType::Q8_0_C; default: return std::nullopt; @@ -331,6 +409,16 @@ ggml_openvino_extracted_layout ggml_openvino_get_extracted_layout(const ggml_ten layout.weights_per_block = 128; layout.is_symmetric = true; break; + case ExtraQuantType::Q4_1_64: + layout.is_u4 = true; + layout.weights_per_block = 64; + layout.is_symmetric = false; + break; + case ExtraQuantType::Q4_0_64: + layout.is_u4 = true; + layout.weights_per_block = 64; + layout.is_symmetric = true; + break; case ExtraQuantType::Q4_0_C: layout.is_u4 = true; layout.weights_per_block = tensor->ne[0]; diff --git a/ggml/src/ggml-openvino/ggml-openvino-extra.h b/ggml/src/ggml-openvino/ggml-openvino-extra.h index 0916b4162..9d827d969 100644 --- a/ggml/src/ggml-openvino/ggml-openvino-extra.h +++ b/ggml/src/ggml-openvino/ggml-openvino-extra.h @@ -15,7 +15,10 @@ #include // ExtraQuantType enum - defines requantization target formats -enum class ExtraQuantType { F16, Q4_0_C, Q8_1_C, Q4_0_128, Q8_0_C, Q8_0_32 }; +// Q4_1_64: u4, group 64, *true* asymmetric (per-group scale and zero point). Note that +// Q4_0_128/Q4_0_C are symmetric despite taking the unsigned branch of quantize_q4_0 -- that branch +// pins zp to 8 with d = max/-8, which is algebraically symmetric. +enum class ExtraQuantType { F16, Q4_0_C, Q8_1_C, Q4_0_128, Q4_0_64, Q8_0_C, Q8_0_32, Q4_1_64 }; ov::Core & ov_singleton_core(); diff --git a/ggml/src/ggml-openvino/ggml-openvino.cpp b/ggml/src/ggml-openvino/ggml-openvino.cpp index a79562278..044b4da1c 100644 --- a/ggml/src/ggml-openvino/ggml-openvino.cpp +++ b/ggml/src/ggml-openvino/ggml-openvino.cpp @@ -10,7 +10,10 @@ #include "ggml.h" #include +#include +#include #include +#include #include #include #include @@ -25,6 +28,11 @@ #include #include +#ifndef _WIN32 +# include +# include +#endif + #if defined(_WIN32) # define WIN32_LEAN_AND_MEAN # ifndef NOMINMAX @@ -64,6 +72,11 @@ struct ggml_backend_openvino_buffer_context { size_t size; bool is_remote; + // Set when the buffer is a file-backed spill mapping (GGML_OPENVINO_SPILL_DIR); it must be + // munmap'd rather than freed. + void * spill_mapping = nullptr; + size_t spill_size = 0; + // Wrapping of the buffer std::shared_ptr ov_buffer; @@ -98,10 +111,56 @@ struct ggml_backend_openvino_buffer_context { data = usm_tensor.get(); ov_buffer = std::make_shared(std::move(usm_tensor)); } else { - data = ggml_aligned_malloc(size); - GGML_ASSERT(data); - memset(data, 0, size); - ov_buffer = std::make_shared(ov::element::u8, ov::Shape{size}, data); +#ifndef _WIN32 + if (const char * spill_dir = ggml_openvino_getenv_str("GGML_OPENVINO_SPILL_DIR")) { + // Disk-backed weight buffer: back the repacked weights with a temp file via MAP_SHARED + // instead of anonymous memory. Anonymous pages can only be evicted to swap, so the + // repacked buffer stays pinned alongside the mmap'd source and both are resident at once + // -- that double residency is the load-time peak. File-backed pages are reclaimable: the + // kernel can write them back and drop them under pressure, then re-read on demand, so RSS + // becomes a working set rather than the whole buffer. The file is unlinked immediately, + // so it disappears when the process exits. + // + // The directory must be real storage. Pointing this at a tmpfs mount (/tmp on many + // systems) backs the "spill" with RAM and makes matters worse. + char path[PATH_MAX]; + snprintf(path, sizeof(path), "%s/ggml-ov-weights-%d-XXXXXX", spill_dir, (int) getpid()); + int fd = mkstemp(path); + if (fd < 0) { + GGML_LOG_ERROR("%s: mkstemp(%s) failed: %s\n", __func__, path, strerror(errno)); + return; + } + unlink(path); // anonymous-but-file-backed: freed on process exit + if (ftruncate(fd, (off_t) size) != 0) { + GGML_LOG_ERROR("%s: ftruncate(%zu) failed: %s\n", __func__, size, strerror(errno)); + close(fd); + return; + } + void * m = mmap(nullptr, size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0); + close(fd); // the mapping keeps the file alive + if (m == MAP_FAILED) { + GGML_LOG_ERROR("%s: mmap(%zu) failed: %s\n", __func__, size, strerror(errno)); + return; + } + data = m; + spill_mapping = m; + spill_size = size; + GGML_LOG_INFO("%s: weight buffer spilled to %s (%zu MB, file-backed)\n", __func__, spill_dir, + size / 1024 / 1024); + ov_buffer = std::make_shared(ov::element::u8, ov::Shape{size}, data); + } else +#endif + { +#ifdef _WIN32 + if (ggml_openvino_getenv_str("GGML_OPENVINO_SPILL_DIR")) { + GGML_LOG_WARN("%s: GGML_OPENVINO_SPILL_DIR is not supported on Windows, ignoring\n", __func__); + } +#endif + data = ggml_aligned_malloc(size); + GGML_ASSERT(data); + memset(data, 0, size); + ov_buffer = std::make_shared(ov::element::u8, ov::Shape{size}, data); + } } if (data == nullptr) { @@ -124,6 +183,11 @@ struct ggml_backend_openvino_buffer_context { delete pair.second; } tensor_extras.clear(); +#ifndef _WIN32 + if (spill_mapping != nullptr) { + munmap(spill_mapping, spill_size); + } else +#endif if (!is_remote && data != nullptr) { ggml_aligned_free(data, size); } @@ -611,9 +675,7 @@ GGML_BACKEND_API ggml_backend_buffer_type_t ggml_backend_openvino_buffer_type(in static const char * ggml_backend_openvino_host_buffer_type_get_name(ggml_backend_buffer_type_t buft) { ggml_backend_openvino_buffer_type_context * ctx = (ggml_backend_openvino_buffer_type_context *) buft->context; - static std::string name; - name = ctx->name + "_HOST"; - return name.c_str(); + return ctx->name.c_str(); } static bool ggml_backend_openvino_host_buffer_type_is_host(ggml_backend_buffer_type_t buft) { @@ -646,7 +708,7 @@ GGML_BACKEND_API ggml_backend_buffer_type_t ggml_backend_openvino_host_buffer_ty for (int i = 0; i < device_count; i++) { buffer_type_contexts[i].device = i; - buffer_type_contexts[i].name = std::string(GGML_OPENVINO_NAME) + std::to_string(i); + buffer_type_contexts[i].name = std::string(GGML_OPENVINO_NAME) + std::to_string(i) + "_HOST"; buffer_types[i] = ggml_backend_buffer_type{ /* .iface = */ ggml_backend_openvino_host_buffer_type_interface, @@ -711,13 +773,16 @@ static void ggml_backend_openvino_free(ggml_backend_t backend) { if (ctx->runtime_context) { auto r_ctx = std::static_pointer_cast(ctx->runtime_context); - if (--r_ctx->backend_count == 0) { + auto cache = r_ctx->compiled_cache; + r_ctx->clear_caches(); + std::lock_guard cache_lock(cache->mutex); + if (--cache->backend_count == 0) { // If host weight buffers were released (GGML_OPENVINO_RELEASE_WEIGHTS), the // dropped pages can never be repopulated, so a recompile is impossible. Keep // the compiled-model cache alive across backend teardown so the next context // reuses it instead of recompiling against zeroed weights. if (!ggml_openvino_weight_buffers_released()) { - r_ctx->clear_caches(); + cache->graphs.clear(); } } } @@ -766,12 +831,14 @@ static ggml_guid_t ggml_backend_openvino_guid(void) { } static std::shared_ptr get_ov_runtime_context_ptr() { - static std::shared_ptr r_ctx = [] { - auto ctx = std::make_shared(); - ctx->device = ggml_openvino_get_device_name(); - ctx->stateful = is_stateful_enabled() && !ggml_openvino_is_npu(); - return ctx; - }(); + // Share compiled models, but give every backend its own requests and KV state. + static auto cache = std::make_shared(); + auto r_ctx = std::make_shared(); + r_ctx->device = ggml_openvino_get_device_name(); + r_ctx->stateful = is_stateful_enabled() && !ggml_openvino_is_npu(); + r_ctx->compiled_cache = cache; + std::lock_guard cache_lock(cache->mutex); + ++cache->backend_count; return r_ctx; } @@ -795,9 +862,6 @@ GGML_BACKEND_API ggml_backend_t ggml_backend_openvino_init(int device) { return nullptr; } - std::shared_ptr r_ctx = std::static_pointer_cast(ctx->runtime_context); - r_ctx->backend_count++; - ggml_backend_t openvino_backend = new ggml_backend{ /* .guid = */ ggml_backend_openvino_guid(), /* .interface = */ ggml_backend_openvino_interface, @@ -928,6 +992,10 @@ static bool is_supported_flash_attn_pattern(const ggml_tensor * op) { if (src->src[0] == nullptr || src->src[0]->view_src != nullptr) { return false; } + } else if (src->op == GGML_OP_CPY) { + if (src->src[0] == nullptr || src->src[0]->op != GGML_OP_PERMUTE || src->src[0]->src[0] == nullptr) { + return false; + } } else { return false; } @@ -995,7 +1063,7 @@ static bool cpy_output_view_is_supported(const ggml_tensor * op) { return false; } - return ggml_nbytes(op) == 0 || ggml_is_contiguous(op); + return ggml_nbytes(op) == 0 || ggml_is_contiguous(op) || GgmlOvDecoder::is_conv_state_writeback(op); } static bool mul_mat_id_requires_large_tmp(const ggml_tensor * op) { @@ -1123,6 +1191,10 @@ static ggml_openvino_op_support is_op_supported_case(const ggml_tensor * op) { if (op->src[1]->op == GGML_OP_PERMUTE) { return {false, "ADD/MUL/SUB with PERMUTE src1 is not supported"}; } + // >8-expert MoE ReduceSum drifts past the 1e-7 tolerance (f32 order vs CPU); intermittent. + if (op->op == GGML_OP_ADD && is_moe_expert_sum_add(op) && op->src[1]->src[0]->ne[1] > 8) { + return {false, "MoE expert-plane sum with more than 8 experts is not supported"}; + } for (int i = 0; i < 4; i++) { if (op->src[0]->ne[i] != op->src[1]->ne[i] && (op->src[0]->ne[i] != 1 && op->src[1]->ne[i] != 1)) { return {false, "ADD/MUL/SUB with incompatible broadcast shapes: src0->ne[" + std::to_string(i) + "]=" + @@ -1207,8 +1279,11 @@ static ggml_openvino_op_support is_op_supported_case(const ggml_tensor * op) { break; } case GGML_OP_CPY: { - if (op->src[0]->type == GGML_TYPE_BF16 || op->src[1]->type == GGML_TYPE_BF16) { - return {false, "CPY with BF16 src type is not supported"}; + if (op->src[0]->type != GGML_TYPE_BF16 && op->src[1]->type == GGML_TYPE_BF16) { + return {false, "CPY with BF16 src[1] type is not supported"}; + } + if (ggml_openvino_get_device_name() == "NPU" && (op->src[0]->type == GGML_TYPE_BF16 || op->src[1]->type == GGML_TYPE_BF16)) { + return {false, "CPY with BF16 is not supported is not supported on NPU"}; } // CPY to a quantized destination (e.g. f32 -> q4_0) is numerically unstable with OpenVINO backend. if (ggml_is_quantized(op->type)) { @@ -1238,6 +1313,10 @@ static ggml_openvino_op_support is_op_supported_case(const ggml_tensor * op) { op->src[0]->ne[0] == 256 && op->src[1]->ne[0] == 256) { return {false, "MUL_MAT quantized benchmark test case on GPU is not supported"}; } + if (ggml_openvino_get_device_name() == "GPU" && op->type == GGML_TYPE_F32 && op->ne[0] == 1 && op->ne[1] == 1 && + (op->src[0]->buffer == nullptr || op->src[0]->buffer->usage != GGML_BACKEND_BUFFER_USAGE_WEIGHTS)) { + return {false, "MUL_MAT scalar dot product with non-weight src[0] on GPU is not supported"}; + } if (op->src[0]->ne[3] != op->src[1]->ne[3] && op->src[0]->ne[3] != 1 && op->src[1]->ne[3] != 1) { return {false, "MUL_MAT with incompatible broadcast on ne[3]: src0->ne[3]=" + std::to_string(op->src[0]->ne[3]) + ", src1->ne[3]=" + std::to_string(op->src[1]->ne[3])}; @@ -1254,14 +1333,23 @@ static ggml_openvino_op_support is_op_supported_case(const ggml_tensor * op) { return {false, "MUL_MAT_ID with single-expert or empty ne[2] <= 1 (ne[2]=" + std::to_string(op->src[0]->ne[2]) + ") is not supported"}; } - if (ggml_openvino_get_device_name() == "GPU" && op->src[0] != nullptr && op->src[0]->type == GGML_TYPE_BF16) { - return {false, "MUL_MAT_ID with BF16 weights on GPU is not supported"}; + if (ggml_openvino_get_device_name() == "GPU" && op->src[0] != nullptr && !ggml_is_quantized(op->src[0]->type)) { + return {false, "MUL_MAT_ID with non-quantized weights on GPU is not supported"}; } - // GPU MUL_MAT_ID uses a Gather+MatMul fallback because the GPU plugin rejects internal - // GatherMatmul for these test shapes. Skip cases that would materialize a large selected - // expert-weight temporary. - if (ggml_openvino_get_device_name() == "GPU" && mul_mat_id_requires_large_tmp(op)) { - return {false, "MUL_MAT_ID requires large temporary on GPU"}; + // The GPU plugin's GatherMatmul returns wrong values for the layouts test-backend-ops + // produces: it builds a rank-4 input layout ([n_used, n_tokens, k, 1]) instead of rank 3 + // and the kernel misreads it, silently returning garbage (NMSE ~86) rather than asserting. + // The same graph is correct on the CPU plugin, and correct on GPU for every real model, + // which always feeds experts from a bound tensor buffer. Standalone op-test tensors have + // no buffer at all, so use that to exclude them and let the scheduler run them on CPU. + if (ggml_openvino_get_device_name() == "GPU" && op->src[0] != nullptr && op->src[0]->buffer == nullptr) { + return {false, "MUL_MAT_ID with unbound expert tensors on GPU is not supported"}; + } + // Only MXFP4 still needs the large-temporary guard; every other quantized type goes + // through GatherMatmul, which never materializes the selected expert weights. + if (ggml_openvino_get_device_name() == "GPU" && op->src[0] != nullptr && op->src[0]->type == GGML_TYPE_MXFP4 && + mul_mat_id_requires_large_tmp(op)) { + return {false, "MUL_MAT_ID with MXFP4 weights requires large temporary on GPU"}; } break; } @@ -1269,36 +1357,39 @@ static ggml_openvino_op_support is_op_supported_case(const ggml_tensor * op) { const int32_t * op_params = op->op_params; const int n_dims = op_params[1]; const int mode = op_params[2]; - if (op_params[15] != 0) { - // FIXME: support ggml_rope_set_offset - return {false, "ggml_rope_set_offset is not supported"}; - } + const int64_t n_offs = op_params[15]; if (mode != GGML_ROPE_TYPE_NORMAL && mode != GGML_ROPE_TYPE_NEOX && mode != GGML_ROPE_TYPE_IMROPE) { return {false, "ROPE with mode " + std::to_string(mode) + " is not supported"}; } + if (n_offs < 0 || (n_offs % 2) != 0) { + return {false, "ROPE with invalid n_offs=" + std::to_string(n_offs)}; + } const int64_t head_dim = op->src[0]->ne[0]; const int64_t rope_dims = n_dims == 0 ? head_dim : n_dims; - if (rope_dims <= 0 || rope_dims > head_dim || (rope_dims % 2) != 0) { - return {false, "ROPE with n_dims=" + std::to_string(n_dims) + ", head_dim=" + std::to_string(head_dim) + " is not supported"}; + if (rope_dims <= 0 || rope_dims + n_offs > head_dim || (rope_dims % 2) != 0) { + return {false, "ROPE with n_dims=" + std::to_string(n_dims) + ", n_offs=" + std::to_string(n_offs) + + ", head_dim=" + std::to_string(head_dim) + " is not supported"}; } if (op->type != GGML_TYPE_F32 && op->type != GGML_TYPE_F16) { return {false, "ROPE with type " + std::string(ggml_type_name(op->type)) + " is not supported"}; } - if (op->src[0]->op == GGML_OP_VIEW) { - const struct ggml_tensor * view = op->src[0]; - const struct ggml_tensor * view_src = view->view_src; - if (view_src->ne[1] != view->ne[1] || view_src->ne[2] != view->ne[2] || view_src->ne[3] != view->ne[3]) { - return {false, "ROPE with view_src->ne [" + std::to_string(view_src->ne[1]) + ", " + - std::to_string(view_src->ne[2]) + ", " + std::to_string(view_src->ne[3]) + - "] != view->ne [" + std::to_string(view->ne[1]) + ", " + - std::to_string(view->ne[2]) + ", " + std::to_string(view->ne[3]) + - "] is not supported"}; - } + if (op->view_src != nullptr && !ggml_is_contiguous(op->src[0])) { + return {false, "ROPE on VIEW / non-contiguous input is not supported"}; } + if (op->src[0]->ne[3] > 1) { + // translate_rope's cos/sin tables cover one sequence only; ne[3] > 1 fails to broadcast. + return {false, "ROPE with multiple sequences (ne[3]=" + std::to_string(op->src[0]->ne[3]) + + ") is not supported"}; + } + float freq_scale; + float ext_factor; + float attn_factor; + memcpy(&freq_scale, op_params + 6, sizeof(float)); + memcpy(&ext_factor, op_params + 7, sizeof(float)); + memcpy(&attn_factor, op_params + 8, sizeof(float)); if (mode == GGML_ROPE_TYPE_IMROPE && - (op->src[2] != 0 || ((const float *) op_params)[6] != 1 || ((const float *) op_params)[7] != 0 || - ((const float *) op_params)[8] != 1)) { - return {false, "IMROPE with freq_factors, freq_scale, ext_factor, and attn_factor is not supported"}; + (op->src[2] != nullptr || freq_scale != 1.0f || ext_factor != 0.0f || attn_factor != 1.0f)) { + return {false, "IMROPE with freq_factors, freq_scale, ext_factor, or attn_factor is not supported"}; } break; } diff --git a/ggml/src/ggml-openvino/ggml-quants.cpp b/ggml/src/ggml-openvino/ggml-quants.cpp index 120db01e1..93f9e8254 100644 --- a/ggml/src/ggml-openvino/ggml-quants.cpp +++ b/ggml/src/ggml-openvino/ggml-quants.cpp @@ -851,7 +851,8 @@ std::shared_ptr requantize_to_buffers(const ggml_tensor * tensor, const auto * type_traits = ggml_get_type_traits(tensor->type); const size_t src_row_bytes = ggml_row_size(tensor->type, ne0); - bool is_u4 = (requant_type == ExtraQuantType::Q4_0_C || requant_type == ExtraQuantType::Q4_0_128); + bool is_u4 = (requant_type == ExtraQuantType::Q4_0_C || requant_type == ExtraQuantType::Q4_0_128 || + requant_type == ExtraQuantType::Q4_0_64 || requant_type == ExtraQuantType::Q4_1_64); // Streaming dequant (opt-in via GGML_OPENVINO_REDUCE_COMPILE_MEM or // GGML_OPENVINO_MEMORY_OPTIMIZE): instead of @@ -879,7 +880,9 @@ std::shared_ptr requantize_to_buffers(const ggml_tensor * tensor, result->set_friendly_name(tensor->name); return result; } - if (is_u4) { + if (requant_type == ExtraQuantType::Q4_1_64) { + quantize_q4_1_asym(weights_f32.data(), weights, scales, zp, n_elements, block_size); + } else if (is_u4) { quantize_q4_0(weights_f32.data(), weights, scales, zp, n_elements, block_size); } else if (requant_type == ExtraQuantType::Q8_1_C) { quantize_q8_1(weights_f32.data(), weights, scales, zp, n_elements, block_size); @@ -1178,6 +1181,71 @@ void quantize_q4_0(const float * x, } } +// Asymmetric u4 quantization with a per-group scale and zero point. +// +// Unlike quantize_q4_0's unsigned branch, which pins the zero point to 8 and is therefore +// symmetric, this keeps a real per-group zero point, so a group whose values are not centred on +// zero does not waste half its range. +void quantize_q4_1_asym(const float * x, + ov::Tensor & weights_arr, + ov::Tensor & scales_arr, + ov::Tensor & zp_arr, + int64_t k, + int64_t qk) { + assert(k % qk == 0); + const int nb = k / qk; + + auto * weights = static_cast(weights_arr.data()); + auto * scales = scales_arr.data::value_type>(); + auto * zp = static_cast(zp_arr.data()); + + // u4 zero points are packed two per byte, low nibble first, indexed by group -- the same + // convention as the unsigned branch of quantize_q4_0. + auto store_zp = [zp](int i, uint8_t v) { + if (i % 2 == 0) { + zp[i / 2] = v & 0x0F; + } else { + zp[i / 2] |= (uint8_t) ((v & 0x0F) << 4); + } + }; + + for (int i = 0; i < nb; i++) { + float vmin = x[i * qk]; + float vmax = x[i * qk]; + for (int j = 1; j < qk; j++) { + const float v = x[i * qk + j]; + vmin = std::min(vmin, v); + vmax = std::max(vmax, v); + } + // Include 0 in the range so an all-positive or all-negative group still represents zero + // exactly -- these are weights, so an exact zero matters. + vmin = std::min(vmin, 0.0f); + vmax = std::max(vmax, 0.0f); + + const float d = (vmax - vmin) / 15.0f; + if (d == 0.0f) { + scales[i] = ov::float16(1.0f); + store_zp(i, 0); + memset(weights + i * qk / 2, 0, qk / 2); + continue; + } + const float id = 1.0f / d; + + // The zero point is itself a 4-bit integer, so round it and dequantize as (q - zq) * d. + const int zq = std::max(0, std::min(15, (int) lroundf(-vmin * id))); + scales[i] = ov::float16(d); + store_zp(i, (uint8_t) zq); + + for (int j = 0; j < qk / 2; ++j) { + const float x0 = x[i * qk + 2 * j] * id; + const float x1 = x[i * qk + 2 * j + 1] * id; + const uint8_t q0 = (uint8_t) std::max(0, std::min(15, (int) lroundf(x0) + zq)); + const uint8_t q1 = (uint8_t) std::max(0, std::min(15, (int) lroundf(x1) + zq)); + weights[i * qk / 2 + j] = (uint8_t) (q0 | (q1 << 4)); + } + } +} + void quantize_q8_0(const float * x, ov::Tensor & weights_arr, ov::Tensor & scales_arr, diff --git a/ggml/src/ggml-openvino/ggml-quants.h b/ggml/src/ggml-openvino/ggml-quants.h index e247255a7..d5273727e 100644 --- a/ggml/src/ggml-openvino/ggml-quants.h +++ b/ggml/src/ggml-openvino/ggml-quants.h @@ -122,6 +122,10 @@ inline const char * extra_quant_type_name(ExtraQuantType t) { return "Q8_0_32"; case ExtraQuantType::Q8_1_C: return "Q8_1_C"; + case ExtraQuantType::Q4_0_64: + return "Q4_0_64"; + case ExtraQuantType::Q4_1_64: + return "Q4_1_64"; default: return "unknown"; } @@ -166,6 +170,12 @@ void quantize_q8_1(const float * x, int64_t k, int64_t qk, int64_t block_offset = 0); +void quantize_q4_1_asym(const float * x, + ov::Tensor & weights_arr, + ov::Tensor & scales_arr, + ov::Tensor & zp_arr, + int64_t k, + int64_t qk); void quantize_q8_0(const float * x, ov::Tensor & weights_arr, ov::Tensor & scales_arr, diff --git a/ggml/src/ggml-openvino/openvino/frontend.cpp b/ggml/src/ggml-openvino/openvino/frontend.cpp index c2ba14e66..88de86fea 100644 --- a/ggml/src/ggml-openvino/openvino/frontend.cpp +++ b/ggml/src/ggml-openvino/openvino/frontend.cpp @@ -3,6 +3,7 @@ #include "input_model.h" #include "op_table.h" #include "translate_session.h" +#include namespace ov { namespace frontend { @@ -11,7 +12,7 @@ namespace ggml { FrontEnd::FrontEnd() {} std::shared_ptr FrontEnd::convert(const InputModel::Ptr & model, bool naive) { - auto ggml_model = std::dynamic_pointer_cast(model); + auto ggml_model = ov::as_type_ptr(model); FRONT_END_GENERAL_CHECK(ggml_model, "Invalid input model"); std::shared_ptr converted_model; const auto & supported_ops = get_supported_ops(); diff --git a/ggml/src/ggml-openvino/openvino/node_context.h b/ggml/src/ggml-openvino/openvino/node_context.h index 2e2756037..f1ea0e4f0 100644 --- a/ggml/src/ggml-openvino/openvino/node_context.h +++ b/ggml/src/ggml-openvino/openvino/node_context.h @@ -143,6 +143,10 @@ public: bool has_input(const std::string & name) const { return m_tensor_map->find(name) != m_tensor_map->end(); } + void put_shared(const std::string & name, const Output & value) const { + m_tensor_map->insert({name, value}); + } + const std::string & get_name() const override { return m_decoder->get_op_name(m_node_idx); } ov::Any get_attribute_as_any(const std::string & name) const override { return m_decoder->get_attribute(name); } diff --git a/ggml/src/ggml-openvino/openvino/op/add.cpp b/ggml/src/ggml-openvino/openvino/op/add.cpp index c43eb67f8..a45520d92 100644 --- a/ggml/src/ggml-openvino/openvino/op/add.cpp +++ b/ggml/src/ggml-openvino/openvino/op/add.cpp @@ -5,6 +5,7 @@ #include #include #include +#include #include #include @@ -35,7 +36,20 @@ OutputVector translate_add(const NodeContext & context) { auto input_0 = process_view_input_new(context, 0); auto input_1 = process_view_input_new(context, 1); - auto res = std::make_shared(input_0, input_1); + // opset1::Add needs matching types (e.g. fused ADD_ADD mixes f16/f32); add in f32, cast once. + auto output_type = context.get_output_type(); + if (input_0.get_element_type() != input_1.get_element_type()) { + if (input_0.get_element_type() != ov::element::f32) { + input_0 = std::make_shared(input_0, ov::element::f32); + } + if (input_1.get_element_type() != ov::element::f32) { + input_1 = std::make_shared(input_1, ov::element::f32); + } + } + ov::Output res = std::make_shared(input_0, input_1); + if (res.get_element_type() != output_type) { + res = std::make_shared(res, output_type); + } return rename_outputs_with_suffix({res}, context.get_name()); } diff --git a/ggml/src/ggml-openvino/openvino/op/diag.cpp b/ggml/src/ggml-openvino/openvino/op/diag.cpp index dacea2f05..05e064892 100644 --- a/ggml/src/ggml-openvino/openvino/op/diag.cpp +++ b/ggml/src/ggml-openvino/openvino/op/diag.cpp @@ -3,11 +3,8 @@ #include "../utils.h" #include -#include +#include #include -#include -#include -#include namespace ov { namespace frontend { @@ -23,31 +20,13 @@ namespace op { OutputVector translate_diag(const NodeContext & context) { num_inputs_check(context, 1, 1); - auto x = context.get_input(0); // OV shape: [ne3, ne2, 1, ne0] + auto x = process_view_input_new(context, 0); // OV shape: [ne3, ne2, 1, ne0] - auto out_shape = context.get_output_shape().to_shape(); - int64_t n = static_cast(out_shape[3]); // ne0 + auto n = get_dimensions(x.get_node_shared_ptr(), {3}); + auto zero_diag = ov::op::v0::Constant::create(ov::element::i64, {}, {0}); - // Build index range [0, 1, ..., n-1] - auto start = ov::op::v0::Constant::create(ov::element::i64, {}, {int64_t(0)}); - auto stop = ov::op::v0::Constant::create(ov::element::i64, {}, {n}); - auto step = ov::op::v0::Constant::create(ov::element::i64, {}, {int64_t(1)}); - auto range = std::make_shared(start, stop, step, ov::element::i64); - - // col_idx shape [1, 1, 1, n] - auto col_shape = ov::op::v0::Constant::create(ov::element::i64, {4}, std::vector{1, 1, 1, n}); - auto col_idx = std::make_shared(range, col_shape, false); - - // row_idx shape [1, 1, n, 1] - auto row_shape = ov::op::v0::Constant::create(ov::element::i64, {4}, std::vector{1, 1, n, 1}); - auto row_idx = std::make_shared(range, row_shape, false); - - // mask: true where col == row (diagonal) - auto mask = std::make_shared(col_idx, row_idx); - - // Broadcast input from [ne3, ne2, 1, ne0] to [ne3, ne2, ne0, ne0] via select - auto zero = ov::op::v0::Constant::create(ov::element::f32, {}, {0.0f}); - auto res = std::make_shared(mask, x, zero); + auto eye = std::make_shared(n, n, zero_diag, x.get_element_type()); + auto res = std::make_shared(x, eye); return rename_outputs_with_suffix({res}, context.get_name()); } diff --git a/ggml/src/ggml-openvino/openvino/op/div.cpp b/ggml/src/ggml-openvino/openvino/op/div.cpp index 11dd9dece..2089ffd4c 100644 --- a/ggml/src/ggml-openvino/openvino/op/div.cpp +++ b/ggml/src/ggml-openvino/openvino/op/div.cpp @@ -4,12 +4,14 @@ #include "ggml.h" #include +#include #include #include #include #include #include #include +#include #include #include #include @@ -33,22 +35,12 @@ bool is_silu_div_pattern(const ov::Output & numerator, return false; } - auto mul = std::dynamic_pointer_cast(numerator.get_node_shared_ptr()); - if (!mul) { - return false; - } - const auto denom_node = denominator.get_node_shared_ptr(); - const auto mul_input_0 = mul->input_value(0).get_node_shared_ptr(); - const auto mul_input_1 = mul->input_value(1).get_node_shared_ptr(); - auto sigmoid = std::dynamic_pointer_cast(mul_input_1); - if (mul_input_0 == denom_node && sigmoid && sigmoid->input_value(0).get_node_shared_ptr() == denom_node) { - return true; + if (auto swish = ov::as_type_ptr(numerator.get_node_shared_ptr())) { + return swish->input_value(0).get_node_shared_ptr() == denom_node; } - - sigmoid = std::dynamic_pointer_cast(mul_input_0); - return mul_input_1 == denom_node && sigmoid && sigmoid->input_value(0).get_node_shared_ptr() == denom_node; + return false; } ov::Output repeat_input_to_match(const NodeContext & context, diff --git a/ggml/src/ggml-openvino/openvino/op/glu_geglu_quick.cpp b/ggml/src/ggml-openvino/openvino/op/glu_geglu_quick.cpp index c6d64aed4..385d75f5f 100644 --- a/ggml/src/ggml-openvino/openvino/op/glu_geglu_quick.cpp +++ b/ggml/src/ggml-openvino/openvino/op/glu_geglu_quick.cpp @@ -6,8 +6,8 @@ #include #include #include -#include #include +#include namespace ov { namespace frontend { @@ -50,9 +50,7 @@ OutputVector translate_glu_geglu_quick(const NodeContext & context) { // Create the constant in the same type as src0 to avoid f16/f32 mismatch. auto input_type = src0.get_element_type(); auto coef = ov::op::v0::Constant::create(input_type, ov::Shape{}, {1.702f}); - auto scaled = std::make_shared(src0, coef); - auto sigmoid = std::make_shared(scaled); - auto gated = std::make_shared(src0, sigmoid); + auto gated = std::make_shared(src0, coef); auto res = std::make_shared(gated, src1); return rename_outputs_with_suffix({res}, context.get_name()); diff --git a/ggml/src/ggml-openvino/openvino/op/glu_swiglu.cpp b/ggml/src/ggml-openvino/openvino/op/glu_swiglu.cpp index d81fc53b5..7eea81d96 100644 --- a/ggml/src/ggml-openvino/openvino/op/glu_swiglu.cpp +++ b/ggml/src/ggml-openvino/openvino/op/glu_swiglu.cpp @@ -9,9 +9,10 @@ #include #include #include +#include #include -#include #include +#include namespace ov { namespace frontend { @@ -61,8 +62,7 @@ static std::pair, ov::Output> get_glu_inputs(cons OutputVector translate_glu_swiglu(const NodeContext & context) { auto [src0, src1] = get_glu_inputs(context); - auto sigmoid = std::make_shared(src0); - auto silu = std::make_shared(src0, sigmoid); + auto silu = std::make_shared(src0); auto res = std::make_shared(silu, src1); return rename_outputs_with_suffix({res}, context.get_name()); @@ -77,9 +77,7 @@ OutputVector translate_glu_swiglu_oai(const NodeContext & context) { auto gate = std::make_shared(src0, -std::numeric_limits::infinity(), limit); auto alpha_const = ov::op::v0::Constant::create(ov::element::f32, {}, {alpha}); - auto scaled_gate = std::make_shared(gate, alpha_const); - auto sigmoid = std::make_shared(scaled_gate); - auto out_glu = std::make_shared(gate, sigmoid); + auto out_glu = std::make_shared(gate, alpha_const); auto up = std::make_shared(src1, -limit, limit); auto one = ov::op::v0::Constant::create(ov::element::f32, {}, {1.0f}); @@ -95,11 +93,22 @@ OutputVector translate_glu_swiglu_clamp(const NodeContext & context) { const int32_t * params = context.get_output_op_params(); const float limit = reinterpret_cast(params)[3]; + // Compute in f32: f16 Swish/Clamp rounding drifts past the 1e-7 test tolerance. + auto output_type = context.get_output_type(); + if (src0.get_element_type() != ov::element::f32) { + src0 = std::make_shared(src0, ov::element::f32); + } + if (src1.get_element_type() != ov::element::f32) { + src1 = std::make_shared(src1, ov::element::f32); + } + auto gate = std::make_shared(src0, -std::numeric_limits::infinity(), limit); - auto sigmoid = std::make_shared(gate); - auto silu = std::make_shared(gate, sigmoid); + auto silu = std::make_shared(gate); auto up = std::make_shared(src1, -limit, limit); - auto res = std::make_shared(silu, up); + ov::Output res = std::make_shared(silu, up); + if (res.get_element_type() != output_type) { + res = std::make_shared(res, output_type); + } return rename_outputs_with_suffix({res}, context.get_name()); } diff --git a/ggml/src/ggml-openvino/openvino/op/moe_compressed.hpp b/ggml/src/ggml-openvino/openvino/op/moe_compressed.hpp new file mode 100644 index 000000000..07e94c690 --- /dev/null +++ b/ggml/src/ggml-openvino/openvino/op/moe_compressed.hpp @@ -0,0 +1,90 @@ +// Copyright (C) 2018-2026 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// +// Local mirror of OpenVINO's internal ov::op::internal::MOE and MOECompressed ops. +// +// The class bodies are provided by the linked libopenvino.so; only the declarations are +// needed here so the backend can construct the node directly (same approach as +// GatherMatmul and GatedDeltaNet). The class layout must stay in sync with +// openvino/src/core/dev_api/openvino/op/moe.hpp +// openvino/src/common/transformations/include/ov_ops/moe_compressed.hpp +// +// \note MOE op classes are under development and subject to change. + +#pragma once + +#include + +#include "openvino/core/type/element_type.hpp" +#include "openvino/op/op.hpp" + +namespace ov::op::internal { + +class OPENVINO_API MOE : public ov::op::Op { +public: + OPENVINO_OP("MOE") + + MOE() = default; + + MOE(const OutputVector & args) : Op(args) {} + + enum class Expert_type { GEMM2_BIAS_SWIGLU_CLAMP, GEMM3_SWIGLU }; + + enum class Activation_type { SWIGLU, GEGLU_TANH, GEGLU_ERF }; + + struct Config { + Expert_type expert_type{ Expert_type::GEMM2_BIAS_SWIGLU_CLAMP }; + float expert_alpha{ 0.0f }; + float expert_beta{ 1.0f }; + size_t gate_idx{ 0 }; + Activation_type activation_type{ Activation_type::SWIGLU }; + }; + + MOE(const OutputVector & args, const Config & config); + + const Config & get_config() const; + void set_config(const Config & config); + + bool visit_attributes(AttributeVisitor & visitor) override; + void validate_and_infer_types() override; + std::shared_ptr clone_with_new_inputs(const OutputVector & new_args) const override; + +private: + Config m_config; +}; + +class OPENVINO_API MOECompressed : public MOE { +public: + OPENVINO_OP("MOECompressed", "", ov::op::internal::MOE) + + MOECompressed() = default; + + struct Config : public MOE::Config { + size_t hidden_size = 0; + size_t inter_size = 0; + size_t num_expert = 0; + size_t num_shared_expert = 0; + size_t top_k = 0; + // numeric_limits::max() means per_channel compression (single group) + size_t group_size = 0; + bool has_batch_dim = false; + bool has_zp = false; + ov::element::Type out_type = ov::element::dynamic; + std::optional scale_factor; + }; + + MOECompressed(const OutputVector & args, const Config & config); + + const Config & get_config() const { return m_config; } + + void set_scale_factor(float scale_factor) { m_config.scale_factor = scale_factor; } + + bool visit_attributes(AttributeVisitor & visitor) override; + void validate_and_infer_types() override; + std::shared_ptr clone_with_new_inputs(const OutputVector & new_args) const override; + +protected: + Config m_config; +}; + +} // namespace ov::op::internal diff --git a/ggml/src/ggml-openvino/openvino/op/mul_mat_id.cpp b/ggml/src/ggml-openvino/openvino/op/mul_mat_id.cpp index f1b28c85d..0de6161be 100644 --- a/ggml/src/ggml-openvino/openvino/op/mul_mat_id.cpp +++ b/ggml/src/ggml-openvino/openvino/op/mul_mat_id.cpp @@ -56,54 +56,6 @@ ov::Output static_shape_dims_or_shapeof(const ov::Output & i return get_dimensions(shape, dims); } -ov::Output translate_mul_mat_id_gather_matmul_fallback(const NodeContext & context, - ov::Output expert_weights, - ov::Output activations, - ov::Output ids) { - auto gather_axis = ov::op::v0::Constant::create(ov::element::i32, ov::Shape{}, {0}); - ov::Output selected_weights = std::make_shared(expert_weights, ids, gather_axis); - - const auto output_type = context.get_output_type(); - if (selected_weights.get_element_type() != ov::element::f32) { - selected_weights = std::make_shared(selected_weights, ov::element::f32); - } - if (activations.get_element_type() != ov::element::f32) { - activations = std::make_shared(activations, ov::element::f32); - } - - auto activations_shape = std::make_shared(activations, ov::element::i64); - auto ids_shape = std::make_shared(ids, ov::element::i64); - ov::Output acts_target_dims = std::make_shared( - ov::OutputVector{ - get_dimensions(activations_shape, {0}), - get_dimensions(ids_shape, {1}), - get_dimensions(activations_shape, {2}), - }, - 0); - ov::Output acts_broadcasted = - std::make_shared(activations, acts_target_dims, ov::op::BroadcastType::BIDIRECTIONAL); - - auto activations_expanded = std::make_shared(acts_broadcasted, const_i64({2})); - ov::Output result = - std::make_shared(activations_expanded, selected_weights, false, true); - - auto output_shape = context.get_output_shape(); - FRONT_END_OP_CONVERSION_CHECK(output_shape.rank().is_static() && output_shape.rank().get_length() == 4, - "Unexpected MUL_MAT_ID output rank"); - FRONT_END_OP_CONVERSION_CHECK(output_shape[3].is_static(), "Expected static row dimension for MUL_MAT_ID output"); - - auto batch_dim = ov::op::v0::Constant::create(ov::element::i64, {1}, {1}); - auto row_dim = ov::op::v0::Constant::create(ov::element::i64, {1}, {output_shape[3].get_length()}); - auto result_target_dims = std::make_shared( - ov::OutputVector{batch_dim, get_dimensions(ids_shape, {0, 1}), row_dim}, 0); - result = std::make_shared(result, result_target_dims, false); - - if (result.get_element_type() != output_type) { - result = std::make_shared(result, output_type); - } - return result; -} - ov::Output translate_mul_mat_id_mxfp4_packed(const NodeContext & context, ov::Output expert_weights, ov::Output activations, @@ -229,7 +181,6 @@ OutputVector translate_mul_mat_id(const NodeContext & context) { auto expert_weights_rank = expert_weights.get_partial_shape().rank(); FRONT_END_OP_CONVERSION_CHECK(expert_weights_rank.is_static(), "Expected static rank for MUL_MAT_ID expert weights"); - const bool use_gpu_fallback = ggml_openvino_get_device_name() == "GPU"; if (expert_weights_rank.get_length() == 4) { auto expert_weights_shape_3d = static_shape_dims_or_shapeof(expert_weights, {1, 2, 3}); expert_weights = std::make_shared(expert_weights, expert_weights_shape_3d, false); @@ -246,14 +197,9 @@ OutputVector translate_mul_mat_id(const NodeContext & context) { } const auto output_type = context.get_output_type(); - if (activations.get_element_type() != ov::element::f32) { - activations = std::make_shared(activations, ov::element::f32); - } - - if (use_gpu_fallback || !expert_weights.get_partial_shape().is_static() || !activations.get_partial_shape().is_static() || - !ids.get_partial_shape().is_static()) { - return rename_outputs_with_suffix({translate_mul_mat_id_gather_matmul_fallback(context, expert_weights, activations, ids)}, - context.get_name()); + const auto activations_type = ggml_openvino_get_device_name() == "GPU" ? ov::element::f16 : ov::element::f32; + if (activations.get_element_type() != activations_type) { + activations = std::make_shared(activations, activations_type); } // GatherMatmul's A input is [n_used_or_1, n_tokens, k]; activations_3d is diff --git a/ggml/src/ggml-openvino/openvino/op/mulmat.cpp b/ggml/src/ggml-openvino/openvino/op/mulmat.cpp index 41d7c54ae..9d4315aa4 100644 --- a/ggml/src/ggml-openvino/openvino/op/mulmat.cpp +++ b/ggml/src/ggml-openvino/openvino/op/mulmat.cpp @@ -29,19 +29,11 @@ OutputVector translate_mulmat(const NodeContext & context) { int op_case = context.get_op_case(); - ov::Output res; - ov::Output B; - ov::Output A; - if (op_case == 3) { - B = process_view_input(context, 0); - A = process_view_input(context, 1); - } else { - B = process_view_input_new(context, 0); - A = process_view_input_new(context, 1); - } + ov::Output B = process_view_input_new(context, 0); + ov::Output A = process_view_input_new(context, 1); if (A.get_element_type() != B.get_element_type()) { - B = std::make_shared(context.get_input(0), context.get_input_type(1)); + B = std::make_shared(B, context.get_input_type(1)); } auto B_shape = context.get_input_shape(0).to_shape(); @@ -84,7 +76,7 @@ OutputVector translate_mulmat(const NodeContext & context) { } bool transpose_b = true; - res = std::make_shared(A, B, false, transpose_b); + ov::Output res = std::make_shared(A, B, false, transpose_b); const auto output_type = context.get_output_type(); if (res.get_element_type() != output_type) { diff --git a/ggml/src/ggml-openvino/openvino/op/norm.cpp b/ggml/src/ggml-openvino/openvino/op/norm.cpp index c8bedb6db..8660c6521 100644 --- a/ggml/src/ggml-openvino/openvino/op/norm.cpp +++ b/ggml/src/ggml-openvino/openvino/op/norm.cpp @@ -2,15 +2,10 @@ #include "../op_table.h" #include "../utils.h" +#include #include -#include #include -#include -#include -#include -#include -#include -#include +#include namespace ov { namespace frontend { @@ -21,33 +16,11 @@ OutputVector translate_norm(const NodeContext & context) { num_inputs_check(context, 1, 1); auto input_node = process_view_input_new(context, 0); - - // Step 1: Calculate mean along the last dimension - // mean = reduce_mean(input, axis=-1, keepdims=true) - auto mean = std::make_shared( - input_node, ov::op::v0::Constant::create(ov::element::i64, ov::Shape{1}, {-1}), true); - - // Step 2: Calculate (input - mean) - auto centered = std::make_shared(input_node, mean); - - // Step 3: Calculate squared differences (input - mean)^2 - auto squared = std::make_shared( - centered, ov::op::v0::Constant::create(ov::element::f32, ov::Shape{1}, {2.0f})); - - // Step 4: Calculate variance = mean((input - mean)^2) - auto variance = std::make_shared( - squared, ov::op::v0::Constant::create(ov::element::i64, ov::Shape{1}, {-1}), true); - - // Step 5: Get epsilon from op_params float eps; memcpy(&eps, context.get_output_op_params(), sizeof(float)); - // Step 6: Calculate std = sqrt(variance + eps) - auto std_dev = std::make_shared(std::make_shared( - variance, ov::op::v0::Constant::create(ov::element::f32, ov::Shape{1}, {eps}))); - - // Step 7: Normalize: output = (input - mean) / std - auto res = std::make_shared(centered, std_dev); + auto axes = ov::op::v0::Constant::create(ov::element::i64, ov::Shape{1}, {-1}); + auto res = std::make_shared(input_node, axes, true, eps, ov::op::MVNEpsMode::INSIDE_SQRT); return rename_outputs_with_suffix({res}, context.get_name()); } diff --git a/ggml/src/ggml-openvino/openvino/op/pad.cpp b/ggml/src/ggml-openvino/openvino/op/pad.cpp index 492033d1b..d2b861142 100644 --- a/ggml/src/ggml-openvino/openvino/op/pad.cpp +++ b/ggml/src/ggml-openvino/openvino/op/pad.cpp @@ -60,9 +60,7 @@ OutputVector translate_pad(const NodeContext & context) { auto input = process_view_input_new(context, 0); if (context.get_input_shape(0) == context.get_output_shape()) { - auto input_shape = std::make_shared(input); - auto res = std::make_shared(input, input_shape, false); - return rename_outputs_with_suffix({res}, context.get_name()); + return rename_outputs_with_suffix({input}, context.get_name()); } const int32_t * op_params = context.get_output_op_params(); diff --git a/ggml/src/ggml-openvino/openvino/op/permute.cpp b/ggml/src/ggml-openvino/openvino/op/permute.cpp index 85550bff3..df4f03898 100644 --- a/ggml/src/ggml-openvino/openvino/op/permute.cpp +++ b/ggml/src/ggml-openvino/openvino/op/permute.cpp @@ -45,11 +45,22 @@ OutputVector translate_permute(const NodeContext & context) { static_cast(perm_values.size() - 1 - input_axis); } } - auto perm = ov::op::v0::Constant::create(ov::element::i64, {4}, perm_values); - if (op_case == 1 || context.is_stateful()) { + // The stateful path carries hidden-state tensors in a rank-3 layout (the + // leading batch dim is dropped, e.g. Gemma4's per-layer-embedding path). The + // perm above is rank-4; when the actual input is rank-3, drop the batch axis + // (perm[0], which is always the identity 0 here) and shift the rest down by 1 + // so the transpose order matches the input rank. + std::vector perm_used = perm_values; + const auto & src_ps = src.get_partial_shape(); + if (src_ps.rank().is_static() && src_ps.rank().get_length() == 3 && perm_values.size() == 4 && + perm_values[0] == 0) { + perm_used = {perm_values[1] - 1, perm_values[2] - 1, perm_values[3] - 1}; + } + auto perm = ov::op::v0::Constant::create(ov::element::i64, ov::Shape{perm_used.size()}, perm_used); res = std::make_shared(src, perm); } else if (op_case == 2) { + auto perm = ov::op::v0::Constant::create(ov::element::i64, {4}, perm_values); auto output_shape = context.get_output_shape().to_shape(); auto n_heads = ov::op::v0::Constant::create(ov::element::i64, {1}, {output_shape[1]}); auto head_size = ov::op::v0::Constant::create(ov::element::i64, {1}, {output_shape[3]}); @@ -68,6 +79,7 @@ OutputVector translate_permute(const NodeContext & context) { auto reshaped = std::make_shared(src, new_shape, true); res = std::make_shared(reshaped, perm); } else { + auto perm = ov::op::v0::Constant::create(ov::element::i64, {4}, perm_values); auto cache_shape = src.get_partial_shape(); auto output_shape = context.get_output_shape().to_shape(); int64_t head_size = output_shape[3]; diff --git a/ggml/src/ggml-openvino/openvino/op/rope.cpp b/ggml/src/ggml-openvino/openvino/op/rope.cpp index 8f20a0d19..a3da7d1fb 100644 --- a/ggml/src/ggml-openvino/openvino/op/rope.cpp +++ b/ggml/src/ggml-openvino/openvino/op/rope.cpp @@ -11,16 +11,11 @@ #include #include #include -#include -#include #include #include -#include -#include #include #include #include -#include #include #include #include @@ -37,13 +32,14 @@ OutputVector translate_rope(const NodeContext & context) { ov::Output res; - auto data_node = context.get_input(0).get_node_shared_ptr(); + auto data_node = process_view_input_new(context, 0).get_node_shared_ptr(); auto output_shape = context.get_output_shape().to_shape(); int32_t * op_params = context.get_output_op_params(); const int mode = op_case; const int64_t head_dim = static_cast(output_shape[3]); const int64_t configured_n_dims = static_cast(op_params[1]); const int64_t n_dims = configured_n_dims == 0 ? head_dim : configured_n_dims; + const int64_t n_offs = static_cast(op_params[15]); constexpr int TYPE_NORMAL = 0; constexpr int TYPE_NEOX = 1; @@ -55,27 +51,27 @@ OutputVector translate_rope(const NodeContext & context) { cos_theta_node = context.get_input("rope_cos"); sin_theta_node = context.get_input("rope_sin"); } else { - auto inp_pos = context.get_input(1).get_node_shared_ptr(); - std::shared_ptr rope_freqs_weight; - if (context.get_input_size() == 3) { - rope_freqs_weight = context.get_input(2).get_node_shared_ptr(); + std::string cache_key = "rope_sin_cos"; + for (int i = 0; i < 15; i++) { + cache_key += "_" + std::to_string(op_params[i]); } - auto sin_cos = make_sin_cos(op_params, inp_pos, rope_freqs_weight, mode == TYPE_IMROPE, false); - sin_theta_node = sin_cos.first; - cos_theta_node = sin_cos.second; - } - - if (context.get_view_input_size(0) > 0) { - data_node = process_view_input_new(context, 0).get_node_shared_ptr(); - if (context.is_stateful()) { - auto data_shape = ov::op::v0::Constant::create( - ov::element::i64, {3}, std::vector{-1, (int64_t) output_shape[2], (int64_t) output_shape[3]}); - data_node = std::make_shared(data_node, data_shape, false); + if (context.get_input_size() == 3) { + cache_key += "_ff_" + context.get_input_names()[2]; + } + if (context.has_input(cache_key + "_cos")) { + cos_theta_node = context.get_input(cache_key + "_cos"); + sin_theta_node = context.get_input(cache_key + "_sin"); } else { - auto data_shape = ov::op::v0::Constant::create( - ov::element::i64, {4}, - std::vector{1, -1, (int64_t) output_shape[2], (int64_t) output_shape[3]}); - data_node = std::make_shared(data_node, data_shape, false); + auto inp_pos = context.get_input(1).get_node_shared_ptr(); + std::shared_ptr rope_freqs_weight; + if (context.get_input_size() == 3) { + rope_freqs_weight = context.get_input(2).get_node_shared_ptr(); + } + auto sin_cos = make_sin_cos(op_params, inp_pos, rope_freqs_weight, mode == TYPE_IMROPE, false); + sin_theta_node = sin_cos.first; + cos_theta_node = sin_cos.second; + context.put_shared(cache_key + "_cos", cos_theta_node); + context.put_shared(cache_key + "_sin", sin_theta_node); } } @@ -84,52 +80,34 @@ OutputVector translate_rope(const NodeContext & context) { data_node = std::make_shared(data_node, ov::element::f32); } - FRONT_END_OP_CONVERSION_CHECK(n_dims > 0 && n_dims <= head_dim && (n_dims % 2 == 0), - "ROPE expects even n_dims in [1, head_dim]"); + FRONT_END_OP_CONVERSION_CHECK(n_offs >= 0 && (n_offs % 2 == 0), + "ROPE expects non-negative even n_offs"); + FRONT_END_OP_CONVERSION_CHECK(n_dims > 0 && n_dims + n_offs <= head_dim && (n_dims % 2 == 0), + "ROPE expects even n_dims in [1, head_dim - n_offs]"); - // TODO(openvino-gpu-rope-fusion): TEMPORARY WORKAROUND - do NOT revert until the - // OpenVINO GPU plugin is updated. - // + // RoPEFusionFlux requires rank_equals(4) on x, t_cos and t_sin. The cos/sin + // tables are already built rank-4 ([1, S, 1, head_size/2]) for both modes. In + // stateful mode the data arrives rank-3 ([S, n_heads, head_size]), so lift it + // to rank-4 ([1, S, n_heads, head_size]) here. Stateful RoPE already produced + // rank-4 output, so downstream attention is unaffected. + if (context.is_stateful()) { + auto r4_shape = ov::op::v0::Constant::create( + ov::element::i64, {4}, + std::vector{1, -1, (int64_t) output_shape[2], (int64_t) output_shape[3]}); + data_node = std::make_shared(data_node, r4_shape, false); + } // For TYPE_NORMAL rope (both stateful and stateless) we emit the Flux-style // interleaved pattern below so the GPU plugin's RoPEFusionFlux matcher folds it - // into ov::op::internal::RoPE. The matcher requires rank-4 inputs, which is why - // the original even/odd Slice translation (kept in the `else if (mode == - // TYPE_NORMAL)` branch below for reference) does not get fused. - // - // Once the GPU plugin's RoPE fusion is extended to also recognize the original - // even/odd Slice form, this Flux rewrite should be removed and both modes should - // be restored to the captured even/odd translation. Until then, keep both paths: - // the active Flux rewrite here and the previous translation preserved below. + // into ov::op::internal::RoPE. if (mode == TYPE_NORMAL) { auto axis_last = ov::op::v0::Constant::create(ov::element::i64, {1}, {-1}); - auto zero = ov::op::v0::Constant::create(ov::element::i64, {1}, {0}); auto step_one = ov::op::v0::Constant::create(ov::element::i64, {1}, {1}); - // Emit the Flux-style interleaved-RoPE pattern so the GPU plugin's - // RoPEFusionFlux matcher folds this subgraph into ov::op::internal::RoPE: - // x_paired = Reshape(x_rot, [1, S, n_heads, n_dims/2, 2]) - // x0, x1 = Split(x_paired, axis=-1, num_splits=2) - // x1_neg = x1 * -1 - // x_rotated = Reshape(Concat([x1_neg, x0], axis=-1), [1, S, n_heads, n_dims]) - // y_rot = x_rot * t_cos + x_rotated * t_sin - // y = Concat([y_rot, x_tail], axis=-1) if n_dims < head_dim - // Mathematically equivalent to the even/odd Slice form below. - // - // RoPEFusionFlux requires rank_equals(4) on x, t_cos and t_sin. The cos/sin - // tables are already built rank-4 ([1, S, 1, head_size/2]) for both modes. In - // stateful mode the data arrives rank-3 ([S, n_heads, head_size]), so lift it - // to rank-4 ([1, S, n_heads, head_size]) here. Stateful RoPE already produced - // rank-4 output, so downstream attention is unaffected. - if (context.is_stateful()) { - auto r4_shape = ov::op::v0::Constant::create( - ov::element::i64, {4}, - std::vector{1, -1, (int64_t) output_shape[2], (int64_t) output_shape[3]}); - data_node = std::make_shared(data_node, r4_shape, false); - } const int64_t n_heads = static_cast(output_shape[2]); const int64_t half = n_dims / 2; - auto rot_end = ov::op::v0::Constant::create(ov::element::i64, {1}, {n_dims}); - auto rot_data = std::make_shared(data_node, zero, rot_end, step_one, axis_last); + auto rot_start = ov::op::v0::Constant::create(ov::element::i64, {1}, {n_offs}); + auto rot_end = ov::op::v0::Constant::create(ov::element::i64, {1}, {n_offs + n_dims}); + auto rot_data = std::make_shared(data_node, rot_start, rot_end, step_one, axis_last); auto neg_one_f = ov::op::v0::Constant::create(data_node->get_element_type(), ov::Shape{}, {-1.0f}); @@ -153,7 +131,7 @@ OutputVector translate_rope(const NodeContext & context) { // Expand cos/sin from [..., n_dims/2] to [..., n_dims] by repeating each // entry twice. Use special_zero on the final Reshape so the seq dim passes // through dynamically. Final rank is 4 to satisfy the matcher's predicate. - auto expand_cos_sin = [&](Output cs) { + auto expand_cos_sin = [&](const Output& cs) { auto cs_unsq = std::make_shared( cs, ov::op::v0::Constant::create(ov::element::i64, {1}, {-1})); auto bcast_target = ov::op::v0::Constant::create( @@ -170,123 +148,80 @@ OutputVector translate_rope(const NodeContext & context) { auto y2 = std::make_shared(x_rotated, sin_full); auto rotated = std::make_shared(y1, y2); - if (n_dims < head_dim) { - auto tail_start = ov::op::v0::Constant::create(ov::element::i64, {1}, {n_dims}); + ov::OutputVector concat_parts; + if (n_offs > 0) { + auto head_start = ov::op::v0::Constant::create(ov::element::i64, {1}, {0}); + auto head_end = ov::op::v0::Constant::create(ov::element::i64, {1}, {n_offs}); + auto head = std::make_shared(data_node, head_start, head_end, step_one, axis_last); + concat_parts.push_back(head); + } + concat_parts.push_back(rotated); + if (n_offs + n_dims < head_dim) { + auto tail_start = ov::op::v0::Constant::create(ov::element::i64, {1}, {n_offs + n_dims}); auto tail_end = ov::op::v0::Constant::create(ov::element::i64, {1}, {head_dim}); auto tail = std::make_shared(data_node, tail_start, tail_end, step_one, axis_last); - res = std::make_shared(ov::OutputVector{rotated, tail}, -1); - } else { + concat_parts.push_back(tail); + } + if (concat_parts.size() == 1) { res = rotated; - } - } - // PRESERVED PREVIOUS TRANSLATION - Re-enable this branch (and remove the Flux branch above) once - // the GPU plugin's RoPE fusion is updated to recognize the even/odd Slice form; - // see the TODO(openvino-gpu-rope-fusion) note above. Do not delete. - // - // Original even/odd Slice form. In stateless mode it ran on rank-4 data - // ([1, S, n_heads, head_size]); in stateful mode on rank-3 data - // ([S, n_heads, head_size]). Either way it does not match RoPEFusionFlux - // (which needs rank-4 x in the interleaved layout), so the RoPE stays as - // discrete elementwise ops. - // - // } else if (mode == TYPE_NORMAL) { - // auto neg_one = ov::op::v0::Constant::create(ov::element::i64, {1}, {-1}); - // auto zero = ov::op::v0::Constant::create(ov::element::i64, {1}, {0}); - // auto one = ov::op::v0::Constant::create(ov::element::i64, {1}, {1}); - // auto two = ov::op::v0::Constant::create(ov::element::i64, {1}, {2}); - // auto end = ov::op::v0::Constant::create(ov::element::i64, {1}, {output_shape[3]}); - // Output even_slice; - // Output odd_slice; - // // stateful data is rank 3 (unsqueeze at axis 3), stateless is rank 4 (axis 4) - // int32_t unsqueeze_dim = context.is_stateful() ? 3 : 4; - // even_slice = std::make_shared(data_node, zero, end, two, neg_one); - // odd_slice = std::make_shared(data_node, one, end, two, neg_one); - // - // Output first_half = - // std::make_shared(std::make_shared(even_slice, cos_theta_node), - // std::make_shared(odd_slice, sin_theta_node)); - // Output second_half = - // std::make_shared(std::make_shared(even_slice, sin_theta_node), - // std::make_shared(odd_slice, cos_theta_node)); - // - // first_half = std::make_shared(first_half, - // ov::op::v0::Constant::create(ov::element::i64, {1}, {unsqueeze_dim})); - // second_half = std::make_shared(second_half, - // ov::op::v0::Constant::create(ov::element::i64, {1}, {unsqueeze_dim})); - // auto stack = std::make_shared(OutputVector{first_half, second_half}, unsqueeze_dim); - // - // auto data_shape = ov::op::v0::Constant::create( - // ov::element::i64, {4}, std::vector{1, -1, (int64_t) output_shape[2], (int64_t) output_shape[3]}); - // res = std::make_shared(stack, data_shape, false); - else if (mode == TYPE_NEOX) { - // In stateful mode the data arrives rank-3 ([S, n_heads, head_size]) while the - // cos/sin tables are rank-4 ([1, S, 1, n_dims/2]). The resulting mixed-rank - // broadcast in the Multiply below is miscomputed by the OpenVINO GPU plugin, - // corrupting the rotated Q/K. Lift the data to rank-4 ([1, S, n_heads, head_size]) - // first so the RoPE Multiplies are equal-rank, matching the TYPE_NORMAL branch. - // Stateful RoPE already produced rank-4 output, so downstream attention is unaffected. - if (context.is_stateful()) { - auto r4_shape = ov::op::v0::Constant::create( - ov::element::i64, {4}, - std::vector{1, -1, (int64_t) output_shape[2], (int64_t) output_shape[3]}); - data_node = std::make_shared(data_node, r4_shape, false); - } - auto axis_last = ov::op::v0::Constant::create(ov::element::i64, ov::Shape{}, {-1}); - std::vector split_lengths = {n_dims / 2, n_dims / 2}; - if (n_dims < head_dim) { - split_lengths.push_back(head_dim - n_dims); - } - - auto data_split = std::make_shared( - data_node, axis_last, - ov::op::v0::Constant::create(ov::element::i64, {split_lengths.size()}, split_lengths)); - Output slice_data_node_0 = data_split->outputs()[0]; - Output slice_data_node_1 = data_split->outputs()[1]; - - auto first_half_node = std::make_shared( - std::make_shared(slice_data_node_0, cos_theta_node), - std::make_shared(slice_data_node_1, sin_theta_node)); - - auto second_half_node = std::make_shared( - std::make_shared(slice_data_node_0, sin_theta_node), - std::make_shared(slice_data_node_1, cos_theta_node)); - - if (n_dims < head_dim) { - Output tail = data_split->outputs()[2]; - res = std::make_shared(ov::OutputVector{first_half_node, second_half_node, tail}, -1); } else { - res = std::make_shared(ov::OutputVector{first_half_node, second_half_node}, -1); + res = std::make_shared(concat_parts, -1); } - } else if (mode == TYPE_IMROPE) { - auto cos_sin_shape = std::make_shared(ov::element::i64, ov::Shape{4}, - std::vector{1, -1, 1, (n_dims >> 1)}); - auto cos_reshaped = std::make_shared(cos_theta_node, cos_sin_shape, true); - auto sin_reshaped = std::make_shared(sin_theta_node, cos_sin_shape, true); + } else if (mode == TYPE_NEOX || mode == TYPE_IMROPE) { + if (mode == TYPE_IMROPE) { + auto cos_sin_shape = std::make_shared(ov::element::i64, ov::Shape{4}, + std::vector{1, -1, 1, (n_dims >> 1)}); + cos_theta_node = std::make_shared(cos_theta_node, cos_sin_shape, true); + sin_theta_node = std::make_shared(sin_theta_node, cos_sin_shape, true); + } + + auto axis_last = ov::op::v0::Constant::create(ov::element::i64, {1}, {-1}); + auto step_one = ov::op::v0::Constant::create(ov::element::i64, {1}, {1}); + + Output rot_data = data_node; + if (n_offs > 0 || n_offs + n_dims < head_dim) { + auto rot_start = ov::op::v0::Constant::create(ov::element::i64, {1}, {n_offs}); + auto rot_end = ov::op::v0::Constant::create(ov::element::i64, {1}, {n_offs + n_dims}); + rot_data = std::make_shared(data_node, rot_start, rot_end, step_one, axis_last); + } + + const int64_t half = n_dims / 2; + auto neg_one_f = ov::op::v0::Constant::create(data_node->get_element_type(), ov::Shape{}, {-1.0f}); auto split_axis = ov::op::v0::Constant::create(ov::element::i64, ov::Shape{}, {3}); - std::vector split_lengths = {n_dims / 2, n_dims / 2}; - if (n_dims < head_dim) { - split_lengths.push_back(head_dim - n_dims); + auto split_lengths = ov::op::v0::Constant::create(ov::element::i64, {2}, {half, half}); + auto data_split = std::make_shared(rot_data, split_axis, split_lengths); + Output x1 = data_split->outputs()[0]; + Output x2 = data_split->outputs()[1]; + + auto x2_neg = std::make_shared(x2, neg_one_f); + auto x_rotate_half = std::make_shared(ov::OutputVector{x2_neg, x1}, -1); + + auto cos_full = std::make_shared(ov::OutputVector{cos_theta_node, cos_theta_node}, -1); + auto sin_full = std::make_shared(ov::OutputVector{sin_theta_node, sin_theta_node}, -1); + + auto y1 = std::make_shared(rot_data, cos_full); + auto y2 = std::make_shared(x_rotate_half, sin_full); + auto rotated = std::make_shared(y1, y2); + + ov::OutputVector concat_parts; + if (n_offs > 0) { + auto head_start = ov::op::v0::Constant::create(ov::element::i64, {1}, {0}); + auto head_end = ov::op::v0::Constant::create(ov::element::i64, {1}, {n_offs}); + auto head = std::make_shared(data_node, head_start, head_end, step_one, axis_last); + concat_parts.push_back(head); } - - auto split_a = std::make_shared( - data_node, split_axis, - ov::op::v0::Constant::create(ov::element::i64, {split_lengths.size()}, split_lengths)); - auto x0 = split_a->output(0); - auto x1 = split_a->output(1); - auto mul_a = std::make_shared(x0, cos_reshaped); - auto mul_b = std::make_shared(x1, sin_reshaped); - auto sub = std::make_shared(mul_a, mul_b); - - auto mul_c = std::make_shared(x0, sin_reshaped); - auto mul_d = std::make_shared(x1, cos_reshaped); - auto add = std::make_shared(mul_c, mul_d); - - if (n_dims < head_dim) { - auto tail = split_a->output(2); - res = std::make_shared(ov::OutputVector{sub, add, tail}, 3); + concat_parts.push_back(rotated); + if (n_offs + n_dims < head_dim) { + auto tail_start = ov::op::v0::Constant::create(ov::element::i64, {1}, {n_offs + n_dims}); + auto tail_end = ov::op::v0::Constant::create(ov::element::i64, {1}, {head_dim}); + auto tail = std::make_shared(data_node, tail_start, tail_end, step_one, axis_last); + concat_parts.push_back(tail); + } + if (concat_parts.size() == 1) { + res = rotated; } else { - res = std::make_shared(ov::OutputVector{sub, add}, 3); + res = std::make_shared(concat_parts, -1); } } diff --git a/ggml/src/ggml-openvino/openvino/op/set_rows.cpp b/ggml/src/ggml-openvino/openvino/op/set_rows.cpp index 0fe8e0a8d..3b606c82a 100644 --- a/ggml/src/ggml-openvino/openvino/op/set_rows.cpp +++ b/ggml/src/ggml-openvino/openvino/op/set_rows.cpp @@ -7,6 +7,7 @@ #include #include #include +#include #include #include #include @@ -75,7 +76,7 @@ OutputVector translate_set_rows(const NodeContext & context) { res = std::make_shared(dst, ind_squeezed, data_reshaped, axes); } - auto dst_reshape = std::dynamic_pointer_cast(dst.get_node_shared_ptr()); + auto dst_reshape = ov::as_type_ptr(dst.get_node_shared_ptr()); if (!multidim_indices && dst_reshape) { // Fix the case of multiple sequences, reshape back to original shape [1, n_seq, ctx_per_seq, emb] // ctx_per_seq is not fixed due to llama-bench compatibility diff --git a/ggml/src/ggml-openvino/openvino/op/transpose.cpp b/ggml/src/ggml-openvino/openvino/op/transpose.cpp index 8d89ca556..0651a410a 100644 --- a/ggml/src/ggml-openvino/openvino/op/transpose.cpp +++ b/ggml/src/ggml-openvino/openvino/op/transpose.cpp @@ -14,9 +14,7 @@ OutputVector translate_transpose(const NodeContext & context) { // Compute permute order from input/output shape and stride information // so it adapts to different input and output layouts. - auto input_shape = context.get_input_shape(0).to_shape(); auto input_stride = context.get_input_stride(0); - auto output_shape = context.get_output_shape().to_shape(); auto output_stride = context.get_output_stride(); // Compute permute order by matching output and input stride rankings. diff --git a/ggml/src/ggml-openvino/openvino/op/unary_silu.cpp b/ggml/src/ggml-openvino/openvino/op/unary_silu.cpp deleted file mode 100644 index 48ee0431f..000000000 --- a/ggml/src/ggml-openvino/openvino/op/unary_silu.cpp +++ /dev/null @@ -1,27 +0,0 @@ -#include "../node_context.h" -#include "../op_table.h" -#include "../utils.h" - -#include -#include -#include - -namespace ov { -namespace frontend { -namespace ggml { -namespace op { - -OutputVector translate_unary_silu(const NodeContext & context) { - num_inputs_check(context, 1, 1); - - auto input = process_view_input_new(context, 0); - auto sigmoid = std::make_shared(input); - auto res = std::make_shared(input, sigmoid); - - return rename_outputs_with_suffix({res}, context.get_name()); -} - -} // namespace op -} // namespace ggml -} // namespace frontend -} // namespace ov diff --git a/ggml/src/ggml-openvino/openvino/op/unary_softplus.cpp b/ggml/src/ggml-openvino/openvino/op/unary_softplus.cpp index 756d9c33d..a9e495c37 100644 --- a/ggml/src/ggml-openvino/openvino/op/unary_softplus.cpp +++ b/ggml/src/ggml-openvino/openvino/op/unary_softplus.cpp @@ -1,6 +1,7 @@ #include "../node_context.h" #include "../op_table.h" #include "../utils.h" +#include "ggml-openvino/ggml-openvino-extra.h" #include #include @@ -9,6 +10,7 @@ #include #include #include +#include namespace ov { namespace frontend { @@ -18,6 +20,10 @@ namespace op { OutputVector translate_unary_softplus(const NodeContext & context) { num_inputs_check(context, 1, 1); + if (ggml_openvino_getenv_int("GGML_OPENVINO_NATIVE_SOFTPLUS") != 0) { + return translate_1to1_match_1_input(context); + } + auto input = process_view_input_new(context, 0); const auto element_type = input.get_element_type(); auto one = ov::op::v0::Constant::create(element_type, ov::Shape{}, {1.0f}); diff --git a/ggml/src/ggml-openvino/openvino/op_table.cpp b/ggml/src/ggml-openvino/openvino/op_table.cpp index d4f5ac307..f249a06bb 100644 --- a/ggml/src/ggml-openvino/openvino/op_table.cpp +++ b/ggml/src/ggml-openvino/openvino/op_table.cpp @@ -13,6 +13,7 @@ #include #include #include +#include #include namespace ov { @@ -50,10 +51,9 @@ std::unordered_map get_supported_ops() { {"GGML_OP_TRANSPOSE", op::translate_transpose }, {"GGML_UNARY_OP_GELU", op::translate_1to1_match_1_input }, {"GGML_UNARY_OP_SIGMOID", op::translate_1to1_match_1_input }, - {"GGML_UNARY_OP_SILU", op::translate_unary_silu }, + {"GGML_UNARY_OP_SILU", op::translate_1to1_match_1_input }, {"GGML_UNARY_OP_SOFTPLUS", op::translate_unary_softplus }, {"GGML_UNARY_OP_TANH", op::translate_1to1_match_1_input }, - {"GGML_UNARY_OP_SIGMOID", op::translate_1to1_match_1_input }, {"GGML_UNARY_OP_EXP", op::translate_1to1_match_1_input }, {"GGML_UNARY_OP_NEG", op::translate_1to1_match_1_input }, {"GGML_UNARY_OP_RELU", op::translate_1to1_match_1_input }, diff --git a/ggml/src/ggml-openvino/openvino/op_table.h b/ggml/src/ggml-openvino/openvino/op_table.h index a0a42bff3..3dc98bd96 100644 --- a/ggml/src/ggml-openvino/openvino/op_table.h +++ b/ggml/src/ggml-openvino/openvino/op_table.h @@ -30,7 +30,6 @@ GGML_OP_CONVERTER(translate_sqr); GGML_OP_CONVERTER(translate_rope); GGML_OP_CONVERTER(translate_scale); GGML_OP_CONVERTER(translate_sqrt); -GGML_OP_CONVERTER(translate_unary_silu); GGML_OP_CONVERTER(translate_unary_softplus); GGML_OP_CONVERTER(translate_soft_max); GGML_OP_CONVERTER(translate_transpose); diff --git a/ggml/src/ggml-openvino/openvino/pass/fuse_moe_compressed.cpp b/ggml/src/ggml-openvino/openvino/pass/fuse_moe_compressed.cpp new file mode 100644 index 000000000..c4872ac2e --- /dev/null +++ b/ggml/src/ggml-openvino/openvino/pass/fuse_moe_compressed.cpp @@ -0,0 +1,273 @@ +#include "fuse_moe_compressed.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "../op/gather_matmul.hpp" +#include "../op/moe_compressed.hpp" + +namespace ov { +namespace frontend { +namespace ggml { +namespace pass { + +namespace { + +struct dequant_inputs { + ov::Output weight; + ov::Output scale; + ov::Output zp; + bool has_zp = false; + bool ok = false; +}; + +// Peel the chain built by make_int4_weights/make_int8_weights back to its Constant inputs. +// Grouped weights keep the pre-Reshape rank-4 form [n_expert, n, k/group, group] with scale +// and zp at [n_expert, n, k/group, 1], which is the layout MOECompressed expects. Channel-wise +// weights stay rank-3 with a rank-3 scale and carry no zp. +dequant_inputs unwrap_dequant(const ov::Output & b) { + dequant_inputs res; + + auto node = b.get_node_shared_ptr(); + while (ov::is_type(node) || ov::is_type(node)) { + node = node->get_input_node_shared_ptr(0); + } + + auto mul = ov::as_type_ptr(node); + if (!mul) { + return res; + } + res.scale = mul->input_value(1); + + auto lhs = mul->get_input_node_shared_ptr(0); + if (auto sub = ov::as_type_ptr(lhs)) { + // Take the zero point down to its Constant: an integer zp is wrapped in a Convert to f16, + // and the op wants the integer form. A natively quantized expert instead carries an exact + // f16 zp (-min/scale) with no integer behind it, which the MoE kernel does not accept. + auto zp_node = sub->get_input_node_shared_ptr(1); + while (ov::is_type(zp_node)) { + zp_node = zp_node->get_input_node_shared_ptr(0); + } + res.zp = zp_node->output(0); + res.has_zp = true; + lhs = sub->get_input_node_shared_ptr(0); + } + while (ov::is_type(lhs)) { + lhs = lhs->get_input_node_shared_ptr(0); + } + if (!ov::is_type(lhs)) { + return res; + } + + res.weight = lhs->output(0); + res.ok = res.scale.get_partial_shape().is_static() && res.weight.get_partial_shape().is_static(); + return res; +} + +size_t logical_k(const ov::Shape & shape) { + return shape.size() == 4 ? shape[2] * shape[3] : shape.back(); +} + +} // namespace + +FuseMoeCompressed::FuseMoeCompressed() { + using namespace ov::pass::pattern; + + // The gate and up projections each get their own Reshape/Transpose of the hidden state and + // their own Reshape of the routing ids, so every branch needs its own sub-pattern. On GPU + // mul_mat_id also converts the activations to f16 before the op and back to f32 after it, + // so those Converts are matched as optional. + auto hidden_gate_m = any_input(); + auto a_gate_reshape_m = wrap_type({ hidden_gate_m, any_input() }); + auto a_gate_m = + wrap_type({ optional({ a_gate_reshape_m }), any_input() }); + auto hidden_up_m = any_input(); + auto a_up_m = wrap_type( + { optional({ wrap_type({ hidden_up_m, any_input() }) }), + any_input() }); + + auto gate_w_m = any_input(); + auto up_w_m = any_input(); + auto down_w_m = any_input(); + auto ids_gate_m = any_input(); + auto ids_up_m = any_input(); + auto ids_down_m = any_input(); + + auto bgm_gate_m = wrap_type({ a_gate_m, gate_w_m, ids_gate_m, any_input() }); + auto gate_u_m = optional({ wrap_type( + { wrap_type({ bgm_gate_m, any_input() }), any_input() }) }); + + auto silu_m = wrap_type({ gate_u_m }); + + auto bgm_up_m = wrap_type({ a_up_m, up_w_m, ids_up_m, any_input() }); + auto up_u_m = optional({ wrap_type( + { wrap_type({ bgm_up_m, any_input() }), any_input() }) }); + auto swiglu_m = wrap_type({ silu_m, up_u_m }); + + auto d_t_m = wrap_type( + { optional({ wrap_type({ swiglu_m, any_input() }) }), + any_input() }); + auto bgm_down_m = wrap_type({ d_t_m, down_w_m, ids_down_m, any_input() }); + auto down_u_m = optional({ wrap_type( + { wrap_type({ bgm_down_m, any_input() }), any_input() }) }); + + auto routing_m = any_input(); + auto weighted_m = wrap_type({ down_u_m, routing_m }); + auto root_m = wrap_type({ weighted_m, any_input() }); + + const auto callback = [=](Matcher & m) { + auto & pm = m.get_pattern_value_map(); + + const auto gate = unwrap_dequant(pm.at(gate_w_m)); + const auto up = unwrap_dequant(pm.at(up_w_m)); + const auto down = unwrap_dequant(pm.at(down_w_m)); + if (!gate.ok || !up.ok || !down.ok) { + return false; + } + + const auto gate_shape = gate.weight.get_shape(); + const auto up_shape = up.weight.get_shape(); + const auto down_shape = down.weight.get_shape(); + if (gate_shape != up_shape || gate_shape.size() < 3 || down_shape.size() < 3) { + return false; + } + + // MOECompressed carries one group_size and one has_zp for all three projections, so a + // model whose down-proj is quantized differently from gate/up cannot be described. This + // happens when ggml requantizes Q5_K/Q6_K experts to channel-wise int8. + if (gate.has_zp != down.has_zp || gate_shape.size() != down_shape.size()) { + return false; + } + + // The kernel only takes an integer zero point (moe_3gemm_swiglu_opt validate_impl). + if (gate.has_zp) { + static const std::set int_zp_types = { ov::element::u4, ov::element::i4, + ov::element::u8, ov::element::i8 }; + if (int_zp_types.count(gate.zp.get_element_type()) == 0 || + int_zp_types.count(down.zp.get_element_type()) == 0) { + return false; + } + } + + // Config holds a single group_size for all three projections. + const auto group_of = [](const dequant_inputs & w) { + const auto s = w.weight.get_shape(); + return s.size() == 4 ? s[3] : logical_k(s); + }; + if (group_of(gate) != group_of(up) || group_of(gate) != group_of(down)) { + return false; + } + + // all three branches must route the same hidden state through the same experts + if (pm.at(hidden_gate_m) != pm.at(hidden_up_m)) { + return false; + } + + auto ids = pm.at(ids_down_m); + const auto ids_pshape = ids.get_partial_shape(); + if (ids_pshape.rank().is_dynamic() || ids_pshape[ids_pshape.rank().get_length() - 1].is_dynamic()) { + return false; + } + const size_t top_k = ids_pshape[ids_pshape.rank().get_length() - 1].get_length(); + + // routing weights arrive as [1, n_tokens, top_k, 1]; the op wants [..., top_k] + auto routing = pm.at(routing_m); + const auto routing_pshape = routing.get_partial_shape(); + if (routing_pshape.rank().is_dynamic() || routing_pshape.rank().get_length() != 4 || + routing_pshape[3] != 1) { + return false; + } + // MOE requires routing weights and ids to have the same shape. Drop the trailing 1 of the + // routing weights and give the ids the leading batch dim, so both become [1, n_tokens, top_k]. + routing = std::make_shared( + routing, ov::op::v0::Constant::create(ov::element::i64, ov::Shape{ 1 }, { 3 })); + if (ids_pshape.rank().get_length() == 2) { + ids = std::make_shared( + ids, ov::op::v0::Constant::create(ov::element::i64, ov::Shape{ 1 }, { 0 })); + } + if (routing.get_partial_shape() != ids.get_partial_shape()) { + return false; + } + + const size_t down_k = logical_k(down_shape); + const auto down_scale_shape = down.scale.get_shape(); + const size_t down_groups = down_scale_shape.size() >= 3 ? down_scale_shape[2] : 1; + + ov::op::internal::MOECompressed::Config config; + config.expert_type = ov::op::internal::MOE::Expert_type::GEMM3_SWIGLU; + config.activation_type = ov::op::internal::MOE::Activation_type::SWIGLU; + config.expert_alpha = 0.0f; + config.expert_beta = 1.0f; + config.gate_idx = 0; + config.hidden_size = logical_k(gate_shape); + config.inter_size = gate_shape[1]; + config.num_expert = gate_shape[0]; + config.num_shared_expert = 0; + config.top_k = top_k; + config.group_size = down_groups <= 1 ? std::numeric_limits::max() : down_k / down_groups; + config.has_batch_dim = true; + config.has_zp = gate.has_zp; + // dynamic makes the output follow the hidden state, so the plugin can lower this region + // to f16 together with the rest of the graph + config.out_type = ov::element::dynamic; + + auto absent_zp = [] { + auto zp = std::make_shared(ov::element::dynamic, ov::Shape{ 0 }); + ov::pass::disable_constant_folding(zp); + return zp->output(0); + }; + + // MOE takes its output type from the hidden state. Transpose the activations before the + // f16 Convert that mul_mat_id adds on GPU, so the op stays f32 like the block it replaces + // and the plugin can lower the whole region uniformly. + const auto a_transpose = pm.at(a_gate_m).get_node_shared_ptr(); + ov::Output hidden = + std::make_shared(pm.at(a_gate_reshape_m), a_transpose->input_value(1)); + + const ov::OutputVector args = { + hidden, routing, ids, + gate.weight, gate.scale, gate.has_zp ? gate.zp : absent_zp(), + up.weight, up.scale, up.has_zp ? up.zp : absent_zp(), + down.weight, down.scale, down.has_zp ? down.zp : absent_zp(), + }; + + auto moe = std::make_shared(args, config); + + // MOE takes its output type from the hidden state, which is f16 on GPU, while the rest of + // the ggml graph works in f32. + ov::Output result = moe->output(0); + const auto root_type = m.get_match_root()->get_output_element_type(0); + if (result.get_element_type() != root_type) { + result = std::make_shared(result, root_type); + } + + result.get_node_shared_ptr()->set_friendly_name(m.get_match_root()->get_friendly_name()); + ov::copy_runtime_info(m.get_matched_nodes(), result.get_node_shared_ptr()); + ov::replace_node(m.get_match_root(), result.get_node_shared_ptr()); + register_new_node(moe); + return true; + }; + + register_matcher(std::make_shared(root_m, "ov::frontend::ggml::pass::FuseMoeCompressed"), callback); +} + +} // namespace pass +} // namespace ggml +} // namespace frontend +} // namespace ov diff --git a/ggml/src/ggml-openvino/openvino/pass/fuse_moe_compressed.h b/ggml/src/ggml-openvino/openvino/pass/fuse_moe_compressed.h new file mode 100644 index 000000000..5500bed68 --- /dev/null +++ b/ggml/src/ggml-openvino/openvino/pass/fuse_moe_compressed.h @@ -0,0 +1,19 @@ +#include "openvino/pass/matcher_pass.hpp" + +namespace ov { +namespace frontend { +namespace ggml { +namespace pass { + +// Folds the MoE expert block emitted for MUL_MAT_ID (3 GatherMatmul + SwiGLU + routing +// weighting + expert reduction) into a single ov::op::internal::MOECompressed. +class FuseMoeCompressed : public ov::pass::MatcherPass { +public: + OPENVINO_MATCHER_PASS_RTTI("ov::frontend::ggml::pass::FuseMoeCompressed") + FuseMoeCompressed(); +}; + +} // namespace pass +} // namespace ggml +} // namespace frontend +} // namespace ov diff --git a/ggml/src/ggml-openvino/openvino/pass/kv_state_seq_axis.cpp b/ggml/src/ggml-openvino/openvino/pass/kv_state_seq_axis.cpp new file mode 100644 index 000000000..c9952b1d5 --- /dev/null +++ b/ggml/src/ggml-openvino/openvino/pass/kv_state_seq_axis.cpp @@ -0,0 +1,114 @@ +#include "kv_state_seq_axis.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace ov { +namespace frontend { +namespace ggml { +namespace pass { + +namespace { + +const std::vector & seq_axis_perm() { + // [1, seq, n_heads_kv, head_size] <-> [1, n_heads_kv, seq, head_size] + static const std::vector perm{0, 2, 1, 3}; + return perm; +} + +// True when the state still has the frontend's stateful KV layout, so the sequence axis +// can be moved: rank 4, batch and both head dims static, and seq the only dynamic dim, +// at dim 1. Any KV head count is fine. With a single head the rewrite is pure metadata +// ([1, seq, 1, head] and [1, 1, seq, head] are the same memory); with several heads it +// also drops the reader-side transpose of the whole accumulated state, which is where +// most of the gain comes from at depth. +bool can_move_seq_axis(const ov::PartialShape & shape) { + return shape.rank().is_static() && shape.rank().get_length() == 4 && shape[0].is_static() && + shape[1].is_dynamic() && shape[2].is_static() && shape[3].is_static(); +} + +std::shared_ptr match_kv_append(const std::shared_ptr & assign) { + auto concat = ov::as_type_ptr(assign->get_input_node_shared_ptr(0)); + if (!concat || concat->get_input_size() != 2 || concat->get_axis() != 1) { + return nullptr; + } + auto read_value = ov::as_type_ptr(concat->get_input_node_shared_ptr(0)); + if (!read_value || read_value->get_variable() != assign->get_variable()) { + return nullptr; + } + if (!can_move_seq_axis(read_value->get_output_partial_shape(0))) { + return nullptr; + } + return concat; +} + +} // namespace + +bool KVStateSeqAxis::run_on_model(const std::shared_ptr & model) { + std::vector> assigns; + for (const auto & op : model->get_ops()) { + if (auto assign = ov::as_type_ptr(op)) { + assigns.push_back(assign); + } + } + + bool changed = false; + for (const auto & assign : assigns) { + auto concat = match_kv_append(assign); + if (!concat) { + continue; + } + auto read_value = ov::as_type_ptr(concat->get_input_node_shared_ptr(0)); + + auto variable = read_value->get_variable(); + auto info = variable->get_info(); + const auto & shape = info.data_shape; + info.data_shape = ov::PartialShape{shape[0], shape[2], shape[1], shape[3]}; + variable->update(info); + read_value->validate_and_infer_types(); + + auto readers = concat->output(0).get_target_inputs(); + + auto new_rows = concat->input_value(1); + auto perm_in = ov::op::v0::Constant::create(ov::element::i64, {4}, seq_axis_perm()); + concat->set_argument(1, std::make_shared(new_rows, perm_in)); + concat->set_axis(2); + concat->validate_and_infer_types(); + + // Readers still expect seq at dim 1. A reader that is itself the inverse + // Transpose wanted seq at dim 2 all along, so drop it; give anything else the + // inverse Transpose so its input is unchanged. + for (auto & reader : readers) { + auto * node = reader.get_node(); + if (ov::is_type(node)) { + continue; + } + bool dropped = false; + if (auto * transpose = ov::as_type(node)) { + auto order = ov::as_type_ptr(transpose->get_input_node_shared_ptr(1)); + if (order && order->cast_vector() == seq_axis_perm()) { + ov::replace_output_update_name(transpose->output(0), concat->output(0)); + dropped = true; + } + } + if (!dropped) { + auto perm_out = ov::op::v0::Constant::create(ov::element::i64, {4}, seq_axis_perm()); + reader.replace_source_output(std::make_shared(concat->output(0), perm_out)); + } + } + changed = true; + } + + return changed; +} + +} // namespace pass +} // namespace ggml +} // namespace frontend +} // namespace ov diff --git a/ggml/src/ggml-openvino/openvino/pass/kv_state_seq_axis.h b/ggml/src/ggml-openvino/openvino/pass/kv_state_seq_axis.h new file mode 100644 index 000000000..579022c45 --- /dev/null +++ b/ggml/src/ggml-openvino/openvino/pass/kv_state_seq_axis.h @@ -0,0 +1,24 @@ +#include "openvino/pass/pass.hpp" + +namespace ov { +namespace frontend { +namespace ggml { +namespace pass { + +// Moves the sequence axis of the stateful KV cache from dim 1 to dim 2, i.e. from +// [1, seq, n_heads_kv, head_size] to [1, n_heads_kv, seq, head_size], and updates the +// Concat that appends to it. Two wins: the GPU plugin only appends new tokens in place +// when the growing axis is a spatial axis, and the reader no longer has to transpose the +// whole accumulated state every token (that cost grows with context length, so it is the +// larger win at depth for a model with several KV heads). Only rewrites states that still +// match the frontend layout, so it no-ops if that layout ever changes. +class KVStateSeqAxis : public ov::pass::ModelPass { +public: + OPENVINO_MODEL_PASS_RTTI("ov::frontend::ggml::pass::KVStateSeqAxis") + bool run_on_model(const std::shared_ptr & model) override; +}; + +} // namespace pass +} // namespace ggml +} // namespace frontend +} // namespace ov diff --git a/ggml/src/ggml-openvino/openvino/pass/squeeze_matmul.cpp b/ggml/src/ggml-openvino/openvino/pass/squeeze_matmul.cpp index 20a3a3749..09c213f3e 100644 --- a/ggml/src/ggml-openvino/openvino/pass/squeeze_matmul.cpp +++ b/ggml/src/ggml-openvino/openvino/pass/squeeze_matmul.cpp @@ -2,6 +2,7 @@ #include #include +#include #include #include #include @@ -26,7 +27,7 @@ SqueezeMatmul::SqueezeMatmul() { const auto callback = [=](ov::pass::pattern::Matcher & m) { const auto & pattern_map = m.get_pattern_value_map(); auto matmul_node = - std::dynamic_pointer_cast(pattern_map.at(m_matmul).get_node_shared_ptr()); + ov::as_type_ptr(pattern_map.at(m_matmul).get_node_shared_ptr()); auto act = pattern_map.at(m_act); auto wei = pattern_map.at(m_wei); auto act_shape = act.get_partial_shape(); diff --git a/ggml/src/ggml-openvino/openvino/translate_session.cpp b/ggml/src/ggml-openvino/openvino/translate_session.cpp index df3a72f32..3170c2e4c 100644 --- a/ggml/src/ggml-openvino/openvino/translate_session.cpp +++ b/ggml/src/ggml-openvino/openvino/translate_session.cpp @@ -5,7 +5,9 @@ #include "ggml-openvino/openvino/node_context.h" #include "ggml-openvino/openvino/utils.h" #include "input_model.h" +#include "pass/fuse_moe_compressed.h" #include "pass/fuse_to_conv.h" +#include "pass/kv_state_seq_axis.h" #include "pass/mark_decompression_convert_constant_folding.h" #include "pass/mark_dequantization_subgraph.h" #include "pass/squeeze_matmul.h" @@ -19,28 +21,36 @@ #include #include #include +#include #include #include #include #include +#include #include #include #include #include #include +#include +#include +#include #include #include #include #include #include +#include #include #include #include #include +#include #include #include #include #include +#include #include namespace ov { @@ -143,6 +153,64 @@ void add_sliced_mask_stateful(TensorMap & tensor_map) { create_sliced_mask("self_kq_mask_swa", "KQ_mask_swa_sliced"); } +// Rebuild the sliding-window mask from absolute positions. +// ggml caps self_kq_mask_swa at the size of its own SWA cache, but the stateful KV state is +// Concat-appended and grows without bound, so past that cap the two disagree on length and the +// mask add fails. A pure-Concat state is ordered by position, so positions can rebuild the mask. +// swa_window holds the real n_swa, read back from the ggml mask in ggml-decoder.cpp. +// No-op when the graph has no SWA mask, or when the window could not be read back. +void add_position_mask_stateful_swa(TensorMap & tensor_map) { + if (tensor_map.find("self_kq_mask_swa") == tensor_map.end() || tensor_map.find("inp_pos") == tensor_map.end() || + tensor_map.find("swa_window") == tensor_map.end()) { + return; + } + + auto inp_pos = tensor_map.at("inp_pos").get_node_shared_ptr(); + + auto zero_i64 = ov::op::v0::Constant::create(ov::element::i64, {1}, {0}); + auto one_i64 = ov::op::v0::Constant::create(ov::element::i64, {1}, {1}); + auto three = ov::op::v0::Constant::create(ov::element::i64, {1}, {3}); + auto neg_one = ov::op::v0::Constant::create(ov::element::i64, {1}, {-1}); + + auto query_pos = std::make_shared(inp_pos, ov::element::i64); + auto query_pos_1d = std::make_shared( + query_pos, ov::op::v0::Constant::create(ov::element::i64, {1}, {-1}), false); + + auto last_pos = std::make_shared(inp_pos, neg_one, three); + auto last_pos_1d = std::make_shared(last_pos, one_i64, false); + auto last_pos_cvt = std::make_shared(last_pos_1d, ov::element::i64); + auto total_len = std::make_shared(last_pos_cvt, one_i64); + auto total_len_scalar = std::make_shared(total_len); + + auto cached_pos = std::make_shared( + ov::op::v0::Constant::create(ov::element::i64, {}, {0}), total_len_scalar, + ov::op::v0::Constant::create(ov::element::i64, {}, {1}), ov::element::i64); + + auto query_col = std::make_shared( + query_pos_1d, ov::op::v0::Constant::create(ov::element::i64, {2}, {-1, 1}), false); + auto cached_row = std::make_shared( + cached_pos, ov::op::v0::Constant::create(ov::element::i64, {2}, {1, -1}), false); + auto diff = std::make_shared(query_col, cached_row); + + auto swa_window = tensor_map.at("swa_window").get_node_shared_ptr(); + auto window = std::make_shared(swa_window, ov::element::i64); + auto causal_ok = std::make_shared(diff, zero_i64); + auto window_ok = std::make_shared(diff, window); + auto keep = std::make_shared(causal_ok, window_ok); + + auto zero_f = ov::op::v0::Constant::create(ov::element::f32, {}, {0.0f}); + auto neg_inf_f = ov::op::v0::Constant::create(ov::element::f32, {}, {-std::numeric_limits::infinity()}); + std::shared_ptr mask = std::make_shared(keep, zero_f, neg_inf_f); + + auto batch_axis = ov::op::v0::Constant::create(ov::element::i64, {1}, {0}); + mask = std::make_shared(mask, batch_axis); + mask = std::make_shared(mask, batch_axis); + mask = std::make_shared(mask, ov::element::f16); + mask->set_friendly_name("KQ_mask_swa_sliced"); + + tensor_map["KQ_mask_swa_sliced"] = mask->output(0); +} + void add_rope_sin_cos(TensorMap & tensor_map, GgmlDecoder & ggml_model_decoder) { // When ROPE ops in the graph have divergent op_params (e.g. gemma4's mixed // SWA/non-SWA layers with different n_dims or freq_base), a shared sin/cos @@ -175,6 +243,7 @@ void add_rope_sin_cos(TensorMap & tensor_map, GgmlDecoder & ggml_model_decoder) void preprocess(TensorMap & tensor_map, GgmlDecoder & ggml_model_decoder) { if (ggml_model_decoder.is_stateful()) { add_sliced_mask_stateful(tensor_map); + add_position_mask_stateful_swa(tensor_map); } // This optimization is error-prone // add_rope_sin_cos(tensor_map, ggml_model_decoder); @@ -204,7 +273,7 @@ std::shared_ptr TranslateSession::translate_graph(const frontend::InputMo auto tensor_map = std::make_shared(); std::shared_ptr resulting_model; - const auto & ggml_model = std::dynamic_pointer_cast(input_model); + const auto & ggml_model = ov::as_type_ptr(input_model); std::shared_ptr ggml_model_decoder = ggml_model->get_model_decoder(); for (const auto & it : ggml_model_decoder->get_model_inputs()) { @@ -216,7 +285,7 @@ std::shared_ptr TranslateSession::translate_graph(const frontend::InputMo for (const auto & it : ggml_model_decoder->get_model_extra_inputs()) { auto input_node = create_extra_input(it.first, it.second); if (it.second.is_parameter) { - params.push_back(std::dynamic_pointer_cast(input_node)); + params.push_back(ov::as_type_ptr(input_node)); } (*tensor_map)[it.first] = input_node; } @@ -387,7 +456,7 @@ std::shared_ptr TranslateSession::translate_graph(const frontend::InputMo } std::shared_ptr TranslateSession::apply_transformations(std::shared_ptr model) { - auto ggml_model_decoder = std::dynamic_pointer_cast(m_input_model)->get_model_decoder(); + auto ggml_model_decoder = ov::as_type_ptr(m_input_model)->get_model_decoder(); { ov::pass::Manager manager; manager.set_per_pass_validation(true); @@ -400,10 +469,20 @@ std::shared_ptr TranslateSession::apply_transformations(std::shared_ptr{ov::element::u8, ov::element::i8, ov::element::u4, ov::element::i4}); manager.register_pass(); + // MOECompressed has no CPU plugin implementation, so keep the GatherMatmul path + // everywhere else. Opt-in while the fused path is being brought up. + if (ggml_openvino_get_device_name() == "GPU" && getenv("GGML_OPENVINO_MOE_OP")) { + manager.register_pass(); + } + if (ggml_model_decoder->is_stateful()) { const auto kv_param_res_names = ggml_model_decoder->get_kv_param_res_names(); const auto kv_param_res_pairs = get_kv_param_res_pairs(model, kv_param_res_names); manager.register_pass(kv_param_res_pairs); + // Must run after MakeStateful, which is what creates the ReadValue/Assign pairs. + if (!ggml_openvino_getenv_int("GGML_OPENVINO_DISABLE_KV_STATE_RELAYOUT")) { + manager.register_pass(); + } } if (ggml_model_decoder->is_static()) { diff --git a/ggml/src/ggml-openvino/utils.cpp b/ggml/src/ggml-openvino/utils.cpp index 93b1ccbe9..44a9b2c78 100644 --- a/ggml/src/ggml-openvino/utils.cpp +++ b/ggml/src/ggml-openvino/utils.cpp @@ -1,6 +1,7 @@ #include "utils.h" #include "ggml-impl.h" +#include "ggml-openvino.h" #include "ggml-openvino-extra.h" #include "ggml-openvino/ggml-decoder.h" #include "ggml.h" @@ -42,6 +43,11 @@ #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wdeprecated-declarations" +// Both execution paths use two cache levels: +// 1. Reuse this backend's decoder/request via graph_key and compatibility checks. +// 2. On a local miss, look up compiled_graph_key in the shared compilation cache, +// compile if needed, then create a private request from the compiled model. +// The shared lock covers compilation and frontend cleanup, never inference. enum ggml_status ov_graph_compute(ggml_cgraph * cgraph, ggml_backend_t backend) { ggml_backend_openvino_context * ctx = (ggml_backend_openvino_context *) backend->context; try { @@ -54,6 +60,7 @@ enum ggml_status ov_graph_compute(ggml_cgraph * cgraph, ggml_backend_t backend) GGML_ASSERT(ctx->runtime_context != nullptr); std::shared_ptr r_ctx = std::static_pointer_cast(ctx->runtime_context); + std::lock_guard execution_lock(r_ctx->execution_mutex); return is_static ? ov_graph_compute_static(cgraph, r_ctx) : ov_graph_compute_dynamic(cgraph, r_ctx); } catch (const ov::Exception & e) { @@ -143,7 +150,7 @@ static uint64_t ggml_openvino_model_cache_extra_cfg(const std::string & device, ggml_openvino_getenv_int("GGML_OPENVINO_MANUAL_GQA_ATTN") > 0 : device == "GPU"; - uint64_t extra_cfg = 0; + uint64_t extra_cfg = 1; // Graph-ordinal port names (invalidate older disk-cache blobs). extra_cfg = extra_cfg * 131 + (stateful ? 1u : 0u); extra_cfg = extra_cfg * 131 + (ggml_openvino_reduce_compile_mem_enabled() ? 1u : 0u); extra_cfg = extra_cfg * 131 + (ggml_openvino_getenv_int("GGML_OPENVINO_DISABLE_KV_SLICE") ? 1u : 0u); @@ -151,6 +158,91 @@ static uint64_t ggml_openvino_model_cache_extra_cfg(const std::string & device, return extra_cfg; } +static std::map> get_weight_names(ggml_cgraph * cgraph) { + std::map> names; + for (const auto & name : GgmlOvDecoder::collect_weight_names(cgraph)) { + names[name] = nullptr; + } + return names; +} + +// A conservative, exact in-process key, evaluated only on a context-local cache +// miss. Include topology, layouts, op parameters, constant extra inputs and weight +// allocation identities. Never use a sampled weight hash or a graph name alone: +// different models can have identical topology. OV buffer IDs survive address reuse. +static std::string compiled_graph_key(const ggml_cgraph * graph, const GgmlOvDecoder & decoder, + const std::string & device, int prefill_chunk_size = 0) { + std::string key; + auto append = [&key](const auto & value) { + key.append(reinterpret_cast(&value), sizeof(value)); + }; + auto append_string = [&](const std::string & value) { + append(value.size()); + key.append(value); + }; + append_string(device); + append(decoder.is_static()); + append(decoder.is_stateful()); + append(prefill_chunk_size); + bool has_weight_buffer_id = false; + std::unordered_map ids; + std::function visit = [&](const ggml_tensor * tensor) { + if (!tensor) { + append(size_t(0)); + return; + } + auto inserted = ids.emplace(tensor, ids.size() + 1); + append(inserted.first->second); + if (!inserted.second) { + return; + } + append_string(tensor->name); + append(tensor->type); + append(tensor->op); + append(tensor->flags); + append(tensor->ne); + append(tensor->nb); + append(tensor->op_params); + append(tensor->view_offs); + const auto * base = tensor->view_src ? tensor->view_src : tensor; + const bool weight = base->buffer && base->buffer->usage == GGML_BACKEND_BUFFER_USAGE_WEIGHTS; + append(weight); + if (weight) { + const size_t buffer_id = ggml_backend_openvino_buffer_get_ctx_id(base->buffer); + has_weight_buffer_id |= buffer_id != 0; + append(buffer_id); + append(tensor->data); + } + visit(tensor->view_src); + for (const auto * src : tensor->src) { + visit(src); + } + }; + append(graph->n_nodes); + for (int i = 0; i < graph->n_nodes; ++i) { + visit(graph->nodes[i]); + } + append(graph->n_leafs); + for (int i = 0; i < graph->n_leafs; ++i) { + visit(graph->leafs[i]); + } + for (const auto & input : decoder.get_model_extra_inputs()) { + append_string(input.first); + append_string(input.second.type.get_type_name()); + append(input.second.shape.size()); + for (auto dim : input.second.shape) { + append(dim); + } + append(input.second.is_parameter); + if (!input.second.is_parameter) { + append(input.second.value); + } + } + // Without an allocation generation, pointer reuse could select stale weights. + // Such graphs still get private requests; they simply do not share compilation. + return has_weight_buffer_id ? key : std::string{}; +} + ov::Tensor create_ov_output_tensor(std::shared_ptr ggml_decoder, std::shared_ptr infer_request, int output_index, @@ -191,6 +283,26 @@ ov::Tensor create_ov_output_tensor(std::shared_ptr ggml_decoder, return output_tensor; } +// Rewrite ggml's KV rows into a relayout state that keeps the sequence on dim 2. +// ggml stores [seq][n_heads_kv * head_size]; the state wants [1, n_heads_kv, seq, head_size], +// a different element order, so the rows are copied instead of reinterpreted. +static ov::Tensor kv_rows_to_seq_axis_2(const ov::Tensor & kv_tensor, size_t n_heads_kv) { + const size_t rows = kv_tensor.get_shape()[2]; + const size_t head_size = kv_tensor.get_shape()[3] / n_heads_kv; + const size_t elem = kv_tensor.get_element_type().size(); + const size_t head_bytes = head_size * elem; + + ov::Tensor out(kv_tensor.get_element_type(), ov::Shape{1, n_heads_kv, rows, head_size}); + const auto * src = static_cast(kv_tensor.data()); + auto * dst = static_cast(out.data()); + for (size_t s = 0; s < rows; s++) { + for (size_t h = 0; h < n_heads_kv; h++) { + memcpy(dst + (h * rows + s) * head_bytes, src + (s * n_heads_kv + h) * head_bytes, head_bytes); + } + } + return out; +} + enum ggml_status ov_graph_compute_dynamic(ggml_cgraph * cgraph, std::shared_ptr r_ctx) { auto & core = ov_singleton_core(); const auto & config = ggml_openvino_get_compile_config(); @@ -216,7 +328,7 @@ enum ggml_status ov_graph_compute_dynamic(ggml_cgraph * cgraph, std::shared_ptr< if (is_naive(cgraph)) { if (!model_is_splitted) { - return naive_compute(cgraph, core, device, config); + return naive_compute(cgraph, core, device, config, *r_ctx->compiled_cache); } } @@ -260,6 +372,7 @@ enum ggml_status ov_graph_compute_dynamic(ggml_cgraph * cgraph, std::shared_ptr< } std::lock_guard lock(*(entry->mutex)); + cache_hit = cache_hit && entry->ptr && r_ctx->infer_request_cache.count(key) != 0; if (cache_hit) { ggml_decoder = entry->ptr; @@ -297,32 +410,90 @@ enum ggml_status ov_graph_compute_dynamic(ggml_cgraph * cgraph, std::shared_ptr< } else if (r_ctx->stateful_kv_size == static_cast(pos_data[0])) { r_ctx->stateful_kv_size += pos_shape[3]; } else { + const size_t pos_begin = static_cast(pos_data[0]); + const bool refill = pos_begin > r_ctx->stateful_kv_size; + + // A refill seeds the state from ggml's KV cache, so it needs that cache to be a + // plain prefix: cell i must hold position i. An SWA layer keeps only the last + // n_swa positions, so once a position leaves the window ggml drops it and the + // remaining cells shift - cell i stops holding position i. While every position + // is still inside the window nothing has been dropped and the refill is sound. + if (refill && !ggml_decoder->get_model_params().swa_layers.empty()) { + const int n_swa = ggml_decoder->get_compute_params().swa_window; + if (n_swa < 0 || static_cast(n_swa) < pos_begin) { + GGML_LOG_ERROR( + "GGML OpenVINO backend stateful inference failed: cannot resume at position %zu from a " + "state that holds %zu tokens, because the sliding-window layers keep only the last %d " + "positions. Run without GGML_OPENVINO_STATEFUL_EXECUTION.\n", + pos_begin, r_ctx->stateful_kv_size, n_swa); + return GGML_STATUS_FAILED; + } + } + + const bool relayout_enabled = + !ggml_openvino_getenv_int("GGML_OPENVINO_DISABLE_KV_STATE_RELAYOUT"); + auto states = infer_request->query_state(); for (auto state : states) { auto state_tensor = state.get_state(); auto state_tensor_shape = state_tensor.get_shape(); - if (static_cast(pos_data[0]) > r_ctx->stateful_kv_size) { - std::string state_name; - try { - state_name = r_ctx->kv_state_input_name_map.at(state.get_name()); - } catch (...) { + + std::string state_name; + if (auto it = r_ctx->kv_state_input_name_map.find(state.get_name()); + it != r_ctx->kv_state_input_name_map.end()) { + state_name = it->second; + } + + // Which axis holds the sequence: pass::KVStateSeqAxis moves it from dim 1 + // to dim 2. The head count is still needed below, because only a 1-head + // state stays byte-compatible with ggml's cache buffer. gemma-4 12B mixes + // 1-head full layers with 8-head sliding layers, so it is per state. + int n_heads_kv = ggml_decoder->get_model_params().n_heads_kv; + if (auto layer = extract_layer_from_name(state_name); layer.has_value()) { + n_heads_kv = ggml_decoder->get_n_heads_kv_for_layer(layer.value()); + } + const bool relayout_this_state = relayout_enabled; + const size_t seq_axis = relayout_this_state ? 2 : 1; + const size_t head_axis = seq_axis == 2 ? 1 : 2; + + if (refill) { + if (state_name.empty()) { GGML_LOG_ERROR( "GGML OpenVINO backend stateful inference failed: no input found for the state\n"); return GGML_STATUS_FAILED; } auto kv_tensor = get_ov_input_tensor(ggml_decoder, state_name); - kv_tensor.set_shape({state_tensor_shape[0], kv_tensor.get_shape()[2], state_tensor_shape[2], - state_tensor_shape[3]}); - state_tensor = kv_tensor; + if (relayout_this_state && n_heads_kv != 1) { + // several heads with seq on dim 2: not the same bytes as ggml's + // buffer, so the rows have to be copied into the new order + state_tensor = kv_rows_to_seq_axis_2(kv_tensor, (size_t) n_heads_kv); + } else { + ov::Shape refill_shape(4); + refill_shape[0] = state_tensor_shape[0]; + refill_shape[seq_axis] = kv_tensor.get_shape()[2]; + refill_shape[head_axis] = state_tensor_shape[head_axis]; + refill_shape[3] = state_tensor_shape[3]; + kv_tensor.set_shape(refill_shape); + state_tensor = kv_tensor; + } state_tensor_shape = state_tensor.get_shape(); } + // Only ever shrink to a prefix the source really has. Slicing past it used to + // surface as a bare ov::Exception from the ROI constructor. + if (state_tensor_shape[seq_axis] < pos_begin) { + GGML_LOG_ERROR( + "GGML OpenVINO backend stateful inference failed: state '%s' holds %zu tokens on axis " + "%zu, cannot resume at position %zu\n", + state.get_name().c_str(), state_tensor_shape[seq_axis], seq_axis, pos_begin); + return GGML_STATUS_FAILED; + } ov::Coordinate begin = {0, 0, 0, 0}; - ov::Coordinate end = {state_tensor_shape[0], static_cast(pos_data[0]), - state_tensor_shape[2], state_tensor_shape[3]}; + ov::Coordinate end(state_tensor_shape.begin(), state_tensor_shape.end()); + end[seq_axis] = pos_begin; ov::Tensor new_state_tensor(state_tensor, begin, end); state.set_state(new_state_tensor); } - r_ctx->stateful_kv_size = pos_data[0] + pos_shape[3]; + r_ctx->stateful_kv_size = pos_begin + pos_shape[3]; } } @@ -330,11 +501,30 @@ enum ggml_status ov_graph_compute_dynamic(ggml_cgraph * cgraph, std::shared_ptr< conversion_end_time = decoder_end_time; compile_end_time = decoder_end_time; } else { + // Compilation can mutate shared weight nodes, so serialize cold paths. + // The lock is released before binding tensors or running inference. + auto shared_cache = r_ctx->compiled_cache; + std::unique_lock compile_lock(shared_cache->mutex); + auto weight_names = get_weight_names(cgraph); + ggml_decoder = std::make_shared(cgraph, m_params, c_params, weight_names, + is_static, stateful, model_is_splitted); + const std::string shared_key = cache_enabled ? compiled_graph_key(cgraph, *ggml_decoder, device) : ""; + ov::CompiledModel shared_model; + bool imported = false; + auto shared_it = shared_cache->graphs.find(shared_key); + if (!shared_key.empty() && shared_it != shared_cache->graphs.end()) { + shared_model = shared_it->second.decode; + infer_request = std::make_shared(shared_model.create_infer_request()); + ov_input_names = shared_it->second.input_names; + ov_output_names = shared_it->second.output_names; + imported = true; + GGML_LOG_DEBUG("ggml-openvino: shared compiled model HIT (dynamic)\n"); + } // Fail fast: a cache-miss recompile feeds weight data to compile_model, but // GGML_OPENVINO_RELEASE_WEIGHTS (or GGML_OPENVINO_MEMORY_OPTIMIZE on GPU) // may have already dropped the host weight pages // (they would read as zeros). That mode requires stable graph shapes. - if (ggml_openvino_weight_buffers_released()) { + if (!imported && ggml_openvino_weight_buffers_released()) { GGML_ABORT( "ggml-openvino: a new graph needs to be compiled but host weight buffers were already " "released via GGML_OPENVINO_RELEASE_WEIGHTS/GGML_OPENVINO_MEMORY_OPTIMIZE. This mode requires " @@ -354,7 +544,6 @@ enum ggml_status ov_graph_compute_dynamic(ggml_cgraph * cgraph, std::shared_ptr< const std::string model_cache_dir = ggml_openvino_model_cache_dir(); uint64_t model_fp = 0; std::string blob_path, manifest_path; - bool imported = false; // When the frontend model cache is active it supersedes the plugin-level // ov::cache_dir: a blob exported from a model compiled WITH cache_dir cannot // be re-imported (import returns an uninitialized model). Strip cache_dir / @@ -364,10 +553,10 @@ enum ggml_status ov_graph_compute_dynamic(ggml_cgraph * cgraph, std::shared_ptr< mc_config.erase("CACHE_DIR"); mc_config.erase("CACHE_MODE"); } - if (!model_cache_dir.empty() && !model_is_splitted) { + if (!imported && !model_cache_dir.empty() && !model_is_splitted) { const uint64_t extra_cfg = ggml_openvino_model_cache_extra_cfg(device, stateful); model_fp = ggml_openvino_model_fingerprint(cgraph, device, /*fa=*/true, m_params.rope_params, - 15, extra_cfg); + 16, extra_cfg); blob_path = ggml_openvino_model_cache_blob_path(model_cache_dir, model_fp); manifest_path = ggml_openvino_model_cache_manifest_path(model_cache_dir, model_fp); @@ -393,6 +582,7 @@ enum ggml_status ov_graph_compute_dynamic(ggml_cgraph * cgraph, std::shared_ptr< ggml_decoder = std::make_shared(cgraph, m_params, c_params, weight_names, is_static, stateful, model_is_splitted); infer_request = std::make_shared(cm.create_infer_request()); + shared_model = cm; entry->ptr = ggml_decoder; // Names must match the decoder's ggml-tensor keys. The non-cached // path keys off Parameter/Result *friendly names* (set by the @@ -486,6 +676,7 @@ enum ggml_status ov_graph_compute_dynamic(ggml_cgraph * cgraph, std::shared_ptr< } infer_request = std::make_shared(compiled_model.create_infer_request()); + shared_model = compiled_model; entry->ptr = ggml_decoder; for (const auto & ov_param : model->get_parameters()) { @@ -496,6 +687,11 @@ enum ggml_status ov_graph_compute_dynamic(ggml_cgraph * cgraph, std::shared_ptr< } } // end non-imported (compile) path + entry->ptr = ggml_decoder; + if (!shared_key.empty() && shared_it == shared_cache->graphs.end()) { + shared_cache->graphs.emplace(shared_key, ov_compiled_graph{shared_model, {}, ov_input_names, + ov_output_names}); + } if (cache_enabled) { std::lock_guard map_lock(r_ctx->ctx_mutex); r_ctx->infer_request_cache[key] = infer_request; @@ -506,6 +702,18 @@ enum ggml_status ov_graph_compute_dynamic(ggml_cgraph * cgraph, std::shared_ptr< if (stateful && cache_enabled) { const auto * inp_pos = get_inp_pos_tensor(cgraph); auto pos_shape = ggml_decoder->get_shape(inp_pos); + // A freshly compiled model starts with an empty state, so it can only serve a + // sequence from its beginning. A non-zero start position means the KV history was + // built elsewhere (a restored ggml cache), which the state cannot adopt. + const int32_t pos_begin = ((int32_t *) inp_pos->data)[0]; + if (pos_begin != 0) { + GGML_LOG_ERROR( + "GGML OpenVINO backend stateful inference failed: a new model was compiled for a sequence that " + "starts at position %d, but its state is empty. Run without " + "GGML_OPENVINO_STATEFUL_EXECUTION.\n", + pos_begin); + return GGML_STATUS_FAILED; + } r_ctx->stateful_kv_size = pos_shape[3]; const auto kv_param_res_names = ggml_decoder->get_kv_param_res_names(); for (const auto & pair : kv_param_res_names) { @@ -570,21 +778,31 @@ enum ggml_status ov_graph_compute_dynamic(ggml_cgraph * cgraph, std::shared_ptr< // GGML_OPENVINO_RELEASE_WEIGHTS (or GGML_OPENVINO_MEMORY_OPTIMIZE on GPU): the plugin holds its own device copy of // every weight after compile, so the host weight buffers can be dropped to reclaim - // RSS. The GPU backend uses a single dynamic-shape model for both prefill and decode, - // so once a graph is compiled it is reused for the whole session — the only thing - // that forces a recompile is clear_caches() on backend teardown. We therefore release - // on the first cache-hit (model compiled, plugin has its copy) and, crucially, pin the - // compiled-model cache so it survives backend teardown (see ggml_backend_openvino_free). - // Without the pin, a later test/context would recompile against the now-dropped pages. - // A genuinely new graph still fails fast at the cache-miss compile branch. - if (cache_hit && ggml_openvino_release_weights_enabled(device) && - !ggml_openvino_weight_buffers_released()) { - ggml_openvino_release_weight_buffers(); + // RSS. Release only while holding the compilation mutex so another context cannot + // be reading host weights during conversion/compilation. Pin the shared compiled + // models across backend teardown; a later context can create its own request without + // reading the dropped pages. A new, uncached graph still fails fast above. + if (cache_hit && ggml_openvino_release_weights_enabled(device)) { + std::lock_guard compile_lock(r_ctx->compiled_cache->mutex); + if (!ggml_openvino_weight_buffers_released()) { + ggml_openvino_release_weight_buffers(); + } } return GGML_STATUS_SUCCESS; } +static ov::AnyMap without_npuw(const ov::AnyMap & config) { + ov::AnyMap out; + for (const auto & kv : config) { + if (kv.first.rfind("NPUW", 0) == 0 || kv.first == "NPU_USE_NPUW") { + continue; + } + out.insert(kv); + } + return out; +} + enum ggml_status ov_graph_compute_static(ggml_cgraph * cgraph, std::shared_ptr r_ctx) { auto & core = ov_singleton_core(); @@ -606,7 +824,7 @@ enum ggml_status ov_graph_compute_static(ggml_cgraph * cgraph, std::shared_ptrcompiled_cache); } auto start_time = ggml_time_us(); @@ -618,7 +836,12 @@ enum ggml_status ov_graph_compute_static(ggml_cgraph * cgraph, std::shared_ptrne[0]; + } graph_key key(cgraph); static const bool cache_enabled = !ggml_openvino_getenv_int("GGML_OPENVINO_DISABLE_CACHE"); bool cache_hit = false; @@ -652,6 +875,8 @@ enum ggml_status ov_graph_compute_static(ggml_cgraph * cgraph, std::shared_ptr lock(*(entry->mutex)); + cache_hit = cache_hit && entry->ptr && r_ctx->infer_request_cache.count(key) != 0 && + r_ctx->infer_request_cache_prefill.count(key) != 0; if (cache_hit) { ggml_decoder = entry->ptr; @@ -689,88 +914,122 @@ enum ggml_status ov_graph_compute_static(ggml_cgraph * cgraph, std::shared_ptrinfer_request_cache_prefill.erase(key); } - std::shared_ptr model; - auto model_weights = GgmlOvDecoder::create_weight_nodes(cgraph); - - if (m_params.n_heads_kv == -1) { - // graph is not a LLM, e.g. context-shift graph - prefill_chunk_size = inp_pos->ne[0]; - } - auto ggml_decoder_prefill = std::make_shared( - cgraph, m_params, c_params, model_weights, is_static, stateful, false, true, prefill_chunk_size); - auto ggml_decoder_decode = std::make_shared(cgraph, m_params, c_params, model_weights, is_static, - stateful, false, false, prefill_chunk_size); - decoder_end_time = ggml_time_us(); - - const bool dump_ir = ggml_openvino_getenv_int("GGML_OPENVINO_DUMP_IR"); - const auto dump_ir_timestamp = static_cast(ggml_time_us()); - - auto build_static_model = [&core, &config, dump_ir, dump_ir_timestamp]( - std::shared_ptr decoder, - const char * tag, - std::shared_ptr & model, - ov::CompiledModel & compiled_model, - std::shared_ptr & infer_request, - int64_t & local_conversion_end_time, - int64_t & local_compile_end_time) { - auto input_model = std::make_shared(decoder); - model = ov::frontend::ggml::FrontEnd::convert(input_model); - decoder->clear_model_weights(); - local_conversion_end_time = ggml_time_us(); - - if (dump_ir) { - char timestamped_filename[64]; - snprintf(timestamped_filename, sizeof(timestamped_filename), "model_%s_%lld.xml", tag, - dump_ir_timestamp); - ov::serialize(model, timestamped_filename); - } - - compiled_model = core.compile_model(model, device, config); - infer_request = std::make_shared(compiled_model.create_infer_request()); - local_compile_end_time = ggml_time_us(); - }; - std::shared_ptr model_prefill; - std::shared_ptr model_decode; - ov::CompiledModel compiled_model_prefill; - ov::CompiledModel compiled_model_decode; - std::shared_ptr infer_request_prefill; - std::shared_ptr infer_request_decode; - int64_t prefill_conversion_end_time; - int64_t decode_conversion_end_time; - int64_t prefill_compile_end_time; - int64_t decode_compile_end_time; - auto prefill_future = std::async(std::launch::async, build_static_model, ggml_decoder_prefill, "prefill", - std::ref(model_prefill), std::ref(compiled_model_prefill), - std::ref(infer_request_prefill), std::ref(prefill_conversion_end_time), - std::ref(prefill_compile_end_time)); - auto decode_future = std::async(std::launch::async, build_static_model, ggml_decoder_decode, "decode", - std::ref(model_decode), std::ref(compiled_model_decode), - std::ref(infer_request_decode), std::ref(decode_conversion_end_time), - std::ref(decode_compile_end_time)); - prefill_future.get(); - decode_future.get(); - conversion_end_time = std::max(prefill_conversion_end_time, decode_conversion_end_time); - compile_end_time = std::max(prefill_compile_end_time, decode_compile_end_time); - - model = is_prefill ? model_prefill : model_decode; - ggml_decoder = is_prefill ? ggml_decoder_prefill : ggml_decoder_decode; - infer_request = is_prefill ? infer_request_prefill : infer_request_decode; - entry->ptr = ggml_decoder; - - for (const auto & ov_param : model->get_parameters()) { - ov_input_names_local.push_back(ov_param->get_friendly_name()); - } - for (const auto & ov_output : model->get_results()) { - ov_output_names_local.push_back(ov_output->get_friendly_name()); - } - - if (cache_enabled) { - std::lock_guard map_lock(r_ctx->ctx_mutex); - r_ctx->infer_request_cache_prefill[key] = infer_request_prefill; - r_ctx->infer_request_cache[key] = infer_request_decode; + // Static execution shares a compiled prefill/decode pair. Each backend + // creates and retains its own requests for both phases. + auto shared_cache = r_ctx->compiled_cache; + std::unique_lock compile_lock(shared_cache->mutex); + auto weight_names = get_weight_names(cgraph); + auto local_decoder = std::make_shared( + cgraph, m_params, c_params, weight_names, is_static, stateful, false, is_prefill, prefill_chunk_size); + const std::string shared_key = cache_enabled ? + compiled_graph_key(cgraph, *local_decoder, device, prefill_chunk_size) : ""; + auto shared_it = shared_cache->graphs.find(shared_key); + if (!shared_key.empty() && shared_it != shared_cache->graphs.end()) { + auto & compiled = shared_it->second; + auto prefill_request = std::make_shared(compiled.prefill.create_infer_request()); + auto decode_request = no_kv_cache ? prefill_request : + std::make_shared(compiled.decode.create_infer_request()); + ggml_decoder = local_decoder; + entry->ptr = ggml_decoder; + infer_request = is_prefill ? prefill_request : decode_request; + ov_input_names_local = compiled.input_names; + ov_output_names_local = compiled.output_names; + r_ctx->infer_request_cache_prefill[key] = prefill_request; + r_ctx->infer_request_cache[key] = decode_request; r_ctx->ov_input_names_cache[key] = ov_input_names_local; r_ctx->ov_output_names_cache[key] = ov_output_names_local; + decoder_end_time = conversion_end_time = compile_end_time = ggml_time_us(); + GGML_LOG_DEBUG("ggml-openvino: shared compiled model HIT (static)\n"); + } else { + std::shared_ptr model; + auto model_weights = GgmlOvDecoder::create_weight_nodes(cgraph); + + auto ggml_decoder_prefill = std::make_shared( + cgraph, m_params, c_params, model_weights, is_static, stateful, false, true, prefill_chunk_size); + auto ggml_decoder_decode = + no_kv_cache ? ggml_decoder_prefill : + std::make_shared(cgraph, m_params, c_params, model_weights, is_static, + stateful, false, false, prefill_chunk_size); + decoder_end_time = ggml_time_us(); + + const bool dump_ir = ggml_openvino_getenv_int("GGML_OPENVINO_DUMP_IR"); + const auto dump_ir_timestamp = static_cast(ggml_time_us()); + + auto build_static_model = [&core, &compile_config, dump_ir, dump_ir_timestamp]( + std::shared_ptr decoder, + const char * tag, + std::shared_ptr & model, + ov::CompiledModel & compiled_model, + std::shared_ptr & infer_request, + int64_t & local_conversion_end_time, + int64_t & local_compile_end_time) { + auto input_model = std::make_shared(decoder); + model = ov::frontend::ggml::FrontEnd::convert(input_model); + decoder->clear_model_weights(); + local_conversion_end_time = ggml_time_us(); + + if (dump_ir) { + char timestamped_filename[64]; + snprintf(timestamped_filename, sizeof(timestamped_filename), "model_%s_%lld.xml", tag, + dump_ir_timestamp); + ov::serialize(model, timestamped_filename); + } + + compiled_model = core.compile_model(model, device, compile_config); + infer_request = std::make_shared(compiled_model.create_infer_request()); + local_compile_end_time = ggml_time_us(); + }; + std::shared_ptr model_prefill; + std::shared_ptr model_decode; + ov::CompiledModel compiled_model_prefill; + ov::CompiledModel compiled_model_decode; + std::shared_ptr infer_request_prefill; + std::shared_ptr infer_request_decode; + int64_t prefill_conversion_end_time; + int64_t decode_conversion_end_time; + int64_t prefill_compile_end_time; + int64_t decode_compile_end_time; + build_static_model(ggml_decoder_prefill, "prefill", model_prefill, compiled_model_prefill, + infer_request_prefill, prefill_conversion_end_time, prefill_compile_end_time); + if (no_kv_cache) { + model_decode = model_prefill; + compiled_model_decode = compiled_model_prefill; + infer_request_decode = infer_request_prefill; + decode_conversion_end_time = prefill_conversion_end_time; + decode_compile_end_time = prefill_compile_end_time; + } else { + build_static_model(ggml_decoder_decode, "decode", model_decode, compiled_model_decode, infer_request_decode, + decode_conversion_end_time, decode_compile_end_time); + } + conversion_end_time = std::max(prefill_conversion_end_time, decode_conversion_end_time); + compile_end_time = std::max(prefill_compile_end_time, decode_compile_end_time); + + model = is_prefill ? model_prefill : model_decode; + ggml_decoder = is_prefill ? ggml_decoder_prefill : ggml_decoder_decode; + infer_request = is_prefill ? infer_request_prefill : infer_request_decode; + entry->ptr = ggml_decoder; + + for (const auto & ov_param : model->get_parameters()) { + ov_input_names_local.push_back(ov_param->get_friendly_name()); + } + for (const auto & ov_output : model->get_results()) { + ov_output_names_local.push_back(ov_output->get_friendly_name()); + } + + if (!shared_key.empty()) { + shared_cache->graphs.emplace(shared_key, ov_compiled_graph{compiled_model_decode, compiled_model_prefill, + ov_input_names_local, ov_output_names_local}); + } + + if (cache_enabled) { + std::lock_guard map_lock(r_ctx->ctx_mutex); + r_ctx->infer_request_cache_prefill[key] = infer_request_prefill; + r_ctx->infer_request_cache[key] = infer_request_decode; + r_ctx->ov_input_names_cache[key] = ov_input_names_local; + r_ctx->ov_output_names_cache[key] = ov_output_names_local; + } } + } if (is_prefill) { @@ -961,11 +1220,13 @@ bool is_naive(ggml_cgraph * cgraph) { enum ggml_status naive_compute(ggml_cgraph * cgraph, ov::Core & core, const std::string & device, - const ov::AnyMap & config) { + const ov::AnyMap & config, + ov_compiled_model_cache & cache) { if (cgraph->n_nodes == 1 && (cgraph->nodes[0]->op == GGML_OP_NONE || cgraph->nodes[0]->op == GGML_OP_VIEW)) { return GGML_STATUS_SUCCESS; } + std::unique_lock compile_lock(cache.mutex); bool naive = true; auto model_weights = GgmlOvDecoder::create_weight_nodes(cgraph, naive); auto decoder = std::make_shared(cgraph, model_weights); @@ -977,23 +1238,38 @@ enum ggml_status naive_compute(ggml_cgraph * cgraph, std::shared_ptr infer_request; auto remote_context = ggml_openvino_get_remote_context(); + ov::AnyMap compile_config = config; if (cgraph->nodes[0]->op == GGML_OP_MUL_MAT) { // TODO ACCURACY hint triggers a bug in GPU plugin/driver on Lunar Lake. Remove once CVS-182166 is resolved - core.set_property(device, ov::hint::execution_mode(ov::hint::ExecutionMode::PERFORMANCE)); + compile_config[ov::hint::execution_mode.name()] = ov::hint::ExecutionMode::PERFORMANCE; } else { - core.set_property(device, ov::hint::execution_mode(ov::hint::ExecutionMode::ACCURACY)); + compile_config[ov::hint::execution_mode.name()] = ov::hint::ExecutionMode::ACCURACY; } if (remote_context.has_value()) { infer_request = std::make_shared( - core.compile_model(model, remote_context.value(), config).create_infer_request()); + core.compile_model(model, remote_context.value(), compile_config).create_infer_request()); } else { infer_request = - std::make_shared(core.compile_model(model, device, config).create_infer_request()); + std::make_shared(core.compile_model(model, device, compile_config).create_infer_request()); } + std::vector input_names; + std::vector output_names; + for (const auto & param : model->get_parameters()) { + input_names.push_back(param->get_friendly_name()); + } + for (const auto & result : model->get_results()) { + output_names.push_back(result->get_friendly_name()); + } + // Destroy the frontend graph under the compilation lock as well: it can + // still own edges into the shared weight nodes. + model.reset(); + input_model.reset(); + decoder->clear_model_weights(); + model_weights.clear(); + compile_lock.unlock(); - auto ov_params = model->get_parameters(); - for (size_t i = 0; i < ov_params.size(); i++) { - auto param_name = ov_params[i]->get_friendly_name(); + for (size_t i = 0; i < input_names.size(); i++) { + const auto & param_name = input_names[i]; auto input_tensor = get_ov_input_tensor(decoder, param_name); infer_request->set_input_tensor(i, input_tensor); } @@ -1003,16 +1279,15 @@ enum ggml_status naive_compute(ggml_cgraph * cgraph, infer_request->infer(); - auto ov_results = model->get_results(); - for (size_t i = 0; i < ov_results.size(); i++) { + for (size_t i = 0; i < output_names.size(); i++) { auto output_tensor = infer_request->get_output_tensor(i); const auto & model_outputs = decoder->get_model_outputs(); - auto model_output_it = model_outputs.find(ov_results[i]->get_friendly_name()); + auto model_output_it = model_outputs.find(output_names[i]); if (model_output_it == model_outputs.end()) { // Debug-only output added via GGML_OPENVINO_DEBUG_NODE; nothing to copy into. if (ggml_openvino_getenv_int("GGML_OPENVINO_DEBUG_OUTPUT") || ggml_openvino_getenv_str("GGML_OPENVINO_DEBUG_NODE")) { - print_output_tensor_info(ov_results[i]->get_friendly_name(), output_tensor, output_tensor.data()); + print_output_tensor_info(output_names[i], output_tensor, output_tensor.data()); } continue; } @@ -1262,6 +1537,20 @@ ov::Tensor get_ov_input_tensor_static_prefill(std::shared_ptr ggm return input_tensor; } + if (GgmlOvDecoder::is_inp_mean(ggml_tensor, op)) { + const size_t n_seqs = ggml_tensor->ne[1]; + const size_t src_stride = ggml_tensor->ne[0]; + const size_t copy_len = std::min(chunk_valid_size, src_stride - chunk_index * chunk_size); + ov::Tensor input_tensor(ov::element::f32, ov::Shape{1, 1, n_seqs, chunk_size}); + auto * dst = input_tensor.data(); + std::fill(dst, dst + n_seqs * chunk_size, 0.0f); + const auto * src = static_cast(ggml_tensor->data) + chunk_index * chunk_size; + for (size_t s = 0; s < n_seqs; s++) { + std::memcpy(dst + s * chunk_size, src + s * src_stride, copy_len * sizeof(float)); + } + return input_tensor; + } + if (GgmlOvDecoder::is_inp_mask(ggml_tensor, op)) { size_t cols = ggml_tensor->ne[0]; size_t rows = ggml_tensor->ne[1]; diff --git a/ggml/src/ggml-openvino/utils.h b/ggml/src/ggml-openvino/utils.h index 5aa74da38..235b15d7e 100644 --- a/ggml/src/ggml-openvino/utils.h +++ b/ggml/src/ggml-openvino/utils.h @@ -2,7 +2,6 @@ #include "ggml-impl.h" #include -#include #include #include #include @@ -14,6 +13,8 @@ #include #include +// Local execution-cache key. A match still needs the ModelParams compatibility +// check; this key alone does not identify weights or a compiled model. struct graph_key { int n_nodes; std::string first_node_name; @@ -26,14 +27,13 @@ struct graph_key { last_node_name = cgraph->nodes[n_nodes - 1]->name; } - auto get_input_key_name = [](const ggml_cgraph * graph, const ggml_tensor * tensor) { - std::string name = tensor->name; - const size_t hash_pos = ggml_hash_find(&graph->visited_hash_set, tensor); - if (((tensor->flags & GGML_TENSOR_FLAG_COMPUTE) || GgmlOvDecoder::is_kvcache(tensor, nullptr)) && - hash_pos != GGML_HASHSET_FULL && ggml_bitset_get(graph->visited_hash_set.used, hash_pos)) { - name += "#" + std::to_string(hash_pos); + std::unordered_map names; + auto get_input_key_name = [&names](const ggml_cgraph * graph, const ggml_tensor * tensor) { + auto it = names.find(tensor); + if (it == names.end()) { + it = names.emplace(tensor, GgmlOvDecoder::get_tensor_name(graph, tensor)).first; } - return name; + return it->second; }; std::vector node_names; @@ -90,7 +90,27 @@ struct decoder_runtime_ctx { std::shared_ptr ptr; }; +struct ov_compiled_graph { + ov::CompiledModel decode; + ov::CompiledModel prefill; + std::vector input_names; + std::vector output_names; +}; + +// Only compilation and cache publication use this mutex. Requests, decoders and +// sequence state belong to individual backend contexts and never enter this cache. +struct ov_compiled_model_cache { + std::mutex mutex; + std::unordered_map graphs; + size_t backend_count = 0; +}; + +// Private to one backend instance. Only compiled_cache is shared with other +// instances; clearing these local caches cannot invalidate their requests. struct ov_runtime_context { + // Serializes calls on this backend only, not inference in other contexts. + std::mutex execution_mutex; + std::shared_ptr compiled_cache; mutable std::mutex ctx_mutex; std::string device; bool stateful; @@ -99,13 +119,10 @@ struct ov_runtime_context { std::unordered_map, graph_key_hash> infer_request_cache_prefill; std::unordered_map, graph_key_hash> ov_input_names_cache; std::unordered_map, graph_key_hash> ov_output_names_cache; - //TODO: Stateful is only supported for single request at a time. - // Simultanous stateful inference request support to be added. size_t stateful_kv_size; std::map kv_state_input_name_map; - std::atomic backend_count; - ov_runtime_context() : device("CPU"), stateful(false), stateful_kv_size(0), backend_count(0) {} + ov_runtime_context() : device("CPU"), stateful(false), stateful_kv_size(0) {} void clear_caches_locked() { decoder_cache.clear(); @@ -192,4 +209,5 @@ bool is_model_splitted(struct ggml_cgraph * cgraph); enum ggml_status naive_compute(struct ggml_cgraph * cgraph, ov::Core & core, const std::string & device, - const ov::AnyMap & config); + const ov::AnyMap & config, + ov_compiled_model_cache & cache); From fc82583e65ad753710fbd69a9244d9a35dca667a Mon Sep 17 00:00:00 2001 From: Ruben Ortlam Date: Tue, 15 Sep 2026 11:30:27 +0200 Subject: [PATCH 14/23] vulkan: support sparse Flash Attention (#28105) * vulkan: add sparse Flash Attention support for DSV4/GLM * tune implementation * add tests * avoid nondeterministic atomicAdd * add cm2 decode vector support * simplify logic and make variable names more consistent * add cm2 f16vec4 binding for decode vector --- ggml/src/ggml-vulkan/ggml-vulkan.cpp | 142 +++++++++++++++--- .../vulkan-shaders/flash_attn.comp | 46 +++--- .../vulkan-shaders/flash_attn_base.glsl | 33 +++- .../vulkan-shaders/flash_attn_cm1.comp | 48 ++++-- .../vulkan-shaders/flash_attn_cm2.comp | 95 +++++++++++- .../flash_attn_sparse_compact.comp | 102 +++++++++++++ .../vulkan-shaders/vulkan-shaders-gen.cpp | 2 + tests/test-backend-ops.cpp | 11 ++ 8 files changed, 412 insertions(+), 67 deletions(-) create mode 100644 ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_sparse_compact.comp diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vulkan.cpp index 0dfa44dbf..f936127a6 100644 --- a/ggml/src/ggml-vulkan/ggml-vulkan.cpp +++ b/ggml/src/ggml-vulkan/ggml-vulkan.cpp @@ -1171,6 +1171,10 @@ struct vk_device_struct { std::map, vk_pipeline> pipeline_fa_mask_opt; + vk_pipeline pipeline_fa_sparse_compact; + vk_pipeline pipeline_fa_sparse_compact_subgroup; + bool fa_sparse_compact_use_subgroups; + vk_pipeline pipeline_flash_attn_split_k_reduce; vk_pipeline pipeline_count_experts; @@ -2196,6 +2200,16 @@ struct vk_op_flash_attn_mask_opt_push_constants { uint32_t nbd3; }; +struct vk_op_flash_attn_sparse_compact_push_constants { + uint32_t KV; + uint32_t nem1; + uint32_t nem2; + uint32_t nbm1; + uint32_t nbm2; + uint32_t nbm3; + uint32_t n_kv_max; +}; + // Allow pre-recording command buffers struct vk_staging_memcpy { vk_staging_memcpy(void * _dst, const void * _src, size_t _n) : dst(_dst), src(_src), n(_n) {} @@ -4119,14 +4133,15 @@ static vk_fa_tuning_params get_fa_tuning_params(const vk_device& device, uint32_ } static vk_fa_pipeline_state get_fa_pipeline_state(const vk_device& device, const vk_fa_tuning_params& params, uint32_t hsk, uint32_t hsv, bool aligned, bool f32acc, - bool use_mask, bool use_mask_opt, bool use_logit_softcap, ggml_type k_type, ggml_type v_type) { + bool use_mask, bool use_mask_opt, bool use_logit_softcap, bool use_sparse, ggml_type k_type, ggml_type v_type) { const bool old_amd_windows = device->vendor_id == VK_VENDOR_ID_AMD && device->driver_id == vk::DriverId::eAmdProprietary && (device->architecture == AMD_GCN || device->architecture == AMD_RDNA1 || device->architecture == AMD_RDNA2); uint32_t flags = (use_mask_opt ? 1 : 0) | (use_mask ? 2 : 0) | (use_logit_softcap ? 4 : 0) | - (old_amd_windows ? 8 : 0); + (old_amd_windows ? 8 : 0) | + (use_sparse ? 16 : 0); const uint32_t subgroup_size = params.disable_subgroups ? 0 : params.subgroup_size; @@ -4746,7 +4761,7 @@ static void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) { } name = aligned ? "flash_attn_f32_f16_aligned" : "flash_attn_f32_f16"; } - ggml_vk_create_pipeline(device, fa.second, name, spv_size, spv_data, "main", 7, + ggml_vk_create_pipeline(device, fa.second, name, spv_size, spv_data, "main", 8, sizeof(vk_flash_attn_push_constants), {Br, 1, 1}, get_fa_spec_constants(fa.first), aligned ? Bc : 1, true, !fa_ds, !fa_ds ? fa_sgs : 0); @@ -4782,7 +4797,7 @@ static void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) { else { spv_data = flash_attn_f32_f16_f16acc_cm1_data; spv_size = flash_attn_f32_f16_f16acc_cm1_len; } name = aligned ? "flash_attn_f32_f16_aligned_cm1" : "flash_attn_f32_f16_cm1"; } - ggml_vk_create_pipeline(device, fa.second, name, spv_size, spv_data, "main", 7, + ggml_vk_create_pipeline(device, fa.second, name, spv_size, spv_data, "main", 8, sizeof(vk_flash_attn_push_constants), {Br, 1, 1}, get_fa_spec_constants(fa.first), aligned ? Bc : 1, true, !fa_ds, !fa_ds ? fa_sgs : 0); @@ -4819,7 +4834,7 @@ static void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) { if (f32acc) { spv_data = flash_attn_f32_f16_cm2_data; spv_size = flash_attn_f32_f16_cm2_len; name = "flash_attn_f32_f16_f32acc_cm2"; } else { spv_data = flash_attn_f32_f16_f16acc_cm2_data; spv_size = flash_attn_f32_f16_f16acc_cm2_len; name = "flash_attn_f32_f16_f16acc_cm2"; } } - ggml_vk_create_pipeline(device, fa.second, name, spv_size, spv_data, "main", 7, + ggml_vk_create_pipeline(device, fa.second, name, spv_size, spv_data, "main", 8, sizeof(vk_flash_attn_push_constants), {Br, 1, 1}, get_fa_spec_constants(fa.first), aligned ? Bc : 1, true, false, 0); } @@ -5783,6 +5798,22 @@ static void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) { ggml_vk_create_pipeline(device, it.second, "fa_mask_opt", fa_mask_opt_len, fa_mask_opt_data, "main", 2, sizeof(vk_op_flash_attn_mask_opt_push_constants), {1, 1, 1}, {128, 128 / device->subgroup_size, BrBc.first, BrBc.second}, 1, true, true, device->subgroup_size); } + { + // Large workgroup so the per-row KV scan parallelizes; capped to device limits. + const uint32_t compact_max = std::min({1024u, device->properties.limits.maxComputeWorkGroupInvocations, device->properties.limits.maxComputeWorkGroupSize[0]}); + + // Fast ballot prefix-sum path when the device supports full subgroups; otherwise + // a shared-memory prefix-sum fallback. Both emit a deterministic ascending list. + device->fa_sparse_compact_use_subgroups = device->subgroup_ballot && device->subgroup_require_full_support; + if (device->fa_sparse_compact_use_subgroups) { + const uint32_t compact_wg = std::max(device->subgroup_size, (compact_max / device->subgroup_size) * device->subgroup_size); + const uint32_t compact_num_sg = compact_wg / device->subgroup_size; + ggml_vk_create_pipeline(device, device->pipeline_fa_sparse_compact_subgroup, "fa_sparse_compact_subgroup", fa_sparse_compact_subgroup_len, fa_sparse_compact_subgroup_data, "main", 2, sizeof(vk_op_flash_attn_sparse_compact_push_constants), {1, 1, 1}, {compact_wg, compact_num_sg}, 1, true, true, device->subgroup_size); + } else { + ggml_vk_create_pipeline(device, device->pipeline_fa_sparse_compact, "fa_sparse_compact", fa_sparse_compact_len, fa_sparse_compact_data, "main", 2, sizeof(vk_op_flash_attn_sparse_compact_push_constants), {1, 1, 1}, {compact_max}, 1, true); + } + } + if (device->subgroup_clustered && device->subgroup_require_full_support) { ggml_vk_create_pipeline(device, device->pipeline_quantize_q8_1_x4, "quantize_q8_1_x4", quantize_q8_1_x4_subgroup_len, quantize_q8_1_x4_subgroup_data, "main", 2, sizeof(vk_quantize_q8_1_push_constants), {32 * device->subgroup_size / 8, 1, 1}, { device->subgroup_size }, 1, true, true); } else { @@ -11276,6 +11307,30 @@ static void ggml_vk_flash_attn(ggml_backend_vk_context * ctx, vk_context& subctx tuning_params = get_fa_tuning_params(ctx->device, HSK, HSV, N, KV, k_type_eff, v_type_eff, f32acc); + float scale = 1.0f; + float max_bias = 0.0f; + float logit_softcap = 0.0f; + + memcpy(&scale, (const float *) dst->op_params + 0, sizeof(float)); + memcpy(&max_bias, (const float *) dst->op_params + 1, sizeof(float)); + memcpy(&logit_softcap, (const float *) dst->op_params + 2, sizeof(float)); + + if (logit_softcap != 0) { + scale /= logit_softcap; + } + + // Sparse mask hint (op_params[4]): compact the <= n_kv_max finite positions and gather only those. + const int32_t n_kv_max = mask ? ggml_get_op_params_i32(dst, 4) : 0; + static const bool disable_sparse = getenv("GGML_VK_FA_SPARSE_DISABLE") != nullptr; + // cm2 dense is fast, so it needs a larger reduction to win. + const int64_t min_ratio = tuning_params.path == FA_COOPMAT2 ? 4 : 2; + const bool use_sparse = !disable_sparse && n_kv_max > 0 && mask && + max_bias == 0.0f && logit_softcap == 0.0f && + k_type_eff == GGML_TYPE_F16 && v_type_eff == GGML_TYPE_F16 && + nem0 == KV && + (int64_t)KV >= std::max(4096, min_ratio * (int64_t)n_kv_max) && + (gqa_ratio > 1 || (tuning_params.path == FA_SCALAR && N == 1)); + const uint32_t q_stride = (uint32_t)(nbq1 / ggml_type_size(q->type)); uint32_t k_stride = (uint32_t)(nbk1 / ggml_type_size(k->type)); uint32_t v_stride = (uint32_t)(nbv1 / ggml_type_size(v->type)); @@ -11298,7 +11353,6 @@ static void ggml_vk_flash_attn(ggml_backend_vk_context * ctx, vk_context& subctx nbv2_eff = (uint32_t)((uint64_t)HSV * KV * sizeof(ggml_fp16_t)); nbv3_eff = (uint32_t)((uint64_t)HSV * KV * nev2 * sizeof(ggml_fp16_t)); } - const uint32_t alignment = tuning_params.block_cols; bool aligned = (KV % alignment) == 0 && // the "aligned" shader variant will forcibly align strides, for performance @@ -11309,23 +11363,11 @@ static void ggml_vk_flash_attn(ggml_backend_vk_context * ctx, vk_context& subctx aligned = false; } - float scale = 1.0f; - float max_bias = 0.0f; - float logit_softcap = 0.0f; - - memcpy(&scale, (const float *) dst->op_params + 0, sizeof(float)); - memcpy(&max_bias, (const float *) dst->op_params + 1, sizeof(float)); - memcpy(&logit_softcap, (const float *) dst->op_params + 2, sizeof(float)); - - if (logit_softcap != 0) { - scale /= logit_softcap; - } - // Only use mask opt when the mask is fairly large. This hasn't been tuned extensively. - bool use_mask_opt = mask && nem1 >= 32 && nem0 * nem1 > 32768 && nem0 >= tuning_params.block_cols * 16 + bool use_mask_opt = mask && !use_sparse && nem1 >= 32 && nem0 * nem1 > 32768 && nem0 >= tuning_params.block_cols * 16 && (ctx->device->architecture != vk_device_architecture::AMD_GCN || HSK > 256 || HSV > 256); vk_fa_pipeline_state fa_pipeline_state = get_fa_pipeline_state(ctx->device, tuning_params, HSK, HSV, aligned, f32acc, - mask != nullptr, use_mask_opt, logit_softcap != 0, k_type_eff, v_type_eff); + mask != nullptr, use_mask_opt, logit_softcap != 0, use_sparse, k_type_eff, v_type_eff); vk_pipeline pipeline = nullptr; @@ -11360,7 +11402,19 @@ static void ggml_vk_flash_attn(ggml_backend_vk_context * ctx, vk_context& subctx const uint32_t Tr = CEIL_DIV(N, Br); // Try to use split_k when KV is large enough to be worth the overhead. - if (gqa_ratio > 1 && workgroups_x <= Br) { + // Sparse: split_kv carries n_kv_max, split_k partitions its blocks for occupancy. + if (use_sparse) { + split_kv = (uint32_t)n_kv_max; + const uint32_t total_blocks = CEIL_DIV((uint32_t)n_kv_max, Bc); + const uint32_t base_wgs = (gqa_ratio > 1 ? workgroups_x : Tr) * workgroups_y * workgroups_z; + if (base_wgs < shader_core_count * 2) { + split_k = shader_core_count * 2 / base_wgs; + } + split_k = std::max(1u, std::min(split_k, total_blocks)); + // Match the shader's per-split block count so no split is empty. + const uint32_t per_blocks = CEIL_DIV(total_blocks, split_k); + split_k = CEIL_DIV(total_blocks, per_blocks); + } else if (gqa_ratio > 1 && workgroups_x <= Br) { split_k = shader_core_count * 2 / (workgroups_x * workgroups_y * workgroups_z); } else if (gqa_ratio <= 1) { uint32_t total_wgs_no_split = Tr * workgroups_y * workgroups_z; @@ -11369,7 +11423,7 @@ static void ggml_vk_flash_attn(ggml_backend_vk_context * ctx, vk_context& subctx } } - if (split_k > 1) { + if (!use_sparse && split_k > 1) { // Try to evenly split KV into split_k chunks, but it needs to be a multiple // of "align", so recompute split_k based on that. split_kv = ROUNDUP_POW2(std::max(1u, KV / split_k), alignment); @@ -11416,6 +11470,24 @@ static void ggml_vk_flash_attn(ggml_backend_vk_context * ctx, vk_context& subctx } } + // Sparse index scratch reuses prealloc_y (mutually exclusive with mask opt). + const uint64_t sparse_idx_size = use_sparse + ? sizeof(int32_t) * (uint64_t)n_kv_max * nem1 * nem2 * nem3 + : 0; + vk_pipeline sparse_compact_pipeline = ctx->device->fa_sparse_compact_use_subgroups + ? ctx->device->pipeline_fa_sparse_compact_subgroup + : ctx->device->pipeline_fa_sparse_compact; + if (use_sparse) { + ggml_pipeline_request_descriptor_sets(ctx, sparse_compact_pipeline, 1); + if (ctx->prealloc_size_y < sparse_idx_size) { + ctx->prealloc_size_y = sparse_idx_size; + ggml_vk_preallocate_buffers(ctx, subctx); + } + if (ctx->prealloc_y_need_sync) { + ggml_vk_sync_buffers(ctx, subctx); + } + } + const uint32_t n_head_kv = neq2; const uint32_t n_head_log2 = 1u << (uint32_t) floorf(log2f((float) n_head_kv)); const float m0 = powf(2.0f, -(max_bias ) / n_head_log2); @@ -11428,6 +11500,7 @@ static void ggml_vk_flash_attn(ggml_backend_vk_context * ctx, vk_context& subctx vk_subbuffer mask_buf = mask ? ggml_vk_tensor_subbuffer(ctx, mask) : q_buf; vk_subbuffer sinks_buf = sinks ? ggml_vk_tensor_subbuffer(ctx, sinks) : q_buf; vk_subbuffer mask_opt_buf = use_mask_opt ? ggml_vk_subbuffer(ctx, ctx->prealloc_y, 0) : q_buf; + vk_subbuffer sparse_buf = use_sparse ? ggml_vk_subbuffer(ctx, ctx->prealloc_y, 0) : q_buf; if (use_dequant_kv) { const uint64_t fp = sizeof(ggml_fp16_t); @@ -11479,6 +11552,24 @@ static void ggml_vk_flash_attn(ggml_backend_vk_context * ctx, vk_context& subctx ggml_vk_sync_buffers(ctx, subctx); } + if (use_sparse) + { + const vk_op_flash_attn_sparse_compact_push_constants sc_pc = { + KV, + nem1, + nem2, + (uint32_t)(mask->nb[1] / sizeof(ggml_fp16_t)), + (uint32_t)(mask->nb[2] / sizeof(ggml_fp16_t)), + (uint32_t)(mask->nb[3] / sizeof(ggml_fp16_t)), + (uint32_t)n_kv_max, + }; + + ggml_vk_dispatch_pipeline(ctx, subctx, sparse_compact_pipeline, + { mask_buf, sparse_buf }, sc_pc, + { nem1, nem2, nem3 }); + ggml_vk_sync_buffers(ctx, subctx); + } + const vk_flash_attn_push_constants pc = { N, KV, (uint32_t)ne1, (uint32_t)ne2, (uint32_t)ne3, (uint32_t)neq2, (uint32_t)neq3, @@ -11511,7 +11602,7 @@ static void ggml_vk_flash_attn(ggml_backend_vk_context * ctx, vk_context& subctx vk_subbuffer split_k_buf = ggml_vk_subbuffer(ctx, ctx->prealloc_split_k, 0); ggml_vk_dispatch_pipeline(ctx, subctx, pipeline, - {q_buf, k_buf, v_buf, mask_buf, sinks_buf, split_k_buf, mask_opt_buf}, + {q_buf, k_buf, v_buf, mask_buf, sinks_buf, split_k_buf, mask_opt_buf, sparse_buf}, pc, { dispatch_x, workgroups_y, workgroups_z }); ggml_vk_sync_buffers(ctx, subctx); @@ -11526,13 +11617,16 @@ static void ggml_vk_flash_attn(ggml_backend_vk_context * ctx, vk_context& subctx workgroups_x *= pipeline->wg_denoms[0]; } ggml_vk_dispatch_pipeline(ctx, subctx, pipeline, - {q_buf, k_buf, v_buf, mask_buf, sinks_buf, dst_buf, mask_opt_buf}, + {q_buf, k_buf, v_buf, mask_buf, sinks_buf, dst_buf, mask_opt_buf, sparse_buf}, pc, { workgroups_x, workgroups_y, workgroups_z }); } if (use_dequant_kv) { ctx->prealloc_x_need_sync = true; } + if (use_mask_opt || use_sparse) { + ctx->prealloc_y_need_sync = true; + } } static vk_conv_shapes ggml_vk_conv_select_shape(ggml_backend_vk_context * ctx, uint32_t K, uint32_t NPQ) { diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn.comp b/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn.comp index 9a12cdfb8..107d44aaa 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn.comp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn.comp @@ -218,12 +218,14 @@ void main() { uint32_t c = (idx + tid) % Bc; uint32_t r = (idx + tid) / Bc; if (idx + tid < Bc * Br) { - if ((!KV_bounds_check || j * Bc + c < KV) && (!nem1_bounds_check || i * Br + r < p.nem1)) { - FLOAT_TYPE m = FLOAT_TYPE(data_m[m_offset + (i * Br + r) * m_stride + (j * Bc + c)]); + uint32_t kcol; + bool kv_active = fa_kv_index(j * Bc + c, kcol); + if (kv_active && (!nem1_bounds_check || i * Br + r < p.nem1)) { + FLOAT_TYPE m = FLOAT_TYPE(data_m[m_offset + (i * Br + r) * m_stride + kcol]); masksh[c * masksh_stride + r] = m; max_mask = max(max_mask, float(m)); } else { - masksh[c * masksh_stride + r] = FLOAT_TYPE(0); + masksh[c * masksh_stride + r] = USE_SPARSE ? FLOAT_TYPE(NEG_FLT_MAX_OVER_2) : FLOAT_TYPE(0); } } } @@ -258,14 +260,15 @@ void main() { uint32_t c = (idx + tid) / (HSK / 4); if (idx + gl_WorkGroupSize.x <= Bc * HSK / 4 || c < Bc) { FLOAT_TYPEV4 K_Tf = FLOAT_TYPEV4(0); - if (!KV_bounds_check || j * Bc + c < KV) { + uint32_t kcol; + if (fa_kv_index(j * Bc + c, kcol)) { if (USE_DECODE_K) { - uint coord = (j * Bc + c) * k_stride * BLOCK_SIZE_K + 4 * d; + uint coord = kcol * k_stride * BLOCK_SIZE_K + 4 * d; uint ib = coord / BLOCK_SIZE_K; uint iqs = (coord % BLOCK_SIZE_K); K_Tf = dequantize4(ib, iqs, k_offset, BINDING_IDX_K); } else { - K_Tf = FLOAT_TYPEV4(data_kv4[k_offset / 4 + (j * Bc + c) * k_stride / 4 + d]); + K_Tf = FLOAT_TYPEV4(data_kv4[k_offset / 4 + kcol * k_stride / 4 + d]); } } @@ -305,7 +308,9 @@ void main() { } [[unroll]] for (uint32_t c = 0; c < cols_per_thread; ++c) { - if (KV_bounds_check && j * Bc + c * cols_per_iter + col_tid >= KV) { + uint32_t kcol; + bool kv_active = fa_kv_index(j * Bc + c * cols_per_iter + col_tid, kcol); + if (!kv_active) { continue; } @@ -313,12 +318,12 @@ void main() { if (SHMEM_STAGING != 0) { K_Tf = kvsh[(c * cols_per_iter + col_tid) * kvsh_stride + (d * D_split + d_tid)]; } else if (USE_DECODE_K) { - uint coord = (j * Bc + c * cols_per_iter + col_tid) * k_stride * BLOCK_SIZE_K + 4 * (d * D_split + d_tid); + uint coord = kcol * k_stride * BLOCK_SIZE_K + 4 * (d * D_split + d_tid); uint ib = coord / BLOCK_SIZE_K; uint iqs = (coord % BLOCK_SIZE_K); K_Tf = dequantize4(ib, iqs, k_offset, BINDING_IDX_K); } else { - K_Tf = FLOAT_TYPEV4(data_kv4[k_offset / 4 + (j * Bc + c * cols_per_iter + col_tid) * k_stride / 4 + d * D_split + d_tid]); + K_Tf = FLOAT_TYPEV4(data_kv4[k_offset / 4 + kcol * k_stride / 4 + d * D_split + d_tid]); } [[unroll]] for (uint32_t r = 0; r < rows_per_thread; ++r) { Sf[r][c] = dot_product(Q_cache[r], K_Tf, Sf[r][c]); @@ -327,7 +332,9 @@ void main() { } } else { [[unroll]] for (uint32_t c = 0; c < cols_per_thread; ++c) { - if (KV_bounds_check && j * Bc + c * cols_per_iter + col_tid >= KV) { + uint32_t kcol; + bool kv_active = fa_kv_index(j * Bc + c * cols_per_iter + col_tid, kcol); + if (!kv_active) { continue; } @@ -336,12 +343,12 @@ void main() { if (SHMEM_STAGING != 0) { K_Tf = kvsh[(c * cols_per_iter + col_tid) * kvsh_stride + (d * D_split + d_tid)]; } else if (USE_DECODE_K) { - uint coord = (j * Bc + c * cols_per_iter + col_tid) * k_stride * BLOCK_SIZE_K + 4 * (d * D_split + d_tid); + uint coord = kcol * k_stride * BLOCK_SIZE_K + 4 * (d * D_split + d_tid); uint ib = coord / BLOCK_SIZE_K; uint iqs = (coord % BLOCK_SIZE_K); K_Tf = dequantize4(ib, iqs, k_offset, BINDING_IDX_K); } else { - K_Tf = FLOAT_TYPEV4(data_kv4[k_offset / 4 + (j * Bc + c * cols_per_iter + col_tid) * k_stride / 4 + d * D_split + d_tid]); + K_Tf = FLOAT_TYPEV4(data_kv4[k_offset / 4 + kcol * k_stride / 4 + d * D_split + d_tid]); } [[unroll]] for (uint32_t r = 0; r < rows_per_thread; ++r) { Sf[r][c] = dot_product(Qf[tile_row(r) * qf_stride + d * D_split + d_tid], K_Tf, Sf[r][c]); @@ -489,14 +496,15 @@ void main() { uint32_t c = (idx + tid) / (HSV / 4); if (idx + gl_WorkGroupSize.x <= Bc * HSV / 4 || c < Bc) { FLOAT_TYPEV4 V_Tf = FLOAT_TYPEV4(0); - if (!KV_bounds_check || j * Bc + c < KV) { + uint32_t vcol; + if (fa_kv_index(j * Bc + c, vcol)) { if (USE_DECODE_V) { - uint coord = (j * Bc + c) * v_stride * BLOCK_SIZE_V + 4 * d; + uint coord = vcol * v_stride * BLOCK_SIZE_V + 4 * d; uint ib = coord / BLOCK_SIZE_V; uint iqs = (coord % BLOCK_SIZE_V); V_Tf = dequantize4(ib, iqs, v_offset, BINDING_IDX_V); } else { - V_Tf = FLOAT_TYPEV4(data_vv4[v_offset / 4 + (j * Bc + c) * v_stride / 4 + d]); + V_Tf = FLOAT_TYPEV4(data_vv4[v_offset / 4 + vcol * v_stride / 4 + d]); } } @@ -507,7 +515,9 @@ void main() { } [[unroll]] for (uint32_t c = 0; c < cols_per_thread; ++c) { - if (KV_bounds_check && j * Bc + c * cols_per_iter + col_tid >= KV) { + uint32_t vcol; + bool kv_active = fa_kv_index(j * Bc + c * cols_per_iter + col_tid, vcol); + if (!kv_active) { continue; } @@ -522,12 +532,12 @@ void main() { if (SHMEM_STAGING != 0) { Vf = kvsh[(c * cols_per_iter + col_tid) * kvsh_stride + (d * D_split + d_tid)]; } else if (USE_DECODE_V) { - uint coord = (j * Bc + c * cols_per_iter + col_tid) * v_stride * BLOCK_SIZE_V + 4 * (d * D_split + d_tid); + uint coord = vcol * v_stride * BLOCK_SIZE_V + 4 * (d * D_split + d_tid); uint ib = coord / BLOCK_SIZE_V; uint iqs = (coord % BLOCK_SIZE_V); Vf = dequantize4(ib, iqs, v_offset, BINDING_IDX_V); } else { - Vf = FLOAT_TYPEV4(data_vv4[v_offset / 4 + (j * Bc + c * cols_per_iter + col_tid) * v_stride / 4 + d * D_split + d_tid]); + Vf = FLOAT_TYPEV4(data_vv4[v_offset / 4 + vcol * v_stride / 4 + d * D_split + d_tid]); } [[unroll]] for (uint32_t r = 0; r < rows_per_thread; ++r) { Of[r][d] += FLOAT_TYPEV4(Pf[r] * Vf); diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_base.glsl b/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_base.glsl index a4be1ebf9..2e0e23bc1 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_base.glsl +++ b/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_base.glsl @@ -24,6 +24,8 @@ const bool USE_MASK_OPT = (Flags & 1) != 0; const bool MASK_ENABLE = (Flags & 2) != 0; const bool LOGIT_SOFTCAP = (Flags & 4) != 0; const bool OLD_AMD_WINDOWS = (Flags & 8) != 0; +// Sparse: gather binding-7 indices instead of scanning [0,KV); p.split_kv = n_kv_max. +const bool USE_SPARSE = (Flags & 16) != 0; // Round up head sizes to a multiple of 16, for coopmat1/coopmat2 paths const uint32_t HSK_pad = (HSK + 15) & ~15; @@ -82,6 +84,8 @@ layout (binding = 5) writeonly buffer OV4 {D_TYPEV4 data_ov4[];}; layout (binding = 6) readonly buffer MO {uint32_t data_mask_opt[];}; +layout (binding = 7) readonly buffer SP {int32_t data_sparse[];}; + #define MASK_OPT_ALL_NEG_INF 1 #define MASK_OPT_ALL_ZERO 2 @@ -144,7 +148,7 @@ ACC_TYPE perElemOpGetSink(const in uint32_t r, const in uint32_t c, const in ACC uint32_t i, N, KV, split_k_index, Tr, start_j, end_j, gqa_iq1, iq2, iq3, rk2, rk3, rv2, rv3, ik2, ik3, iv2, iv3, - q_stride, k_stride, v_stride, m_stride; + q_stride, k_stride, v_stride, m_stride, sparse_base; void init_indices() { @@ -208,6 +212,33 @@ void init_indices() // that prevents the compiler from folding the "&" through the select // and breaking the alignment detection. m_stride = (p.gqa_ratio > 1) ? (p.gqa_ratio >> 16) : KV; + + // Sparse: the tile shares one mask row (gqa heads, or Br==1). split_k + // partitions the n_kv_max blocks. + if (USE_SPARSE) { + uint32_t qrow = (p.gqa_ratio > 1) ? gqa_iq1 : (i * Br); + sparse_base = (((iq3 % p.nem3) * p.nem2 + (iq2 % p.nem2)) * p.nem1 + qrow) * p.split_kv; + + uint32_t total_blocks = CEIL_DIV(p.split_kv, Bc); + uint32_t per_blocks = CEIL_DIV(total_blocks, p.k_num); + start_j = min(split_k_index * per_blocks, total_blocks); + end_j = min((split_k_index + 1) * per_blocks, total_blocks); + } +} + +// Resolve a linear KV slot to a real column; false for inactive (sparse padding/-1, or dense OOB). +bool fa_kv_index(uint lin, out uint kv_col) { + if (USE_SPARSE) { + if (lin >= p.split_kv) { + kv_col = 0; + return false; + } + int idx = data_sparse[sparse_base + lin]; + kv_col = idx >= 0 ? uint(idx) : 0; + return idx >= 0; + } + kv_col = lin; + return !KV_bounds_check || lin < KV; } // Bias applied to softmax to stay in fp16 range. diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_cm1.comp b/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_cm1.comp index 057ed739a..aa9dd624b 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_cm1.comp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_cm1.comp @@ -176,9 +176,16 @@ void main() { uint32_t c = (idx + tid) / (Br / 4); uint32_t r = (idx + tid) % (Br / 4); if (idx + tid < Bc * Br / 4 || idx + gl_WorkGroupSize.x <= Bc * Br / 4) { - if ((!KV_bounds_check || j * Bc + c < KV)) { + uint32_t kcol; + bool kv_active = fa_kv_index(j * Bc + c, kcol); + if (kv_active) { f16vec4 m; - if (!nem1_bounds_check || i * Br + r * 4 + 3 < p.nem1) { + if (USE_SPARSE) { + // sparse is gqa-gated (m_stride == 0): all four rows share the value + FLOAT_TYPE mv = FLOAT_TYPE(data_m[m_offset + kcol]); + m = f16vec4(mv); + max_mask = max(max_mask, float(mv)); + } else if (!nem1_bounds_check || i * Br + r * 4 + 3 < p.nem1) { m = f16vec4(data_m[m_offset + (i * Br + r * 4 ) * m_stride + (j * Bc + c)], data_m[m_offset + (i * Br + r * 4 + 1) * m_stride + (j * Bc + c)], data_m[m_offset + (i * Br + r * 4 + 2) * m_stride + (j * Bc + c)], @@ -206,6 +213,8 @@ void main() { m = f16vec4(0.0); } mask_cache[idx / WorkGroupSize] = m; + } else if (USE_SPARSE) { + mask_cache[idx / WorkGroupSize] = f16vec4(NEG_FLT_MAX_OVER_2); } } } @@ -231,17 +240,19 @@ void main() { uint32_t c = (idx + tid) / (HSK_pad / 4); if (idx + gl_WorkGroupSize.x <= Bc * HSK_pad / 4 || c < Bc) { FLOAT_TYPEV4 K_Tf = FLOAT_TYPEV4(0); - if ((!KV_bounds_check || j * Bc + c < KV) && (HSK == HSK_pad || d < HSK / 4)) { + uint32_t kcol; + bool kv_active = fa_kv_index(j * Bc + c, kcol); + if (kv_active && (HSK == HSK_pad || d < HSK / 4)) { #if !defined(BFLOAT16) if (USE_DECODE_K) { - uint coord = (j * Bc + c) * k_stride * BLOCK_SIZE_K + 4 * d; + uint coord = kcol * k_stride * BLOCK_SIZE_K + 4 * d; uint ib = coord / BLOCK_SIZE_K; uint iqs = (coord % BLOCK_SIZE_K); K_Tf = dequantize4(ib, iqs, k_offset, BINDING_IDX_K); } else #endif { - K_Tf = FLOAT_TYPEV4(data_kv4[k_offset / 4 + (j * Bc + c) * k_stride / 4 + d]); + K_Tf = FLOAT_TYPEV4(data_kv4[k_offset / 4 + kcol * k_stride / 4 + d]); } } @@ -266,7 +277,7 @@ void main() { if (SHMEM_STAGING == 0) { // For quants we always need to dequant into kvsh; for f16/bf16 we can load // directly from global memory when alignment / bounds allow it. - const bool stage_k = USE_DECODE_K || KV_bounds_check || d * 16 + 16 > HSK; + const bool stage_k = USE_DECODE_K || KV_bounds_check || USE_SPARSE || d * 16 + 16 > HSK; if (stage_k) { barrier(); [[unroll]] for (uint32_t idx = 0; idx < Bc * MatBr / 4; idx += gl_WorkGroupSize.x) { @@ -274,17 +285,19 @@ void main() { uint32_t row = (idx + tid) / (MatBr / 4); if (idx + tid < Bc * MatBr / 4) { FLOAT_TYPEV4 K_Tf = FLOAT_TYPEV4(0); - if ((!KV_bounds_check || j * Bc + row < KV) && (HSK == HSK_pad || d * 16 + col_vec * 4 < HSK)) { + uint32_t kcol; + bool kv_active = fa_kv_index(j * Bc + row, kcol); + if (kv_active && (HSK == HSK_pad || d * 16 + col_vec * 4 < HSK)) { #if !defined(BFLOAT16) if (USE_DECODE_K) { - uint coord = (j * Bc + row) * k_stride * BLOCK_SIZE_K + d * 16 + col_vec * 4; + uint coord = kcol * k_stride * BLOCK_SIZE_K + d * 16 + col_vec * 4; uint ib = coord / BLOCK_SIZE_K; uint iqs = (coord % BLOCK_SIZE_K); K_Tf = dequantize4(ib, iqs, k_offset, BINDING_IDX_K); } else #endif { - K_Tf = FLOAT_TYPEV4(data_kv4[k_offset / 4 + (j * Bc + row) * k_stride / 4 + d * 16 / 4 + col_vec]); + K_Tf = FLOAT_TYPEV4(data_kv4[k_offset / 4 + kcol * k_stride / 4 + d * 16 / 4 + col_vec]); } } @@ -401,17 +414,19 @@ void main() { uint32_t c = (idx + tid) / (HSV_pad / 4); if (idx + gl_WorkGroupSize.x <= Bc * HSV_pad / 4 || c < Bc) { FLOAT_TYPEV4 V_Tf = FLOAT_TYPEV4(0); - if ((!KV_bounds_check || j * Bc + c < KV) && (HSV == HSV_pad || d < HSV / 4)) { + uint32_t v_row; + bool kv_active = fa_kv_index(j * Bc + c, v_row); + if (kv_active && (HSV == HSV_pad || d < HSV / 4)) { #if !defined(BFLOAT16) if (USE_DECODE_V) { - uint coord = (j * Bc + c) * v_stride * BLOCK_SIZE_V + 4 * d; + uint coord = v_row * v_stride * BLOCK_SIZE_V + 4 * d; uint ib = coord / BLOCK_SIZE_V; uint iqs = (coord % BLOCK_SIZE_V); V_Tf = dequantize4(ib, iqs, v_offset, BINDING_IDX_V); } else #endif { - V_Tf = FLOAT_TYPEV4(data_vv4[v_offset / 4 + (j * Bc + c) * v_stride / 4 + d]); + V_Tf = FLOAT_TYPEV4(data_vv4[v_offset / 4 + v_row * v_stride / 4 + d]); } } @@ -441,21 +456,22 @@ void main() { if (SHMEM_STAGING == 0) { // For quants we always preload via kvsh. For f16/bf16 we only preload when // alignment / bounds force it (otherwise we coopMatLoad direct from data_vv4). - const bool stage_v = USE_DECODE_V || KV_bounds_check; + const bool stage_v = USE_DECODE_V || KV_bounds_check || USE_SPARSE; if (stage_v) { [[unroll]] for (uint32_t i = 0; i < v_loads_per_thread; ++i) { const uint idx = i * gl_WorkGroupSize.x + tid; const uint row = idx / v_cols; const uint col = idx % v_cols; - const uint v_row = j * Bc + row; + uint32_t v_row; + bool kv_active = fa_kv_index(j * Bc + row, v_row); const uint v_col = hsv_tile * MatBc * row_split + col * 4; const uint coord = v_row * v_stride * BLOCK_SIZE_V + v_col; const uint ib = coord / BLOCK_SIZE_V; const uint iqs = coord % BLOCK_SIZE_V; - if (!KV_bounds_check || (v_row < KV && v_col < HSV)) { + if (USE_SPARSE ? (kv_active && v_col < HSV) : (!KV_bounds_check || (v_row < KV && v_col < HSV))) { #if !defined(BFLOAT16) if (USE_DECODE_V) { kvsh[row * vsh_stride + col] = dequantize4(ib, iqs, v_offset, BINDING_IDX_V); @@ -479,7 +495,7 @@ void main() { coopMatLoad(KMat, Psh, bc_chunk * MatBc * psh_stride, psh_stride, gl_CooperativeMatrixLayoutColumnMajor); if (SHMEM_STAGING == 0) { - if (!USE_DECODE_V && !KV_bounds_check) { + if (!USE_DECODE_V && !KV_bounds_check && !USE_SPARSE) { // F16/BF16 values can be loaded directly from global memory const uint v_tile_row = j * Bc + bc_chunk * MatBc; const uint v_tile_offset = v_offset / 4 + v_tile_row * v_stride / 4 + hsv_offset / 4; diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_cm2.comp b/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_cm2.comp index 5a9abe226..c6ed63dd4 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_cm2.comp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_cm2.comp @@ -29,6 +29,12 @@ #include "dequant_funcs_cm2.glsl" #endif +#ifdef GL_NV_cooperative_matrix_decode_vector +#define FA_GATHER_BS 4u +#else +#define FA_GATHER_BS 1u +#endif + // buffer_reference stride = sizeof(struct) = FaBlockBytesK/V. layout(buffer_reference, std430, buffer_reference_align = 1) buffer decodeBufFA_K { uint8_t raw[FaBlockBytesK]; @@ -107,6 +113,67 @@ layout (binding = 1) readonly buffer K {uint8_t data_k[];}; layout (binding = 2) readonly buffer V {uint8_t data_v[];}; layout (binding = 3) readonly buffer M {uint8_t data_m[];}; +// f16 aliases for the sparse gather callbacks. +layout (binding = 1) readonly buffer KF16 {float16_t data_kf16[];}; +layout (binding = 2) readonly buffer VF16 {float16_t data_vf16[];}; +layout (binding = 3) readonly buffer MF16 {float16_t data_mf16[];}; +#ifdef GL_NV_cooperative_matrix_decode_vector +layout (binding = 1) readonly buffer KF16V4 {f16vec4 data_kf16v4[];}; +layout (binding = 2) readonly buffer VF16V4 {f16vec4 data_vf16v4[];}; +#endif + +// K/V/mask f16-element offsets for the current head/batch, set in main(). +uint32_t g_k_off_elem, g_v_off_elem, g_m_off_elem; + +#if !defined(BFLOAT16) +// blockCoords are in block units: KV slot = blockCoords[0], +// head dim = blockCoords[1]*FA_GATHER_BS + coordInBlock[1]. +float16_t faGatherK(const decodeBufFA_K unused, const uint32_t blockCoords[2], const uint32_t coordInBlock[2]) { + if (blockCoords[0] >= p.split_kv) { return float16_t(0); } + const int r = data_sparse[sparse_base + blockCoords[0]]; + return r < 0 ? float16_t(0) : data_kf16[g_k_off_elem + uint(r) * k_stride + blockCoords[1] * FA_GATHER_BS + coordInBlock[1]]; +} + +float16_t faGatherV(const decodeBufFA_V unused, const uint32_t blockCoords[2], const uint32_t coordInBlock[2]) { + if (blockCoords[0] >= p.split_kv) { return float16_t(0); } + const int r = data_sparse[sparse_base + blockCoords[0]]; + return r < 0 ? float16_t(0) : data_vf16[g_v_off_elem + uint(r) * v_stride + blockCoords[1] * FA_GATHER_BS + coordInBlock[1]]; +} + +#ifdef GL_NV_cooperative_matrix_decode_vector +f16vec4 faGatherKVector(const decodeBufFA_K unused, const uint32_t blockCoords[2], const uint32_t coordInBlock[2]) { + if (blockCoords[0] >= p.split_kv) { return f16vec4(0); } + const int r = data_sparse[sparse_base + blockCoords[0]]; + if (r < 0) { return f16vec4(0); } + const uint32_t o = g_k_off_elem + uint(r) * k_stride + blockCoords[1] * FA_GATHER_BS + coordInBlock[1]; + return data_kf16v4[o / 4]; +} + +f16vec4 faGatherVVector(const decodeBufFA_V unused, const uint32_t blockCoords[2], const uint32_t coordInBlock[2]) { + if (blockCoords[0] >= p.split_kv) { return f16vec4(0); } + const int r = data_sparse[sparse_base + blockCoords[0]]; + if (r < 0) { return f16vec4(0); } + const uint32_t o = g_v_off_elem + uint(r) * v_stride + blockCoords[1] * FA_GATHER_BS + coordInBlock[1]; + return data_vf16v4[o / 4]; +} + +#define FAGATHERK , faGatherK, faGatherKVector +#define FAGATHERV , faGatherV, faGatherVVector +#else +#define FAGATHERK , faGatherK +#define FAGATHERV , faGatherV +#endif +#endif + +// Add gathered mask to S (slope==1 since sparse requires max_bias==0). col = slot in block jblk. +ACC_TYPE faAddSparseMask(const uint32_t row, const uint32_t col, const ACC_TYPE elem, const uint32_t jblk) { + const float NEG = uintBitsToFloat(0xFEFFFFFF); + const uint32_t kvslot = jblk * Bc + col; + if (kvslot >= p.split_kv) { return ACC_TYPE(NEG); } + const int r = data_sparse[sparse_base + kvslot]; + return r < 0 ? ACC_TYPE(NEG) : elem + ACC_TYPE(data_mf16[g_m_off_elem + row * m_stride + uint(r)]); +} + ACC_TYPE maxReduce(const in ACC_TYPE x, const in ACC_TYPE y) { return max(x, y); } @@ -185,14 +252,16 @@ void main() { tensorViewNV<2, false, 1, 0> tensorViewTranspose = createTensorViewNV(2, false, 1, 0); - const uint bs_k = fa_block_elems(FaTypeK); - const uint bs_v = fa_block_elems(FaTypeV); + const uint bs_k = USE_SPARSE ? FA_GATHER_BS : fa_block_elems(FaTypeK); + const uint bs_v = USE_SPARSE ? FA_GATHER_BS : fa_block_elems(FaTypeV); tensorLayoutK = setTensorLayoutBlockSizeNV(tensorLayoutK, 1, bs_k); tensorLayoutV = setTensorLayoutBlockSizeNV(tensorLayoutV, 1, bs_v); + // Sparse iterates n_kv_max (in split_kv); the decode callbacks remap each slot. + const uint32_t KV_iter = USE_SPARSE ? p.split_kv : KV; tensorLayoutQ = setTensorLayoutDimensionNV(tensorLayoutQ, N, HSK); - tensorLayoutK = setTensorLayoutDimensionNV(tensorLayoutK, KV, HSK); - tensorLayoutV = setTensorLayoutDimensionNV(tensorLayoutV, KV, HSV); + tensorLayoutK = setTensorLayoutDimensionNV(tensorLayoutK, KV_iter, HSK); + tensorLayoutV = setTensorLayoutDimensionNV(tensorLayoutV, KV_iter, HSV); // hint to the compiler that strides are aligned for the aligned variant of the shader if (Clamp != gl_CooperativeMatrixClampModeConstantNV) @@ -250,6 +319,10 @@ void main() { mo_offset += ((iq3 % p.nem3) * p.nem2 + (iq2 % p.nem2)) * CEIL_DIV(p.nem1, Br) * mo_stride; } + g_k_off_elem = (ik2*p.nb12 + ik3*p.nb13) / 2; + g_v_off_elem = (iv2*p.nb22 + iv3*p.nb23) / 2; + g_m_off_elem = m_offset / 2; + uint32_t mask_opt = 0; uint32_t mask_opt_idx = ~0; @@ -257,7 +330,7 @@ void main() { for (uint32_t j = start_j; j < end_j; ++j) { coopmat mv = coopmat(0); - if (MASK_ENABLE) { + if (MASK_ENABLE && !USE_SPARSE) { if (USE_MASK_OPT && mask_opt_idx != j / 16) { mask_opt_idx = j / 16; @@ -315,7 +388,9 @@ void main() { coopMatLoadTensorNV(K_T, data_k, k_offset, sliceTensorLayoutNV(tensorLayoutK, j * Bc, Bc, 0, HSK_pad), tensorViewTranspose); #else const bool k_use_decode = (bs_k > 1u); - if (k_use_decode) { + if (USE_SPARSE) { + coopMatLoadTensorNV(K_T, data_k, k_offset, sliceTensorLayoutNV(tensorLayoutK, j * Bc, Bc, 0, HSK_pad), tensorViewTranspose FAGATHERK); + } else if (k_use_decode) { coopMatLoadTensorNV(K_T, data_k, k_offset, sliceTensorLayoutNV(tensorLayoutK, j * Bc, Bc, 0, HSK_pad), tensorViewTranspose FADECODEK); } else { coopMatLoadTensorNV(K_T, data_k, k_offset, sliceTensorLayoutNV(tensorLayoutK, j * Bc, Bc, 0, HSK_pad), tensorViewTranspose); @@ -330,7 +405,9 @@ void main() { } } - if (MASK_ENABLE) { + if (MASK_ENABLE && USE_SPARSE) { + coopMatPerElementNV(S, S, faAddSparseMask, j); + } else if (MASK_ENABLE) { S += slopeMat*coopmat(mv); } @@ -385,7 +462,9 @@ void main() { coopMatLoadTensorNV(V, data_v, v_offset, sliceTensorLayoutNV(tensorLayoutV, j * Bc, Bc, 0, HSV_pad)); #else const bool v_use_decode = (bs_v > 1u); - if (v_use_decode) { + if (USE_SPARSE) { + coopMatLoadTensorNV(V, data_v, v_offset, sliceTensorLayoutNV(tensorLayoutV, j * Bc, Bc, 0, HSV_pad) FAGATHERV); + } else if (v_use_decode) { coopMatLoadTensorNV(V, data_v, v_offset, sliceTensorLayoutNV(tensorLayoutV, j * Bc, Bc, 0, HSV_pad) FADECODEV); } else { coopMatLoadTensorNV(V, data_v, v_offset, sliceTensorLayoutNV(tensorLayoutV, j * Bc, Bc, 0, HSV_pad)); diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_sparse_compact.comp b/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_sparse_compact.comp new file mode 100644 index 000000000..3d3136266 --- /dev/null +++ b/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_sparse_compact.comp @@ -0,0 +1,102 @@ +#version 450 + +#extension GL_EXT_control_flow_attributes : enable +#extension GL_EXT_shader_16bit_storage : require +#extension GL_EXT_shader_explicit_arithmetic_types_int32 : require +#ifdef USE_SUBGROUPS +#extension GL_KHR_shader_subgroup_basic : require +#extension GL_KHR_shader_subgroup_ballot : require +#endif + +layout(local_size_x_id = 0, local_size_y = 1, local_size_z = 1) in; +layout(constant_id = 0) const uint BLOCK_SIZE = 128; +layout(constant_id = 1) const uint NUM_SUBGROUPS = 1; + +layout (binding = 0) readonly buffer M {float16_t data_m[];}; +layout (binding = 1) writeonly buffer I {int32_t data_i[];}; + +layout (push_constant) uniform parameter { + uint KV; + uint nem1; + uint nem2; + uint nbm1; + uint nbm2; + uint nbm3; + uint n_kv_max; +} p; + +#ifdef USE_SUBGROUPS +shared uvec4 ballots_sh[NUM_SUBGROUPS]; +#else +shared uint scan[BLOCK_SIZE]; +#endif + +// One workgroup per mask row: compact the finite-mask KV positions into a +// per-row index list of length n_kv_max, -1 padded. Emitted in ascending KV +// order so the downstream attention accumulation is deterministic. +void main() { + const uint i1 = gl_WorkGroupID.x; + const uint i2 = gl_WorkGroupID.y; + const uint i3 = gl_WorkGroupID.z; + const uint tid = gl_LocalInvocationIndex; + + const uint m_base = i3 * p.nbm3 + i2 * p.nbm2 + i1 * p.nbm1; + const uint out_base = ((i3 * p.nem2 + i2) * p.nem1 + i1) * p.n_kv_max; + + uint base = 0; + for (uint chunk = 0; chunk < p.KV; chunk += BLOCK_SIZE) { + const uint k = chunk + tid; + bool selected = false; + if (k < p.KV) { + const float v = float(data_m[m_base + k]); + selected = !isinf(v) && !isnan(v); + } + +#ifdef USE_SUBGROUPS + const uvec4 ballot = subgroupBallot(selected); + if (subgroupElect()) { + ballots_sh[gl_SubgroupID] = ballot; + } + barrier(); + + uint subgroup_base = 0; + uint total = 0; + [[unroll]] for (uint s = 0; s < gl_NumSubgroups; ++s) { + if (s == gl_SubgroupID) { + subgroup_base = total; + } + total += subgroupBallotBitCount(ballots_sh[s]); + } + barrier(); + + const uint slot = base + subgroup_base + subgroupBallotExclusiveBitCount(ballot); +#else + // Hillis-Steele inclusive prefix sum over the workgroup. + scan[tid] = selected ? 1u : 0u; + barrier(); + for (uint off = 1; off < BLOCK_SIZE; off <<= 1) { + uint add = 0; + if (tid >= off) { + add = scan[tid - off]; + } + barrier(); + scan[tid] += add; + barrier(); + } + + const uint inclusive = scan[tid]; + const uint total = scan[BLOCK_SIZE - 1]; + const uint slot = base + inclusive - 1u; +#endif + + if (selected && slot < p.n_kv_max) { + data_i[out_base + slot] = int32_t(k); + } + base += total; + barrier(); + } + + for (uint s = min(base, p.n_kv_max) + tid; s < p.n_kv_max; s += BLOCK_SIZE) { + data_i[out_base + s] = int32_t(-1); + } +} diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp b/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp index 30fe0884e..d3f425968 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp @@ -922,6 +922,8 @@ void process_shaders() { string_to_spv("fa_split_k_reduce", "flash_attn_split_k_reduce.comp", {}); string_to_spv("fa_mask_opt", "flash_attn_mask_opt.comp", {}); + string_to_spv("fa_sparse_compact", "flash_attn_sparse_compact.comp", {}); + string_to_spv("fa_sparse_compact_subgroup", "flash_attn_sparse_compact.comp", {{"USE_SUBGROUPS", "1"}}); string_to_spv("quantize_q8_1", "quantize_q8_1.comp", {}); string_to_spv("quantize_q8_1_subgroup", "quantize_q8_1.comp", {{"USE_SUBGROUPS", "1"}}); diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp index 4f2665497..0e074770d 100644 --- a/tests/test-backend-ops.cpp +++ b/tests/test-backend-ops.cpp @@ -10707,6 +10707,9 @@ static std::vector> make_test_cases_eval() { test_cases.emplace_back(new test_flash_attn_ext(128, 128, 1, { 8, 1}, 4096, 1, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q8_0, GGML_TYPE_Q8_0, {0, 1, 2, 3}, true, false, 512)); test_cases.emplace_back(new test_flash_attn_ext(128, 128, 1, { 8, 1}, 4096, 64, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q8_0, GGML_TYPE_Q8_0, {0, 1, 2, 3}, true, false, 512)); + // Qwen QSA: 256/256, gqa 12, budget 2048. + test_cases.emplace_back(new test_flash_attn_ext(256, 256, 2, {12, 1}, 8192, 1, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16, {0, 1, 2, 3}, true, false, 2048)); + // more V-is-sub-view-of-K cases: other head shapes, and full views with equal head sizes test_cases.emplace_back(new test_flash_attn_ext(320, 256, 1, {32, 1}, 512, 1, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16, {0, 1, 2, 3}, true, true)); test_cases.emplace_back(new test_flash_attn_ext(192, 128, 4, {8, 1}, 512, 1, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16, {0, 1, 2, 3}, true, true)); @@ -11165,6 +11168,14 @@ static std::vector> make_test_cases_perf() { // Qwen3-VL-8B https://github.com/ggml-org/llama.cpp/issues/17012 test_cases.emplace_back(new test_flash_attn_ext(72, 72, 16, {1, 1}, 5776, 5776, false, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16)); + // Sparse flash attention (n_kv_max hint) decode across KV depths. + // Shapes: 576/512 DeepSeek MLA, 512/512 DeepSeek-V4/GLM-5.2, 256/256 gqa12 Qwen QSA. + for (int64_t kv : {4096, 16384, 32768}) { + test_cases.emplace_back(new test_flash_attn_ext(512, 512, 1, { 8, 1}, kv, 1, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16, {0, 1, 2, 3}, true, false, 512)); + test_cases.emplace_back(new test_flash_attn_ext(576, 512, 1, {16, 1}, kv, 1, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16, {0, 1, 2, 3}, true, true, 512)); + test_cases.emplace_back(new test_flash_attn_ext(256, 256, 2, {12, 1}, kv, 1, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16, {0, 1, 2, 3}, true, false, 2048)); + } + test_cases.emplace_back(new test_flash_attn_ext(64, 64, 8, {8, 1}, 7680, 1, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16)); test_cases.emplace_back(new test_flash_attn_ext(64, 64, 8, {8, 1}, 7680, 4, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16)); test_cases.emplace_back(new test_flash_attn_ext(64, 64, 8, {8, 1}, 7680, 1, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q4_0, GGML_TYPE_Q4_0)); From 9e71716247113b47bb831d1e0680cbf5f242f094 Mon Sep 17 00:00:00 2001 From: Chris Peterson Date: Tue, 15 Sep 2026 02:33:26 -0700 Subject: [PATCH 15/23] models : move build_arch_graph() after graph() template specialization (#28934) Move build_arch_graph()'s function definitions after the graph and graph template specializations have been explicitly defined. --- src/models/dflash.cpp | 30 +++++++++++++++--------------- src/models/eagle3.cpp | 24 ++++++++++++------------ src/models/t5.cpp | 24 ++++++++++++------------ 3 files changed, 39 insertions(+), 39 deletions(-) diff --git a/src/models/dflash.cpp b/src/models/dflash.cpp index da84f30b6..ed5366d80 100644 --- a/src/models/dflash.cpp +++ b/src/models/dflash.cpp @@ -240,21 +240,6 @@ void llama_model_dflash::load_arch_tensors(llama_model_loader &) { } } -std::unique_ptr llama_model_dflash::build_arch_graph(const llm_graph_params & params) const { - switch (params.gtype) { - case LLM_GRAPH_TYPE_ENCODER: - return std::make_unique>(*this, params); - case LLM_GRAPH_TYPE_DEFAULT: - case LLM_GRAPH_TYPE_DECODER: - if (hparams.dsv4_hc_mult > 0) { - return std::make_unique(*this, params); - } - return std::make_unique>(*this, params); - default: - GGML_ABORT("invalid graph type"); - }; -} - template <> ggml_tensor * llama_model_dflash::graph::build_inp_embd_enc() const { const int64_t n_embd_inp = hparams.n_embd_inp_enc(); @@ -999,3 +984,18 @@ llama_model_dflash::graph_dsv4::graph_dsv4(const llama_model & model, const llm_ build_dspark_markov_head(*this, model, inp_tokens); } } + +std::unique_ptr llama_model_dflash::build_arch_graph(const llm_graph_params & params) const { + switch (params.gtype) { + case LLM_GRAPH_TYPE_ENCODER: + return std::make_unique>(*this, params); + case LLM_GRAPH_TYPE_DEFAULT: + case LLM_GRAPH_TYPE_DECODER: + if (hparams.dsv4_hc_mult > 0) { + return std::make_unique(*this, params); + } + return std::make_unique>(*this, params); + default: + GGML_ABORT("invalid graph type"); + }; +} diff --git a/src/models/eagle3.cpp b/src/models/eagle3.cpp index be466056d..bfde35e43 100644 --- a/src/models/eagle3.cpp +++ b/src/models/eagle3.cpp @@ -100,18 +100,6 @@ void llama_model_eagle3::load_arch_tensors(llama_model_loader &) { } } -std::unique_ptr llama_model_eagle3::build_arch_graph(const llm_graph_params & params) const { - switch (params.gtype) { - case LLM_GRAPH_TYPE_ENCODER: - return std::make_unique>(*this, params); - case LLM_GRAPH_TYPE_DEFAULT: - case LLM_GRAPH_TYPE_DECODER: - return std::make_unique>(*this, params); - default: - GGML_ABORT("invalid graph type"); - }; -} - template <> ggml_tensor * llama_model_eagle3::graph::build_inp_embd_enc() const { ggml_tensor * cur = nullptr; @@ -336,3 +324,15 @@ llama_model_eagle3::graph::graph(const llama_model & model, const llm_gra ggml_build_forward_expand(gf, cur); } + +std::unique_ptr llama_model_eagle3::build_arch_graph(const llm_graph_params & params) const { + switch (params.gtype) { + case LLM_GRAPH_TYPE_ENCODER: + return std::make_unique>(*this, params); + case LLM_GRAPH_TYPE_DEFAULT: + case LLM_GRAPH_TYPE_DECODER: + return std::make_unique>(*this, params); + default: + GGML_ABORT("invalid graph type"); + }; +} diff --git a/src/models/t5.cpp b/src/models/t5.cpp index b0e3f0625..e2bf12b6b 100644 --- a/src/models/t5.cpp +++ b/src/models/t5.cpp @@ -106,18 +106,6 @@ void llama_model_t5::load_arch_tensors(llama_model_loader &) { } } -std::unique_ptr llama_model_t5::build_arch_graph(const llm_graph_params & params) const { - switch (params.gtype) { - case LLM_GRAPH_TYPE_ENCODER: - return std::make_unique>(*this, params); - case LLM_GRAPH_TYPE_DEFAULT: - case LLM_GRAPH_TYPE_DECODER: - return std::make_unique>(*this, params); - default: - GGML_ABORT("invalid graph type"); - }; -} - template <> llama_model_t5::graph::graph(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) { const int64_t n_embd_head = hparams.n_embd_head_v(); @@ -368,3 +356,15 @@ llama_model_t5::graph::graph(const llama_model & model, const llm_graph_pa ggml_build_forward_expand(gf, cur); } + +std::unique_ptr llama_model_t5::build_arch_graph(const llm_graph_params & params) const { + switch (params.gtype) { + case LLM_GRAPH_TYPE_ENCODER: + return std::make_unique>(*this, params); + case LLM_GRAPH_TYPE_DEFAULT: + case LLM_GRAPH_TYPE_DECODER: + return std::make_unique>(*this, params); + default: + GGML_ABORT("invalid graph type"); + }; +} From 54315813269112dd0baed7112ec87ad93a8218ca Mon Sep 17 00:00:00 2001 From: Mohamed Elashri Date: Tue, 15 Sep 2026 12:39:29 +0200 Subject: [PATCH 16/23] cuda: support row-contiguous SUM_ROWS (#26308) * cuda: support row-contiguous SUM_ROWS * organize the code and add GGML_OP_MEAN to support row-contiguous tensors using the same shared kernel, and add a test to MEAN permute/slice * Keep original comments and add if/else branch --- ggml/src/ggml-cuda/ggml-cuda.cu | 2 ++ ggml/src/ggml-cuda/mean.cu | 21 ++++++++++----- ggml/src/ggml-cuda/reduce_rows.cuh | 43 +++++++++++++++++++++++++----- ggml/src/ggml-cuda/sumrows.cu | 20 +++++++++----- tests/test-backend-ops.cpp | 21 ++++++++++++--- 5 files changed, 83 insertions(+), 24 deletions(-) diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu index 43003245c..74bb47145 100644 --- a/ggml/src/ggml-cuda/ggml-cuda.cu +++ b/ggml/src/ggml-cuda/ggml-cuda.cu @@ -5466,7 +5466,9 @@ static bool ggml_backend_cuda_device_supports_op(ggml_backend_dev_t dev, const g return true; #endif case GGML_OP_SUM_ROWS: + return op->src[0]->type == GGML_TYPE_F32 && op->type == GGML_TYPE_F32 && ggml_is_contiguous_rows(op->src[0]); case GGML_OP_MEAN: + return op->src[0]->type == GGML_TYPE_F32 && op->type == GGML_TYPE_F32 && ggml_is_contiguous_rows(op->src[0]); case GGML_OP_GROUP_NORM: return ggml_is_contiguous(op->src[0]); case GGML_OP_PAD: diff --git a/ggml/src/ggml-cuda/mean.cu b/ggml/src/ggml-cuda/mean.cu index a8f6046e4..64ad7e1d5 100644 --- a/ggml/src/ggml-cuda/mean.cu +++ b/ggml/src/ggml-cuda/mean.cu @@ -18,7 +18,7 @@ void ggml_cuda_op_mean(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { GGML_ASSERT(src0->type == GGML_TYPE_F32); GGML_ASSERT(dst->type == GGML_TYPE_F32); - GGML_ASSERT(ggml_is_contiguous(src0)); + GGML_ASSERT(ggml_is_contiguous_rows(src0)); const int64_t ncols = src0->ne[0]; const int64_t nrows = ggml_nrows(src0); @@ -65,13 +65,20 @@ void ggml_cuda_op_mean(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { // Heuristic for block size selection to optimize occupancy. // See discussion in: https://github.com/ggml-org/llama.cpp/pull/15132 + dim3 block_dims; if ((nrows / nsm) < 2) { - const dim3 block_dims(512, 1, 1); - const ggml_cuda_kernel_launch_params launch_params = ggml_cuda_kernel_launch_params(block_nums, block_dims, 0, stream); - ggml_cuda_kernel_launch(reduce_rows_f32, launch_params, src0_d, dst_d, ncols); + block_dims = dim3(512, 1, 1); } else { - const dim3 block_dims(ncols < 1024 ? 32 : 128, 1, 1); - const ggml_cuda_kernel_launch_params launch_params = ggml_cuda_kernel_launch_params(block_nums, block_dims, 0, stream); - ggml_cuda_kernel_launch(reduce_rows_f32, launch_params, src0_d, dst_d, ncols); + block_dims = dim3(ncols < 1024 ? 32 : 128, 1, 1); } + const ggml_cuda_kernel_launch_params launch_params = ggml_cuda_kernel_launch_params(block_nums, block_dims, 0, stream); + + if (ggml_is_contiguous(src0)) { + ggml_cuda_kernel_launch(reduce_rows_f32, launch_params, src0_d, dst_d, ncols); + return; + } + + const char * src0_d_bytes = (const char *) src0->data; + ggml_cuda_kernel_launch(reduce_rows_f32_strided, launch_params, src0_d_bytes, dst_d, ncols, + src0->ne[1], src0->ne[2], src0->nb[1], src0->nb[2], src0->nb[3]); } diff --git a/ggml/src/ggml-cuda/reduce_rows.cuh b/ggml/src/ggml-cuda/reduce_rows.cuh index 968c47aa2..111fd838a 100644 --- a/ggml/src/ggml-cuda/reduce_rows.cuh +++ b/ggml/src/ggml-cuda/reduce_rows.cuh @@ -1,11 +1,6 @@ #include "common.cuh" -// Row reduction kernel template - compute sum (norm=false) or mean (norm=true) -template -static __global__ void reduce_rows_f32(const float * x_ptr, float * dst_ptr, const int ncols) { - const float * GGML_CUDA_RESTRICT x = x_ptr; - float * GGML_CUDA_RESTRICT dst = dst_ptr; - const int row = blockIdx.x; +static __device__ __forceinline__ float reduce_row_f32(const float * x, const int ncols) { const int col = threadIdx.x; float sum = 0.0f; @@ -17,7 +12,7 @@ static __global__ void reduce_rows_f32(const float * x_ptr, float * dst_ptr, con for (int i = col; i < ncols;) { for (int j = 0; j < num_unroll; ++j) { if (i < ncols) { - temp[j] = x[row * ncols + i]; + temp[j] = x[i]; } else { temp[j] = 0; } @@ -35,6 +30,40 @@ static __global__ void reduce_rows_f32(const float * x_ptr, float * dst_ptr, con __shared__ float shared_vals[32]; sum = block_reduce(sum, shared_vals); + return sum; +} + +// Row reduction kernel template - compute sum (norm=false) or mean (norm=true) +template +static __global__ void reduce_rows_f32(const float * x_ptr, float * dst_ptr, const int ncols) { + float * GGML_CUDA_RESTRICT dst = dst_ptr; + const int64_t row = blockIdx.x; + const int col = threadIdx.x; + + const float * GGML_CUDA_RESTRICT x = x_ptr + row*ncols; + const float sum = reduce_row_f32(x, ncols); + + if (col != 0) { + return; + } + + dst[row] = norm ? sum / ncols : sum; +} + +template +static __global__ void reduce_rows_f32_strided(const char * x_ptr, float * dst_ptr, const int ncols, + const int64_t ne1, const int64_t ne2, const int64_t nb1, const int64_t nb2, const int64_t nb3) { + float * GGML_CUDA_RESTRICT dst = dst_ptr; + const int64_t row = blockIdx.x; + const int col = threadIdx.x; + + const int64_t i1 = row % ne1; + const int64_t i2 = (row / ne1) % ne2; + const int64_t i3 = row / (ne1 * ne2); + + const float * GGML_CUDA_RESTRICT x = (const float *) (x_ptr + i1*nb1 + i2*nb2 + i3*nb3); + const float sum = reduce_row_f32(x, ncols); + if (col != 0) { return; } diff --git a/ggml/src/ggml-cuda/sumrows.cu b/ggml/src/ggml-cuda/sumrows.cu index 0003658ca..aa8342b5f 100644 --- a/ggml/src/ggml-cuda/sumrows.cu +++ b/ggml/src/ggml-cuda/sumrows.cu @@ -24,24 +24,30 @@ void ggml_cuda_op_sum_rows(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { GGML_ASSERT(src0->type == GGML_TYPE_F32); GGML_ASSERT( dst->type == GGML_TYPE_F32); - GGML_ASSERT(ggml_is_contiguous(src0)); + GGML_ASSERT(ggml_is_contiguous_rows(src0)); const int64_t ncols = src0->ne[0]; const int64_t nrows = ggml_nrows(src0); + if (ggml_is_contiguous(src0)) { + sum_rows_f32_cuda(src0_d, dst_d, ncols, nrows, stream); + return; + } + const dim3 block_nums(nrows, 1, 1); const int id = ggml_cuda_get_device(); const int nsm = ggml_cuda_info().devices[id].nsm; + dim3 block_dims; if ((nrows / nsm) < 2) { // Increase num threads to 512 for small nrows to better hide the latency - const dim3 block_dims(512, 1, 1); - const ggml_cuda_kernel_launch_params launch_params = ggml_cuda_kernel_launch_params(block_nums, block_dims, 0, stream); - ggml_cuda_kernel_launch(reduce_rows_f32, launch_params, src0_d, dst_d, ncols); + block_dims = dim3(512, 1, 1); } else { // Enough active SMs to hide latency, use smaller blocks to allow better scheduling - const dim3 block_dims(ncols < 1024 ? 32 : 128, 1, 1); - const ggml_cuda_kernel_launch_params launch_params = ggml_cuda_kernel_launch_params(block_nums, block_dims, 0, stream); - ggml_cuda_kernel_launch(reduce_rows_f32, launch_params, src0_d, dst_d, ncols); + block_dims = dim3(ncols < 1024 ? 32 : 128, 1, 1); } + const ggml_cuda_kernel_launch_params launch_params = ggml_cuda_kernel_launch_params(block_nums, block_dims, 0, stream); + const char * src0_d_bytes = (const char *) src0->data; + ggml_cuda_kernel_launch(reduce_rows_f32_strided, launch_params, src0_d_bytes, dst_d, ncols, + src0->ne[1], src0->ne[2], src0->nb[1], src0->nb[2], src0->nb[3]); } diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp index 0e074770d..1616004e0 100644 --- a/tests/test-backend-ops.cpp +++ b/tests/test-backend-ops.cpp @@ -7126,20 +7126,32 @@ struct test_sum_rows : public test_case { struct test_mean : public test_case { const ggml_type type; const std::array ne; + const bool permute; + const bool slice; std::string vars() override { - return VARS_TO_STR2(type, ne); + return VARS_TO_STR4(type, ne, permute, slice); } test_mean(ggml_type type = GGML_TYPE_F32, - std::array ne = {10, 5, 4, 3}) - : type(type), ne(ne) {} + std::array ne = {10, 5, 4, 3}, + bool permute = false, bool slice = false) + : type(type), ne(ne), permute(permute), slice(slice) {} ggml_tensor * build_graph(ggml_context * ctx) override { ggml_tensor * a = ggml_new_tensor(ctx, type, 4, ne.data()); ggml_set_param(a); ggml_set_name(a, "a"); + if (slice) { + a = ggml_view_4d(ctx, a, + ne[0], ne[1], ne[2] / 2, ne[3] - 1, + a->nb[1], a->nb[2] * 2, a->nb[3], /*offset=*/a->nb[3]); + } + if (permute) { + a = ggml_permute(ctx, a, 0, 2, 3, 1); + } + ggml_tensor * out = ggml_mean(ctx, a); ggml_set_name(out, "out"); @@ -10470,6 +10482,9 @@ static std::vector> make_test_cases_eval() { test_cases.emplace_back(new test_mean(GGML_TYPE_F32, { 32, 1, 1, 1 })); test_cases.emplace_back(new test_mean(GGML_TYPE_F32, { 32, 256, 1, 1 })); test_cases.emplace_back(new test_mean(GGML_TYPE_F32, { 32768, 1, 1, 1 })); + test_cases.emplace_back(new test_mean(GGML_TYPE_F32, { 11, 5, 6, 3 }, true, false)); + test_cases.emplace_back(new test_mean(GGML_TYPE_F32, { 11, 5, 6, 3 }, false, true)); + test_cases.emplace_back(new test_mean(GGML_TYPE_F32, { 11, 5, 6, 3 }, true, true)); test_cases.emplace_back(new test_sum(GGML_TYPE_F32, { 33, 1, 1, 1 })); test_cases.emplace_back(new test_sum(GGML_TYPE_F32, { 33, 1024, 1, 1 })); test_cases.emplace_back(new test_sum(GGML_TYPE_F32, { 33, 256, 1, 1 })); From 7609846557c50f9d984719a9e1e8c5f3d02f807b Mon Sep 17 00:00:00 2001 From: Patrick Hoffmann Date: Tue, 15 Sep 2026 13:50:20 +0200 Subject: [PATCH 17/23] rpc : hash-cache only weights (#28789) * rpc : hash-cache only weights ggml_backend_rpc_buffer_set_tensor and ggml_backend_rpc_set_tensor_async hashed every transfer above HASH_THRESHOLD and let `rpc-server -c` serve it from its file cache. The cache is meant for weights, but the activations ggml_backend_sched copies between backends took the same path: with a two-node split of Qwen3.8-Flash-Next every prefill ubatch above 10 MB was hashed, written to the worker's cache directory (1.4 TB after a day) and later served from there. Use the hash path only for tensors in buffers marked GGML_BACKEND_BUFFER_USAGE_WEIGHTS. Co-Authored-By: Claude Opus 5 * rpc : save a cache entry only for the tensor that missed the hash check With the client hashing weights only, the server still wrote every SET_TENSOR above HASH_THRESHOLD to the cache directory, so the compute data the scheduler sends kept filling the disk. Remember the hash of the last SET_TENSOR_HASH that missed and save only the SET_TENSOR that follows it with that hash - the weight the client is re-sending. * rpc : signal the cache decision in the SET_TENSOR payload Replace the server-side `pending_cache` state with a `cache_flag` byte in the SET_TENSOR message: the client sets it when SET_TENSOR_HASH reported a miss, the server saves a cache entry only when it is set. Bump RPC_PROTO_MAJOR_VERSION since the wire format changes. --------- Co-authored-by: Patrick Hoffmann Co-authored-by: Claude Opus 5 --- ggml/include/ggml-rpc.h | 2 +- ggml/src/ggml-rpc/ggml-rpc.cpp | 69 ++++++++++++++++++++++------------ 2 files changed, 45 insertions(+), 26 deletions(-) diff --git a/ggml/include/ggml-rpc.h b/ggml/include/ggml-rpc.h index cbfe40013..1f8cb7906 100644 --- a/ggml/include/ggml-rpc.h +++ b/ggml/include/ggml-rpc.h @@ -6,7 +6,7 @@ extern "C" { #endif -#define RPC_PROTO_MAJOR_VERSION 6 +#define RPC_PROTO_MAJOR_VERSION 7 #define RPC_PROTO_MINOR_VERSION 0 #define RPC_PROTO_PATCH_VERSION 0 diff --git a/ggml/src/ggml-rpc/ggml-rpc.cpp b/ggml/src/ggml-rpc/ggml-rpc.cpp index cc7d72069..adb88a245 100644 --- a/ggml/src/ggml-rpc/ggml-rpc.cpp +++ b/ggml/src/ggml-rpc/ggml-rpc.cpp @@ -697,10 +697,31 @@ static void ggml_backend_rpc_buffer_memset_tensor( ctx->dispatcher->send(RPC_CMD_MEMSET_TENSOR, request, sizeof(*request)); } +// input serialization format: | rpc_tensor | cache_flag (1 byte) | offset (8 bytes) | data (size bytes) +static std::shared_ptr serialize_set_tensor(const rpc_tensor & rpc_tensor, uint8_t cache_flag, uint64_t offset, const void * data, size_t size, size_t & input_size) { + input_size = sizeof(rpc_tensor) + sizeof(cache_flag) + sizeof(offset) + size; + uint8_t * input = new uint8_t[input_size](); + uint8_t * p = input; + memcpy(p, &rpc_tensor, sizeof(rpc_tensor)); p += sizeof(rpc_tensor); + memcpy(p, &cache_flag, sizeof(cache_flag)); p += sizeof(cache_flag); + memcpy(p, &offset, sizeof(offset)); p += sizeof(offset); + memcpy(p, data, size); + return std::shared_ptr(input, std::default_delete()); +} + +// the hash cache is meant for weights, so that a model reload can skip re-sending them. +// compute-buffer inputs (the activations ggml_backend_sched copies between backends) must not +// take this path, otherwise with `rpc-server -c` every ubatch above the threshold is written +// to the cache directory and later served from there. +static bool rpc_use_hash_cache(const ggml_tensor * tensor, size_t size) { + return size > HASH_THRESHOLD && tensor->buffer->usage == GGML_BACKEND_BUFFER_USAGE_WEIGHTS; +} + static void ggml_backend_rpc_buffer_set_tensor(ggml_backend_buffer_t buffer, ggml_tensor * tensor, const void * data, size_t offset, size_t size) { ggml_backend_rpc_buffer_context * ctx = (ggml_backend_rpc_buffer_context *)buffer->context; rpc_tensor rpc_tensor = serialize_tensor(tensor); - if (size > HASH_THRESHOLD) { + uint8_t cache_flag = 0; + if (rpc_use_hash_cache(tensor, size)) { auto request = std::make_shared(); request->tensor = rpc_tensor; request->offset = offset; @@ -711,15 +732,12 @@ static void ggml_backend_rpc_buffer_set_tensor(ggml_backend_buffer_t buffer, ggm // the server has the same data, no need to send it return; } + // the server has no cache entry for this tensor - ask it to save one + cache_flag = 1; } - // input serialization format: | rpc_tensor | offset (8 bytes) | data (size bytes) - size_t input_size = sizeof(rpc_tensor) + sizeof(uint64_t) + size; - uint8_t * input = new uint8_t[input_size](); - memcpy(input, &rpc_tensor, sizeof(rpc_tensor)); - memcpy(input + sizeof(rpc_tensor), &offset, sizeof(offset)); - memcpy(input + sizeof(rpc_tensor) + sizeof(offset), data, size); - std::shared_ptr input_ptr(input, std::default_delete()); - ctx->dispatcher->send(RPC_CMD_SET_TENSOR, input_ptr, input_size); + size_t input_size; + auto input = serialize_set_tensor(rpc_tensor, cache_flag, offset, data, size, input_size); + ctx->dispatcher->send(RPC_CMD_SET_TENSOR, input, input_size); } static void ggml_backend_rpc_buffer_get_tensor(ggml_backend_buffer_t buffer, const ggml_tensor * tensor, void * data, size_t offset, size_t size) { @@ -927,7 +945,8 @@ static void ggml_backend_rpc_free(ggml_backend_t backend) { static void ggml_backend_rpc_set_tensor_async(ggml_backend_t backend, ggml_tensor * tensor, const void * data, size_t offset, size_t size) { ggml_backend_rpc_context * ctx = (ggml_backend_rpc_context *)backend->context; rpc_tensor rpc_tensor = serialize_tensor(tensor); - if (size > HASH_THRESHOLD) { + uint8_t cache_flag = 0; + if (rpc_use_hash_cache(tensor, size)) { auto request = std::make_shared(); request->tensor = rpc_tensor; request->offset = offset; @@ -939,15 +958,12 @@ static void ggml_backend_rpc_set_tensor_async(ggml_backend_t backend, ggml_tenso // the server has the same data, no need to send it return; } + // the server has no cache entry for this tensor - ask it to save one + cache_flag = 1; } - // input serialization format: | rpc_tensor | offset (8 bytes) | data (size bytes) - size_t input_size = sizeof(rpc_tensor) + sizeof(uint64_t) + size; - uint8_t * input = new uint8_t[input_size](); - memcpy(input, &rpc_tensor, sizeof(rpc_tensor)); - memcpy(input + sizeof(rpc_tensor), &offset, sizeof(offset)); - memcpy(input + sizeof(rpc_tensor) + sizeof(offset), data, size); - std::shared_ptr input_ptr(input, std::default_delete()); - ctx->dispatcher->send_async(RPC_CMD_SET_TENSOR, input_ptr, input_size); + size_t input_size; + auto input = serialize_set_tensor(rpc_tensor, cache_flag, offset, data, size, input_size); + ctx->dispatcher->send_async(RPC_CMD_SET_TENSOR, input, input_size); } static void ggml_backend_rpc_get_tensor_async(ggml_backend_t backend, const ggml_tensor * tensor, void * data, size_t offset, size_t size) { @@ -1398,14 +1414,17 @@ ggml_tensor * rpc_server::deserialize_tensor(struct ggml_context * ctx, const rp bool rpc_server::set_tensor(const std::vector & input) { - // serialization format: | rpc_tensor | offset (8 bytes) | data (size bytes) | - if (input.size() < sizeof(rpc_tensor) + sizeof(uint64_t)) { + // serialization format: | rpc_tensor | cache_flag (1 byte) | offset (8 bytes) | data (size bytes) | + uint8_t cache_flag; + uint64_t offset; + const size_t header_size = sizeof(rpc_tensor) + sizeof(cache_flag) + sizeof(offset); + if (input.size() < header_size) { return false; } const rpc_tensor * in_tensor = (const rpc_tensor *)input.data(); - uint64_t offset; - memcpy(&offset, input.data() + sizeof(rpc_tensor), sizeof(offset)); - const size_t size = input.size() - sizeof(rpc_tensor) - sizeof(offset); + memcpy(&cache_flag, input.data() + sizeof(rpc_tensor), sizeof(cache_flag)); + memcpy(&offset, input.data() + sizeof(rpc_tensor) + sizeof(cache_flag), sizeof(offset)); + const size_t size = input.size() - header_size; struct ggml_init_params params { /*.mem_size =*/ ggml_tensor_overhead(), @@ -1434,8 +1453,8 @@ bool rpc_server::set_tensor(const std::vector & input) { } } - const void * data = input.data() + sizeof(rpc_tensor) + sizeof(offset); - if (cache_dir && size > HASH_THRESHOLD) { + const void * data = input.data() + header_size; + if (cache_dir && cache_flag) { uint64_t hash = fnv_hash((const uint8_t*)data, size); char hash_str[17]; snprintf(hash_str, sizeof(hash_str), "%016" PRIx64, hash); From 6011c34ce6099646ccdf0d39a61c6e681477c178 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20G=C3=A4=C3=9Fler?= Date: Tue, 15 Sep 2026 14:11:16 +0200 Subject: [PATCH 18/23] docs: Rule of thumb for AI review time [no ci] (#28945) --- CONTRIBUTING.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 6aac3cb87..59ec3f311 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -20,8 +20,8 @@ If AI is used to generate any portion of the code, contributors must adhere to t 1. Explicitly disclose the manner in which AI was employed. 2. Check for an existing PR addressing the same change; if one exists, comment there to work with its author instead of opening a duplicate. -3. Perform a comprehensive manual review prior to submitting the pull request. -4. Be prepared to explain every line of code they submitted when asked about it by a maintainer. +3. Perform a comprehensive manual review prior to submitting the pull request. A proper code review usually takes something like one hour per 200-400 LOC and you should be spending **at least that much time on code review alone**. +4. Be prepared to explain every line of code you submit when asked about it by a maintainer. 5. It is strictly prohibited to use AI to write your posts for you (bug reports, feature requests, pull request descriptions, Github discussions, responding to humans, ...). For more info, please refer to the [AGENTS.md](AGENTS.md) file. From d1d3c3396aa13a5f239109a822666c4870490ad5 Mon Sep 17 00:00:00 2001 From: Aman Gupta Date: Tue, 15 Sep 2026 21:48:15 +0800 Subject: [PATCH 19/23] ci: build MUSA for only 1 arch (#28944) * ci: optimize * keep only the MUSA changes --- .github/workflows/build-cuda-ubuntu.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build-cuda-ubuntu.yml b/.github/workflows/build-cuda-ubuntu.yml index da61b3353..30029887c 100644 --- a/.github/workflows/build-cuda-ubuntu.yml +++ b/.github/workflows/build-cuda-ubuntu.yml @@ -177,7 +177,8 @@ jobs: id: cmake_build run: | cmake -B build -S . \ - -DGGML_MUSA=ON + -DGGML_MUSA=ON \ + -DMUSA_ARCHITECTURES=21 time cmake --build build --config Release -j $(nproc) - name: ccache-buckets-save From 9f31776c3773cf03f98535c19b7e6d394af374b4 Mon Sep 17 00:00:00 2001 From: Hongqiang Wang Date: Tue, 15 Sep 2026 11:21:05 -0700 Subject: [PATCH 20/23] opencl: choose the MoE expert matmul by batch size for speculative decoding/MTP (#27637) * opencl: gate the prebuilt q4_0 MoE GEMM on routing count * opencl: stop writing zeros into the padded MoE activation slots * opencl: rephrase claude's comments --------- Co-authored-by: Li He --- ggml/src/ggml-opencl/ggml-opencl.cpp | 45 ++++++++++++------- ggml/src/ggml-opencl/kernels/moe_reorder_b.cl | 12 ++--- 2 files changed, 35 insertions(+), 22 deletions(-) diff --git a/ggml/src/ggml-opencl/ggml-opencl.cpp b/ggml/src/ggml-opencl/ggml-opencl.cpp index 5b99f5d00..bd5af9e37 100644 --- a/ggml/src/ggml-opencl/ggml-opencl.cpp +++ b/ggml/src/ggml-opencl/ggml-opencl.cpp @@ -24566,10 +24566,33 @@ static void ggml_cl_mul_mat_id(ggml_backend_t backend, const ggml_tensor * src0, CL_CHECK(clReleaseMemObject(buf_src2)); } else { // for gemm - kernel = backend_ctx->kernel_gemm_moe_q4_0_f32_ns; - if (backend_ctx->kernel_gemm_moe_q4_0_f32_ns_bin) { - kernel = backend_ctx->kernel_gemm_moe_q4_0_f32_ns_bin; - } + // dp4a (int8) prefill GEMM variant + static const char * q4_0_moe_dp4a_env = getenv("GGML_OPENCL_Q4_0_MOE_DP4A"); + + // It turns out that the prebuilt kernel only outperforms the dp4a variant (on X2-90) + // at very large routing counts, so we gate its use accordingly using moe_bin_min, + // which can be overridden via the GGML_OPENCL_MOE_BIN_MIN_ROUTINGS environment variable. + // The routing count is ne20 * ne21 (n_expert_used * n_tokens). + static const char * moe_bin_min_env = getenv("GGML_OPENCL_MOE_BIN_MIN_ROUTINGS"); + const int moe_bin_min = moe_bin_min_env ? atoi(moe_bin_min_env) : 4096; + + // whether bin kernels are available + const bool bin_available = backend_ctx->kernel_gemm_moe_q4_0_f32_ns_bin != nullptr; + const bool dp4a_bin_available = backend_ctx->kernel_gemm_moe_q4_0_q8_1_dp4a_bin != nullptr; + + bool use_moe_dp4a = q4_0_moe_dp4a_env + ? (atoi(q4_0_moe_dp4a_env) != 0) + : (backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E + && (dp4a_bin_available || !bin_available + || (int)(ne20 * ne21) < moe_bin_min)); + // dot prod has to be available + use_moe_dp4a = backend_ctx->has_integer_dot && use_moe_dp4a; + + const bool use_bin_kernel = bin_available && !use_moe_dp4a; + + kernel = use_bin_kernel + ? backend_ctx->kernel_gemm_moe_q4_0_f32_ns_bin + : backend_ctx->kernel_gemm_moe_q4_0_f32_ns; // Reorder router if called from test-backend-ops or when new router is generated. // Otherwise reuse the reordered result from previous mul_mat_id call. @@ -24582,18 +24605,6 @@ static void ggml_cl_mul_mat_id(ggml_backend_t backend, const ggml_tensor * src0, cl_mem buf_src1_reordered = nullptr, image_src1_reordered = nullptr; cl_mem buf_src2, buf_src2_emap; - // dp4a (int8) prefill GEMM variant - static const char * q4_0_moe_dp4a_env = getenv("GGML_OPENCL_Q4_0_MOE_DP4A"); - bool use_moe_dp4a = q4_0_moe_dp4a_env - ? (atoi(q4_0_moe_dp4a_env) != 0) - : (backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E); - // dot prod has to be available - use_moe_dp4a = backend_ctx->has_integer_dot && use_moe_dp4a; - // bin kernel takes precedence - if (backend_ctx->kernel_gemm_moe_q4_0_q8_1_dp4a_bin == nullptr) { - use_moe_dp4a = use_moe_dp4a && backend_ctx->kernel_gemm_moe_q4_0_f32_ns_bin == nullptr; - } - cl_buffer_region region; region.origin = 0; region.size = sizeof(int) * max_post_router_tile * n_tile_size; @@ -24632,7 +24643,7 @@ static void ggml_cl_mul_mat_id(ggml_backend_t backend, const ggml_tensor * src0, cl_image_desc image_desc_buf_src1; image_format_buf_src1 = {CL_RGBA, CL_FLOAT}; image_desc_buf_src1 = {CL_MEM_OBJECT_IMAGE1D_BUFFER, static_cast(ne00 * max_post_router_tile * n_tile_size / 4), 0,0,0,0,0,0,0, {buf_src1_reordered}}; - if (backend_ctx->kernel_gemm_moe_q4_0_f32_ns_bin) { + if (use_bin_kernel) { // bin kernel uses slightly different image format image_format_buf_src1 = {CL_R, CL_FLOAT}; image_desc_buf_src1.image_width = static_cast(ne00 * max_post_router_tile * n_tile_size); diff --git a/ggml/src/ggml-opencl/kernels/moe_reorder_b.cl b/ggml/src/ggml-opencl/kernels/moe_reorder_b.cl index e6295c816..2f5c110bf 100644 --- a/ggml/src/ggml-opencl/kernels/moe_reorder_b.cl +++ b/ggml/src/ggml-opencl/kernels/moe_reorder_b.cl @@ -20,11 +20,13 @@ kernel void kernel_moe_reorder_b( uint router_idx = router[post_router_idx]; - float4 out = (float4)(0); - if (router_idx != 0xFFFFFFFF) { - ushort activation_idx = router_idx / map_ratio; - out = src[activation_idx * K / 4 + k_4]; + // Padded slots need not be written at all. The MoE GEMMs accumulate per output + // column and scatter only the real columns, so whatever sits in a padded slot + // never reaches dst + if (router_idx == 0xFFFFFFFF) { + return; } - dst[post_router_idx * K / 4 + k_4] = out; + ushort activation_idx = router_idx / map_ratio; + dst[post_router_idx * K / 4 + k_4] = src[activation_idx * K / 4 + k_4]; } From 38a5b42d9a3e82e0a586bcd1caed121f36c87a73 Mon Sep 17 00:00:00 2001 From: Sandro Steeger <78495486+Stastez@users.noreply.github.com> Date: Tue, 15 Sep 2026 20:57:41 +0200 Subject: [PATCH 21/23] HIP: Enable AllReduce for ROCm (#27825) --- ggml/src/ggml-cuda/allreduce.cu | 58 ++++++++++++++++++-------------- ggml/src/ggml-cuda/allreduce.cuh | 2 +- ggml/src/ggml-cuda/vendors/hip.h | 4 +++ 3 files changed, 37 insertions(+), 27 deletions(-) diff --git a/ggml/src/ggml-cuda/allreduce.cu b/ggml/src/ggml-cuda/allreduce.cu index d56129a22..39b23bed7 100644 --- a/ggml/src/ggml-cuda/allreduce.cu +++ b/ggml/src/ggml-cuda/allreduce.cu @@ -1,6 +1,6 @@ #include "allreduce.cuh" -#if !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA) +#if !defined(GGML_USE_MUSA) #include "convert.cuh" #include "ggml-impl.h" @@ -11,11 +11,12 @@ #include // --------------------------------------------------------------------------- -// CUDA AllReduce for tensor-parallel inference across two GPUs. +// AllReduce for tensor-parallel inference across two GPUs (CUDA or +// ROCm/HIP). // -// Provides an in-place sum reduction over matching tensors on two CUDA -// devices in the same process. Used by the tensor-split path alongside -// NCCL; targets setups without NVLink, where data is exchanged between the +// Provides an in-place sum reduction over matching tensors on two GPUs +// in the same process. Used by the tensor-split path alongside NCCL; +// targets setups without NVLink/xGMI, where data is exchanged between the // GPUs by staging it through pinned host memory over PCIe. // // Two reduction strategies are selected per call by tensor size: @@ -161,11 +162,14 @@ static __global__ void ggml_cuda_ar_kernel( __threadfence_system(); // make our signal visible system-wide while (ggml_cuda_ar_signal_get(other_slot) != token) { -#if __CUDA_ARCH__ >= GGML_CUDA_CC_VOLTA +#ifdef GGML_USE_HIP + // Equals ~100ns at 2500 MHz (sleeps for n * [1,64] clock cycles) + __builtin_amdgcn_s_sleep(4); +#elif __CUDA_ARCH__ >= GGML_CUDA_CC_VOLTA __nanosleep(100); #else NO_DEVICE_CODE; -#endif // __CUDA_ARCH__ >= GGML_CUDA_CC_VOLTA +#endif // GGML_USE_HIP } } @@ -280,7 +284,7 @@ struct ggml_cuda_ar_host_mapping { } rc = cudaHostGetDevicePointer(reinterpret_cast(&dev), host, 0); if (rc != cudaSuccess) { - cudaFreeHost(host); + CUDA_CHECK(cudaFreeHost(host)); host = nullptr; dev = nullptr; } @@ -289,7 +293,7 @@ struct ggml_cuda_ar_host_mapping { void free() { if (host) { - cudaFreeHost(host); + CUDA_CHECK(cudaFreeHost(host)); host = nullptr; dev = nullptr; } @@ -401,7 +405,8 @@ ggml_cuda_ar_pipeline * ggml_cuda_ar_pipeline_init(const int * devices, size_t n return nullptr; } - // The chunked kernel uses __nanosleep, which is sm70+ (Volta+). + // The chunked kernel uses __nanosleep (NVIDIA, sm70+) or + // __builtin_amdgcn_s_sleep (AMD). for (size_t i = 0; i < n_devices; ++i) { const int cc = ggml_cuda_info().devices[devices[i]].cc; if (cc < GGML_CUDA_CC_VOLTA) { @@ -543,7 +548,7 @@ void ggml_cuda_ar_pipeline_free(ggml_cuda_ar_pipeline * p) { for (int i = 0; i < p->n_devices; ++i) { if (p->streams[i]) { ggml_cuda_set_device(p->devices[i]); - cudaStreamSynchronize(p->streams[i]); + CUDA_CHECK(cudaStreamSynchronize(p->streams[i])); } } @@ -552,28 +557,28 @@ void ggml_cuda_ar_pipeline_free(ggml_cuda_ar_pipeline * p) { p->host_large[i].free(); if (p->dev_tmp[i]) { ggml_cuda_set_device(p->devices[i]); - cudaFree(p->dev_tmp[i]); + CUDA_CHECK(cudaFree(p->dev_tmp[i])); } ggml_cuda_set_device(p->devices[i]); for (int s = 0; s < GGML_CUDA_AR_POOL_SIZE; ++s) { - if (p->ev_pool[i][s].app) { cudaEventDestroy(p->ev_pool[i][s].app); } + if (p->ev_pool[i][s].app) { CUDA_CHECK(cudaEventDestroy(p->ev_pool[i][s].app)); } for (int c = 0; c < GGML_CUDA_AR_COPY_MAX_CHUNKS; ++c) { - if (p->ev_pool[i][s].cpy[c]) { cudaEventDestroy(p->ev_pool[i][s].cpy[c]); } + if (p->ev_pool[i][s].cpy[c]) { CUDA_CHECK(cudaEventDestroy(p->ev_pool[i][s].cpy[c])); } } - if (p->ev_pool[i][s].h2d) { cudaEventDestroy(p->ev_pool[i][s].h2d); } - if (p->ev_pool[i][s].ker) { cudaEventDestroy(p->ev_pool[i][s].ker); } + if (p->ev_pool[i][s].h2d) { CUDA_CHECK(cudaEventDestroy(p->ev_pool[i][s].h2d)); } + if (p->ev_pool[i][s].ker) { CUDA_CHECK(cudaEventDestroy(p->ev_pool[i][s].ker)); } } if (p->host_large_read_done[i]) { ggml_cuda_set_device(p->devices[i]); - cudaEventDestroy(p->host_large_read_done[i]); + CUDA_CHECK(cudaEventDestroy(p->host_large_read_done[i])); } if (p->dev_tmp_kernel_done[i]) { ggml_cuda_set_device(p->devices[i]); - cudaEventDestroy(p->dev_tmp_kernel_done[i]); + CUDA_CHECK(cudaEventDestroy(p->dev_tmp_kernel_done[i])); } if (p->streams[i]) { ggml_cuda_set_device(p->devices[i]); - cudaStreamDestroy(p->streams[i]); + CUDA_CHECK(cudaStreamDestroy(p->streams[i])); } } p->arrival.free(); @@ -952,13 +957,14 @@ bool ggml_cuda_ar_allreduce( return ok; } -#else // defined(GGML_USE_HIP) || defined(GGML_USE_MUSA) +#else // defined(GGML_USE_MUSA) -// HIP and MUSA lack the host-mapped pinned-memory APIs (cudaHostAllocPortable -// / cudaHostAllocMapped / cudaHostGetDevicePointer) and __nanosleep that this -// implementation relies on, so the internal AllReduce is a CUDA-only feature. -// The dispatcher in ggml-cuda.cu treats a nullptr pipeline as "init failed" -// and silently falls back to the meta backend's generic AllReduce. +// MUSA lacks the host-mapped pinned-memory APIs (cudaHostAllocPortable +// / cudaHostAllocMapped / cudaHostGetDevicePointer) and a device-side +// sleep intrinsic that this implementation relies on, so the internal +// AllReduce is unavailable there. The dispatcher in ggml-cuda.cu treats +// a nullptr pipeline as "init failed" and silently falls back to the meta +// backend's generic AllReduce. ggml_cuda_ar_pipeline * ggml_cuda_ar_pipeline_init(const int *, size_t) { return nullptr; } @@ -968,4 +974,4 @@ bool ggml_cuda_ar_allreduce(ggml_cuda_ar_pipeline *, ggml_backend_t *, ggml_tens return false; } -#endif // !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA) +#endif // !defined(GGML_USE_MUSA) diff --git a/ggml/src/ggml-cuda/allreduce.cuh b/ggml/src/ggml-cuda/allreduce.cuh index 0f2c9518d..76205d323 100644 --- a/ggml/src/ggml-cuda/allreduce.cuh +++ b/ggml/src/ggml-cuda/allreduce.cuh @@ -9,7 +9,7 @@ struct ggml_cuda_ar_pipeline; // Allocate a pipeline for n_devices GPUs. -// devices[] holds the CUDA device IDs in rank order. +// devices[] holds the GPU device IDs in rank order. // Returns nullptr on allocation failure. ggml_cuda_ar_pipeline * ggml_cuda_ar_pipeline_init( const int * devices, size_t n_devices); diff --git a/ggml/src/ggml-cuda/vendors/hip.h b/ggml/src/ggml-cuda/vendors/hip.h index 2fc0fe9fd..48d4eb2ce 100644 --- a/ggml/src/ggml-cuda/vendors/hip.h +++ b/ggml/src/ggml-cuda/vendors/hip.h @@ -73,6 +73,10 @@ #define cudaGetDeviceProperties hipGetDeviceProperties #define cudaGetErrorString hipGetErrorString #define cudaGetLastError hipGetLastError +#define cudaHostAlloc hipHostMalloc +#define cudaHostAllocPortable hipHostMallocPortable +#define cudaHostAllocMapped hipHostMallocMapped +#define cudaHostGetDevicePointer hipHostGetDevicePointer #define cudaHostRegister hipHostRegister #define cudaHostRegisterPortable hipHostRegisterPortable #define cudaHostRegisterReadOnly hipHostRegisterReadOnly From 72b590d65f04adabbb6403d75188edc77bc5a867 Mon Sep 17 00:00:00 2001 From: Trivikram Reddy <127072883+trivikram-reddy1@users.noreply.github.com> Date: Tue, 15 Sep 2026 17:45:28 -0500 Subject: [PATCH 22/23] hex-cpy: use dma if src and dst are contiguous (#28906) --- ggml/src/ggml-hexagon/htp/cpy-ops.c | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/ggml/src/ggml-hexagon/htp/cpy-ops.c b/ggml/src/ggml-hexagon/htp/cpy-ops.c index 7f01a8c1e..e68b2d3db 100644 --- a/ggml/src/ggml-hexagon/htp/cpy-ops.c +++ b/ggml/src/ggml-hexagon/htp/cpy-ops.c @@ -294,6 +294,18 @@ static inline void cpy_dma_sametype_sameshape( dma_queue_flush(q); } +static inline void cpy_dma_sametype_reshape_contig( + struct htp_ops_context * octx, + const struct htp_tensor * dst, + const struct htp_tensor * src0, + uint32_t total_bytes +) { + dma_queue * q = octx->ctx->dma[0]; + dma_queue_push(q, dma_make_ptr((void *) dst->data, (const void *) src0->data), + total_bytes, total_bytes, total_bytes, /*nrows=*/ 1); + dma_queue_pop(q); +} + static int exec_cpy(struct htp_ops_context * octx, bool * use_dma) { cpy_preamble; *use_dma = false; @@ -327,6 +339,7 @@ static int exec_cpy(struct htp_ops_context * octx, bool * use_dma) { const uint32_t n_threads = octx->n_threads; + const bool src_is_contiguous = htp_tensor_is_contiguous(src0, ct.src0_type_size); const bool dst_is_contiguous = htp_tensor_is_contiguous(dst, ct.dst_type_size); if (sameshape) { @@ -375,6 +388,12 @@ static int exec_cpy(struct htp_ops_context * octx, bool * use_dma) { const uint32_t total_elems = ne0 * ne1 * ne2 * ne3; const uint32_t elems_per_line = (ct.dst_type_size == 4) ? 32 : 64; + if (octx->ctx->mdev.count <= 1 && dst_is_contiguous && src_is_contiguous) { + *use_dma = true; + cpy_dma_sametype_reshape_contig(octx, dst, src0, total_elems * ct.dst_type_size); + return HTP_STATUS_OK; + } + ct.div_ne0 = init_fastdiv_values(ne0); ct.div_ne1_ne0 = init_fastdiv_values(ne1 * ne0); ct.div_ne2_ne1_ne0 = init_fastdiv_values(ne2 * ne1 * ne0); From 930e2fa5995789efbf249a8bf61325bb626e417b Mon Sep 17 00:00:00 2001 From: Jhen-Jie Hong Date: Wed, 16 Sep 2026 07:02:06 +0800 Subject: [PATCH 23/23] hexagon: add back missing contiguous fast-path and hvx_copy_uu for each run (#28886) --- ggml/src/ggml-hexagon/htp/cpy-ops.c | 30 +++++++++++++++++++++++------ 1 file changed, 24 insertions(+), 6 deletions(-) diff --git a/ggml/src/ggml-hexagon/htp/cpy-ops.c b/ggml/src/ggml-hexagon/htp/cpy-ops.c index e68b2d3db..490efd687 100644 --- a/ggml/src/ggml-hexagon/htp/cpy-ops.c +++ b/ggml/src/ggml-hexagon/htp/cpy-ops.c @@ -126,6 +126,13 @@ static void cpy_thread_##NAME##_reshape(unsigned int nth, unsigned int ith, void const uint32_t th_end = MIN(th_start + th_nelem, ct->elem_start + ct->nelem); \ if (th_start >= th_end) return; \ \ + if (htp_tensor_is_contiguous(src0, ELEM_SIZE) && htp_tensor_is_contiguous(dst, ELEM_SIZE)) { \ + hvx_copy_uu((uint8_t *) dst->data + (size_t) th_start * ELEM_SIZE, \ + (const uint8_t *) src0->data + (size_t) th_start * ELEM_SIZE, \ + th_end - th_start, ELEM_SIZE); \ + return; \ + } \ + \ const uint32_t ne01_ne00 = ne01 * ne00; \ const uint32_t ne02_ne01_ne00 = ne02 * ne01_ne00; \ const uint32_t ne1_ne0 = ne1 * ne0; \ @@ -149,11 +156,21 @@ static void cpy_thread_##NAME##_reshape(unsigned int nth, unsigned int ith, void char * dst_ptr = (char *) dst->data + i10*nb0 + i11*nb1 + i12*nb2 + i13*nb3; \ const char * src0_ptr = (const char *) src0->data + i00*nb00 + i01*nb01 + i02*nb02 + i03*nb03; \ \ - for (; e < th_end; e++) { \ - *((ELEM_TYPE *) dst_ptr) = *((const ELEM_TYPE *) src0_ptr); \ + const bool rows_contig = (nb00 == ELEM_SIZE) && (nb0 == ELEM_SIZE); \ \ - dst_ptr += nb0; \ - if (++i10 == ne0) { \ + while (e < th_end) { \ + uint32_t run = 1; \ + if (rows_contig) { \ + run = MIN(MIN(ne00 - i00, ne0 - i10), th_end - e); \ + hvx_copy_uu((uint8_t *) dst_ptr, (const uint8_t *) src0_ptr, run, ELEM_SIZE); \ + } else { \ + *((ELEM_TYPE *) dst_ptr) = *((const ELEM_TYPE *) src0_ptr); \ + } \ + e += run; \ + \ + dst_ptr += run * nb0; \ + i10 += run; \ + if (i10 == ne0) { \ i10 = 0; \ if (++i11 == ne1) { \ i11 = 0; \ @@ -165,8 +182,9 @@ static void cpy_thread_##NAME##_reshape(unsigned int nth, unsigned int ith, void dst_ptr = (char *) dst->data + i11*nb1 + i12*nb2 + i13*nb3; \ } \ \ - src0_ptr += nb00; \ - if (++i00 == ne00) { \ + src0_ptr += run * nb00; \ + i00 += run; \ + if (i00 == ne00) { \ i00 = 0; \ if (++i01 == ne01) { \ i01 = 0; \