sycl: fuse mul_mat(gate) + mul_mat(up) + GLU for q4_K dense FFN (#26779)

Measured on Arc Pro B70 (Battlemage, Level Zero), llama-bench -r 20, two
interleaved rounds, tg128:

    qwen2.5-3B-Instruct Q4_K_M    154.18 -> 158.53 t/s   +2.8%
    gemma-2-2b-it Q4_K_M          162.45 -> 165.62 t/s   +2.0%

llama-batched-bench on qwen2.5-3B, S_TG by batch size:

      B=1   142.72 -> 147.57 t/s    +3.4%
      B=2   243.72 -> 268.26 t/s   +10.1%
      B=4   359.58 -> 398.02 t/s   +10.7%
      B=8   449.75 -> 505.63 t/s   +12.4%

.github/workflows/build-cache.yml

@@ -119,27 +119,27 @@ jobs:
           version_major: ${{ env.OPENVINO_VERSION_MAJOR }}
           version_full: ${{ env.OPENVINO_VERSION_FULL }}
 
-  windows-2022-rocm-cache:
-    runs-on: windows-2022
+  # windows-2022-rocm-cache:
+  #   runs-on: windows-2022
 
-    env:
-      # Make sure this is in sync with release.yml and build-cuda-windows.yml
-      ROCM_VERSION: "7.14.0"
+  #   env:
+  #     # Make sure this is in sync with release.yml and build-cuda-windows.yml
+  #     ROCM_VERSION: "7.14.0"
 
-    steps:
-      - name: Clone
-        id: checkout
-        uses: actions/checkout@v6
+  #   steps:
+  #     - name: Clone
+  #       id: checkout
+  #       uses: actions/checkout@v6
 
-      - name: Setup Cache
-        uses: actions/cache@v5
-        id: cache-rocm
-        with:
-          path: C:\TheRock\build
-          key: rocm-wheels-${{ env.ROCM_VERSION }}-multi-arch-${{ runner.os }}
+  #     - name: Setup Cache
+  #       uses: actions/cache@v5
+  #       id: cache-rocm
+  #       with:
+  #         path: C:\TheRock\build
+  #         key: rocm-wheels-${{ env.ROCM_VERSION }}-multi-arch-${{ runner.os }}
 
-      - name: Setup ROCm
-        if: steps.cache-rocm.outputs.cache-hit != 'true'
-        uses: ./.github/actions/windows-setup-rocm
-        with:
-          version: ${{ env.ROCM_VERSION }}
+  #     - name: Setup ROCm
+  #       if: steps.cache-rocm.outputs.cache-hit != 'true'
+  #       uses: ./.github/actions/windows-setup-rocm
+  #       with:
+  #         version: ${{ env.ROCM_VERSION }}

.github/workflows/build-cmake-pkg.yml

@@ -5,7 +5,7 @@ on:
 
 jobs:
   linux:
-    runs-on: [self-hosted, Linux, CPU]
+    runs-on: [self-hosted, Linux]
     steps:
       - uses: actions/checkout@v6
         with:
@@ -21,15 +21,21 @@ jobs:
                 -DLLAMA_BUILD_TOOLS=OFF \
                 -DLLAMA_BUILD_EXAMPLES=OFF \
                 -DLLAMA_BUILD_APP=OFF \
+                -DLLAMA_BUILD_IS_DEV=OFF \
                 -DCMAKE_BUILD_TYPE=Release
-          cmake --build build --config Release
+          cmake --build build --config Release -j $(nproc)
           cmake --install build --prefix "$PREFIX" --config Release
 
           export LLAMA_CONFIG="$PREFIX"/lib/cmake/llama/llama-config.cmake
           tclsh <<'EOF'
           set build(commit)  [string trim [exec git rev-parse --short HEAD]]
           set build(number)  [string trim [exec git rev-list  --count HEAD]]
-          set build(version) "0.0.$build(number)"
+
+          set cmakelists [read [open "CMakeLists.txt" r]]
+          regexp {set\(LLAMA_VERSION_MAJOR\s+(\d+)\)} $cmakelists -> major
+          regexp {set\(LLAMA_VERSION_MINOR\s+(\d+)\)} $cmakelists -> minor
+          regexp {set\(LLAMA_VERSION_PATCH\s+(\d+)\)} $cmakelists -> patch
+          set build(version) "$major.$minor.$patch"
 
           set llamaconfig [read [open "$env(LLAMA_CONFIG)" r]]
           set checks [list "set\\(LLAMA_VERSION     \\s+$build(version)\\)" \
@@ -48,4 +54,4 @@ jobs:
 
           cd examples/simple-cmake-pkg
           cmake -S . -B build -DCMAKE_PREFIX_PATH="$PREFIX"/lib/cmake
-          cmake --build build
+          cmake --build build -j $(nproc)

.github/workflows/build-cpu.yml

@@ -94,8 +94,10 @@ jobs:
         id: cmake_build
         run: |
           cmake -B build \
+            -DGGML_NATIVE=OFF \
             -DLLAMA_FATAL_WARNINGS=ON \
-            -DGGML_RPC=ON
+            -DGGML_RPC=ON \
+            -DGGML_NATIVE=OFF
           time cmake --build build --config Release -j $(nproc)
 
       - name: Test

.github/workflows/build-cuda-windows.yml

@@ -97,15 +97,15 @@ jobs:
         id: checkout
         uses: actions/checkout@v6
 
-      - name: Cache ROCm Installation
-        uses: actions/cache@v5
-        id: cache-rocm
-        with:
-          path: C:\TheRock\build
-          key: rocm-wheels-${{ env.ROCM_VERSION }}-multi-arch-${{ runner.os }}
+      # - name: Cache ROCm Installation
+      #   uses: actions/cache@v5
+      #   id: cache-rocm
+      #   with:
+      #     path: C:\TheRock\build
+      #     key: rocm-wheels-${{ env.ROCM_VERSION }}-multi-arch-${{ runner.os }}
 
       - name: Setup ROCm
-        if: steps.cache-rocm.outputs.cache-hit != 'true'
+        # if: steps.cache-rocm.outputs.cache-hit != 'true'
         uses: ./.github/actions/windows-setup-rocm
         with:
           version: ${{ env.ROCM_VERSION }}

.github/workflows/build-sanitize.yml

@@ -39,9 +39,9 @@ jobs:
     strategy:
       matrix:
         include:
+        # thread and address doesn't run properly on some self hosted machines, so run it on Github instead
         - sanitizer: ADDRESS
-          machine: [self-hosted, X64, Linux]
-        # thread doesn't run properly on some self hosted machines, so run it on Github instead
+          machine: ubuntu-24.04
         - sanitizer: THREAD
           machine: ubuntu-24.04
         - sanitizer: UNDEFINED
@@ -54,14 +54,14 @@ jobs:
         id: checkout
         uses: actions/checkout@v6
 
-      - name: ccache
-        uses: ggml-org/ccache-action@v1.2.21
-        if: ${{ matrix.sanitizer == 'THREAD' }}
-        with:
-          key: ctest-thread-ubuntu-24.04
-          variant: ccache
-          evict-old-files: 1d
-          save: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
+      # - name: ccache
+      #   uses: ggml-org/ccache-action@v1.2.21
+      #   if: ${{ matrix.sanitizer != 'UNDEFINED' }}
+      #   with:
+      #     key: ctest-${{ matrix.sanitizer }}-ubuntu-24.04
+      #     variant: ccache
+      #     evict-old-files: 1d
+      #     save: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
 
       # with UNDEFINED sanitizer, we have to build in Debug to avoid GCC 13 false-positive warnings
       - name: Build (undefined)

.github/workflows/make-release.yml

@@ -0,0 +1,46 @@
+name: Make Release
+
+on:
+  workflow_dispatch:
+    inputs:
+      dry_run:
+        description: 'Dry run - validate without creating the tag'
+        required: true
+        type: boolean
+        default: true
+
+env:
+  GH_TOKEN: ${{ github.token }}
+
+permissions:
+  contents: write
+
+jobs:
+  make-release:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v6
+
+      - name: Run release checks
+        id: checks
+        run: bash scripts/make-release-checks.sh ${{ github.event.inputs.dry_run == 'true' && '--dry-run' || '' }}
+        env:
+          GITHUB_REPOSITORY: ${{ github.repository }}
+
+      - name: Create release tag
+        if: ${{ github.event.inputs.dry_run == 'false' }}
+        run: |
+          VERSION="${{ steps.checks.outputs.version }}"
+          git config user.name "github-actions[bot]"
+          git config user.email "github-actions[bot]@users.noreply.github.com"
+          git tag -a "${VERSION}" -m "Release ${VERSION}"
+          git push origin "${VERSION}"
+          echo "Created and pushed tag ${VERSION}"
+
+      - name: Dry run summary
+        if: ${{ github.event.inputs.dry_run == 'true' }}
+        run: |
+          echo "Dry run complete - all checks passed."
+          echo "Would have created tag: ${{ steps.checks.outputs.version }}"

.github/workflows/release.yml

@@ -750,6 +750,8 @@ jobs:
 
   windows-rocm:
     needs: [check-release]
+    if: ${{ needs.check-release.outputs.should_release == 'true' }}
+
     runs-on: windows-2022
 
     strategy:
@@ -772,15 +774,15 @@ jobs:
           key: windows-rocm-${{ matrix.ROCM_VERSION }}-${{ matrix.build }}
           evict-old-files: 1d
 
-      - name: Cache ROCm Installation
-        id: cache-rocm
-        uses: actions/cache@v5
-        with:
-          path: C:\TheRock\build
-          key: rocm-wheels-${{ matrix.ROCM_VERSION }}-multi-arch-${{ runner.os }}
+      # - name: Cache ROCm Installation
+      #   id: cache-rocm
+      #   uses: actions/cache@v5
+      #   with:
+      #     path: C:\TheRock\build
+      #     key: rocm-wheels-${{ matrix.ROCM_VERSION }}-multi-arch-${{ runner.os }}
 
       - name: Setup ROCm
-        if: steps.cache-rocm.outputs.cache-hit != 'true'
+        # if: steps.cache-rocm.outputs.cache-hit != 'true'
         uses: ./.github/actions/windows-setup-rocm
         with:
           version: ${{ matrix.ROCM_VERSION }}
@@ -1283,123 +1285,123 @@ jobs:
           path: llama-${{ steps.tag.outputs.name }}-bin-ubuntu-sycl-${{ matrix.build }}-x64.tar.gz
           name: llama-bin-ubuntu-sycl-${{ matrix.build }}-x64.tar.gz
 
-  ubuntu-22-rocm:
-    needs: [check-release, get-version]
-    if: ${{ needs.check-release.outputs.should_release == 'true' }}
+#   ubuntu-22-rocm:
+#     needs: [check-release, get-version]
+#     if: ${{ needs.check-release.outputs.should_release == 'true' }}
 
-    runs-on: ubuntu-22.04
+#     runs-on: ubuntu-22.04
 
-    permissions:
-      actions: write
+#     permissions:
+#       actions: write
 
-    strategy:
-      matrix:
-        include:
-          - ROCM_VERSION: "7.14.0"
-            gpu_targets: "gfx908;gfx90a;gfx942;gfx950;gfx1010;gfx1011;gfx1012;gfx1030;gfx1031;gfx1032;gfx1033;gfx1034;gfx1035;gfx1036;gfx1100;gfx1101;gfx1102;gfx1150;gfx1151;gfx1152;gfx1200;gfx1201"
-            build: 'x64'
+#     strategy:
+#       matrix:
+#         include:
+#           - ROCM_VERSION: "7.14.0"
+#             gpu_targets: "gfx908;gfx90a;gfx942;gfx950;gfx1010;gfx1011;gfx1012;gfx1030;gfx1031;gfx1032;gfx1033;gfx1034;gfx1035;gfx1036;gfx1100;gfx1101;gfx1102;gfx1150;gfx1151;gfx1152;gfx1200;gfx1201"
+#             build: 'x64'
 
-    steps:
-      - name: Clone
-        id: checkout
-        uses: actions/checkout@v6
-        with:
-          fetch-depth: 0
+#     steps:
+#       - name: Clone
+#         id: checkout
+#         uses: actions/checkout@v6
+#         with:
+#           fetch-depth: 0
 
-      - name: Setup Node.js
-        uses: actions/setup-node@v6
-        with:
-          node-version: "24"
-          cache: "npm"
-          cache-dependency-path: "tools/ui/package-lock.json"
+#       - name: Setup Node.js
+#         uses: actions/setup-node@v6
+#         with:
+#           node-version: "24"
+#           cache: "npm"
+#           cache-dependency-path: "tools/ui/package-lock.json"
 
-      - name: Free up disk space
-        uses: ggml-org/free-disk-space@v1.3.1
-        with:
-          tool-cache: true
+#       - name: Free up disk space
+#         uses: ggml-org/free-disk-space@v1.3.1
+#         with:
+#           tool-cache: true
 
-      - name: ccache
-        uses: ggml-org/ccache-action@v1.2.21
-        with:
-          key: release-ubuntu-22.04-rocm-${{ matrix.ROCM_VERSION }}
+#       # - name: ccache
+#       #   uses: ggml-org/ccache-action@v1.2.21
+#       #   with:
+#       #     key: release-ubuntu-22.04-rocm-${{ matrix.ROCM_VERSION }}
 
-      - name: Dependencies
-        id: depends
-        run: |
-          sudo apt install -y build-essential git cmake wget
+#       - name: Dependencies
+#         id: depends
+#         run: |
+#           sudo apt install -y build-essential git cmake wget
 
-      - name: Setup TheRock with Wheels
-        id: therock_env
-        run: |
-          # Create Python virtual environment
-          python3 -m venv .venv
-          source .venv/bin/activate
+#       - name: Setup TheRock with Wheels
+#         id: therock_env
+#         run: |
+#           # Create Python virtual environment
+#           python3 -m venv .venv
+#           source .venv/bin/activate
 
-          # Install ROCm wheels for build
-          # libraries = HIP runtime and CMake configs needed for linking
-          # devel = compilers, headers, static libs
-          python -m pip install --upgrade pip
-          python -m pip install --index-url https://repo.amd.com/rocm/whl-multi-arch/ "rocm[libraries,devel]==${{ matrix.ROCM_VERSION }}"
+#           # Install ROCm wheels for build
+#           # libraries = HIP runtime and CMake configs needed for linking
+#           # devel = compilers, headers, static libs
+#           python -m pip install --upgrade pip
+#           python -m pip install --index-url https://repo.amd.com/rocm/whl-multi-arch/ "rocm[libraries,devel]==${{ matrix.ROCM_VERSION }}"
 
-          # Get ROCm installation paths using the rocm-sdk CLI tool
-          ROCM_PATH=$(rocm-sdk path --root)
-          CMAKE_PATH=$(rocm-sdk path --cmake)
-          BIN_PATH=$(rocm-sdk path --bin)
-          echo "ROCM_PATH=$ROCM_PATH"
-          echo "CMAKE_PATH=$CMAKE_PATH"
-          echo "BIN_PATH=$BIN_PATH"
+#           # Get ROCm installation paths using the rocm-sdk CLI tool
+#           ROCM_PATH=$(rocm-sdk path --root)
+#           CMAKE_PATH=$(rocm-sdk path --cmake)
+#           BIN_PATH=$(rocm-sdk path --bin)
+#           echo "ROCM_PATH=$ROCM_PATH"
+#           echo "CMAKE_PATH=$CMAKE_PATH"
+#           echo "BIN_PATH=$BIN_PATH"
 
-          # Set environment variables
-          echo "ROCM_PATH=$ROCM_PATH" >> $GITHUB_ENV
-          echo "CMAKE_PREFIX_PATH=$CMAKE_PATH" >> $GITHUB_ENV
-          echo "HIP_PATH=$ROCM_PATH" >> $GITHUB_ENV
-          echo "PATH=$BIN_PATH:${PATH}" >> $GITHUB_ENV
-          echo "LD_LIBRARY_PATH=$ROCM_PATH/lib:${LD_LIBRARY_PATH:-}" >> $GITHUB_ENV
+#           # Set environment variables
+#           echo "ROCM_PATH=$ROCM_PATH" >> $GITHUB_ENV
+#           echo "CMAKE_PREFIX_PATH=$CMAKE_PATH" >> $GITHUB_ENV
+#           echo "HIP_PATH=$ROCM_PATH" >> $GITHUB_ENV
+#           echo "PATH=$BIN_PATH:${PATH}" >> $GITHUB_ENV
+#           echo "LD_LIBRARY_PATH=$ROCM_PATH/lib:${LD_LIBRARY_PATH:-}" >> $GITHUB_ENV
 
-          # Keep venv activated for subsequent steps
-          echo "$(pwd)/.venv/bin" >> $GITHUB_PATH
+#           # Keep venv activated for subsequent steps
+#           echo "$(pwd)/.venv/bin" >> $GITHUB_PATH
 
-      - name: Build with native CMake HIP support
-        id: cmake_build
-        run: |
-          cmake -B build -S . \
-            -DCMAKE_HIP_COMPILER="$(hipconfig -l)/clang" \
-            -DCMAKE_BUILD_TYPE=Release \
-            -DGGML_BACKEND_DL=ON \
-            -DGGML_NATIVE=OFF \
-            -DCMAKE_INSTALL_RPATH='$ORIGIN' \
-            -DCMAKE_BUILD_WITH_INSTALL_RPATH=ON \
-            -DGGML_CPU_ALL_VARIANTS=ON \
-            -DGPU_TARGETS="${{ matrix.gpu_targets }}" \
-            -DGGML_HIP=ON \
-            -DHIP_PLATFORM=amd \
-            -DHF_UI_VERSION=${{ needs.get-version.outputs.ui_version }} \
-            ${{ env.CMAKE_ARGS }}
-          cmake --build build --config Release -j $(nproc)
+#       - name: Build with native CMake HIP support
+#         id: cmake_build
+#         run: |
+#           cmake -B build -S . \
+#             -DCMAKE_HIP_COMPILER="$(hipconfig -l)/clang" \
+#             -DCMAKE_BUILD_TYPE=Release \
+#             -DGGML_BACKEND_DL=ON \
+#             -DGGML_NATIVE=OFF \
+#             -DCMAKE_INSTALL_RPATH='$ORIGIN' \
+#             -DCMAKE_BUILD_WITH_INSTALL_RPATH=ON \
+#             -DGGML_CPU_ALL_VARIANTS=ON \
+#             -DGPU_TARGETS="${{ matrix.gpu_targets }}" \
+#             -DGGML_HIP=ON \
+#             -DHIP_PLATFORM=amd \
+#             -DHF_UI_VERSION=${{ needs.get-version.outputs.ui_version }} \
+#             ${{ env.CMAKE_ARGS }}
+#           cmake --build build --config Release -j $(nproc)
 
-      - name: ccache-clear
-        uses: ./.github/actions/ccache-clear
-        with:
-          key: release-ubuntu-22.04-rocm-${{ matrix.ROCM_VERSION }}
+#       # - name: ccache-clear
+#       #   uses: ./.github/actions/ccache-clear
+#       #   with:
+#       #     key: release-ubuntu-22.04-rocm-${{ matrix.ROCM_VERSION }}
 
-      - name: Determine tag name
-        id: tag
-        uses: ./.github/actions/get-tag-name
+#       - name: Determine tag name
+#         id: tag
+#         uses: ./.github/actions/get-tag-name
 
-      - name: Get ROCm short version
-        run: echo "ROCM_VERSION_SHORT=$(echo '${{ matrix.ROCM_VERSION }}' | cut -d '.' -f 1,2)" >> $GITHUB_ENV
+#       - name: Get ROCm short version
+#         run: echo "ROCM_VERSION_SHORT=$(echo '${{ matrix.ROCM_VERSION }}' | cut -d '.' -f 1,2)" >> $GITHUB_ENV
 
-      - name: Pack artifacts
-        id: pack_artifacts
-        run: |
-          cp LICENSE ./build/bin/
-          tar -czvf llama-${{ steps.tag.outputs.name }}-bin-ubuntu-rocm-${{ env.ROCM_VERSION_SHORT }}-${{ matrix.build }}.tar.gz --transform "s,^\.,llama-${{ steps.tag.outputs.name }}," -C ./build/bin .
+#       - name: Pack artifacts
+#         id: pack_artifacts
+#         run: |
+#           cp LICENSE ./build/bin/
+#           tar -czvf llama-${{ steps.tag.outputs.name }}-bin-ubuntu-rocm-${{ env.ROCM_VERSION_SHORT }}-${{ matrix.build }}.tar.gz --transform "s,^\.,llama-${{ steps.tag.outputs.name }}," -C ./build/bin .
 
-      - name: Upload artifacts
-        uses: actions/upload-artifact@v6
-        with:
-          path: llama-${{ steps.tag.outputs.name }}-bin-ubuntu-rocm-${{ env.ROCM_VERSION_SHORT }}-${{ matrix.build }}.tar.gz
-          name: llama-bin-ubuntu-rocm-${{ env.ROCM_VERSION_SHORT }}-${{ matrix.build }}.tar.gz
+#       - name: Upload artifacts
+#         uses: actions/upload-artifact@v6
+#         with:
+#           path: llama-${{ steps.tag.outputs.name }}-bin-ubuntu-rocm-${{ env.ROCM_VERSION_SHORT }}-${{ matrix.build }}.tar.gz
+#           name: llama-bin-ubuntu-rocm-${{ env.ROCM_VERSION_SHORT }}-${{ matrix.build }}.tar.gz
 
   ios-xcode:
     needs: [check-release, get-version]
@@ -1576,7 +1578,7 @@ jobs:
       #- windows-sycl
       - windows-rocm
       - windows-openvino
-      - ubuntu-22-rocm
+      #- ubuntu-22-rocm
       - ubuntu-cpu
       - ubuntu-vulkan
       - ubuntu-24-openvino
@@ -1686,7 +1688,7 @@ jobs:
             - [Ubuntu s390x (CPU)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-s390x.tar.gz)
             - [Ubuntu x64 (Vulkan)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-vulkan-x64.tar.gz)
             - [Ubuntu arm64 (Vulkan)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-vulkan-arm64.tar.gz)
-            - [Ubuntu x64 (ROCm 7.14)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-rocm-7.14-x64.tar.gz)
+            - Ubuntu x64 (ROCm 7.14)[DISABLED](https://github.com/ggml-org/llama.cpp/pull/26969)
             - [Ubuntu x64 (OpenVINO)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-openvino-${{ needs.ubuntu-24-openvino.outputs.openvino_version }}-x64.tar.gz)
             - [Ubuntu x64 (SYCL FP32)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-sycl-fp32-x64.tar.gz)
             - [Ubuntu x64 (SYCL FP16)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-sycl-fp16-x64.tar.gz)

.github/workflows/winget.yml

@@ -19,6 +19,8 @@ jobs:
         run: |
           cargo binstall komac@2.16.0 -y
 
+      # TODO: This should later be updated to publish releases instead of
+      # development release builds.
       - name: Find latest release
         id: find_latest_release
         uses: actions/github-script@v8

CMakeLists.txt

@@ -2,6 +2,26 @@ cmake_minimum_required(VERSION 3.14...3.28) # for add_link_options and implicit
 project("llama.cpp" C CXX)
 include(CheckIncludeFileCXX)
 
+### llama.cpp version
+set(LLAMA_VERSION_MAJOR 0)
+set(LLAMA_VERSION_MINOR 1)
+set(LLAMA_VERSION_PATCH 0)
+set(LLAMA_VERSION_BASE "${LLAMA_VERSION_MAJOR}.${LLAMA_VERSION_MINOR}.${LLAMA_VERSION_PATCH}")
+
+# whether this is a development/nightly build
+# set this to OFF when making a release from a release tag (vX.Y.Z)
+# ref: https://github.com/ggml-org/ggml/discussions/1579
+option(LLAMA_BUILD_IS_DEV "llama: dev build" ON)
+
+if (LLAMA_BUILD_IS_DEV)
+    set(LLAMA_VERSION "${LLAMA_VERSION_BASE}-dev")
+else()
+    # TODO: check that the current commit is tagged correctly according to the version specified above
+    set(LLAMA_VERSION "${LLAMA_VERSION_BASE}")
+endif()
+
+message(STATUS "llama.cpp version: ${LLAMA_VERSION}")
+
 #set(CMAKE_WARN_DEPRECATED YES)
 set(CMAKE_WARN_UNUSED_CLI YES)
 
@@ -24,9 +44,6 @@ if (CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR)
     set(LLAMA_STANDALONE ON)
 
     include(git-vars)
-
-    # configure project version
-    # TODO
 else()
     set(LLAMA_STANDALONE OFF)
 endif()
@@ -139,7 +156,6 @@ endif()
 if (NOT DEFINED LLAMA_BUILD_COMMIT)
     set(LLAMA_BUILD_COMMIT        ${BUILD_COMMIT})
 endif()
-set(LLAMA_INSTALL_VERSION 0.0.${LLAMA_BUILD_NUMBER})
 
 # override ggml options
 set(GGML_ALL_WARNINGS   ${LLAMA_ALL_WARNINGS})
@@ -275,12 +291,12 @@ configure_package_config_file(
               LLAMA_BIN_INSTALL_DIR )
 
 write_basic_package_version_file(
-        ${CMAKE_CURRENT_BINARY_DIR}/llama-version.cmake
-    VERSION ${LLAMA_INSTALL_VERSION}
+        ${CMAKE_CURRENT_BINARY_DIR}/llama-config-version.cmake
+        VERSION ${LLAMA_VERSION}
     COMPATIBILITY SameMajorVersion)
 
 install(FILES ${CMAKE_CURRENT_BINARY_DIR}/llama-config.cmake
-              ${CMAKE_CURRENT_BINARY_DIR}/llama-version.cmake
+              ${CMAKE_CURRENT_BINARY_DIR}/llama-config-version.cmake
         DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/llama)
 
 configure_file(cmake/llama.pc.in

README.md

@@ -106,6 +106,7 @@ The `llama.cpp` project is build on top of the [ggml](https://github.com/ggml-or
 - [XCFramework](docs/xcframework.md)
 - [Completions](docs/completions.md)
 - [Models](docs/models.md)
+- [Release process](docs/release.md)
 
 ## Contributing
 

app/llama.cpp

@@ -1,5 +1,7 @@
 #include "build-info.h"
 
+#include "llama.h"
+
 #include <cstdio>
 #include <cstdlib>
 #include <string>
@@ -77,12 +79,12 @@ static const command cmds[] = {
 
 #undef UPDATE_HIDDEN
 
-static int version(int argc, char ** argv) {
-    printf("%s\n", llama_build_info());
+static int version(int /*argc*/, char ** /*argv*/) {
+    llama_print_build_info(llama_version());
     return 0;
 }
 
-static int licenses(int argc, char ** argv) {
+static int licenses(int /*argc*/, char ** /*argv*/) {
     for (int i = 0; LICENSES[i]; ++i) {
         printf("%s\n", LICENSES[i]);
     }

cmake/llama-config.cmake.in

@@ -1,4 +1,4 @@
-set(LLAMA_VERSION      @LLAMA_INSTALL_VERSION@)
+set(LLAMA_VERSION      @LLAMA_VERSION@)
 set(LLAMA_BUILD_COMMIT @LLAMA_BUILD_COMMIT@)
 set(LLAMA_BUILD_NUMBER @LLAMA_BUILD_NUMBER@)
 set(LLAMA_SHARED_LIB   @BUILD_SHARED_LIBS@)

cmake/llama.pc.in

@@ -5,6 +5,6 @@ includedir=@CMAKE_INSTALL_FULL_INCLUDEDIR@
 
 Name: llama
 Description: Port of Facebook's LLaMA model in C/C++
-Version: @LLAMA_INSTALL_VERSION@
+Version: @LLAMA_VERSION@
 Libs: -L${libdir} -lggml -lggml-base -lllama
 Cflags: -I${includedir}

common/CMakeLists.txt

@@ -121,8 +121,8 @@ add_library(${TARGET}
     )
 
 set_target_properties(${TARGET} PROPERTIES
-    VERSION ${LLAMA_INSTALL_VERSION}
-    SOVERSION 0
+    VERSION ${LLAMA_VERSION_BASE}
+    SOVERSION ${LLAMA_VERSION_MAJOR}
     MACHO_CURRENT_VERSION 0 # keep macOS linker from seeing oversized version number
 )
 

common/arg.cpp

@@ -35,6 +35,7 @@
 #include <regex>
 #include <set>
 #include <string>
+#include <system_error>
 #include <thread> // for hardware_concurrency
 #include <vector>
 
@@ -560,6 +561,15 @@ void common_models_handler_apply(common_models_handler & handler, common_params
         }
     }
 
+    // infer the speculative type from the draft GGUF metadata when none is requested
+    // note: reads only the first split - sharded drafts need an explicit --spec-type
+    if (spec_types_is_default(params) && !params.speculative.draft.mparams.path.empty()) {
+        const auto types_gguf = common_speculative_types_from_gguf(params.speculative.draft.mparams.path);
+        if (!types_gguf.empty()) {
+            params.speculative.types = types_gguf;
+        }
+    }
+
     // when a sidecar type is requested, the draft repo resolves to its sidecar instead of a full model
     const bool spec_sidecar_found = !plan_spec.mtp.local_path.empty() ||
                                     !plan_spec.dflash.local_path.empty() ||
@@ -704,12 +714,61 @@ void common_models_handler_apply(common_models_handler & handler, common_params
 // CLI argument parsing functions
 //
 
+// apply config files (if present), a later file overrides an earlier one:
+// 1. system-wide: /etc/llama.cpp/config.ini (%PROGRAMDATA%\llama.cpp\config.ini on windows)
+// 2. user-level: ${XDG_CONFIG_HOME:-~/.config}/llama.cpp/config.ini (%APPDATA%\llama.cpp\config.ini on windows)
+static void common_params_apply_system_config(common_params & params, llama_example ex) {
+    std::vector<std::string> paths;
+
+#if defined(_WIN32)
+    const std::string program_data = common_get_env("PROGRAMDATA");
+    if (!program_data.empty()) {
+        paths.push_back(program_data + "\\llama.cpp\\config.ini");
+    }
+#else
+    paths.push_back("/etc/llama.cpp/config.ini");
+#endif
+
+    try {
+        paths.push_back(fs_get_config_directory() + "config.ini");
+    } catch (const std::exception & e) {
+        LOG_DBG("cannot read user-level config file, skipping: %s\n", e.what());
+    }
+
+    std::vector<std::string> found;
+    for (const auto & path : paths) {
+        std::error_code ec;
+        if (std::filesystem::exists(path, ec)) {
+            found.push_back(path);
+        }
+    }
+    if (found.empty()) {
+        return;
+    }
+
+    common_preset_context ctx(ex);
+    ctx.ignore_unknown_keys = true; // the same config file is shared by all programs
+    for (const auto & path : found) {
+        LOG_INF("using config file: %s\n", path.c_str());
+        common_preset global;
+        common_presets presets = ctx.load_from_ini(path, global);
+        global.apply_to_params(params);
+        auto it = presets.find(COMMON_PRESET_DEFAULT_NAME);
+        if (it != presets.end()) {
+            it->second.apply_to_params(params);
+        }
+    }
+}
+
 static bool common_params_parse_ex(int argc, char ** argv, common_params_context & ctx_arg) {
     common_params & params = ctx_arg.params;
 
     // setup log directly from params.verbosity: see tools/cli/cli.cpp
     common_log_set_verbosity_thold(params.verbosity);
 
+    // config file applies first, so env variables and CLI arguments override it
+    common_params_apply_system_config(params, ctx_arg.ex);
+
     std::unordered_map<std::string, std::pair<common_arg *, bool>> arg_to_options;
     for (auto & opt : ctx_arg.options) {
         for (const auto & arg : opt.args) {
@@ -1390,8 +1449,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
         {"--version"},
         "show version and build info",
         [](common_params &) {
-            fprintf(stderr, "version: %d (%s)\n", llama_build_number(), llama_commit());
-            fprintf(stderr, "built with %s for %s\n", llama_compiler(), llama_build_target());
+            llama_print_build_info(llama_version());
             exit(0);
         }
     ));

common/build-info.cpp.in

@@ -29,7 +29,7 @@ const char * llama_build_info(void) {
     return s.c_str();
 }
 
-void llama_print_build_info(void) {
-    fprintf(stderr, "%s: build = %d (%s)\n",      __func__, llama_build_number(), llama_commit());
-    fprintf(stderr, "%s: built with %s for %s\n", __func__, llama_compiler(), llama_build_target());
+void llama_print_build_info(const char * llama_version) {
+    fprintf(stderr, "version: %s (build %d, commit %s)\n", llama_version, llama_build_number(), llama_commit());
+    fprintf(stderr, "built with %s for %s\n", llama_compiler(), llama_build_target());
 }

common/build-info.h

@@ -8,4 +8,4 @@ const char * llama_compiler(void);
 const char * llama_build_target(void);
 const char * llama_build_info(void);
 
-void llama_print_build_info(void);
+void llama_print_build_info(const char *);

common/chat-peg-parser.cpp

@@ -594,9 +594,7 @@ common_peg_parser common_chat_peg_builder::python_style_tool_calls(
 
                 // Full argument: name="value" or name=value
                 auto arg_rule = tool_arg(
-                    tool_arg_open(eps()) +
-                    tool_arg_name(arg_name_parser) +
-                    literal("=") +
+                    tool_arg_open(tool_arg_name(arg_name_parser) + literal("=")) +
                     arg_value_parser +
                     tool_arg_close(eps())
                 );

common/chat.cpp

@@ -3155,7 +3155,8 @@ static common_chat_params common_chat_params_init_muse_glimmer(const common_chat
         auto analysis = p.ref("analysis");
 
         auto recipient  = p.optional(p.literal(" to=user"));
-        auto final_msg  = p.rule("final", recipient + p.literal("<|message|>") + p.content(p.until("<|eot|>")));
+        auto final_msg  = p.rule("final", recipient + p.literal("<|message|>") +
+                                              p.content(p.until_one_of({ "<|eot|>", "<|eom|>" })));
 
         if (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE) {
             auto string_value = p.ac(
@@ -3211,7 +3212,8 @@ static common_chat_params common_chat_params_init_muse_glimmer(const common_chat
             if (inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED) {
                 return p.zero_or_more(start + analysis) + start + tool_calls;
             }
-            return p.zero_or_more(start + analysis) + start + (tool_calls | final_msg);
+            auto trailing_calls = p.optional(p.literal("<|eom|>") + start + tool_calls);
+            return p.zero_or_more(start + analysis) + start + (tool_calls | (final_msg + trailing_calls));
         }
 
         return p.zero_or_more(start + analysis) + start + final_msg;

common/common.cpp

@@ -1019,20 +1019,21 @@ std::string fs_get_cache_directory() {
     std::string cache_directory = "";
     auto ensure_trailing_slash = [](std::string p) {
         // Make sure to add trailing slash
-        if (p.back() != DIRECTORY_SEPARATOR) {
+        if (p.empty() || p.back() != DIRECTORY_SEPARATOR) {
             p += DIRECTORY_SEPARATOR;
         }
         return p;
     };
-    if (getenv("LLAMA_CACHE")) {
-        cache_directory = std::getenv("LLAMA_CACHE");
-    } else {
+    cache_directory = common_get_env("LLAMA_CACHE");
+    if (cache_directory.empty()) {
 #if defined(__linux__) || defined(__FreeBSD__) || defined(_AIX) || \
         defined(__OpenBSD__) || defined(__NetBSD__)
-        if (std::getenv("XDG_CACHE_HOME")) {
-            cache_directory = std::getenv("XDG_CACHE_HOME");
-        } else if (std::getenv("HOME")) {
-            cache_directory = std::getenv("HOME") + std::string("/.cache/");
+        const std::string xdg_cache_home = common_get_env("XDG_CACHE_HOME");
+        const std::string home           = common_get_env("HOME");
+        if (!xdg_cache_home.empty()) {
+            cache_directory = xdg_cache_home;
+        } else if (!home.empty()) {
+            cache_directory = home + "/.cache/";
         } else {
 #if defined(__linux__)
             /* no $HOME is defined, fallback to getpwuid */
@@ -1047,9 +1048,16 @@ std::string fs_get_cache_directory() {
 #endif /* defined(__linux__) */
         }
 #elif defined(__APPLE__)
-        cache_directory = std::getenv("HOME") + std::string("/Library/Caches/");
+        cache_directory = common_get_env("HOME");
+        if (cache_directory.empty()) {
+            throw std::runtime_error("Failed to find $HOME directory");
+        }
+        cache_directory += "/Library/Caches/";
 #elif defined(_WIN32)
-        cache_directory = std::getenv("LOCALAPPDATA");
+        cache_directory = common_get_env("LOCALAPPDATA");
+        if (cache_directory.empty()) {
+            throw std::runtime_error("Failed to find %LOCALAPPDATA% directory");
+        }
 #elif defined(__EMSCRIPTEN__)
         GGML_ABORT("not implemented on this platform");
 #else
@@ -1061,6 +1069,51 @@ std::string fs_get_cache_directory() {
     return ensure_trailing_slash(cache_directory);
 }
 
+std::string fs_get_config_directory() {
+    std::string config_directory = "";
+    auto ensure_trailing_slash = [](std::string p) {
+        if (p.empty() || p.back() != DIRECTORY_SEPARATOR) {
+            p += DIRECTORY_SEPARATOR;
+        }
+        return p;
+    };
+#if defined(__linux__) || defined(__FreeBSD__) || defined(_AIX) || \
+        defined(__OpenBSD__) || defined(__NetBSD__) || defined(__APPLE__)
+    const std::string xdg_config_home = common_get_env("XDG_CONFIG_HOME");
+    const std::string home            = common_get_env("HOME");
+    if (!xdg_config_home.empty()) {
+        config_directory = xdg_config_home;
+    } else if (!home.empty()) {
+        config_directory = home + "/.config/";
+    } else {
+#if defined(__linux__)
+        /* no $HOME is defined, fallback to getpwuid */
+        struct passwd *pw = getpwuid(getuid());
+        if ((!pw) || (!pw->pw_dir)) {
+            throw std::runtime_error("Failed to find $HOME directory");
+        }
+
+        config_directory = std::string(pw->pw_dir) + std::string("/.config/");
+#else
+        throw std::runtime_error("Failed to find $HOME directory");
+#endif
+    }
+#elif defined(_WIN32)
+    config_directory = common_get_env("APPDATA");
+    if (config_directory.empty()) {
+        throw std::runtime_error("Failed to find %APPDATA% directory");
+    }
+#elif defined(__EMSCRIPTEN__)
+    // caller decides what to do when there is no config directory
+    throw std::runtime_error("not implemented on this platform");
+#else
+#  error Unknown architecture
+#endif
+    config_directory = ensure_trailing_slash(config_directory);
+    config_directory += "llama.cpp";
+    return ensure_trailing_slash(config_directory);
+}
+
 std::string fs_get_cache_file(const std::string & filename) {
     GGML_ASSERT(filename.find(DIRECTORY_SEPARATOR) == std::string::npos);
     std::string cache_directory = fs_get_cache_directory();
@@ -1222,6 +1275,8 @@ struct common_init_result::impl {
 
     // note: the order in which model, context, etc. are declared matters because their destructors will be called bottom-to-top
 
+    common_threadpools threadpools;
+
     llama_model_ptr   model;
     llama_context_ptr context;
 
@@ -1323,6 +1378,10 @@ common_init_result::common_init_result(common_params & params, bool model_only)
     }
 
     pimpl->context.reset(lctx);
+
+    set_process_priority(params.cpuparams.priority);
+
+    pimpl->threadpools.init(lctx, params);
 }
 
 llama_model * common_init_result::model() {
@@ -1671,6 +1730,10 @@ struct llama_context_params common_context_params_to_llama(const common_params &
     return cparams;
 }
 
+//
+// Threadpool utils
+//
+
 struct ggml_threadpool_params ggml_threadpool_params_from_cpu_params(const common_cpu_params & params) {
     struct ggml_threadpool_params tpp;
 
@@ -1687,6 +1750,56 @@ struct ggml_threadpool_params ggml_threadpool_params_from_cpu_params(const commo
     return tpp;
 }
 
+common_threadpools::~common_threadpools() {
+    if (!free_fn) {
+        return;
+    }
+    free_fn(threadpool);
+    free_fn(threadpool_batch);
+}
+
+void common_threadpools::init(llama_context * ctx, const common_params & params) {
+    GGML_ASSERT(!threadpool);
+    GGML_ASSERT(!threadpool_batch);
+
+    COM_INF("llama threadpool init, n_threads = %d\n", (int) params.cpuparams.n_threads);
+
+    auto * cpu_dev = ggml_backend_dev_by_type(GGML_BACKEND_DEVICE_TYPE_CPU);
+    if (!cpu_dev) {
+        COM_WRN("%s", "no CPU backend found\n");
+        return;
+    }
+    auto * reg = ggml_backend_dev_backend_reg(cpu_dev);
+    auto * ggml_threadpool_new_fn = (decltype(ggml_threadpool_new) *) ggml_backend_reg_get_proc_address(reg, "ggml_threadpool_new");
+    free_fn = (decltype(ggml_threadpool_free) *) ggml_backend_reg_get_proc_address(reg, "ggml_threadpool_free");
+
+    struct ggml_threadpool_params tpp_batch =
+            ggml_threadpool_params_from_cpu_params(params.cpuparams_batch);
+    struct ggml_threadpool_params tpp =
+            ggml_threadpool_params_from_cpu_params(params.cpuparams);
+
+    if (!ggml_threadpool_params_match(&tpp, &tpp_batch)) {
+        threadpool_batch = ggml_threadpool_new_fn(&tpp_batch);
+        if (!threadpool_batch) {
+            COM_WRN("batch threadpool create failed : n_threads %d\n", tpp_batch.n_threads);
+            return;
+        }
+
+        // start the non-batch threadpool in the paused state
+        tpp.paused = true;
+    }
+
+    threadpool = ggml_threadpool_new_fn(&tpp);
+    if (!threadpool) {
+        COM_WRN("threadpool create failed : n_threads %d\n", tpp.n_threads);
+        free_fn(threadpool_batch);
+        threadpool_batch = nullptr;
+        return;
+    }
+
+    llama_attach_threadpool(ctx, threadpool, threadpool_batch);
+}
+
 //
 // Batch utils
 //

common/common.h

@@ -881,6 +881,7 @@ bool fs_is_directory(const std::string & path);
 
 std::string fs_get_cache_directory();
 std::string fs_get_cache_file(const std::string & filename);
+std::string fs_get_config_directory();
 
 struct common_file_info {
     std::string path;
@@ -928,9 +929,8 @@ using common_init_result_ptr = std::unique_ptr<common_init_result>;
 
 common_init_result_ptr common_init_from_params(common_params & params, bool model_only = false);
 
-struct llama_model_params     common_model_params_to_llama  (      common_params & params);
-struct llama_context_params   common_context_params_to_llama(const common_params & params);
-struct ggml_threadpool_params ggml_threadpool_params_from_cpu_params(const common_cpu_params & params);
+struct llama_model_params   common_model_params_to_llama  (      common_params & params);
+struct llama_context_params common_context_params_to_llama(const common_params & params);
 
 // clear LoRA adapters from context, then apply new list of adapters
 void common_set_adapter_lora(struct llama_context * ctx, std::vector<common_adapter_lora_info> & lora);
@@ -941,6 +941,28 @@ std::string common_get_model_endpoint();
 // for testing purposes
 char * common_get_model_or_exit(int, char*[]);
 
+//
+// Threadpool utils
+//
+
+struct ggml_threadpool_params ggml_threadpool_params_from_cpu_params(const common_cpu_params & params);
+
+struct common_threadpools {
+    common_threadpools() = default;
+    ~common_threadpools();
+
+    common_threadpools(const common_threadpools &) = delete;
+    common_threadpools & operator=(const common_threadpools &) = delete;
+
+    void init(llama_context * ctx, const common_params & params);
+
+private:
+    ggml_threadpool * threadpool       = nullptr;
+    ggml_threadpool * threadpool_batch = nullptr;
+
+    decltype(ggml_threadpool_free) * free_fn = nullptr;
+};
+
 //
 // Context utils
 //

common/preset.cpp

@@ -322,6 +322,8 @@ common_presets common_preset_context::load_from_ini(const std::string & path, co
                     preset.options[opt] = value;
                 }
                 LOG_DBG("accepted option: %s = %s\n", key.c_str(), preset.options[opt].c_str());
+            } else if (ignore_unknown_keys) {
+                LOG_WRN("ignoring option '%s' from %s: not supported by this program\n", key.c_str(), path.c_str());
             } else {
                 throw std::runtime_error(string_format(
                     "option '%s' not recognized in preset '%s'",

common/preset.h

@@ -59,6 +59,10 @@ struct common_preset_context {
     bool filter_allowed_keys = false;
     std::set<std::string> allowed_keys;
 
+    // if true, options unknown to the current example are skipped instead of being an error
+    // used for config files shared by all binaries, where each binary only knows a subset of options
+    bool ignore_unknown_keys = false;
+
     // if only_remote_allowed is true, only accept whitelisted keys
     common_preset_context(llama_example ex);
 

common/speculative.cpp

@@ -2,6 +2,7 @@
 
 #include "common.h"
 #include "ggml.h"
+#include "ggml-cpp.h"
 #include "llama.h"
 #include "log.h"
 #include "ngram-cache.h"
@@ -171,12 +172,6 @@ struct common_speculative_impl {
     // (optional) serialize/restore per-seq internal state (e.g. eagle3's deferred boundary).
     virtual bool get_state(llama_seq_id /*seq_id*/, std::vector<uint8_t> & /*data*/) const { return false; }
     virtual void set_state(llama_seq_id /*seq_id*/, const std::vector<uint8_t> & /*data*/) {}
-
-    // true if this implementation requires the target context to extract post-norm embeddings
-    virtual bool need_embd() const = 0;
-
-    // true if this implementation requires the target context to extract pre-norm embeddings
-    virtual bool need_embd_nextn() const { return false; }
 };
 
 struct common_speculative_impl_draft_simple : public common_speculative_impl {
@@ -193,6 +188,10 @@ struct common_speculative_impl_draft_simple : public common_speculative_impl {
         auto * ctx_dft = this->params.ctx_dft;
         auto * ctx_tgt = this->params.ctx_tgt;
 
+        if (!ctx_dft) {
+            throw std::runtime_error("draft-simple requires a draft context");
+        }
+
         SPC_TRC("%s", "adding speculative implementation 'draft-simple'\n");
         SPC_TRC("- n_max=%d, n_min=%d, p_min=%f\n", this->params.n_max, this->params.n_min, this->params.p_min);
         SPC_TRC("- gpu_layers=%d, cache_k=%s, cache_v=%s, ctx_tgt=%s, ctx_dft=%s, devices=[%s]\n",
@@ -385,10 +384,6 @@ struct common_speculative_impl_draft_simple : public common_speculative_impl {
     void accept(llama_seq_id /*seq_id*/, uint16_t /*n_accepted*/, bool /*is_other*/) override {
         // noop
     }
-
-    bool need_embd() const override {
-        return false;
-    }
 };
 
 
@@ -907,10 +902,6 @@ struct common_speculative_impl_draft_eagle3 : public common_speculative_impl {
         pending_g_last[seq_id].resize(n_embd_dec);
         std::memcpy(pending_g_last[seq_id].data(), data.data() + sizeof(llama_pos), (size_t) n_embd_dec * sizeof(float));
     }
-
-    bool need_embd() const override {
-        return false;
-    }
 };
 
 // DFlash: block-diffusion drafting with a draft-side KV cache injection
@@ -922,6 +913,9 @@ struct common_speculative_impl_draft_dflash : public common_speculative_impl {
 
     std::vector<common_sampler_ptr> smpls;
 
+    // backend sampler chain per seq, attached to ctx_dft
+    std::vector<llama_sampler *> backend_chains;
+
     int32_t n_embd_dec = 0;  // draft hidden size
     int32_t n_embd_enc = 0;  // target_layer_ids_n * target_hidden_size
     int32_t n_embd_tgt = 0;  // target model hidden size
@@ -995,6 +989,22 @@ struct common_speculative_impl_draft_dflash : public common_speculative_impl {
             s.reset(common_sampler_init(model_dft, sparams));
         }
 
+        // offload draft sampling to the backend
+        backend_chains.assign(n_seq, nullptr);
+        if (this->params.backend_sampling) {
+            for (llama_seq_id seq_id = 0; seq_id < (llama_seq_id) n_seq; ++seq_id) {
+                llama_sampler * chain = llama_sampler_chain_init(llama_sampler_chain_default_params());
+                llama_sampler_chain_add(chain, llama_sampler_init_top_k(10));
+
+                if (!llama_set_sampler(ctx_dft, seq_id, chain)) {
+                    SPC_WRN("backend offload failed for seq_id=%d; using CPU sampler\n", (int) seq_id);
+                    llama_sampler_free(chain);
+                    chain = nullptr;
+                }
+                backend_chains[seq_id] = chain;
+            }
+        }
+
         // turn on extraction of the target layers' input embeddings
         for (uint32_t k = 0; k < target_layer_ids_n; ++k) {
             llama_set_embeddings_layer_inp(ctx_tgt, (uint32_t) target_layer_ids[k], true);
@@ -1005,6 +1015,18 @@ struct common_speculative_impl_draft_dflash : public common_speculative_impl {
     }
 
     ~common_speculative_impl_draft_dflash() override {
+        auto * ctx_dft = this->params.ctx_dft;
+        for (llama_seq_id seq_id = 0; seq_id < (llama_seq_id) backend_chains.size(); ++seq_id) {
+            if (backend_chains[seq_id] == nullptr) {
+                continue;
+            }
+            if (ctx_dft) {
+                llama_set_sampler(ctx_dft, seq_id, nullptr);
+            }
+            llama_sampler_free(backend_chains[seq_id]);
+        }
+        backend_chains.clear();
+
         llama_batch_free(batch);
         llama_batch_free(batch_inject);
     }
@@ -1247,10 +1269,6 @@ struct common_speculative_impl_draft_dflash : public common_speculative_impl {
     void accept(llama_seq_id /*seq_id*/, uint16_t /*n_accepted*/, bool /*is_other*/) override {
         // noop
     }
-
-    bool need_embd() const override {
-        return false;
-    }
 };
 
 struct common_speculative_impl_draft_mtp : public common_speculative_impl {
@@ -1689,14 +1707,6 @@ struct common_speculative_impl_draft_mtp : public common_speculative_impl {
         const size_t row_bytes = (size_t) n_embd * sizeof(float);
         std::memcpy(pending_h[seq_id].data(), verify_h[seq_id].data() + (size_t) i_h * n_embd, row_bytes);
     }
-
-    bool need_embd() const override {
-        return false;
-    }
-
-    bool need_embd_nextn() const override {
-        return true;
-    }
 };
 
 // state of self-speculation (simple implementation, not ngram-map)
@@ -1743,10 +1753,6 @@ struct common_speculative_impl_ngram_simple : public common_speculative_impl {
     void accept(llama_seq_id /*seq_id*/, uint16_t /*n_accepted*/, bool /*is_other*/) override {
         // noop
     }
-
-    bool need_embd() const override {
-        return false;
-    }
 };
 
 struct common_speculative_impl_ngram_map_k : public common_speculative_impl {
@@ -1801,10 +1807,6 @@ struct common_speculative_impl_ngram_map_k : public common_speculative_impl {
 
         common_ngram_map_accept(config[seq_id], n_accepted);
     }
-
-    bool need_embd() const override {
-        return false;
-    }
 };
 
 struct common_speculative_impl_ngram_mod : public common_speculative_impl {
@@ -1980,10 +1982,6 @@ struct common_speculative_impl_ngram_mod : public common_speculative_impl {
             }
         }
     }
-
-    bool need_embd() const override {
-        return false;
-    }
 };
 
 struct common_speculative_impl_ngram_cache : public common_speculative_impl {
@@ -2123,10 +2121,6 @@ struct common_speculative_impl_ngram_cache : public common_speculative_impl {
     void accept(llama_seq_id /*seq_id*/, uint16_t /*n_accepted*/, bool /*is_other*/) override {
         // noop
     }
-
-    bool need_embd() const override {
-        return false;
-    }
 };
 
 struct common_speculative {
@@ -2234,6 +2228,43 @@ common_speculative_type common_speculative_type_from_name(const std::string & na
     return it->second;
 }
 
+std::vector<common_speculative_type> common_speculative_types_from_gguf(const std::string & path) {
+    struct gguf_init_params gguf_params = {
+        /* .no_alloc = */ true,
+        /* .ctx      = */ nullptr,
+    };
+
+    gguf_context_ptr gguf_ctx(gguf_init_from_file(path.c_str(), gguf_params));
+    if (!gguf_ctx) {
+        return {};
+    }
+
+    const int64_t arch_id = gguf_find_key(gguf_ctx.get(), "general.architecture");
+    if (arch_id < 0 || gguf_get_kv_type(gguf_ctx.get(), arch_id) != GGUF_TYPE_STRING) {
+        return {};
+    }
+
+    const std::string arch = gguf_get_val_str(gguf_ctx.get(), arch_id);
+    if (arch != "dflash") {
+        const uint32_t block_count = gguf_get_val_u32(gguf_ctx.get(), gguf_find_key(gguf_ctx.get(), (arch + ".block_count").c_str()));
+
+        if (gguf_find_tensor(gguf_ctx.get(), ("blk." + std::to_string(block_count - 1) + ".nextn.eh_proj.weight").c_str()) >= 0) {
+            return { COMMON_SPECULATIVE_TYPE_DRAFT_MTP };
+        }
+
+        return {};
+    }
+
+    // the Markov head distinguishes draft-dspark from draft-dflash
+    const auto type = gguf_find_tensor(gguf_ctx.get(), "markov_w1.weight") >= 0
+                    ? COMMON_SPECULATIVE_TYPE_DRAFT_DSPARK
+                    : COMMON_SPECULATIVE_TYPE_DRAFT_DFLASH;
+
+    SPC_INF("auto-detected speculative type '%s' from the draft model metadata\n", common_speculative_type_to_str(type).c_str());
+
+    return { type };
+}
+
 static uint32_t common_get_enabled_speculative_configs(const std::vector<common_speculative_type> & configs) {
     uint32_t result = 0;
     for (size_t i = 0; i < configs.size(); i++) {
@@ -2301,6 +2332,23 @@ common_params common_base_params_to_speculative(const common_params & params) {
     result.n_outputs_max = params.n_parallel;
     result.n_outputs_max_per_seq = 1;
 
+    // dflash/dspark decode the whole noise block in a single pass and sample every block position on the backend
+    // TODO: refactor such properties to be announced by the speculative types
+    //       something like `struct common_speculative_type_props common_speculative_type_get_props(...);`
+    const bool has_block_draft = std::any_of(
+        params.speculative.types.begin(), params.speculative.types.end(),
+        [](common_speculative_type t) {
+            return t == COMMON_SPECULATIVE_TYPE_DRAFT_DFLASH || t == COMMON_SPECULATIVE_TYPE_DRAFT_DSPARK;
+        });
+    if (has_block_draft) {
+        // per-seq output positions: DFlash decodes anchor + n_max masks (n_max + 1); DSpark n_max -> +1 covers both
+        const int32_t per_seq = std::max(1, params_spec.n_max + 1);
+        result.n_outputs_max = params.n_parallel * per_seq;
+        if (params_spec.backend_sampling) {
+            result.n_outputs_max_per_seq = per_seq;
+        }
+    }
+
     return result;
 }
 
@@ -2322,7 +2370,6 @@ common_speculative_init_result::common_speculative_init_result(
     const bool spec_mtp = std::find(params.speculative.types.begin(),
                                     params.speculative.types.end(),
                                     COMMON_SPECULATIVE_TYPE_DRAFT_MTP) != params.speculative.types.end();
-    GGML_ASSERT(has_draft || spec_mtp);
 
     auto mparams = common_model_params_to_llama(params);
     auto cparams = common_context_params_to_llama(params);
@@ -2560,34 +2607,6 @@ bool common_speculative_process(common_speculative * spec, const llama_batch & b
     return result;
 }
 
-bool common_speculative_need_embd(common_speculative * spec) {
-    if (spec == nullptr) {
-        return false;
-    }
-
-    for (auto & impl : spec->impls) {
-        if (impl->need_embd()) {
-            return true;
-        }
-    }
-
-    return false;
-}
-
-bool common_speculative_need_embd_nextn(common_speculative * spec) {
-    if (spec == nullptr) {
-        return false;
-    }
-
-    for (auto & impl : spec->impls) {
-        if (impl->need_embd_nextn()) {
-            return true;
-        }
-    }
-
-    return false;
-}
-
 void common_speculative_draft(common_speculative * spec) {
     if (spec == nullptr) {
         return;
@@ -2672,7 +2691,10 @@ void common_speculative_draft(common_speculative * spec) {
 void common_speculative_accept(common_speculative * spec, llama_seq_id seq_id, uint16_t n_accepted) {
     common_speculative_impl * impl = spec->impl_last[seq_id];
 
-    GGML_ASSERT(impl);
+    if (impl == nullptr) {
+        GGML_ASSERT(n_accepted == 0);
+        return;
+    }
 
     {
         common_time_meas tm(impl->t_accept_us, !impl->gen_perf);

common/speculative.h

@@ -14,6 +14,9 @@ const char * common_speculative_all_types_str();
 // parse user provided types
 std::vector<enum common_speculative_type> common_speculative_types_from_names(const std::vector<std::string> & names);
 
+// infer the spec types from the GGUF metadata of a draft model; empty if unknown
+std::vector<enum common_speculative_type> common_speculative_types_from_gguf(const std::string & path);
+
 // convert string to type
 enum common_speculative_type common_speculative_type_from_name(const std::string & name);
 
@@ -67,12 +70,6 @@ void common_speculative_begin(common_speculative * spec, llama_seq_id seq_id, co
 // process the batch and update the internal state of the speculative context
 bool common_speculative_process(common_speculative * spec, const llama_batch & batch);
 
-// true if any implementation requires target post-norm embeddings to be extracted
-bool common_speculative_need_embd(common_speculative * spec);
-
-// true if any implementation requires target nextn embeddings to be extracted
-bool common_speculative_need_embd_nextn(common_speculative * spec);
-
 // generate drafts for the sequences specified with `common_speculative_get_draft_params`
 void common_speculative_draft(common_speculative * spec);
 

conversion/gemma.py

@@ -665,7 +665,18 @@ class Gemma4Model(Gemma3Model):
         swa_layers = [t == "sliding_attention" for t in self.hparams["layer_types"]]
         self.gguf_writer.add_sliding_window_pattern(swa_layers)
 
-        head_dim_full = self.hparams["global_head_dim"]
+        per_layer_config = self.hparams.get("per_layer_config")
+        layer_types = self.hparams.get("layer_types", [])
+        if (head_dim_full := self.hparams.get("global_head_dim")) is None and per_layer_config is not None:
+            for layer_idx, layer_config in per_layer_config.items():
+                layer_idx = int(layer_idx)
+                if layer_idx < len(layer_types):
+                    if layer_types[layer_idx] == "full_attention" and "head_dim" in layer_config:
+                        head_dim_full = layer_config["head_dim"]
+                        break
+
+        assert head_dim_full is not None
+
         head_dim_swa = self.hparams["head_dim"]
         # correct the head dim for global/swa layers
         self.gguf_writer.add_key_length(head_dim_full)
@@ -685,8 +696,14 @@ class Gemma4Model(Gemma3Model):
             n_ff_arr = [n_ff if il < first_kv_shared_layer_idx else n_ff * 2 for il in range(self.block_count)]
             self.gguf_writer.add_feed_forward_length(n_ff_arr)
 
-        # handle num_global_key_value_heads
-        num_key_value_heads_full = self.hparams.get("num_global_key_value_heads")
+        if (num_key_value_heads_full := self.hparams.get("num_global_key_value_heads")) is None and per_layer_config is not None:
+            for layer_idx, layer_config in per_layer_config.items():
+                layer_idx = int(layer_idx)
+                if layer_idx < len(layer_types):
+                    if layer_types[layer_idx] == "full_attention" and "num_key_value_heads" in layer_config:
+                        num_key_value_heads_full = layer_config["num_key_value_heads"]
+                        break
+
         num_key_value_heads_swa = self.hparams.get("num_key_value_heads")
         if num_key_value_heads_full is not None and num_key_value_heads_swa is not None:
             value_arr = [num_key_value_heads_swa if is_swa else num_key_value_heads_full for is_swa in swa_layers]
@@ -708,7 +725,19 @@ class Gemma4Model(Gemma3Model):
         # IMPORTANT: this ROPE_FREQS tensor is ONLY used by the full_attention layers
         rope_params_full = self.hparams["rope_parameters"]["full_attention"]
         assert rope_params_full["rope_type"] == "proportional"
-        head_dim_full = (self.hparams["global_head_dim"])
+
+        per_layer_config = self.hparams.get("per_layer_config")
+        if (head_dim_full := self.hparams.get("global_head_dim")) is None and per_layer_config is not None:
+            layer_types = self.hparams.get("layer_types", [])
+            for layer_idx, layer_config in per_layer_config.items():
+                layer_idx = int(layer_idx)
+                if layer_idx < len(layer_types):
+                    if layer_types[layer_idx] == "full_attention" and "head_dim" in layer_config:
+                        head_dim_full = layer_config["head_dim"]
+                        break
+
+        assert head_dim_full is not None
+
         partial_rotary_factor_full = rope_params_full["partial_rotary_factor"]
         n_rot_full = int(head_dim_full * partial_rotary_factor_full / 2)
         n_unrot_full = int(head_dim_full / 2) - n_rot_full

docs/backend/OPENVINO.md

@@ -206,7 +206,7 @@ cmake -B build/ReleaseOV -G Ninja -DCMAKE_BUILD_TYPE=Release -DGGML_OPENVINO=ON
 cmake --build build/ReleaseOV --parallel
 ```
 
-- **Windows:** Open a **Developer Command Prompt for VS 2022** (so the MSVC toolchain is on `PATH`), then run:
+- **Windows:** Open **x64 Native Tools Command Prompt for VS** (so the MSVC toolchain is on `PATH`), then run:
 
 ```cmd
 C:\Intel\openvino\setupvars.bat
@@ -710,11 +710,15 @@ Boolean flags follow a uniform convention: set to a **positive integer** (e.g. `
 |-----------------------------------|-----------|------------|-------------------------------------------------------------------------------------------------------------|
 | `GGML_OPENVINO_DEVICE`            | String    | `CPU`      | Specify the target device (CPU, GPU, NPU). On systems with multiple GPUs, use `GPU.0` or `GPU.1` to explicitly target specific GPU. See [OpenVINO GPU Device](https://docs.openvino.ai/2026/openvino-workflow/running-inference/inference-devices-and-modes/gpu-device.html). When set to **NPU**, static compilation mode is enabled for optimal performance. |
 | `GGML_OPENVINO_CACHE_DIR`         | String    | `not set`  | Directory for OpenVINO model caching (recommended: `/tmp/ov_cache`). Enables model caching when set. **Not supported on NPU devices.** |
+| `GGML_OPENVINO_COMPILED_MODEL_CACHE_DIR` | String | `not set` | Directory for the frontend compiled-model cache. When set, OpenVINO compiled models are exported as blobs and imported on later runs to skip weight requantization, graph conversion, and compilation for matching single-graph models. |
 | `GGML_OPENVINO_PREFILL_CHUNK_SIZE`| Integer   | `256`      | Token chunk size for **NPU** prefill (NPU-only; ignored on CPU/GPU). Must be a positive integer; otherwise the default is used. |
 | `GGML_OPENVINO_STATEFUL_EXECUTION`| Boolean   | `0`        | Enable stateful KV cache for better performance. Recommended on CPU, GPU.                                   |
 | `GGML_OPENVINO_DISABLE_CACHE`     | Boolean   | `0`        | Disable the in-process compiled-model / decoder cache (cache is on by default). Set to `1` to disable.      |
 | `GGML_OPENVINO_DISABLE_KV_SLICE`  | Boolean   | `0`        | Disable the KV-cache input-tensor slicing optimization (slicing is on by default on CPU/GPU). Set to `1` to disable. |
 | `GGML_OPENVINO_MANUAL_GQA_ATTN`   | Boolean   | device-based | Tri-state. When **unset**, manual GQA attention is enabled by default on `GPU` and disabled on other devices. Set to a positive integer to force-enable, or `0` to force-disable. |
+| `GGML_OPENVINO_MEMORY_OPTIMIZE`   | Boolean   | `0`        | Umbrella switch for compile-time memory reductions. Enables `GGML_OPENVINO_REDUCE_COMPILE_MEM` and, on GPU, `GGML_OPENVINO_RELEASE_WEIGHTS` unless those fine-grained variables are explicitly set. |
+| `GGML_OPENVINO_REDUCE_COMPILE_MEM`| Boolean   | inherits from `GGML_OPENVINO_MEMORY_OPTIMIZE` | Reduce compile-time host memory use by streaming weight requantization and avoiding extra weight-node materialization where possible. Set explicitly to override the umbrella switch. |
+| `GGML_OPENVINO_RELEASE_WEIGHTS`   | Boolean   | inherits from `GGML_OPENVINO_MEMORY_OPTIMIZE` on GPU | GPU-only. Release host weight buffers after the compiled model cache can reuse the device/plugin copy. Requires stable graph shapes; dynamic workloads that need recompilation should leave this disabled. |
 | `GGML_OPENVINO_PROFILING`         | Boolean   | `0`        | Enable execution-time profiling.                                                                            |
 | `GGML_OPENVINO_DUMP_CGRAPH`       | Boolean   | `0`        | Dump the GGML compute graph to `cgraph_ov.txt`.                                                             |
 | `GGML_OPENVINO_DUMP_IR`           | Boolean   | `0`        | Serialize OpenVINO IR files with timestamps.                                                                |

docs/backend/SYCL.md

@@ -795,6 +795,7 @@ User can use the device management in [docs/multi-gpu.md](https://github.com/ggm
 | GGML_SYCL_ENABLE_FLASH_ATTN | 1 (default) or 0| Enable Flash-Attention. It can reduce memory usage. The performance impact depends on the LLM.|
 | GGML_SYCL_ENABLE_OPT | 0 or 1 (default)| Enable optimize features for Intel GPUs. (Recommended to 0 for Intel devices older than Gen 10) |
 | GGML_SYCL_ENABLE_GRAPH | 0 (default) or 1 | Enable running computations through SYCL Graphs feature. Disabled by default because SYCL Graph is still on development, no better performance. |
+| GGML_SYCL_ENABLE_HOST_PINNED_MEM | 0 or 1 (default) | Enable host pinned memory to speed up copy data from host to device. When disable it, host memory will common malloc() on CPU.|
 | GGML_SYCL_USE_LEVEL_ZERO_API | 1 (default) or 0 | Use Level Zero API for device memory allocation instead of SYCL. Reduces system RAM usage on Intel dGPUs by avoiding DMA-buf/TTM host memory staging. Requires GGML_SYCL_SUPPORT_LEVEL_ZERO_API=ON at build time. SYCL backend always runs on Level Zero running time even if it's set as OFF (The SYCL api will be usage for memory allocation).|
 | GGML_SYCL_ENABLE_DNN | 0 or 1 (default)| Enable running computations through oneDNN and always use oneMKL. |
 | GGML_SYCL_FA_ONEDNN | 1 (default) or 0 | Enable the oneDNN fused SDPA (flash-attention) path on supported GPUs. Set to 0 to always use the native SYCL flash-attention kernel. |
@@ -803,7 +804,8 @@ User can use the device management in [docs/multi-gpu.md](https://github.com/ggm
 | GGML_SYCL_ENABLE_MKL_FA | 1 (default) or 0 | Enable oneMKL GEMM flash attention for XMX-accelerated prompt processing with quantized KV cache. Automatically activates during prefill (prompt processing) when all conditions are met: (1) flash-attn enabled (`-fa` or `--flash-attn on`), (2) KV cache quantized (`--cache-type-k q8_0 --cache-type-v q8_0` or other `*_0/*_1` types), (3) batch size ≥ 1024 (`--batch-size 1024`), (4) prompt length ≥ 1024 tokens. Set to 0 to force the TILE kernel for A/B testing. Example minimum command: `llama-cli -m model.gguf -fa -ngl 99 --cache-type-k q8_0 --cache-type-v q8_0 --batch-size 1024 -p "your prompt"` |
 | GGML_SYCL_MKL_FA_DEBUG | 0 (default) or 1 | Enable per-call diagnostic logging for MKL flash attention: GEMM/softmax timings, interleaved-head detection, and buffer memory usage. |
 | GGML_SYCL_MKL_FA_DIAG | 0 (default) or 1 | Enable output fingerprinting for MKL flash attention. Dumps the first 64 float output values for the first 6 FA calls with n_kv ≥ 1024, labeled with kernel type (MKL/TILE/VEC) for cross-kernel comparison. |
-| GGML_SYCL_ENABLE_FUSION | 0 or 1 (default) | Enable fused-kernel dispatch in graph compute (currently top-k MoE gating). |
+| GGML_SYCL_ENABLE_FUSION | 0 or 1 (default) | Enable fused-kernel dispatch in graph compute. |
+| GGML_SYCL_ENABLE_ESIMD | 0 or 1 (default)| Enable ESIMD kernels when available. |
 | ZES_ENABLE_SYSMAN | 0 (default) or 1 | Support to get free memory of GPU by sycl::aspect::ext_intel_free_memory.<br>Recommended to use when --split-mode = layer |
 | UR_L0_ENABLE_RELAXED_ALLOCATION_LIMITS | 0 (default) or 1 | Allow SYCL/Unified Runtime Level Zero device allocations larger than 4 GiB. llama.cpp's direct Level Zero allocation path requests the relaxed maximum-size limit itself when GGML_SYCL_ENABLE_LEVEL_ZERO=1. |
 | GGML_SYCL_USM_SYSTEM | 0 (default) or 1 | Enable experimental support for [USM system allocations](https://github.khronos.org/SYCL_Reference/iface/usm_basic_concept.html#system-allocations) for large GPU buffers. This requires enough host memory for model weights and caches, an Intel Xe2+ GPU such as BMG or newer and supported on Linux only, with CONFIG_DRM_XE_GPUSVM enabled. |

docs/preset.md

@@ -4,7 +4,7 @@
 
 The INI preset feature, introduced in [PR#17859](https://github.com/ggml-org/llama.cpp/pull/17859), allows users to create reusable and shareable parameter configurations for llama.cpp.
 
-### Using Presets with the Server
+## Using Presets with the Server
 
 When running multiple models on the server (router mode), INI preset files can be used to configure model-specific parameters. Please refer to the [server documentation](../tools/server/README.md) for more details.
 
@@ -93,3 +93,18 @@ llama-server -hf user/repo:gpt-oss-120b-hf
 ```
 
 Please make sure to provide the correct `hf-repo` for each child preset. Otherwise, you may get error: `The specified tag is not a valid quantization scheme.`
+
+## System-level config
+
+The system-level config, added in PR [#26118](https://github.com/ggml-org/llama.cpp/pull/26118), allows sharing the same set of options among multiple tools and examples. Unlike the sections above, it is not limited to the server.
+
+These files are loaded on startup if present. A later file overrides an earlier one:
+1. System-wide: `/etc/llama.cpp/config.ini` (or `%PROGRAMDATA%\llama.cpp\config.ini` on Windows)
+2. User-level: `$XDG_CONFIG_HOME/llama.cpp/config.ini`, `~/.config/llama.cpp/config.ini` by default (or `%APPDATA%\llama.cpp\config.ini` on Windows)
+
+The config file is applied first, then its options are overridden by ENV variables, CLI arguments and model presets (in router mode).
+
+Note:
+- Only the `[*]` and default sections are used; options written before any section header belong to "default. Named sections are ignored
+- Tool-specific options can be specified, but will be ignored (with a warning) if the example doesn't support it<br/>Example: if you specify `port = 1234`, only `llama-server` will use it, other examples will ignore it
+- `model` or `hf-repo` are not recommended to be configured system-level, because it may introduce conflicts<br/>Example: a `hf-repo` in the config file still takes effect when you pass `-m` on the command line, so you may load a different model than expected

docs/release.md

@@ -0,0 +1,49 @@
+# Release process
+
+llama.cpp uses [semantic versioning](https://semver.org) (`MAJOR.MINOR.PATCH`).
+
+## Version bump guidelines
+
+| Change type | Version component |
+|---|---|
+| Breaking change to the public C API (`include/llama.h`)         | `MAJOR` |
+| Backward-compatible features, model support, or API addition    | `MINOR` |
+| Bug fix with no API change                                      | `PATCH` |
+
+The version is set in the three variables at the top of the root `CMakeLists.txt`:
+
+```cmake
+set(LLAMA_VERSION_MAJOR 0)
+set(LLAMA_VERSION_MINOR 1)
+set(LLAMA_VERSION_PATCH 0)
+```
+
+_A version bump should be included in the PR that introduces the change, or in a
+dedicated bump commit merged before the release is cut._
+
+_TODO: add PR labels (`semver: patch`, `semver: minor`, `semver: major`) to help
+identify which PRs require a version bump before cutting a release._
+
+## Making a release
+
+Releases are created by running the [make-release](.github/workflows/make-release.yml)
+which is a manual workflow.
+
+The workflow creates an annotated git tag (e.g. `v0.1.0`) and pushes it to the
+remote. No GitHub Release object is created, the tag is the release artifact.
+
+## Building a release
+
+By default, `LLAMA_BUILD_IS_DEV=ON` which appends a `-dev` suffix to `LLAMA_VERSION`,
+marking the build as a nightly/development build. Distributors building from a
+release tag must pass `-DLLAMA_BUILD_IS_DEV=OFF` to produce a clean version string
+(e.g. `0.1.0` instead of `0.1.0-dev`).
+
+## How releases reach users
+Currently releases are not published to github releases, only nightly/development
+builds are available there. The way users can access releases are using the following
+channels:
+
+- **llama-install.sh**  — downloads pre-built binaries built from the release tag.
+- **Package managers**  — consume the git tag directly.
+- **Build from source** — users clone the repo and check out the tag.

examples/speculative-simple/README.md

@@ -3,10 +3,47 @@
 Demonstration of basic greedy speculative decoding
 
 ```bash
+# spec-type draft-simple
 ./bin/llama-speculative-simple \
-    -m  ../models/qwen2.5-32b-coder-instruct/ggml-model-q8_0.gguf \
-    -md ../models/qwen2.5-1.5b-coder-instruct/ggml-model-q4_0.gguf \
-    -f test.txt -c 0 -ngl 99 --color on \
-    --sampling-seq k --top-k 1 -fa on --temp 0.0 \
-    -ngld 99 --spec-draft-n-max 16 --spec-draft-n-draft-min 5 --draft-p-min 0.9
+        -hf  ggml-org/Qwen3-8B-Base-GGUF:Q8_0 \
+        -hfd ggml-org/Qwen3-0.6B-Base-GGUF \
+        -p "Here is a quick sort implementation in C++. Just code, no comments:\n\n#include" \
+        --spec-type draft-simple --spec-draft-n-max 7 -ngld 99 --color on \
+        -n 256 --temp 0 --top-k 1 --seed 42 -ngl 99 -lv 4
+
+# spec-type draft-mtp
+./bin/llama-speculative-simple \
+        -hf ggml-org/Qwen3.6-27B-GGUF:Q8_0 \
+        -p "Here is a quick sort implementation in C++. Just code, no comments:\n\n#include" \
+        --spec-type draft-mtp --spec-draft-n-max 3 -ngld 99 --color on \
+        -n 256 --temp 0 --top-k 1 --seed 42 -ngl 99 -lv 4
+
+# spec-type draft-mtp (with shared KV cache)
+# note: this model needs a <s> token at the start to somewhat work without the chat template
+./bin/llama-speculative-simple \
+        -hf ggml-org/Gemma-4-31B-it-GGUF:Q8_0 \
+        -p "<s>Here is a quick sort implementation in C++. Just code, no comments:\n\n#include" \
+        --spec-type draft-mtp --spec-draft-n-max 3 -ngld 99 --color on \
+        -n 256 --temp 0 --top-k 1 --seed 42 -ngl 99 -lv 4
+
+# spec-type draft-eagle3
+./bin/llama-speculative-simple \
+        -hf ggml-org/gpt-oss-20b-GGUF \
+        -p "Here is a quick sort implementation in C++. Just code, no comments:\n\n#include" \
+        --spec-type draft-eagle3 --spec-draft-n-max 3 -ngld 99 --color on \
+        -n 256 --temp 0 --top-k 1 --seed 42 -ngl 99 -lv 4
+
+# spec-type draft-dflash
+./bin/llama-speculative-simple \
+        -hf ggml-org/Qwen3-8B-GGUF \
+        -p "Here is a quick sort implementation in C++. Just code, no comments:\n\n#include" \
+        --spec-type draft-dflash --spec-draft-n-max 7 -ngld 99 --color on \
+        -n 256 --temp 0 --top-k 1 --seed 42 -ngl 99 -lv 4
+
+# spec-type draft-dspark
+./bin/llama-speculative-simple \
+        -hf ggml-org/Qwen3-8B-GGUF \
+        -p "Here is a quick sort implementation in C++. Just code, no comments:\n\n#include" \
+        --spec-type draft-dspark --spec-draft-n-max 7 -ngld 99 --color on \
+        -n 256 --temp 0 --top-k 1 --seed 42 -ngl 99 -lv 4
 ```

examples/speculative-simple/speculative-simple.cpp

@@ -51,48 +51,23 @@ int main(int argc, char ** argv) {
 
     const llama_vocab * vocab = llama_model_get_vocab(model_tgt);
 
-    // load the draft model
-    llama_model_ptr model_dft;
-    llama_context_ptr ctx_dft;
+    // load the draft model (if any) - this also creates the MTP draft context when MTP speculation is enabled
+    common_speculative_init_result_ptr spec_init;
 
-    // TODO: simplify this logic
     {
-        const auto & params_spec = params.speculative.draft;
+        common_params params_dft = common_base_params_to_speculative(params);
 
-        auto params_dft = params;
-
-        params_dft.n_outputs_max = params.n_parallel;
-        params_dft.n_outputs_max_per_seq = 1;
-
-        params_dft.devices      = params_spec.devices;
-        params_dft.model        = params_spec.mparams;
-        params_dft.n_gpu_layers = params_spec.n_gpu_layers;
-
-        if (params_spec.cpuparams.n_threads > 0) {
-            params_dft.cpuparams.n_threads       = params.speculative.draft.cpuparams.n_threads;
-            params_dft.cpuparams_batch.n_threads = params.speculative.draft.cpuparams_batch.n_threads;
-        }
-
-        params_dft.tensor_buft_overrides = params.speculative.draft.tensor_buft_overrides;
-
-        auto mparams_dft = common_model_params_to_llama(params_dft);
-
-        model_dft.reset(llama_model_load_from_file(params_dft.model.path.c_str(), mparams_dft));
-        if (model_dft == nullptr) {
-            LOG_ERR("failed to load draft model, '%s'\n", params_dft.model.path.c_str());
-            return 1;
-        }
-
-        auto cparams = common_context_params_to_llama(params_dft);
-        ctx_dft.reset(llama_init_from_model(model_dft.get(), cparams));
+        spec_init = common_speculative_init_from_params(params_dft, model_tgt, ctx_tgt);
 
         params.speculative.draft.ctx_tgt = ctx_tgt;
-        params.speculative.draft.ctx_dft = ctx_dft.get();
+        params.speculative.draft.ctx_dft = spec_init->context();
     }
 
+    llama_context * ctx_dft = params.speculative.draft.ctx_dft;
+
     // check if the context supports partial sequence removal
-    const bool use_ckpt_tgt = (common_context_can_seq_rm(ctx_tgt)       == COMMON_CONTEXT_SEQ_RM_TYPE_FULL);
-    const bool use_ckpt_dft = (common_context_can_seq_rm(ctx_dft.get()) == COMMON_CONTEXT_SEQ_RM_TYPE_FULL);
+    const bool use_ckpt_tgt = common_context_can_seq_rm(ctx_tgt) == COMMON_CONTEXT_SEQ_RM_TYPE_FULL;
+    const bool use_ckpt_dft = common_context_can_seq_rm(ctx_dft) == COMMON_CONTEXT_SEQ_RM_TYPE_FULL;
 
     if (use_ckpt_tgt) {
         LOG_INF("speculative decoding will use checkpoints (context does not support partial sequence removal)\n");
@@ -138,9 +113,30 @@ int main(int argc, char ** argv) {
     // target model sampling context
     common_sampler_ptr smpl(common_sampler_init(model_tgt, params.sampling));
 
-    // eval the prompt
-    llama_decode(ctx_tgt,       llama_batch_get_one(inp.data(), inp.size() - 1));
-    llama_decode(ctx_dft.get(), llama_batch_get_one(inp.data(), inp.size() - 1));
+    // init the speculator
+    const auto & params_spec = params.speculative;
+
+    struct common_speculative * spec = common_speculative_init(params.speculative, 1);
+
+    if (spec == nullptr) {
+        LOG_ERR("%s", "failed to initialize speculative decoding\n");
+        return 1;
+    }
+
+    // eval the prompt on the target and feed it to the speculative implementation(s)
+    {
+        llama_batch batch_prompt = llama_batch_init(inp.size(), 0, 1);
+        for (size_t i = 0; i < inp.size() - 1; ++i) {
+            common_batch_add(batch_prompt, inp[i], i, { seq_id }, false);
+        }
+
+        llama_decode(ctx_tgt, batch_prompt);
+
+        if (!common_speculative_process(spec, batch_prompt)) {
+            LOG_ERR("%s", "failed to process speculative prompt\n");
+            return 1;
+        }
+    }
 
     // note: keep the last token separate!
     llama_token id_last = inp.back();
@@ -151,18 +147,12 @@ int main(int argc, char ** argv) {
 
     int n_past = inp.size() - 1;
 
-    // init the speculator
-    const auto & params_spec = params.speculative;
-
-    struct common_speculative * spec = common_speculative_init(params.speculative, 1);
-
     common_speculative_begin(spec, seq_id, prompt_tgt);
 
     llama_batch batch_tgt = llama_batch_init(llama_n_batch(ctx_tgt), 0, 1);
 
-    size_t n_draft = 0;
-
     llama_tokens draft;
+
     common_prompt_checkpoint ckpt;
 
     const auto t_enc_end = ggml_time_us();
@@ -184,13 +174,20 @@ int main(int argc, char ** argv) {
                     llama_memory_seq_pos_max(llama_get_memory(ctx_tgt), seq_id));
 
             if (use_ckpt_dft) {
-                ckpt.update_dft(ctx_dft.get(), seq_id, LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY);
+                ckpt.update_dft(ctx_dft, seq_id, LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY);
             }
 
+            // determine the max draft that fits the remaining context and generation budget
+            int n_draft_max = (int) llama_n_ctx(ctx_tgt) - n_past - 2;
+            if (params.n_predict >= 0) {
+                n_draft_max = std::min(n_draft_max, params.n_predict - n_predict - 1);
+            }
+            n_draft_max = std::max(n_draft_max, 0);
+
             // generate a new draft
             common_speculative_get_draft_params(spec, seq_id) = {
                 /* .drafting   = */ true,
-                /* .n_max      = */ -1,
+                /* .n_max      = */ n_draft_max,
                 /* .n_past     = */ n_past,
                 /* .id_last    = */ id_last,
                 /* .prompt     = */ &prompt_tgt,
@@ -198,9 +195,6 @@ int main(int argc, char ** argv) {
             };
             common_speculative_draft(spec);
 
-            // save the original draft size
-            n_draft = draft.size();
-
             // save a checkpoint of the target context before evaluating the draft
             // this allows us to restore the state if partial draft acceptance occurs
             if (!draft.empty()) {
@@ -209,10 +203,13 @@ int main(int argc, char ** argv) {
                 }
             }
 
-            {
-                ckpt.load_dft(ctx_dft.get(), seq_id, LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY);
+            // reset the draft context to the checkpoint before verification
+            if (ctx_dft) {
+                if (use_ckpt_dft) {
+                    ckpt.load_dft(ctx_dft, seq_id, LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY);
+                }
 
-                llama_memory_seq_rm(llama_get_memory(ctx_dft.get()), seq_id, ckpt.pos_max + 1, -1);
+                llama_memory_seq_rm(llama_get_memory(ctx_dft), seq_id, ckpt.pos_max + 1, -1);
             }
         } else {
             // we have a previous (partial) draft to reuse from checkpoint restoration
@@ -236,10 +233,10 @@ int main(int argc, char ** argv) {
             llama_decode(ctx_tgt, batch_tgt);
         }
 
-        // evaluate the same batch with the draft model
-        {
-            // TODO: extend to support MTP, Eagle, etc. See server code for reference
-            llama_decode(ctx_dft.get(), batch_tgt);
+        // feed the batch to the speculative implementation(s) - this drives the draft model, MTP, Eagle3, etc.
+        if (!common_speculative_process(spec, batch_tgt)) {
+            LOG_ERR("%s", "failed to process speculative batch\n");
+            break;
         }
 
         // only save the sampler sampler state if we use checkpoints
@@ -248,6 +245,9 @@ int main(int argc, char ** argv) {
             smpl_save.reset(common_sampler_clone(smpl.get()));
         }
 
+        // save the size of the draft being verified
+        const size_t n_draft = draft.size();
+
         // sample from the full target batch and return the accepted tokens based on the target sampler
         //
         // for each token to be accepted, the sampler would have to sample that same token
@@ -264,8 +264,8 @@ int main(int argc, char ** argv) {
         // check for partial draft acceptance:
         // if the context doesn't support partial sequence removal, restore the checkpoint
         // and make the accepted tokens the new partial draft for the next iteration
-        if (use_ckpt_tgt && ids.size() - 1 < draft.size()) {
-            LOG_DBG("partial acceptance: %zu < %zu, restoring checkpoint\n", ids.size() - 1, draft.size());
+        if (use_ckpt_tgt && ids.size() - 1 < n_draft) {
+            LOG_DBG("partial acceptance: %zu < %zu, restoring checkpoint\n", ids.size() - 1, n_draft);
 
             draft = std::move(ids);
 
@@ -275,10 +275,10 @@ int main(int argc, char ** argv) {
                 llama_memory_seq_rm(llama_get_memory(ctx_tgt), seq_id, ckpt.pos_max + 1, -1);
             }
 
-            {
-                ckpt.load_dft(ctx_dft.get(), seq_id, LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY);
+            if (ctx_dft) {
+                ckpt.load_dft(ctx_dft, seq_id, LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY);
 
-                llama_memory_seq_rm(llama_get_memory(ctx_dft.get()), seq_id, ckpt.pos_max + 1, -1);
+                llama_memory_seq_rm(llama_get_memory(ctx_dft), seq_id, ckpt.pos_max + 1, -1);
             }
 
             prompt_tgt.resize(ckpt.n_tokens);
@@ -329,8 +329,11 @@ int main(int argc, char ** argv) {
         {
             LOG_DBG("clear kv cache from any extra tokens, n_past = %d\n", n_past);
 
-            llama_memory_seq_rm(llama_get_memory(ctx_tgt),       seq_id, n_past, -1);
-            llama_memory_seq_rm(llama_get_memory(ctx_dft.get()), seq_id, n_past, -1);
+            llama_memory_seq_rm(llama_get_memory(ctx_tgt), seq_id, n_past, -1);
+
+            if (ctx_dft) {
+                llama_memory_seq_rm(llama_get_memory(ctx_dft), seq_id, n_past, -1);
+            }
         }
 
         if ((params.n_predict >= 0 && n_predict > params.n_predict) || has_eos) {
@@ -356,6 +359,7 @@ int main(int argc, char ** argv) {
 
     LOG_INF("\n");
     LOG_INF("draft:\n\n");
+    common_speculative_print_stats(spec);
 
     LOG_INF("\n");
     LOG_INF("target:\n\n");

examples/test-cmake/.gitignore

@@ -0,0 +1,3 @@
+llama-build-install
+install
+build

examples/test-cmake/CMakeLists.txt

@@ -0,0 +1,13 @@
+cmake_minimum_required(VERSION 3.14)
+project(llama-simple)
+
+set(CMAKE_CXX_STANDARD 17)
+
+find_package(llama 0.1.0 REQUIRED)
+
+add_executable(test-cmake test-cmake.cpp)
+target_link_libraries(test-cmake PRIVATE llama)
+target_compile_definitions(test-cmake PRIVATE
+    LLAMA_BUILD_NUMBER=${LLAMA_BUILD_NUMBER}
+    LLAMA_BUILD_COMMIT="${LLAMA_BUILD_COMMIT}"
+)

examples/test-cmake/README.md

@@ -0,0 +1,36 @@
+## cmake-test
+
+This is just for manually testing/developing of a llama.cpp installation to
+enable troubleshooting issues and exploration. The idea is that this can be used
+after making changes to llama.cpp installation cmake configuration and then
+verify it locally.
+
+### Usage
+The following will configure, build, and install llama.cpp
+
+Configuring/build/install:
+```console
+./build-install.sh
+```
+The above command will create a directory named `install` in the current directory
+which will have the follwing files in its lib directory:
+```console
+(venv) $ ls install/lib/
+cmake                   libggml.so          libllama-common.so.0      libllama.so.0.1.0  llama.cpp
+libggml-base.so         libggml.so.0        libllama-common.so.0.1.0  libmtmd.so         pkgconfig
+libggml-base.so.0       libggml.so.0.19.0   libllama.so               libmtmd.so.0
+libggml-base.so.0.19.0  libllama-common.so  libllama.so.0             libmtmd.so.0.1.0
+```
+
+Build/run this project using the installation created above:
+```console
+(venv) $ ./build.sh
+-- Configuring done (0.0s)
+-- Generating done (0.0s)
+-- Build files have been written to: /home/danbev/work/ai/llama.cpp/examples/test-cmake/build
+[100%] Built target test-cmake
+[test-cmake] Using llama.cpp version 0.1.0-dev-b10335
+[test-cmake] Initializing backend...
+load_backend: loaded CPU backend from /home/danbev/work/ai/llama.cpp/examples/test-cmake/install/lib/llama.cpp/libggml-cpu-alderlake.so
+[test-cmake] Backend initialized.
+```

examples/test-cmake/build-install.sh

@@ -0,0 +1,19 @@
+#!/bin/bash
+
+set -e
+
+rm -rf llama-build-install install
+
+cmake --fresh -S ../../. -B llama-build-install -DCMAKE_BUILD_TYPE=Release \
+  -DBUILD_SHARED_LIBS=ON \
+  -DGGML_BACKEND_DL=ON \
+  -DGGML_CPU_ALL_VARIANTS=ON \
+  -DLLAMA_TESTS_INSTALL=OFF \
+  -DCMAKE_INSTALL_PREFIX="${PWD}/install" \
+  -DGGML_BACKEND_DIR="${PWD}/install/lib/llama.cpp" \
+  -DGGML_LIB_INSTALL_DIR="${PWD}/install/lib/llama.cpp" \
+  -DLLAMA_LIB_INSTALL_DIR="${PWD}/install/lib/llama.cpp" \
+  -DLLAMA_TOOLS_INSTALL=OFF
+
+cmake --build llama-build-install --parallel 12
+cmake --install llama-build-install

examples/test-cmake/build.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+set -e
+
+cmake -S . -B build -DCMAKE_PREFIX_PATH="${PWD}/install"
+cmake --build build
+LD_LIBRARY_PATH="${PWD}/install/lib/llama.cpp:${PWD}/install/lib${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" ./build/test-cmake

examples/test-cmake/test-cmake.cpp

@@ -0,0 +1,12 @@
+#include "llama.h"
+#include <cstdio>
+
+int main(void) {
+    printf("[test-cmake] version: %s, build: %d (%s)\n",
+           llama_version(), LLAMA_BUILD_NUMBER, LLAMA_BUILD_COMMIT);
+    printf("[test-cmake] Initializing backend...\n");
+    llama_backend_init();
+    printf("[test-cmake] Backend initialized.\n");
+    llama_backend_free();
+    return 0;
+}

ggml/CMakeLists.txt

@@ -402,7 +402,7 @@ configure_package_config_file(
               GGML_BIN_INSTALL_DIR)
 
 write_basic_package_version_file(
-        ${CMAKE_CURRENT_BINARY_DIR}/ggml-version.cmake
+        ${CMAKE_CURRENT_BINARY_DIR}/ggml-config-version.cmake
     VERSION ${GGML_INSTALL_VERSION}
     COMPATIBILITY SameMajorVersion)
 
@@ -414,7 +414,7 @@ message(STATUS "ggml version: ${GGML_INSTALL_VERSION}")
 message(STATUS "ggml commit:  ${GGML_BUILD_COMMIT}")
 
 install(FILES ${CMAKE_CURRENT_BINARY_DIR}/ggml-config.cmake
-              ${CMAKE_CURRENT_BINARY_DIR}/ggml-version.cmake
+              ${CMAKE_CURRENT_BINARY_DIR}/ggml-config-version.cmake
         DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/ggml)
 
 if (MSVC)

ggml/cmake/ggml-config.cmake.in

@@ -113,6 +113,7 @@ set_and_check(GGML_LIB_DIR "@PACKAGE_GGML_LIB_INSTALL_DIR@")
 if(NOT TARGET ggml::ggml)
     find_package(Threads REQUIRED)
 
+    unset(GGML_LIBRARY CACHE)
     find_library(GGML_LIBRARY ggml
         REQUIRED
         HINTS ${GGML_LIB_DIR}
@@ -121,8 +122,10 @@ if(NOT TARGET ggml::ggml)
     add_library(ggml::ggml UNKNOWN IMPORTED)
     set_target_properties(ggml::ggml
         PROPERTIES
-            IMPORTED_LOCATION "${GGML_LIBRARY}")
+            IMPORTED_LOCATION "${GGML_LIBRARY}"
+            INTERFACE_INCLUDE_DIRECTORIES "${GGML_INCLUDE_DIR}")
 
+    unset(GGML_BASE_LIBRARY CACHE)
     find_library(GGML_BASE_LIBRARY ggml-base
         REQUIRED
         HINTS ${GGML_LIB_DIR}
@@ -132,6 +135,7 @@ if(NOT TARGET ggml::ggml)
     set_target_properties(ggml::ggml-base
         PROPERTIES
             IMPORTED_LOCATION "${GGML_BASE_LIBRARY}"
+            INTERFACE_INCLUDE_DIRECTORIES "${GGML_INCLUDE_DIR}"
             INTERFACE_LINK_LIBRARIES "${GGML_BASE_INTERFACE_LINK_LIBRARIES}")
 
     set(_ggml_all_targets "")
@@ -140,6 +144,7 @@ if(NOT TARGET ggml::ggml)
             string(REPLACE "-" "_" _ggml_backend_pfx "${_ggml_backend}")
             string(TOUPPER "${_ggml_backend_pfx}" _ggml_backend_pfx)
 
+            unset(${_ggml_backend_pfx}_LIBRARY CACHE)
             find_library(${_ggml_backend_pfx}_LIBRARY ${_ggml_backend}
                 REQUIRED
                 HINTS ${GGML_LIB_DIR}

ggml/src/ggml-cpu/arch/arm/cpu-feats.cpp

@@ -1,90 +1,12 @@
 #include "ggml-backend-impl.h"
+#include "ggml-feats.h"
 
-#if defined(__aarch64__)
-
-#if defined(__linux__)
-#include <sys/auxv.h>
-#elif defined(__APPLE__)
-#include <sys/sysctl.h>
-#endif
-
-#if !defined(HWCAP_FPHP)
-#define HWCAP_FPHP (1 << 9)
-#endif
-
-#if !defined(HWCAP_ASIMDHP)
-#define HWCAP_ASIMDHP (1 << 10)
-#endif
-
-#if !defined(HWCAP_ASIMDDP)
-#define HWCAP_ASIMDDP (1 << 20)
-#endif
-
-#if !defined(HWCAP_SVE)
-#define HWCAP_SVE (1 << 22)
-#endif
-
-#if !defined(HWCAP2_SVE2)
-#define HWCAP2_SVE2 (1 << 1)
-#endif
-
-#if !defined(HWCAP2_I8MM)
-#define HWCAP2_I8MM (1 << 13)
-#endif
-
-#if !defined(HWCAP2_SME)
-#define HWCAP2_SME (1 << 23)
-#endif
-
-struct aarch64_features {
-    // has_neon not needed, aarch64 has NEON guaranteed
-    bool has_dotprod     = false;
-    bool has_fp16        = false;
-    bool has_sve         = false;
-    bool has_sve2        = false;
-    bool has_i8mm        = false;
-    bool has_sme         = false;
-    bool has_sme2        = false;
-
-    aarch64_features() {
-#if defined(__linux__)
-        uint32_t hwcap = getauxval(AT_HWCAP);
-        uint32_t hwcap2 = getauxval(AT_HWCAP2);
-
-        has_dotprod = !!(hwcap & HWCAP_ASIMDDP);
-        has_fp16    = !!(hwcap & HWCAP_FPHP) && !!(hwcap & HWCAP_ASIMDHP);
-        has_sve     = !!(hwcap & HWCAP_SVE);
-        has_sve2    = !!(hwcap2 & HWCAP2_SVE2);
-        has_i8mm    = !!(hwcap2 & HWCAP2_I8MM);
-        has_sme     = !!(hwcap2 & HWCAP2_SME);
-#elif defined(__APPLE__)
-        int oldp = 0;
-        size_t size = sizeof(oldp);
-
-        if (sysctlbyname("hw.optional.arm.FEAT_DotProd", &oldp, &size, NULL, 0) == 0) {
-            has_dotprod = static_cast<bool>(oldp);
-        }
-
-        if (sysctlbyname("hw.optional.arm.FEAT_I8MM", &oldp, &size, NULL, 0) == 0) {
-            has_i8mm = static_cast<bool>(oldp);
-        }
-
-        if (sysctlbyname("hw.optional.arm.FEAT_SME", &oldp, &size, NULL, 0) == 0) {
-            has_sme = static_cast<bool>(oldp);
-        }
-
-        if (sysctlbyname("hw.optional.arm.FEAT_SME2", &oldp, &size, NULL, 0) == 0) {
-            has_sme2 = static_cast<bool>(oldp);
-        }
-
-        // Apple apparently does not implement SVE yet
-#endif
-    }
-};
+#if defined(__aarch64__) || defined(_M_ARM64)
 
 static int ggml_backend_cpu_aarch64_score() {
     int score = 1;
-    aarch64_features af;
+    const ggml_feats_arch64_runtime_t af = ggml_feats_get_arch64_runtime();
+    GGML_UNUSED(af);
 
 #ifdef GGML_USE_DOTPROD
     if (!af.has_dotprod) { return 0; }
@@ -116,4 +38,4 @@ static int ggml_backend_cpu_aarch64_score() {
 
 GGML_BACKEND_DL_SCORE_IMPL(ggml_backend_cpu_aarch64_score)
 
-# endif // defined(__aarch64__)
+# endif // defined(__aarch64__) || defined(_M_ARM64)

ggml/src/ggml-cpu/ggml-cpu.c

@@ -2795,6 +2795,11 @@ struct ggml_cplan ggml_graph_plan(
     n_threads = 1;
 #endif
 
+#if defined(__wasi__)
+    // WASI doesn't support parallelism yet
+    n_threads = 1;
+#endif
+
     size_t work_size = 0;
 
     struct ggml_cplan cplan;

ggml/src/ggml-cpu/kleidiai/kleidiai.cpp

@@ -2,10 +2,12 @@
 // SPDX-License-Identifier: MIT
 //
 #include <arm_neon.h>
-#include <assert.h>
-#include <stdio.h>
+#include <cassert>
+#include <cstdio>
+#include <cstdlib>
 #include <atomic>
 #include <cfloat>
+#include <cctype>
 #include <algorithm>
 #include <cmath>
 #include <stdexcept>
@@ -17,25 +19,21 @@
 #include <cstddef>
 #include <cstdint>
 #include <fstream>
-#include <set>
+#include <map>
 #include <iostream>
 #include <climits>
+#include <charconv>
+#include <system_error>
 #if defined(__linux__)
 #include <asm/hwcap.h>
+#include <dirent.h>
 #include <sys/auxv.h>
 #include <sys/types.h>
 #include <sys/stat.h>
 #include <unistd.h>
-#ifndef HWCAP2_SME2
-#define HWCAP2_SME2 (1UL << 37)
-#endif
 #elif defined(__APPLE__)
-#include <string_view>
 #include <sys/sysctl.h>
 #include <sys/types.h>
-#elif defined(_WIN32)
-#include <windows.h>
-#include <excpt.h>
 #endif
 
 #include "kleidiai.h"
@@ -43,6 +41,7 @@
 #include "ggml-cpu.h"
 #include "ggml-cpu-impl.h"
 #include "ggml-impl.h"
+#include "ggml-feats.h"
 #include "ggml-backend-impl.h"
 #include "ggml-threading.h"
 #include "traits.h"
@@ -64,8 +63,8 @@ struct ggml_kleidiai_context {
     ggml_kleidiai_kernels * kernels_q4;
     ggml_kleidiai_kernels * kernels_q8;
     ggml_kleidiai_kernels * kernels_f32;
-    int sme_thread_cap; // <= 0 means “SME disabled/unknown”;
-    int thread_hint;    // <= 0 means “no hint”
+    int sme_thread_cap; // <= 0 means "SME disabled/unknown"
+    int thread_hint;    // <= 0 means "no hint"
     int chunk_multiplier;
 } static ctx = { CPU_FEATURE_NONE, nullptr, nullptr, nullptr, 0, -1, 4 };
 
@@ -93,24 +92,117 @@ static const char* cpu_feature_to_string(cpu_feature f) {
     }
 }
 
+#if defined(__linux__) && defined(__aarch64__)
+static bool parse_cpu_dir_name(const char* name, size_t* cpu) {
+    if (strncmp(name, "cpu", 3) != 0 ||
+        name[3] < '0' || name[3] > '9') {
+        return false;
+    }
+
+    const char* first = name + 3;
+    const char* last = name + strlen(name);
+
+    size_t value = 0;
+    const auto [end, ec] = std::from_chars(first, last, value, 10);
+
+    if (ec != std::errc{} || end != last) {
+        return false;
+    }
+
+    *cpu = value;
+    return true;
+}
+
+static std::vector<size_t> detect_cpu_ids() {
+    std::vector<size_t> cpus;
+
+    DIR * dir = opendir("/sys/devices/system/cpu");
+    if (dir == nullptr) {
+        return cpus;
+    }
+
+    while (dirent * entry = readdir(dir)) {
+        size_t cpu = 0;
+        if (parse_cpu_dir_name(entry->d_name, &cpu)) {
+            cpus.push_back(cpu);
+        }
+    }
+    closedir(dir);
+
+    std::sort(cpus.begin(), cpus.end());
+    cpus.erase(std::unique(cpus.begin(), cpus.end()), cpus.end());
+    return cpus;
+}
+#endif
+
+#if defined(__APPLE__) && defined(__aarch64__)
+static bool apple_sme_counted_perf_level(std::string name) {
+    for (std::string::size_type i = 0; i < name.size(); ++i) {
+        name[i] = (char) std::tolower((unsigned char) name[i]);
+    }
+
+    // Conservative ceiling: only count perf-level names observed to provide full SME throughput.
+    // Future names should be calibrated here before they raise the automatic SME thread cap.
+    return name.find("super") != std::string::npos ||
+           name.find("performance") != std::string::npos;
+}
+#endif
+
+static void add_smcus_from_smidr(uint64_t smidr, size_t & num_private, std::map<uint32_t, size_t> & shared_counts) {
+    // Arm ARM: SMIDR_EL1. SH==0 is implementation-defined; keep the existing
+    // conservative policy and only treat zero affinity as private.
+    const uint32_t sh = (uint32_t)((smidr >> 13) & 0x3);
+    const uint32_t nsmc = (uint32_t)((smidr >> 56) & 0xF);
+    const size_t shared_count = nsmc == 0xF ? 1 : (size_t)nsmc + 1;
+    const uint32_t affinity = (uint32_t)(smidr & 0xFFFu);
+    const uint32_t affinity2 = (uint32_t)((smidr >> 32) & 0xFFFFFu);
+    const uint32_t id = (affinity2 << 12) | affinity;
+
+    if (nsmc == 0xF) {
+        GGML_LOG_WARN("kleidiai: NSMC detected as 0xF indicating reseved value, setting min safe shared SMCU count to 1");
+    }
+
+    switch (sh) {
+        case 2: // private SMCU
+            ++num_private;
+            break;
+        case 3: // shared SMCU
+            if (shared_counts[id] < shared_count) {
+                shared_counts[id] = shared_count;
+            }
+            break;
+        case 0:
+            if (id == 0) {
+                ++num_private;
+            } else if (shared_counts[id] < shared_count) {
+                shared_counts[id] = shared_count;
+            }
+            break;
+        default:
+            break;
+    }
+}
+
 static size_t detect_num_smcus() {
-    if (!ggml_cpu_has_sme()) {
+    const auto runtime_feat = ggml_feats_get_arch64_runtime();
+    if (!runtime_feat.has_sme) {
         return 0;
     }
 
 #if defined(__linux__) && defined(__aarch64__)
     // Linux/aarch64: Best-effort count of Streaming Mode Compute Units (SMCUs) via SMIDR_EL1 sysfs.
     size_t num_private = 0;
-    std::set<uint32_t> shared_ids;
+    std::map<uint32_t, size_t> shared_counts;
 
-    for (size_t cpu = 0;; ++cpu) {
+    const std::vector<size_t> cpus = detect_cpu_ids();
+    for (const size_t cpu : cpus) {
         const std::string path =
             "/sys/devices/system/cpu/cpu" + std::to_string(cpu) +
             "/regs/identification/smidr_el1";
 
         std::ifstream file(path);
         if (!file.is_open()) {
-            break;
+            continue;
         }
 
         uint64_t smidr = 0;
@@ -118,54 +210,69 @@ static size_t detect_num_smcus() {
             continue;
         }
 
-        // Arm ARM: SMIDR_EL1
-        const uint32_t sh = (uint32_t)((smidr >> 13) & 0x3);
-        // Build an "affinity-like" identifier for shared SMCUs.
-        // Keep the original packing logic, but isolate it here.
-        const uint32_t id = (uint32_t)((smidr & 0xFFFu) | ((smidr >> 20) & 0xFFFFF000u));
-
-        switch (sh) {
-            case 0b10: // private SMCU
-                ++num_private;
-                break;
-            case 0b11: // shared SMCU
-                shared_ids.emplace(id);
-                break;
-            case 0b00:
-                // Ambiguous / implementation-defined. Be conservative:
-                // treat id==0 as private, otherwise as shared.
-                if (id == 0) ++num_private;
-                else shared_ids.emplace(id);
-                break;
-            default:
-                break;
-        }
+        add_smcus_from_smidr(smidr, num_private, shared_counts);
     }
 
-    return num_private + shared_ids.size();
+    size_t total = num_private;
+    for (const auto & entry : shared_counts) {
+        total += entry.second;
+    }
+    return total;
 
 #elif defined(__APPLE__) && defined(__aarch64__)
-    // table for known M4 variants. Users can override via GGML_KLEIDIAI_SME=<n>.
-    char chip_name[256] = {};
-    size_t size = sizeof(chip_name);
+    int perf_levels = 0;
+    size_t size = sizeof(perf_levels);
+    if (sysctlbyname("hw.nperflevels", &perf_levels, &size, nullptr, 0) != 0 ||
+        size != sizeof(perf_levels) || perf_levels <= 0) {
+        return 0;
+    }
 
-    if (sysctlbyname("machdep.cpu.brand_string", chip_name, &size, nullptr, 0) == 0) {
-        const std::string brand(chip_name);
+    size_t units = 0;
+    for (int i = 0; i < perf_levels; ++i) {
+        char key[64] = {};
+        int physical_cpus = 0;
+        int cpus_per_l2 = 0;
 
-        struct ModelSMCU { const char *match; size_t smcus; };
-        static const ModelSMCU table[] = {
-            { "M4 Ultra", 2 },
-            { "M4 Max",   2 },
-            { "M4 Pro",   2 },
-            { "M4",       1 },
-        };
+        snprintf(key, sizeof(key), "hw.perflevel%d.physicalcpu", i);
+        size = sizeof(physical_cpus);
+        if (sysctlbyname(key, &physical_cpus, &size, nullptr, 0) != 0 ||
+            size != sizeof(physical_cpus) || physical_cpus <= 0) {
+            continue;
+        }
 
-        for (const auto &e : table) {
-            if (brand.find(e.match) != std::string::npos) {
-                return e.smcus;
-            }
+        snprintf(key, sizeof(key), "hw.perflevel%d.cpusperl2", i);
+        size = sizeof(cpus_per_l2);
+        if (sysctlbyname(key, &cpus_per_l2, &size, nullptr, 0) != 0 ||
+            size != sizeof(cpus_per_l2) || cpus_per_l2 <= 0) {
+            continue;
+        }
+
+        snprintf(key, sizeof(key), "hw.perflevel%d.name", i);
+        size = 0;
+        if (sysctlbyname(key, nullptr, &size, nullptr, 0) != 0 || size == 0) {
+            continue;
+        }
+
+        std::string name(size, '\0');
+        if (sysctlbyname(key, &name[0], &size, nullptr, 0) != 0) {
+            continue;
+        }
+        name.resize(size);
+        while (!name.empty() && name.back() == '\0') {
+            name.pop_back();
+        }
+
+        if (apple_sme_counted_perf_level(name)) {
+            units += (size_t) ((physical_cpus + cpus_per_l2 - 1) / cpus_per_l2);
         }
     }
+
+    return units;
+
+#elif defined(_WIN32) && (defined(_M_ARM64) || defined(__aarch64__))
+    // No verified Windows arm64 SMCU detection path yet. Return unknown and use
+    // GGML_KLEIDIAI_SME=N as a diagnostics/debug override for SME thread cap
+    // calibration until a detection mechanism is verified on real hardware.
     return 0;
 
 #else
@@ -198,15 +305,18 @@ static void init_kleidiai_context(void) {
     if (!initialized) {
         initialized = true;
 
+        // Optional diagnostics/debug overrides; production defaults come from runtime detection.
         const char *env_sme         = getenv("GGML_KLEIDIAI_SME");
         const char *env_threads     = getenv("GGML_TOTAL_THREADS");
         const char *env_chunk_mult  = getenv("GGML_KLEIDIAI_CHUNK_MULTIPLIER");
 
+        const auto runtime_feat = ggml_feats_get_arch64_runtime();
+
         size_t detected_smcus = 0;
 
-        ctx.features  = (ggml_cpu_has_dotprod()     ? CPU_FEATURE_DOTPROD : CPU_FEATURE_NONE) |
-                        (ggml_cpu_has_matmul_int8() ? CPU_FEATURE_I8MM    : CPU_FEATURE_NONE) |
-                        ((ggml_cpu_has_sve() && ggml_cpu_get_sve_cnt() == QK8_0) ? CPU_FEATURE_SVE : CPU_FEATURE_NONE);
+        ctx.features  = (runtime_feat.has_dotprod  ? CPU_FEATURE_DOTPROD : CPU_FEATURE_NONE) |
+                        (runtime_feat.has_i8mm     ? CPU_FEATURE_I8MM    : CPU_FEATURE_NONE) |
+                        (runtime_feat.sve_cnt == QK8_0 ? CPU_FEATURE_SVE : CPU_FEATURE_NONE);
 
         if (env_threads) {
             bool ok = false;
@@ -224,54 +334,54 @@ static void init_kleidiai_context(void) {
             }
         }
 
-        // SME policy:
-        // - env unset => auto-detect SMCUs; enable SME only if detected > 0.
-        // - env=0     => force off.
-        // - env>0     => force N cores, if the binary was built with SME.
         int sme_cores = 0;
         bool sme_env_ok = false;
         bool sme_env_set = (env_sme != nullptr);
 
+        const bool has_supported_sme_family = runtime_feat.has_sme;
+        bool sme_cap_detected = false;
+
+        if (has_supported_sme_family) {
+            detected_smcus = detect_num_smcus();
+            sme_cap_detected = detected_smcus > 0;
+            // Some platforms expose SME without exposing a calibrated SMCU count.
+            // Use one SME thread as the conservative default; add platform SMCU detection to raise it.
+            sme_cores = sme_cap_detected ? (int)detected_smcus : 1;
+
+            if (!sme_env_set && !sme_cap_detected) {
+                GGML_LOG_INFO("kleidiai: SME detected; SMCU count unavailable, using conservative SME thread cap=1\n");
+            }
+        }
+
+        // Runtime-detect SME support and available SMCUs first. The detected SMCU
+        // count is used as the SME thread cap, and GGML_KLEIDIAI_SME can debug-override that:
+        //   - unset: use runtime detection.
+        //   - 0:     disable SME-family kernels.
+        //   - N > 0: use N as the SME thread cap, if an SME-family kernel is selectable.
         if (sme_env_set) {
             bool ok = false;
             int v = parse_uint_env(env_sme, "GGML_KLEIDIAI_SME", &ok);
             sme_env_ok = ok;
 
-            if (!ok) {
-                GGML_LOG_WARN("kleidiai: GGML_KLEIDIAI_SME set but parsing failed; falling back to runtime SME-core detection\n");
-                detected_smcus = detect_num_smcus();
-                sme_cores = detected_smcus > 0 ? (int)detected_smcus : 0;
-            } else if (v == 0) {
-                sme_cores = 0;
-            } else if (!ggml_cpu_has_sme()) {
-                GGML_LOG_WARN("kleidiai: GGML_KLEIDIAI_SME=%d but the binary was not built with SME; disabling SME\n", v);
-                sme_cores = 0;
+            if (ok) {
+                if (has_supported_sme_family) {
+                    sme_cores = v;
+                } else {
+                    if (v > 0) {
+                        GGML_LOG_WARN("kleidiai: GGML_KLEIDIAI_SME=%d but SME is not supported on this CPU; disabling SME-family kernels\n", v);
+                    }
+                    sme_cores = 0;
+                }
             } else {
-                sme_cores = v;
+                GGML_LOG_WARN("kleidiai: GGML_KLEIDIAI_SME set but parsing failed; using automatic SME thread cap\n");
             }
-        } else {
-            detected_smcus = detect_num_smcus();
-            sme_cores = detected_smcus > 0 ? (int)detected_smcus : 0;
         }
 
-        if (!sme_env_set && ggml_cpu_has_sme() && sme_cores == 0) {
-            GGML_LOG_WARN("kleidiai: runtime SME-core detection returned 0; falling back to NEON\n");
-        }
-
-        if (sme_cores > 0) {
+        if (sme_cores > 0 && has_supported_sme_family) {
             ctx.features |= CPU_FEATURE_SME;
-#if defined(__aarch64__) && defined(__linux__)
-            // ARM guarantees SME2 implies SME, so only check SME2 when SME is enabled.
-            if (getauxval(AT_HWCAP2) & HWCAP2_SME2) {
+            if (runtime_feat.has_sme2) {
                 ctx.features |= CPU_FEATURE_SME2;
             }
-#elif defined(__aarch64__) && defined(__APPLE__)
-            int feat_sme2 = 0;
-            size_t size = sizeof(feat_sme2);
-            if (sysctlbyname("hw.optional.arm.FEAT_SME2", &feat_sme2, &size, NULL, 0) == 0 && feat_sme2) {
-                ctx.features |= CPU_FEATURE_SME2;
-            }
-#endif
         }
 
         // Kernel selection
@@ -297,16 +407,19 @@ static void init_kleidiai_context(void) {
             GGML_LOG_INFO("kleidiai: primary f32 kernel feature %s\n", cpu_feature_to_string(ctx.kernels_f32->required_cpu));
         }
 
-        ctx.sme_thread_cap = (ctx.features & CPU_FEATURE_SME) ? sme_cores : 0;
+        const bool has_selected_sme_family_kernel =
+            (ctx.kernels_q4  && is_sme_family(ctx.kernels_q4->required_cpu)) ||
+            (ctx.kernels_q8  && is_sme_family(ctx.kernels_q8->required_cpu)) ||
+            (ctx.kernels_f32 && is_sme_family(ctx.kernels_f32->required_cpu));
+        ctx.sme_thread_cap = has_selected_sme_family_kernel ? sme_cores : 0;
 
-        if (ctx.features & CPU_FEATURE_SME) {
-            const bool has_sme2 = (ctx.features & CPU_FEATURE_SME2) != CPU_FEATURE_NONE;
+        if (has_selected_sme_family_kernel) {
             if (sme_env_set && sme_env_ok && sme_cores > 0) {
-                GGML_LOG_INFO("kleidiai: SME%s enabled (GGML_KLEIDIAI_SME=%d override)\n",
-                    has_sme2 ? "2" : "", sme_cores);
+                GGML_LOG_INFO("kleidiai: SME enabled (GGML_KLEIDIAI_SME=%d debug override)\n", sme_cores);
+            } else if (sme_cap_detected) {
+                GGML_LOG_INFO("kleidiai: SME enabled (runtime-detected SME thread cap=%d)\n", sme_cores);
             } else {
-                GGML_LOG_INFO("kleidiai: SME%s enabled (runtime-detected SME cores=%d)\n",
-                    has_sme2 ? "2" : "", sme_cores);
+                GGML_LOG_INFO("kleidiai: SME enabled (runtime SME detected, conservative thread cap=%d)\n", sme_cores);
             }
         } else {
             GGML_LOG_INFO("kleidiai: SME disabled\n");
@@ -467,7 +580,7 @@ static int kleidiai_collect_kernel_chain_common(
     }
 
     if (is_sme_family(primary->required_cpu)) {
-        const cpu_feature fallback_mask = static_cast<cpu_feature>(features & ~CPU_FEATURE_SME & ~CPU_FEATURE_SME2);
+        const cpu_feature fallback_mask = static_cast<cpu_feature>(features & ~(CPU_FEATURE_SME | CPU_FEATURE_SME2));
         if (fallback_mask != CPU_FEATURE_NONE) {
             ggml_kleidiai_kernels * fallback = select_fallback(fallback_mask);
             if (fallback && fallback != primary &&
@@ -1077,13 +1190,14 @@ class tensor_traits : public ggml::cpu::tensor_traits {
         const int ith_total = params->ith;
 
         int sme_slot = -1;
+        int non_sme_slot = -1;
         for (int i = 0; i < runtime_count; ++i) {
             if (is_sme_family(runtime[i].kernels->required_cpu)) {
                 sme_slot = i;
                 break;
             }
         }
-        int non_sme_slot = -1;
+
         for (int i = 0; i < runtime_count; ++i) {
             if (!is_sme_family(runtime[i].kernels->required_cpu)) {
                 non_sme_slot = i;

ggml/src/ggml-cpu/ops.cpp

@@ -8941,7 +8941,7 @@ static void ggml_compute_forward_flash_attn_ext_tiled(
             for (int tk = 0; tk < kv_tile; tk++) {
                 const char * v_data = (const char *)v->data + (ic + tk)*nbv1 + iv2*nbv2 + iv3*nbv3;
                 if (kv_type == GGML_TYPE_F16) {
-                    ggml_fp16_to_fp32_row((const ggml_fp16_t *)v_data, V32 + tk * DV, DV);
+                    ggml_cpu_fp16_to_fp32((const ggml_fp16_t *)v_data, V32 + tk * DV, DV);
                 } else {
                     memcpy(V32 + tk * DV, v_data, DV * sizeof(float));
                 }

ggml/src/ggml-cuda/ggml-cuda.cu

@@ -1865,6 +1865,37 @@ static void ggml_cuda_mul_mat(ggml_backend_cuda_context & ctx, const ggml_tensor
     ggml_cuda_mul_mat_cublas(ctx, src0, src1, dst);
 }
 
+// returns true when ggml_cuda_mul_mat_id takes the fallback path that requires stream synchronization
+// [TAG_MUL_MAT_ID_CUDA_GRAPHS]
+static bool ggml_cuda_mul_mat_id_needs_sync(const ggml_tensor * dst, const int cc) {
+    const ggml_tensor * src0 = dst->src[0];
+    const ggml_tensor * src1 = dst->src[1];
+
+    if (src1->type != GGML_TYPE_F32 || dst->type != GGML_TYPE_F32) {
+        return true;
+    }
+
+    if (dst->ne[2] <= MMVQ_MAX_BATCH_SIZE) {
+        if (ggml_is_quantized(src0->type)) {
+            if (dst->ne[2] <= get_mmvq_mmid_max_batch(src0->type, cc)) {
+                return false;
+            }
+        } else if (GGML_CUDA_CC_IS_AMD(cc)) {
+            return false;
+        }
+    }
+
+    if (ggml_cuda_should_use_mmq(src0->type, cc, src1->ne[2], /*n_experts=*/src0->ne[2])) {
+        return false;
+    }
+
+    if (ggml_cuda_should_use_mmf(src0->type, cc, WARP_SIZE, src0->ne, src0->nb, src1->ne[2], /*mul_mat_id=*/true)) {
+        return false;
+    }
+
+    return true;
+}
+
 static void ggml_cuda_mul_mat_id(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {
     const ggml_tensor * src0 = dst->src[0];
     const ggml_tensor * src1 = dst->src[1];
@@ -1907,7 +1938,7 @@ static void ggml_cuda_mul_mat_id(ggml_backend_cuda_context & ctx, ggml_tensor *
     }
 
     // note: this path should not be reached when recording CUDA graphs, because it requires stream synchronization
-    // TODO: add asserts to verify this. should work with CUDA, HIP, etc.
+    GGML_ASSERT(ggml_cuda_mul_mat_id_needs_sync(dst, cc));
     cudaStream_t stream = ctx.stream();
 
     GGML_ASSERT(nb12 % nb11 == 0);
@@ -2522,10 +2553,8 @@ static bool ggml_cuda_graph_check_compability(ggml_cgraph * cgraph) {
         // [TAG_MUL_MAT_ID_CUDA_GRAPHS]
         if (node->op == GGML_OP_MUL_MAT_ID) {
             const int cc = ggml_cuda_info().devices[ggml_cuda_get_device()].cc;
-            const int mmvq_mmid_max = get_mmvq_mmid_max_batch(node->src[0]->type, cc);
-            if (!ggml_is_quantized(node->src[0]->type) || node->ne[2] > mmvq_mmid_max) {
-                // under these conditions, the mul_mat_id operation will need to synchronize the stream, so we cannot use CUDA graphs
-                // TODO: figure out a way to enable for larger batch sizes, without hurting performance
+            if (ggml_cuda_mul_mat_id_needs_sync(node, cc)) {
+                // the mul_mat_id fallback path synchronizes the stream, so we cannot use CUDA graphs
                 // ref: https://github.com/ggml-org/llama.cpp/pull/18958
                 use_cuda_graph = false;
 #ifndef NDEBUG

ggml/src/ggml-cuda/wkv.cu

@@ -141,6 +141,57 @@ static __global__ void rwkv_wkv7_f32(const int B, const int T, const int C, cons
     }
 }
 
+template <int rows_per_block>
+static __global__ void __launch_bounds__(WARP_SIZE * rows_per_block, 2)
+rwkv_wkv7_f32_t1_warp_row(const int T, const int C, const int H, const float * r, const float * w, const float * k, const float * v, const float * a, const float * b, const float * s, float * dst) {
+    constexpr int head_size = CUDA_WKV_BLOCK_SIZE;
+    constexpr int half_head = head_size / 2;
+
+    const int lane = threadIdx.x;
+    const int row  = blockIdx.y * rows_per_block + threadIdx.y;
+    const int bid  = blockIdx.x;
+
+    const int batch_i = bid / H;
+    const int head_i  = bid % H;
+    const int state_size = C * head_size;
+    const int head_off = head_i * head_size;
+    const int t = batch_i * C + head_off + row;
+
+    __shared__ float _r[head_size], _w[head_size], _k[head_size], _a[head_size], _b[head_size];
+
+    if (threadIdx.y == 0) {
+        _r[lane] = r[batch_i * C + head_off + lane];
+        _w[lane] = w[batch_i * C + head_off + lane];
+        _k[lane] = k[batch_i * C + head_off + lane];
+        _a[lane] = a[batch_i * C + head_off + lane];
+        _b[lane] = b[batch_i * C + head_off + lane];
+
+        _r[lane + half_head] = r[batch_i * C + head_off + lane + half_head];
+        _w[lane + half_head] = w[batch_i * C + head_off + lane + half_head];
+        _k[lane + half_head] = k[batch_i * C + head_off + lane + half_head];
+        _a[lane + half_head] = a[batch_i * C + head_off + lane + half_head];
+        _b[lane + half_head] = b[batch_i * C + head_off + lane + half_head];
+    }
+    __syncthreads();
+
+    const int64_t state_base = batch_i * state_size + head_i * head_size * head_size + row * head_size;
+    const float s0 = s[state_base + lane];
+    const float s1 = s[state_base + lane + half_head];
+    const float sa = warp_reduce_sum(_a[lane] * s0 + _a[lane + half_head] * s1);
+
+    const float vt  = v[t];
+    const float st0 = s0 * _w[lane]             + _k[lane]             * vt + sa * _b[lane];
+    const float st1 = s1 * _w[lane + half_head] + _k[lane + half_head] * vt + sa * _b[lane + half_head];
+    const float y   = warp_reduce_sum(st0 * _r[lane] + st1 * _r[lane + half_head]);
+
+    dst[T * C + state_base + lane]             = st0;
+    dst[T * C + state_base + lane + half_head] = st1;
+
+    if (lane == 0) {
+        dst[t] = y;
+    }
+}
+
 void ggml_cuda_op_rwkv_wkv6(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {
     const float * k_d  = (const float *)dst->src[0]->data;
     const float * v_d  = (const float *)dst->src[1]->data;
@@ -191,7 +242,10 @@ void ggml_cuda_op_rwkv_wkv7(ggml_backend_cuda_context & ctx, ggml_tensor * dst)
     GGML_ASSERT(C % H == 0);
     GGML_ASSERT(C / H == CUDA_WKV_BLOCK_SIZE || C / H == CUDA_WKV_BLOCK_SIZE * 2);
 
-    if (C / H == CUDA_WKV_BLOCK_SIZE) {
+    if (T / B == 1 && C / H == CUDA_WKV_BLOCK_SIZE) {
+        constexpr int rows_per_block = 4;
+        rwkv_wkv7_f32_t1_warp_row<rows_per_block><<<dim3(B * H, CUDA_WKV_BLOCK_SIZE / rows_per_block), dim3(WARP_SIZE, rows_per_block), 0, stream>>>(T, C, H, r_d, w_d, k_d, v_d, a_d, b_d, s_d, dst_d);
+    } else if (C / H == CUDA_WKV_BLOCK_SIZE) {
         rwkv_wkv7_f32<CUDA_WKV_BLOCK_SIZE><<<B * H, C / H, 0, stream>>>(B, T, C, H, r_d, w_d, k_d, v_d, a_d, b_d, s_d, dst_d);
     } else {
         rwkv_wkv7_f32<CUDA_WKV_BLOCK_SIZE * 2><<<B * H, C / H, 0, stream>>>(B, T, C, H, r_d, w_d, k_d, v_d, a_d, b_d, s_d, dst_d);

ggml/src/ggml-feats.h

@@ -0,0 +1,166 @@
+#pragma once
+
+#if defined(__aarch64__) || defined(_M_ARM64)
+
+#if defined(__linux__)
+#include <sys/auxv.h>
+#include <sys/prctl.h>
+
+#if !defined(HWCAP2_SVE2)
+#define HWCAP2_SVE2 (1ULL << 1)
+#endif
+
+#if !defined(HWCAP_FPHP)
+#define HWCAP_FPHP (1 << 9)
+#endif
+
+#if !defined(HWCAP_ASIMDHP)
+#define HWCAP_ASIMDHP (1 << 10)
+#endif
+
+#if !defined(HWCAP2_I8MM)
+#define HWCAP2_I8MM (1ULL << 13)
+#endif
+
+#if !defined(HWCAP_ASIMDDP)
+#define HWCAP_ASIMDDP (1 << 20)
+#endif
+
+#if !defined(HWCAP_SVE)
+#define HWCAP_SVE (1 << 22)
+#endif
+
+#if !defined(HWCAP2_SME)
+#define HWCAP2_SME (1ULL << 23)
+#endif
+
+#if !defined(HWCAP2_SME2)
+#define HWCAP2_SME2 (1ULL << 37)
+#endif
+
+#if !defined(PR_SVE_GET_VL)
+#define PR_SVE_GET_VL 51
+#endif
+
+#if !defined(PR_SVE_VL_LEN_MASK)
+#define PR_SVE_VL_LEN_MASK 0xffff
+#endif
+
+#elif defined(__APPLE__)
+#include <sys/sysctl.h>
+#elif defined(_WIN32)
+#include <windows.h>
+
+#if !defined(PF_ARM_V82_DP_INSTRUCTIONS_AVAILABLE)
+#define PF_ARM_V82_DP_INSTRUCTIONS_AVAILABLE 43
+#endif
+
+#if !defined(PF_ARM_SVE_INSTRUCTIONS_AVAILABLE)
+#define PF_ARM_SVE_INSTRUCTIONS_AVAILABLE 46
+#endif
+
+#if !defined(PF_ARM_SVE2_INSTRUCTIONS_AVAILABLE)
+#define PF_ARM_SVE2_INSTRUCTIONS_AVAILABLE 47
+#endif
+
+#if !defined(PF_ARM_V82_I8MM_INSTRUCTIONS_AVAILABLE)
+#define PF_ARM_V82_I8MM_INSTRUCTIONS_AVAILABLE 66
+#endif
+
+#if !defined(PF_ARM_V82_FP16_INSTRUCTIONS_AVAILABLE)
+#define PF_ARM_V82_FP16_INSTRUCTIONS_AVAILABLE 67
+#endif
+
+#if !defined(PF_ARM_SME_INSTRUCTIONS_AVAILABLE)
+#define PF_ARM_SME_INSTRUCTIONS_AVAILABLE 70
+#endif
+
+#if !defined(PF_ARM_SME2_INSTRUCTIONS_AVAILABLE)
+#define PF_ARM_SME2_INSTRUCTIONS_AVAILABLE 71
+#endif
+
+#endif
+
+typedef struct ggml_feats_arch64_runtime {
+    bool has_dotprod;
+    bool has_fp16;
+    bool has_sve;
+    bool has_sve2;
+    bool has_i8mm;
+    bool has_sme;
+    bool has_sme2;
+    int sve_cnt;
+} ggml_feats_arch64_runtime_t;
+
+static inline ggml_feats_arch64_runtime_t ggml_feats_get_arch64_runtime(void) {
+    ggml_feats_arch64_runtime_t runtime_feat = {};
+
+#if defined(__linux__)
+    const unsigned long hwcap  = getauxval(AT_HWCAP);
+    const unsigned long hwcap2 = getauxval(AT_HWCAP2);
+
+    runtime_feat.has_dotprod = !!(hwcap & HWCAP_ASIMDDP);
+    runtime_feat.has_fp16    = !!(hwcap & HWCAP_FPHP) && !!(hwcap & HWCAP_ASIMDHP);;
+    runtime_feat.has_sve     = !!(hwcap & HWCAP_SVE);
+    runtime_feat.has_sve2    = !!(hwcap2 & HWCAP2_SVE2);
+    runtime_feat.has_i8mm    = !!(hwcap2 & HWCAP2_I8MM);
+    runtime_feat.has_sme     = !!(hwcap2 & HWCAP2_SME);
+    runtime_feat.has_sme2    = !!(hwcap2 & HWCAP2_SME2);
+
+    if (runtime_feat.has_sve) {
+        const int vl = prctl(PR_SVE_GET_VL);
+        if (vl >= 0) {
+            runtime_feat.sve_cnt = vl & PR_SVE_VL_LEN_MASK;
+        }
+    }
+#elif defined(__APPLE__)
+    int oldp = 0;
+    size_t size = sizeof(oldp);
+
+    if (sysctlbyname("hw.optional.arm.FEAT_DotProd", &oldp, &size, nullptr, 0) == 0) {
+        runtime_feat.has_dotprod = static_cast<bool>(oldp);
+    }
+
+    if (sysctlbyname("hw.optional.arm.FEAT_FP16", &oldp, &size, nullptr, 0) == 0) {
+        runtime_feat.has_fp16 = static_cast<bool>(oldp);
+    }
+
+    if (sysctlbyname("hw.optional.arm.FEAT_SVE", &oldp, &size, nullptr, 0) == 0) {
+        runtime_feat.has_sve = static_cast<bool>(oldp);
+    }
+
+    if (sysctlbyname("hw.optional.arm.FEAT_SVE2", &oldp, &size, nullptr, 0) == 0) {
+        runtime_feat.has_sve2 = static_cast<bool>(oldp);
+    }
+
+    if (sysctlbyname("hw.optional.arm.FEAT_I8MM", &oldp, &size, nullptr, 0) == 0) {
+        runtime_feat.has_i8mm = static_cast<bool>(oldp);
+    }
+
+    if (sysctlbyname("hw.optional.arm.FEAT_SME", &oldp, &size, nullptr, 0) == 0) {
+        runtime_feat.has_sme = static_cast<bool>(oldp);
+    }
+
+    if (sysctlbyname("hw.optional.arm.FEAT_SME2", &oldp, &size, nullptr, 0) == 0) {
+        runtime_feat.has_sme2 = static_cast<bool>(oldp);
+    }
+
+    // Apple does not support userspace non-streaming SVE; keep SVE vector length unknown.
+    runtime_feat.sve_cnt = 0;
+#elif defined (_WIN32)
+    runtime_feat.has_dotprod = IsProcessorFeaturePresent(PF_ARM_V82_DP_INSTRUCTIONS_AVAILABLE) != 0;
+    runtime_feat.has_fp16    = IsProcessorFeaturePresent(PF_ARM_V82_FP16_INSTRUCTIONS_AVAILABLE) != 0;
+    runtime_feat.has_sve     = IsProcessorFeaturePresent(PF_ARM_SVE_INSTRUCTIONS_AVAILABLE) != 0;
+    runtime_feat.has_sve2    = IsProcessorFeaturePresent(PF_ARM_SVE2_INSTRUCTIONS_AVAILABLE) != 0;
+    runtime_feat.has_i8mm    = IsProcessorFeaturePresent(PF_ARM_V82_I8MM_INSTRUCTIONS_AVAILABLE) != 0;
+    runtime_feat.has_sme     = IsProcessorFeaturePresent(PF_ARM_SME_INSTRUCTIONS_AVAILABLE) != 0;
+    runtime_feat.has_sme2    = IsProcessorFeaturePresent(PF_ARM_SME2_INSTRUCTIONS_AVAILABLE) != 0;
+
+    // Windows exposes SVE feature presence, but not the runtime SVE vector length here.
+    runtime_feat.sve_cnt = 0;
+#endif
+
+    return runtime_feat;
+}
+
+#endif // defined(__aarch64__) || defined(_M_ARM64)

ggml/src/ggml-hip/CMakeLists.txt

@@ -126,9 +126,6 @@ if (GGML_HIP_EXPORT_METRICS)
     set(CMAKE_HIP_FLAGS "${CMAKE_HIP_FLAGS} -Rpass-analysis=kernel-resource-usage --save-temps")
 endif()
 
-# Fast math for HIP, like CUDA's -use_fast_math. Not -ffast-math: that implies -ffinite-math-only, which breaks ggml's INFINITY masking and produces NaNs.
-set(CMAKE_HIP_FLAGS "${CMAKE_HIP_FLAGS} -funsafe-math-optimizations")
-
 if (NOT GGML_CUDA_FA)
     add_compile_definitions(GGML_CUDA_NO_FA)
 endif()

ggml/src/ggml-metal/ggml-metal-device.cpp

@@ -953,6 +953,11 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_mul_mv(ggml_meta
                 nr0 = N_R0_IQ4_XS;
                 smem = 32*sizeof(float);
             } break;
+        case GGML_TYPE_TQ2_0:
+            {
+                nsg = N_SG_TQ2_0;
+                nr0 = N_R0_TQ2_0;
+            } break;
         default:
             {
                 GGML_LOG_ERROR("Asserting on type %d\n", (int) tsrc0);
@@ -1182,6 +1187,11 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_mul_mv_id(ggml_m
                 nr0 = N_R0_IQ4_XS;
                 smem = 32*sizeof(float);
             } break;
+        case GGML_TYPE_TQ2_0:
+            {
+                nsg = N_SG_TQ2_0;
+                nr0 = N_R0_TQ2_0;
+            } break;
         default:
             {
                 GGML_LOG_ERROR("Asserting on type %d\n", (int)op->src[2]->type);

ggml/src/ggml-metal/ggml-metal-device.m

@@ -1407,6 +1407,7 @@ bool ggml_metal_device_supports_op(ggml_metal_device_t dev, const struct ggml_te
                            case GGML_TYPE_Q5_0:
                            case GGML_TYPE_Q5_1:
                            case GGML_TYPE_IQ4_NL:
+                           case GGML_TYPE_TQ2_0:
                            case GGML_TYPE_I32:
                                 return true;
                            default:
@@ -1435,6 +1436,7 @@ bool ggml_metal_device_supports_op(ggml_metal_device_t dev, const struct ggml_te
                     case GGML_TYPE_Q5_0:
                     case GGML_TYPE_Q5_1:
                     case GGML_TYPE_Q8_0:
+                    case GGML_TYPE_TQ2_0:
                         switch (op->type) {
                             case GGML_TYPE_F32:
                             case GGML_TYPE_F16:
@@ -1470,6 +1472,7 @@ bool ggml_metal_device_supports_op(ggml_metal_device_t dev, const struct ggml_te
                     case GGML_TYPE_Q5_0:
                     case GGML_TYPE_Q5_1:
                     case GGML_TYPE_IQ4_NL:
+                    case GGML_TYPE_TQ2_0:
                         return true;
                     default:
                         return false;

ggml/src/ggml-metal/ggml-metal-impl.h

@@ -87,6 +87,9 @@
 #define N_R0_IQ4_XS 2
 #define N_SG_IQ4_XS 2
 
+#define N_R0_TQ2_0 4
+#define N_SG_TQ2_0 2
+
 // function constants offsets
 #define FC_FLASH_ATTN_EXT_PAD          100
 #define FC_FLASH_ATTN_EXT_BLK          200

ggml/src/ggml-metal/ggml-metal.metal

@@ -468,6 +468,34 @@ void quantize_iq4_nl(device const float * src, device block_iq4_nl & dst) {
     dst.d = sumq2 > 0 ? sumqx/sumq2 : d;
 }
 
+void quantize_tq2_0(device const float * src, device block_tq2_0 & dst) {
+#pragma METAL fp math_mode(safe)
+    float amax = 0.0f; // absolute max
+
+    for (int j = 0; j < QK_K; j++) {
+        const float v = src[j];
+        amax = MAX(amax, fabs(v));
+    }
+
+    const float d = amax;
+    const float id = d ? 1.0f/d : 0.0f;
+
+    dst.d = (half) d;
+
+    for (int j = 0; j < QK_K/4; j += 32) {
+        for (int m = 0; m < 32; ++m) {
+            uint8_t q = 0;
+            for (int n = 0; n < 4; ++n) {
+                // -1, 0, 1 -> 0, 1, 2
+                int xi = (int)round(src[m + n*32] * id) + 1;
+                q += (uint8_t)((xi & 3) << (2*n));
+            }
+            dst.qs[j + m] = q;
+        }
+        src += 4*32;
+    }
+}
+
 template <typename type4x4>
 void dequantize_q4_1(device const block_q4_1 * xb, short il, thread type4x4 & reg) {
     device const uint16_t * qs = ((device const uint16_t *)xb + 2);
@@ -1021,6 +1049,25 @@ void dequantize_iq4_xs(device const block_iq4_xs * xb, short il, thread type4x4
     }
 }
 
+template <typename type4x4>
+void dequantize_tq2_0(device const block_tq2_0 * xb, short il, thread type4x4 & reg) {
+    device const uint8_t * qs = xb->qs;
+    const float d = xb->d;
+
+    float4x4 reg_f;
+
+    // 2 bits per element, 4 elements per byte, 128 elements per 32-byte group
+    const short base = il * 16;
+    for (int k = 0; k < 16; k++) {
+        const int i = base + k;
+        const int byte = ((i >> 7) & 1) * 32 + (i & 31);
+        const int l = (i >> 5) & 3;
+        reg_f[k/4][k%4] = d * (float)(((qs[byte] >> (2*l)) & 3) - 1);
+    }
+
+    reg = (type4x4) reg_f;
+}
+
 enum ggml_sort_order {
     GGML_SORT_ORDER_ASC,
     GGML_SORT_ORDER_DESC,
@@ -8001,6 +8048,7 @@ template [[host_name("kernel_cpy_f32_q4_1")]]   kernel cpy_f_q_t kernel_cpy_f32_
 template [[host_name("kernel_cpy_f32_q5_0")]]   kernel cpy_f_q_t kernel_cpy_f32_q<QK5_0,  block_q5_0,   quantize_q5_0>;
 template [[host_name("kernel_cpy_f32_q5_1")]]   kernel cpy_f_q_t kernel_cpy_f32_q<QK5_1,  block_q5_1,   quantize_q5_1>;
 template [[host_name("kernel_cpy_f32_iq4_nl")]] kernel cpy_f_q_t kernel_cpy_f32_q<QK4_NL, block_iq4_nl, quantize_iq4_nl>;
+template [[host_name("kernel_cpy_f32_tq2_0")]]  kernel cpy_f_q_t kernel_cpy_f32_q<QK_K,   block_tq2_0,  quantize_tq2_0>;
 
 template<typename T4x4, typename block_q, short nl, void (*dequantize_func)(device const block_q *, short, thread T4x4 &)>
 kernel void kernel_cpy_q_f32(
@@ -8048,6 +8096,8 @@ template [[host_name("kernel_cpy_q5_0_f32")]] kernel cpy_q_f_t kernel_cpy_q_f32<
 template [[host_name("kernel_cpy_q5_1_f32")]] kernel cpy_q_f_t kernel_cpy_q_f32<float4x4, block_q5_1, 2, dequantize_q5_1>;
 template [[host_name("kernel_cpy_q8_0_f32")]] kernel cpy_q_f_t kernel_cpy_q_f32<float4x4, block_q8_0, 2, dequantize_q8_0>;
 
+template [[host_name("kernel_cpy_tq2_0_f32")]] kernel cpy_q_f_t kernel_cpy_q_f32<float4x4, block_tq2_0, QK_NL, dequantize_tq2_0>;
+
 template [[host_name("kernel_cpy_q1_0_f16")]] kernel cpy_q_f_t kernel_cpy_q_f32<half4x4, block_q1_0, 8, dequantize_q1_0>;
 template [[host_name("kernel_cpy_q2_0_f16")]] kernel cpy_q_f_t kernel_cpy_q_f32<half4x4, block_q2_0, 4, dequantize_q2_0>;
 template [[host_name("kernel_cpy_q4_0_f16")]] kernel cpy_q_f_t kernel_cpy_q_f32<half4x4, block_q4_0, 2, dequantize_q4_0>;
@@ -8056,6 +8106,8 @@ template [[host_name("kernel_cpy_q5_0_f16")]] kernel cpy_q_f_t kernel_cpy_q_f32<
 template [[host_name("kernel_cpy_q5_1_f16")]] kernel cpy_q_f_t kernel_cpy_q_f32<half4x4, block_q5_1, 2, dequantize_q5_1>;
 template [[host_name("kernel_cpy_q8_0_f16")]] kernel cpy_q_f_t kernel_cpy_q_f32<half4x4, block_q8_0, 2, dequantize_q8_0>;
 
+template [[host_name("kernel_cpy_tq2_0_f16")]] kernel cpy_q_f_t kernel_cpy_q_f32<half4x4, block_tq2_0, QK_NL, dequantize_tq2_0>;
+
 template<typename T>
 kernel void kernel_concat(
         constant ggml_metal_kargs_concat & args,
@@ -9822,6 +9874,121 @@ kernel void kernel_mul_mv_mxfp4_f32(
     kernel_mul_mv_mxfp4_f32_impl<N_R0_MXFP4, constant ggml_metal_kargs_mul_mv &>(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg);
 }
 
+template<int nr0, typename args_t>
+void kernel_mul_mv_tq2_0_f32_impl(
+        args_t args,
+        device const char * src0,
+        device const char * src1,
+        device       char * dst,
+        threadgroup  char * shmem,
+        uint3  tgpig,
+        ushort tiisg,
+        ushort sgitg) {
+    const short NSG = FC_mul_mv_nsg;
+
+    const int nb = args.ne00/QK_K;
+
+    const int r0 = tgpig.x;
+    const int r1 = tgpig.y;
+    const int im = tgpig.z;
+
+    const int first_row = (r0 * NSG + sgitg) * nr0;
+
+    const uint i12 = im%FC_mul_mv_ne12;
+    const uint i13 = im/FC_mul_mv_ne12;
+
+    const uint64_t offset1 =        r1*args.nb11 + (i12        )*args.nb12 + (i13        )*args.nb13;
+
+    device const float * y = (device const float *) (src1 + offset1);
+
+    device const block_tq2_0 * ax[nr0];
+    for (int row = 0; row < nr0; ++row) {
+        const uint64_t offset0 = (first_row + row)*args.nb01 + (i12/FC_mul_mv_r2)*args.nb02 + (i13/FC_mul_mv_r3)*args.nb03;
+        ax[row] = (device const block_tq2_0 *) ((device char *) src0 + offset0);
+    }
+
+    float sumf[nr0] = {0.f};
+
+    // 8 threads per block, NBLOCK blocks per pass, 2 halves per block per pass
+    constexpr short NBLOCK = 4;
+
+    constexpr short NB = N_SIMDWIDTH/NBLOCK; // threads per block
+
+    const short blk = tiisg / NB;    // 0..NBLOCK-1, block handled by this thread
+    const short htg = tiisg % NB;    // 0..NB-1, thread within block (0..7)
+
+    // byte and y base offsets within the block (32 elements per thread, 4 per byte)
+    device const float4 * yb4 = (device const float4 *)(y + 4*htg + blk*QK_K);
+
+    // hoisted per-byte coefficients (from y) and total y-sum, shared across rows
+    // ref: https://github.com/ggml-org/llama.cpp/pull/26980
+    float4 coef[4];
+
+    for (int ib = blk; ib < nb; ib += NBLOCK) {
+        FOR_UNROLL (short h0 = 0; h0 < 2; ++h0) {
+            const float4 y0 = yb4[ 0 + 32*h0];
+            const float4 y1 = yb4[ 8 + 32*h0];
+            const float4 y2 = yb4[16 + 32*h0];
+            const float4 y3 = yb4[24 + 32*h0];
+
+            float sumy = 0.f;
+            FOR_UNROLL (short j = 0; j < 4; ++j) {
+                coef[j] = float4(
+                        y0[j],
+                        y1[j] - 4.0f*y0[j],
+                        y2[j] - 4.0f*y1[j],
+                        y3[j] - 4.0f*y2[j]);
+
+                sumy += (y0[j] + y1[j]) + (y2[j] + y3[j]);
+            }
+
+            FOR_UNROLL (short row = 0; row < nr0; ++row) {
+                device const block_tq2_0 & xb = ax[row][ib];
+                device const uchar * qs = xb.qs + 4*htg + 32*h0;
+
+                float sum = -sumy;
+                FOR_UNROLL (short j = 0; j < 4; ++j) {
+                    // express the 2-bit field shifts (v>>2, v>>4, v>>6) as float floor ops
+                    const float v = (float)qs[j];
+
+                    const float f0 = v;
+                    const float f1 = floor(v*0.25f);    // v>>2
+                    const float f2 = floor(v*0.0625);   // v>>4
+                    const float f3 = floor(v*0.015625); // v>>6
+
+                    sum += coef[j][0]*f0 + coef[j][1]*f1 + coef[j][2]*f2 + coef[j][3]*f3;
+                }
+
+                sumf[row] += xb.d * sum;
+            }
+        }
+
+        yb4 += QK_K * NBLOCK / 4;
+    }
+
+    device float * dst_f32 = (device float *) dst + (uint64_t)im*args.ne0*args.ne1 + (uint64_t)r1*args.ne0;
+
+    for (int row = 0; row < nr0; ++row) {
+        const float tot = simd_sum(sumf[row]);
+        if (tiisg == 0 && first_row + row < args.ne01) {
+            dst_f32[first_row + row] = tot;
+        }
+    }
+}
+
+[[host_name("kernel_mul_mv_tq2_0_f32")]]
+kernel void kernel_mul_mv_tq2_0_f32(
+        constant ggml_metal_kargs_mul_mv & args,
+        device const char * src0,
+        device const char * src1,
+        device       char * dst,
+        uint3  tgpig[[threadgroup_position_in_grid]],
+        ushort tiisg[[thread_index_in_simdgroup]],
+        ushort sgitg[[simdgroup_index_in_threadgroup]]) {
+
+    kernel_mul_mv_tq2_0_f32_impl<N_R0_TQ2_0, constant ggml_metal_kargs_mul_mv &>(args, src0, src1, dst, nullptr, tgpig, tiisg, sgitg);
+}
+
 template<typename block_q, short nl, void (*dequantize_func)(device const block_q *, short, thread float4x4 &)>
 kernel void kernel_get_rows_q(
         constant ggml_metal_kargs_get_rows & args,
@@ -9915,6 +10082,38 @@ template [[host_name("kernel_get_rows_iq1_s")]]   kernel get_rows_q_t kernel_get
 template [[host_name("kernel_get_rows_iq1_m")]]   kernel get_rows_q_t kernel_get_rows_q<block_iq1_m,   QK_NL, dequantize_iq1_m>;
 template [[host_name("kernel_get_rows_iq4_nl")]]  kernel get_rows_q_t kernel_get_rows_q<block_iq4_nl,  2,     dequantize_iq4_nl>;
 template [[host_name("kernel_get_rows_iq4_xs")]]  kernel get_rows_q_t kernel_get_rows_q<block_iq4_xs,  QK_NL, dequantize_iq4_xs>;
+template [[host_name("kernel_get_rows_tq2_0")]]   kernel get_rows_q_t kernel_get_rows_q<block_tq2_0,   QK_NL, dequantize_tq2_0>;
+
+template<typename TS, typename TI, short QK, typename block_q, void (*quantize_func)(device const float *, device block_q &)>
+kernel void kernel_set_rows_q(
+        constant ggml_metal_kargs_set_rows & args,
+        device const  void * src0,
+        device const  void * src1,
+        device       float * dst,
+        uint3                tgpig[[threadgroup_position_in_grid]],
+        uint                 tiitg[[thread_index_in_threadgroup]],
+        uint3                tptg [[threads_per_threadgroup]]) {
+    const int32_t i03 = tgpig.z;
+    const int32_t i02 = tgpig.y;
+
+    const int32_t i12 = i03%args.ne12;
+    const int32_t i11 = i02%args.ne11;
+
+    const int32_t i01 = tgpig.x*tptg.y + tiitg/tptg.x;
+    if (i01 >= args.ne01) {
+        return;
+    }
+
+    const int32_t i10 = i01;
+    const TI      i1  = ((const device TI *) ((const device char *) src1 + i10*args.nb10 + i11*args.nb11 + i12*args.nb12))[0];
+
+          device block_q * dst_row = (      device block_q *) ((      device char *) dst  +  i1*args.nb1  + i02*args.nb2  + i03*args.nb3);
+    const device TS      * src_row = (const device TS      *) ((const device char *) src0 + i01*args.nb01 + i02*args.nb02 + i03*args.nb03);
+
+    for (int ind = tiitg%tptg.x; ind < args.nk0; ind += tptg.x) {
+        quantize_func(src_row + QK*ind, dst_row[ind]);
+    }
+}
 
 template<typename TS, typename TI, typename block_q, void (*quantize_func)(device const float *, device block_q &)>
 kernel void kernel_set_rows_q32(
@@ -10011,6 +10210,11 @@ template [[host_name("kernel_set_rows_f32_i32_q5_1")]]   kernel set_rows_q32_t k
 template [[host_name("kernel_set_rows_f32_i64_iq4_nl")]] kernel set_rows_q32_t kernel_set_rows_q32<float, int64_t, block_iq4_nl, quantize_iq4_nl>;
 template [[host_name("kernel_set_rows_f32_i32_iq4_nl")]] kernel set_rows_q32_t kernel_set_rows_q32<float, int32_t, block_iq4_nl, quantize_iq4_nl>;
 
+typedef decltype(kernel_set_rows_q<float, int64_t, QK_K, block_tq2_0, quantize_tq2_0>) set_rows_qK_t;
+
+template [[host_name("kernel_set_rows_f32_i64_tq2_0")]]  kernel set_rows_qK_t kernel_set_rows_q<float, int64_t, QK_K, block_tq2_0, quantize_tq2_0>;
+template [[host_name("kernel_set_rows_f32_i32_tq2_0")]]  kernel set_rows_qK_t kernel_set_rows_q<float, int32_t, QK_K, block_tq2_0, quantize_tq2_0>;
+
 kernel void kernel_diag_f32(
         constant ggml_metal_kargs_diag & args,
         device   const char * src0,
@@ -10786,6 +10990,7 @@ template [[host_name("kernel_mul_mm_iq1_s_f32")]]   kernel mul_mm_t kernel_mul_m
 template [[host_name("kernel_mul_mm_iq1_m_f32")]]   kernel mul_mm_t kernel_mul_mm<half,   half4x4,   simdgroup_half8x8,   half,   half2x4,   simdgroup_half8x8,   block_iq1_m,   QK_NL, dequantize_iq1_m,   float,  float4x4,  float, float2x4>;
 template [[host_name("kernel_mul_mm_iq4_nl_f32")]]  kernel mul_mm_t kernel_mul_mm<half,   half4x4,   simdgroup_half8x8,   half,   half2x4,   simdgroup_half8x8,   block_iq4_nl,  2,     dequantize_iq4_nl,  float,  float4x4,  float, float2x4>;
 template [[host_name("kernel_mul_mm_iq4_xs_f32")]]  kernel mul_mm_t kernel_mul_mm<half,   half4x4,   simdgroup_half8x8,   half,   half2x4,   simdgroup_half8x8,   block_iq4_xs,  QK_NL, dequantize_iq4_xs,  float,  float4x4,  float, float2x4>;
+template [[host_name("kernel_mul_mm_tq2_0_f32")]]   kernel mul_mm_t kernel_mul_mm<half,   half4x4,   simdgroup_half8x8,   half,   half2x4,   simdgroup_half8x8,   block_tq2_0,   QK_NL, dequantize_tq2_0,   float,  float4x4,  float, float2x4>;
 
 template [[host_name("kernel_mul_mm_f32_f16")]]     kernel mul_mm_t kernel_mul_mm<half,   half4x4,   simdgroup_half8x8,   half,   half2x4,   simdgroup_half8x8,   float4x4,      1,     dequantize_f32,     float,  float4x4,  half, half2x4>;
 template [[host_name("kernel_mul_mm_f16_f16")]]     kernel mul_mm_t kernel_mul_mm<half,   half4x4,   simdgroup_half8x8,   half,   half2x4,   simdgroup_half8x8,   half4x4,       1,     dequantize_f16,     half,   half4x4,   half, half2x4>;
@@ -10811,6 +11016,7 @@ template [[host_name("kernel_mul_mm_iq1_s_f16")]]   kernel mul_mm_t kernel_mul_m
 template [[host_name("kernel_mul_mm_iq1_m_f16")]]   kernel mul_mm_t kernel_mul_mm<half,   half4x4,   simdgroup_half8x8,   half,   half2x4,   simdgroup_half8x8,   block_iq1_m,   QK_NL, dequantize_iq1_m,   float,  float4x4,  half, half2x4>;
 template [[host_name("kernel_mul_mm_iq4_nl_f16")]]  kernel mul_mm_t kernel_mul_mm<half,   half4x4,   simdgroup_half8x8,   half,   half2x4,   simdgroup_half8x8,   block_iq4_nl,  2,     dequantize_iq4_nl,  float,  float4x4,  half, half2x4>;
 template [[host_name("kernel_mul_mm_iq4_xs_f16")]]  kernel mul_mm_t kernel_mul_mm<half,   half4x4,   simdgroup_half8x8,   half,   half2x4,   simdgroup_half8x8,   block_iq4_xs,  QK_NL, dequantize_iq4_xs,  float,  float4x4,  half, half2x4>;
+template [[host_name("kernel_mul_mm_tq2_0_f16")]]   kernel mul_mm_t kernel_mul_mm<half,   half4x4,   simdgroup_half8x8,   half,   half2x4,   simdgroup_half8x8,   block_tq2_0,   QK_NL, dequantize_tq2_0,   float,  float4x4,  half, half2x4>;
 
 //
 // indirect matrix-matrix multiplication
@@ -10845,6 +11051,7 @@ template [[host_name("kernel_mul_mm_id_iq1_s_f32")]]   kernel mul_mm_id kernel_m
 template [[host_name("kernel_mul_mm_id_iq1_m_f32")]]   kernel mul_mm_id kernel_mul_mm_id<half,   half4x4,   simdgroup_half8x8,   half,   half2x4,   simdgroup_half8x8,   block_iq1_m,   QK_NL, dequantize_iq1_m,   float,  float4x4,  float, float2x4>;
 template [[host_name("kernel_mul_mm_id_iq4_nl_f32")]]  kernel mul_mm_id kernel_mul_mm_id<half,   half4x4,   simdgroup_half8x8,   half,   half2x4,   simdgroup_half8x8,   block_iq4_nl,  2,     dequantize_iq4_nl,  float,  float4x4,  float, float2x4>;
 template [[host_name("kernel_mul_mm_id_iq4_xs_f32")]]  kernel mul_mm_id kernel_mul_mm_id<half,   half4x4,   simdgroup_half8x8,   half,   half2x4,   simdgroup_half8x8,   block_iq4_xs,  QK_NL, dequantize_iq4_xs,  float,  float4x4,  float, float2x4>;
+template [[host_name("kernel_mul_mm_id_tq2_0_f32")]]   kernel mul_mm_id kernel_mul_mm_id<half,   half4x4,   simdgroup_half8x8,   half,   half2x4,   simdgroup_half8x8,   block_tq2_0,   QK_NL, dequantize_tq2_0,   float,  float4x4,  float, float2x4>;
 
 template [[host_name("kernel_mul_mm_id_f32_f16")]]     kernel mul_mm_id kernel_mul_mm_id<half,   half4x4,   simdgroup_half8x8,   half,   half2x4,   simdgroup_half8x8,   float4x4,      1,     dequantize_f32,     float,  float4x4,  half, half2x4>;
 template [[host_name("kernel_mul_mm_id_f16_f16")]]     kernel mul_mm_id kernel_mul_mm_id<half,   half4x4,   simdgroup_half8x8,   half,   half2x4,   simdgroup_half8x8,   half4x4,       1,     dequantize_f16,     half,   half4x4,   half, half2x4>;
@@ -10870,6 +11077,7 @@ template [[host_name("kernel_mul_mm_id_iq1_s_f16")]]   kernel mul_mm_id kernel_m
 template [[host_name("kernel_mul_mm_id_iq1_m_f16")]]   kernel mul_mm_id kernel_mul_mm_id<half,   half4x4,   simdgroup_half8x8,   half,   half2x4,   simdgroup_half8x8,   block_iq1_m,   QK_NL, dequantize_iq1_m,   float,  float4x4,  half, half2x4>;
 template [[host_name("kernel_mul_mm_id_iq4_nl_f16")]]  kernel mul_mm_id kernel_mul_mm_id<half,   half4x4,   simdgroup_half8x8,   half,   half2x4,   simdgroup_half8x8,   block_iq4_nl,  2,     dequantize_iq4_nl,  float,  float4x4,  half, half2x4>;
 template [[host_name("kernel_mul_mm_id_iq4_xs_f16")]]  kernel mul_mm_id kernel_mul_mm_id<half,   half4x4,   simdgroup_half8x8,   half,   half2x4,   simdgroup_half8x8,   block_iq4_xs,  QK_NL, dequantize_iq4_xs,  float,  float4x4,  half, half2x4>;
+template [[host_name("kernel_mul_mm_id_tq2_0_f16")]]   kernel mul_mm_id kernel_mul_mm_id<half,   half4x4,   simdgroup_half8x8,   half,   half2x4,   simdgroup_half8x8,   block_tq2_0,   QK_NL, dequantize_tq2_0,   float,  float4x4,  half, half2x4>;
 
 //
 // matrix-vector multiplication
@@ -11027,6 +11235,7 @@ template [[host_name("kernel_mul_mv_id_iq3_s_f32")]]   kernel kernel_mul_mv_id_t
 template [[host_name("kernel_mul_mv_id_iq2_s_f32")]]   kernel kernel_mul_mv_id_t kernel_mul_mv_id<mmv_fn<kernel_mul_mv_iq2_s_f32_impl  <N_R0_IQ2_S>>>;
 template [[host_name("kernel_mul_mv_id_iq4_nl_f32")]]  kernel kernel_mul_mv_id_t kernel_mul_mv_id<mmv_fn<kernel_mul_mv_iq4_nl_f32_impl <N_R0_IQ4_NL>>>;
 template [[host_name("kernel_mul_mv_id_iq4_xs_f32")]]  kernel kernel_mul_mv_id_t kernel_mul_mv_id<mmv_fn<kernel_mul_mv_iq4_xs_f32_impl <N_R0_IQ4_XS>>>;
+template [[host_name("kernel_mul_mv_id_tq2_0_f32")]]   kernel kernel_mul_mv_id_t kernel_mul_mv_id<mmv_fn<kernel_mul_mv_tq2_0_f32_impl  <N_R0_TQ2_0>>>;
 
 kernel void kernel_pool_2d_max_f32(
         constant    ggml_metal_kargs_pool_2d & args,

ggml/src/ggml-opencl/ggml-opencl.cpp

@@ -4929,8 +4929,13 @@ static bool ggml_opencl_ensure_fa_variant(ggml_backend_opencl_context * backend_
         const int x = (e && e[0]) ? atoi(e) : 0;
         return (x == 8 || x == 16 || x == 32) ? x : 0;   // 0 = per-gen default
     }();
+    // X2E needs 16 to keep per-lane o_acc at 128B (the compiler spills the
+    // kernel-default width); X1E does not spill, but C=16 is still a measured
+    // +28-30% DK128-GQA4 decode win there (X1-85, kv 4096/8192), neutral on
+    // DK64 / GQA1 / quant-KV.
     const int fa_cl_c_gqa4 = fa_cl_c_env ? fa_cl_c_env
-        : (backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E ? 16 : 0);
+        : (backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E ||
+           backend_ctx->adreno_gen == ADRENO_GPU_GEN::X1E ? 16 : 0);
     const std::string opts_cl_c_gqa4 = fa_cl_c_gqa4
         ? " -D FA_CL_C=" + std::to_string(fa_cl_c_gqa4) : std::string();
     const std::string fa_cl_c_g8_val = std::to_string(fa_cl_c_gqa4 ? fa_cl_c_gqa4 * 2 : 16);
@@ -7076,6 +7081,19 @@ inline bool enable_adreno_trans_weight(const ggml_backend_opencl_context *backen
     return ((elem_num < 128 * 1024 * 1024) && adreno_kernel && shape_ok);  // max element num: 2**27
 }
 
+inline bool enable_adreno_trans_weight_q5_K(const ggml_backend_opencl_context *backend_ctx, const ggml_tensor *tensor) {
+    if (!use_adreno_kernels(backend_ctx, tensor)) {
+        return false;
+    }
+
+    const size_t elem_num = ggml_nelements(tensor);
+    const size_t q_img_width = elem_num / 8;
+    const size_t qh_img_width = elem_num / 16;
+
+    return q_img_width <= backend_ctx->image_max_buffer_size &&
+           qh_img_width <= backend_ctx->image_max_buffer_size;
+}
+
 static inline bool use_flat_gemv_for_large_m_q4_K(const ggml_tensor *tensor) {
     // gemv_noshuffle variant perf drops for large M, use flat variant for large M.
     // threshold is well above typical hidden/FFN dims, but below typical vocab sizes.
@@ -9255,7 +9273,7 @@ static void ggml_backend_opencl_buffer_set_tensor(ggml_backend_buffer_t buffer,
 
 #ifdef GGML_OPENCL_USE_ADRENO_KERNELS
         cl_kernel kernel = backend_ctx->kernel_convert_block_q5_K;
-        if (use_adreno_kernels(backend_ctx, tensor)) {
+        if (enable_adreno_trans_weight_q5_K(backend_ctx, tensor)) {
             kernel = backend_ctx->kernel_convert_block_q5_K_noshuffle;
         }
 #else
@@ -9290,7 +9308,7 @@ static void ggml_backend_opencl_buffer_set_tensor(ggml_backend_buffer_t buffer,
 
         tensor->extra = extra;
 #ifdef GGML_OPENCL_USE_ADRENO_KERNELS
-        if (use_adreno_kernels(backend_ctx, tensor)) {
+        if (enable_adreno_trans_weight_q5_K(backend_ctx, tensor)) {
 
             int M = tensor->ne[1];
             int K = tensor->ne[0];
@@ -10388,7 +10406,7 @@ static void ggml_backend_opencl_buffer_get_tensor(ggml_backend_buffer_t buffer,
             CL_CHECK(clReleaseMemObject(data_device));
             return;
         }
-        if (use_adreno_kernels(backend_ctx, tensor)) {
+        if (enable_adreno_trans_weight_q5_K(backend_ctx, tensor)) {
             int M = tensor->ne[1];
             int K = tensor->ne[0];
 
@@ -18928,7 +18946,8 @@ static void ggml_cl_mul_mat(ggml_backend_t backend, const ggml_tensor * src0, co
         }
 
         // q5_K x fp32
-        if (src0t == GGML_TYPE_Q5_K && src1t == GGML_TYPE_F32) {
+        if (src0t == GGML_TYPE_Q5_K && src1t == GGML_TYPE_F32 &&
+            enable_adreno_trans_weight_q5_K(backend_ctx, src0)) {
             ggml_cl_mul_mat_q5_K_f32_adreno(backend, src0, src1, dst);
             return;
         }

ggml/src/ggml-openvino/ggml-decoder.cpp

@@ -16,6 +16,7 @@
 #include <iomanip>
 #include <map>
 #include <memory>
+#include <mutex>
 #include <openvino/core/dimension.hpp>
 #include <openvino/core/except.hpp>
 #include <openvino/core/node.hpp>
@@ -25,12 +26,13 @@
 #include <openvino/core/type/float16.hpp>
 #include <openvino/op/constant.hpp>
 #include <openvino/op/convert.hpp>
-#include <openvino/op/parameter.hpp>
 #include <openvino/runtime/tensor.hpp>
 #include <ostream>
 #include <set>
 #include <stdexcept>
 #include <string>
+#include <cstring>
+#include <unordered_map>
 #include <vector>
 
 GgmlOvDecoder::GgmlOvDecoder(ggml_cgraph * cgraph,
@@ -98,27 +100,119 @@ GgmlOvDecoder::GgmlOvDecoder(ggml_cgraph * cgraph, std::map<std::string, std::sh
     }
 }
 
+namespace {
+bool is_inplace_op(const ggml_tensor * node) {
+    return node->op == GGML_OP_SET_ROWS || node->op == GGML_OP_CPY || (node->op == GGML_OP_SCALE && node->view_src);
+}
+
+bool is_same_shape(const ggml_tensor * a, const ggml_tensor * b) {
+    for (int i = 0; i < GGML_MAX_DIMS; i++) {
+        if (a->ne[i] != b->ne[i]) {
+            return false;
+        }
+    }
+    return true;
+}
+
+bool is_conv_states_all_tensor(const ggml_tensor * tensor) {
+    return tensor != nullptr && strncmp(tensor->name, "conv_states_all", strlen("conv_states_all")) == 0;
+}
+
+// CPY writing the tail of conv_input (the concat of the previous conv state and the new tokens)
+// back into a slot block of the recurrent state cache. Detected structurally because the rollback
+// variant (cparams.n_rs_seq > 0) emits one such CPY per snapshot slot without naming them.
+bool is_conv_state_writeback(const ggml_tensor * node) {
+    return node->op == GGML_OP_CPY && node->view_src != nullptr && GgmlOvDecoder::is_kvcache(node->view_src, nullptr) &&
+           node->src[0] != nullptr && node->src[0]->op == GGML_OP_VIEW && node->src[0]->src[0] != nullptr &&
+           node->src[0]->src[0]->op == GGML_OP_CONCAT && node->src[1] != nullptr && node->src[1]->op == GGML_OP_VIEW &&
+           node->src[1]->view_src == node->view_src;
+}
+
+// MoE expert aggregation (build_moe_ffn in llama-graph.cpp): each expert plane is
+// `ggml_view_2d(experts, n_embd, n_tokens, experts->nb[2], i*experts->nb[1])` and the planes
+// are summed with a chain of ADDs: moe_out = ((view_0 + view_1) + view_2) + ... + view_{n-1}.
+// Detected structurally by walking the ADD chain and checking every leaf is a same-shape,
+// same-stride VIEW of one common base tensor, indexed by a distinct expert-plane offset, and
+// that the chain covers every plane of that base (leaf count == base->ne[1]). Only the
+// outermost ADD of the chain satisfies this (inner ADDs see fewer leaves than base->ne[1]).
+bool is_moe_expert_sum_add(const ggml_tensor * node) {
+    std::vector<const ggml_tensor *> leaves;
+    const ggml_tensor * cur = node;
+    while (cur->op == GGML_OP_ADD) {
+        if (cur->src[0] == nullptr || cur->src[1] == nullptr) {
+            return false;
+        }
+        leaves.push_back(cur->src[1]);
+        cur = cur->src[0];
+    }
+    leaves.push_back(cur);
+
+    const ggml_tensor * base = nullptr;
+    std::set<int64_t> plane_indices;
+    for (const ggml_tensor * leaf : leaves) {
+        if (leaf->op != GGML_OP_VIEW || leaf->src[0] == nullptr) {
+            return false;
+        }
+        const ggml_tensor * leaf_base = leaf->src[0];
+        if (base == nullptr) {
+            base = leaf_base;
+        } else if (leaf_base != base) {
+            return false;
+        }
+        if (leaf->ne[0] != base->ne[0] || leaf->ne[1] != base->ne[2] || leaf->ne[2] != 1 || leaf->ne[3] != 1 ||
+            leaf->nb[1] != base->nb[2]) {
+            return false;
+        }
+        if (base->nb[1] == 0 || leaf->view_offs % base->nb[1] != 0) {
+            return false;
+        }
+        int64_t plane = static_cast<int64_t>(leaf->view_offs / base->nb[1]);
+        if (plane < 0 || plane >= base->ne[1] || !plane_indices.insert(plane).second) {
+            return false;
+        }
+    }
+
+    return base != nullptr && base->ne[1] > 1 && plane_indices.size() == static_cast<size_t>(base->ne[1]);
+}
+}  // namespace
+
+static std::string get_tensor_ov_name(const ggml_cgraph * cgraph, const ggml_tensor * tensor) {
+    if (tensor == nullptr) {
+        return "";
+    }
+    const size_t hash_pos = ggml_hash_find(&cgraph->visited_hash_set, tensor);
+    if (((tensor->flags & GGML_TENSOR_FLAG_COMPUTE) || GgmlOvDecoder::is_kvcache(tensor, nullptr)) &&
+        hash_pos != GGML_HASHSET_FULL && ggml_bitset_get(cgraph->visited_hash_set.used, hash_pos)) {
+        return std::string(tensor->name) + "#" + std::to_string(hash_pos);
+    }
+    return tensor->name;
+}
+
+static std::string get_tensor_graph_input_ov_name(const GgmlOvDecoder * decoder,
+                                                  const ggml_cgraph * cgraph,
+                                                  const ggml_tensor * tensor,
+                                                  const ggml_tensor * op) {
+    if (GgmlOvDecoder::is_inp_pos(tensor, op)) {
+        return "inp_pos";
+    }
+    if (GgmlOvDecoder::is_inp_emb(tensor, op)) {
+        return "embd";
+    }
+    if (decoder->is_stateful() && GgmlOvDecoder::is_inp_mask(tensor, op)) {
+        return std::string(tensor->name).find("swa") == std::string::npos ? "self_kq_mask" : "self_kq_mask_swa";
+    }
+    return get_tensor_ov_name(cgraph, tensor);
+}
+
 void GgmlOvDecoder::set_input_output() {
     for (int node_n = 0; node_n < m_cgraph->n_nodes; node_n++) {
-        auto node = m_cgraph->nodes[node_n];
+        auto * node = m_cgraph->nodes[node_n];
 
         NodeInfo current_node_info;
-        auto node_name = std::string(node->name);
-        auto node_output_name = node_name;
-        auto * node_output = node;
-        if (node->op == GGML_OP_SET_ROWS) {
-            // SET_ROWS updates the tensor in place. For later ov op that uses the
-            // the view_src of SET_ROWS, we need to make sure they get the updated tensor
-            // by putting the view_src name in the tensor_map in
-            // <openvino>/src/frontends/ggml/src/translate_session.cpp
-            node_output_name = std::string(node->view_src->name);
-            node_output = node->view_src;
-        }
+        auto node_name = get_tensor_ov_name(m_cgraph, node);
 
         current_node_info.node = node;
         current_node_info.node_name = node_name;
-        current_node_info.node_output = node_output;
-        current_node_info.node_output_name = node_output_name;
         current_node_info.node_op_case = 0;
         current_node_info.data_addr = node->data;
 
@@ -127,9 +221,9 @@ void GgmlOvDecoder::set_input_output() {
             if (src == nullptr) {
                 continue;
             }
-            auto src_name = std::string(src->name);
+            auto src_name = get_tensor_ov_name(m_cgraph, src);
             if (src->flags & GGML_TENSOR_FLAG_INPUT) {
-                src_name = get_graph_input_ov_name(src, node);
+                src_name = get_tensor_graph_input_ov_name(this, m_cgraph, src, node);
             }
             current_node_info.node_inputs[src_name] = src;
             current_node_info.node_inputs_names.push_back(src_name);
@@ -140,9 +234,9 @@ void GgmlOvDecoder::set_input_output() {
                 auto current = src;
 
                 while (current != nullptr) {
-                    auto current_name = std::string(current->name);
+                    auto current_name = get_tensor_ov_name(m_cgraph, current);
                     if (current->flags & GGML_TENSOR_FLAG_INPUT) {
-                        current_name = get_graph_input_ov_name(current, node);
+                        current_name = get_tensor_graph_input_ov_name(this, m_cgraph, current, node);
                     }
                     view_chain.emplace_back(current_name, current);
                     // If current src is also a VIEW, continue traversing
@@ -166,6 +260,7 @@ int GgmlOvDecoder::compute_op_case(const ggml_tensor * node) const {
     int op_case = 0;
     switch (node->op) {
     case GGML_OP_RESHAPE: {
+        auto name = std::string(node->name);
         auto * src = node->src[0];
         if (src->op == GGML_OP_RESHAPE && src->src[0]->ne[0] == node->ne[0] && src->src[0]->ne[1] == node->ne[1]) {
             op_case = 4;
@@ -178,11 +273,12 @@ int GgmlOvDecoder::compute_op_case(const ggml_tensor * node) const {
             }
         } else if (src->ne[0] * src->ne[1] * src->ne[2] == node->ne[1]) {
             op_case = 3;
-        } else if (src->ne[1] * src->ne[2] == node->ne[1]) {
-            op_case = 6;
-        }
-        if (op_case == 0 && ggml_nelements(node) == ggml_nelements(src)) {
+        } else if (name.find("linear_attn_qkv_mixed") == 0 || name.find("alpha") == 0) {
             op_case = 6;
+        } else if (name.find("linear_attn_out") == 0) {
+            op_case = 7;
+        } else if (name.find("state_predelta") == 0) {
+            op_case = 8;
         }
         break;
     }
@@ -232,7 +328,14 @@ int GgmlOvDecoder::compute_op_case(const ggml_tensor * node) const {
     }
     case GGML_OP_GET_ROWS: {
         if (node->src[1]->op == GGML_OP_VIEW) {
-            op_case = 2;
+            // GET_ROWS gathering recurrent state cache rows via the inp->s_copy index list:
+            // src[0] is a reshape of cache_r/cache_s, src[1] is a view of the s_copy leaf.
+            // op_case 3: main view (active sequences, view offset 0)
+            // op_case 4: extra view (defrag remainder, nonzero view offset)
+            if (node->src[0]->op == GGML_OP_RESHAPE && node->src[0]->src[0] != nullptr &&
+                is_kvcache(node->src[0]->src[0], nullptr)) {
+                op_case = node->src[1]->view_offs == 0 ? 1 : 2;
+            }
         }
         break;
     }
@@ -260,7 +363,7 @@ int GgmlOvDecoder::compute_op_case(const ggml_tensor * node) const {
                 // throw std::runtime_error("Unsupported VIEW case");
             }
             op_case = 0;
-            if (m_model_is_splitted && m_model_inputs.find(std::string(src->name)) != m_model_inputs.end()) {
+            if (m_model_is_splitted && m_model_inputs.find(get_tensor_ov_name(m_cgraph, src)) != m_model_inputs.end()) {
                 op_case = 0;
             }
         }
@@ -295,6 +398,56 @@ int GgmlOvDecoder::compute_op_case(const ggml_tensor * node) const {
         }
         break;
     }
+    case GGML_OP_RMS_NORM: {
+        if (node->src[0]->op == GGML_OP_VIEW) {
+            if (is_same_shape(node->src[0]->src[0], node->src[0])) {
+                op_case = 1;
+            } else if (node->src[0]->src[0]->op == GGML_OP_GATED_DELTA_NET) {
+                op_case = 2;
+            }
+        }
+        break;
+    }
+    case GGML_OP_CPY: {
+        if (node->src[0]->op == GGML_OP_VIEW) {
+            if (node->src[0]->src[0]->op == GGML_OP_GATED_DELTA_NET) {
+                op_case = 1;
+            } else if (is_conv_state_writeback(node)) {
+                op_case = 2;
+                break;
+            } else if (is_conv_states_all_tensor(node->view_src) && node->src[1] != nullptr &&
+                       node->src[1]->op == GGML_OP_VIEW && node->src[1]->view_src == node->view_src) {
+                op_case = 4;
+                break;
+            }
+        } else if (node->src[0]->op == GGML_OP_GET_ROWS && node->src[1] != nullptr &&
+                   node->src[1]->op == GGML_OP_VIEW && node->src[1]->view_src != nullptr &&
+                   is_kvcache(node->src[1]->view_src, nullptr)) {
+            // s_copy defrag remainder writeback: gathered extra state rows copied back into the cache
+            op_case = 3;
+        }
+        break;
+    }
+    case GGML_OP_ADD: {
+        if (is_moe_expert_sum_add(node)) {
+            // Outermost ADD of a MoE expert-plane sum chain: translated as a single
+            // ReduceSum over the base tensor instead of N-1 chained Adds over N Slices.
+            op_case = 1;
+        }
+        break;
+    }
+    case GGML_OP_SCALE: {
+        if (node->view_src && node->buffer->usage == GGML_BACKEND_BUFFER_USAGE_ANY) {
+            op_case = 1;
+        }
+        break;
+    }
+    case GGML_OP_L2_NORM: {
+        if (std::string(node->name).find("predelta") != std::string::npos) {
+            op_case = 1;
+        }
+        break;
+    }
     default:
         break;
     }
@@ -476,6 +629,43 @@ std::pair<ModelParams, ComputeParams> GgmlOvDecoder::compute_llm_params(ggml_cgr
                 model_params.mixed_rope_params = true;
             }
         }
+        if (node->op == GGML_OP_GATED_DELTA_NET) {
+            model_params.state_size = node->src[0]->ne[0];
+        }
+        if (node->op == GGML_OP_SCALE && node->view_src != nullptr && is_kvcache(node->view_src, nullptr)) {
+            compute_params.cache_rs_reset_len = ggml_nelements(node) / node->view_src->ne[0];
+            compute_params.cache_rs_reset_idx = node->src[0]->view_offs / node->view_src->ne[0];
+        }
+        // Capture the destination slot block of every recurrent state cache writeback, plus the
+        // conv_input window the conv state writeback copies. The active sequences occupy a
+        // contiguous slot block [begin, begin + n_seqs) of the cache; the block and the window move
+        // with the batch, so they are fed to the cached model as runtime inputs.
+        if (node->op == GGML_OP_CPY && node->view_src != nullptr && is_kvcache(node->view_src, nullptr) &&
+            node->src[1] != nullptr && node->src[1]->op == GGML_OP_VIEW && node->src[1]->view_src == node->view_src) {
+            const bool is_conv = is_conv_state_writeback(node);
+            const bool is_gdn = node->src[0]->op == GGML_OP_VIEW && node->src[0]->src[0] != nullptr &&
+                                node->src[0]->src[0]->op == GGML_OP_GATED_DELTA_NET;
+            const bool is_extra = node->src[0]->op == GGML_OP_GET_ROWS;
+
+            const ggml_tensor * dest_view = node->src[1];
+            const ggml_tensor * cache = node->view_src;
+            const size_t row_bytes = cache->ne[0] * ggml_type_size(cache->type);
+            if (row_bytes > 0 && (is_conv || is_gdn || is_extra)) {
+                ComputeParams::RsWriteback writeback;
+                writeback.slot_begin = (int) (dest_view->view_offs / row_bytes);
+                if (is_conv) {
+                    // conv_input column the copied window starts at
+                    writeback.src_begin = (int) (node->src[0]->view_offs / node->src[0]->view_src->nb[0]);
+                } else if (is_gdn) {
+                    // first row of the state part of the gated-delta-net output
+                    writeback.src_begin = (int) (node->src[0]->view_offs / node->src[0]->view_src->nb[1]);
+                }
+                compute_params.rs_writebacks[get_tensor_ov_name(cgraph, node)] = writeback;
+            }
+            if (is_conv || is_gdn) {
+                compute_params.s_copy_active_slot_len = (int) dest_view->ne[1];
+            }
+        }
     }
     auto * output_tensor = cgraph->nodes[cgraph->n_nodes - 1];
     compute_params.output_len = output_tensor->ne[1];
@@ -505,6 +695,10 @@ ov::PartialShape GgmlOvDecoder::get_graph_input_shape(const ggml_tensor * op,
     if (is_inp_tok(input, op) || is_inp_pos(input, op)) {
         // tokens or positions
         int len = m_is_static ? (m_is_prefill ? m_prefill_chunk_size : 1) : -1;
+        if (m_is_static && is_inp_pos(input, op)) {
+            // IMROPE stacks n_planes (t/h/w/e) position planes back to back
+            len *= get_inp_pos_n_planes(op);
+        }
         input_shape = ov::PartialShape{1, 1, 1, len};
 
     } else if (is_output_idx(input, op)) {
@@ -543,6 +737,9 @@ ov::PartialShape GgmlOvDecoder::get_graph_input_shape(const ggml_tensor * op,
         int len = m_is_static ? (m_is_prefill ? m_prefill_chunk_size : 1) : -1;
         input_shape = ov::PartialShape{1, 1, 1, len};
 
+    } else if (is_inp_s_copy(input, op) || is_s_copy_leaf(input)) {
+        input_shape = ov::PartialShape{1, 1, 1, -1};
+
     } else {
         input_shape = ov::PartialShape{get_shape(input)};
     }
@@ -558,6 +755,35 @@ ov::PartialShape GgmlOvDecoder::get_graph_input_shape(const ggml_tensor * op,
     return input_shape;
 }
 
+bool GgmlOvDecoder::is_s_copy_leaf(const ggml_tensor * tensor) const {
+    if (tensor == nullptr || tensor->op != GGML_OP_NONE || m_cgraph == nullptr) {
+        return false;
+    }
+    for (int i = 0; i < m_cgraph->n_nodes; i++) {
+        const ggml_tensor * node = m_cgraph->nodes[i];
+        if (node->op != GGML_OP_GET_ROWS || node->src[0] == nullptr || node->src[1] == nullptr) {
+            continue;
+        }
+        // The index list may reach the s_copy leaf through one or more VIEWs.
+        const ggml_tensor * idx = node->src[1];
+        while (idx != nullptr && idx->op == GGML_OP_VIEW) {
+            idx = idx->src[0];
+        }
+        if (idx != tensor) {
+            continue;
+        }
+        // The gathered data must be a recurrent state cache (cache_r/cache_s).
+        const ggml_tensor * data = node->src[0];
+        while (data != nullptr && (data->op == GGML_OP_VIEW || data->op == GGML_OP_RESHAPE)) {
+            data = data->src[0];
+        }
+        if (data != nullptr && is_kvcache(data, nullptr)) {
+            return true;
+        }
+    }
+    return false;
+}
+
 void GgmlOvDecoder::add_extra_inputs() {
     // Extra inputs:
     // 1. `attention_size`, used in FLASH_ATTN where the shape of the matmul's are 256 aligned,
@@ -565,21 +791,7 @@ void GgmlOvDecoder::add_extra_inputs() {
     // 2. `n_seq_active` and `seq_active_start`, used in FLASH_ATTN_EXT to indicate the active sequences in the batch
 
     auto create_1d_input = [this](const std::string & name, int64_t value) {
-        if (m_is_static) {
-            auto constant =
-                std::make_shared<ov::op::v0::Constant>(ov::element::i64, ov::Shape{1}, std::vector<int64_t>{value});
-            constant->set_friendly_name(name);
-            m_model_extra_inputs[name] = constant;
-        } else {
-            auto param_node = std::make_shared<ov::op::v0::Parameter>(ov::element::i64, ov::Shape{1});
-            param_node->set_friendly_name(name);
-            param_node->output(0).get_tensor().set_names({name});
-            m_model_extra_inputs[name] = param_node;
-
-            auto tensor = std::make_shared<ov::Tensor>(ov::element::i64, ov::Shape{1});
-            *tensor->data<int64_t>() = value;
-            m_model_extra_input_values[name] = tensor;
-        }
+        m_model_extra_inputs[name] = {ov::element::i64, ov::Shape{1}, value, !m_is_static};
     };
 
     if (m_compute_params.attention_size != -1) {
@@ -595,6 +807,20 @@ void GgmlOvDecoder::add_extra_inputs() {
         create_1d_input("token_len_per_seq", m_compute_params.token_len_per_seq);
     }
     // create_1d_input("token_len", m_compute_params.token_len_per_seq * m_compute_params.n_seq_active);
+
+    if (m_compute_params.cache_rs_reset_idx != -1) {
+        create_1d_input("cache_rs_reset_idx", m_compute_params.cache_rs_reset_idx);
+        create_1d_input("cache_rs_reset_len", m_compute_params.cache_rs_reset_len);
+    }
+
+    if (m_compute_params.s_copy_active_slot_len != -1) {
+        create_1d_input("s_copy_active_slot_len", m_compute_params.s_copy_active_slot_len);
+    }
+
+    for (const auto & [node_name, writeback] : m_compute_params.rs_writebacks) {
+        create_1d_input("rs_slot_begin_" + node_name, writeback.slot_begin);
+        create_1d_input("rs_src_begin_" + node_name, writeback.src_begin);
+    }
 }
 
 bool GgmlOvDecoder::node_is_used_as_src(const int node_idx) {
@@ -617,14 +843,11 @@ void GgmlOvDecoder::compute_model_inputs() {
         ggml_tensor * node = m_cgraph->nodes[i];
         // the node op is NONE means this node maybe as input of later nodes, we should add it to model inputs for this node.
         if (node->op == GGML_OP_NONE && node_is_used_as_src(i)) {
-            std::string node_name(node->name);
+            std::string node_name = get_tensor_ov_name(m_cgraph, node);
             if (m_model_weights.find(node_name) == m_model_weights.end()) {
                 m_inputs[node_name] = node;
-                auto param_node = std::make_shared<ov::op::v0::Parameter>(
-                    get_ov_type(node), get_graph_input_shape(node, nullptr, m_node_dynamic_dims[node]));
-                param_node->set_friendly_name(node_name);
-                param_node->output(0).get_tensor().set_names({node_name});
-                m_model_inputs[node_name] = param_node;
+                m_model_inputs[node_name] = {get_ov_type(node),
+                                             get_graph_input_shape(node, nullptr, m_node_dynamic_dims[node])};
             }
             continue;
         }
@@ -633,9 +856,9 @@ void GgmlOvDecoder::compute_model_inputs() {
             if (src == nullptr) {
                 continue;
             }
-            std::string src_name = std::string(src->name);
+            std::string src_name = get_tensor_ov_name(m_cgraph, src);
             if (src->flags & GGML_TENSOR_FLAG_INPUT) {
-                src_name = get_graph_input_ov_name(src, node);
+                src_name = get_tensor_graph_input_ov_name(this, m_cgraph, src, node);
             }
             if (m_model_weights.find(src_name) != m_model_weights.end()) {
                 continue;
@@ -668,14 +891,11 @@ void GgmlOvDecoder::compute_model_inputs() {
             // Resolve nested VIEW nodes by following src[0] until the first non-VIEW tensor.
             while (src->op == GGML_OP_VIEW && src->src[0] != nullptr) {
                 src = src->src[0];
-                src_name = std::string(src->name);
+                src_name = get_tensor_ov_name(m_cgraph, src);
             }
             m_inputs[src_name] = src;
-            ov::PartialShape param_shape = get_graph_input_shape(node, src, m_node_dynamic_dims[src]);
-            auto param_node = std::make_shared<ov::op::v0::Parameter>(get_ov_type(src), param_shape);
-            param_node->set_friendly_name(src_name);
-            param_node->output(0).get_tensor().set_names({src_name});
-            m_model_inputs[src_name] = param_node;
+            m_model_inputs[src_name] = {get_ov_type(src),
+                                        get_graph_input_shape(node, src, m_node_dynamic_dims[src])};
         }
     }
 }
@@ -691,8 +911,8 @@ void GgmlOvDecoder::compute_model_outputs() {
         }
         auto cur_node_use_count = m_cgraph->use_counts[ggml_hash_find(&m_cgraph->visited_hash_set, cur_node)];
         if (cur_node_use_count == 0) {
-            // The output of SET_ROWS is the view_src tensor, which is updated in place. We should use the view_src name as the output name to make sure it can be correctly matched with the later ops that use the view_src.
-            if (cur_node != nullptr && cur_node->op == GGML_OP_SET_ROWS) {
+            // The output of in-place ops is the view_src tensor, which is updated in place. We should use the view_src name as the output name to make sure it can be correctly matched with the later ops that use the view_src.
+            if (cur_node != nullptr && ::is_inplace_op(cur_node) && ggml_nbytes(cur_node) > 0) {
                 cur_node = cur_node->view_src;
             }
         } else {
@@ -710,9 +930,9 @@ void GgmlOvDecoder::compute_model_outputs() {
             }
         }
         if (cur_node != nullptr) {
-            std::string node_output_name(cur_node->name);
-            m_model_outputs[node_output_name] = cur_node;
-            m_model_output_names.push_back(node_output_name);
+            std::string cur_node_name = get_tensor_ov_name(m_cgraph, cur_node);
+            m_model_outputs[cur_node_name] = cur_node;
+            m_model_output_names.insert(cur_node_name);
         }
     }
 }
@@ -740,7 +960,7 @@ const ggml_tensor * GgmlOvDecoder::get_tensor_from_name(const std::string & name
             if (src == nullptr) {
                 break;
             }
-            if (std::string(src->name) == name) {
+            if (get_tensor_ov_name(m_cgraph, src) == name) {
                 return src;
             }
         }
@@ -756,6 +976,16 @@ std::map<std::string, std::string> GgmlOvDecoder::get_kv_param_res_names() const
     return kv_param_res_names;
 }
 
+// MUL_MAT_ID's src[0] is the [k, m, n_expert] expert-weight tensor. It is always a constant per-expert
+// weight table -- never a computed activation -- regardless of whether the backend happened to mark its
+// buffer as GGML_BACKEND_BUFFER_USAGE_WEIGHTS (test-backend-ops, for example, never sets that usage
+// flag, unlike real inference). Without this, non-quantized (F16/F32/BF16) expert weights would fall
+// through the check below as "not a weight", get decoded as a Parameter/activation instead of a
+// Constant, and crash GatherMatmul's "only constant weights are supported" check.
+static bool is_mul_mat_id_expert_weight(const ggml_tensor * node, int src_index) {
+    return node->op == GGML_OP_MUL_MAT_ID && src_index == 0;
+}
+
 std::map<std::string, std::shared_ptr<ov::Node>> GgmlOvDecoder::create_weight_nodes(ggml_cgraph * cgraph, bool naive) {
     std::map<std::string, std::shared_ptr<ov::Node>> model_weights;
     auto * nodes = cgraph->nodes;
@@ -768,13 +998,14 @@ std::map<std::string, std::shared_ptr<ov::Node>> GgmlOvDecoder::create_weight_no
                 continue;
             }
 
-            std::string src_name(src->name);
+            std::string src_name = get_tensor_ov_name(cgraph, src);
             if (is_rope_freqs_weight(src, node)) {
                 src_name = "rope_freqs.weight";
             }
             if (!src->view_src) {
                 ggml_backend_buffer * buffer = src->buffer;
-                if (buffer->usage == GGML_BACKEND_BUFFER_USAGE_WEIGHTS || ggml_is_quantized(src->type)) {
+                if (buffer->usage == GGML_BACKEND_BUFFER_USAGE_WEIGHTS || ggml_is_quantized(src->type) ||
+                    is_mul_mat_id_expert_weight(node, i)) {
                     if (model_weights.find(src_name) == model_weights.end()) {
                         auto weight_node = create_weight_node(src, naive);
                         weight_node->set_friendly_name(src_name);
@@ -787,6 +1018,42 @@ std::map<std::string, std::shared_ptr<ov::Node>> GgmlOvDecoder::create_weight_no
     return model_weights;
 }
 
+// Process-lifetime cache for weight nodes built from NON-OpenVINO buffers (e.g. the
+// token_embd.weight copy that lives in a CPU/mmap buffer and feeds GET_ROWS). Such
+// tensors have no OV buffer context to own a cached extra, so without this they are
+// re-extracted/re-requantized on every (re)compile — for token_embd that is a ~1-2 GB
+// F32 dequant each time. Keyed by tensor->data, which is stable for the process and
+// uniquely identifies the immutable weight bytes. OV-buffer weights keep using the
+// per-tensor extra cache and never reach here.
+static std::mutex g_nonov_weight_cache_mutex;
+static std::unordered_map<const void *, std::shared_ptr<ov::Node>> g_nonov_weight_cache;
+
+std::set<std::string> GgmlOvDecoder::collect_weight_names(ggml_cgraph * cgraph) {
+    // Mirrors the name-selection logic of create_weight_nodes() but builds no nodes,
+    // so topology checks don't trigger weight extraction/requantization.
+    std::set<std::string> names;
+    for (int node_i = 0; node_i < cgraph->n_nodes; node_i++) {
+        auto * node = cgraph->nodes[node_i];
+        for (int i = 0; i < GGML_MAX_SRC; i++) {
+            auto * src = node->src[i];
+            if (src == nullptr) {
+                continue;
+            }
+            std::string src_name(src->name);
+            if (is_rope_freqs_weight(src, node)) {
+                src_name = "rope_freqs.weight";
+            }
+            if (!src->view_src) {
+                ggml_backend_buffer * buffer = src->buffer;
+                if (buffer->usage == GGML_BACKEND_BUFFER_USAGE_WEIGHTS || ggml_is_quantized(src->type)) {
+                    names.insert(src_name);
+                }
+            }
+        }
+    }
+    return names;
+}
+
 std::shared_ptr<ov::Node> GgmlOvDecoder::create_weight_node(ggml_tensor * tensor, bool naive) {
     const bool is_ov_buffer = ggml_backend_buffer_is_openvino(tensor->buffer);
 
@@ -826,6 +1093,21 @@ std::shared_ptr<ov::Node> GgmlOvDecoder::create_weight_node(ggml_tensor * tensor
         return weight_node;
     }
 
+    // Non-OV-buffer weights (CPU/mmap, e.g. the GET_ROWS token_embd copy) have no buffer
+    // context to cache an extra in, so memoize them here keyed by their (stable) data
+    // pointer to avoid re-extracting on every recompile. Opt-in via
+    // GGML_OPENVINO_REDUCE_COMPILE_MEM or GGML_OPENVINO_MEMORY_OPTIMIZE. Skip
+    // for `naive` (test/naive path) since use_bias changes the produced node.
+    const bool cacheable_nonov = ggml_openvino_reduce_compile_mem_enabled() && !is_ov_buffer &&
+                                 !naive && tensor->data != nullptr;
+    if (cacheable_nonov) {
+        std::lock_guard<std::mutex> lock(g_nonov_weight_cache_mutex);
+        auto it = g_nonov_weight_cache.find(tensor->data);
+        if (it != g_nonov_weight_cache.end()) {
+            return it->second;
+        }
+    }
+
     // There are three cases where we need to create a new weight node:
     // 1. weights are in openvino_host_buffer. Weight loading to host buffer will not trigger backend_buffer_set_tensor
     // 2. weights are in cpu/cpu_mapped buffer. On token_embd.weight goes to case 1 or 2, depending on whether mmap or direct_io is used
@@ -834,7 +1116,7 @@ std::shared_ptr<ov::Node> GgmlOvDecoder::create_weight_node(ggml_tensor * tensor
     // GGML_LOG_DEBUG("%s: creating new weight node for %s\n", __func__, tensor->name);
     static const std::set<ggml_type> weight_types = {GGML_TYPE_F32,  GGML_TYPE_F16,  GGML_TYPE_BF16, GGML_TYPE_Q8_0,
                                                      GGML_TYPE_Q4_0, GGML_TYPE_Q4_1, GGML_TYPE_Q5_1, GGML_TYPE_Q4_K,
-                                                     GGML_TYPE_Q5_K, GGML_TYPE_Q6_K};
+                                                     GGML_TYPE_Q5_K, GGML_TYPE_Q6_K, GGML_TYPE_MXFP4};
     if (weight_types.find(tensor->type) == weight_types.end()) {
         throw std::runtime_error("Unexpected weight tensor type: " + std::string(tensor->name) + " with type " +
                                  ggml_type_name(tensor->type));
@@ -863,6 +1145,12 @@ std::shared_ptr<ov::Node> GgmlOvDecoder::create_weight_node(ggml_tensor * tensor
 
     ov_weight.weight_node->set_friendly_name(tensor->name);
     if (!is_ov_buffer) {
+        if (cacheable_nonov) {
+            std::lock_guard<std::mutex> lock(g_nonov_weight_cache_mutex);
+            // Another thread may have inserted concurrently; keep the first.
+            auto [it, inserted] = g_nonov_weight_cache.emplace(tensor->data, ov_weight.weight_node);
+            return it->second;
+        }
         return ov_weight.weight_node;
     }
 
@@ -1178,7 +1466,7 @@ std::string GgmlOvDecoder::get_view_input_name(int node_idx, const std::string &
     auto it = m_node_info_list[node_idx].node_inputs_views.find(name);
     if (it != m_node_info_list[node_idx].node_inputs_views.end()) {
         if (view_index < it->second.size()) {
-            return it->second[view_index].second->name;
+            return it->second[view_index].first;
         }
     }
     return "";
@@ -1190,7 +1478,7 @@ std::string GgmlOvDecoder::get_view_input_src_name(int node_idx, const std::stri
         if (view_index < it->second.size()) {
             auto * view_tensor = it->second[view_index].second;
             if (view_tensor && view_tensor->src[0]) {
-                return view_tensor->src[0]->name;
+                return get_tensor_ov_name(m_cgraph, view_tensor->src[0]);
             }
         }
     }
@@ -1214,7 +1502,7 @@ std::vector<std::string> GgmlOvDecoder::get_input_names(int node_idx) const {
 }
 
 ov::PartialShape GgmlOvDecoder::get_output_shape(int node_idx) const {
-    auto * ggml_tensor = m_node_info_list[node_idx].node_output;
+    auto * ggml_tensor = m_node_info_list[node_idx].node;
     return ov::PartialShape(get_shape(ggml_tensor));
 }
 
@@ -1228,7 +1516,28 @@ std::vector<size_t> GgmlOvDecoder::get_output_stride(int node_idx) const {
 }
 
 std::vector<std::string> GgmlOvDecoder::get_output_names(int node_idx) const {
-    return {m_node_info_list[node_idx].node_output_name};
+    return {m_node_info_list[node_idx].node_name};
+}
+
+std::string GgmlOvDecoder::get_inplace_op_src(int node_idx) const {
+    auto * node = m_node_info_list[node_idx].node;
+    if (!::is_inplace_op(node) || node->view_src == nullptr || ggml_nbytes(node) == 0) {
+        return "";
+    }
+    const int op_case = m_node_info_list[node_idx].node_op_case;
+    if (node->op == GGML_OP_CPY && (op_case == 1 || op_case == 2 || op_case == 3) &&
+        m_compute_params.s_copy_active_slot_len == -1) {
+        return "";
+    }
+    return get_tensor_ov_name(m_cgraph, node->view_src);
+}
+
+bool GgmlOvDecoder::is_view_like_alias_of(int node_idx, const std::string & view_src_name) const {
+    auto * node = m_node_info_list[node_idx].node;
+    if (node->view_src == nullptr || get_tensor_ov_name(m_cgraph, node->view_src) != view_src_name) {
+        return false;
+    }
+    return node->op == GGML_OP_RESHAPE || node->op == GGML_OP_VIEW;
 }
 
 const std::string & GgmlOvDecoder::get_op_name() const {
@@ -1404,14 +1713,18 @@ void GgmlOvDecoder::compute_node_dynamic_dims() {
                 }
                 if (m_node_dynamic_dims[node] != -1 && dynamic_dim_value != node->ne[m_node_dynamic_dims[node]]) {
                     m_node_dynamic_dims[node] = -1;
-                    // std::cout << "Warning: Dynamic dim value mismatch for node: " << node->name
-                    //           << " and its src[0]: " << node->src[0]->name << std::endl;
+                    GGML_LOG_WARN("ggml-openvino: dynamic dim value mismatch for VIEW node '%s', src[0]: '%s'\n",
+                                  node->name, node->src[0]->name);
                 }
             }
             break;
         }
         case GGML_OP_TRANSPOSE:
         case GGML_OP_RESHAPE: {
+            if (is_same_shape(node->src[0], node)) {
+                m_node_dynamic_dims[node] = m_node_dynamic_dims[node->src[0]];
+                break;
+            }
             // RESHAPE requires src[0] to be contiguous, so both src and result
             // have standard compact strides: nb[i] = type_size * prod(ne[0..i-1]).
             // Match src->nb[dynamic_dim] against result->nb[i] to find the output
@@ -1429,7 +1742,7 @@ void GgmlOvDecoder::compute_node_dynamic_dims() {
                     }
                 }
                 if (m_node_dynamic_dims[node] == -1) {
-                    // std::cout << "Cannot determine dynamic dim for RESHAPE node: " << node->name << std::endl;
+                    GGML_LOG_WARN("ggml-openvino: cannot determine dynamic dim for RESHAPE node '%s'\n", node->name);
                 }
             }
             break;
@@ -1480,15 +1793,29 @@ void GgmlOvDecoder::compute_node_dynamic_dims() {
                     }
                     if (matched_dim_count != 1) {
                         m_node_dynamic_dims[node] = -1;
-                        // std::cout << "Warning: Cannot determine dynamic dim for CONT node: " << node->name
-                        //           << " and its src[0]: " << node->src[0]->name << std::endl;
+                        GGML_LOG_WARN("ggml-openvino: cannot determine dynamic dim for CONT node '%s', src[0]: '%s'\n",
+                                      node->name, node->src[0]->name);
                     }
                 }
             }
             break;
+        case GGML_OP_CONCAT:
+            for (int i = 0; i < GGML_MAX_DIMS; i++) {
+                if (node->src[0]->ne[i] != node->ne[i]) {
+                    m_node_dynamic_dims[node] = i;
+                    break;
+                }
+            }
+            break;
+        case GGML_OP_SSM_CONV:
+        case GGML_OP_GATED_DELTA_NET:
+            m_node_dynamic_dims[node] = 1;
+            break;
         case GGML_OP_RMS_NORM:
+        case GGML_OP_L2_NORM:
         case GGML_OP_NORM:
         case GGML_OP_ADD:
+        case GGML_OP_SUB:
         case GGML_OP_GLU:
         case GGML_OP_ROPE:
         case GGML_OP_SCALE:
@@ -1496,9 +1823,31 @@ void GgmlOvDecoder::compute_node_dynamic_dims() {
         case GGML_OP_ARGSORT:
         case GGML_OP_ADD_ID:
         case GGML_OP_UNARY:
+        case GGML_OP_CUMSUM:
+        case GGML_OP_FILL:
+        case GGML_OP_SET:
+        case GGML_OP_DIAG:
+        case GGML_OP_TRI:
+        case GGML_OP_REPEAT:
+        // Shape-preserving elementwise ops: the dynamic dim is unchanged from src[0].
+        // DIV/CLAMP are used in the MoE routing-weight normalization
+        // (sum_rows -> clamp -> div). If they are left untracked here the dynamic
+        // (token) dim is lost there, the captured prefill token count gets baked into
+        // the downstream reshapes, and every decoder layer after layer 0 turns static
+        // (which then triggers the GPU in-place-concat KV-cache corruption).
+        case GGML_OP_DIV:
+        case GGML_OP_CLAMP:
+        case GGML_OP_PAD:
             m_node_dynamic_dims[node] = m_node_dynamic_dims[node->src[0]];
             break;
+        case GGML_OP_SUM_ROWS:
+            // SUM_ROWS reduces ggml axis 0 to size 1 and preserves all other axes, so the
+            // dynamic dim is preserved unless it was axis 0 (then it is summed away).
+            m_node_dynamic_dims[node] =
+                (m_node_dynamic_dims[node->src[0]] == 0) ? -1 : m_node_dynamic_dims[node->src[0]];
+            break;
         case GGML_OP_MUL_MAT_ID:
+        case GGML_OP_SOLVE_TRI:
             m_node_dynamic_dims[node] = m_node_dynamic_dims[node->src[1]];
             break;
         case GGML_OP_CPY:
@@ -1534,7 +1883,8 @@ void GgmlOvDecoder::compute_node_dynamic_dims() {
             break;
         }
         default:
-            // std::cout << "Doesn't handle node name: " << node->name << " op: " << ggml_op_name(node->op) << std::endl;
+            GGML_LOG_DEBUG("ggml-openvino: compute_node_dynamic_dims: unhandled op %s for node '%s'\n",
+                           ggml_op_name(node->op), node->name);
             break;
         }
     };

ggml/src/ggml-openvino/ggml-decoder.h

@@ -11,6 +11,8 @@
 #include <memory>
 #include <openvino/core/partial_shape.hpp>
 #include <optional>
+#include <set>
+#include <string>
 #include <vector>
 
 struct ModelParams {
@@ -20,6 +22,7 @@ struct ModelParams {
     int n_seq = 1;
     int n_heads_kv = -1;
     int head_size = -1;
+    int state_size = -1;  // for SSM molels, eg qwen35
     int32_t rope_params[15];
     bool mixed_rope_params = false;
     std::vector<int> swa_layers;
@@ -48,6 +51,47 @@ struct ComputeParams {
     int token_len_per_seq = -1;
     int past_kv_len = -1;
     int output_len = 1;
+
+    int cache_rs_reset_idx = -1;
+    int cache_rs_reset_len = -1;
+    // SSM/DeltaNet models otionally clear cache_r and cache_s of certain slots in the cgraph
+    // 3: [ 18432,     4,     1,     1] RESHAPE              cache_r_l0 (reshaped)
+    //    [ 18432,     4,     1,     1]            0: NONE        cache_r_l0
+    // 4: [ 18432,     1,     1,     1] VIEW                 cache_r_l0 (reshaped) (view)
+    //    [ 18432,     4,     1,     1]            0: RESHAPE     cache_r_l0 (reshaped)
+    // 5: [ 18432,     1,     1,     1] SCALE                cache_r_l0 (reshaped) (view) (view)
+    //    [ 18432,     1,     1,     1]            0: VIEW        cache_r_l0 (reshaped) (view)
+
+    int s_copy_active_slot_len = -1;
+    // SSM/DeltaNet models otionally reorder slots of state cache, to make the active slots contiguous
+    // leaf_5 is the inp->s_copy in llama-graph.cpp, eg if there are 8 slots in total and slot 3 and 7
+    // are active in the current batch, leaf_5 will be [3, 7, 5, 6, 4]
+    //  6: [     2,     1,     1,     1] VIEW                  (view)
+    //      [     2,     1,     1,     1]            0: NONE        leaf_5
+    //  7: [ 18432,     2,     1,     1] GET_ROWS             conv_states-0
+    //      [ 18432,     4,     1,     1]            0: RESHAPE     cache_r_l0 (reshaped)
+    //      [     2,     1,     1,     1]            1: VIEW         (view)
+    //  8: [     0,     1,     1,     1] VIEW                  (view)
+    //      [     2,     1,     1,     1]            0: NONE        leaf_5
+    //  9: [ 18432,     0,     1,     1] GET_ROWS             node_9
+    //      [ 18432,     4,     1,     1]            0: RESHAPE     cache_r_l0 (reshaped)
+    //      [     0,     1,     1,     1]            1: VIEW         (view)
+    // 10: [ 18432,     0,     1,     1] VIEW                 cache_r_l0 (view)
+    //      [ 18432,     4,     1,     1]            0: NONE        cache_r_l0
+    // 11: [ 18432,     0,     1,     1] CPY                  cache_r_l0 (view) (copy of )
+    //      [ 18432,     0,     1,     1]            0: GET_ROWS    node_9
+    //      [ 18432,     0,     1,     1]            1: VIEW        cache_r_l0 (view)
+
+    struct RsWriteback {
+        int slot_begin = 0;  // first cache slot written by the CPY
+        int src_begin = 0;   // where the copied data starts in the source tensor (in rows of it)
+    };
+
+    std::map<std::string, RsWriteback> rs_writebacks;
+    // Offsets of the state cache writeback CPY nodes, keyed by node name. They change with the
+    // batch (kv head, active sequence count, token count) and, with rollback enabled
+    // (cparams.n_rs_seq > 0), the conv state is written back once per snapshot slot, each snapshot
+    // taking a different conv_input window. Passed to the cached model as runtime inputs.
 };
 
 class GgmlOvDecoder : public ov::frontend::ggml::GgmlDecoder {
@@ -59,8 +103,6 @@ public:
         std::map<std::string, ggml_tensor *> node_inputs;
         std::map<std::string, std::vector<std::pair<std::string, ggml_tensor *>>> node_inputs_views;
         std::vector<std::string> node_inputs_names;
-        ggml_tensor * node_output;
-        std::string node_output_name;
         int node_op_case = 0;
         void * data_addr;
     };
@@ -156,6 +198,10 @@ public:
 
     virtual std::vector<std::string> get_output_names(int node_idx) const override;
 
+    virtual std::string get_inplace_op_src(int node_idx) const override;
+
+    virtual bool is_view_like_alias_of(int node_idx, const std::string & view_src_name) const override;
+
     virtual const std::string & get_op_type() const override;
 
     virtual const std::string & get_op_type(int node_idx) const override;
@@ -173,23 +219,19 @@ public:
 
     virtual int get_op_case(int node_idx) const override { return m_node_info_list[node_idx].node_op_case; }
 
-    virtual const std::map<std::string, std::shared_ptr<ov::Node>> & get_model_inputs() const override {
+    virtual const std::map<std::string, ov::frontend::ggml::ModelInputInfo> & get_model_inputs() const override {
         return m_model_inputs;
     }
 
-    virtual const std::map<std::string, std::shared_ptr<ov::Node>> & get_model_extra_inputs() const override {
+    virtual const std::map<std::string, ov::frontend::ggml::ModelExtraInputInfo> & get_model_extra_inputs() const override {
         return m_model_extra_inputs;
     }
 
-    virtual const std::map<std::string, std::shared_ptr<ov::Tensor>> & get_model_extra_input_values() const {
-        return m_model_extra_input_values;
-    }
-
     virtual const std::map<std::string, std::shared_ptr<ov::Node>> & get_model_weights() const override {
         return m_model_weights;
     }
 
-    virtual std::vector<std::string> get_model_output_names() const override { return m_model_output_names; }
+    virtual std::set<std::string> get_model_output_names() const override { return m_model_output_names; }
 
     const std::map<std::string, ggml_tensor *> & get_model_outputs() const { return m_model_outputs; }
 
@@ -214,6 +256,8 @@ public:
 
     virtual bool has_mixed_rope_params() const override { return m_model_params.mixed_rope_params; }
 
+    virtual int get_ssm_state_size() const override { return m_model_params.state_size; }
+
     virtual std::map<std::string, std::string> get_kv_param_res_names() const override;
 
     virtual bool is_static() const override { return m_is_static; }
@@ -235,6 +279,11 @@ public:
     static std::map<std::string, std::shared_ptr<ov::Node>> create_weight_nodes(ggml_cgraph * cgraph,
                                                                                 bool naive = false);
 
+    // Collect just the set of weight-tensor names referenced by the graph, without
+    // building (or requantizing) any OV weight nodes. Used by topology checks like
+    // is_model_splitted that only need name membership.
+    static std::set<std::string> collect_weight_names(ggml_cgraph * cgraph);
+
     const ggml_tensor * get_tensor_used_op(const ggml_tensor * tensor) const;
 
     const ggml_tensor * get_tensor_from_name(const std::string & name) const;
@@ -274,6 +323,12 @@ public:
         return op->op == GGML_OP_ROPE && tensor == op->src[1];
     }
 
+    // IMROPE packs 4 stacked position planes (t/h/w/e) into inp_pos, each of length
+    // n_tokens; other modes carry a single position per token.
+    inline static int get_inp_pos_n_planes(const ggml_tensor * op) {
+        return op->op_params[2] == GGML_ROPE_TYPE_IMROPE ? 4 : 1;
+    }
+
     inline static bool is_inp_emb(const ggml_tensor * tensor, const ggml_tensor * op) {
         return tensor->op == GGML_OP_GET_ROWS && op->op == GGML_OP_RMS_NORM;
     }
@@ -287,8 +342,12 @@ public:
         return op->op == GGML_OP_ROPE && tensor == op->src[2];
     }
 
+    // also returns true for cache_s and cache_r in SSM/DeltaNet models
     inline static bool is_kvcache(const ggml_tensor * tensor, const ggml_tensor * op) {
-        return tensor->buffer->usage == GGML_BACKEND_BUFFER_USAGE_ANY ||
+        if (tensor == nullptr) {
+            return false;
+        }
+        return (tensor->buffer != nullptr && tensor->buffer->usage == GGML_BACKEND_BUFFER_USAGE_ANY) ||
                (op != nullptr && op->op == GGML_OP_SET_ROWS && op->src[2] == tensor);
     }
 
@@ -301,7 +360,13 @@ public:
                op->src[1]->op == GGML_OP_NONE;
     }
 
-    std::string get_graph_input_ov_name(const ggml_tensor * tensor, const ggml_tensor * op) {
+    // the state permutation index input used in SSM/DeltaNet models (inp->s_copy in llama-graph.cpp)
+    inline static bool is_inp_s_copy(const ggml_tensor * tensor, const ggml_tensor * op) {
+        return op->op == GGML_OP_GET_ROWS && tensor == op->src[1] &&
+               op->src[0]->buffer->usage == GGML_BACKEND_BUFFER_USAGE_ANY;
+    }
+
+    std::string get_graph_input_ov_name(const ggml_tensor * tensor, const ggml_tensor * op) const {
         if (is_inp_pos(tensor, op)) {
             return "inp_pos";
         }
@@ -321,6 +386,10 @@ private:
     void compute_model_inputs();
     void compute_model_outputs();
 
+    // True if tensor is the inp->s_copy index leaf gathered by a recurrent state cache GET_ROWS
+    // (possibly through a VIEW), so it gets a dynamic [1,1,1,-1] graph-input shape.
+    bool is_s_copy_leaf(const ggml_tensor * tensor) const;
+
     // Infer and propagate dynamic-dimension indices for all tensors in the GGML graph.
     void compute_node_dynamic_dims();
 
@@ -329,12 +398,11 @@ private:
     ggml_cgraph * m_cgraph = nullptr;
     std::map<std::string, ggml_tensor *> m_inputs;
 
-    std::map<std::string, std::shared_ptr<ov::Node>> m_model_inputs;
-    std::map<std::string, std::shared_ptr<ov::Node>> m_model_extra_inputs;
-    std::map<std::string, std::shared_ptr<ov::Tensor>> m_model_extra_input_values;
+    std::map<std::string, ov::frontend::ggml::ModelInputInfo> m_model_inputs;
+    std::map<std::string, ov::frontend::ggml::ModelExtraInputInfo> m_model_extra_inputs;
     std::map<std::string, std::shared_ptr<ov::Node>> m_model_weights;
     std::map<std::string, ggml_tensor *> m_model_outputs;
-    std::vector<std::string> m_model_output_names;
+    std::set<std::string> m_model_output_names;
     std::vector<NodeInfo> m_node_info_list;
     std::map<ggml_tensor *, int> m_node_dynamic_dims;
 

ggml/src/ggml-openvino/ggml-openvino-extra.cpp

@@ -31,6 +31,7 @@ void ggml_openvino_device_config::init() {
         // String values (use ggml_openvino_getenv_str)
         "GGML_OPENVINO_DEVICE",
         "GGML_OPENVINO_CACHE_DIR",
+        "GGML_OPENVINO_DEBUG_NODE",
         // Integer values (use ggml_openvino_getenv_int)
         "GGML_OPENVINO_PREFILL_CHUNK_SIZE",
         // Boolean toggles (treated as int flags via ggml_openvino_getenv_int)
@@ -44,7 +45,12 @@ void ggml_openvino_device_config::init() {
         "GGML_OPENVINO_ENABLE_CACHE",
         "GGML_OPENVINO_DISABLE_CACHE",
         "GGML_OPENVINO_DISABLE_KV_SLICE",
+        "GGML_OPENVINO_ENABLE_FALLBACK",
         "GGML_OPENVINO_MANUAL_GQA_ATTN",
+        "GGML_OPENVINO_MEMORY_OPTIMIZE",
+        "GGML_OPENVINO_RELEASE_WEIGHTS",
+        "GGML_OPENVINO_REDUCE_COMPILE_MEM",
+        "GGML_OPENVINO_COMPILED_MODEL_CACHE_DIR",
     };
 
     for (const char * const & env_var : env_var_names) {
@@ -168,6 +174,22 @@ int ggml_openvino_getenv_int(const char * var, int default_value) {
     return v ? std::atoi(v) : default_value;
 }
 
+bool ggml_openvino_reduce_compile_mem_enabled() {
+    const char * reduce_compile_mem = ggml_openvino_getenv_str("GGML_OPENVINO_REDUCE_COMPILE_MEM");
+    if (reduce_compile_mem != nullptr) {
+        return ggml_openvino_getenv_int("GGML_OPENVINO_REDUCE_COMPILE_MEM") != 0;
+    }
+    return ggml_openvino_getenv_int("GGML_OPENVINO_MEMORY_OPTIMIZE") != 0;
+}
+
+bool ggml_openvino_release_weights_enabled(const std::string & device) {
+    const char * release_weights = ggml_openvino_getenv_str("GGML_OPENVINO_RELEASE_WEIGHTS");
+    if (release_weights != nullptr) {
+        return device == "GPU" && ggml_openvino_getenv_int("GGML_OPENVINO_RELEASE_WEIGHTS") != 0;
+    }
+    return device == "GPU" && ggml_openvino_getenv_int("GGML_OPENVINO_MEMORY_OPTIMIZE") != 0;
+}
+
 // Check if running on NPU
 bool ggml_openvino_is_npu() {
     return ggml_openvino_get_device_config().is_npu;
@@ -252,14 +274,31 @@ ggml_openvino_extracted_layout ggml_openvino_get_extracted_layout(const ggml_ten
         return layout;
     }
 
-    // Only handle 2D weight tensors
-    if (tensor->ne[2] != 1 || tensor->ne[3] != 1) {
+    // Most quantized weights use the existing 2D extraction path. 3D expert weights for
+    // MUL_MAT_ID (MoE) are also supported, either as MXFP4 (packed, dedicated branch below) or via the
+    // generic sizing math below, which is shape-agnostic (based on total element count). Only reject 4D.
+    if (tensor->ne[3] != 1) {
         return layout;
     }
 
+    // 3D MoE expert weights that are not requantized (see below) always use the exact f16
+    // zero-point extraction (see extract_quantized_weights), which needs a wider zp slot than
+    // the packed integer zero point -- must be kept in sync with that function so the buffer
+    // sizing here matches what process_weight_tensor actually writes.
+    const bool for_gather_matmul = tensor->ne[2] > 1;
+
     int64_t n_elements = ggml_nelements(tensor);
     const size_t alignment = 64;  // Good for SIMD
 
+    if (tensor->type == GGML_TYPE_MXFP4 && (tensor->ne[2] > 1 || tensor->ne[3] > 1)) {
+        layout.weights_per_block = 32;
+        layout.is_symmetric = true;
+        layout.weights_size = ggml_nbytes(tensor);
+        layout.weights_offset = 0;
+        layout.total_size = layout.weights_size;
+        return layout;
+    }
+
     // Check if requantization is needed (NPU-specific)
     auto requant_type = ggml_openvino_get_requant_type(tensor, use_bias);
     if (requant_type.has_value()) {
@@ -334,6 +373,11 @@ ggml_openvino_extracted_layout ggml_openvino_get_extracted_layout(const ggml_ten
     layout.is_symmetric = false;
 
     switch (tensor->type) {
+    case GGML_TYPE_MXFP4:
+        layout.is_u4 = true;
+        layout.is_symmetric = true;
+        break;
+
     case GGML_TYPE_Q4_0:
         layout.is_u4 = true;
         layout.is_symmetric = true;
@@ -369,12 +413,17 @@ ggml_openvino_extracted_layout ggml_openvino_get_extracted_layout(const ggml_ten
     // Weights: U4 = n_elements/2 bytes, U8 = n_elements bytes
     layout.weights_size = layout.is_u4 ? (n_elements / 2) : n_elements;
 
-    // Scales: F16 per block
+    // Scales: F16 per block, except MXFP4 which stores one E8M0 byte per block.
     int64_t n_blocks = n_elements / layout.weights_per_block;
-    layout.scales_size = n_blocks * sizeof(uint16_t);  // F16 = 2 bytes
-    // For symmetric quantization, no zp needed (weights stored as signed)
+    layout.scales_size = n_blocks * (tensor->type == GGML_TYPE_MXFP4 ? sizeof(uint8_t) : sizeof(uint16_t));
+    // For symmetric quantization, no zp needed (weights stored as signed). Asymmetric
+    // for_gather_matmul (3D MoE expert) weights use an exact f16 zero point (see
+    // extract_quantized_weights/make_int8_weights/make_int4_weights), which needs one f16 per
+    // block instead of a packed u4/u8 integer zero point.
     if (layout.is_symmetric) {
         layout.zp_size = 0;
+    } else if (use_bias || for_gather_matmul) {
+        layout.zp_size = n_blocks * sizeof(uint16_t);
     } else {
         layout.zp_size = layout.is_u4 ? ((n_blocks + 1) / 2) : n_blocks;
     }

ggml/src/ggml-openvino/ggml-openvino-extra.h

@@ -96,9 +96,22 @@ const std::string & ggml_openvino_get_device_name();
 const char * ggml_openvino_getenv_str(const char * var, const char * default_value = nullptr);
 int ggml_openvino_getenv_int(const char * var, int default_value = 0);
 
+// Memory optimization toggles. GGML_OPENVINO_MEMORY_OPTIMIZE is an umbrella
+// switch; the fine-grained env vars still override it when explicitly set.
+bool ggml_openvino_reduce_compile_mem_enabled();
+bool ggml_openvino_release_weights_enabled(const std::string & device);
+
 // Check if running on NPU
 bool ggml_openvino_is_npu();
 
+// Host weight-buffer release (GGML_OPENVINO_RELEASE_WEIGHTS, GPU only).
+// register: record a host weight buffer (idempotent per data pointer).
+// release:  madvise(MADV_DONTNEED) all registered buffers, dropping their RSS.
+// released: true once release has run (used to fail-fast on post-release recompile).
+void ggml_openvino_register_weight_buffer(void * data, size_t size);
+void ggml_openvino_release_weight_buffers();
+bool ggml_openvino_weight_buffers_released();
+
 // Get requantization type for a tensor type (returns nullopt if no requant needed)
 std::optional<ExtraQuantType> ggml_openvino_get_requant_type(const ggml_tensor * tensor, bool no_requant = false);
 

ggml/src/ggml-openvino/ggml-openvino.cpp

@@ -32,6 +32,7 @@
 #    endif
 #    include <windows.h>
 #else
+#    include <sys/mman.h>
 #    include <unistd.h>
 #endif
 
@@ -135,6 +136,81 @@ struct ggml_backend_openvino_buffer_type_context {
     std::string name;
 };
 
+// =====================================================
+// Host weight-buffer release (GGML_OPENVINO_RELEASE_WEIGHTS)
+// =====================================================
+// The OpenVINO weight Constants are zero-copy views into the host buffers
+// allocated here (ggml_aligned_malloc, anonymous memory). On GPU the plugin
+// holds its own device copy after compile_model, so the host pages are dead
+// weight for inference and can be dropped to reclaim RSS (~weights size).
+//
+// We do NOT free the buffer (ggml owns its lifetime and tensors still point
+// into it); instead madvise(MADV_DONTNEED) drops the resident pages while
+// keeping the mapping valid. A later recompile would re-read these Constants
+// from now-zeroed memory and produce garbage, so once released we fail fast
+// if the cache-miss compile branch is reached again (see utils.cpp).
+namespace {
+struct ov_weight_buffer_registry {
+    std::mutex mutex;
+    // (data, size) of every non-remote weight buffer, for madvise.
+    std::vector<std::pair<void *, size_t>> buffers;
+    bool released = false;
+};
+
+ov_weight_buffer_registry & ov_weight_registry() {
+    static ov_weight_buffer_registry reg;
+    return reg;
+}
+}  // namespace
+
+void ggml_openvino_register_weight_buffer(void * data, size_t size) {
+    if (data == nullptr || size == 0) {
+        return;
+    }
+    auto & reg = ov_weight_registry();
+    std::lock_guard<std::mutex> lock(reg.mutex);
+    for (const auto & b : reg.buffers) {
+        if (b.first == data) {
+            return;  // already registered
+        }
+    }
+    reg.buffers.emplace_back(data, size);
+}
+
+bool ggml_openvino_weight_buffers_released() {
+    auto & reg = ov_weight_registry();
+    std::lock_guard<std::mutex> lock(reg.mutex);
+    return reg.released;
+}
+
+void ggml_openvino_release_weight_buffers() {
+    auto & reg = ov_weight_registry();
+    std::lock_guard<std::mutex> lock(reg.mutex);
+    if (reg.released) {
+        return;
+    }
+    size_t total = 0;
+#if !defined(_WIN32)
+    for (const auto & b : reg.buffers) {
+        // Align down/up to page boundaries so madvise only drops whole pages
+        // fully owned by this buffer.
+        const long page = sysconf(_SC_PAGESIZE);
+        uintptr_t start = reinterpret_cast<uintptr_t>(b.first);
+        uintptr_t end = start + b.second;
+        uintptr_t astart = (start + page - 1) & ~(uintptr_t) (page - 1);
+        uintptr_t aend = end & ~(uintptr_t) (page - 1);
+        if (aend > astart) {
+            if (madvise(reinterpret_cast<void *>(astart), aend - astart, MADV_DONTNEED) == 0) {
+                total += aend - astart;
+            }
+        }
+    }
+#endif
+    reg.released = true;
+    GGML_LOG_INFO("%s: released %zu MB of host weight buffers (%zu buffers)\n", __func__, total / 1024 / 1024,
+                  reg.buffers.size());
+}
+
 // Buffer interface functions
 static void ggml_backend_openvino_buffer_free_buffer(ggml_backend_buffer_t buffer) {
     ggml_backend_openvino_buffer_context * ctx = (ggml_backend_openvino_buffer_context *) buffer->context;
@@ -235,10 +311,12 @@ static void ggml_backend_openvino_buffer_set_tensor(ggml_backend_buffer_t buffer
     bool is_weight_buffer = (buffer->usage == GGML_BACKEND_BUFFER_USAGE_WEIGHTS);
     // Full tensor set: offset=0, full size, not a view
     bool is_full_tensor_set = (offset == 0 && size == ggml_nbytes(tensor) && tensor->view_src == nullptr);
-    // 2D tensor (typical weight shape)
+    // 2D tensor (typical weight shape), or a 3D quantized MoE expert weight (MUL_MAT_ID). Dense 3D
+    // expert weights are handled later in create_weight_node instead.
     bool is_2d = (tensor->ne[2] == 1 && tensor->ne[3] == 1);
+    bool is_supported_weight_shape = is_2d || (tensor->ne[3] == 1 && ggml_is_quantized(tensor->type));
 
-    if (is_weight_buffer && is_full_tensor_set && is_2d) {
+    if (is_weight_buffer && is_full_tensor_set && is_supported_weight_shape) {
         try {
             auto result = process_weight_tensor(tensor, data, tensor->data);
             result.weight_node->set_friendly_name(tensor->name);
@@ -274,6 +352,22 @@ static void ggml_backend_openvino_buffer_set_tensor(ggml_backend_buffer_t buffer
             ctx->tensor_extras[tensor] = extra;
             tensor->extra = extra;
 
+            // Register the host buffer so its pages can be dropped after the GPU
+            // plugin has its own device copy (GGML_OPENVINO_RELEASE_WEIGHTS).
+            if (!ctx->is_remote) {
+                // Weights are set once at model load. Setting a weight after a release
+                // means a second model is loading while the first's compiled graph is
+                // pinned — that graph would be wrongly reused with this model's key.
+                // Fail loud rather than return silently-wrong results.
+                if (ggml_openvino_weight_buffers_released()) {
+                    GGML_ABORT(
+                        "ggml-openvino: loading a new model while GGML_OPENVINO_RELEASE_WEIGHTS pinned a previous "
+                        "model's compiled graph. This mode supports a single model per process; unset it for "
+                        "multi-model runs.");
+                }
+                ggml_openvino_register_weight_buffer(ctx->data, ctx->size);
+            }
+
         } catch (const std::exception & e) {
             GGML_LOG_ERROR("%s: failed to process weight tensor for %s: %s\n", __func__, tensor->name, e.what());
             memcpy((char *) tensor->data + offset, data, size);
@@ -458,8 +552,8 @@ static size_t ggml_backend_openvino_buffer_type_get_alloc_size(ggml_backend_buff
                                                                const ggml_tensor * tensor) {
     GGML_UNUSED(buft);
 
-    // For quantized 2D tensors (weights), we need extra space for extracted data
-    if (ggml_is_quantized(tensor->type) && tensor->ne[2] == 1 && tensor->ne[3] == 1) {
+    // For quantized weight tensors, we need extra space for extracted data.
+    if (ggml_is_quantized(tensor->type) && tensor->ne[3] == 1) {
         ggml_openvino_extracted_layout layout = ggml_openvino_get_extracted_layout(tensor);
         if (layout.total_size > 0) {
             // GGML_LOG_DEBUG("%s: tensor %s needs %zu bytes (original %zu, extracted: weights=%zu scales=%zu zp=%zu)\n",
@@ -618,7 +712,13 @@ static void ggml_backend_openvino_free(ggml_backend_t backend) {
     if (ctx->runtime_context) {
         auto r_ctx = std::static_pointer_cast<ov_runtime_context>(ctx->runtime_context);
         if (--r_ctx->backend_count == 0) {
-            r_ctx->clear_caches();
+            // If host weight buffers were released (GGML_OPENVINO_RELEASE_WEIGHTS), the
+            // dropped pages can never be repopulated, so a recompile is impossible. Keep
+            // the compiled-model cache alive across backend teardown so the next context
+            // reuses it instead of recompiling against zeroed weights.
+            if (!ggml_openvino_weight_buffers_released()) {
+                r_ctx->clear_caches();
+            }
         }
     }
 
@@ -856,6 +956,32 @@ static bool checked_mul_size(size_t a, size_t b, size_t & out) {
     return true;
 }
 
+static bool tensor_view_fits_src_buffer(const ggml_tensor * tensor) {
+    if (tensor->view_src == nullptr) {
+        return true;
+    }
+
+    const size_t src_nbytes = ggml_nbytes(tensor->view_src);
+    if (tensor->view_offs > src_nbytes) {
+        return false;
+    }
+
+    const size_t tensor_nbytes = ggml_nbytes(tensor);
+    return tensor_nbytes <= src_nbytes - tensor->view_offs;
+}
+
+static bool cpy_output_view_is_supported(const ggml_tensor * op) {
+    if (op->view_src == nullptr) {
+        return true;
+    }
+
+    if (!tensor_view_fits_src_buffer(op)) {
+        return false;
+    }
+
+    return ggml_nbytes(op) == 0 || ggml_is_contiguous(op);
+}
+
 static bool mul_mat_id_requires_large_tmp(const ggml_tensor * op) {
     const ggml_tensor * as = op->src[0];
     const ggml_tensor * ids = op->src[2];
@@ -863,9 +989,10 @@ static bool mul_mat_id_requires_large_tmp(const ggml_tensor * op) {
         return true;
     }
 
-    // The current OpenVINO translation materializes selected expert weights with
-    // shape [n_tokens, n_used, rows, k]. Skip cases that would create a very
-    // large temporary on GPU and let the scheduler fall back instead.
+    // The MXFP4 MUL_MAT_ID translation (translate_mul_mat_id_mxfp4_packed in mul_mat_id.cpp)
+    // materializes selected expert weights with shape [n_tokens, n_used, rows, k]. Skip cases that
+    // would create a very large temporary and let the scheduler fall back instead. Every other weight
+    // type goes through GatherMatmul, which never materializes this temporary.
     size_t tmp_elems = 1;
     if (!checked_mul_size(tmp_elems, static_cast<size_t>(ids->ne[1]), tmp_elems) ||
         !checked_mul_size(tmp_elems, static_cast<size_t>(ids->ne[0]), tmp_elems) ||
@@ -883,12 +1010,56 @@ static bool mul_mat_id_requires_large_tmp(const ggml_tensor * op) {
     return tmp_bytes > mul_mat_id_tmp_limit;
 }
 
+static bool tensor_name_starts_with(const ggml_tensor * tensor, const char * prefix) {
+    return tensor != nullptr && strncmp(tensor->name, prefix, strlen(prefix)) == 0;
+}
+
+static bool is_msa_block_mask_expansion(const ggml_tensor * op) {
+    if (tensor_name_starts_with(op, "msa_")) {
+        return true;
+    }
+
+    const ggml_tensor * src = op->src[0];
+    while (src != nullptr && (src->op == GGML_OP_RESHAPE || src->op == GGML_OP_REPEAT)) {
+        if (tensor_name_starts_with(src, "msa_block_mask")) {
+            return true;
+        }
+        src = src->src[0];
+    }
+
+    return tensor_name_starts_with(src, "msa_block_mask");
+}
+
 static bool is_op_unsupported_case(const ggml_tensor * op) {
+    if (is_msa_block_mask_expansion(op)) {
+        return true;
+    }
+
     switch (op->op) {
     case GGML_OP_CONCAT: {
         if (op->type == GGML_TYPE_I64) {
             return true;
         }
+        if (ggml_openvino_get_device_name() == "GPU" && op->type == GGML_TYPE_BF16 && has_view_op_input(op)) {
+            return true;
+        }
+        break;
+    }
+    case GGML_OP_SET: {
+        const auto nb1 = static_cast<size_t>(op->op_params[0]);
+        const auto nb2 = static_cast<size_t>(op->op_params[1]);
+        const auto nb3 = static_cast<size_t>(op->op_params[2]);
+
+        // OpenVINO SET translation currently supports dst layouts that match src0 strides.
+        if (op->src[0] == nullptr || nb1 != op->src[0]->nb[1] || nb2 != op->src[0]->nb[2] || nb3 != op->src[0]->nb[3]) {
+            // std::cout << "Unsupported SET op with dst nb1=" << nb1 << ", nb2=" << nb2 << ", nb3=" << nb3
+            //           << " that does not match src0 strides nb[1]="
+            //           << (op->src[0] != nullptr ? std::to_string(op->src[0]->nb[1]) : "null")
+            //           << ", nb[2]=" << (op->src[0] != nullptr ? std::to_string(op->src[0]->nb[2]) : "null")
+            //           << ", nb[3]=" << (op->src[0] != nullptr ? std::to_string(op->src[0]->nb[3]) : "null")
+            //           << std::endl;
+            return true;
+        }
         break;
     }
     case GGML_OP_GET_ROWS:
@@ -896,23 +1067,24 @@ static bool is_op_unsupported_case(const ggml_tensor * op) {
         if (op->ne[3] != 1) {
             return true;
         }
-        if (op->ne[0] == 256 && (op->src[0]->type == GGML_TYPE_Q4_K || op->src[0]->type == GGML_TYPE_Q5_K)) {
-            // ERR = 0.000000306 > 0.000000100   GET_ROWS(type=q4_K,n=256,m=5,r=4,be1=1,be2=1,v=0)
-            // ERR = 0.000000197 > 0.000000100   GET_ROWS(type=q5_K,n=256,m=5,r=4,be1=1,be2=1,v=0)
+        if (op->op == GGML_OP_GET_ROWS && ggml_openvino_get_device_name() == "GPU" &&
+            op->src[0]->type == GGML_TYPE_BF16) {
+            return true;
+        }
+        if (op->ne[0] == 256 && (op->src[0]->type == GGML_TYPE_Q4_K || op->src[0]->type == GGML_TYPE_Q5_K ||
+                                 op->src[0]->type == GGML_TYPE_Q4_1 || op->src[0]->type == GGML_TYPE_Q5_1)) {
+            // These are all f16-arithmetic dequant rounding errors that intermittently exceed the
+            // tight 1e-7 NMSE threshold depending on the random test data (see ggml-quants.cpp
+            // make_int8_weights/make_int4_weights: dequant is done in f16, not f32, to keep the
+            // Convert/Subtract/Multiply chain fusable into GatherMatmulCompressed/FullyConnectedCompressed
+            // for the shared non-test code paths).
             return true;
         }
 
-        // Keep the MoE routing weights gather on CPU for GPU runs. Splitting
-        // only at the later SUM/CLAMP/DIV nodes still leaves this routing path
-        // numerically unstable for arctic-style MoE graphs.
-        if (strncmp(op->name, "ffn_moe_weights", sizeof("ffn_moe_weights") - 1) == 0) {
-            return true;
-        }
         break;
     }
     case GGML_OP_RESHAPE: {
-        if (strncmp(op->name, "ffn_moe_weights", sizeof("ffn_moe_weights") - 1) == 0 ||
-            strncmp(op->name, "ffn_norm_exps", sizeof("ffn_norm_exps") - 1) == 0) {
+        if (strncmp(op->name, "ffn_norm_exps", sizeof("ffn_norm_exps") - 1) == 0) {
             return true;
         }
         break;
@@ -939,69 +1111,22 @@ static bool is_op_unsupported_case(const ggml_tensor * op) {
         break;
     }
     case GGML_OP_DIV: {
-        bool requires_broadcast = false;
-        for (int i = 0; i < 4; i++) {
-            if (op->src[0]->ne[i] == op->src[1]->ne[i]) {
-                continue;
-            }
-
-            if (op->src[0]->ne[i] != 1 && op->src[1]->ne[i] != 1) {
-                return true;
-            }
-
-            requires_broadcast = true;
-        }
-
         // The GPU plugin can fuse broadcast DIV into the preceding FFN GEMM path
         // and produce infs for per-channel scale vectors. Keep those DIVs on CPU
         // until the fused GPU kernel is reliable. (falied case llama-arch-test mpt)
-        if (requires_broadcast && ggml_openvino_get_device_name() == "GPU") {
-            return true;
-        }
-
-        // qwen3next MoE weight normalization is numerically sensitive on the GPU
-        // path. Keep the normalization divide on CPU to match the reference.
-        if (strncmp(op->name, "ffn_moe_weights_norm", sizeof("ffn_moe_weights_norm") - 1) == 0) {
-            return true;
-        }
-        break;
-    }
-    case GGML_OP_SOFT_MAX: {
-        if (op->src[2] != nullptr) {
-            // GGML_LOG_WARN("OpenVINO backend does not support SOFT_MAX with sinks\n");
-            return true;
-        }
-
-        if (strncmp(op->name, "ffn_moe_probs", sizeof("ffn_moe_probs") - 1) == 0) {
-            return true;
-        }
-
-        // GPU execution of the MoE routing weights softmax is numerically unstable
-        // when fused with the surrounding GET_ROWS/reshape path. Keep this softmax
-        // on CPU so the scheduler splits at the same boundary that restores parity.
-        if (op->src[0] != nullptr && op->src[0]->op == GGML_OP_RESHAPE && op->src[0]->src[0] != nullptr &&
-            strncmp(op->src[0]->src[0]->name, "ffn_moe_weights", sizeof("ffn_moe_weights") - 1) == 0) {
+        if (ggml_openvino_get_device_name() == "GPU" && op->src[1]->ne[0] == op->ne[0] &&
+            op->src[1]->ne[1] == 1 && op->src[1]->ne[2] == 1 && op->src[1]->ne[3] == 1) {
             return true;
         }
         break;
     }
     case GGML_OP_SUM_ROWS: {
-        if (strncmp(op->name, "ffn_moe_weights_sum", sizeof("ffn_moe_weights_sum") - 1) == 0) {
-            return true;
-        }
-
         // if the input is PERMUTE skip
         if (op->src[0]->op == GGML_OP_PERMUTE) {
             return true;
         }
         break;
     }
-    case GGML_OP_CLAMP: {
-        if (strncmp(op->name, "ffn_moe_weights_sum_clamped", sizeof("ffn_moe_weights_sum_clamped") - 1) == 0) {
-            return true;
-        }
-        break;
-    }
     case GGML_OP_FLASH_ATTN_EXT: {
         float scale = 1.0f;
         float max_bias = 0.0f;
@@ -1048,23 +1173,29 @@ static bool is_op_unsupported_case(const ggml_tensor * op) {
             // GGML_LOG_WARN("OpenVINO backend does not support CPY with non-contiguous data or bf16 types\n");
             return true;
         }
+        // CPY to a quantized destination (e.g. f32 -> q4_0) is numerically unstable with OpenVINO backend.
+        if (ggml_is_quantized(op->type)) {
+            return true;
+        }
+        if (ggml_nelements(op->src[0]) != ggml_nelements(op->src[1])) {
+            return true;
+        }
         // op test case with non-contiguous src or dst
         if ((op->ne[0] == 3 && op->ne[1] == 4 && op->ne[2] == 3 && op->ne[3] == 2) ||
             (op->ne[0] == 1 && op->ne[1] == 4 && op->ne[2] == 3 && op->ne[3] == 2) ||
             (op->ne[0] == 2 && op->ne[1] == 4 && op->ne[2] == 3 && op->ne[3] == 2)) {
             return true;
         }
-        // CPY into a strided view of a larger buffer (recurrent-state snapshots) not supported
-        if (op->view_src && ggml_nbytes(op) != ggml_nbytes(op->view_src)) {
+        if (!cpy_output_view_is_supported(op)) {
             return true;
         }
         break;
     }
     case GGML_OP_MUL_MAT: {
-        if (ggml_openvino_get_device_name() == "GPU" && op->src[1]->op == GGML_OP_SOFT_MAX &&
-            op->src[0]->op == GGML_OP_CONT && op->src[0]->src[0] != nullptr &&
-            op->src[0]->src[0]->op == GGML_OP_TRANSPOSE && op->src[0]->src[0]->src[0] != nullptr &&
-            op->src[0]->src[0]->src[0]->op == GGML_OP_PERMUTE) {
+        if (ggml_openvino_get_device_name() == "GPU" && op->src[0] != nullptr && op->src[1] != nullptr &&
+            ggml_is_quantized(op->src[0]->type) && strcmp(op->src[0]->name, "a") == 0 &&
+            strcmp(op->src[1]->name, "b") == 0 && op->src[0]->ne[1] == 1 && op->src[1]->ne[1] == 64 &&
+            op->src[0]->ne[0] == 256 && op->src[1]->ne[0] == 256) {
             return true;
         }
         if (op->src[0]->ne[3] != op->src[1]->ne[3] && op->src[0]->ne[3] != 1 && op->src[1]->ne[3] != 1) {
@@ -1076,12 +1207,18 @@ static bool is_op_unsupported_case(const ggml_tensor * op) {
         break;
     }
     case GGML_OP_MUL_MAT_ID: {
-        if (strncmp(op->name, "ffn_moe_gate_up", sizeof("ffn_moe_gate_up") - 1) == 0 ||
-            strncmp(op->name, "ffn_moe_down", sizeof("ffn_moe_down") - 1) == 0) {
+        // Single-expert (or empty) MUL_MAT_ID is a degenerate shape that stresses GatherMatmul edge
+        // cases and never occurs in real MoE; let it fall back to CPU.
+        if (op->src[0] != nullptr && op->src[0]->ne[2] <= 1) {
             return true;
         }
-
-        if (mul_mat_id_requires_large_tmp(op)) {
+        if (ggml_openvino_get_device_name() == "GPU" && op->src[0] != nullptr && op->src[0]->type == GGML_TYPE_BF16) {
+            return true;
+        }
+        // GPU MUL_MAT_ID uses a Gather+MatMul fallback because the GPU plugin rejects internal
+        // GatherMatmul for these test shapes. Skip cases that would materialize a large selected
+        // expert-weight temporary.
+        if (ggml_openvino_get_device_name() == "GPU" && mul_mat_id_requires_large_tmp(op)) {
             return true;
         }
         break;
@@ -1094,8 +1231,10 @@ static bool is_op_unsupported_case(const ggml_tensor * op) {
             // GGML_LOG_WARN("OpenVINO backend does not support ROPE with mode %d\n", mode);
             return true;
         }
-        if (n_dims != 0.0f && n_dims != op->src[0]->ne[0]) {
-            // GGML_LOG_WARN("OpenVINO backend does not support ROPE with n_dims %d != src[0]->ne[0] %ld\n", n_dims,
+        const int64_t head_dim = op->src[0]->ne[0];
+        const int64_t rope_dims = n_dims == 0 ? head_dim : n_dims;
+        if (rope_dims <= 0 || rope_dims > head_dim || (rope_dims % 2) != 0) {
+            // GGML_LOG_WARN("OpenVINO backend does not support ROPE with n_dims %d and src[0]->ne[0] %ld\n", n_dims,
             //               op->src[0]->ne[0]);
             return true;
         }
@@ -1128,9 +1267,15 @@ static bool is_op_unsupported_case(const ggml_tensor * op) {
         }
         break;
     }
+    case GGML_OP_REPEAT: {
+        if (ggml_openvino_get_device_name() == "GPU" && op->type == GGML_TYPE_BF16) {
+            return true;
+        }
+        break;
+    }
     case GGML_OP_GATED_DELTA_NET: {
         // enable after https://github.com/openvinotoolkit/openvino/pull/35917 is included in OV release
-        return true;
+        // return true;
         // if (ggml_openvino_get_device_name() == "GPU" && op->src[0]->ne[2] > 1) {
         //     // CVS-186471
         //     return true;
@@ -1142,13 +1287,8 @@ static bool is_op_unsupported_case(const ggml_tensor * op) {
         if (op->src[3]->ne[0] != 1) {
             return true;
         }
-        // v_repeat > 1 (GQA): ggml uses modulo head mapping (h_q = h_v % H_k)
-        // but the fused op uses consecutive mapping (h_q = h_v / group_size)
-        if (op->src[2]->ne[1] != op->src[0]->ne[1]) {
-            return true;
-        }
         // K > 1 (multiple state snapshots) not supported by fused op
-        if (op->src[5]->ne[1] > 1) {
+        if (((const int32_t *) op->op_params)[0] > 1) {
             return true;
         }
         break;
@@ -1156,11 +1296,12 @@ static bool is_op_unsupported_case(const ggml_tensor * op) {
     case GGML_OP_SSM_CONV: {
         // qwen3next is numerically unstable with OpenVINO SSM_CONV.
         // Keep this op on CPU until the OpenVINO implementation is fixed.
-        return true;
+        // return true;
+        break;
     }
     case GGML_OP_VIEW: {
-        // Skip TOPK_MOE fused tests until it is fully supported
-        // the argsort_top_k VIEW wrapping ARGSORT is named "selected_experts" in test_topk_moe
+        // Skip TOPK_MOE fused tests until it is fully supported.
+        // The argsort_top_k VIEW wrapping ARGSORT is named "selected_experts" in test_topk_moe.
         if (strcmp(op->name, "selected_experts") == 0) {
             return true;
         }
@@ -1177,7 +1318,8 @@ static bool ggml_backend_openvino_device_supports_op(ggml_backend_dev_t dev, con
 
     static std::unordered_set<ggml_type> supported_types{
         GGML_TYPE_F32,  GGML_TYPE_F16,  GGML_TYPE_BF16, GGML_TYPE_I64,  GGML_TYPE_I32,  GGML_TYPE_Q4_0,
-        GGML_TYPE_Q4_1, GGML_TYPE_Q4_K, GGML_TYPE_Q5_1, GGML_TYPE_Q5_K, GGML_TYPE_Q8_0, GGML_TYPE_Q6_K};
+        GGML_TYPE_Q4_1, GGML_TYPE_Q4_K, GGML_TYPE_Q5_1, GGML_TYPE_Q5_K, GGML_TYPE_Q8_0, GGML_TYPE_Q6_K,
+        GGML_TYPE_MXFP4};
 
     // derive supported op sets from the op_table map, keys in
     // the map use the full macro name (e.g. "GGML_OP_ADD"), while
@@ -1224,6 +1366,9 @@ static bool ggml_backend_openvino_device_supports_op(ggml_backend_dev_t dev, con
             // GGML_LOG_WARN("OpenVINO backend does not support unary op %s\n", ggml_unary_op_name(ggml_get_unary_op(op)));
             return false;
         }
+        if (ggml_get_unary_op(op) == GGML_UNARY_OP_EXP && op->type == GGML_TYPE_F32) {
+            return false;
+        }
         break;
     }
     case GGML_OP_GLU: {
@@ -1232,11 +1377,11 @@ static bool ggml_backend_openvino_device_supports_op(ggml_backend_dev_t dev, con
             // GGML_LOG_WARN("OpenVINO backend does not support GLU op %s\n", ggml_glu_op_name(ggml_get_glu_op(op)));
             return false;
         }
-        if (has_view_op_input(op)) {
-            // GGML_LOG_WARN("OpenVINO backend does not support unary op %s with view input\n",
-            //               ggml_glu_op_name(ggml_get_glu_op(op)));
-            return false;
-        }
+        // if (has_view_op_input(op)) {
+        //     // GGML_LOG_WARN("OpenVINO backend does not support unary op %s with view input\n",
+        //     //               ggml_glu_op_name(ggml_get_glu_op(op)));
+        //     return false;
+        // }
         if (op->src[1] == nullptr && op->src[0]->ne[0] % 2 != 0) {
             // triggers bug in ov gpu
             return false;
@@ -1249,16 +1394,11 @@ static bool ggml_backend_openvino_device_supports_op(ggml_backend_dev_t dev, con
             // GGML_LOG_WARN("OpenVINO backend does not support op %s\n", ggml_op_name(op->op));
             return false;
         }
-        static std::set<ggml_op> ops_not_support_view_input{
-            GGML_OP_L2_NORM,
-        };
+        static std::set<ggml_op> ops_not_support_view_input{};
         if (ops_not_support_view_input.find(op->op) != ops_not_support_view_input.end() && has_view_op_input(op)) {
             // GGML_LOG_WARN("OpenVINO backend does not support op %s with view input\n", ggml_op_name(op->op));
             return false;
         }
-        if (op->op == GGML_OP_RMS_NORM && has_non_contiguous_view_input(op)) {
-            return false;
-        }
     }
     }
 
@@ -1275,7 +1415,9 @@ static bool ggml_backend_openvino_device_supports_op(ggml_backend_dev_t dev, con
             // GGML_LOG_WARN("OpenVINO backend does not support tensor type %s\n", ggml_type_name(src->type));
             return false;
         }
-        if (ggml_is_quantized(src->type) && src->ne[2] != 1) {
+        const bool is_supported_3d_moe_expert =
+            op->op == GGML_OP_MUL_MAT_ID && i == 0 && (src->type == GGML_TYPE_MXFP4 || src->ne[3] == 1);
+        if (ggml_is_quantized(src->type) && src->ne[2] != 1 && !is_supported_3d_moe_expert) {
             // GGML_LOG_WARN("OpenVINO backend does not support 3D quantized tensors\n");
             return false;
         }

ggml/src/ggml-openvino/ggml-quants.cpp

@@ -2,6 +2,7 @@
 
 #include "ggml-common.h"
 #include "ggml-impl.h"
+#include "ggml-openvino-extra.h"
 #include "ggml.h"
 
 #include <algorithm>
@@ -19,6 +20,8 @@
 #include <openvino/core/type/element_type.hpp>
 #include <openvino/core/type/element_type_traits.hpp>
 #include <openvino/core/type/float16.hpp>
+#include <openvino/core/type/float4_e2m1.hpp>
+#include <openvino/core/type/float8_e8m0.hpp>
 #include <openvino/op/add.hpp>
 #include <openvino/op/constant.hpp>
 #include <openvino/op/convert.hpp>
@@ -26,6 +29,7 @@
 #include <openvino/op/reshape.hpp>
 #include <openvino/op/subtract.hpp>
 #include <openvino/op/util/attr_types.hpp>
+#include <openvino/pass/constant_folding.hpp>
 #include <openvino/runtime/tensor.hpp>
 #include <string>
 #include <vector>
@@ -44,6 +48,38 @@ void unpack_32_4(const uint8_t * data, uint8_t * dst) {
     }
 }
 
+static constexpr size_t MXFP4_BLOCK_SIZE = 32;
+static constexpr size_t MXFP4_BLOCK_QS_SIZE = MXFP4_BLOCK_SIZE / 2;
+static constexpr size_t MXFP4_BLOCK_BYTES = sizeof(uint8_t) + MXFP4_BLOCK_QS_SIZE;
+
+static void pack_32_mxfp4_for_openvino(const uint8_t * data, uint8_t * dst) {
+    for (int j = 0; j < static_cast<int>(MXFP4_BLOCK_QS_SIZE); j += 2) {
+        const uint8_t v0 = data[j] & 0x0F;
+        const uint8_t v1 = (data[j + 1] & 0x0F) << 4;
+        const uint8_t v16 = data[j] >> 4;
+        const uint8_t v17 = data[j + 1] & 0xF0;
+        dst[j / 2] = v0 | v1;
+        dst[MXFP4_BLOCK_SIZE / 4 + j / 2] = v16 | v17;
+    }
+}
+
+void extract_mxfp4_data(const ggml_tensor * tensor, ov::Tensor & weights_arr, ov::Tensor & scales_arr) {
+    GGML_ASSERT(tensor->type == GGML_TYPE_MXFP4);
+    GGML_ASSERT(weights_arr.get_element_type() == ov::element::f4e2m1);
+    GGML_ASSERT(scales_arr.get_element_type() == ov::element::f8e8m0);
+
+    const auto * data = static_cast<const uint8_t *>(tensor->data);
+    auto * weights = static_cast<uint8_t *>(weights_arr.data());
+    auto * scales = scales_arr.data<ov::element_type_traits<ov::element::f8e8m0>::value_type>();
+    const size_t n_blocks = scales_arr.get_size();
+
+    ov::parallel_for(n_blocks, [&](size_t i) {
+        const uint8_t * block = data + i * MXFP4_BLOCK_BYTES;
+        pack_32_mxfp4_for_openvino(block + sizeof(uint8_t), weights + i * MXFP4_BLOCK_QS_SIZE);
+        scales[i] = ov::float8_e8m0::from_bits(block[0]);
+    });
+}
+
 // Extracts (weight, scales, zp) from Q4_0 tensors.
 // Data layout is: |16 bit scale|32 x 4bit weights|.
 // When zp_arr is empty (symmetric), weights are stored as signed i4 (value - 8).
@@ -470,22 +506,34 @@ void extract_q5_k_data(const ggml_tensor * tensor,
 
 // TODO Reorder for make_intX_weights
 
+// If for_gather_matmul is true, weight may be N-D (e.g. 3D MoE expert weights [n_expert, rows, cols]).
+// The dequantization chain below is built as usual but left in f16 (no final Convert to f32) --
+// ov::pass::MarkDequantization (registered in translate_session.cpp) marks the chain so it survives
+// model-build-time ConstantFolding. mul_mat_id.cpp constructs ov::op::internal::GatherMatmul directly
+// on top of the resulting f16 chain.
 ov::Output<ov::Node> make_int8_weights(ov::Tensor & weight,
                                        ov::Tensor & scales,
                                        ov::Tensor & zp,
                                        size_t group_size,
-                                       bool use_bias) {
+                                       bool use_bias,
+                                       bool for_gather_matmul) {
     ov::Shape orig_shape = weight.get_shape();
     bool is_signed = (weight.get_element_type() == ov::element::i8);  // Symmetric: signed weights, no ZP
 
     // Expand dimensions for scales and zp/bias
     auto scale_shape = scales.get_shape();
 
-    ov::Shape packed_shape = {orig_shape[0], orig_shape[1] / group_size, group_size};
+    // Group the innermost (last) dimension. For 2D weights [rows, cols] this yields
+    // [rows, cols/group_size, group_size]; for 3D MoE experts [n_expert, rows, cols] this yields
+    // [n_expert, rows, cols/group_size, group_size].
+    ov::Shape packed_shape = orig_shape;
+    packed_shape.back() /= group_size;
+    packed_shape.push_back(group_size);
+    const size_t group_dim = packed_shape.size() - 2;
 
-    if (packed_shape[1] == 1) {
+    if (packed_shape[group_dim] == 1) {
         // Requantized channel-wise case
-        packed_shape.erase(packed_shape.begin() + 1);
+        packed_shape.erase(packed_shape.begin() + group_dim);
     } else {
         scale_shape.push_back(1);
         scales.set_shape(scale_shape);
@@ -505,7 +553,8 @@ ov::Output<ov::Node> make_int8_weights(ov::Tensor & weight,
                                                                    static_cast<uint8_t *>(weight.data()), nullptr);
         weights_node->get_rt_info()["__gguf_tensor_holder"] = weight;
         auto weights_f16 = std::make_shared<ov::op::v0::Convert>(weights_node, ov::element::f16);
-        result = std::make_shared<ov::op::v1::Multiply>(weights_f16, scales_f16, ov::op::AutoBroadcastType::NUMPY);
+        auto mul = std::make_shared<ov::op::v1::Multiply>(weights_f16, scales_f16, ov::op::AutoBroadcastType::NUMPY);
+        result = mul;
     } else {
         // Unsigned path
         auto weights_node = std::make_shared<ov::op::v0::Constant>(ov::element::u8, packed_shape,
@@ -514,11 +563,25 @@ ov::Output<ov::Node> make_int8_weights(ov::Tensor & weight,
         auto weights_f16 = std::make_shared<ov::op::v0::Convert>(weights_node, ov::element::f16);
 
         if (use_bias && zp.get_size() > 0) {
-            // Bias path: w * s + b (zp tensor holds f16 bias values)
-            auto bias_f16 = std::make_shared<ov::op::v0::Constant>(zp);
-            auto w_s =
-                std::make_shared<ov::op::v1::Multiply>(weights_f16, scales_f16, ov::op::AutoBroadcastType::NUMPY);
-            result = std::make_shared<ov::op::v1::Add>(w_s, bias_f16, ov::op::AutoBroadcastType::NUMPY);
+            // Accurate dequant in the FUSABLE zero-point form: (w - zp) * s, where the zero
+            // point is an exact f16 value zp = -bias/scale (the zp tensor holds bias values
+            // coming in). Algebraically equal to w*s + bias, but unlike an Add(bias) graph this
+            // matches CompressedWeightsBlock's pattern (Constant->Convert->Subtract->Multiply),
+            // so for_gather_matmul weights still fuse into GatherMatmulCompressed. Also avoids
+            // the round(min/scale) error of an integer zero point. Convert bias -> zero-point IN
+            // PLACE in the (possibly buffer-backed) zp tensor to avoid a duplicate allocation.
+            auto * bias_zp_data = zp.data<ov::float16>();
+            const auto * scale_data = scales.data<ov::float16>();
+            const size_t n = zp.get_size();
+            for (size_t i = 0; i < n; i++) {
+                float s = static_cast<float>(scale_data[i]);
+                float b = static_cast<float>(bias_zp_data[i]);
+                bias_zp_data[i] = ov::float16(s != 0.0f ? -b / s : 0.0f);
+            }
+            auto zero_point_f16 = std::make_shared<ov::op::v0::Constant>(zp);
+            auto w_zp =
+                std::make_shared<ov::op::v1::Subtract>(weights_f16, zero_point_f16, ov::op::AutoBroadcastType::NUMPY);
+            result = std::make_shared<ov::op::v1::Multiply>(w_zp, scales_f16, ov::op::AutoBroadcastType::NUMPY);
         } else {
             // Zero point path: (w - zp) * s
             auto zero_point = std::make_shared<ov::op::v0::Constant>(zp);
@@ -529,37 +592,49 @@ ov::Output<ov::Node> make_int8_weights(ov::Tensor & weight,
             auto zero_point_f16 = std::make_shared<ov::op::v0::Convert>(zero_point, ov::element::f16);
             auto w_zp =
                 std::make_shared<ov::op::v1::Subtract>(weights_f16, zero_point_f16, ov::op::AutoBroadcastType::NUMPY);
-            result = std::make_shared<ov::op::v1::Multiply>(w_zp, scales_f16, ov::op::AutoBroadcastType::NUMPY);
+            auto mul = std::make_shared<ov::op::v1::Multiply>(w_zp, scales_f16, ov::op::AutoBroadcastType::NUMPY);
+            result = mul;
         }
     }
 
-    if (packed_shape.size() != 2) {
+    if (packed_shape.size() != orig_shape.size()) {
         // If not requantized channel-wise case, reshape back to original shape
         auto final_shape =
             std::make_shared<ov::op::v0::Constant>(ov::element::i64, ov::Shape{orig_shape.size()}, orig_shape);
-        result = std::make_shared<ov::op::v1::Reshape>(result, final_shape, false);
+        auto reshaped = std::make_shared<ov::op::v1::Reshape>(result, final_shape, false);
+        result = reshaped;
     }
 
+    if (for_gather_matmul) {
+        return result;
+    }
     return std::make_shared<ov::op::v0::Convert>(result, ov::element::f32);
 }
 
+// See make_int8_weights for the meaning of for_gather_matmul.
 ov::Output<ov::Node> make_int4_weights(ov::Tensor & weight,
                                        ov::Tensor & scales,
                                        ov::Tensor & zp,
                                        size_t group_size,
-                                       bool use_bias) {
+                                       bool use_bias,
+                                       bool for_gather_matmul) {
     ov::Shape orig_weight_shape = weight.get_shape();
     bool is_signed = (weight.get_element_type() == ov::element::i4);  // Symmetric: signed weights, no ZP
 
     // Expand dimensions for scales and zp/bias
     ov::Shape scale_shape = scales.get_shape();
 
-    // Create INT4 weight tensor
-    ov::Shape packed_shape = {orig_weight_shape[0], orig_weight_shape[1] / group_size, group_size};
+    // Create INT4 weight tensor. Group the innermost (last) dimension: for 2D weights
+    // [rows, cols] this yields [rows, cols/group_size, group_size]; for 3D MoE experts
+    // [n_expert, rows, cols] this yields [n_expert, rows, cols/group_size, group_size].
+    ov::Shape packed_shape = orig_weight_shape;
+    packed_shape.back() /= group_size;
+    packed_shape.push_back(group_size);
+    const size_t group_dim = packed_shape.size() - 2;
 
-    if (packed_shape[1] == 1) {
+    if (packed_shape[group_dim] == 1) {
         // Requantized channel-wise case
-        packed_shape.erase(packed_shape.begin() + 1);
+        packed_shape.erase(packed_shape.begin() + group_dim);
     } else {
         scale_shape.push_back(1);
         scales.set_shape(scale_shape);
@@ -579,7 +654,8 @@ ov::Output<ov::Node> make_int4_weights(ov::Tensor & weight,
                                                                    static_cast<uint8_t *>(weight.data()), nullptr);
         weights_node->get_rt_info()["__gguf_tensor_holder"] = weight;
         auto weights_f16 = std::make_shared<ov::op::v0::Convert>(weights_node, ov::element::f16);
-        result = std::make_shared<ov::op::v1::Multiply>(weights_f16, scales_f16, ov::op::AutoBroadcastType::NUMPY);
+        auto mul = std::make_shared<ov::op::v1::Multiply>(weights_f16, scales_f16, ov::op::AutoBroadcastType::NUMPY);
+        result = mul;
     } else {
         // Unsigned path
         auto weights_node = std::make_shared<ov::op::v0::Constant>(ov::element::u4, packed_shape,
@@ -588,11 +664,23 @@ ov::Output<ov::Node> make_int4_weights(ov::Tensor & weight,
         auto weights_f16 = std::make_shared<ov::op::v0::Convert>(weights_node, ov::element::f16);
 
         if (use_bias && zp.get_size() > 0) {
-            // Bias path: w * s + b (zp tensor holds f16 bias values)
-            auto bias_f16 = std::make_shared<ov::op::v0::Constant>(zp);
-            auto w_s =
-                std::make_shared<ov::op::v1::Multiply>(weights_f16, scales_f16, ov::op::AutoBroadcastType::NUMPY);
-            result = std::make_shared<ov::op::v1::Add>(w_s, bias_f16, ov::op::AutoBroadcastType::NUMPY);
+            // Accurate dequant in the FUSABLE zero-point form: (w - zp) * s with an exact f16
+            // zp = -bias/scale. Equivalent to w*s + bias but matches CompressedWeightsBlock's
+            // pattern so for_gather_matmul weights still fuse into GatherMatmulCompressed, and
+            // avoids the round(min/scale) error of an integer zp. Convert bias -> zero-point IN
+            // PLACE in the (possibly buffer-backed) zp tensor to avoid a duplicate allocation.
+            auto * bias_zp_data = zp.data<ov::float16>();
+            const auto * scale_data = scales.data<ov::float16>();
+            const size_t n = zp.get_size();
+            for (size_t i = 0; i < n; i++) {
+                float s = static_cast<float>(scale_data[i]);
+                float b = static_cast<float>(bias_zp_data[i]);
+                bias_zp_data[i] = ov::float16(s != 0.0f ? -b / s : 0.0f);
+            }
+            auto zero_points_f16 = std::make_shared<ov::op::v0::Constant>(zp);
+            auto w_zp =
+                std::make_shared<ov::op::v1::Subtract>(weights_f16, zero_points_f16, ov::op::AutoBroadcastType::NUMPY);
+            result = std::make_shared<ov::op::v1::Multiply>(w_zp, scales_f16, ov::op::AutoBroadcastType::NUMPY);
         } else {
             // Zero point path: (w - zp) * s
             auto zero_points_node = std::make_shared<ov::op::v0::Constant>(zp);
@@ -603,20 +691,61 @@ ov::Output<ov::Node> make_int4_weights(ov::Tensor & weight,
             auto zero_points_f16 = std::make_shared<ov::op::v0::Convert>(zero_points_node, ov::element::f16);
             auto w_zp =
                 std::make_shared<ov::op::v1::Subtract>(weights_f16, zero_points_f16, ov::op::AutoBroadcastType::NUMPY);
-            result = std::make_shared<ov::op::v1::Multiply>(w_zp, scales_f16, ov::op::AutoBroadcastType::NUMPY);
+            auto mul = std::make_shared<ov::op::v1::Multiply>(w_zp, scales_f16, ov::op::AutoBroadcastType::NUMPY);
+            result = mul;
         }
     }
 
-    if (packed_shape.size() != 2) {
+    if (packed_shape.size() != orig_weight_shape.size()) {
         // If not requantized channel-wise case, reshape back to original shape
         auto final_shape = std::make_shared<ov::op::v0::Constant>(ov::element::i64, ov::Shape{orig_weight_shape.size()},
                                                                   orig_weight_shape);
-        result = std::make_shared<ov::op::v1::Reshape>(result, final_shape, false);
+        auto reshaped = std::make_shared<ov::op::v1::Reshape>(result, final_shape, false);
+        result = reshaped;
     }
 
+    if (for_gather_matmul) {
+        return result;
+    }
     return std::make_shared<ov::op::v0::Convert>(result, ov::element::f32);
 }
 
+ov::Output<ov::Node> make_mxfp4_weights(ov::Tensor & weight, ov::Tensor & scales) {
+    const ov::Shape final_shape = weight.get_shape();
+    GGML_ASSERT(!final_shape.empty());
+    GGML_ASSERT(final_shape.back() % MXFP4_BLOCK_SIZE == 0);
+
+    ov::Shape packed_shape = final_shape;
+    packed_shape.back() /= MXFP4_BLOCK_SIZE;
+    packed_shape.push_back(MXFP4_BLOCK_SIZE);
+
+    ov::Shape scale_shape = packed_shape;
+    scale_shape.back() = 1;
+    scales.set_shape(scale_shape);
+
+    auto weights_node = std::make_shared<ov::op::v0::Constant>(ov::element::f4e2m1, packed_shape,
+                                                               static_cast<uint8_t *>(weight.data()), nullptr);
+    weights_node->get_rt_info()["__gguf_tensor_holder"] = weight;
+    auto weights_f32 = std::make_shared<ov::op::v0::Convert>(weights_node, ov::element::f32);
+
+    auto scales_node = std::make_shared<ov::op::v0::Constant>(scales);
+    auto scales_f32 = std::make_shared<ov::op::v0::Convert>(scales_node, ov::element::f32);
+    ov::Output<ov::Node> result =
+        std::make_shared<ov::op::v1::Multiply>(weights_f32, scales_f32, ov::op::AutoBroadcastType::NUMPY);
+
+    auto final_shape_node =
+        std::make_shared<ov::op::v0::Constant>(ov::element::i64, ov::Shape{final_shape.size()}, final_shape);
+    return std::make_shared<ov::op::v1::Reshape>(result, final_shape_node, false);
+}
+
+ov::Output<ov::Node> make_mxfp4_moe_packed_weights(ov::Tensor & weight) {
+    auto weights_node = std::make_shared<ov::op::v0::Constant>(ov::element::u8, weight.get_shape(),
+                                                               static_cast<uint8_t *>(weight.data()), nullptr);
+    weights_node->get_rt_info()["__gguf_tensor_holder"] = weight;
+    weights_node->get_rt_info()["__ggml_openvino_mxfp4_moe_packed"] = true;
+    return weights_node;
+}
+
 // Extract quantized weights from tensor and create weight subgraph
 std::shared_ptr<ov::Node> extract_quantized_weights(const ggml_tensor * tensor,
                                                     const void * data,
@@ -628,6 +757,13 @@ std::shared_ptr<ov::Node> extract_quantized_weights(const ggml_tensor * tensor,
     ggml_tensor temp_tensor = *tensor;
     temp_tensor.data = const_cast<void *>(data);
 
+    if (tensor->type == GGML_TYPE_MXFP4) {
+        extract_mxfp4_data(&temp_tensor, weights, scales);
+        auto result = make_mxfp4_weights(weights, scales).get_node_shared_ptr();
+        result->set_friendly_name(tensor->name);
+        return result;
+    }
+
     // Determine block size based on tensor type
     int64_t weights_per_block;
     bool is_u4;
@@ -653,6 +789,13 @@ std::shared_ptr<ov::Node> extract_quantized_weights(const ggml_tensor * tensor,
                                  std::string(ggml_type_name(tensor->type)));
     }
 
+    // 3D MoE expert weights (for_gather_matmul) always use the exact f16 zero-point extraction
+    // (see make_int8_weights/make_int4_weights) rather than the rounded integer zero point --
+    // round(min/scale) error is what corrupts Q4_K/Q5_1 experts, and the f16-zp form still fuses
+    // into GatherMatmulCompressed since it stays a Subtract, not an Add.
+    const bool for_gather_matmul = tensor->ne[2] > 1;
+    use_bias = use_bias || for_gather_matmul;
+
     // Extract quantized data
     switch (tensor->type) {
     case GGML_TYPE_Q4_0:
@@ -680,12 +823,13 @@ std::shared_ptr<ov::Node> extract_quantized_weights(const ggml_tensor * tensor,
         throw std::runtime_error("Unsupported quantized type: " + std::string(ggml_type_name(tensor->type)));
     }
 
-    // Create the OpenVINO weight subgraph
+    // Create the OpenVINO weight subgraph. 3D expert weights (MoE) are routed through the
+    // GatherMatmul-oriented path: dequantized in f16, with constant folding disabled on the chain.
     ov::Output<ov::Node> weight_node;
     if (is_u4) {
-        weight_node = make_int4_weights(weights, scales, zp, weights_per_block, use_bias);
+        weight_node = make_int4_weights(weights, scales, zp, weights_per_block, use_bias, for_gather_matmul);
     } else {
-        weight_node = make_int8_weights(weights, scales, zp, weights_per_block, use_bias);
+        weight_node = make_int8_weights(weights, scales, zp, weights_per_block, use_bias, for_gather_matmul);
     }
 
     auto result = weight_node.get_node_shared_ptr();
@@ -702,28 +846,76 @@ std::shared_ptr<ov::Node> requantize_to_buffers(const ggml_tensor * tensor,
                                                 ov::Tensor & scales,
                                                 ov::Tensor & zp) {
     int64_t n_elements = ggml_nelements(tensor);
+    const int64_t ne0 = tensor->ne[0];                 // elements per row
+    const int64_t n_rows = n_elements / ne0;
+    const auto * type_traits = ggml_get_type_traits(tensor->type);
+    const size_t src_row_bytes = ggml_row_size(tensor->type, ne0);
 
-    // First dequantize to F32
-    std::vector<float> weights_f32(n_elements);
-    ggml_get_type_traits(tensor->type)->to_float(data, weights_f32.data(), n_elements);
-
-    // Handle F16 case - just convert and create constant
-    if (requant_type == ExtraQuantType::F16) {
-        ggml_get_type_traits(GGML_TYPE_F16)->from_float_ref(weights_f32.data(), weights.data(), n_elements);
-        auto result = std::make_shared<ov::op::v0::Constant>(weights);
-        result->set_friendly_name(tensor->name);
-        return result;
-    }
-
-    // Requantize to target quantized format
     bool is_u4 = (requant_type == ExtraQuantType::Q4_0_C || requant_type == ExtraQuantType::Q4_0_128);
 
-    if (is_u4) {
-        quantize_q4_0(weights_f32.data(), weights, scales, zp, n_elements, block_size);
-    } else if (requant_type == ExtraQuantType::Q8_1_C) {
-        quantize_q8_1(weights_f32.data(), weights, scales, zp, n_elements, block_size);
+    // Streaming dequant (opt-in via GGML_OPENVINO_REDUCE_COMPILE_MEM or
+    // GGML_OPENVINO_MEMORY_OPTIMIZE): instead of
+    // materializing the full n_elements F32 array (e.g. ~1 GB for token_embd), dequantize
+    // a chunk of complete rows into a small scratch and quantize/convert it straight into
+    // the output buffers, capping the transient F32 footprint at CHUNK_ROWS*ne0 floats.
+    //
+    // Only valid (and only used) for the Q8_0_C / Q8_1_C / F16 targets whose block size
+    // divides a row (channel-wise _C uses block_size == ne0) so no target block straddles
+    // a row boundary, and Q8/F16 have no cross-block packing. The u4 (Q4_0) path packs two
+    // weights per byte with running zp ORs that assume a single whole-array call, so it is
+    // never streamed. When the flag is off, behavior is identical to the original
+    // full-materialization path.
+    const bool stream_requant = ggml_openvino_reduce_compile_mem_enabled() && !is_u4 &&
+                                !(block_size > 0 && ne0 % block_size != 0);
+
+    if (!stream_requant) {
+        // Full materialization (original behavior): dequantize the whole tensor to F32,
+        // then convert/quantize in one call.
+        std::vector<float> weights_f32(n_elements);
+        type_traits->to_float(data, weights_f32.data(), n_elements);
+        if (requant_type == ExtraQuantType::F16) {
+            ggml_get_type_traits(GGML_TYPE_F16)->from_float_ref(weights_f32.data(), weights.data(), n_elements);
+            auto result = std::make_shared<ov::op::v0::Constant>(weights);
+            result->set_friendly_name(tensor->name);
+            return result;
+        }
+        if (is_u4) {
+            quantize_q4_0(weights_f32.data(), weights, scales, zp, n_elements, block_size);
+        } else if (requant_type == ExtraQuantType::Q8_1_C) {
+            quantize_q8_1(weights_f32.data(), weights, scales, zp, n_elements, block_size);
+        } else {
+            quantize_q8_0(weights_f32.data(), weights, scales, zp, n_elements, block_size);
+        }
     } else {
-        quantize_q8_0(weights_f32.data(), weights, scales, zp, n_elements, block_size);
+        // Streaming path for Q8_0_C / Q8_1_C / F16 (covers token_embd, output.weight,
+        // and per-layer Q6_K/Q5_K requant — the large transient cases).
+        const int64_t CHUNK_ROWS = std::min<int64_t>(n_rows, 256);
+        std::vector<float> scratch(CHUNK_ROWS * ne0);
+        // F16 destination: 2 bytes/element, advanced per chunk by r0*ne0 elements.
+        auto * f16_base = static_cast<uint8_t *>(weights.data());
+        for (int64_t r0 = 0; r0 < n_rows; r0 += CHUNK_ROWS) {
+            const int64_t rows = std::min(CHUNK_ROWS, n_rows - r0);
+            const int64_t elems = rows * ne0;
+            const auto * src = static_cast<const uint8_t *>(data) + r0 * src_row_bytes;
+            type_traits->to_float(src, scratch.data(), elems);
+
+            if (requant_type == ExtraQuantType::F16) {
+                ggml_get_type_traits(GGML_TYPE_F16)
+                    ->from_float_ref(scratch.data(), f16_base + (r0 * ne0) * sizeof(uint16_t), elems);
+            } else {
+                const int64_t block_offset = (r0 * ne0) / block_size;
+                if (requant_type == ExtraQuantType::Q8_1_C) {
+                    quantize_q8_1(scratch.data(), weights, scales, zp, elems, block_size, block_offset);
+                } else {
+                    quantize_q8_0(scratch.data(), weights, scales, zp, elems, block_size, block_offset);
+                }
+            }
+        }
+        if (requant_type == ExtraQuantType::F16) {
+            auto result = std::make_shared<ov::op::v0::Constant>(weights);
+            result->set_friendly_name(tensor->name);
+            return result;
+        }
     }
 
     // Create the OpenVINO weight subgraph
@@ -745,8 +937,11 @@ OvWeight process_weight_tensor(const ggml_tensor * tensor, const void * data, vo
 
     OvWeight result;
 
-    // Get 2D shape for weights [rows, cols]
-    ov::Shape node_shape = {static_cast<size_t>(tensor->ne[1]), static_cast<size_t>(tensor->ne[0])};
+    // Get shape for weights: [rows, cols], or [n_expert, rows, cols] for 3D MoE expert weights.
+    ov::Shape node_shape = (tensor->ne[2] > 1) ?
+                               ov::Shape{static_cast<size_t>(tensor->ne[2]), static_cast<size_t>(tensor->ne[1]),
+                                         static_cast<size_t>(tensor->ne[0])} :
+                               ov::Shape{static_cast<size_t>(tensor->ne[1]), static_cast<size_t>(tensor->ne[0])};
 
     // Handle F16/F32/BF16 weights
     if (tensor->type == GGML_TYPE_F32 || tensor->type == GGML_TYPE_F16 || tensor->type == GGML_TYPE_BF16) {
@@ -788,6 +983,35 @@ OvWeight process_weight_tensor(const ggml_tensor * tensor, const void * data, vo
         OPENVINO_THROW("Unsupported quantized type: ", ggml_type_name(tensor->type));
     }
 
+    // 3D MoE expert weights (for_gather_matmul) always use the exact f16 zero-point path (see
+    // extract_quantized_weights) -- must be kept in sync with the "use_bias || for_gather_matmul"
+    // check in ggml_openvino_get_extracted_layout, which sizes/offsets the zp slot accordingly.
+    // Requantized tensors (layout.is_requant) are handled by requantize_to_buffers instead, whose
+    // zp sizing/type is unaffected by for_gather_matmul, so they are excluded here.
+    const bool for_gather_matmul = tensor->ne[2] > 1;
+    const bool zp_is_f16 = !layout.is_requant && (use_bias || for_gather_matmul);
+
+    const bool is_3d_mxfp4_moe = tensor->type == GGML_TYPE_MXFP4 && (tensor->ne[2] > 1 || tensor->ne[3] > 1);
+    if (is_3d_mxfp4_moe) {
+        ov::Shape packed_shape = {static_cast<size_t>(tensor->ne[3]),
+                                  static_cast<size_t>(tensor->ne[2]),
+                                  static_cast<size_t>(tensor->ne[1]),
+                                  static_cast<size_t>(tensor->ne[0] / MXFP4_BLOCK_SIZE),
+                                  MXFP4_BLOCK_BYTES};
+        const size_t tensor_bytes = ggml_nbytes(tensor);
+        if (output_base_ptr) {
+            auto * buf_base = static_cast<uint8_t *>(output_base_ptr);
+            memcpy(buf_base + layout.weights_offset, data, tensor_bytes);
+            result.weights = ov::Tensor(ov::element::u8, packed_shape, buf_base + layout.weights_offset);
+        } else {
+            result.weights = ov::Tensor(ov::element::u8, packed_shape);
+            memcpy(result.weights.data(), data, tensor_bytes);
+        }
+        result.weight_node = make_mxfp4_moe_packed_weights(result.weights).get_node_shared_ptr();
+        result.weight_node->set_friendly_name(tensor->name);
+        return result;
+    }
+
     if (use_bias) {
         OPENVINO_ASSERT(!layout.is_requant,
                         "use_bias is only used for test-backend-ops, which should not have requantization");
@@ -812,24 +1036,44 @@ OvWeight process_weight_tensor(const ggml_tensor * tensor, const void * data, vo
     // Quantized path (normal extraction or quantized requant)
     // Create weight/scale/zp tensors - shared between both paths
     // For symmetric quantization, use signed types (i4/i8) and no ZP tensor
-    ov::element::Type weight_type = layout.is_symmetric ? (layout.is_u4 ? ov::element::i4 : ov::element::i8) :
-                                                          (layout.is_u4 ? ov::element::u4 : ov::element::u8);
-    ov::Shape scale_shape = {node_shape[0], node_shape[1] / layout.weights_per_block};
+    ov::element::Type weight_type = tensor->type == GGML_TYPE_MXFP4 ?
+                                        ov::element::f4e2m1 :
+                                        (layout.is_symmetric ? (layout.is_u4 ? ov::element::i4 : ov::element::i8) :
+                                                               (layout.is_u4 ? ov::element::u4 : ov::element::u8));
+    ov::Shape scale_shape = node_shape;
+    scale_shape.back() /= layout.weights_per_block;
+
+    if (tensor->type == GGML_TYPE_MXFP4) {
+        if (tensor->ne[2] == 1 && tensor->ne[3] == 1) {
+            node_shape = {static_cast<size_t>(tensor->ne[1]), static_cast<size_t>(tensor->ne[0])};
+        } else {
+            node_shape.clear();
+            for (int i = GGML_MAX_DIMS - 1; i >= 0; --i) {
+                node_shape.push_back(static_cast<size_t>(tensor->ne[i]));
+            }
+        }
+
+        scale_shape = node_shape;
+        scale_shape.back() /= layout.weights_per_block;
+    }
 
     if (output_base_ptr) {
         uint8_t * buf_base = static_cast<uint8_t *>(output_base_ptr);
         result.weights = ov::Tensor(weight_type, node_shape, buf_base + layout.weights_offset);
-        result.scales = ov::Tensor(ov::element::f16, scale_shape, buf_base + layout.scales_offset);
+        const ov::element::Type scale_type = tensor->type == GGML_TYPE_MXFP4 ? ov::element::f8e8m0 : ov::element::f16;
+        result.scales = ov::Tensor(scale_type, scale_shape, buf_base + layout.scales_offset);
         if (!layout.is_symmetric) {
-            ov::element::Type zp_type = layout.is_u4 ? ov::element::u4 : ov::element::u8;
+            ov::element::Type zp_type =
+                zp_is_f16 ? ov::element::f16 : (layout.is_u4 ? ov::element::u4 : ov::element::u8);
             result.zp = ov::Tensor(zp_type, scale_shape, buf_base + layout.zp_offset);
         }
         // else: result.zp remains default-constructed (empty) for symmetric
     } else {
         result.weights = ov::Tensor(weight_type, node_shape);
-        result.scales = ov::Tensor(ov::element::f16, scale_shape);
+        const ov::element::Type scale_type = tensor->type == GGML_TYPE_MXFP4 ? ov::element::f8e8m0 : ov::element::f16;
+        result.scales = ov::Tensor(scale_type, scale_shape);
         if (!layout.is_symmetric) {
-            if (use_bias) {
+            if (zp_is_f16) {
                 result.zp = ov::Tensor(ov::element::f16, scale_shape);
             } else {
                 ov::element::Type zp_type = layout.is_u4 ? ov::element::u4 : ov::element::u8;
@@ -939,16 +1183,21 @@ void quantize_q8_0(const float * x,
                    ov::Tensor & scales_arr,
                    ov::Tensor & zp_arr,
                    int64_t k,
-                   int64_t qk) {
+                   int64_t qk,
+                   int64_t block_offset) {
     assert(k % qk == 0);
     const int nb = k / qk;
 
-    auto * weights = static_cast<uint8_t *>(weights_arr.data());
-    auto * scales = scales_arr.data<ov::element_type_traits<ov::element::f16>::value_type>();
+    // block_offset lets a caller quantize a chunk of blocks into the right place in the
+    // output buffers (used for streaming requant). x points at this chunk's first block;
+    // outputs are advanced by block_offset blocks. Q8 has one scale/zp per block (no
+    // nibble packing), so any block boundary is safe.
+    auto * weights = static_cast<uint8_t *>(weights_arr.data()) + block_offset * qk;
+    auto * scales = scales_arr.data<ov::element_type_traits<ov::element::f16>::value_type>() + block_offset;
     bool is_symmetric = (weights_arr.get_element_type() == ov::element::i8);  // Signed i8 path
 
     if (!is_symmetric) {
-        auto * zp = static_cast<uint8_t *>(zp_arr.data());
+        auto * zp = static_cast<uint8_t *>(zp_arr.data()) + block_offset;
         for (int i = 0; i < nb; i++) {
             float amax = 0.0f;
             for (int j = 0; j < qk; j++) {
@@ -990,13 +1239,15 @@ void quantize_q8_1(const float * x,
                    ov::Tensor & scales_arr,
                    ov::Tensor & zp_arr,
                    int64_t k,
-                   int64_t qk) {
+                   int64_t qk,
+                   int64_t block_offset) {
     assert(k % qk == 0);
     const int nb = k / qk;
 
-    auto * weights = static_cast<uint8_t *>(weights_arr.data());
-    auto * scales = scales_arr.data<ov::element_type_traits<ov::element::f16>::value_type>();
-    auto * zp = static_cast<uint8_t *>(zp_arr.data());
+    // See quantize_q8_0: block_offset places this chunk's output at the right block.
+    auto * weights = static_cast<uint8_t *>(weights_arr.data()) + block_offset * qk;
+    auto * scales = scales_arr.data<ov::element_type_traits<ov::element::f16>::value_type>() + block_offset;
+    auto * zp = static_cast<uint8_t *>(zp_arr.data()) + block_offset;
     for (int i = 0; i < nb; i++) {
         float min = std::numeric_limits<float>::max();
         float max = std::numeric_limits<float>::lowest();

ggml/src/ggml-openvino/ggml-quants.h

@@ -4,6 +4,7 @@
 
 #include <cstdint>
 #include <openvino/op/constant.hpp>
+#include <openvino/core/node_output.hpp>
 #include <openvino/runtime/tensor.hpp>
 
 void unpack_32_4(const uint8_t * data, uint8_t * dst);
@@ -49,19 +50,38 @@ void extract_q6_k_data(const ggml_tensor * tensor,
                        ov::Tensor & scales_arr,
                        ov::Tensor & zp_arr);
 
+void extract_mxfp4_data(const ggml_tensor * tensor, ov::Tensor & weights_arr, ov::Tensor & scales_arr);
+
 static constexpr size_t GGML_QUANTIZATION_GROUP_SIZE = 32;
 
+// If for_gather_matmul is true, the weight tensor may be N-D (e.g. 3D MoE expert weights
+// [n_expert, rows, cols]). The dequantization chain (Convert->[Subtract]->Multiply) is built as
+// usual but left in f16 (no final Convert to f32) -- ov::pass::MarkDequantization (registered in
+// translate_session.cpp) marks the chain so it survives model-build-time ConstantFolding -- see
+// make_int8_weights.cpp/make_int4_weights.cpp. mul_mat_id.cpp constructs ov::op::internal::GatherMatmul
+// directly from the resulting f16 dequant chain.
+//
+// When use_bias is true (explicitly, or implicitly because for_gather_matmul is true), the zp
+// tensor is expected to hold an exact f16 bias value (rather than a rounded integer zero point);
+// it is converted in place into an exact zero_point = -bias/scale and consumed via Subtract, not
+// Add, so the chain still matches OpenVINO's Convert->Subtract->Multiply decompression pattern.
 ov::Output<ov::Node> make_int8_weights(ov::Tensor & weight,
                                        ov::Tensor & scales,
                                        ov::Tensor & zp,
                                        size_t group_size = GGML_QUANTIZATION_GROUP_SIZE,
-                                       bool use_bias = false);
+                                       bool use_bias = false,
+                                       bool for_gather_matmul = false);
 
 ov::Output<ov::Node> make_int4_weights(ov::Tensor & weight,
                                        ov::Tensor & scales,
                                        ov::Tensor & zp,
                                        size_t group_size = GGML_QUANTIZATION_GROUP_SIZE,
-                                       bool use_bias = false);
+                                       bool use_bias = false,
+                                       bool for_gather_matmul = false);
+
+ov::Output<ov::Node> make_mxfp4_weights(ov::Tensor & weight, ov::Tensor & scales);
+
+ov::Output<ov::Node> make_mxfp4_moe_packed_weights(ov::Tensor & weight);
 
 // Extract quantized weights from tensor and create weight subgraph
 // If weights/scales/zp are provided (non-empty), uses them as output buffers
@@ -73,7 +93,9 @@ std::shared_ptr<ov::Node> extract_quantized_weights(
     ov::Tensor & weights,
     ov::Tensor & scales,
     ov::Tensor & zp,
-    bool use_bias = false);  // Use fp bias instead of quantized zero_point (for test-backend-ops)
+    bool use_bias = false);  // Use an exact f16 zero point (vs. a rounded integer one); always
+                             // used for for_gather_matmul (3D MoE expert) weights regardless of
+                             // this flag, and also settable explicitly for test-backend-ops.
 
 // Requantize weights from tensor to target format, writing to provided buffers
 // For F16 target, only weights buffer is used (scales/zp ignored)
@@ -126,7 +148,10 @@ OvWeight process_weight_tensor(
     const ggml_tensor * tensor,
     const void * data,                 // Source data pointer (may differ from tensor->data)
     void * output_base_ptr = nullptr,  // Base pointer for output buffers (or nullptr for internal allocation)
-    bool use_bias = false);            // Use fp bias instead of quantized zero_point, only used in test-backend-ops
+    bool use_bias = false);            // Use an exact f16 zero point (vs. a rounded integer one);
+                                       // always used for for_gather_matmul (3D MoE expert) weights
+                                       // regardless of this flag, and also settable explicitly for
+                                       // test-backend-ops.
 
 void quantize_q4_0(const float * x,
                    ov::Tensor & weights_arr,
@@ -139,13 +164,15 @@ void quantize_q8_1(const float * x,
                    ov::Tensor & scales_arr,
                    ov::Tensor & zp_arr,
                    int64_t k,
-                   int64_t qk);
+                   int64_t qk,
+                   int64_t block_offset = 0);
 void quantize_q8_0(const float * x,
                    ov::Tensor & weights_arr,
                    ov::Tensor & scales_arr,
                    ov::Tensor & zp_arr,
                    int64_t k,
-                   int64_t qk);
+                   int64_t qk,
+                   int64_t block_offset = 0);
 
 namespace ov {
 namespace op {

ggml/src/ggml-openvino/model-cache.cpp

@@ -0,0 +1,272 @@
+#include "model-cache.h"
+
+#include "ggml-backend-impl.h"
+#include "ggml-backend.h"
+#include "ggml-impl.h"
+#include "ggml-openvino-extra.h"
+
+#include <cerrno>
+#include <cstdio>
+#include <cstring>
+#include <fstream>
+#include <openvino/core/version.hpp>
+#include <string>
+#include <sys/stat.h>
+#include <sys/types.h>
+#include <vector>
+
+#if defined(_WIN32)
+#    include <direct.h>
+#endif
+
+namespace {
+
+// 64-bit FNV-1a, the mixing primitive for all fingerprints here.
+inline uint64_t fnv1a(uint64_t h, const void * data, size_t n) {
+    const uint8_t * p = static_cast<const uint8_t *>(data);
+    for (size_t i = 0; i < n; ++i) {
+        h ^= p[i];
+        h *= 0x100000001b3ull;
+    }
+    return h;
+}
+
+inline uint64_t fnv1a_u64(uint64_t h, uint64_t v) {
+    return fnv1a(h, &v, sizeof(v));
+}
+
+constexpr uint64_t FNV_OFFSET = 0xcbf29ce484222325ull;
+
+// Bytes sampled from each end of a weight tensor for the sampled hash. The whole
+// model is never hashed (that would cost seconds every run); instead we sample a
+// bounded window from the head and tail of each weight's bytes. The manifest
+// re-verify (same sample) guards the residual collision risk.
+constexpr size_t WEIGHT_SAMPLE_BYTES = 4096;
+
+// Is this src a model weight, mirroring create_weight_nodes()'s selection:
+// non-view tensor whose buffer is USAGE_WEIGHTS or whose type is quantized.
+bool is_weight_src(const ggml_tensor * src) {
+    if (src == nullptr || src->view_src != nullptr || src->buffer == nullptr) {
+        return false;
+    }
+    return src->buffer->usage == GGML_BACKEND_BUFFER_USAGE_WEIGHTS || ggml_is_quantized(src->type);
+}
+
+// Per-weight sampled fingerprint: identity (name/shape/type) + a bounded byte
+// sample. Returns FNV offset basis if data is unavailable (kept deterministic).
+uint64_t weight_fingerprint(const ggml_tensor * t) {
+    uint64_t h = FNV_OFFSET;
+    h = fnv1a(h, t->name, strlen(t->name));
+    for (int i = 0; i < GGML_MAX_DIMS; ++i) {
+        h = fnv1a_u64(h, static_cast<uint64_t>(t->ne[i]));
+    }
+    h = fnv1a_u64(h, static_cast<uint64_t>(t->type));
+    const size_t nbytes = ggml_nbytes(t);
+    h = fnv1a_u64(h, nbytes);
+    if (t->data != nullptr && nbytes > 0) {
+        const size_t head = nbytes < WEIGHT_SAMPLE_BYTES ? nbytes : WEIGHT_SAMPLE_BYTES;
+        h = fnv1a(h, t->data, head);
+        if (nbytes > WEIGHT_SAMPLE_BYTES) {
+            const size_t tail = nbytes < 2 * WEIGHT_SAMPLE_BYTES ? nbytes - WEIGHT_SAMPLE_BYTES : WEIGHT_SAMPLE_BYTES;
+            h = fnv1a(h, static_cast<const uint8_t *>(t->data) + (nbytes - tail), tail);
+        }
+    }
+    return h;
+}
+
+// Walk the cgraph and invoke fn(weight_tensor) for each distinct weight, in node
+// order. De-duplicates by tensor pointer so a weight used by several nodes is
+// fingerprinted once, deterministically.
+template <typename F>
+void for_each_weight(const ggml_cgraph * cgraph, F && fn) {
+    std::vector<const ggml_tensor *> seen;
+    for (int i = 0; i < cgraph->n_nodes; ++i) {
+        const ggml_tensor * node = cgraph->nodes[i];
+        for (int s = 0; s < GGML_MAX_SRC; ++s) {
+            const ggml_tensor * src = node->src[s];
+            if (!is_weight_src(src)) {
+                continue;
+            }
+            bool dup = false;
+            for (const auto * p : seen) {
+                if (p == src) {
+                    dup = true;
+                    break;
+                }
+            }
+            if (dup) {
+                continue;
+            }
+            seen.push_back(src);
+            fn(src);
+        }
+    }
+}
+
+std::string ov_version_string() {
+    const ov::Version v = ov::get_openvino_version();
+    return std::string(v.buildNumber ? v.buildNumber : "unknown");
+}
+
+std::string hex64(uint64_t v) {
+    char buf[17];
+    snprintf(buf, sizeof(buf), "%016llx", static_cast<unsigned long long>(v));
+    return std::string(buf);
+}
+
+// Portable mkdir for a single path component. Returns true if the directory
+// exists after the call (created now or already present).
+bool make_dir(const std::string & path) {
+#if defined(_WIN32)
+    int rc = _mkdir(path.c_str());
+#else
+    int rc = ::mkdir(path.c_str(), 0755);
+#endif
+    if (rc == 0 || errno == EEXIST) {
+        return true;
+    }
+    return false;
+}
+
+// Create `path` and any missing parents (like `mkdir -p`). Best-effort:
+// returns true only if the full directory exists afterwards.
+bool make_dirs(const std::string & path) {
+    if (path.empty()) {
+        return false;
+    }
+    std::string acc;
+    for (size_t i = 0; i < path.size(); ++i) {
+        const char c = path[i];
+        acc.push_back(c);
+        const bool sep = (c == '/'
+#if defined(_WIN32)
+                          || c == '\\'
+#endif
+        );
+        // Create each intermediate component (skip a leading "/" root).
+        if (sep && acc.size() > 1) {
+            std::string component = acc.substr(0, acc.size() - 1);
+            if (!make_dir(component)) {
+                return false;
+            }
+        }
+    }
+    return make_dir(path);
+}
+
+}  // namespace
+
+std::string ggml_openvino_model_cache_dir() {
+    const char * dir = ggml_openvino_getenv_str("GGML_OPENVINO_COMPILED_MODEL_CACHE_DIR");
+    if (!dir || strlen(dir) == 0) {
+        return std::string();
+    }
+    std::string path(dir);
+    // Create the cache directory (and parents) on first use so callers don't
+    // have to pre-create it; a missing dir would otherwise silently disable the
+    // cache (manifest/blob writes fail with no directory to write into).
+    if (!make_dirs(path)) {
+        GGML_LOG_WARN("ggml-openvino: could not create model cache dir '%s' (errno=%d); caching disabled\n",
+                      path.c_str(), errno);
+        return std::string();
+    }
+    return path;
+}
+
+uint64_t ggml_openvino_model_fingerprint(const ggml_cgraph * cgraph,
+                                         const std::string & device,
+                                         bool fa,
+                                         const int32_t * rope_params,
+                                         int rope_len,
+                                         uint64_t extra_cfg) {
+    uint64_t h = FNV_OFFSET;
+
+    // Topology: node count + each node's op and name (cheap, and distinguishes
+    // graphs that share weights but differ structurally).
+    h = fnv1a_u64(h, static_cast<uint64_t>(cgraph->n_nodes));
+    for (int i = 0; i < cgraph->n_nodes; ++i) {
+        const ggml_tensor * node = cgraph->nodes[i];
+        h = fnv1a_u64(h, static_cast<uint64_t>(node->op));
+        h = fnv1a(h, node->name, strlen(node->name));
+    }
+
+    // Weights: the model identity.
+    for_each_weight(cgraph, [&](const ggml_tensor * t) { h = fnv1a_u64(h, weight_fingerprint(t)); });
+
+    // Config that changes the produced blob.
+    h = fnv1a(h, device.data(), device.size());
+    h = fnv1a_u64(h, fa ? 1u : 0u);
+    if (rope_params && rope_len > 0) {
+        h = fnv1a(h, rope_params, sizeof(int32_t) * static_cast<size_t>(rope_len));
+    }
+    h = fnv1a_u64(h, extra_cfg);
+    const std::string ver = ov_version_string();
+    h = fnv1a(h, ver.data(), ver.size());
+
+    return h;
+}
+
+std::string ggml_openvino_model_cache_blob_path(const std::string & dir, uint64_t fingerprint) {
+    return dir + "/" + hex64(fingerprint) + ".blob";
+}
+
+std::string ggml_openvino_model_cache_manifest_path(const std::string & dir, uint64_t fingerprint) {
+    return dir + "/" + hex64(fingerprint) + ".manifest";
+}
+
+bool ggml_openvino_model_cache_write_manifest(const std::string & path,
+                                              const ggml_cgraph * cgraph,
+                                              uint64_t fingerprint) {
+    std::ofstream f(path, std::ios::trunc);
+    if (!f.is_open()) {
+        return false;
+    }
+    f << "fingerprint " << hex64(fingerprint) << "\n";
+    f << "ov_version " << ov_version_string() << "\n";
+    for_each_weight(cgraph, [&](const ggml_tensor * t) {
+        f << t->name << " " << t->ne[0] << " " << t->ne[1] << " " << t->ne[2] << " " << t->ne[3] << " "
+          << static_cast<int>(t->type) << " " << hex64(weight_fingerprint(t)) << "\n";
+    });
+    return f.good();
+}
+
+bool ggml_openvino_model_cache_verify_manifest(const std::string & path,
+                                               const ggml_cgraph * cgraph,
+                                               uint64_t fingerprint) {
+    std::ifstream f(path);
+    if (!f.is_open()) {
+        return false;
+    }
+    std::string tag, val;
+    // header: fingerprint
+    if (!(f >> tag >> val) || tag != "fingerprint" || val != hex64(fingerprint)) {
+        return false;
+    }
+    // header: ov_version
+    if (!(f >> tag >> val) || tag != "ov_version" || val != ov_version_string()) {
+        return false;
+    }
+
+    // Build the expected per-weight lines from the live cgraph, then require an
+    // exact match (same set, same order) against the manifest.
+    std::vector<std::string> expected;
+    for_each_weight(cgraph, [&](const ggml_tensor * t) {
+        expected.push_back(std::string(t->name) + " " + std::to_string(t->ne[0]) + " " + std::to_string(t->ne[1]) +
+                           " " + std::to_string(t->ne[2]) + " " + std::to_string(t->ne[3]) + " " +
+                           std::to_string(static_cast<int>(t->type)) + " " + hex64(weight_fingerprint(t)));
+    });
+
+    size_t idx = 0;
+    std::string line;
+    std::getline(f, line);  // consume rest of ov_version line
+    while (std::getline(f, line)) {
+        if (line.empty()) {
+            continue;
+        }
+        if (idx >= expected.size() || line != expected[idx]) {
+            return false;
+        }
+        ++idx;
+    }
+    return idx == expected.size();
+}

ggml/src/ggml-openvino/model-cache.h

@@ -0,0 +1,56 @@
+#pragma once
+
+// Frontend-level compiled-model cache (GGML_OPENVINO_COMPILED_MODEL_CACHE_DIR).
+//
+// The OpenVINO plugin's own ov::cache_dir caches the compiled blob keyed by the
+// *OV model*, but producing that model still runs the full frontend every time:
+// weight requantization (incl. the large token_embd F32 transient) and the
+// ggml->OV graph conversion. This cache keys off a fingerprint computed directly
+// from the ggml cgraph, so a hit skips requant + convert + compile entirely and
+// instead imports a previously exported CompiledModel blob.
+//
+// Opt-in and independent from GGML_OPENVINO_CACHE_DIR. Default off.
+
+#include "ggml.h"
+
+#include <cstdint>
+#include <string>
+
+// Returns the compiled-model cache directory from GGML_OPENVINO_COMPILED_MODEL_CACHE_DIR,
+// or empty if unset/disabled. When empty, callers must not use the cache.
+std::string ggml_openvino_model_cache_dir();
+
+// Compute a stable 64-bit fingerprint identifying the model+config that a cgraph
+// would compile to. Combines graph topology, a sampled hash of every weight
+// tensor (name/shape/dtype + bounded byte sample), and the config that changes
+// the produced blob (device, flash-attention, rope params, the compile-memory
+// flags, stateful, and the OpenVINO version). `device` is the resolved device
+// string; `fa` is the flash-attention flag; `rope_params`/`rope_len` cover the
+// model's rope configuration; `extra_cfg` folds in any other blob-affecting bits.
+uint64_t ggml_openvino_model_fingerprint(const ggml_cgraph * cgraph,
+                                         const std::string & device,
+                                         bool fa,
+                                         const int32_t * rope_params,
+                                         int rope_len,
+                                         uint64_t extra_cfg);
+
+// Path to the compiled-blob file for a fingerprint (<dir>/<hex>.blob).
+std::string ggml_openvino_model_cache_blob_path(const std::string & dir, uint64_t fingerprint);
+
+// Path to the sidecar manifest (<dir>/<hex>.manifest) holding the per-weight
+// fingerprints, used to re-verify a hit before trusting the blob.
+std::string ggml_openvino_model_cache_manifest_path(const std::string & dir, uint64_t fingerprint);
+
+// Write/read the manifest. The manifest is a newline-separated list of
+// "name ne0 ne1 ne2 ne3 type sample_hash" lines plus a header line with the
+// fingerprint and OV version. Returns false on I/O error.
+bool ggml_openvino_model_cache_write_manifest(const std::string & path,
+                                              const ggml_cgraph * cgraph,
+                                              uint64_t fingerprint);
+
+// Verify that the cgraph's weights still match the stored manifest (guards the
+// sampled-hash collision risk: a blob is only trusted if every weight's
+// name/shape/type/sample-hash matches what was cached). Returns true on match.
+bool ggml_openvino_model_cache_verify_manifest(const std::string & path,
+                                               const ggml_cgraph * cgraph,
+                                               uint64_t fingerprint);

ggml/src/ggml-openvino/openvino/decoder.h

@@ -6,12 +6,25 @@
 #include <openvino/core/partial_shape.hpp>
 #include <openvino/core/shape.hpp>
 #include <openvino/frontend/decoder.hpp>
+#include <set>
 #include <string>
 
 namespace ov {
 namespace frontend {
 namespace ggml {
 
+struct ModelInputInfo {
+    element::Type type;
+    PartialShape shape;
+};
+
+struct ModelExtraInputInfo {
+    element::Type type;
+    Shape shape;
+    int64_t value;
+    bool is_parameter;
+};
+
 class GgmlDecoder : public DecoderBase {
 public:
     virtual ov::Any get_attribute(const std::string & name) const = 0;
@@ -75,6 +88,10 @@ public:
 
     virtual std::vector<std::string> get_output_names(int node_idx) const = 0;
 
+    virtual std::string get_inplace_op_src(int node_idx) const = 0;
+
+    virtual bool is_view_like_alias_of(int node_idx, const std::string & view_src_name) const = 0;
+
     virtual const std::string & get_op_type() const = 0;
 
     virtual const std::string & get_op_type(int node_idx) const = 0;
@@ -87,15 +104,17 @@ public:
 
     virtual int get_op_case(int node_idx) const = 0;
 
-    virtual const std::map<std::string, std::shared_ptr<ov::Node>> & get_model_inputs() const = 0;
-    virtual const std::map<std::string, std::shared_ptr<ov::Node>> & get_model_extra_inputs() const = 0;
+    virtual const std::map<std::string, ModelInputInfo> & get_model_inputs() const = 0;
+    virtual const std::map<std::string, ModelExtraInputInfo> & get_model_extra_inputs() const = 0;
     virtual const std::map<std::string, std::shared_ptr<ov::Node>> & get_model_weights() const = 0;
-    virtual std::vector<std::string> get_model_output_names() const = 0;
+    virtual std::set<std::string> get_model_output_names() const = 0;
 
     virtual int32_t * get_rope_params() const = 0;
 
     virtual bool has_mixed_rope_params() const = 0;
 
+    virtual int get_ssm_state_size() const = 0;
+
     virtual std::map<std::string, std::string> get_kv_param_res_names() const = 0;
 
     virtual bool is_static() const = 0;

ggml/src/ggml-openvino/openvino/node_context.h

@@ -153,6 +153,8 @@ public:
 
     bool is_stateful() const { return m_decoder->is_stateful(); }
 
+    int get_ssm_state_size() const { return m_decoder->get_ssm_state_size(); }
+
 private:
     std::shared_ptr<GgmlDecoder> m_decoder;
     std::shared_ptr<TensorMap> & m_tensor_map;

ggml/src/ggml-openvino/openvino/op/add.cpp

@@ -0,0 +1,45 @@
+#include "../node_context.h"
+#include "../op_table.h"
+#include "../utils.h"
+
+#include <memory>
+#include <openvino/op/add.hpp>
+#include <openvino/op/constant.hpp>
+#include <openvino/op/reduce_sum.hpp>
+#include <openvino/op/unsqueeze.hpp>
+
+namespace ov {
+namespace frontend {
+namespace ggml {
+namespace op {
+
+OutputVector translate_add(const NodeContext & context) {
+    num_inputs_check(context, 2, 2);
+
+    if (context.get_op_case() == 1) {
+        // MoE expert-plane sum (see is_moe_expert_sum_add): input 1 is a VIEW plane of the
+        // shared base tensor `experts` = [n_embd, n_expert_used, n_tokens, 1] (ggml order) ->
+        // [1, n_tokens, n_expert_used, n_embd] (OV order). The whole ADD chain is equivalent to
+        // reducing the expert axis (OV axis 2) of that base, so bypass the chain and the
+        // per-plane Slices entirely.
+        size_t view_size = context.get_view_input_size(1);
+        auto base_name = context.get_view_input_src_name(1, view_size - 1);
+        auto base = context.get_input(base_name);
+
+        auto reduced = std::make_shared<ov::op::v1::ReduceSum>(
+            base, ov::op::v0::Constant::create(ov::element::i64, ov::Shape{1}, {2}), false);
+        auto res =
+            std::make_shared<ov::op::v0::Unsqueeze>(reduced, ov::op::v0::Constant::create(ov::element::i64, {1}, {1}));
+        return rename_outputs_with_suffix({res}, context.get_name());
+    }
+
+    auto input_0 = process_view_input_new(context, 0);
+    auto input_1 = process_view_input_new(context, 1);
+    auto res = std::make_shared<ov::op::v1::Add>(input_0, input_1);
+    return rename_outputs_with_suffix({res}, context.get_name());
+}
+
+}  // namespace op
+}  // namespace ggml
+}  // namespace frontend
+}  // namespace ov

ggml/src/ggml-openvino/openvino/op/cpy.cpp

@@ -2,10 +2,19 @@
 #include "../op_table.h"
 #include "../utils.h"
 
+#include <climits>
 #include <memory>
+#include <vector>
+#include <openvino/op/add.hpp>
+#include <openvino/op/concat.hpp>
 #include <openvino/op/constant.hpp>
 #include <openvino/op/convert.hpp>
+#include <openvino/op/gather.hpp>
+#include <openvino/op/multiply.hpp>
+#include <openvino/op/negative.hpp>
 #include <openvino/op/reshape.hpp>
+#include <openvino/op/shape_of.hpp>
+#include <openvino/op/slice.hpp>
 
 namespace ov {
 namespace frontend {
@@ -13,18 +22,158 @@ namespace ggml {
 namespace op {
 
 OutputVector translate_cpy(const NodeContext & context) {
-    auto input = process_view_input_new(context, 0);
+    auto op_case = context.get_op_case();
     auto input_shape = context.get_input_shape(0);
-    auto output_shape = context.get_output_shape();
+    auto output_shape = context.get_input_shape(1);
+
+    if (op_case == 4) {
+        auto src = process_view_input_new(context, 0);
+        auto base = context.get_input(1);
+
+        int64_t n_elems = 1;
+        for (const auto & dim : context.get_output_shape().to_shape()) {
+            n_elems *= static_cast<int64_t>(dim);
+        }
+
+        const auto output_stride = context.get_output_stride();
+        const size_t elem_size = output_stride.empty() ? context.get_output_type().size() : output_stride.back();
+        FRONT_END_OP_CONVERSION_CHECK(elem_size > 0, "CPY conv state view update has invalid element size");
+
+        const int64_t begin_val = static_cast<int64_t>(context.get_output_op_offset() / elem_size);
+        const int64_t end_val = begin_val + n_elems;
+
+        auto flat_shape = ov::op::v0::Constant::create(ov::element::i64, {4}, std::vector<int64_t>{1, 1, 1, -1});
+        src = std::make_shared<ov::op::v1::Reshape>(src, flat_shape, false);
+        if (src.get_element_type() != context.get_output_type()) {
+            src = std::make_shared<ov::op::v0::Convert>(src, context.get_output_type());
+        }
+
+        auto zero = ov::op::v0::Constant::create(ov::element::i64, {1}, {0});
+        auto begin = ov::op::v0::Constant::create(ov::element::i64, {1}, {begin_val});
+        auto end = ov::op::v0::Constant::create(ov::element::i64, {1}, {end_val});
+        auto int_max = ov::op::v0::Constant::create(ov::element::i64, {1}, {INT_MAX});
+        auto one = ov::op::v0::Constant::create(ov::element::i64, {1}, {1});
+        auto axis = ov::op::v0::Constant::create(ov::element::i64, {1}, {3});
+
+        auto head_part = std::make_shared<ov::op::v8::Slice>(base, zero, begin, one, axis);
+        auto tail_part = std::make_shared<ov::op::v8::Slice>(base, end, int_max, one, axis);
+        auto res = std::make_shared<ov::op::v0::Concat>(ov::OutputVector{head_part, src, tail_part}, 3);
+        return rename_outputs_with_suffix({res}, context.get_name());
+    }
+
+    // Recurrent state cache writeback into a slot block of the cache. Where the block starts and
+    // where the copied data starts in the source are runtime inputs, so the cached model works for
+    // any kv head, active sequence count and token count. The result is the full updated cache.
+    // op_case 1: gated-delta-net state, op_case 2: conv state, op_case 3: defrag remainder.
+    const std::string slot_begin_name = "rs_slot_begin_" + context.get_name();
+    const bool slice_assign =
+        context.has_input(slot_begin_name) && !context.is_stateful() && (op_case >= 1 && op_case <= 3);
+    if (slice_assign) {
+        const int64_t slot_axis = 2;
+        auto zero = ov::op::v0::Constant::create(ov::element::i64, {1}, {0});
+        auto one = ov::op::v0::Constant::create(ov::element::i64, {1}, {1});
+        auto int_max = ov::op::v0::Constant::create(ov::element::i64, {1}, {INT_MAX});
+        auto axis = ov::op::v0::Constant::create(ov::element::i64, {1}, {slot_axis});
+        auto feature = ov::op::v0::Constant::create(ov::element::i64, {4},
+                                                    std::vector<int64_t>{1, 1, -1, output_shape[3].get_length()});
+
+        ov::Output<ov::Node> src;
+        ov::Output<ov::Node> begin = context.get_input(slot_begin_name);
+        auto base = context.get_input(1);
+        if (op_case == 1) {
+            // GDN packs [attn | state snapshots]; the state part runs from src_begin to the end.
+            auto src_begin = context.get_input("rs_src_begin_" + context.get_name());
+            auto state_part = std::make_shared<ov::op::v8::Slice>(context.get_input(0), src_begin, int_max, one, axis);
+            src = std::make_shared<ov::op::v1::Reshape>(state_part, feature, false);
+        } else if (op_case == 2) {
+            // conv_input is [previous conv state | new tokens]; copy the conv_kernel_size - 1 wide
+            // window starting at src_begin, which is the snapshot this writeback corresponds to.
+            auto window_size = (int64_t) input_shape[3].get_length();
+            auto src_begin = context.get_input("rs_src_begin_" + context.get_name());
+            auto src_end = std::make_shared<ov::op::v1::Add>(
+                src_begin, ov::op::v0::Constant::create(ov::element::i64, {1}, {window_size}));
+            auto window = std::make_shared<ov::op::v8::Slice>(context.get_input(0), src_begin, src_end, one,
+                                                              ov::op::v0::Constant::create(ov::element::i64, {1}, {3}));
+            const auto base_shape = base.get_partial_shape();
+            FRONT_END_OP_CONVERSION_CHECK(base_shape.rank().is_static() && base_shape.rank().get_length() == 4,
+                                          "CPY conv state cache update requires rank-4 base cache");
+            FRONT_END_OP_CONVERSION_CHECK(base_shape[3].is_static(),
+                                          "CPY conv state cache update requires static feature size");
+            FRONT_END_OP_CONVERSION_CHECK(input_shape.rank().is_static() && input_shape.rank().get_length() == 4 &&
+                                              input_shape[2].is_static() && input_shape[3].is_static(),
+                                          "CPY conv state cache update requires static source feature view");
+
+            const int64_t full_feature_size = base_shape[3].get_length();
+            const int64_t update_feature_size = input_shape[2].get_length() * input_shape[3].get_length();
+            const auto output_stride = context.get_output_stride();
+            const size_t elem_size = output_stride.empty() ? context.get_output_type().size() : output_stride.back();
+            FRONT_END_OP_CONVERSION_CHECK(elem_size > 0,
+                                          "CPY conv state cache update has invalid element size");
+            const int64_t feature_begin = static_cast<int64_t>(context.get_output_op_offset() / elem_size) %
+                                          full_feature_size;
+            const int64_t feature_end = feature_begin + update_feature_size;
+            FRONT_END_OP_CONVERSION_CHECK(feature_begin >= 0 && feature_end <= full_feature_size,
+                                          "CPY conv state cache update feature range is out of bounds");
+
+            auto partial_feature = ov::op::v0::Constant::create(
+                ov::element::i64, {4}, std::vector<int64_t>{1, 1, -1, update_feature_size});
+            src = std::make_shared<ov::op::v1::Reshape>(window, partial_feature, false);
+            if (src.get_element_type() != context.get_output_type()) {
+                src = std::make_shared<ov::op::v0::Convert>(src, context.get_output_type());
+            }
+
+            auto src_len = std::make_shared<ov::op::v8::Gather>(
+                std::make_shared<ov::op::v3::ShapeOf>(src, ov::element::i64), axis,
+                ov::op::v0::Constant::create(ov::element::i64, {}, {0}));
+            auto slot_end = std::make_shared<ov::op::v1::Add>(begin, src_len);
+            auto active_slots = std::make_shared<ov::op::v8::Slice>(base, begin, slot_end, one, axis);
+
+            auto feature_axis = ov::op::v0::Constant::create(ov::element::i64, {1}, {3});
+            auto feature_begin_node = ov::op::v0::Constant::create(ov::element::i64, {1}, {feature_begin});
+            auto feature_end_node = ov::op::v0::Constant::create(ov::element::i64, {1}, {feature_end});
+            auto feature_head = std::make_shared<ov::op::v8::Slice>(active_slots, zero, feature_begin_node, one,
+                                                                    feature_axis);
+            auto feature_tail = std::make_shared<ov::op::v8::Slice>(active_slots, feature_end_node, int_max, one,
+                                                                    feature_axis);
+            src = std::make_shared<ov::op::v0::Concat>(ov::OutputVector{feature_head, src, feature_tail}, 3);
+        } else {
+            // op_case 3: gathered remainder rows already have the cache slot layout [1, 1, extra, feature]
+            src = context.get_input(0);
+        }
+
+        if (src.get_element_type() != context.get_output_type()) {
+            src = std::make_shared<ov::op::v0::Convert>(src, context.get_output_type());
+        }
+
+        auto src_len =
+            std::make_shared<ov::op::v8::Gather>(std::make_shared<ov::op::v3::ShapeOf>(src, ov::element::i64), axis,
+                                                 ov::op::v0::Constant::create(ov::element::i64, {}, {0}));
+        auto end = std::make_shared<ov::op::v1::Add>(begin, src_len);
+        auto head_part = std::make_shared<ov::op::v8::Slice>(base, zero, begin, one, axis);
+        auto tail_part = std::make_shared<ov::op::v8::Slice>(base, end, int_max, one, axis);
+        auto res = std::make_shared<ov::op::v0::Concat>(ov::OutputVector{head_part, src, tail_part}, slot_axis);
+        return rename_outputs_with_suffix({res}, context.get_name());
+    }
+
+    auto input = process_view_input_new(context, 0);
 
-    // Non-cast CPY may need a reshape (e.g. [3,192,1,1] -> [576,1,1,1])
     if (input_shape != output_shape) {
         auto new_shape = ov::op::v0::Constant::create(
             ov::element::i64, {static_cast<size_t>(output_shape.rank().get_length())}, output_shape.to_shape());
         input = std::make_shared<ov::op::v1::Reshape>(input, new_shape, false);
     }
 
-    auto res = std::make_shared<ov::op::v0::Convert>(input, context.get_output_type());
+    ov::Output<Node> res;
+    if (context.get_input_type(0) != context.get_output_type()) {
+        res = std::make_shared<ov::op::v0::Convert>(input, context.get_output_type());
+    } else {
+        res = input;
+    }
+
+    if (res.get_node_shared_ptr() == context.get_input(0).get_node_shared_ptr()) {
+        return {res};
+    }
+
     return rename_outputs_with_suffix({res}, context.get_name());
 }
 

ggml/src/ggml-openvino/openvino/op/cumsum.cpp

@@ -0,0 +1,29 @@
+#include "../node_context.h"
+#include "../op_table.h"
+#include "../utils.h"
+
+#include <openvino/op/constant.hpp>
+#include <openvino/op/cum_sum.hpp>
+
+namespace ov {
+namespace frontend {
+namespace ggml {
+namespace op {
+
+// GGML cumsum computes prefix sum along dim 0 (the innermost/fastest dimension).
+// In OV layout the dims are reversed: ggml [ne0, ne1, ne2, ne3] → OV [ne3, ne2, ne1, ne0],
+// so ggml dim 0 maps to OV axis 3 (last axis).
+OutputVector translate_cumsum(const NodeContext & context) {
+    num_inputs_check(context, 1, 1);
+
+    auto x    = context.get_input(0);
+    auto axis = ov::op::v0::Constant::create(ov::element::i64, {}, {3});
+    auto res  = std::make_shared<ov::op::v0::CumSum>(x, axis);
+
+    return rename_outputs_with_suffix({res}, context.get_name());
+}
+
+}  // namespace op
+}  // namespace ggml
+}  // namespace frontend
+}  // namespace ov

ggml/src/ggml-openvino/openvino/op/diag.cpp

@@ -0,0 +1,58 @@
+#include "../node_context.h"
+#include "../op_table.h"
+#include "../utils.h"
+
+#include <openvino/op/constant.hpp>
+#include <openvino/op/equal.hpp>
+#include <openvino/op/multiply.hpp>
+#include <openvino/op/range.hpp>
+#include <openvino/op/reshape.hpp>
+#include <openvino/op/select.hpp>
+
+namespace ov {
+namespace frontend {
+namespace ggml {
+namespace op {
+
+// GGML DIAG takes a 1D vector (ne0, 1, ne2, ne3) and produces a diagonal matrix
+// of shape (ne0, ne0, ne2, ne3).
+// In OV layout (ggml [ne0, ne1, ne2, ne3] → OV [ne3, ne2, ne1, ne0]):
+//   input:  [ne3, ne2, 1, ne0]
+//   output: [ne3, ne2, ne0, ne0]
+// The diagonal: output[..., i, j] = input[..., 0, j] if i == j, else 0.
+OutputVector translate_diag(const NodeContext & context) {
+    num_inputs_check(context, 1, 1);
+
+    auto x = context.get_input(0);  // OV shape: [ne3, ne2, 1, ne0]
+
+    auto out_shape = context.get_output_shape().to_shape();
+    int64_t n = static_cast<int64_t>(out_shape[3]);  // ne0
+
+    // Build index range [0, 1, ..., n-1]
+    auto start = ov::op::v0::Constant::create(ov::element::i64, {}, {int64_t(0)});
+    auto stop  = ov::op::v0::Constant::create(ov::element::i64, {}, {n});
+    auto step  = ov::op::v0::Constant::create(ov::element::i64, {}, {int64_t(1)});
+    auto range = std::make_shared<ov::op::v4::Range>(start, stop, step, ov::element::i64);
+
+    // col_idx shape [1, 1, 1, n]
+    auto col_shape = ov::op::v0::Constant::create(ov::element::i64, {4}, std::vector<int64_t>{1, 1, 1, n});
+    auto col_idx   = std::make_shared<ov::op::v1::Reshape>(range, col_shape, false);
+
+    // row_idx shape [1, 1, n, 1]
+    auto row_shape = ov::op::v0::Constant::create(ov::element::i64, {4}, std::vector<int64_t>{1, 1, n, 1});
+    auto row_idx   = std::make_shared<ov::op::v1::Reshape>(range, row_shape, false);
+
+    // mask: true where col == row (diagonal)
+    auto mask = std::make_shared<ov::op::v1::Equal>(col_idx, row_idx);
+
+    // Broadcast input from [ne3, ne2, 1, ne0] to [ne3, ne2, ne0, ne0] via select
+    auto zero = ov::op::v0::Constant::create(ov::element::f32, {}, {0.0f});
+    auto res  = std::make_shared<ov::op::v1::Select>(mask, x, zero);
+
+    return rename_outputs_with_suffix({res}, context.get_name());
+}
+
+}  // namespace op
+}  // namespace ggml
+}  // namespace frontend
+}  // namespace ov

ggml/src/ggml-openvino/openvino/op/fill.cpp

@@ -0,0 +1,34 @@
+#include "../node_context.h"
+#include "../op_table.h"
+#include "../utils.h"
+
+#include <openvino/op/broadcast.hpp>
+#include <openvino/op/constant.hpp>
+
+namespace ov {
+namespace frontend {
+namespace ggml {
+namespace op {
+
+// GGML FILL sets all elements of a tensor to a constant value.
+// The constant is stored as a float in op_params[0].
+OutputVector translate_fill(const NodeContext & context) {
+    num_inputs_check(context, 1, 1);
+
+    float c;
+    memcpy(&c, context.get_output_op_params(), sizeof(float));
+
+    auto shape = context.get_input_shape(0).to_shape();
+
+    auto val = ov::op::v0::Constant::create(ov::element::f32, {}, {c});
+    auto target_shape = ov::op::v0::Constant::create(ov::element::i64, {shape.size()},
+        std::vector<int64_t>(shape.begin(), shape.end()));
+    auto res = std::make_shared<ov::op::v3::Broadcast>(val, target_shape);
+
+    return rename_outputs_with_suffix({res}, context.get_name());
+}
+
+}  // namespace op
+}  // namespace ggml
+}  // namespace frontend
+}  // namespace ov

ggml/src/ggml-openvino/openvino/op/gated_delta_net.cpp

@@ -19,6 +19,7 @@
 #include <openvino/op/reshape.hpp>
 #include <openvino/op/squeeze.hpp>
 #include <openvino/op/subtract.hpp>
+#include <openvino/op/tile.hpp>
 #include <openvino/op/transpose.hpp>
 #include <openvino/op/unsqueeze.hpp>
 #include <vector>
@@ -31,57 +32,76 @@ namespace op {
 static OutputVector translate_gated_delta_net_ref(const NodeContext & context);
 
 OutputVector translate_gated_delta_net(const NodeContext & context) {
-    // auto v_shape = context.get_input_shape(2).to_shape();  // [B, T, H_v, S_v]
-    // auto q_shape = context.get_input_shape(0).to_shape();  // [B, T, H_k, S_k]
+    auto v_shape = context.get_input_shape(2).to_shape();  // [B, T, H_v, S_v]
+    auto q_shape = context.get_input_shape(0).to_shape();  // [B, T, H_k, S_k]
 
-    // // Fused GatedDeltaNet op only supports scalar gate (kda=0).
-    // // Fall back to reference implementation for per-key-dimension gating.
-    // // if (kda) {
-    // //     return translate_gated_delta_net_ref(context);
-    // // }
-
-    // auto q = context.get_input(0);
-    // auto k = context.get_input(1);
-    // auto v = context.get_input(2);
-    // auto g = context.get_input(3);
-    // auto beta = context.get_input(4);
-    // auto state = context.get_input(5);
+    // Fused GatedDeltaNet op only supports scalar gate (kda=0).
+    // Fall back to reference implementation for per-key-dimension gating.
+    // if (kda) {
+    //     return translate_gated_delta_net_ref(context);
+    // }
 
     // const int64_t B = v_shape[0];
     // const int64_t T = v_shape[1];
-    // const int64_t H_v = v_shape[2];
-    // const int64_t S_v = v_shape[3];
+    const int64_t H_v = v_shape[2];
+    const int64_t S_v = v_shape[3];
+    const int64_t H_k = q_shape[2];
     // const int64_t S_k = q_shape[3];
 
-    // // ggml state layout (OV notation): [B, H_v, value_dim, key_dim]
-    // // GatedDeltaNet op expects: [B, H_v, key_dim, value_dim]
-    // auto state_reshape_shape =
-    //     ov::op::v0::Constant::create(ov::element::i64, {4}, std::vector<int64_t>{B, H_v, S_v, S_k});
-    // state = std::make_shared<ov::op::v1::Reshape>(state, state_reshape_shape, false);
-    // auto state_perm = ov::op::v0::Constant::create(ov::element::i64, {4}, std::vector<int64_t>{0, 1, 3, 2});
-    // state = std::make_shared<ov::op::v1::Transpose>(state, state_perm);
+    auto q = context.get_input(0);
+    auto k = context.get_input(1);
+    auto v = process_view_input(context, 2, H_v * S_v);
+    auto g = context.get_input(3);
+    auto beta = context.get_input(4);
+    auto state = context.get_input(5);
 
-    // g = std::make_shared<ov::op::v0::Squeeze>(g, ov::op::v0::Constant::create(ov::element::i64, {1}, {3}));
-    // beta = std::make_shared<ov::op::v0::Squeeze>(beta, ov::op::v0::Constant::create(ov::element::i64, {1}, {3}));
+    // ggml maps GQA heads in tiled order, while OV GDN maps repeated heads in grouped order.
+    if (H_v != H_k) {
+        const int64_t repeat = H_v / H_k;
+        auto repeats = ov::op::v0::Constant::create(ov::element::i64, {4}, std::vector<int64_t>{1, 1, repeat, 1});
+        q = std::make_shared<ov::op::v0::Tile>(q, repeats);
+        k = std::make_shared<ov::op::v0::Tile>(k, repeats);
+    }
 
-    // auto gdn = std::make_shared<ov::op::internal::GatedDeltaNet>(q, k, v, state, g, beta);
+    if (context.get_view_input_size(2)) {
+        // Same as l2_norm case 1
+        v = std::make_shared<ov::op::v0::Squeeze>(v, ov::op::v0::Constant::create(ov::element::i64, {1}, {0}));
+        auto v_shape = context.get_input_shape(2).to_shape();
+        std::vector<int64_t> reshape_pattern = {0, 0, (int64_t) v_shape[2], (int64_t) v_shape[3]};
+        v = std::make_shared<ov::op::v1::Reshape>(
+            v, ov::op::v0::Constant::create(ov::element::i64, {4}, reshape_pattern), true);
+    }
 
-    // auto attn_4d = gdn->output(0);
-    // auto state_4d = gdn->output(1);  // [B, H_v, key_dim, value_dim]
-    // // Transpose output state back to ggml layout [B, H_v, value_dim, key_dim]
-    // auto state_transposed = std::make_shared<ov::op::v1::Transpose>(state_4d, state_perm);
-    // auto flat_shape_1d = ov::op::v0::Constant::create(ov::element::i64, {1}, {-1});
-    // auto attn = std::make_shared<ov::op::v1::Reshape>(attn_4d, flat_shape_1d, false);
-    // auto new_state = std::make_shared<ov::op::v1::Reshape>(state_transposed, flat_shape_1d, false);
-    // auto packed = std::make_shared<ov::op::v0::Concat>(ov::OutputVector{attn, new_state}, 0);
-    // auto out_shape =
-    //     ov::op::v0::Constant::create(ov::element::i64, {4}, std::vector<int64_t>{1, 1, T * B + S_v * B, S_v * H_v});
-    // auto res = std::make_shared<ov::op::v1::Reshape>(packed, out_shape, false);
+    // ggml state layout (OV notation): [B, H_v, value_dim, key_dim]
+    // GatedDeltaNet op expects: [B, H_v, key_dim, value_dim]
+    auto state_perm = ov::op::v0::Constant::create(ov::element::i64, {4}, std::vector<int64_t>{0, 1, 3, 2});
+    state = std::make_shared<ov::op::v1::Transpose>(state, state_perm);
 
-    // return rename_outputs_with_suffix({res}, context.get_name());
+    g = std::make_shared<ov::op::v0::Squeeze>(g, ov::op::v0::Constant::create(ov::element::i64, {1}, {3}));
+    beta = std::make_shared<ov::op::v0::Squeeze>(beta, ov::op::v0::Constant::create(ov::element::i64, {1}, {3}));
 
-    // The OV version in CI does not have the GatedDeltaNet op, so use reference implementation for now.
-    return translate_gated_delta_net_ref(context);
+    // std::cout << "GatedDeltaNet input shapes: q=" << q.get_partial_shape() << ", k=" << k.get_partial_shape()
+    //           << ", v=" << v.get_partial_shape() << ", g=" << g.get_partial_shape()
+    //           << ", beta=" << beta.get_partial_shape() << ", state=" << state.get_partial_shape() << std::endl;
+
+    auto gdn = std::make_shared<ov::op::internal::GatedDeltaNet>(q, k, v, state, g, beta);
+    auto attn_4d = gdn->output(0);
+    auto state_4d = gdn->output(1);  // [B, H_v, key_dim, value_dim]
+
+    // std::cout << "GatedDeltaNet output shapes: attn=" << gdn->output(0).get_partial_shape()
+    //           << ", new_state=" << gdn->output(1).get_partial_shape() << std::endl;
+
+    // Transpose output state back to ggml layout [B, H_v, value_dim, key_dim]
+    auto state_transposed = std::make_shared<ov::op::v1::Transpose>(state_4d, state_perm);
+    auto flat_shape_1d = ov::op::v0::Constant::create(ov::element::i64, {1}, {-1});
+    auto attn = std::make_shared<ov::op::v1::Reshape>(attn_4d, flat_shape_1d, false);
+    auto new_state = std::make_shared<ov::op::v1::Reshape>(state_transposed, flat_shape_1d, false);
+    auto packed = std::make_shared<ov::op::v0::Concat>(ov::OutputVector{attn, new_state}, 0);
+    auto out_shape = ov::op::v0::Constant::create(ov::element::i64, {4},
+                                                  std::vector<int64_t>{1, 1, -1 /*T * B + S_v * B*/, S_v * H_v});
+    auto res = std::make_shared<ov::op::v1::Reshape>(packed, out_shape, false);
+
+    return rename_outputs_with_suffix({res}, context.get_name());
 }
 
 static OutputVector translate_gated_delta_net_ref(const NodeContext & context) {

ggml/src/ggml-openvino/openvino/op/gather_matmul.hpp

@@ -0,0 +1,43 @@
+// Copyright (C) 2018-2026 Intel Corporation
+// SPDX-License-Identifier: Apache-2.0
+//
+// Local mirror of OpenVINO's internal ov::op::internal::GatherMatmul op.
+//
+// The op class body (validate_and_infer_types / clone_with_new_inputs) is
+// provided by the linked libopenvino.so; only the declaration is needed here so
+// the backend can construct the node directly (same approach as GatedDeltaNet).
+// The class layout must stay in sync with
+//   openvino/src/common/transformations/include/ov_ops/gather_matmul.hpp
+//
+// \note GatherMatmul op class is under development and subject to change.
+
+#pragma once
+
+#include "openvino/op/op.hpp"
+
+namespace ov::op::internal {
+
+class OPENVINO_API GatherMatmul : public ov::op::Op {
+public:
+    OPENVINO_OP("GatherMatmul")
+
+    GatherMatmul() = default;
+
+    GatherMatmul(const ov::Output<Node>& A,
+                 const ov::Output<Node>& B,
+                 const ov::Output<Node>& indices,
+                 const ov::Output<Node>& bias);
+
+    GatherMatmul(const ov::Output<Node>& A, const ov::Output<Node>& B, const ov::Output<Node>& indices);
+
+    std::shared_ptr<Node> clone_with_new_inputs(const ov::OutputVector& new_args) const override;
+
+    void validate_and_infer_types() override;
+
+private:
+    // the weights matrix B is expected to have the transposed form [group, N, K]
+    static constexpr bool transp_a = false;
+    static constexpr bool transp_b = true;
+};
+
+}  // namespace ov::op::internal

ggml/src/ggml-openvino/openvino/op/get_rows.cpp

@@ -2,11 +2,16 @@
 #include "../op_table.h"
 #include "../utils.h"
 
+#include <climits>
 #include <openvino/core/node.hpp>
 #include <openvino/core/node_output.hpp>
+#include <openvino/op/broadcast.hpp>
+#include <openvino/op/concat.hpp>
 #include <openvino/op/constant.hpp>
 #include <openvino/op/convert.hpp>
 #include <openvino/op/gather.hpp>
+#include <openvino/op/shape_of.hpp>
+#include <openvino/op/slice.hpp>
 #include <openvino/op/squeeze.hpp>
 #include <openvino/op/unsqueeze.hpp>
 
@@ -20,7 +25,27 @@ OutputVector translate_get_rows(const NodeContext & context) {
 
     Output<Node> res;
     auto data = process_view_input_new(context, 0);
-    auto indices = process_view_input_new(context, 1);
+
+    auto op_case = context.get_op_case();
+    ov::Output<ov::Node> indices;
+    if ((op_case == 1 || op_case == 2) && context.has_input("s_copy_active_slot_len")) {
+        // Recurrent state reorder (inp->s_copy): slice the active (op_case 1) or extra (op_case 2)
+        // segment from the s_copy index list at runtime, instead of baking the static view offset,
+        // so the cached IR works for any number of active sequences.
+        auto s_copy = context.get_input(1);
+        auto len = context.get_input("s_copy_active_slot_len");
+        auto step = ov::op::v0::Constant::create(ov::element::i64, {1}, {1});
+        auto axis = ov::op::v0::Constant::create(ov::element::i64, {1}, {3});
+        if (op_case == 1) {
+            auto begin = ov::op::v0::Constant::create(ov::element::i64, {1}, {0});
+            indices = std::make_shared<ov::op::v8::Slice>(s_copy, begin, len, step, axis);
+        } else {
+            auto end = ov::op::v0::Constant::create(ov::element::i64, {1}, {INT_MAX});
+            indices = std::make_shared<ov::op::v8::Slice>(s_copy, len, end, step, axis);
+        }
+    } else {
+        indices = process_view_input_new(context, 1);
+    }
 
     // data[1,b,x,y] ind[1,1,b,x'] test-backend-ops case
     // data[x,y] ind[1,1,1,x'] normal case
@@ -37,7 +62,62 @@ OutputVector translate_get_rows(const NodeContext & context) {
             auto axis = ov::op::v0::Constant::create(ov::element::i32, ov::Shape{}, {1});
             data =
                 std::make_shared<ov::op::v0::Squeeze>(data, ov::op::v0::Constant::create(ov::element::i64, {1}, {0}));
-            res = std::make_shared<ov::op::v8::Gather>(data, indices, axis, 1);
+            // data: [batch, rows, ...], indices: [batch, n] - this is a batched gather
+            // (batch_dims=1) along the rows axis. The data and indices batch dims are
+            // logically equal (both == n_tokens) but reach this node through independent
+            // reshapes, so the GPU plugin's gather shape inference cannot prove
+            // data.shape[0] == indices.shape[0] and rejects the node. We must tie both
+            // batch dims to the SAME value, and crucially that value must stay DYNAMIC.
+            const auto data_ps = data.get_partial_shape();
+            const auto idx_ps = indices.get_partial_shape();
+            const bool data_batch_static = data_ps.rank().is_static() && data_ps[0].is_static();
+            const bool idx_batch_dynamic = idx_ps.rank().is_dynamic() || idx_ps[0].is_dynamic();
+
+            if (data_batch_static && idx_batch_dynamic) {
+                // MoE per-expert-scale path: `data` is a statically-tiled REPEAT
+                // (ggml_repeat_4d(scale, 1, n_expert, n_tokens, 1)) whose batch dim is a
+                // compile-time-constant n_tokens, and every batch slice is IDENTICAL (it was
+                // tiled from a single [1, n_expert, 1] scale). `indices` (selected_experts)
+                // carries the genuinely dynamic token dim. Broadcasting indices up to the
+                // static data batch (the naive fix) would freeze the token dim to the
+                // captured prefill length, and that static value then flows through the
+                // gather into the residual stream, making every following decoder layer
+                // static -> triggers the GPU in-place-concat KV-cache corruption (only
+                // layer 0 stays dynamic). A static->dynamic Broadcast cannot expand, so
+                // instead collapse the redundant data batch to 1 and broadcast 1->dynamic to
+                // match the indices batch. Mathematically identical (the slices are equal),
+                // and the whole graph stays dynamic.
+                auto zero = ov::op::v0::Constant::create(ov::element::i64, {1}, {0});
+                auto one = ov::op::v0::Constant::create(ov::element::i64, {1}, {1});
+                auto axis0 = ov::op::v0::Constant::create(ov::element::i64, {1}, {0});
+                auto data_b1 = std::make_shared<ov::op::v8::Slice>(data, zero, one, one, axis0);  // [1, rows, ...]
+
+                auto idx_shape = std::make_shared<ov::op::v3::ShapeOf>(indices, ov::element::i64);
+                auto idx_batch = get_dimensions(idx_shape, {0});  // [batch] (dynamic)
+                auto data_b1_shape = std::make_shared<ov::op::v3::ShapeOf>(data_b1, ov::element::i64);
+                const auto rank = data_ps.rank().get_length();
+                std::vector<int> rest_axes;
+                for (int a = 1; a < rank; ++a) {
+                    rest_axes.push_back(a);
+                }
+                auto data_rest = get_dimensions(data_b1_shape, rest_axes);  // [rows, ...]
+                auto data_target = std::make_shared<ov::op::v0::Concat>(ov::OutputVector{idx_batch, data_rest}, 0);
+                data =
+                    std::make_shared<ov::op::v3::Broadcast>(data_b1, data_target, ov::op::BroadcastType::BIDIRECTIONAL);
+                res = std::make_shared<ov::op::v8::Gather>(data, indices, axis, 1);
+            } else {
+                // General case: tie the indices batch to the data batch (the data batch is
+                // already dynamic, e.g. the routing-weights gather whose data comes from the
+                // activations). Broadcast indices to [data_batch, indices_n].
+                auto data_shape = std::make_shared<ov::op::v3::ShapeOf>(data, ov::element::i64);
+                auto data_batch = get_dimensions(data_shape, {0});  // [batch]
+                auto idx_shape = std::make_shared<ov::op::v3::ShapeOf>(indices, ov::element::i64);
+                auto idx_n = get_dimensions(idx_shape, {1});  // [n]
+                auto idx_target = std::make_shared<ov::op::v0::Concat>(ov::OutputVector{data_batch, idx_n}, 0);
+                indices = std::make_shared<ov::op::v3::Broadcast>(indices, idx_target,
+                                                                  ov::op::BroadcastType::BIDIRECTIONAL);
+                res = std::make_shared<ov::op::v8::Gather>(data, indices, axis, 1);
+            }
         }
     } else if (context.is_stateful() && data.get_partial_shape().rank() == 3) {
         auto axis = ov::op::v0::Constant::create(ov::element::i32, ov::Shape{}, {1});

ggml/src/ggml-openvino/openvino/op/l2_norm.cpp

@@ -8,7 +8,9 @@
 #include <openvino/op/maximum.hpp>
 #include <openvino/op/multiply.hpp>
 #include <openvino/op/reduce_sum.hpp>
+#include <openvino/op/reshape.hpp>
 #include <openvino/op/sqrt.hpp>
+#include <openvino/op/squeeze.hpp>
 
 namespace ov {
 namespace frontend {
@@ -20,6 +22,21 @@ OutputVector translate_l2_norm(const NodeContext & context) {
 
     auto input_node = process_view_input_new(context, 0);
 
+    if (context.get_op_case() == 1) {
+        // 92: [   128,    16,     1,     2] VIEW                 q_conv-1
+        //      [  6144,     1,     2,     1]            0: UNARY       conv_output_silu-1
+        // 93: [   128,    16,     1,     2] L2_NORM              q_conv_predelta-1
+        //      [   128,    16,     1,     2]            0: VIEW        q_conv-1
+        auto output_shape = context.get_output_shape().to_shape();
+        input_node = process_view_input(context, 0, output_shape[2] * output_shape[3]);
+        input_node =
+            std::make_shared<ov::op::v0::Squeeze>(input_node, ov::op::v0::Constant::create(ov::element::i64, {1}, {0}));
+
+        std::vector<int64_t> reshape_pattern = {0, 0, (int64_t) output_shape[2], (int64_t) output_shape[3]};
+        input_node = std::make_shared<ov::op::v1::Reshape>(
+            input_node, ov::op::v0::Constant::create(ov::element::i64, {4}, reshape_pattern), true);
+    }
+
     auto squared = std::make_shared<ov::op::v1::Multiply>(input_node, input_node);
 
     auto sum_squared = std::make_shared<ov::op::v1::ReduceSum>(

ggml/src/ggml-openvino/openvino/op/mul_mat_id.cpp

@@ -1,6 +1,8 @@
 #include "../node_context.h"
 #include "../op_table.h"
 #include "../utils.h"
+#include "gather_matmul.hpp"
+#include "ggml-openvino/ggml-openvino-extra.h"
 
 #include <cstdint>
 #include <cstring>
@@ -18,6 +20,7 @@
 #include <openvino/op/reshape.hpp>
 #include <openvino/op/shape_of.hpp>
 #include <openvino/op/slice.hpp>
+#include <openvino/op/transpose.hpp>
 #include <openvino/op/unsqueeze.hpp>
 #include <vector>
 
@@ -37,6 +40,70 @@ ov::Output<ov::Node> slice_axis(const ov::Output<ov::Node> & input, int64_t axis
                                               const_i64({axis}));
 }
 
+ov::Output<ov::Node> static_shape_dims_or_shapeof(const ov::Output<ov::Node> & input,
+                                                  const std::vector<int> & dims) {
+    const auto partial_shape = input.get_partial_shape();
+    if (partial_shape.is_static()) {
+        std::vector<int64_t> values;
+        values.reserve(dims.size());
+        for (const int64_t dim : dims) {
+            values.push_back(partial_shape[dim].get_length());
+        }
+        return const_i64(values);
+    }
+
+    auto shape = std::make_shared<ov::op::v3::ShapeOf>(input, ov::element::i64);
+    return get_dimensions(shape, dims);
+}
+
+ov::Output<ov::Node> translate_mul_mat_id_gather_matmul_fallback(const NodeContext & context,
+                                                                 ov::Output<ov::Node> expert_weights,
+                                                                 ov::Output<ov::Node> activations,
+                                                                 ov::Output<ov::Node> ids) {
+    auto gather_axis = ov::op::v0::Constant::create(ov::element::i32, ov::Shape{}, {0});
+    ov::Output<ov::Node> selected_weights = std::make_shared<ov::op::v8::Gather>(expert_weights, ids, gather_axis);
+
+    const auto output_type = context.get_output_type();
+    if (selected_weights.get_element_type() != ov::element::f32) {
+        selected_weights = std::make_shared<ov::op::v0::Convert>(selected_weights, ov::element::f32);
+    }
+    if (activations.get_element_type() != ov::element::f32) {
+        activations = std::make_shared<ov::op::v0::Convert>(activations, ov::element::f32);
+    }
+
+    auto activations_shape = std::make_shared<ov::op::v3::ShapeOf>(activations, ov::element::i64);
+    auto ids_shape = std::make_shared<ov::op::v3::ShapeOf>(ids, ov::element::i64);
+    ov::Output<ov::Node> acts_target_dims = std::make_shared<ov::op::v0::Concat>(
+        ov::OutputVector{
+            get_dimensions(activations_shape, {0}),
+            get_dimensions(ids_shape, {1}),
+            get_dimensions(activations_shape, {2}),
+        },
+        0);
+    ov::Output<ov::Node> acts_broadcasted =
+        std::make_shared<ov::op::v3::Broadcast>(activations, acts_target_dims, ov::op::BroadcastType::BIDIRECTIONAL);
+
+    auto activations_expanded = std::make_shared<ov::op::v0::Unsqueeze>(acts_broadcasted, const_i64({2}));
+    ov::Output<ov::Node> result =
+        std::make_shared<ov::op::v0::MatMul>(activations_expanded, selected_weights, false, true);
+
+    auto output_shape = context.get_output_shape();
+    FRONT_END_OP_CONVERSION_CHECK(output_shape.rank().is_static() && output_shape.rank().get_length() == 4,
+                                  "Unexpected MUL_MAT_ID output rank");
+    FRONT_END_OP_CONVERSION_CHECK(output_shape[3].is_static(), "Expected static row dimension for MUL_MAT_ID output");
+
+    auto batch_dim = ov::op::v0::Constant::create(ov::element::i64, {1}, {1});
+    auto row_dim = ov::op::v0::Constant::create(ov::element::i64, {1}, {output_shape[3].get_length()});
+    auto result_target_dims = std::make_shared<ov::op::v0::Concat>(
+        ov::OutputVector{batch_dim, get_dimensions(ids_shape, {0, 1}), row_dim}, 0);
+    result = std::make_shared<ov::op::v1::Reshape>(result, result_target_dims, false);
+
+    if (result.get_element_type() != output_type) {
+        result = std::make_shared<ov::op::v0::Convert>(result, output_type);
+    }
+    return result;
+}
+
 ov::Output<ov::Node> translate_mul_mat_id_mxfp4_packed(const NodeContext & context,
                                                        ov::Output<ov::Node> expert_weights,
                                                        ov::Output<ov::Node> activations,
@@ -144,22 +211,33 @@ OutputVector translate_mul_mat_id(const NodeContext & context) {
                                           context.get_name());
     }
 
+    // General (non-packed) path: dense F32/F16/BF16 weights, or the f16 dequantization chain for
+    // quantized MoE experts (see extract_quantized_weights / make_int4_weights / make_int8_weights in
+    // ggml-quants.cpp). Routed through ov::op::internal::GatherMatmul instead of a naive
+    // Gather+Broadcast+MatMul, so the selected expert's full weight matrix is never materialized per
+    // token. The CPU plugin's ConvertGatherMatmulToGatherMatmulCompressed pass (run during
+    // compile_model) fuses the dequantization chain feeding GatherMatmul's B input into a
+    // GatherMatmulCompressed node automatically, as long as MarkDequantization has marked the chain --
+    // see translate_session.cpp's apply_transformations for the MarkDequantization registration.
+    //
     // OpenVINO sees GGML tensors in reversed dimension order:
-    //   weights: [1, n_expert, m, k]
     //   activations: [1, n_tokens, n_used_or_1, k]
     //   ids: [1, 1, n_tokens, n_used]
-    // Rebuild the logical ranks explicitly from the 4D inputs instead of relying
-    // on fixed squeeze axes: real graphs can arrive through VIEW/RESHAPE chains
-    // where singleton axes are still represented differently at this point.
-    auto expert_weights_shape_4d = std::make_shared<ov::op::v3::ShapeOf>(expert_weights, ov::element::i64);
-    auto activations_shape_4d = std::make_shared<ov::op::v3::ShapeOf>(activations, ov::element::i64);
-    auto ids_shape_4d = std::make_shared<ov::op::v3::ShapeOf>(ids, ov::element::i64);
+    // expert_weights is either [1, n_expert, m, k] (4D, e.g. non-quantized weights without a
+    // pre-built extra) or already [n_expert, m, k] (3D, weights routed through
+    // process_weight_tensor) -- GatherMatmul's B input expects the latter.
+    auto expert_weights_rank = expert_weights.get_partial_shape().rank();
+    FRONT_END_OP_CONVERSION_CHECK(expert_weights_rank.is_static(),
+                                  "Expected static rank for MUL_MAT_ID expert weights");
+    const bool use_gpu_fallback = ggml_openvino_get_device_name() == "GPU";
+    if (expert_weights_rank.get_length() == 4) {
+        auto expert_weights_shape_3d = static_shape_dims_or_shapeof(expert_weights, {1, 2, 3});
+        expert_weights = std::make_shared<ov::op::v1::Reshape>(expert_weights, expert_weights_shape_3d, false);
+    }
 
-    auto expert_weights_shape_3d = get_dimensions(expert_weights_shape_4d, {1, 2, 3});
-    auto activations_shape_3d = get_dimensions(activations_shape_4d, {1, 2, 3});
-    auto ids_shape_2d = get_dimensions(ids_shape_4d, {2, 3});
+    auto activations_shape_3d = static_shape_dims_or_shapeof(activations, {1, 2, 3});
+    auto ids_shape_2d = static_shape_dims_or_shapeof(ids, {2, 3});
 
-    expert_weights = std::make_shared<ov::op::v1::Reshape>(expert_weights, expert_weights_shape_3d, false);
     activations = std::make_shared<ov::op::v1::Reshape>(activations, activations_shape_3d, false);
     ids = std::make_shared<ov::op::v1::Reshape>(ids, ids_shape_2d, false);
 
@@ -167,51 +245,30 @@ OutputVector translate_mul_mat_id(const NodeContext & context) {
         ids = std::make_shared<ov::op::v0::Convert>(ids, ov::element::i32);
     }
 
-    auto gather_axis = ov::op::v0::Constant::create(ov::element::i32, ov::Shape{}, {0});
-    ov::Output<ov::Node> selected_weights = std::make_shared<ov::op::v8::Gather>(expert_weights, ids, gather_axis);
-
     const auto output_type = context.get_output_type();
-    if (selected_weights.get_element_type() != ov::element::f32) {
-        selected_weights = std::make_shared<ov::op::v0::Convert>(selected_weights, ov::element::f32);
-    }
     if (activations.get_element_type() != ov::element::f32) {
         activations = std::make_shared<ov::op::v0::Convert>(activations, ov::element::f32);
     }
 
-    auto activations_shape = std::make_shared<ov::op::v3::ShapeOf>(activations, ov::element::i64);
-    auto ids_shape = std::make_shared<ov::op::v3::ShapeOf>(ids, ov::element::i64);
-    ov::Output<ov::Node> acts_target_dims = std::make_shared<ov::op::v0::Concat>(
-        ov::OutputVector{
-            get_dimensions(activations_shape, {0}),
-            get_dimensions(ids_shape, {1}),
-            get_dimensions(activations_shape, {2}),
-        },
-        0);
-    ov::Output<ov::Node> acts_broadcasted =
-        std::make_shared<ov::op::v3::Broadcast>(activations, acts_target_dims, ov::op::BroadcastType::BIDIRECTIONAL);
+    if (use_gpu_fallback || !expert_weights.get_partial_shape().is_static() || !activations.get_partial_shape().is_static() ||
+        !ids.get_partial_shape().is_static()) {
+        return rename_outputs_with_suffix({translate_mul_mat_id_gather_matmul_fallback(context, expert_weights, activations, ids)},
+                                          context.get_name());
+    }
 
-    auto unsqueeze_axes = ov::op::v0::Constant::create(ov::element::i64, {1}, {2});
-    auto activations_expanded = std::make_shared<ov::op::v0::Unsqueeze>(acts_broadcasted, unsqueeze_axes);
+    // GatherMatmul's A input is [n_used_or_1, n_tokens, k]; activations_3d is
+    // [n_tokens, n_used_or_1, k].
+    auto activations_transpose_order = const_i64({1, 0, 2});
+    ov::Output<ov::Node> activations_for_gather =
+        std::make_shared<ov::op::v1::Transpose>(activations, activations_transpose_order);
 
-    auto batch_dim = ov::op::v0::Constant::create(ov::element::i64, {1}, {1});
-    auto output_shape = context.get_output_shape();
-    FRONT_END_OP_CONVERSION_CHECK(output_shape.rank().is_static() && output_shape.rank().get_length() == 4,
-                                  "Unexpected MUL_MAT_ID output rank");
-    FRONT_END_OP_CONVERSION_CHECK(output_shape[3].is_static(), "Expected static row dimension for MUL_MAT_ID output");
-    const auto row_dim_value = output_shape[3].get_length();
-    auto row_dim = ov::op::v0::Constant::create(ov::element::i64, {1}, {row_dim_value});
+    ov::Output<ov::Node> result = std::make_shared<ov::op::internal::GatherMatmul>(activations_for_gather, expert_weights, ids);
 
-    ov::Output<ov::Node> result =
-        std::make_shared<ov::op::v0::MatMul>(activations_expanded, selected_weights, false, true);
-
-    auto result_target_dims = std::make_shared<ov::op::v0::Concat>(
-        ov::OutputVector{
-            batch_dim,
-            get_dimensions(ids_shape, {0, 1}),
-            row_dim,
-        },
-        0);
-    result = std::make_shared<ov::op::v1::Reshape>(result, result_target_dims, false);
+    // result is [n_used, n_tokens, m]; GGML expects [1, n_tokens, n_used, m].
+    auto result_transpose_order = const_i64({1, 0, 2});
+    result = std::make_shared<ov::op::v1::Transpose>(result, result_transpose_order);
+    auto unsqueeze_axes = ov::op::v0::Constant::create(ov::element::i64, {1}, {0});
+    result = std::make_shared<ov::op::v0::Unsqueeze>(result, unsqueeze_axes);
 
     if (result.get_element_type() != output_type) {
         result = std::make_shared<ov::op::v0::Convert>(result, output_type);

ggml/src/ggml-openvino/openvino/op/repeat.cpp

@@ -23,47 +23,21 @@ OutputVector translate_repeat(const NodeContext & context) {
 
     auto input = process_view_input_new(context, 0);
 
-    const auto input_shape = context.get_input_shape(0);
-    const auto output_shape = context.get_output_shape();
+    const auto input_shape = context.get_input_shape(0).to_shape();
+    const auto output_shape = context.get_output_shape().to_shape();
 
-    if (input_shape.rank().is_static() && output_shape.rank().is_static() &&
-        input_shape.rank() == output_shape.rank()) {
-        const auto rank = static_cast<size_t>(input_shape.rank().get_length());
-        std::vector<int64_t> repeats(rank, 1);
-        bool all_static = true;
+    std::vector<int64_t> repeats(4, 1);
+    for (size_t axis = 0; axis < 4; ++axis) {
+        const int64_t input_dim = input_shape[axis];
+        const int64_t output_dim = output_shape[axis];
 
-        for (size_t axis = 0; axis < rank; ++axis) {
-            if (!input_shape[axis].is_static() || !output_shape[axis].is_static()) {
-                all_static = false;
-                break;
-            }
+        FRONT_END_OP_CONVERSION_CHECK(input_dim > 0 && output_dim > 0 && output_dim % input_dim == 0,
+                                      "REPEAT input shape ", input_shape, " cannot tile to match ", output_shape);
 
-            const int64_t input_dim = input_shape[axis].get_length();
-            const int64_t output_dim = output_shape[axis].get_length();
-
-            FRONT_END_OP_CONVERSION_CHECK(input_dim > 0 && output_dim > 0 && output_dim % input_dim == 0,
-                                          "REPEAT input shape ", input_shape, " cannot tile to match ", output_shape);
-
-            repeats[axis] = output_dim / input_dim;
-        }
-
-        if (all_static) {
-            auto repeats_node = ov::op::v0::Constant::create(ov::element::i64, {repeats.size()}, repeats);
-            ov::Output<ov::Node> res = std::make_shared<ov::op::v0::Tile>(input, repeats_node);
-            return rename_outputs_with_suffix({res}, context.get_name());
-        }
+        repeats[axis] = output_dim / input_dim;
     }
 
-    // Dynamic fallback: tile by the ratio of output to input shape.
-    auto input_shape_node = std::make_shared<ov::op::v3::ShapeOf>(input, ov::element::i64);
-    std::shared_ptr<ov::Node> target_shape_node;
-    if (output_shape.rank().is_static() && output_shape.is_static()) {
-        target_shape_node =
-            ov::op::v0::Constant::create(ov::element::i64, {output_shape.to_shape().size()}, output_shape.to_shape());
-    } else {
-        target_shape_node = std::make_shared<ov::op::v3::ShapeOf>(context.get_input(1), ov::element::i64);
-    }
-    auto repeats_node = std::make_shared<ov::op::v1::Divide>(target_shape_node, input_shape_node);
+    auto repeats_node = ov::op::v0::Constant::create(ov::element::i64, {repeats.size()}, repeats);
     ov::Output<ov::Node> res = std::make_shared<ov::op::v0::Tile>(input, repeats_node);
     return rename_outputs_with_suffix({res}, context.get_name());
 }

ggml/src/ggml-openvino/openvino/op/reshape.cpp

@@ -25,13 +25,12 @@ OutputVector translate_reshape(const NodeContext & context) {
     }
 
     int op_case = context.get_op_case();
-    FRONT_END_CHECK_IMPLEMENTED(
-        op_case == 1 || op_case == 2 || op_case == 3 || op_case == 4 || op_case == 5 || op_case == 6,
-        "Unsupported RESHAPE case");
 
     auto output_shape = context.get_output_shape().to_shape();
     std::shared_ptr<ov::Node> new_shape_node;
-    if (op_case == 1) {
+    if (op_case == 0) {
+        new_shape_node = ov::op::v0::Constant::create(ov::element::i64, {4}, context.get_output_shape().to_shape());
+    } else if (op_case == 1) {
         if (context.is_stateful()) {
             new_shape_node = ov::op::v0::Constant::create(
                 ov::element::i64, {3}, std::vector<int64_t>{-1, (int64_t) output_shape[2], (int64_t) output_shape[3]});
@@ -76,9 +75,33 @@ OutputVector translate_reshape(const NodeContext & context) {
         //     ov::op::v0::Constant::create(ov::element::i64, {1}, {(int64_t) context.get_output_shape().to_shape()[3]});
         // auto one = ov::op::v0::Constant::create(ov::element::i64, {1}, {1});
         // new_shape_node = std::make_shared<ov::op::v0::Concat>(ov::OutputVector{one, one, token_len, emb_size}, 0);
-
     } else if (op_case == 6) {
-        new_shape_node = ov::op::v0::Constant::create(ov::element::i64, {4}, context.get_output_shape().to_shape());
+        // 14: [  6144,     1,     2,     1] RESHAPE              linear_attn_qkv_mixed-0
+        //     [  6144,     2,     1,     1]            0: MUL_MAT     node_13
+        // reshape to [1, n_slot_active_len, -1, 6144]
+        if (context.has_input("s_copy_active_slot_len")) {
+            auto n_slot_active_len = context.get_input("s_copy_active_slot_len");
+            auto emb_size = ov::op::v0::Constant::create(ov::element::i64, {1},
+                                                         {(int64_t) context.get_output_shape().to_shape()[3]});
+            auto one = ov::op::v0::Constant::create(ov::element::i64, {1}, {1});
+            auto neg_one = ov::op::v0::Constant::create(ov::element::i64, {1}, {-1});
+            new_shape_node =
+                std::make_shared<ov::op::v0::Concat>(ov::OutputVector{one, n_slot_active_len, neg_one, emb_size}, 0);
+        } else {
+            new_shape_node = ov::op::v0::Constant::create(ov::element::i64, {4}, context.get_output_shape().to_shape());
+        }
+    } else if (op_case == 7) {
+        // 57: [  2048,     2,     1,     1] RESHAPE              linear_attn_out-0 (reshaped)
+        //     [  2048,     1,     2,     1]            0: MUL_MAT     linear_attn_out-0
+        std::vector<int64_t> shape_vec = {1, 1, -1, (int64_t) context.get_output_shape().to_shape()[3]};
+        new_shape_node = ov::op::v0::Constant::create(ov::element::i64, {4}, shape_vec);
+    } else if (op_case == 8) {
+        // 106: [   128,   128,    16,     2] RESHAPE              state_predelta-1
+        //      [ 262144,     2,     1,     1]            0: GET_ROWS    node_86
+        auto output_shape = context.get_output_shape().to_shape();
+        std::vector<int64_t> shape_vec = {-1, (int64_t) output_shape[1], (int64_t) output_shape[2],
+                                          (int64_t) output_shape[3]};
+        new_shape_node = ov::op::v0::Constant::create(ov::element::i64, {4}, shape_vec);
     }
     auto res = std::make_shared<ov::op::v1::Reshape>(context.get_input(0), new_shape_node, false);
     return rename_outputs_with_suffix({res}, context.get_name());

ggml/src/ggml-openvino/openvino/op/rms_norm.cpp

@@ -7,8 +7,11 @@
 #include <openvino/op/constant.hpp>
 #include <openvino/op/divide.hpp>
 #include <openvino/op/multiply.hpp>
+#include <openvino/op/negative.hpp>
 #include <openvino/op/power.hpp>
 #include <openvino/op/reduce_mean.hpp>
+#include <openvino/op/reshape.hpp>
+#include <openvino/op/slice.hpp>
 #include <openvino/op/sqrt.hpp>
 
 namespace ov {
@@ -19,9 +22,41 @@ namespace op {
 OutputVector translate_rms_norm(const NodeContext & context) {
     num_inputs_check(context, 1, 1);
 
-    auto input_node = process_view_input_new(context, 0);
-    auto square = std::make_shared<ov::op::v1::Power>(
-        input_node, ov::op::v0::Constant::create(ov::element::f32, ov::Shape{1}, {2.0f}));
+    auto op_case = context.get_op_case();
+
+    ov::Output<ov::Node> input_node;
+    if (op_case == 1) {
+        input_node = process_view_input_new(context, 0);
+    } else if (op_case == 2) {
+        auto ssm_state_size = context.get_ssm_state_size();
+        // The GDN op packs [attn | new_state] along the row axis; the state occupies the last
+        // ssm_state_size * n_seqs rows. Slice it off (scaling by the active sequence count) to keep
+        // just the attention output.
+        ov::Output<ov::Node> state_end;
+        if (context.has_input("s_copy_active_slot_len")) {
+            auto len = context.get_input("s_copy_active_slot_len");
+            auto state_rows = std::make_shared<ov::op::v1::Multiply>(
+                ov::op::v0::Constant::create(ov::element::i64, {1}, {ssm_state_size}), len);
+            state_end = std::make_shared<ov::op::v0::Negative>(state_rows);
+        } else {
+            state_end = ov::op::v0::Constant::create(ov::element::i64, {1}, {-ssm_state_size});
+        }
+        auto gdn_attn_output = std::make_shared<ov::op::v8::Slice>(
+            context.get_input(0), ov::op::v0::Constant::create(ov::element::i64, {1}, {0}), state_end,
+            ov::op::v0::Constant::create(ov::element::i64, {1}, {1}),
+            ov::op::v0::Constant::create(ov::element::i64, {1}, {2}));
+
+        auto input_shape = context.get_input_shape(0).to_shape();
+        input_node = std::make_shared<ov::op::v1::Reshape>(
+            gdn_attn_output,
+            ov::op::v0::Constant::create(
+                ov::element::i64, {4}, std::vector<int64_t>{1, -1, (int64_t) input_shape[2], (int64_t) input_shape[3]}),
+            false);
+
+    } else {
+        input_node = process_view_input_new(context, 0);
+    }
+    auto square = std::make_shared<ov::op::v1::Multiply>(input_node, input_node);
 
     auto mean = std::make_shared<ov::op::v1::ReduceMean>(
         square, ov::op::v0::Constant::create(ov::element::i64, ov::Shape{1}, {-1}), true);

ggml/src/ggml-openvino/openvino/op/rope.cpp

@@ -22,6 +22,7 @@
 #include <openvino/op/subtract.hpp>
 #include <openvino/op/transpose.hpp>
 #include <openvino/op/unsqueeze.hpp>
+#include <openvino/op/variadic_split.hpp>
 #include <vector>
 
 namespace ov {
@@ -40,6 +41,9 @@ OutputVector translate_rope(const NodeContext & context) {
     auto output_shape = context.get_output_shape().to_shape();
     int32_t * op_params = context.get_output_op_params();
     const int mode = op_case;
+    const int64_t head_dim = static_cast<int64_t>(output_shape[3]);
+    const int64_t configured_n_dims = static_cast<int64_t>(op_params[1]);
+    const int64_t n_dims = configured_n_dims == 0 ? head_dim : configured_n_dims;
 
     constexpr int TYPE_NORMAL = 0;
     constexpr int TYPE_NEOX = 1;
@@ -80,6 +84,9 @@ OutputVector translate_rope(const NodeContext & context) {
         data_node = std::make_shared<ov::op::v0::Convert>(data_node, ov::element::f32);
     }
 
+    FRONT_END_OP_CONVERSION_CHECK(n_dims > 0 && n_dims <= head_dim && (n_dims % 2 == 0),
+                                  "ROPE expects even n_dims in [1, head_dim]");
+
     // TODO(openvino-gpu-rope-fusion): TEMPORARY WORKAROUND - do NOT revert until the
     // OpenVINO GPU plugin is updated.
     //
@@ -94,13 +101,18 @@ OutputVector translate_rope(const NodeContext & context) {
     // be restored to the captured even/odd translation. Until then, keep both paths:
     // the active Flux rewrite here and the previous translation preserved below.
     if (mode == TYPE_NORMAL) {
+        auto axis_last = ov::op::v0::Constant::create(ov::element::i64, {1}, {-1});
+        auto zero = ov::op::v0::Constant::create(ov::element::i64, {1}, {0});
+        auto step_one = ov::op::v0::Constant::create(ov::element::i64, {1}, {1});
+
         // Emit the Flux-style interleaved-RoPE pattern so the GPU plugin's
         // RoPEFusionFlux matcher folds this subgraph into ov::op::internal::RoPE:
-        //   x_paired   = Reshape(x, [1, S, n_heads, head_size/2, 2])
+        //   x_paired   = Reshape(x_rot, [1, S, n_heads, n_dims/2, 2])
         //   x0, x1     = Split(x_paired, axis=-1, num_splits=2)
         //   x1_neg     = x1 * -1
-        //   x_rotated  = Reshape(Concat([x1_neg, x0], axis=-1), [1, S, n_heads, head_size])
-        //   y          = x * t_cos + x_rotated * t_sin
+        //   x_rotated  = Reshape(Concat([x1_neg, x0], axis=-1), [1, S, n_heads, n_dims])
+        //   y_rot      = x_rot * t_cos + x_rotated * t_sin
+        //   y          = Concat([y_rot, x_tail], axis=-1) if n_dims < head_dim
         // Mathematically equivalent to the even/odd Slice form below.
         //
         // RoPEFusionFlux requires rank_equals(4) on x, t_cos and t_sin. The cos/sin
@@ -114,15 +126,16 @@ OutputVector translate_rope(const NodeContext & context) {
                 std::vector<int64_t>{1, -1, (int64_t) output_shape[2], (int64_t) output_shape[3]});
             data_node = std::make_shared<ov::op::v1::Reshape>(data_node, r4_shape, false);
         }
-        const int64_t head_size = static_cast<int64_t>(output_shape[3]);
         const int64_t n_heads = static_cast<int64_t>(output_shape[2]);
-        const int64_t half = head_size / 2;
+        const int64_t half = n_dims / 2;
+        auto rot_end = ov::op::v0::Constant::create(ov::element::i64, {1}, {n_dims});
+        auto rot_data = std::make_shared<ov::op::v8::Slice>(data_node, zero, rot_end, step_one, axis_last);
 
         auto neg_one_f = ov::op::v0::Constant::create(data_node->get_element_type(), ov::Shape{}, {-1.0f});
 
-        auto paired_shape =
-            ov::op::v0::Constant::create(ov::element::i64, {5}, std::vector<int64_t>{1, -1, n_heads, half, 2});
-        auto x_paired = std::make_shared<ov::op::v1::Reshape>(data_node, paired_shape, false);
+        auto paired_shape = ov::op::v0::Constant::create(
+            ov::element::i64, {5}, std::vector<int64_t>{1, -1, n_heads, half, 2});
+        auto x_paired = std::make_shared<ov::op::v1::Reshape>(rot_data, paired_shape, false);
 
         auto split_axis = ov::op::v0::Constant::create(ov::element::i64, ov::Shape{}, {-1});
         auto data_split = std::make_shared<ov::op::v1::Split>(x_paired, split_axis, 2);
@@ -133,28 +146,38 @@ OutputVector translate_rope(const NodeContext & context) {
         auto x_rotated_paired = std::make_shared<ov::op::v0::Concat>(ov::OutputVector{x1_neg, x0}, -1);
 
         auto flat_shape =
-            ov::op::v0::Constant::create(ov::element::i64, {4}, std::vector<int64_t>{1, -1, n_heads, head_size});
-        auto x_rotated = std::make_shared<ov::op::v1::Reshape>(x_rotated_paired, flat_shape, false);
+            ov::op::v0::Constant::create(ov::element::i64, {4}, std::vector<int64_t>{1, -1, n_heads, n_dims});
+        auto x_rotated =
+            std::make_shared<ov::op::v1::Reshape>(x_rotated_paired, flat_shape, false);
 
-        // Expand cos/sin from [..., head_size/2] to [..., head_size] by repeating each
+        // Expand cos/sin from [..., n_dims/2] to [..., n_dims] by repeating each
         // entry twice. Use special_zero on the final Reshape so the seq dim passes
         // through dynamically. Final rank is 4 to satisfy the matcher's predicate.
         auto expand_cos_sin = [&](Output<Node> cs) {
-            auto cs_unsq =
-                std::make_shared<ov::op::v0::Unsqueeze>(cs, ov::op::v0::Constant::create(ov::element::i64, {1}, {-1}));
-            auto bcast_target =
-                ov::op::v0::Constant::create(ov::element::i64, {5}, std::vector<int64_t>{1, 1, 1, half, 2});
-            auto bcast =
-                std::make_shared<ov::op::v3::Broadcast>(cs_unsq, bcast_target, ov::op::BroadcastType::BIDIRECTIONAL);
-            auto flat = ov::op::v0::Constant::create(ov::element::i64, {4}, std::vector<int64_t>{0, 0, 0, head_size});
+            auto cs_unsq = std::make_shared<ov::op::v0::Unsqueeze>(
+                cs, ov::op::v0::Constant::create(ov::element::i64, {1}, {-1}));
+            auto bcast_target = ov::op::v0::Constant::create(
+                ov::element::i64, {5}, std::vector<int64_t>{1, 1, 1, half, 2});
+            auto bcast = std::make_shared<ov::op::v3::Broadcast>(
+                cs_unsq, bcast_target, ov::op::BroadcastType::BIDIRECTIONAL);
+            auto flat = ov::op::v0::Constant::create(ov::element::i64, {4}, std::vector<int64_t>{0, 0, 0, n_dims});
             return std::make_shared<ov::op::v1::Reshape>(bcast, flat, true);
         };
         Output<Node> cos_full = expand_cos_sin(cos_theta_node);
         Output<Node> sin_full = expand_cos_sin(sin_theta_node);
 
-        auto y1 = std::make_shared<ov::op::v1::Multiply>(data_node, cos_full);
+        auto y1 = std::make_shared<ov::op::v1::Multiply>(rot_data, cos_full);
         auto y2 = std::make_shared<ov::op::v1::Multiply>(x_rotated, sin_full);
-        res = std::make_shared<ov::op::v1::Add>(y1, y2);
+        auto rotated = std::make_shared<ov::op::v1::Add>(y1, y2);
+
+        if (n_dims < head_dim) {
+            auto tail_start = ov::op::v0::Constant::create(ov::element::i64, {1}, {n_dims});
+            auto tail_end = ov::op::v0::Constant::create(ov::element::i64, {1}, {head_dim});
+            auto tail = std::make_shared<ov::op::v8::Slice>(data_node, tail_start, tail_end, step_one, axis_last);
+            res = std::make_shared<ov::op::v0::Concat>(ov::OutputVector{rotated, tail}, -1);
+        } else {
+            res = rotated;
+        }
     }
     // PRESERVED PREVIOUS TRANSLATION - Re-enable this branch (and remove the Flux branch above) once
     // the GPU plugin's RoPE fusion is updated to recognize the even/odd Slice form;
@@ -196,8 +219,27 @@ OutputVector translate_rope(const NodeContext & context) {
     //         ov::element::i64, {4}, std::vector<int64_t>{1, -1, (int64_t) output_shape[2], (int64_t) output_shape[3]});
     //     res = std::make_shared<ov::op::v1::Reshape>(stack, data_shape, false);
     else if (mode == TYPE_NEOX) {
-        auto data_split = std::make_shared<ov::op::v1::Split>(
-            data_node, ov::op::v0::Constant::create(ov::element::i64, ov::Shape{}, {-1}), 2);
+        // In stateful mode the data arrives rank-3 ([S, n_heads, head_size]) while the
+        // cos/sin tables are rank-4 ([1, S, 1, n_dims/2]). The resulting mixed-rank
+        // broadcast in the Multiply below is miscomputed by the OpenVINO GPU plugin,
+        // corrupting the rotated Q/K. Lift the data to rank-4 ([1, S, n_heads, head_size])
+        // first so the RoPE Multiplies are equal-rank, matching the TYPE_NORMAL branch.
+        // Stateful RoPE already produced rank-4 output, so downstream attention is unaffected.
+        if (context.is_stateful()) {
+            auto r4_shape = ov::op::v0::Constant::create(
+                ov::element::i64, {4},
+                std::vector<int64_t>{1, -1, (int64_t) output_shape[2], (int64_t) output_shape[3]});
+            data_node = std::make_shared<ov::op::v1::Reshape>(data_node, r4_shape, false);
+        }
+        auto axis_last = ov::op::v0::Constant::create(ov::element::i64, ov::Shape{}, {-1});
+        std::vector<int64_t> split_lengths = {n_dims / 2, n_dims / 2};
+        if (n_dims < head_dim) {
+            split_lengths.push_back(head_dim - n_dims);
+        }
+
+        auto data_split = std::make_shared<ov::op::v1::VariadicSplit>(
+            data_node, axis_last,
+            ov::op::v0::Constant::create(ov::element::i64, {split_lengths.size()}, split_lengths));
         Output<Node> slice_data_node_0 = data_split->outputs()[0];
         Output<Node> slice_data_node_1 = data_split->outputs()[1];
 
@@ -209,16 +251,27 @@ OutputVector translate_rope(const NodeContext & context) {
             std::make_shared<ov::op::v1::Multiply>(slice_data_node_0, sin_theta_node),
             std::make_shared<ov::op::v1::Multiply>(slice_data_node_1, cos_theta_node));
 
-        res = std::make_shared<ov::op::v0::Concat>(ov::OutputVector{first_half_node, second_half_node}, -1);
+        if (n_dims < head_dim) {
+            Output<Node> tail = data_split->outputs()[2];
+            res = std::make_shared<ov::op::v0::Concat>(ov::OutputVector{first_half_node, second_half_node, tail}, -1);
+        } else {
+            res = std::make_shared<ov::op::v0::Concat>(ov::OutputVector{first_half_node, second_half_node}, -1);
+        }
     } else if (mode == TYPE_IMROPE) {
-        int64_t n_dims = data_node->get_output_partial_shape(0)[3].get_length();
         auto cos_sin_shape = std::make_shared<ov::op::v0::Constant>(ov::element::i64, ov::Shape{4},
                                                                     std::vector<int64_t>{1, -1, 1, (n_dims >> 1)});
         auto cos_reshaped = std::make_shared<ov::op::v1::Reshape>(cos_theta_node, cos_sin_shape, true);
         auto sin_reshaped = std::make_shared<ov::op::v1::Reshape>(sin_theta_node, cos_sin_shape, true);
 
         auto split_axis = ov::op::v0::Constant::create(ov::element::i64, ov::Shape{}, {3});
-        auto split_a = std::make_shared<ov::op::v1::Split>(data_node, split_axis, 2);
+        std::vector<int64_t> split_lengths = {n_dims / 2, n_dims / 2};
+        if (n_dims < head_dim) {
+            split_lengths.push_back(head_dim - n_dims);
+        }
+
+        auto split_a = std::make_shared<ov::op::v1::VariadicSplit>(
+            data_node, split_axis,
+            ov::op::v0::Constant::create(ov::element::i64, {split_lengths.size()}, split_lengths));
         auto x0 = split_a->output(0);
         auto x1 = split_a->output(1);
         auto mul_a = std::make_shared<ov::op::v1::Multiply>(x0, cos_reshaped);
@@ -229,7 +282,12 @@ OutputVector translate_rope(const NodeContext & context) {
         auto mul_d = std::make_shared<ov::op::v1::Multiply>(x1, cos_reshaped);
         auto add = std::make_shared<ov::op::v1::Add>(mul_c, mul_d);
 
-        res = std::make_shared<ov::op::v0::Concat>(ov::OutputVector{sub, add}, 3);
+        if (n_dims < head_dim) {
+            auto tail = split_a->output(2);
+            res = std::make_shared<ov::op::v0::Concat>(ov::OutputVector{sub, add, tail}, 3);
+        } else {
+            res = std::make_shared<ov::op::v0::Concat>(ov::OutputVector{sub, add}, 3);
+        }
     }
 
     if (res.get_element_type() != output_type) {

ggml/src/ggml-openvino/openvino/op/scale.cpp

@@ -2,9 +2,24 @@
 #include "../op_table.h"
 #include "../utils.h"
 
+#include <openvino/core/except.hpp>
 #include <openvino/op/add.hpp>
+#include <openvino/op/concat.hpp>
 #include <openvino/op/constant.hpp>
+#include <openvino/op/convert.hpp>
+#include <openvino/op/equal.hpp>
+#include <openvino/op/gather.hpp>
+#include <openvino/op/greater_eq.hpp>
+#include <openvino/op/if.hpp>
+#include <openvino/op/less.hpp>
+#include <openvino/op/logical_or.hpp>
 #include <openvino/op/multiply.hpp>
+#include <openvino/op/range.hpp>
+#include <openvino/op/reshape.hpp>
+#include <openvino/op/shape_of.hpp>
+#include <openvino/op/slice.hpp>
+#include <openvino/op/squeeze.hpp>
+#include <openvino/op/unsqueeze.hpp>
 #include <vector>
 
 namespace ov {
@@ -21,6 +36,36 @@ OutputVector translate_scale(const NodeContext & context) {
     memcpy(&bias, (float *) context.get_output_op_params() + 1, sizeof(float));
 
     auto scale_node = std::make_shared<ov::op::v0::Constant>(ov::element::f32, ov::Shape{}, std::vector<float>{scale});
+
+    if (context.get_op_case() == 1 && context.has_input("cache_rs_reset_len")) {
+        auto cache_rs_reset_idx = context.get_input("cache_rs_reset_idx");
+        auto cache_rs_reset_len = context.get_input("cache_rs_reset_len");
+
+        auto cache_rs = context.get_input(0);
+
+        auto cache_shape = std::make_shared<ov::op::v3::ShapeOf>(cache_rs, ov::element::i64);
+        auto n_slots_1d = std::make_shared<ov::op::v8::Gather>(
+            cache_shape, ov::op::v0::Constant::create(ov::element::i64, ov::Shape{1}, {2}),
+            ov::op::v0::Constant::create(ov::element::i64, ov::Shape{}, {0}));
+        auto n_slots = std::make_shared<ov::op::v0::Squeeze>(n_slots_1d);
+
+        auto iota = std::make_shared<ov::op::v4::Range>(
+            ov::op::v0::Constant::create(ov::element::i64, ov::Shape{}, {0}), n_slots,
+            ov::op::v0::Constant::create(ov::element::i64, ov::Shape{}, {1}), ov::element::i64);
+
+        auto idx_plus_len = std::make_shared<ov::op::v1::Add>(cache_rs_reset_idx, cache_rs_reset_len);
+        auto less_than_idx = std::make_shared<ov::op::v1::Less>(iota, cache_rs_reset_idx);
+        auto greater_equal_idx_plus_len = std::make_shared<ov::op::v1::GreaterEqual>(iota, idx_plus_len);
+        auto keep_mask = std::make_shared<ov::op::v1::LogicalOr>(less_than_idx, greater_equal_idx_plus_len);
+
+        auto keep_mask_f32 = std::make_shared<ov::op::v0::Convert>(keep_mask, ov::element::f32);
+        auto keep_mask_reshape = std::make_shared<ov::op::v0::Unsqueeze>(
+            keep_mask_f32, ov::op::v0::Constant::create(ov::element::i64, ov::Shape{1}, {1}));
+
+        auto cleared_cache_rs = std::make_shared<ov::op::v1::Multiply>(cache_rs, keep_mask_reshape);
+        return rename_outputs_with_suffix({cleared_cache_rs}, context.get_name());
+    }
+
     auto scaled = std::make_shared<ov::op::v1::Multiply>(context.get_input(0), scale_node);
 
     std::shared_ptr<ov::Node> res;

ggml/src/ggml-openvino/openvino/op/set.cpp

@@ -0,0 +1,76 @@
+#include "../node_context.h"
+#include "../op_table.h"
+#include "../utils.h"
+
+#include <cstdint>
+#include <openvino/frontend/exception.hpp>
+#include <openvino/op/add.hpp>
+#include <openvino/op/constant.hpp>
+#include <openvino/op/convert.hpp>
+#include <openvino/op/range.hpp>
+#include <openvino/op/reduce_prod.hpp>
+#include <openvino/op/reshape.hpp>
+#include <openvino/op/scatter_update.hpp>
+#include <openvino/op/shape_of.hpp>
+
+namespace ov {
+namespace frontend {
+namespace ggml {
+namespace op {
+
+// GGML SET writes src1 into a view of src0 and returns the updated tensor.
+OutputVector translate_set(const NodeContext & context) {
+    num_inputs_check(context, 2, 2);
+
+    auto dst = process_view_input_new(context, 0);
+    auto src = process_view_input_new(context, 1);
+
+    src = std::make_shared<ov::op::v0::Convert>(src, context.get_output_type());
+
+    const auto dst_stride = context.get_input_stride(0);
+    FRONT_END_OP_CONVERSION_CHECK(dst_stride.size() >= 4, "SET requires 4D destination strides");
+
+    const auto * op_params = reinterpret_cast<const uint32_t *>(context.get_output_op_params());
+    const size_t offset = static_cast<size_t>(op_params[3]);
+
+    const size_t elem_size = dst_stride.back();
+    FRONT_END_OP_CONVERSION_CHECK(elem_size != 0 && offset % elem_size == 0,
+                                  "SET offset must be aligned to destination element size");
+
+    const int64_t offset_elems = static_cast<int64_t>(offset / elem_size);
+
+    auto dst_flat = std::make_shared<ov::op::v1::Reshape>(
+        dst,
+        ov::op::v0::Constant::create(ov::element::i64, {1}, {-1}),
+        false);
+
+    auto src_flat = std::make_shared<ov::op::v1::Reshape>(
+        src,
+        ov::op::v0::Constant::create(ov::element::i64, {1}, {-1}),
+        false);
+
+    auto src_shape = std::make_shared<ov::op::v3::ShapeOf>(src_flat, ov::element::i64);
+    auto src_len = std::make_shared<ov::op::v1::ReduceProd>(
+        src_shape,
+        ov::op::v0::Constant::create(ov::element::i64, {1}, {0}),
+        false);
+
+    auto start = ov::op::v0::Constant::create(ov::element::i64, {}, {offset_elems});
+    auto stop = std::make_shared<ov::op::v1::Add>(start, src_len);
+    auto step = ov::op::v0::Constant::create(ov::element::i64, {}, {1});
+
+    auto indices = std::make_shared<ov::op::v4::Range>(start, stop, step, ov::element::i64);
+    auto axis = ov::op::v0::Constant::create(ov::element::i64, {}, {0});
+
+    auto updated_flat = std::make_shared<ov::op::v3::ScatterUpdate>(dst_flat, indices, src_flat, axis);
+
+    auto dst_shape = std::make_shared<ov::op::v3::ShapeOf>(dst, ov::element::i64);
+    auto res = std::make_shared<ov::op::v1::Reshape>(updated_flat, dst_shape, false);
+
+    return rename_outputs_with_suffix({res}, context.get_name());
+}
+
+}  // namespace op
+}  // namespace ggml
+}  // namespace frontend
+}  // namespace ov

ggml/src/ggml-openvino/openvino/op/set_rows.cpp

@@ -8,11 +8,13 @@
 #include <openvino/core/node.hpp>
 #include <openvino/core/node_output.hpp>
 #include <openvino/frontend/exception.hpp>
+#include <openvino/op/broadcast.hpp>
 #include <openvino/op/concat.hpp>
 #include <openvino/op/constant.hpp>
 #include <openvino/op/convert.hpp>
 #include <openvino/op/gather.hpp>
 #include <openvino/op/reshape.hpp>
+#include <openvino/op/scatter_elements_update.hpp>
 #include <openvino/op/scatter_update.hpp>
 #include <openvino/op/shape_of.hpp>
 #include <openvino/op/slice.hpp>
@@ -29,20 +31,17 @@ OutputVector translate_set_rows(const NodeContext & context) {
     num_inputs_check(context, 3, 3);
 
     auto data = process_view_input_new(context, 0);
-    auto indices = context.get_input(1);
-    auto dst = context.get_input(2);
+    auto indices = process_view_input_new(context, 1);
+    auto dst = process_view_input_new(context, 2);
 
     data = std::make_shared<ov::op::v0::Convert>(data, context.get_output_type());
 
-    auto row_size = context.get_input_shape(2)[3].get_length();
+    const auto indices_shape = context.get_input_shape(1);
+    const bool multidim_indices = indices_shape.rank().is_static() &&
+                                  indices_shape.rank().get_length() == 4 &&
+                                  ((indices_shape[1].is_static() && indices_shape[1].get_length() > 1) ||
+                                   (indices_shape[2].is_static() && indices_shape[2].get_length() > 1));
 
-    auto ind_squeezed =
-        std::make_shared<ov::op::v0::Squeeze>(indices, ov::op::v0::Constant::create(ov::element::i64, {3}, {0, 1, 2}));
-    auto data_reshaped = std::make_shared<ov::op::v1::Reshape>(
-        data,
-        ov::op::v0::Constant::create(ov::element::i64, {4},
-                                     {(int64_t) 1, (int64_t) 1, (int64_t) -1, (int64_t) row_size}),
-        false);
     auto axes = ov::op::v0::Constant::create(ov::element::i64, ov::Shape{}, {2});
 
     Output<Node> res;
@@ -53,11 +52,31 @@ OutputVector translate_set_rows(const NodeContext & context) {
         data = std::make_shared<ov::op::v1::Reshape>(
             data, ov::op::v0::Constant::create(ov::element::i64, {4}, {(int64_t) 1, (int64_t) -1, dim2, dim3}), false);
         res = std::make_shared<ov::op::v0::Concat>(OutputVector{dst, data}, concat_axis);
+    } else if (multidim_indices) {
+        auto updates_shape = std::make_shared<ov::op::v3::ShapeOf>(data, ov::element::i64);
+
+        auto indices_rank3 = std::make_shared<ov::op::v0::Squeeze>(
+            indices, ov::op::v0::Constant::create(ov::element::i64, {1}, {0}));
+        auto one = ov::op::v0::Constant::create(ov::element::i64, {1}, {1});
+        auto indices_rank4_shape = std::make_shared<ov::op::v0::Concat>(OutputVector{get_dimensions(updates_shape, {0, 1, 2}), one}, 0);
+        auto indices_rank4 = std::make_shared<ov::op::v1::Reshape>(indices_rank3, indices_rank4_shape, false);
+        auto broadcasted_indices = std::make_shared<ov::op::v3::Broadcast>(indices_rank4, updates_shape);
+
+        res = std::make_shared<ov::op::v3::ScatterElementsUpdate>(dst, broadcasted_indices, data, axes);
     } else {
+        auto row_size = context.get_input_shape(2)[3].get_length();
+        auto ind_squeezed = std::make_shared<ov::op::v0::Squeeze>(
+            indices, ov::op::v0::Constant::create(ov::element::i64, {3}, {0, 1, 2}));
+        auto data_reshaped = std::make_shared<ov::op::v1::Reshape>(
+            data,
+            ov::op::v0::Constant::create(ov::element::i64, {4},
+                                         {(int64_t) 1, (int64_t) 1, (int64_t) -1, (int64_t) row_size}),
+            false);
         res = std::make_shared<ov::op::v3::ScatterUpdate>(dst, ind_squeezed, data_reshaped, axes);
     }
 
-    if (auto dst_reshape = std::dynamic_pointer_cast<ov::op::v1::Reshape>(dst.get_node_shared_ptr())) {
+    auto dst_reshape = std::dynamic_pointer_cast<ov::op::v1::Reshape>(dst.get_node_shared_ptr());
+    if (!multidim_indices && dst_reshape) {
         // Fix the case of multiple sequences, reshape back to original shape [1, n_seq, ctx_per_seq, emb]
         // ctx_per_seq is not fixed due to llama-bench compatibility
         auto dst_shape_partial = dst_reshape->get_input_partial_shape(0);

ggml/src/ggml-openvino/openvino/op/solve_tri.cpp

@@ -0,0 +1,108 @@
+#include "../node_context.h"
+#include "../op_table.h"
+#include "../utils.h"
+
+#include <openvino/op/broadcast.hpp>
+#include <openvino/op/constant.hpp>
+#include <openvino/op/divide.hpp>
+#include <openvino/op/gather.hpp>
+#include <openvino/op/loop.hpp>
+#include <openvino/op/matmul.hpp>
+#include <openvino/op/scatter_update.hpp>
+#include <openvino/op/shape_of.hpp>
+#include <openvino/op/subtract.hpp>
+
+namespace ov {
+namespace frontend {
+namespace ggml {
+namespace op {
+
+// GGML SOLVE_TRI: solve Ax = B for lower-triangular A via forward substitution.
+// Currently only lower, right, non-unitriangular variant is implemented.
+//
+// ggml layout: A [n, n, B1, B2], B [k, n, B1, B2] → X [k, n, B1, B2]
+// OV layout:   A [B2, B1, n, n], B [B2, B1, n, k] → X [B2, B1, n, k]
+//
+// Forward substitution row i:
+//   x[i] = (b[i] - sum_{t<i} A[i,t]*x[t]) / A[i,i]
+//
+// Implemented as an OV Loop op iterating n times with a carried X accumulator.
+// Key insight: A is lower-triangular and X starts as zeros, so the full matmul
+//   A_row_i @ X_partial = sum_{t<i} A[i,t]*x[t] exactly (upper triangle of A
+//   is zero; unfilled rows of X are zero).
+OutputVector translate_solve_tri(const NodeContext & context) {
+    num_inputs_check(context, 2, 2);
+
+    auto A = context.get_input(0);  // [B2, B1, n, n]
+    auto B = context.get_input(1);  // [B2, B1, n, k]
+
+    auto A_shape = context.get_input_shape(0).to_shape();
+    int64_t n = static_cast<int64_t>(A_shape[2]);
+
+    // Initial X: zeros with shape of B
+    auto B_shape_node = std::make_shared<ov::op::v3::ShapeOf>(B, ov::element::i64);
+    auto zero_f32  = ov::op::v0::Constant::create(ov::element::f32, {}, {0.0f});
+    auto X_init    = std::make_shared<ov::op::v3::Broadcast>(zero_f32, B_shape_node);
+
+    // --- Loop body parameters ---
+    // body_iter: iteration counter injected by the Loop op (i64, shape {1})
+    auto body_iter = std::make_shared<ov::op::v0::Parameter>(ov::element::i64, ov::Shape{1});
+    auto body_X    = std::make_shared<ov::op::v0::Parameter>(ov::element::f32, ov::PartialShape::dynamic(4));
+    auto body_A    = std::make_shared<ov::op::v0::Parameter>(ov::element::f32, ov::PartialShape::dynamic(4));
+    auto body_B_p  = std::make_shared<ov::op::v0::Parameter>(ov::element::f32, ov::PartialShape::dynamic(4));
+
+    auto c_axis2        = ov::op::v0::Constant::create(ov::element::i64, {1}, {int64_t(2)});
+    auto c_axis3        = ov::op::v0::Constant::create(ov::element::i64, {1}, {int64_t(3)});
+    auto c_axis2_scalar = ov::op::v0::Constant::create(ov::element::i64, {}, {int64_t(2)});
+
+    // b_i      = B[..., i, :]      [B2, B1, 1, k]
+    auto b_i = std::make_shared<ov::op::v8::Gather>(body_B_p, body_iter, c_axis2);
+
+    // A_row_i  = A[..., i, :]      [B2, B1, 1, n]
+    auto A_row_i = std::make_shared<ov::op::v8::Gather>(body_A, body_iter, c_axis2);
+
+    // sum_i    = A_row_i @ X       [B2, B1, 1, k]
+    // (lower-tri zeros + unfilled-X zeros make this equal to the partial sum)
+    auto sum_i = std::make_shared<ov::op::v0::MatMul>(A_row_i, body_X, false, false);
+
+    // diag_i   = A[..., i, i]      [B2, B1, 1, 1]
+    auto diag_i = std::make_shared<ov::op::v8::Gather>(A_row_i, body_iter, c_axis3);
+
+    // x_i      = (b_i - sum_i) / diag_i    [B2, B1, 1, k]
+    auto x_i = std::make_shared<ov::op::v1::Divide>(
+        std::make_shared<ov::op::v1::Subtract>(b_i, sum_i), diag_i);
+
+    // X_updated: scatter x_i into body_X at row i along axis 2
+    auto X_updated = std::make_shared<ov::op::v3::ScatterUpdate>(body_X, body_iter, x_i, c_axis2_scalar);
+
+    auto body_cond = ov::op::v0::Constant::create(ov::element::boolean, ov::Shape{1}, {true});
+
+    auto body = std::make_shared<ov::Model>(
+        ov::OutputVector{body_cond, X_updated},
+        ov::ParameterVector{body_iter, body_X, body_A, body_B_p});
+
+    // --- Assemble Loop ---
+    auto trip_count = ov::op::v0::Constant::create(ov::element::i64, ov::Shape{1}, std::vector<int64_t>{n});
+    auto exec_cond  = ov::op::v0::Constant::create(ov::element::boolean, ov::Shape{1}, {true});
+
+    auto loop = std::make_shared<ov::op::v5::Loop>(trip_count, exec_cond);
+    loop->set_function(body);
+    // iter_counter_body_param_idx=0 (body_iter), exec_condition_body_result_idx=0 (body_cond)
+    loop->set_special_body_ports(ov::op::v5::Loop::SpecialBodyPorts{0, 0});
+
+    // Carried state: X feeds back from X_updated each iteration
+    loop->set_merged_input(body_X, X_init, X_updated);
+    // Invariant inputs passed through unchanged
+    loop->set_invariant_input(body_A,   A);
+    loop->set_invariant_input(body_B_p, B);
+
+    // Final output: value of X_updated after the last iteration
+    auto X_final = loop->get_iter_value(X_updated, -1);
+
+    return rename_outputs_with_suffix({X_final}, context.get_name());
+}
+
+}  // namespace op
+}  // namespace ggml
+}  // namespace frontend
+}  // namespace ov

ggml/src/ggml-openvino/openvino/op/sqr.cpp

@@ -0,0 +1,35 @@
+#include "../node_context.h"
+#include "../op_table.h"
+#include "../utils.h"
+
+#include <memory>
+#include <openvino/op/multiply.hpp>
+#include <openvino/op/sqrt.hpp>
+
+namespace ov {
+namespace frontend {
+namespace ggml {
+namespace op {
+
+OutputVector translate_sqr(const NodeContext & context) {
+    num_inputs_check(context, 1, 1);
+
+    auto input = process_view_input_new(context, 0);
+    auto res = std::make_shared<ov::op::v1::Multiply>(input, input);
+
+    return rename_outputs_with_suffix({res}, context.get_name());
+}
+
+OutputVector translate_sqrt(const NodeContext & context) {
+    num_inputs_check(context, 1, 1);
+
+    auto input = process_view_input_new(context, 0);
+    auto res = std::make_shared<ov::op::v0::Sqrt>(input);
+
+    return rename_outputs_with_suffix({res}, context.get_name());
+}
+
+}  // namespace op
+}  // namespace ggml
+}  // namespace frontend
+}  // namespace ov

ggml/src/ggml-openvino/openvino/op/ssm_conv.cpp

@@ -5,7 +5,9 @@
 #include <openvino/op/constant.hpp>
 #include <openvino/op/group_conv.hpp>
 #include <openvino/op/reshape.hpp>
+#include <openvino/op/squeeze.hpp>
 #include <openvino/op/transpose.hpp>
+#include <openvino/op/unsqueeze.hpp>
 
 namespace ov {
 namespace frontend {
@@ -21,15 +23,15 @@ OutputVector translate_ssm_conv(const NodeContext & context) {
     auto sx_shape = context.get_input_shape(0).to_shape();  // [1, n_s, d_inner, ncs]
     auto c_shape = context.get_input_shape(1).to_shape();   // [1, 1, d_inner, d_conv]
 
-    int64_t n_s = sx_shape[1];
+    // int64_t n_s  = sx_shape[1];
     int64_t d_inner = sx_shape[2];
-    int64_t ncs = sx_shape[3];  // d_conv - 1 + n_t
-    int64_t d_conv = c_shape[3];
-    int64_t n_t = ncs - d_conv + 1;
+    // int64_t ncs  = sx_shape[3];  // d_conv - 1 + n_t
+    int64_t d_conv  = c_shape[3];
+    // int64_t n_t  = ncs - d_conv + 1;
 
     // Reshape sx from [1, n_s, d_inner, ncs] to [n_s, d_inner, ncs] for 1D GroupConvolution
-    auto sx_new_shape = ov::op::v0::Constant::create(ov::element::i64, {3}, std::vector<int64_t>{n_s, d_inner, ncs});
-    auto sx_reshaped = std::make_shared<ov::op::v1::Reshape>(sx, sx_new_shape, false);
+    auto sx_reshaped =
+        std::make_shared<ov::op::v0::Squeeze>(sx, ov::op::v0::Constant::create(ov::element::i64, {1}, {0}));
 
     // Reshape c from [1, 1, d_inner, d_conv] to [d_inner, 1, 1, d_conv]
     // GroupConvolution filter: [groups, out_channels/groups, in_channels/groups, kernel_size]
@@ -47,8 +49,8 @@ OutputVector translate_ssm_conv(const NodeContext & context) {
     auto transposed = std::make_shared<ov::op::v1::Transpose>(conv, perm);
 
     // Reshape to output shape [1, n_s, n_t, d_inner]
-    auto out_shape = ov::op::v0::Constant::create(ov::element::i64, {4}, std::vector<int64_t>{1, n_s, n_t, d_inner});
-    auto res = std::make_shared<ov::op::v1::Reshape>(transposed, out_shape, false);
+    auto res =
+        std::make_shared<ov::op::v0::Unsqueeze>(transposed, ov::op::v0::Constant::create(ov::element::i64, {1}, {0}));
 
     return rename_outputs_with_suffix({res}, context.get_name());
 }

ggml/src/ggml-openvino/openvino/op/tri.cpp

@@ -0,0 +1,82 @@
+#include "../node_context.h"
+#include "../op_table.h"
+#include "../utils.h"
+
+#include <openvino/op/constant.hpp>
+#include <openvino/op/greater.hpp>
+#include <openvino/op/greater_eq.hpp>
+#include <openvino/op/less.hpp>
+#include <openvino/op/less_eq.hpp>
+#include <openvino/op/range.hpp>
+#include <openvino/op/reshape.hpp>
+#include <openvino/op/select.hpp>
+
+namespace ov {
+namespace frontend {
+namespace ggml {
+namespace op {
+
+// GGML TRI zeroes out elements outside a triangular region of a square matrix.
+// The type param (stored in op_params[0]) maps to ggml_tri_type:
+//   0 = UPPER_DIAG : keep where col >= row
+//   1 = UPPER      : keep where col >  row
+//   2 = LOWER_DIAG : keep where col <= row
+//   3 = LOWER      : keep where col <  row
+//
+// In OV layout (ggml [ne0, ne1, ne2, ne3] → OV [ne3, ne2, ne1, ne0]):
+//   ggml dim 0 (ne0, cols) → OV axis 3
+//   ggml dim 1 (ne1, rows) → OV axis 2
+// The matrix is square so ne0 == ne1.
+OutputVector translate_tri(const NodeContext & context) {
+    num_inputs_check(context, 1, 1);
+
+    auto x = context.get_input(0);  // OV shape: [ne3, ne2, ne1, ne0]
+
+    int32_t tri_type = context.get_output_op_params()[0];
+
+    auto shape = context.get_input_shape(0).to_shape();
+    int64_t n = static_cast<int64_t>(shape[3]);  // ne0 == ne1
+
+    // Build index range [0, 1, ..., n-1]
+    auto start = ov::op::v0::Constant::create(ov::element::i64, {}, {int64_t(0)});
+    auto stop  = ov::op::v0::Constant::create(ov::element::i64, {}, {n});
+    auto step  = ov::op::v0::Constant::create(ov::element::i64, {}, {int64_t(1)});
+    auto range = std::make_shared<ov::op::v4::Range>(start, stop, step, ov::element::i64);
+
+    // col_idx shape [1, 1, 1, n] — broadcasts over batch and row dims
+    auto col_shape = ov::op::v0::Constant::create(ov::element::i64, {4}, std::vector<int64_t>{1, 1, 1, n});
+    auto col_idx   = std::make_shared<ov::op::v1::Reshape>(range, col_shape, false);
+
+    // row_idx shape [1, 1, n, 1] — broadcasts over batch and col dims
+    auto row_shape = ov::op::v0::Constant::create(ov::element::i64, {4}, std::vector<int64_t>{1, 1, n, 1});
+    auto row_idx   = std::make_shared<ov::op::v1::Reshape>(range, row_shape, false);
+
+    // Build boolean mask: true where element should be kept
+    std::shared_ptr<ov::Node> mask;
+    switch (tri_type) {
+        case 0:  // UPPER_DIAG: col >= row
+            mask = std::make_shared<ov::op::v1::GreaterEqual>(col_idx, row_idx);
+            break;
+        case 1:  // UPPER: col > row
+            mask = std::make_shared<ov::op::v1::Greater>(col_idx, row_idx);
+            break;
+        case 2:  // LOWER_DIAG: col <= row
+            mask = std::make_shared<ov::op::v1::LessEqual>(col_idx, row_idx);
+            break;
+        case 3:  // LOWER: col < row
+            mask = std::make_shared<ov::op::v1::Less>(col_idx, row_idx);
+            break;
+        default:
+            throw std::runtime_error("translate_tri: invalid tri_type " + std::to_string(tri_type));
+    }
+
+    auto zero = ov::op::v0::Constant::create(ov::element::f32, {}, {0.0f});
+    auto res  = std::make_shared<ov::op::v1::Select>(mask, x, zero);
+
+    return rename_outputs_with_suffix({res}, context.get_name());
+}
+
+}  // namespace op
+}  // namespace ggml
+}  // namespace frontend
+}  // namespace ov

ggml/src/ggml-openvino/openvino/op/view.cpp

@@ -1,8 +1,11 @@
 #include "../op_table.h"
 #include "../utils.h"
 
+#include <openvino/op/concat.hpp>
 #include <openvino/op/constant.hpp>
+#include <openvino/op/gather.hpp>
 #include <openvino/op/reshape.hpp>
+#include <openvino/op/shape_of.hpp>
 #include <openvino/op/slice.hpp>
 #include <set>
 
@@ -15,6 +18,123 @@ OutputVector translate_view(const NodeContext & context) {
     num_inputs_check(context, 1, 1);
 
     if (!context.is_static()) {
+        // On the stateless/non-static path VIEW is normally a no-op (consumers re-slice).
+        // EXCEPTION: the MoE expert aggregation slices each expert plane out of
+        // ffn_moe_weighted [n_embd, n_expert_used, n_tokens] with ggml_view_2d and then
+        // sums the planes with a chain of ADDs (llama-graph.cpp). Those ADDs read this
+        // VIEW node directly from the tensor map and do NOT re-slice, so a no-op here
+        // makes every plane the full tensor and the expert sum collapses. Materialize the
+        // single-expert slice here. Gated by name (ffn_moe_weighted...view) so it can't
+        // affect any other view.
+        const std::string & vname = context.get_name();
+        if (vname.find("ffn_moe_weighted") != std::string::npos) {
+            auto src_ps = context.get_input_shape(0);
+            auto dst_ps = context.get_output_shape();
+            if (src_ps.rank().is_static() && dst_ps.rank().is_static() && src_ps.rank() == dst_ps.rank() &&
+                src_ps.is_static() && dst_ps.is_static()) {
+                auto sst = context.get_input_stride(0);
+                auto dst = context.get_output_stride();
+                size_t voff = context.get_output_op_offset();
+                auto ss = src_ps.to_shape();
+                auto dd = dst_ps.to_shape();
+                const size_t nd = ss.size();
+                if (sst.size() == nd && dst.size() == nd) {
+                    // Map each dst axis of size>1 to a src axis with equal (size,stride);
+                    // the unmatched src axis of size>1 is the indexed expert axis.
+                    // dst_to_src[d] records which src axis each dst axis came from, so we can
+                    // later pull the dynamic (token) dim from the right source axis at runtime.
+                    std::vector<bool> used(nd, false);
+                    std::vector<int> dst_to_src(nd, -1);
+                    bool ok = true;
+                    for (size_t d = 0; d < nd; ++d) {
+                        if (dd[d] == 1) {
+                            continue;
+                        }
+                        int found = -1;
+                        for (size_t s = 0; s < nd; ++s) {
+                            if (!used[s] && ss[s] == dd[d] && sst[s] == dst[d]) {
+                                found = (int) s;
+                                break;
+                            }
+                        }
+                        if (found < 0) {
+                            ok = false;
+                            break;
+                        }
+                        used[found] = true;
+                        dst_to_src[d] = found;
+                    }
+                    int dropped = -1;
+                    if (ok) {
+                        for (size_t s = 0; s < nd; ++s) {
+                            if (!used[s] && ss[s] > 1) {
+                                if (dropped >= 0) {
+                                    ok = false;
+                                    break;
+                                }
+                                dropped = (int) s;
+                            }
+                        }
+                    }
+                    if (ok && dropped >= 0) {
+                        const size_t dstr = sst[dropped];
+                        const int64_t dsz = (int64_t) ss[dropped];
+                        if (dstr > 0 && voff % dstr == 0) {
+                            const int64_t sel = (int64_t) (voff / dstr);
+                            if (sel >= 0 && sel < dsz) {
+                                ov::Output<ov::Node> sl = std::make_shared<ov::op::v8::Slice>(
+                                    context.get_input(0),
+                                    ov::op::v0::Constant::create(ov::element::i64, {1}, {sel}),
+                                    ov::op::v0::Constant::create(ov::element::i64, {1}, {sel + 1}),
+                                    ov::op::v0::Constant::create(ov::element::i64, {1}, {1}),
+                                    ov::op::v0::Constant::create(ov::element::i64, {1}, {dropped}));
+                                // Build the reshape target from the (concrete) dst shape, but
+                                // keep the dynamic token axis dynamic instead of freezing it
+                                // to the captured n_tokens. Without this the constant dst
+                                // shape bakes in the prefill token count and the static value
+                                // flows downstream, turning every later decoder layer static
+                                // (the GPU in-place-concat KV-cache bug). The token axis is
+                                // PERMUTED between the sliced input and the dst (e.g. input
+                                // [1,tok,expert,emb] -> dst [1,1,tok,emb]), so special_zero
+                                // (which copies the same-position dim) is not enough: pull the
+                                // dynamic dim from the correct SOURCE axis via ShapeOf+Gather
+                                // and place it at the dst token position.
+                                const int32_t dyn = context.get_op_dynamic_dim();  // output ggml axis, -1 if none
+                                int dst_ov_axis = (dyn != -1) ? (3 - (int) dyn) : -1;  // get_shape() reverses ggml order
+                                int src_ov_axis = (dst_ov_axis >= 0 && dst_ov_axis < (int) nd)
+                                                      ? dst_to_src[dst_ov_axis]
+                                                      : -1;
+                                if (dst_ov_axis >= 0 && src_ov_axis >= 0) {
+                                    // target = concat of per-axis scalars; the token axis is a
+                                    // runtime Gather of the slice's shape, the rest are constants.
+                                    auto sl_shape = std::make_shared<ov::op::v3::ShapeOf>(sl, ov::element::i64);
+                                    auto tok_dim = std::make_shared<ov::op::v8::Gather>(
+                                        sl_shape,
+                                        ov::op::v0::Constant::create(ov::element::i64, {1}, {src_ov_axis}),
+                                        ov::op::v0::Constant::create(ov::element::i64, {}, {0}));
+                                    ov::OutputVector parts;
+                                    for (int a = 0; a < (int) nd; ++a) {
+                                        if (a == dst_ov_axis) {
+                                            parts.push_back(tok_dim);
+                                        } else {
+                                            parts.push_back(ov::op::v0::Constant::create(
+                                                ov::element::i64, {1}, {(int64_t) dd[a]}));
+                                        }
+                                    }
+                                    auto dc = std::make_shared<ov::op::v0::Concat>(parts, 0);
+                                    auto rs = std::make_shared<ov::op::v1::Reshape>(sl, dc, false);
+                                    return rename_outputs_with_suffix({rs}, context.get_name());
+                                }
+                                auto dc = ov::op::v0::Constant::create(
+                                    ov::element::i64, {nd}, std::vector<int64_t>(dd.begin(), dd.end()));
+                                auto rs = std::make_shared<ov::op::v1::Reshape>(sl, dc, false);
+                                return rename_outputs_with_suffix({rs}, context.get_name());
+                            }
+                        }
+                    }
+                }
+            }
+        }
         return {context.get_input(0)};
     }
 

ggml/src/ggml-openvino/openvino/op_table.cpp

@@ -4,10 +4,13 @@
 
 #include <openvino/op/add.hpp>
 #include <openvino/op/divide.hpp>
+#include <openvino/op/exp.hpp>
 #include <openvino/op/gather.hpp>
 #include <openvino/op/gelu.hpp>
 #include <openvino/op/matmul.hpp>
 #include <openvino/op/multiply.hpp>
+#include <openvino/op/negative.hpp>
+#include <openvino/op/sigmoid.hpp>
 #include <openvino/op/subtract.hpp>
 #include <openvino/op/tanh.hpp>
 
@@ -18,12 +21,13 @@ namespace ggml {
 std::unordered_map<std::string, CreatorFunction> get_supported_ops() {
     using namespace ov::op;
     return {
-        {"GGML_OP_ADD",             op::translate_1to1_match_2_inputs<v1::Add>     },
+        {"GGML_OP_ADD",             op::translate_add                              },
         {"GGML_OP_ADD1",            op::translate_1to1_match_2_inputs<v1::Add>     },
         {"GGML_OP_ADD_ID",          op::translate_add_id                           },
         {"GGML_OP_CONCAT",          op::translate_concat                           },
         {"GGML_OP_CONT",            op::translate_cont                             },
         {"GGML_OP_DIV",             op::translate_div                              },
+        {"GGML_OP_FILL",            op::translate_fill                             },
         {"GGML_OP_GET_ROWS",        op::translate_get_rows                         },
         {"GGML_OP_IM2COL",          op::translate_im2col                           },
         {"GGML_OP_MUL",             op::translate_1to1_match_2_inputs<v1::Multiply>},
@@ -37,14 +41,20 @@ std::unordered_map<std::string, CreatorFunction> get_supported_ops() {
         {"GGML_OP_SUM_ROWS",        op::translate_sum_rows                         },
         {"GGML_OP_ROPE",            op::translate_rope                             },
         {"GGML_OP_SCALE",           op::translate_scale                            },
+        {"GGML_OP_SQR",             op::translate_sqr                              },
+        {"GGML_OP_SQRT",            op::translate_sqrt                             },
         {"GGML_OP_SOFT_MAX",        op::translate_soft_max                         },
         {"GGML_OP_ARGSORT",         op::translate_argsort                          },
         {"GGML_OP_SUB",             op::translate_1to1_match_2_inputs<v1::Subtract>},
         {"GGML_OP_TRANSPOSE",       op::translate_transpose                        },
         {"GGML_UNARY_OP_GELU",      op::translate_1to1_match_1_input<v7::Gelu>     },
+        {"GGML_UNARY_OP_SIGMOID",   op::translate_1to1_match_1_input<v0::Sigmoid>  },
         {"GGML_UNARY_OP_SILU",      op::translate_unary_silu                       },
         {"GGML_UNARY_OP_SOFTPLUS",  op::translate_unary_softplus                   },
         {"GGML_UNARY_OP_TANH",      op::translate_1to1_match_1_input<v0::Tanh>     },
+        {"GGML_UNARY_OP_SIGMOID",   op::translate_1to1_match_1_input<v0::Sigmoid>  },
+        {"GGML_UNARY_OP_EXP",       op::translate_1to1_match_1_input<v0::Exp>      },
+        {"GGML_UNARY_OP_NEG",       op::translate_1to1_match_1_input<v0::Negative> },
         {"GGML_OP_VIEW",            op::translate_view                             },
         {"GGML_GLU_OP_SWIGLU",      op::translate_glu_swiglu                       },
         {"GGML_GLU_OP_SWIGLU_OAI",  op::translate_glu_swiglu_oai                   },
@@ -57,6 +67,13 @@ std::unordered_map<std::string, CreatorFunction> get_supported_ops() {
         {"GGML_OP_SSM_CONV",        op::translate_ssm_conv                         },
         {"GGML_OP_GATED_DELTA_NET", op::translate_gated_delta_net                  },
         {"GGML_OP_REPEAT",          op::translate_repeat                           },
+        {"GGML_OP_CUMSUM",          op::translate_cumsum                           },
+        {"GGML_OP_FILL",            op::translate_fill                             },
+        {"GGML_OP_DIAG",            op::translate_diag                             },
+        {"GGML_OP_TRI",             op::translate_tri                              },
+        {"GGML_OP_SET",             op::translate_set                              },
+        // solve_tri has accuracy issues on GPU
+        // {"GGML_OP_SOLVE_TRI",       op::translate_solve_tri                        },
     };
 }
 

ggml/src/ggml-openvino/openvino/op_table.h

@@ -10,10 +10,12 @@ namespace op {
 
 #define GGML_OP_CONVERTER(op) OutputVector op(const NodeContext & context)
 
+GGML_OP_CONVERTER(translate_add);
 GGML_OP_CONVERTER(translate_cont);
 GGML_OP_CONVERTER(translate_concat);
 GGML_OP_CONVERTER(translate_add_id);
 GGML_OP_CONVERTER(translate_div);
+GGML_OP_CONVERTER(translate_fill);
 GGML_OP_CONVERTER(translate_get_rows);
 GGML_OP_CONVERTER(translate_im2col);
 GGML_OP_CONVERTER(translate_mulmat);
@@ -24,8 +26,10 @@ GGML_OP_CONVERTER(translate_rms_norm);
 GGML_OP_CONVERTER(translate_norm);
 GGML_OP_CONVERTER(translate_l2_norm);
 GGML_OP_CONVERTER(translate_sum_rows);
+GGML_OP_CONVERTER(translate_sqr);
 GGML_OP_CONVERTER(translate_rope);
 GGML_OP_CONVERTER(translate_scale);
+GGML_OP_CONVERTER(translate_sqrt);
 GGML_OP_CONVERTER(translate_unary_silu);
 GGML_OP_CONVERTER(translate_unary_softplus);
 GGML_OP_CONVERTER(translate_soft_max);
@@ -43,6 +47,12 @@ GGML_OP_CONVERTER(translate_pad);
 GGML_OP_CONVERTER(translate_ssm_conv);
 GGML_OP_CONVERTER(translate_gated_delta_net);
 GGML_OP_CONVERTER(translate_repeat);
+GGML_OP_CONVERTER(translate_cumsum);
+GGML_OP_CONVERTER(translate_fill);
+GGML_OP_CONVERTER(translate_set);
+GGML_OP_CONVERTER(translate_diag);
+GGML_OP_CONVERTER(translate_tri);
+GGML_OP_CONVERTER(translate_solve_tri);
 
 }  // namespace op
 

ggml/src/ggml-openvino/openvino/pass/mark_dequantization_subgraph.h

@@ -0,0 +1,44 @@
+// Copyright (C) 2018-2026 Intel Corporation
+// SPDX-License-Identifier: Apache-2.0
+//
+// Local mirror of OpenVINO's ov::pass::MarkDequantization pass declaration.
+//
+// The pass body is provided by the linked libopenvino.so; only the declaration is needed here so
+// we can register it directly in our own TranslateSession::apply_transformations (same approach as
+// MarkCompressedFloatConstants's local mirror in mark_decompression_convert_constant_folding.h). This
+// lets us mark our GatherMatmul dequantization chain with disable_constant_folding regardless of the
+// CPU/GPU plugin's own is_decompression_multiply() consumer allowlist.
+// The class layout must stay in sync with
+//   openvino/src/common/transformations/include/transformations/low_precision/mark_dequantization_subgraph.hpp
+
+#pragma once
+
+#include "openvino/core/type/element_type.hpp"
+#include "openvino/core/visibility.hpp"
+#include "openvino/pass/matcher_pass.hpp"
+
+#ifdef OPENVINO_STATIC_LIBRARY
+#    define TRANSFORMATIONS_API
+#else
+#    ifdef IMPLEMENT_OPENVINO_API
+#        define TRANSFORMATIONS_API OPENVINO_CORE_EXPORTS
+#    else
+#        define TRANSFORMATIONS_API OPENVINO_CORE_IMPORTS
+#    endif  // IMPLEMENT_OPENVINO_API
+#endif      // OPENVINO_STATIC_LIBRARY
+
+namespace ov {
+namespace pass {
+
+class TRANSFORMATIONS_API MarkDequantization;
+
+}  // namespace pass
+}  // namespace ov
+
+class ov::pass::MarkDequantization : public MatcherPass {
+public:
+    OPENVINO_MATCHER_PASS_RTTI("MarkDequantization")
+    explicit MarkDequantization(const element::TypeVector & precisions,
+                                bool fold_subtract_const = false,
+                                bool fold_multiply_const = true);
+};

ggml/src/ggml-openvino/openvino/translate_session.cpp

@@ -1,18 +1,23 @@
 #include "translate_session.h"
 
+#include "ggml-impl.h"
+#include "ggml-openvino/ggml-openvino-extra.h"
 #include "ggml-openvino/openvino/node_context.h"
 #include "ggml-openvino/openvino/utils.h"
 #include "input_model.h"
 #include "pass/mark_decompression_convert_constant_folding.h"
+#include "pass/mark_dequantization_subgraph.h"
 #include "pass/squeeze_matmul.h"
 #include "rt_info/weightless_caching_attributes.hpp"
 
+#include <algorithm>
 #include <cstdint>
 #include <cstdlib>
 #include <map>
 #include <memory>
 #include <openvino/core/node.hpp>
 #include <openvino/core/preprocess/pre_post_process.hpp>
+#include <openvino/core/shape.hpp>
 #include <openvino/core/type/element_type.hpp>
 #include <openvino/op/add.hpp>
 #include <openvino/op/broadcast.hpp>
@@ -35,6 +40,7 @@
 #include <openvino/op/unsqueeze.hpp>
 #include <openvino/pass/constant_folding.hpp>
 #include <openvino/pass/make_stateful.hpp>
+#include <sstream>
 
 namespace ov {
 namespace frontend {
@@ -44,6 +50,28 @@ using namespace ov::op;
 
 namespace {
 
+std::shared_ptr<ov::op::v0::Parameter> create_parameter(const std::string & name,
+                                                        const ModelInputInfo & input_info) {
+    auto param_node = std::make_shared<ov::op::v0::Parameter>(input_info.type, input_info.shape);
+    param_node->set_friendly_name(name);
+    param_node->output(0).get_tensor().set_names({name});
+    return param_node;
+}
+
+std::shared_ptr<ov::Node> create_extra_input(const std::string & name, const ModelExtraInputInfo & input_info) {
+    if (input_info.is_parameter) {
+        auto param_node = std::make_shared<ov::op::v0::Parameter>(input_info.type, input_info.shape);
+        param_node->set_friendly_name(name);
+        param_node->output(0).get_tensor().set_names({name});
+        return param_node;
+    }
+
+    auto constant = std::make_shared<ov::op::v0::Constant>(input_info.type, input_info.shape,
+                                                          std::vector<int64_t>{input_info.value});
+    constant->set_friendly_name(name);
+    return constant;
+}
+
 ov::pass::MakeStateful::ParamResPairs get_kv_param_res_pairs(
     const std::shared_ptr<ov::Model> & model,
     const std::map<std::string, std::string> & kv_param_res_names) {
@@ -177,33 +205,34 @@ std::shared_ptr<Model> TranslateSession::translate_graph(const frontend::InputMo
     std::shared_ptr<GgmlDecoder> ggml_model_decoder = ggml_model->get_model_decoder();
 
     for (const auto & it : ggml_model_decoder->get_model_inputs()) {
-        params.push_back(std::dynamic_pointer_cast<ov::op::v0::Parameter>(it.second));
-        (*tensor_map)[it.first] = it.second;
+        auto param_node = create_parameter(it.first, it.second);
+        params.push_back(param_node);
+        (*tensor_map)[it.first] = param_node;
     }
 
     for (const auto & it : ggml_model_decoder->get_model_extra_inputs()) {
-        if (std::dynamic_pointer_cast<ov::op::v0::Parameter>(it.second)) {
-            params.push_back(std::dynamic_pointer_cast<ov::op::v0::Parameter>(it.second));
+        auto input_node = create_extra_input(it.first, it.second);
+        if (it.second.is_parameter) {
+            params.push_back(std::dynamic_pointer_cast<ov::op::v0::Parameter>(input_node));
         }
-        (*tensor_map)[it.first] = it.second;
+        (*tensor_map)[it.first] = input_node;
     }
 
     for (const auto & it : ggml_model_decoder->get_model_weights()) {
         (*tensor_map)[it.first] = it.second;
     }
 
-    auto node_visitor = [&](std::shared_ptr<GgmlDecoder> decoder, int node_idx) {
+    auto translate_node = [&](const std::shared_ptr<GgmlDecoder> & decoder, int node_idx) {
         auto operation_type = decoder->get_op_type(node_idx);
         if (operation_type == "GGML_OP_NONE") {
-            return;
+            return ov::OutputVector{};
         }
 
-        ov::OutputVector converted_outputs;
         auto it = m_translator_map.find(operation_type);
         FRONT_END_OP_CONVERSION_CHECK(it != m_translator_map.end(), "Translation for operation type ", operation_type,
                                       " is not implemented.");
         NodeContext node_context(decoder, tensor_map, node_idx, this);
-        converted_outputs = it->second(node_context);
+        ov::OutputVector converted_outputs = it->second(node_context);
 
         const auto & node_output_names = decoder->get_output_names(node_idx);
         FRONT_END_OP_CONVERSION_CHECK(node_output_names.size() == converted_outputs.size(), "Number of ",
@@ -216,6 +245,46 @@ std::shared_ptr<Model> TranslateSession::translate_graph(const frontend::InputMo
                 (*tensor_map)[output_name] = converted_outputs[i];
             }
         }
+        return converted_outputs;
+    };
+
+    // To handle cases like this
+    // 3: [ 18432,     1,     1,     1] RESHAPE              cache_r_l0 (reshaped)#3
+    //     [ 18432,     1,     1,     1]            0: NONE        cache_r_l0
+    // 4: [     0,     1,     1,     1] VIEW                 cache_r_l0 (reshaped) (view)#4
+    //     [ 18432,     1,     1,     1]            0: RESHAPE     cache_r_l0 (reshaped)#3
+    // 5: [     0,     1,     1,     1] SCALE                cache_r_l0 (reshaped) (view) (view)#5
+    //     [     0,     1,     1,     1]            0: VIEW        cache_r_l0 (reshaped) (view)#4
+    // 6: [     1,     1,     1,     1] VIEW                  (view)#6
+    //     [     1,     1,     1,     1]            0: NONE        leaf_5
+    // 7: [ 18432,     1,     1,     1] GET_ROWS             conv_states-0#7
+    //     [ 18432,     1,     1,     1]            0: RESHAPE     cache_r_l0 (reshaped)#3
+    //     [     1,     1,     1,     1]            1: VIEW         (view)#6
+    // The scale is in-place which modifies cache_r_l0 (reshaped)#3
+    // The translation of scale overwrites cache_r in the tensor_map,
+    // but we also need to overwrite the old cache_r_l0 (reshaped)#3
+    auto refresh_inplace_aliases = [&](const std::shared_ptr<GgmlDecoder> & decoder, int inplace_node_idx,
+                                       const std::string & view_src_name) {
+        for (int node_idx = 0; node_idx < inplace_node_idx; node_idx++) {
+            if (decoder->is_view_like_alias_of(node_idx, view_src_name)) {
+                translate_node(decoder, node_idx);
+            }
+        }
+    };
+
+    auto node_visitor = [&](std::shared_ptr<GgmlDecoder> decoder, int node_idx) {
+        auto converted_outputs = translate_node(decoder, node_idx);
+        if (converted_outputs.empty()) {
+            return;
+        }
+        const auto inplace_src = decoder->get_inplace_op_src(node_idx);
+        if (inplace_src.empty()) {
+            return;
+        }
+        if (converted_outputs[0].get_node_shared_ptr() != nullptr) {
+            (*tensor_map)[inplace_src] = converted_outputs[0];
+        }
+        refresh_inplace_aliases(decoder, node_idx, inplace_src);
     };
 
     if (!m_naive) {
@@ -231,6 +300,46 @@ std::shared_ptr<Model> TranslateSession::translate_graph(const frontend::InputMo
         results.push_back(result);
     }
 
+    // Debug-only hook: GGML_OPENVINO_DEBUG_NODE=<name1>,<name2>,... adds extra
+    // Result nodes for arbitrary intermediate tensors (looked up by name in
+    // tensor_map), on top of the real model outputs above. These debug
+    // Results are deliberately NOT added to ggml_decoder's model outputs, so
+    // the caller (ov_graph_compute_dynamic in utils.cpp) will not bind them
+    // to any ggml tensor buffer -- OpenVINO allocates its own tensor for
+    // them. This avoids the risk of reading a ggml buffer that has since
+    // been overwritten by a later in-place op (ggml aggressively reuses
+    // buffers), which can happen if trying to inspect an intermediate value
+    // via GGML_OPENVINO_DEBUG_OUTPUT by hacking it into a real output.
+    //
+    // tensor_map keys are usually the plain ggml tensor name (e.g. "embd"),
+    // but tensors that are recomputed multiple times in the same cgraph
+    // (GGML_TENSOR_FLAG_COMPUTE) are disambiguated with a "#<hash>" suffix
+    // (e.g. "cache_k_l0#4853", see get_tensor_ov_name()) which is not
+    // predictable ahead of time. To keep the env var usable, a requested
+    // name is matched either exactly, or as the "name" part before "#" of a
+    // suffixed key (first match wins; ambiguous requests should include the
+    // full "name#hash" form seen in a previous run's log/dump).
+    if (const char * debug_nodes = ggml_openvino_getenv_str("GGML_OPENVINO_DEBUG_NODE")) {
+        std::stringstream ss(debug_nodes);
+        std::string name;
+        while (std::getline(ss, name, ',')) {
+            auto it = tensor_map->find(name);
+            if (it == tensor_map->end()) {
+                it = std::find_if(tensor_map->begin(), tensor_map->end(), [&](const auto & entry) {
+                    return entry.first.compare(0, name.size(), name) == 0 && entry.first.size() > name.size() &&
+                           entry.first[name.size()] == '#';
+                });
+            }
+            if (it == tensor_map->end()) {
+                GGML_LOG_WARN("GGML_OPENVINO_DEBUG_NODE: node '%s' not found in tensor map, skipping\n", name.c_str());
+                continue;
+            }
+            auto result = std::make_shared<v0::Result>(it->second);
+            result->set_friendly_name("__debug_" + it->first);
+            results.push_back(result);
+        }
+    }
+
     ov::ParameterVector used_params;
     for (const auto & param : params) {
         if (!param->output(0).get_target_inputs().empty()) {
@@ -257,10 +366,13 @@ std::shared_ptr<Model> TranslateSession::translate_graph(const frontend::InputMo
     //
     // Small constants (< 16 elements) are excluded since they may be introduced by
     // optimization patterns and the overhead is negligible.
+    //
+    // Note: use shape_size() rather than byte_size()/element_type().size() - GatherMatmul's default
+    // bias is a Constant(element::dynamic, Shape{0}), whose element_type().size() is 0 and would
+    // divide by zero.
     size_t offset = 0;
     for (auto & node : resulting_model->get_ordered_ops()) {
-        if (auto cnst = ov::as_type_ptr<ov::op::v0::Constant>(node);
-            cnst && cnst->get_byte_size() / cnst->get_element_type().size() >= 16) {
+        if (auto cnst = ov::as_type_ptr<ov::op::v0::Constant>(node); cnst && ov::shape_size(cnst->get_shape()) >= 16) {
             auto & rt_info = cnst->get_rt_info();
             if (rt_info.find(ov::WeightlessCacheAttribute::get_type_info_static()) == rt_info.end()) {
                 rt_info[ov::WeightlessCacheAttribute::get_type_info_static()] =
@@ -277,6 +389,12 @@ std::shared_ptr<Model> TranslateSession::apply_transformations(std::shared_ptr<M
         ov::pass::Manager manager;
         manager.set_per_pass_validation(true);
         manager.register_pass<ov::pass::MarkCompressedFloatConstants>();
+        // Marks the Convert/Subtract/Multiply nodes of our GatherMatmul dequantization chain
+        // (make_int4_weights/make_int8_weights, for_gather_matmul=true) with disable_constant_folding,
+        // so it survives ConstantFolding regardless of whether the target plugin's own
+        // is_decompression_multiply() recognizes GatherMatmul as a valid consumer.
+        manager.register_pass<ov::pass::MarkDequantization>(
+            std::vector<ov::element::Type>{ov::element::u8, ov::element::i8, ov::element::u4, ov::element::i4});
 
         if (ggml_model_decoder->is_stateful()) {
             const auto kv_param_res_names = ggml_model_decoder->get_kv_param_res_names();
@@ -289,21 +407,11 @@ std::shared_ptr<Model> TranslateSession::apply_transformations(std::shared_ptr<M
         }
         manager.run_passes(model);
         if (ggml_model_decoder->is_stateful()) {
-            auto output_names = ggml_model_decoder->get_model_output_names();
-            std::map<std::string, int> model_output_indexes;
-            for (size_t i = 0; i < output_names.size(); i++) {
-                model_output_indexes.insert(std::make_pair(output_names[i], i));
-            }
             ov::preprocess::PrePostProcessor ppp(model);
             for (size_t i = 0; i < model->get_output_size(); i++) {
-                auto output_friendly_name = model->output(i).get_node_shared_ptr()->get_friendly_name();
-                auto output_id = model_output_indexes[output_friendly_name];
                 auto model_output_shape = model->output(i).get_partial_shape();
-                auto decoder_output_shape = ggml_model_decoder->get_output_shape(output_id);
-                if (model_output_shape.rank().is_static() && decoder_output_shape.rank().is_static() &&
-                    model_output_shape.rank().get_length() + 1 == decoder_output_shape.rank().get_length() &&
-                    decoder_output_shape[0].is_static() && decoder_output_shape[0].get_length() == 1) {
-                    ppp.output(i).postprocess().custom([](const ov::Output<ov::Node> & node) {
+                if (model_output_shape.rank().is_static() && model_output_shape.rank().get_length() == 3) {
+                    ppp.output(i).postprocess().custom([](const ov::Output<ov::Node>& node) {
                         auto axes = ov::op::v0::Constant::create(ov::element::i32, ov::Shape{1}, {0});
                         return std::make_shared<ov::op::v0::Unsqueeze>(node, axes);
                     });

ggml/src/ggml-openvino/openvino/utils.cpp

@@ -17,6 +17,7 @@
 #include <openvino/op/reshape.hpp>
 #include <openvino/op/shape_of.hpp>
 #include <openvino/op/sin.hpp>
+#include <openvino/op/slice.hpp>
 #include <openvino/op/split.hpp>
 #include <openvino/op/squeeze.hpp>
 #include <openvino/op/subtract.hpp>
@@ -195,7 +196,24 @@ std::pair<ov::Output<Node>, ov::Output<Node>> make_sin_cos(int32_t * rope_params
                 std::make_shared<ov::op::v0::Constant>(ov::element::f32, ov::Shape{1, 1, 1, factor.size()}, factor);
         }
         if (rope_freqs_weight) {
-            freq_factors = std::make_shared<ov::op::v1::Divide>(freq_factors, rope_freqs_weight);
+            Output<Node> rope_factors = std::make_shared<ov::op::v8::Slice>(
+                rope_freqs_weight,
+                ov::op::v0::Constant::create(ov::element::i64, {1}, {0}),
+                ov::op::v0::Constant::create(ov::element::i64, {1}, {(int64_t) n_dims_half}),
+                ov::op::v0::Constant::create(ov::element::i64, {1}, {1}),
+                ov::op::v0::Constant::create(ov::element::i64, {1}, {rope_freqs_weight->get_output_partial_shape(0).rank().get_length() - 1}));
+            if (stateful) {
+                rope_factors = std::make_shared<ov::op::v1::Reshape>(
+                    rope_factors,
+                    ov::op::v0::Constant::create(ov::element::i64, {3}, {(int64_t) 1, (int64_t) 1, (int64_t) n_dims_half}),
+                    false);
+            } else {
+                rope_factors = std::make_shared<ov::op::v1::Reshape>(
+                    rope_factors,
+                    ov::op::v0::Constant::create(ov::element::i64, {4}, {(int64_t) 1, (int64_t) 1, (int64_t) 1, (int64_t) n_dims_half}),
+                    false);
+            }
+            freq_factors = std::make_shared<ov::op::v1::Divide>(freq_factors, rope_factors);
         }
 
         auto theta_extrap = std::make_shared<ov::op::v1::Multiply>(freq_factors, inp_pos);
@@ -234,23 +252,30 @@ std::pair<ov::Output<Node>, ov::Output<Node>> make_sin_cos(int32_t * rope_params
     return std::make_pair(sin_theta, cos_theta);
 }
 
-ov::Output<ov::Node> process_view_input(const NodeContext & context, int input_index, int slice_len) {
-    // Only works for VIEW operations that slice at the lowest dimension
-    // If the VIEW also reshape the result, `slice_len` should be provided
+ov::Output<ov::Node> process_view_input(const NodeContext & context, int input_index, int slice_len, int axis) {
+    // Only works for VIEW operations that does a non-strided slice with optinal reshape on the slice result.
+    // The function only does the slice part, the reshape (if any) should be handled by the caller.
+    // Default axis is -1, which means slicing the last dimension.
+    // If the VIEW reshapes the result, `slice_len` should be provided
     auto input = context.get_input(input_index);
     auto * op_params = (size_t *) context.get_input_op_params(input_index);
-    auto src1_stride = context.get_input_stride(input_index);
+    auto src_stride = context.get_input_stride(input_index);
 
-    int64_t split_addr = op_params[0] / src1_stride[3];
+    int64_t slice_start = op_params[0] / src_stride[3];
     if (slice_len == 0) {
         slice_len = context.get_input_shape(input_index)[3].get_length();
     }
-    int64_t slice_end = split_addr + slice_len;
+    int64_t slice_end = slice_start + slice_len;
 
-    auto begin = ov::op::v0::Constant::create(ov::element::i64, {1}, {split_addr});
+    auto begin = ov::op::v0::Constant::create(ov::element::i64, {1}, {slice_start});
     auto end = ov::op::v0::Constant::create(ov::element::i64, {1}, {slice_end});
     auto stride = ov::op::v0::Constant::create(ov::element::i64, {1}, {1});
-    auto axes = ov::op::v0::Constant::create(ov::element::i64, {1}, {context.is_stateful() ? 2 : 3});
+    ov::Output<ov::Node> axes;
+    if (axis == -1) {
+        axes = ov::op::v0::Constant::create(ov::element::i64, {1}, {context.is_stateful() ? 2 : 3});
+    } else {
+        axes = ov::op::v0::Constant::create(ov::element::i64, {1}, {axis});
+    }
     auto sliced = std::make_shared<ov::op::v8::Slice>(input, begin, end, stride, axes);
     return sliced;
 }
@@ -267,17 +292,40 @@ ov::Output<ov::Node> process_view_input_new(const NodeContext & context, int inp
 
     // If translate_view already resolved this VIEW (produced a Slice), the input
     // will already have the expected shape — skip re-slicing.
+    //
+    // Two notions of "matches" are accepted per axis:
+    //   - both dims static and equal, OR
+    //   - both dims dynamic.
+    // The dynamic case matters for the MoE expert-plane views: translate_view now emits a
+    // DYNAMIC-token slice (so the token dim is not frozen). An all-static-only check would
+    // see the dynamic token dim, decide the shapes "don't match", and fall through to
+    // re-slice/flatten the already-resolved view (a Reshape to the full flattened
+    // n_expert_used*n_embd tail, which then conflicts with the single-plane input). Treat a
+    // dynamic-vs-dynamic axis as matching so the already-resolved view is reused as-is.
+    //
+    // A third case matters for split-model MoE fragments: translate_view resolves the
+    // expert-plane view against the fragment's INPUT parameter. When the graph is split
+    // the token axis of that parameter may already be concrete (static n_tokens) even
+    // though get_view_input_ov_shape() still reports it as dynamic (-1). The resolved
+    // view is then static [1,1,n_tokens,n_embd] while `expected` is [1,1,?,n_embd].
+    // An "expected dynamic, actual static" axis is a valid concretization of the SAME
+    // resolved view, so treat it as matching too. Falling through to process_single_view
+    // here would re-slice/re-flatten the already-resolved single-plane view against the
+    // recorded (multi-plane) source strides and emit a constant-target Reshape whose baked
+    // dims no longer divide the concretized input -> "dimensions do not evenly divide".
     auto expected_ov_shape = context.get_view_input_ov_shape(input_index, 0);
     auto actual_shape = input.get_partial_shape();
     if (expected_ov_shape.rank().is_static() && actual_shape.rank().is_static() &&
         expected_ov_shape.rank() == actual_shape.rank()) {
         bool shapes_match = true;
         for (int64_t i = 0; i < expected_ov_shape.rank().get_length(); ++i) {
-            if (!expected_ov_shape[i].is_static() || !actual_shape[i].is_static()) {
-                shapes_match = false;
-                break;
-            }
-            if (expected_ov_shape[i] != actual_shape[i]) {
+            const bool both_dynamic = expected_ov_shape[i].is_dynamic() && actual_shape[i].is_dynamic();
+            const bool both_static_equal = expected_ov_shape[i].is_static() && actual_shape[i].is_static() &&
+                                           expected_ov_shape[i] == actual_shape[i];
+            // expected dynamic, actual static: the resolved view already carries the
+            // concrete size for this fragment; reuse it rather than re-materializing.
+            const bool expected_dyn_actual_static = expected_ov_shape[i].is_dynamic() && actual_shape[i].is_static();
+            if (!both_dynamic && !both_static_equal && !expected_dyn_actual_static) {
                 shapes_match = false;
                 break;
             }
@@ -758,6 +806,41 @@ ov::Output<ov::Node> process_view_input_new(const NodeContext & context, int inp
         return current;
     };
 
+    // Special case: ggml collapses VIEW-of-VIEW chains so that `view_offs` is always an
+    // ABSOLUTE offset from the true root allocation, regardless of how many VIEW levels
+    // are in between (see ggml_new_tensor_impl). `src[0]` is still the immediate op-graph
+    // parent though, which can be a DIFFERENT (already narrowed) VIEW with the SAME ggml
+    // shape as this one but a different absolute offset -- e.g. a per-layer deepstack
+    // slice `view_2d(embd, n_embd, n_tokens, embd->nb[1], layer*n_embd*sizeof(float))`
+    // whose src[0] ("embd") is itself already a zero-offset VIEW of the true root (the
+    // padded embedding). Chaining through "embd" here would try to re-slice an already
+    // 2-narrowed tensor using a root-relative offset, going out of bounds and silently
+    // falling back to a no-op (returning the wrong, already-resolved sibling slice).
+    // Detect this (same shape as the immediate src, but different absolute offset) and
+    // re-slice directly from the untouched root using the innermost view's absolute
+    // offset against the ROOT's own shape/stride instead of chaining through src[0].
+    {
+        auto innermost_offset = context.get_view_input_offset(input_index, 0);
+        auto innermost_src_offset = context.get_view_input_src_offset(input_index, 0);
+        auto innermost_shape = context.get_view_input_ggml_shape(input_index, 0);
+        auto innermost_src_shape = context.get_view_input_src_ggml_shape(input_index, 0);
+        if (innermost_offset != innermost_src_offset && innermost_shape == innermost_src_shape) {
+            size_t root_view_idx = view_input_size - 1;
+            auto root_ggml_shape = context.get_view_input_src_ggml_shape(input_index, root_view_idx);
+            auto root_stride = context.get_view_input_src_stride(input_index, root_view_idx);
+            auto root_offset = context.get_view_input_src_offset(input_index, root_view_idx);
+            auto root_ov_shape = context.get_view_input_src_ov_shape(input_index, root_view_idx);
+            auto root_name = context.get_view_input_src_name(input_index, root_view_idx);
+            auto innermost_stride = context.get_view_input_stride(input_index, 0);
+            auto innermost_ov_shape = context.get_view_input_ov_shape(input_index, 0);
+            auto innermost_name = context.get_view_input_name(input_index, 0);
+
+            return process_single_view(input, innermost_offset, innermost_stride, innermost_shape, innermost_ov_shape,
+                                       innermost_name, root_offset, root_stride, root_ggml_shape, root_ov_shape,
+                                       root_name);
+        }
+    }
+
     // Process views from the base tensor (last) to the current view (first)
     // Start with the base tensor
     ov::Output<ov::Node> current = input;

ggml/src/ggml-openvino/openvino/utils.h

@@ -62,7 +62,7 @@ std::pair<ov::Output<Node>, ov::Output<Node>> make_sin_cos(int32_t * rope_params
                                                            bool imrope = false,
                                                            bool stateful = false);
 
-ov::Output<ov::Node> process_view_input(const NodeContext & context, int input_index, int slice_len = 0);
+ov::Output<ov::Node> process_view_input(const NodeContext & context, int input_index, int slice_len = 0, int axis = -1);
 
 ov::Output<ov::Node> process_view_input_new(const NodeContext & context, int input_index);
 

ggml/src/ggml-openvino/utils.cpp

@@ -4,6 +4,7 @@
 #include "ggml-openvino-extra.h"
 #include "ggml-openvino/ggml-decoder.h"
 #include "ggml.h"
+#include "model-cache.h"
 #include "openvino/frontend.h"
 #include "openvino/input_model.h"
 
@@ -134,6 +135,20 @@ static std::optional<ov::Tensor> try_make_kv_sliced_tensor(std::shared_ptr<GgmlO
     return ov::Tensor(ggml_decoder->get_ov_type(ggml_tensor), sliced_shape, ggml_tensor->data);
 }
 
+static uint64_t ggml_openvino_model_cache_extra_cfg(const std::string & device, bool stateful) {
+    const char * manual_gqa_env = ggml_openvino_getenv_str("GGML_OPENVINO_MANUAL_GQA_ATTN");
+    const bool manual_gqa_enabled = manual_gqa_env != nullptr ?
+                                        ggml_openvino_getenv_int("GGML_OPENVINO_MANUAL_GQA_ATTN") > 0 :
+                                        device == "GPU";
+
+    uint64_t extra_cfg = 0;
+    extra_cfg = extra_cfg * 131 + (stateful ? 1u : 0u);
+    extra_cfg = extra_cfg * 131 + (ggml_openvino_reduce_compile_mem_enabled() ? 1u : 0u);
+    extra_cfg = extra_cfg * 131 + (ggml_openvino_getenv_int("GGML_OPENVINO_DISABLE_KV_SLICE") ? 1u : 0u);
+    extra_cfg = extra_cfg * 131 + (manual_gqa_enabled ? 1u : 0u);
+    return extra_cfg;
+}
+
 ov::Tensor create_ov_output_tensor(std::shared_ptr<GgmlOvDecoder> ggml_decoder,
                                    std::shared_ptr<ov::InferRequest> infer_request,
                                    int output_index,
@@ -170,8 +185,24 @@ enum ggml_status ov_graph_compute_dynamic(ggml_cgraph * cgraph, std::shared_ptr<
     const auto & stateful = r_ctx->stateful;
     static auto is_static = false;
 
+    static const bool cache_disabled = ggml_openvino_getenv_int("GGML_OPENVINO_DISABLE_CACHE");
+
+    // is_model_splitted is O(n_nodes^2) plus a create_weight_nodes scan and takes ~20 ms
+    // on a Llama-1B decode graph. It is called once per graph_compute invocation but the
+    // graph shape is identical across all decode steps, so memoize by graph_key: compute
+    // graph_key first (a few hundred us), and if the same key is already in decoder_cache
+    // we know the graph is not splitted (only not-splitted graphs get inserted there).
+    graph_key key(cgraph);
+    bool key_seen = false;
+    if (!cache_disabled) {
+        std::lock_guard<std::mutex> map_lock(r_ctx->ctx_mutex);
+        key_seen = r_ctx->decoder_cache.find(key) != r_ctx->decoder_cache.end();
+    }
+
+    bool model_is_splitted = key_seen ? false : is_model_splitted(cgraph);
+
     if (is_naive(cgraph)) {
-        if (!is_model_splitted(cgraph)) {
+        if (!model_is_splitted) {
             return naive_compute(cgraph, core, device, config);
         }
     }
@@ -184,8 +215,7 @@ enum ggml_status ov_graph_compute_dynamic(ggml_cgraph * cgraph, std::shared_ptr<
     ComputeParams c_params;
     std::tie(m_params, c_params) = GgmlOvDecoder::compute_llm_params(cgraph, is_static);
 
-    graph_key key(cgraph);
-    static const bool cache_enabled = !ggml_openvino_getenv_int("GGML_OPENVINO_DISABLE_CACHE");
+    const bool cache_enabled = !model_is_splitted && !cache_disabled;
     bool cache_hit = false;
 
     int64_t decoder_end_time;
@@ -205,6 +235,7 @@ enum ggml_status ov_graph_compute_dynamic(ggml_cgraph * cgraph, std::shared_ptr<
             if (cache_hit) {
                 entry = it->second;
             } else {
+                r_ctx->clear_caches_locked();
                 auto mutex = std::make_shared<std::mutex>();
                 entry = std::make_shared<decoder_runtime_ctx>(mutex);
                 r_ctx->decoder_cache[key] = entry;
@@ -286,48 +317,171 @@ enum ggml_status ov_graph_compute_dynamic(ggml_cgraph * cgraph, std::shared_ptr<
             conversion_end_time = decoder_end_time;
             compile_end_time = decoder_end_time;
         } else {
+            // Fail fast: a cache-miss recompile feeds weight data to compile_model, but
+            // GGML_OPENVINO_RELEASE_WEIGHTS (or GGML_OPENVINO_MEMORY_OPTIMIZE on GPU)
+            // may have already dropped the host weight pages
+            // (they would read as zeros). That mode requires stable graph shapes.
+            if (ggml_openvino_weight_buffers_released()) {
+                GGML_ABORT(
+                    "ggml-openvino: a new graph needs to be compiled but host weight buffers were already "
+                    "released via GGML_OPENVINO_RELEASE_WEIGHTS/GGML_OPENVINO_MEMORY_OPTIMIZE. This mode requires "
+                    "stable graph shapes; disable host weight release for dynamic workloads.");
+            }
             if (cache_enabled) {
                 std::lock_guard<std::mutex> map_lock(r_ctx->ctx_mutex);
                 r_ctx->infer_request_cache.erase(key);
             }
-            bool model_is_splitted = is_model_splitted(cgraph);
+
+            // Frontend-level compiled-model cache (GGML_OPENVINO_COMPILED_MODEL_CACHE_DIR): if this model
+            // was compiled before, import the saved blob and skip requant + convert +
+            // compile. Only the dynamic single-model path is cached (split models compile
+            // two graphs and are left to the plugin-level ov::cache_dir). The decoder is
+            // still needed for I/O mapping, but can be built without weight nodes since
+            // the weights are baked into the imported CompiledModel.
+            const std::string model_cache_dir = ggml_openvino_model_cache_dir();
+            uint64_t model_fp = 0;
+            std::string blob_path, manifest_path;
+            bool imported = false;
+            // When the frontend model cache is active it supersedes the plugin-level
+            // ov::cache_dir: a blob exported from a model compiled WITH cache_dir cannot
+            // be re-imported (import returns an uninitialized model). Strip cache_dir /
+            // cache_mode from the config used for the cached compile and the import.
+            ov::AnyMap mc_config = config;
+            if (!model_cache_dir.empty()) {
+                mc_config.erase("CACHE_DIR");
+                mc_config.erase("CACHE_MODE");
+            }
+            if (!model_cache_dir.empty() && !model_is_splitted) {
+                const uint64_t extra_cfg = ggml_openvino_model_cache_extra_cfg(device, stateful);
+                model_fp = ggml_openvino_model_fingerprint(cgraph, device, /*fa=*/true, m_params.rope_params,
+                                                           15, extra_cfg);
+                blob_path = ggml_openvino_model_cache_blob_path(model_cache_dir, model_fp);
+                manifest_path = ggml_openvino_model_cache_manifest_path(model_cache_dir, model_fp);
+
+                std::ifstream blob_in(blob_path, std::ios::binary);
+                bool blob_ok = blob_in.is_open();
+                bool manifest_ok = blob_ok && ggml_openvino_model_cache_verify_manifest(manifest_path, cgraph, model_fp);
+                if (blob_ok && manifest_ok) {
+                    int64_t import_start = ggml_time_us();
+                    try {
+                        ov::CompiledModel cm;
+                        auto remote_context = ggml_openvino_get_remote_context();
+                        if (remote_context.has_value()) {
+                            cm = core.import_model(blob_in, remote_context.value(), mc_config);
+                        } else {
+                            cm = core.import_model(blob_in, device, mc_config);
+                        }
+                        // Lightweight decoder: names-only weight map (membership is all the
+                        // decoder needs; weights live in the imported model).
+                        std::map<std::string, std::shared_ptr<ov::Node>> weight_names;
+                        for (const auto & n : GgmlOvDecoder::collect_weight_names(cgraph)) {
+                            weight_names[n] = nullptr;
+                        }
+                        ggml_decoder = std::make_shared<GgmlOvDecoder>(cgraph, m_params, c_params, weight_names,
+                                                                       is_static, stateful, model_is_splitted);
+                        infer_request = std::make_shared<ov::InferRequest>(cm.create_infer_request());
+                        entry->ptr = ggml_decoder;
+                        // Names must match the decoder's ggml-tensor keys. The non-cached
+                        // path keys off Parameter/Result *friendly names* (set by the
+                        // frontend); export_model preserves these, and each compiled-model
+                        // port's node is exactly that Parameter/Result. Use the port nodes
+                        // directly (NOT get_runtime_model(), whose graph differs and is
+                        // unsafe to deref this way).
+                        for (const auto & p : cm.inputs()) {
+                            ov_input_names.push_back(p.get_node()->get_friendly_name());
+                        }
+                        for (const auto & o : cm.outputs()) {
+                            ov_output_names.push_back(o.get_node()->get_friendly_name());
+                        }
+                        imported = true;
+                        if (ggml_openvino_getenv_int("GGML_OPENVINO_PROFILING")) {
+                            GGML_LOG_INFO("  - Model cache import time: %.3f ms \n",
+                                          (ggml_time_us() - import_start) / 1000.0);
+                        }
+                        GGML_LOG_INFO("ggml-openvino: model cache HIT %s\n", blob_path.c_str());
+                    } catch (const std::exception & e) {
+                        GGML_LOG_WARN("ggml-openvino: model cache import failed (%s), recompiling\n", e.what());
+                        imported = false;
+                    }
+                }
+            }
 
             std::shared_ptr<ov::Model> model;
-            auto model_weights = GgmlOvDecoder::create_weight_nodes(cgraph);
-
-            ggml_decoder = std::make_shared<GgmlOvDecoder>(cgraph, m_params, c_params, model_weights, is_static,
-                                                           stateful, model_is_splitted);
-            decoder_end_time = ggml_time_us();
-
-            auto input_model = std::make_shared<ov::frontend::ggml::InputModel>(ggml_decoder);
-            model = ov::frontend::ggml::FrontEnd::convert(input_model);
-            ggml_decoder->clear_model_weights();
-            conversion_end_time = ggml_time_us();
-
-            if (ggml_openvino_getenv_int("GGML_OPENVINO_DUMP_IR")) {
-                char timestamped_filename[64];
-                auto timestamp = (long long) ggml_time_us();
-                snprintf(timestamped_filename, sizeof(timestamped_filename), "model_%lld.xml", timestamp);
-                ov::serialize(model, timestamped_filename);
-            }
-
-            ov::CompiledModel compiled_model;
-            auto remote_context = ggml_openvino_get_remote_context();
-            if (remote_context.has_value()) {
-                compiled_model = core.compile_model(model, remote_context.value(), config);
+            if (imported) {
+                decoder_end_time = conversion_end_time = compile_end_time = ggml_time_us();
             } else {
-                compiled_model = core.compile_model(model, device, config);
-            }
-            compile_end_time = ggml_time_us();
-            infer_request = std::make_shared<ov::InferRequest>(compiled_model.create_infer_request());
-            entry->ptr = ggml_decoder;
+                auto model_weights = GgmlOvDecoder::create_weight_nodes(cgraph);
 
-            for (const auto & ov_param : model->get_parameters()) {
-                ov_input_names.push_back(ov_param->get_friendly_name());
-            }
-            for (const auto & ov_output : model->get_results()) {
-                ov_output_names.push_back(ov_output->get_friendly_name());
-            }
+                ggml_decoder = std::make_shared<GgmlOvDecoder>(cgraph, m_params, c_params, model_weights, is_static,
+                                                               stateful, model_is_splitted);
+                decoder_end_time = ggml_time_us();
+
+                auto input_model = std::make_shared<ov::frontend::ggml::InputModel>(ggml_decoder);
+                model = ov::frontend::ggml::FrontEnd::convert(input_model);
+                ggml_decoder->clear_model_weights();
+                conversion_end_time = ggml_time_us();
+
+                if (ggml_openvino_getenv_int("GGML_OPENVINO_DUMP_IR")) {
+                    char timestamped_filename[64];
+                    auto timestamp = (long long) ggml_time_us();
+                    snprintf(timestamped_filename, sizeof(timestamped_filename), "model_%lld.xml", timestamp);
+                    ov::serialize(model, timestamped_filename);
+                }
+
+                // Use the cache-stripped config when the frontend model cache is active, so
+                // the resulting CompiledModel can be exported and later re-imported.
+                const ov::AnyMap & compile_config = model_cache_dir.empty() ? config : mc_config;
+                ov::CompiledModel compiled_model;
+                auto remote_context = ggml_openvino_get_remote_context();
+                if (remote_context.has_value()) {
+                    compiled_model = core.compile_model(model, remote_context.value(), compile_config);
+                } else {
+                    compiled_model = core.compile_model(model, device, compile_config);
+                }
+                compile_end_time = ggml_time_us();
+
+                // Export to the frontend model cache for next time. Publish the blob first,
+                // then the manifest, so a cache hit only sees fully written artifacts.
+                if (!model_cache_dir.empty() && !model_is_splitted && model_fp != 0) {
+                    try {
+                        const std::string blob_tmp = blob_path + ".tmp";
+                        const std::string manifest_tmp = manifest_path + ".tmp";
+                        if (ggml_openvino_model_cache_write_manifest(manifest_tmp, cgraph, model_fp)) {
+                            std::ofstream blob_out(blob_tmp, std::ios::binary | std::ios::trunc);
+                            if (blob_out.is_open()) {
+                                compiled_model.export_model(blob_out);
+                                blob_out.close();
+                                if (blob_out.good()) {
+                                    if (std::rename(blob_tmp.c_str(), blob_path.c_str()) == 0 &&
+                                        std::rename(manifest_tmp.c_str(), manifest_path.c_str()) == 0) {
+                                        GGML_LOG_INFO("ggml-openvino: model cache WROTE %s\n", blob_path.c_str());
+                                    } else {
+                                        std::remove(blob_tmp.c_str());
+                                        std::remove(manifest_tmp.c_str());
+                                    }
+                                } else {
+                                    std::remove(blob_tmp.c_str());
+                                    std::remove(manifest_tmp.c_str());
+                                }
+                            } else {
+                                std::remove(manifest_tmp.c_str());
+                            }
+                        }
+                    } catch (const std::exception & e) {
+                        GGML_LOG_WARN("ggml-openvino: model cache export failed: %s\n", e.what());
+                    }
+                }
+
+                infer_request = std::make_shared<ov::InferRequest>(compiled_model.create_infer_request());
+                entry->ptr = ggml_decoder;
+
+                for (const auto & ov_param : model->get_parameters()) {
+                    ov_input_names.push_back(ov_param->get_friendly_name());
+                }
+                for (const auto & ov_output : model->get_results()) {
+                    ov_output_names.push_back(ov_output->get_friendly_name());
+                }
+            }  // end non-imported (compile) path
 
             if (cache_enabled) {
                 std::lock_guard<std::mutex> map_lock(r_ctx->ctx_mutex);
@@ -358,7 +512,17 @@ enum ggml_status ov_graph_compute_dynamic(ggml_cgraph * cgraph, std::shared_ptr<
         }
 
         for (size_t i = 0; i < ov_output_names.size(); i++) {
-            auto * ggml_tensor = ggml_decoder->get_model_outputs().at(ov_output_names[i]);
+            // Debug-only outputs added via GGML_OPENVINO_DEBUG_NODE (see
+            // translate_session.cpp) have no corresponding ggml tensor; leave
+            // them unbound so OpenVINO allocates its own tensor for them,
+            // rather than aliasing a ggml buffer that may be overwritten by a
+            // later in-place op before we get to read it.
+            const auto & model_outputs = ggml_decoder->get_model_outputs();
+            auto model_output_it = model_outputs.find(ov_output_names[i]);
+            if (model_output_it == model_outputs.end()) {
+                continue;
+            }
+            auto * ggml_tensor = model_output_it->second;
             if (ggml_nbytes(ggml_tensor) == 0) {
                 continue;
             }
@@ -370,7 +534,8 @@ enum ggml_status ov_graph_compute_dynamic(ggml_cgraph * cgraph, std::shared_ptr<
         infer_request->infer();
         infer_end_time = ggml_time_us();
 
-        if (ggml_openvino_getenv_int("GGML_OPENVINO_DEBUG_OUTPUT")) {
+        if (ggml_openvino_getenv_int("GGML_OPENVINO_DEBUG_OUTPUT") ||
+            ggml_openvino_getenv_str("GGML_OPENVINO_DEBUG_NODE")) {
             for (size_t i = 0; i < ov_output_names.size(); i++) {
                 const auto output_tensor = infer_request->get_output_tensor(i);
                 print_output_tensor_info(ov_output_names[i], output_tensor, output_tensor.data());
@@ -390,6 +555,20 @@ enum ggml_status ov_graph_compute_dynamic(ggml_cgraph * cgraph, std::shared_ptr<
         }
     }
 
+    // GGML_OPENVINO_RELEASE_WEIGHTS (or GGML_OPENVINO_MEMORY_OPTIMIZE on GPU): the plugin holds its own device copy of
+    // every weight after compile, so the host weight buffers can be dropped to reclaim
+    // RSS. The GPU backend uses a single dynamic-shape model for both prefill and decode,
+    // so once a graph is compiled it is reused for the whole session — the only thing
+    // that forces a recompile is clear_caches() on backend teardown. We therefore release
+    // on the first cache-hit (model compiled, plugin has its copy) and, crucially, pin the
+    // compiled-model cache so it survives backend teardown (see ggml_backend_openvino_free).
+    // Without the pin, a later test/context would recompile against the now-dropped pages.
+    // A genuinely new graph still fails fast at the cache-miss compile branch.
+    if (cache_hit && ggml_openvino_release_weights_enabled(device) &&
+        !ggml_openvino_weight_buffers_released()) {
+        ggml_openvino_release_weight_buffers();
+    }
+
     return GGML_STATUS_SUCCESS;
 }
 
@@ -446,6 +625,7 @@ enum ggml_status ov_graph_compute_static(ggml_cgraph * cgraph, std::shared_ptr<o
         if (cache_hit) {
             entry = it->second;
         } else {
+            r_ctx->clear_caches_locked();
             auto mutex = std::make_shared<std::mutex>();
             entry = std::make_shared<decoder_runtime_ctx>(mutex);
             r_ctx->decoder_cache[key] = entry;
@@ -576,7 +756,12 @@ enum ggml_status ov_graph_compute_static(ggml_cgraph * cgraph, std::shared_ptr<o
             }
 
             for (size_t i = 0; i < ov_output_names_local.size(); i++) {
-                auto * ggml_tensor = ggml_decoder->get_model_outputs().at(ov_output_names_local[i]);
+                const auto & model_outputs = ggml_decoder->get_model_outputs();
+                auto model_output_it = model_outputs.find(ov_output_names_local[i]);
+                if (model_output_it == model_outputs.end()) {
+                    continue;
+                }
+                auto * ggml_tensor = model_output_it->second;
                 auto output_tensor = create_ov_output_tensor(ggml_decoder, infer_request, i, ggml_tensor);
                 infer_request->set_output_tensor(i, output_tensor);
             }
@@ -585,7 +770,8 @@ enum ggml_status ov_graph_compute_static(ggml_cgraph * cgraph, std::shared_ptr<o
             infer_request->infer();
             ov_raw_infer_total += ggml_time_us() - ov_raw_infer_start;
 
-            if (ggml_openvino_getenv_int("GGML_OPENVINO_DEBUG_OUTPUT")) {
+            if (ggml_openvino_getenv_int("GGML_OPENVINO_DEBUG_OUTPUT") ||
+                ggml_openvino_getenv_str("GGML_OPENVINO_DEBUG_NODE")) {
                 for (size_t i = 0; i < ov_output_names_local.size(); i++) {
                     const auto output_tensor = infer_request->get_output_tensor(i);
                     print_output_tensor_info(ov_output_names_local[i], output_tensor, output_tensor.data());
@@ -606,7 +792,12 @@ enum ggml_status ov_graph_compute_static(ggml_cgraph * cgraph, std::shared_ptr<o
         }
 
         for (size_t i = 0; i < ov_output_names_local.size(); i++) {
-            auto * ggml_tensor = ggml_decoder->get_model_outputs().at(ov_output_names_local[i]);
+            const auto & model_outputs = ggml_decoder->get_model_outputs();
+            auto model_output_it = model_outputs.find(ov_output_names_local[i]);
+            if (model_output_it == model_outputs.end()) {
+                continue;
+            }
+            auto * ggml_tensor = model_output_it->second;
             auto output_tensor = create_ov_output_tensor(ggml_decoder, infer_request, i, ggml_tensor);
             infer_request->set_output_tensor(i, output_tensor);
         }
@@ -616,7 +807,8 @@ enum ggml_status ov_graph_compute_static(ggml_cgraph * cgraph, std::shared_ptr<o
         infer_end_time = ggml_time_us();
         ov_raw_infer_total = infer_end_time - ov_raw_infer_start;
 
-        if (ggml_openvino_getenv_int("GGML_OPENVINO_DEBUG_OUTPUT")) {
+        if (ggml_openvino_getenv_int("GGML_OPENVINO_DEBUG_OUTPUT") ||
+            ggml_openvino_getenv_str("GGML_OPENVINO_DEBUG_NODE")) {
             for (size_t i = 0; i < ov_output_names_local.size(); i++) {
                 const auto output_tensor = infer_request->get_output_tensor(i);
                 print_output_tensor_info(ov_output_names_local[i], output_tensor, output_tensor.data());
@@ -642,6 +834,18 @@ enum ggml_status ov_graph_compute_static(ggml_cgraph * cgraph, std::shared_ptr<o
 // Step 1 compares each node's recorded use_count with actual fan-out references in node->src.
 // Step 2 verifies that node inputs come from model nodes/weights/leafs; external sources imply split.
 bool is_model_splitted(ggml_cgraph * cgraph) {
+    static const bool fallback_enabled = ggml_openvino_getenv_int("GGML_OPENVINO_ENABLE_FALLBACK") != 0;
+    if (!fallback_enabled) {
+        return false;
+    }
+
+    // Backend op tests execute each node through ggml_graph_view(), which preserves the original
+    // graph use_counts while exposing only one node. Treat those single-node views as regular
+    // naive graphs so intermediate ops do not look like split-model fragments.
+    if (cgraph->n_nodes <= 1 && cgraph->n_leafs == 0) {
+        return false;
+    }
+
     // check the nodes of the model are used by the following nodes, through compare the node's use count and the count of nodes that use it as input. If does not match, return true, else return false.
     for (int i = 0; i < cgraph->n_nodes; i++) {
         ggml_tensor * node = cgraph->nodes[i];
@@ -670,7 +874,17 @@ bool is_model_splitted(ggml_cgraph * cgraph) {
         }
     }
     // if all nodes's src node's src is not come from the nodes in the model, we think the model is splitted. This is a complementary check for the above check, because for some special case like the output node is not used by any node, the use count and input use count are both 0, we can not determine whether the model is splitted or not just based on the first check.
-    auto model_weights = GgmlOvDecoder::create_weight_nodes(cgraph, true);
+    // Only weight-name membership is needed below. With GGML_OPENVINO_REDUCE_COMPILE_MEM
+    // use the name-only collector (no weight extraction); otherwise keep the original
+    // behavior of building (naive) weight nodes and take their names.
+    std::set<std::string> model_weights;
+    if (ggml_openvino_reduce_compile_mem_enabled()) {
+        model_weights = GgmlOvDecoder::collect_weight_names(cgraph);
+    } else {
+        for (const auto & kv : GgmlOvDecoder::create_weight_nodes(cgraph, true)) {
+            model_weights.insert(kv.first);
+        }
+    }
     std::set<ggml_tensor *> model_nodes(cgraph->nodes, cgraph->nodes + cgraph->n_nodes);
     // leaf nodes
     std::set<ggml_tensor *> model_leafs(cgraph->leafs, cgraph->leafs + cgraph->n_leafs);
@@ -752,7 +966,17 @@ enum ggml_status naive_compute(ggml_cgraph * cgraph,
     auto ov_results = model->get_results();
     for (size_t i = 0; i < ov_results.size(); i++) {
         auto output_tensor = infer_request->get_output_tensor(i);
-        auto * ggml_tensor = decoder->get_model_outputs().at(ov_results[i]->get_friendly_name());
+        const auto & model_outputs = decoder->get_model_outputs();
+        auto model_output_it = model_outputs.find(ov_results[i]->get_friendly_name());
+        if (model_output_it == model_outputs.end()) {
+            // Debug-only output added via GGML_OPENVINO_DEBUG_NODE; nothing to copy into.
+            if (ggml_openvino_getenv_int("GGML_OPENVINO_DEBUG_OUTPUT") ||
+                ggml_openvino_getenv_str("GGML_OPENVINO_DEBUG_NODE")) {
+                print_output_tensor_info(ov_results[i]->get_friendly_name(), output_tensor, output_tensor.data());
+            }
+            continue;
+        }
+        auto * ggml_tensor = model_output_it->second;
         std::memcpy(ggml_tensor->data, output_tensor.data(), output_tensor.get_byte_size());
     }
     return GGML_STATUS_SUCCESS;
@@ -837,8 +1061,10 @@ ov::Tensor convert_ggml_input_to_ov(std::shared_ptr<GgmlOvDecoder> ggml_decoder,
 
 ov::Tensor get_ov_input_tensor(std::shared_ptr<GgmlOvDecoder> ggml_decoder, const std::string & param_name) {
     ov::Tensor input_tensor;
-    if (ggml_decoder->get_model_extra_inputs().find(param_name) != ggml_decoder->get_model_extra_inputs().end()) {
-        input_tensor = *ggml_decoder->get_model_extra_input_values().at(param_name);
+    auto extra_input = ggml_decoder->get_model_extra_inputs().find(param_name);
+    if (extra_input != ggml_decoder->get_model_extra_inputs().end()) {
+        input_tensor = ov::Tensor(extra_input->second.type, extra_input->second.shape);
+        *input_tensor.data<int64_t>() = extra_input->second.value;
     } else {
         input_tensor = convert_ggml_input_to_ov(ggml_decoder, param_name);
     }
@@ -853,16 +1079,13 @@ ov::Tensor get_ov_input_tensor_static_decode(std::shared_ptr<GgmlOvDecoder> ggml
 
     if (GgmlOvDecoder::is_inp_tok(ggml_tensor, op) || GgmlOvDecoder::is_inp_pos(ggml_tensor, op) ||
         GgmlOvDecoder::is_kv_idx(ggml_tensor, op)) {
-        assert(ggml_tensor->ne[0] == 1);
-        ov::Shape input_shape = {1, 1, 1, 1};
+        // IMROPE's inp_pos holds one value per t/h/w/e plane instead of a single position;
+        // with a single decode token the planes are still contiguous, so a flat copy works.
+        const int n_planes = GgmlOvDecoder::is_inp_pos(ggml_tensor, op) ? GgmlOvDecoder::get_inp_pos_n_planes(op) : 1;
+        assert(ggml_tensor->ne[0] == n_planes);
+        ov::Shape input_shape = {1, 1, 1, (size_t) n_planes};
         ov::Tensor input_tensor(ggml_decoder->get_ov_type(ggml_tensor), input_shape);
-        if (ggml_tensor->type == GGML_TYPE_I32) {
-            *input_tensor.data<int32_t>() = *((int32_t *) ggml_tensor->data);
-        } else if (ggml_tensor->type == GGML_TYPE_I64) {
-            *input_tensor.data<int64_t>() = *((int64_t *) ggml_tensor->data);
-        } else {
-            throw std::runtime_error("Unexpected tensor type for " + param_name);
-        }
+        std::memcpy(input_tensor.data(), ggml_tensor->data, n_planes * ggml_type_size(ggml_tensor->type));
         return input_tensor;
     }
 
@@ -908,6 +1131,35 @@ ov::Tensor get_ov_input_tensor_static_prefill(std::shared_ptr<GgmlOvDecoder> ggm
     const size_t chunk_valid_size = std::min(chunk_size, input_len - chunk_index * chunk_size);
     const size_t chunk_pad_size = chunk_size - chunk_valid_size;
 
+    if (GgmlOvDecoder::is_inp_pos(ggml_tensor, op) && GgmlOvDecoder::get_inp_pos_n_planes(op) > 1) {
+        // IMROPE: inp_pos stacks n_planes (t/h/w/e) position planes, each of length
+        // input_len; pad every plane independently so they stay aligned to chunk_size.
+        const int n_planes = GgmlOvDecoder::get_inp_pos_n_planes(op);
+        const size_t element_size = ggml_type_size(ggml_tensor->type);
+        ov::Shape input_shape = {1, 1, 1, (size_t) n_planes * chunk_size};
+        ov::Tensor input_tensor(ggml_decoder->get_ov_type(ggml_tensor), input_shape);
+        for (int p = 0; p < n_planes; p++) {
+            const char * src =
+                (const char *) ggml_tensor->data + (p * input_len + chunk_index * chunk_size) * element_size;
+            char * dst = (char *) input_tensor.data() + p * chunk_size * element_size;
+            std::memcpy(dst, src, chunk_valid_size * element_size);
+            if (chunk_pad_size > 0) {
+                if (ggml_tensor->type == GGML_TYPE_I32) {
+                    int32_t last_value = *((const int32_t *) src + chunk_valid_size - 1);
+                    int32_t * out = (int32_t *) dst;
+                    std::fill(out + chunk_valid_size, out + chunk_size, last_value + 1);
+                } else if (ggml_tensor->type == GGML_TYPE_I64) {
+                    int64_t last_value = *((const int64_t *) src + chunk_valid_size - 1);
+                    int64_t * out = (int64_t *) dst;
+                    std::fill(out + chunk_valid_size, out + chunk_size, last_value + 1);
+                } else {
+                    throw std::runtime_error("Unexpected tensor type for " + param_name);
+                }
+            }
+        }
+        return input_tensor;
+    }
+
     if (GgmlOvDecoder::is_inp_tok(ggml_tensor, op) || GgmlOvDecoder::is_inp_pos(ggml_tensor, op) ||
         GgmlOvDecoder::is_kv_idx(ggml_tensor, op)) {
         ov::Shape input_shape = {1, 1, 1, chunk_size};

ggml/src/ggml-openvino/utils.h

@@ -4,6 +4,7 @@
 #include <algorithm>
 #include <atomic>
 #include <cstddef>
+#include <functional>
 #include <memory>
 #include <mutex>
 #include <openvino/runtime/core.hpp>
@@ -17,28 +18,68 @@ struct graph_key {
     int n_nodes;
     std::string first_node_name;
     std::string last_node_name;
+    std::vector<std::string> input_src_names;
 
     graph_key(const ggml_cgraph * cgraph) : n_nodes(cgraph->n_nodes) {
         if (n_nodes > 0) {
             first_node_name = cgraph->nodes[0]->name;
             last_node_name = cgraph->nodes[n_nodes - 1]->name;
         }
+
+        auto get_input_key_name = [](const ggml_cgraph * graph, const ggml_tensor * tensor) {
+            std::string name = tensor->name;
+            const size_t hash_pos = ggml_hash_find(&graph->visited_hash_set, tensor);
+            if (((tensor->flags & GGML_TENSOR_FLAG_COMPUTE) || GgmlOvDecoder::is_kvcache(tensor, nullptr)) &&
+                hash_pos != GGML_HASHSET_FULL && ggml_bitset_get(graph->visited_hash_set.used, hash_pos)) {
+                name += "#" + std::to_string(hash_pos);
+            }
+            return name;
+        };
+
+        std::vector<std::string> node_names;
+        node_names.reserve(cgraph->n_nodes);
+        for (int node_idx = 0; node_idx < cgraph->n_nodes; node_idx++) {
+            node_names.emplace_back(cgraph->nodes[node_idx]->name);
+        }
+
+        for (int node_idx = 0; node_idx < cgraph->n_nodes; node_idx++) {
+            const ggml_tensor * node = cgraph->nodes[node_idx];
+            for (int src_idx = 0; src_idx < GGML_MAX_SRC; src_idx++) {
+                const ggml_tensor * src = node->src[src_idx];
+                if (src == nullptr || src->name[0] == '\0') {
+                    continue;
+                }
+
+                const std::string src_name = get_input_key_name(cgraph, src);
+                if (std::find(node_names.begin(), node_names.end(), src_name) != node_names.end()) {
+                    continue;
+                }
+                if (src_name.find("weight") != std::string::npos) {
+                    continue;
+                }
+
+                input_src_names.push_back(std::to_string(node_idx) + ":" + std::to_string(src_idx) + ":" + src_name);
+            }
+        }
     }
 
     bool operator==(const graph_key & other) const {
         return n_nodes == other.n_nodes && first_node_name == other.first_node_name &&
-               last_node_name == other.last_node_name;
+               last_node_name == other.last_node_name && input_src_names == other.input_src_names;
     }
 };
 
 struct graph_key_hash {
     size_t operator()(const graph_key & key) const {
-        size_t h = std::hash<int>{}(key.n_nodes);
+        size_t hash = std::hash<int>{}(key.n_nodes);
         if (key.n_nodes > 0) {
-            h ^= std::hash<std::string>{}(key.first_node_name) + 0x9e3779b9 + (h << 6) + (h >> 2);
-            h ^= std::hash<std::string>{}(key.last_node_name) + 0x9e3779b9 + (h << 6) + (h >> 2);
+            hash ^= std::hash<std::string>{}(key.first_node_name) + 0x9e3779b9 + (hash << 6) + (hash >> 2);
+            hash ^= std::hash<std::string>{}(key.last_node_name) + 0x9e3779b9 + (hash << 6) + (hash >> 2);
         }
-        return h;
+        for (const auto & input_src_name : key.input_src_names) {
+            hash ^= std::hash<std::string>{}(input_src_name) + 0x9e3779b9 + (hash << 6) + (hash >> 2);
+        }
+        return hash;
     }
 };
 
@@ -66,13 +107,19 @@ struct ov_runtime_context {
 
     ov_runtime_context() : device("CPU"), stateful(false), stateful_kv_size(0), backend_count(0) {}
 
-    void clear_caches() {
-        std::lock_guard<std::mutex> lock(ctx_mutex);
+    void clear_caches_locked() {
         decoder_cache.clear();
         infer_request_cache.clear();
         infer_request_cache_prefill.clear();
         ov_input_names_cache.clear();
         ov_output_names_cache.clear();
+        kv_state_input_name_map.clear();
+        stateful_kv_size = 0;
+    }
+
+    void clear_caches() {
+        std::lock_guard<std::mutex> lock(ctx_mutex);
+        clear_caches_locked();
     }
 };
 

ggml/src/ggml-sycl/common.hpp

@@ -61,6 +61,7 @@ void ggml_sycl_host_free(void* ptr);
 extern int g_ggml_sycl_debug;
 extern int g_ggml_sycl_enable_optimize;
 extern int g_ggml_sycl_enable_fusion;
+extern int g_ggml_sycl_enable_esimd;
 extern int g_ggml_sycl_prioritize_dmmv;
 extern int g_ggml_sycl_enable_flash_attention;
 extern int g_ggml_sycl_dev2dev_memcpy;

ggml/src/ggml-sycl/concat.cpp

@@ -184,8 +184,8 @@ void concat_impl_sycl(ggml_backend_sycl_context & ctx, ggml_tensor *dst) {
             const size_t size0 = ggml_nbytes(src0);
             const size_t size1 = ggml_nbytes(src1);
 
-            SYCL_CHECK(CHECK_TRY_ERROR(stream->memcpy(dst_d, src0_d, size0).wait()));
-            SYCL_CHECK(CHECK_TRY_ERROR(stream->memcpy(dst_d + size0 / type_size, src1_d, size1).wait()));
+            SYCL_CHECK(CHECK_TRY_ERROR(stream->memcpy(dst_d, src0_d, size0)));
+            SYCL_CHECK(CHECK_TRY_ERROR(stream->memcpy(dst_d + size0 / type_size, src1_d, size1)));
         }
     } else {
         concat_T_sycl_non_cont<T>(stream, (const char *) src0->data, (const char *) src1->data, (char *) dst->data,
@@ -196,6 +196,270 @@ void concat_impl_sycl(ggml_backend_sycl_context & ctx, ggml_tensor *dst) {
     }
 }
 
+static void concat_impl_q4_0_sycl(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {
+    scope_op_debug_print scope_dbg_print(__func__, dst, /*num_src=*/2);
+    const ggml_tensor * src0   = dst->src[0];
+    const ggml_tensor * src1   = dst->src[1];
+    queue_ptr           stream = ctx.stream();
+
+    const int32_t dim = ((int32_t *) dst->op_params)[0];
+
+    GGML_ASSERT(src0->type == GGML_TYPE_Q4_0);
+    GGML_ASSERT(src1->type == GGML_TYPE_Q4_0);
+    GGML_ASSERT(dst->type == GGML_TYPE_Q4_0);
+    GGML_ASSERT(src0->ne[0] % QK4_0 == 0);
+    GGML_ASSERT(src1->ne[0] % QK4_0 == 0);
+    GGML_ASSERT(dst->ne[0] % QK4_0 == 0);
+
+    const int ne00_blk = src0->ne[0] / QK4_0;
+    const int ne0_blk  = dst->ne[0] / QK4_0;
+
+    if (ggml_is_contiguous(src0) && ggml_is_contiguous(src1)) {
+        const block_q4_0 * src0_d = (const block_q4_0 *) src0->data;
+        const block_q4_0 * src1_d = (const block_q4_0 *) src1->data;
+        block_q4_0 * dst_d        = (block_q4_0 *) dst->data;
+        const size_t type_size    = sizeof(block_q4_0);
+
+        if (dim != 3) {
+            for (int i3 = 0; i3 < dst->ne[3]; i3++) {
+                concat_T_sycl<block_q4_0>(
+                    src0_d + i3 * (src0->nb[3] / type_size),
+                    src1_d + i3 * (src1->nb[3] / type_size),
+                    dst_d + i3 * (dst->nb[3] / type_size),
+                    ne00_blk, src0->ne[1], src0->ne[2], ne0_blk,
+                    dst->ne[1], dst->ne[2], dim, stream);
+            }
+        } else {
+            const size_t size0 = ggml_nbytes(src0);
+            const size_t size1 = ggml_nbytes(src1);
+
+            SYCL_CHECK(CHECK_TRY_ERROR(stream->memcpy(dst_d, src0_d, size0)));
+            SYCL_CHECK(CHECK_TRY_ERROR(stream->memcpy((char *) dst_d + size0, src1_d, size1)));
+        }
+    } else {
+        concat_T_sycl_non_cont<block_q4_0>(
+            stream, (const char *) src0->data, (const char *) src1->data,
+            (char *) dst->data,
+            ne00_blk, src0->ne[1], src0->ne[2], src0->ne[3],
+            src0->nb[0], src0->nb[1], src0->nb[2], src0->nb[3],
+            src1->ne[0] / QK4_0, src1->ne[1], src1->ne[2], src1->ne[3],
+            src1->nb[0], src1->nb[1], src1->nb[2], src1->nb[3],
+            ne0_blk, dst->ne[1], dst->ne[2], dst->ne[3],
+            dst->nb[0], dst->nb[1], dst->nb[2], dst->nb[3], dim);
+    }
+}
+
+static void concat_impl_q4_1_sycl(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {
+    scope_op_debug_print scope_dbg_print(__func__, dst, /*num_src=*/2);
+    const ggml_tensor * src0   = dst->src[0];
+    const ggml_tensor * src1   = dst->src[1];
+    queue_ptr           stream = ctx.stream();
+
+    const int32_t dim = ((int32_t *) dst->op_params)[0];
+
+    GGML_ASSERT(src0->type == GGML_TYPE_Q4_1);
+    GGML_ASSERT(src1->type == GGML_TYPE_Q4_1);
+    GGML_ASSERT(dst->type == GGML_TYPE_Q4_1);
+    GGML_ASSERT(src0->ne[0] % QK4_1 == 0);
+    GGML_ASSERT(src1->ne[0] % QK4_1 == 0);
+    GGML_ASSERT(dst->ne[0] % QK4_1 == 0);
+
+    const int ne00_blk = src0->ne[0] / QK4_1;
+    const int ne0_blk  = dst->ne[0] / QK4_1;
+
+    if (ggml_is_contiguous(src0) && ggml_is_contiguous(src1)) {
+        const block_q4_1 * src0_d = (const block_q4_1 *) src0->data;
+        const block_q4_1 * src1_d = (const block_q4_1 *) src1->data;
+        block_q4_1 * dst_d        = (block_q4_1 *) dst->data;
+        const size_t type_size    = sizeof(block_q4_1);
+
+        if (dim != 3) {
+            for (int i3 = 0; i3 < dst->ne[3]; i3++) {
+                concat_T_sycl<block_q4_1>(
+                    src0_d + i3 * (src0->nb[3] / type_size),
+                    src1_d + i3 * (src1->nb[3] / type_size),
+                    dst_d + i3 * (dst->nb[3] / type_size),
+                    ne00_blk, src0->ne[1], src0->ne[2], ne0_blk,
+                    dst->ne[1], dst->ne[2], dim, stream);
+            }
+        } else {
+            const size_t size0 = ggml_nbytes(src0);
+            const size_t size1 = ggml_nbytes(src1);
+
+            SYCL_CHECK(CHECK_TRY_ERROR(stream->memcpy(dst_d, src0_d, size0)));
+            SYCL_CHECK(CHECK_TRY_ERROR(stream->memcpy((char *) dst_d + size0, src1_d, size1)));
+        }
+    } else {
+        concat_T_sycl_non_cont<block_q4_1>(
+            stream, (const char *) src0->data, (const char *) src1->data,
+            (char *) dst->data,
+            ne00_blk, src0->ne[1], src0->ne[2], src0->ne[3],
+            src0->nb[0], src0->nb[1], src0->nb[2], src0->nb[3],
+            src1->ne[0] / QK4_1, src1->ne[1], src1->ne[2], src1->ne[3],
+            src1->nb[0], src1->nb[1], src1->nb[2], src1->nb[3],
+            ne0_blk, dst->ne[1], dst->ne[2], dst->ne[3],
+            dst->nb[0], dst->nb[1], dst->nb[2], dst->nb[3], dim);
+    }
+}
+
+static void concat_impl_q5_0_sycl(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {
+    scope_op_debug_print scope_dbg_print(__func__, dst, /*num_src=*/2);
+    const ggml_tensor * src0   = dst->src[0];
+    const ggml_tensor * src1   = dst->src[1];
+    queue_ptr           stream = ctx.stream();
+
+    const int32_t dim = ((int32_t *) dst->op_params)[0];
+
+    GGML_ASSERT(src0->type == GGML_TYPE_Q5_0);
+    GGML_ASSERT(src1->type == GGML_TYPE_Q5_0);
+    GGML_ASSERT(dst->type == GGML_TYPE_Q5_0);
+    GGML_ASSERT(src0->ne[0] % QK5_0 == 0);
+    GGML_ASSERT(src1->ne[0] % QK5_0 == 0);
+    GGML_ASSERT(dst->ne[0] % QK5_0 == 0);
+
+    const int ne00_blk = src0->ne[0] / QK5_0;
+    const int ne0_blk  = dst->ne[0] / QK5_0;
+
+    if (ggml_is_contiguous(src0) && ggml_is_contiguous(src1)) {
+        const block_q5_0 * src0_d = (const block_q5_0 *) src0->data;
+        const block_q5_0 * src1_d = (const block_q5_0 *) src1->data;
+        block_q5_0 * dst_d        = (block_q5_0 *) dst->data;
+        const size_t type_size    = sizeof(block_q5_0);
+
+        if (dim != 3) {
+            for (int i3 = 0; i3 < dst->ne[3]; i3++) {
+                concat_T_sycl<block_q5_0>(
+                    src0_d + i3 * (src0->nb[3] / type_size),
+                    src1_d + i3 * (src1->nb[3] / type_size),
+                    dst_d + i3 * (dst->nb[3] / type_size),
+                    ne00_blk, src0->ne[1], src0->ne[2], ne0_blk,
+                    dst->ne[1], dst->ne[2], dim, stream);
+            }
+        } else {
+            const size_t size0 = ggml_nbytes(src0);
+            const size_t size1 = ggml_nbytes(src1);
+
+            SYCL_CHECK(CHECK_TRY_ERROR(stream->memcpy(dst_d, src0_d, size0)));
+            SYCL_CHECK(CHECK_TRY_ERROR(stream->memcpy((char *) dst_d + size0, src1_d, size1)));
+        }
+    } else {
+        concat_T_sycl_non_cont<block_q5_0>(
+            stream, (const char *) src0->data, (const char *) src1->data,
+            (char *) dst->data,
+            ne00_blk, src0->ne[1], src0->ne[2], src0->ne[3],
+            src0->nb[0], src0->nb[1], src0->nb[2], src0->nb[3],
+            src1->ne[0] / QK5_0, src1->ne[1], src1->ne[2], src1->ne[3],
+            src1->nb[0], src1->nb[1], src1->nb[2], src1->nb[3],
+            ne0_blk, dst->ne[1], dst->ne[2], dst->ne[3],
+            dst->nb[0], dst->nb[1], dst->nb[2], dst->nb[3], dim);
+    }
+}
+
+static void concat_impl_q5_1_sycl(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {
+    scope_op_debug_print scope_dbg_print(__func__, dst, /*num_src=*/2);
+    const ggml_tensor * src0   = dst->src[0];
+    const ggml_tensor * src1   = dst->src[1];
+    queue_ptr           stream = ctx.stream();
+
+    const int32_t dim = ((int32_t *) dst->op_params)[0];
+
+    GGML_ASSERT(src0->type == GGML_TYPE_Q5_1);
+    GGML_ASSERT(src1->type == GGML_TYPE_Q5_1);
+    GGML_ASSERT(dst->type == GGML_TYPE_Q5_1);
+    GGML_ASSERT(src0->ne[0] % QK5_1 == 0);
+    GGML_ASSERT(src1->ne[0] % QK5_1 == 0);
+    GGML_ASSERT(dst->ne[0] % QK5_1 == 0);
+
+    const int ne00_blk = src0->ne[0] / QK5_1;
+    const int ne0_blk  = dst->ne[0] / QK5_1;
+
+    if (ggml_is_contiguous(src0) && ggml_is_contiguous(src1)) {
+        const block_q5_1 * src0_d = (const block_q5_1 *) src0->data;
+        const block_q5_1 * src1_d = (const block_q5_1 *) src1->data;
+        block_q5_1 * dst_d        = (block_q5_1 *) dst->data;
+        const size_t type_size    = sizeof(block_q5_1);
+
+        if (dim != 3) {
+            for (int i3 = 0; i3 < dst->ne[3]; i3++) {
+                concat_T_sycl<block_q5_1>(
+                    src0_d + i3 * (src0->nb[3] / type_size),
+                    src1_d + i3 * (src1->nb[3] / type_size),
+                    dst_d + i3 * (dst->nb[3] / type_size),
+                    ne00_blk, src0->ne[1], src0->ne[2], ne0_blk,
+                    dst->ne[1], dst->ne[2], dim, stream);
+            }
+        } else {
+            const size_t size0 = ggml_nbytes(src0);
+            const size_t size1 = ggml_nbytes(src1);
+
+            SYCL_CHECK(CHECK_TRY_ERROR(stream->memcpy(dst_d, src0_d, size0)));
+            SYCL_CHECK(CHECK_TRY_ERROR(stream->memcpy((char *) dst_d + size0, src1_d, size1)));
+        }
+    } else {
+        concat_T_sycl_non_cont<block_q5_1>(
+            stream, (const char *) src0->data, (const char *) src1->data,
+            (char *) dst->data,
+            ne00_blk, src0->ne[1], src0->ne[2], src0->ne[3],
+            src0->nb[0], src0->nb[1], src0->nb[2], src0->nb[3],
+            src1->ne[0] / QK5_1, src1->ne[1], src1->ne[2], src1->ne[3],
+            src1->nb[0], src1->nb[1], src1->nb[2], src1->nb[3],
+            ne0_blk, dst->ne[1], dst->ne[2], dst->ne[3],
+            dst->nb[0], dst->nb[1], dst->nb[2], dst->nb[3], dim);
+    }
+}
+
+static void concat_impl_q8_0_sycl(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {
+    scope_op_debug_print scope_dbg_print(__func__, dst, /*num_src=*/2);
+    const ggml_tensor * src0   = dst->src[0];
+    const ggml_tensor * src1   = dst->src[1];
+    queue_ptr           stream = ctx.stream();
+
+    const int32_t dim = ((int32_t *) dst->op_params)[0];
+
+    GGML_ASSERT(src0->type == GGML_TYPE_Q8_0);
+    GGML_ASSERT(src1->type == GGML_TYPE_Q8_0);
+    GGML_ASSERT(dst->type == GGML_TYPE_Q8_0);
+    GGML_ASSERT(src0->ne[0] % QK8_0 == 0);
+    GGML_ASSERT(src1->ne[0] % QK8_0 == 0);
+    GGML_ASSERT(dst->ne[0] % QK8_0 == 0);
+
+    const int ne00_blk = src0->ne[0] / QK8_0;
+    const int ne0_blk  = dst->ne[0] / QK8_0;
+
+    if (ggml_is_contiguous(src0) && ggml_is_contiguous(src1)) {
+        const block_q8_0 * src0_d = (const block_q8_0 *) src0->data;
+        const block_q8_0 * src1_d = (const block_q8_0 *) src1->data;
+        block_q8_0 * dst_d        = (block_q8_0 *) dst->data;
+        const size_t type_size    = sizeof(block_q8_0);
+
+        if (dim != 3) {
+            for (int i3 = 0; i3 < dst->ne[3]; i3++) {
+                concat_T_sycl<block_q8_0>(
+                    src0_d + i3 * (src0->nb[3] / type_size),
+                    src1_d + i3 * (src1->nb[3] / type_size),
+                    dst_d + i3 * (dst->nb[3] / type_size),
+                    ne00_blk, src0->ne[1], src0->ne[2], ne0_blk,
+                    dst->ne[1], dst->ne[2], dim, stream);
+            }
+        } else {
+            const size_t size0 = ggml_nbytes(src0);
+            const size_t size1 = ggml_nbytes(src1);
+            SYCL_CHECK(CHECK_TRY_ERROR(stream->memcpy(dst_d, src0_d, size0)));
+            SYCL_CHECK(CHECK_TRY_ERROR(stream->memcpy((char *) dst_d + size0, src1_d, size1)));
+        }
+    } else {
+        concat_T_sycl_non_cont<block_q8_0>(
+            stream, (const char *) src0->data, (const char *) src1->data,
+            (char *) dst->data,
+            ne00_blk, src0->ne[1], src0->ne[2], src0->ne[3],
+            src0->nb[0], src0->nb[1], src0->nb[2], src0->nb[3],
+            src1->ne[0] / QK8_0, src1->ne[1], src1->ne[2], src1->ne[3],
+            src1->nb[0], src1->nb[1], src1->nb[2], src1->nb[3],
+            ne0_blk, dst->ne[1], dst->ne[2], dst->ne[3],
+            dst->nb[0], dst->nb[1], dst->nb[2], dst->nb[3], dim);
+    }
+}
+
 void ggml_sycl_op_concat(ggml_backend_sycl_context & ctx, ggml_tensor *dst) {
 
     switch (dst->type) {
@@ -222,6 +486,21 @@ void ggml_sycl_op_concat(ggml_backend_sycl_context & ctx, ggml_tensor *dst) {
     case GGML_TYPE_I8:
         concat_impl_sycl<int8_t>(ctx, dst);
         break;
+    case GGML_TYPE_Q4_0:
+        concat_impl_q4_0_sycl(ctx, dst);
+        break;
+    case GGML_TYPE_Q4_1:
+        concat_impl_q4_1_sycl(ctx, dst);
+        break;
+    case GGML_TYPE_Q5_0:
+        concat_impl_q5_0_sycl(ctx, dst);
+        break;
+    case GGML_TYPE_Q5_1:
+        concat_impl_q5_1_sycl(ctx, dst);
+        break;
+    case GGML_TYPE_Q8_0:
+        concat_impl_q8_0_sycl(ctx, dst);
+        break;
     default:
         fprintf(stderr, "%s: unsupported types: dst: %s\n", __func__, ggml_type_name(dst->type));
         GGML_ASSERT(false);

ggml/src/ggml-sycl/dmmv.cpp

@@ -8,6 +8,9 @@
         #include <sycl/ext/oneapi/bfloat16.hpp>
         #define GGML_SYCL_DMMV_HAS_BF16
     #endif
+    #include <sycl/ext/intel/esimd.hpp>
+    #include "esimd.hpp"
+    #define GGML_SYCL_DMMV_HAS_ESIMD
 #endif
 
 static void convert_f16(const void * vx, const int64_t ib, const int iqs, dfloat2 & v){
@@ -1864,6 +1867,113 @@ static void dequantize_mul_mat_vec_q6_K_sycl(const void *vx, const float *y,
         });
 }
 
+#ifdef GGML_SYCL_DMMV_HAS_ESIMD
+using ggml_sycl_esimd::GGML_SYCL_DMMV_ESIMD_WG_SIZE;
+
+// generic reordered dequantize-matvec: each work-group owns a pair of
+// consecutive output rows and updates one 32-wide accumulator per row
+template <ggml_type T>
+ESIMD_INLINE void dequantize_mul_mat_vec_reorder_esimd(
+        const void * vx, const float * y, float * dst,
+        const int ncols, const int nrows,
+        sycl::local_accessor<float, 1> lmem,
+        const sycl::nd_item<1> & it) {
+    using namespace sycl::ext::intel::esimd;
+    using traits = ggml_sycl_esimd::esimd_reorder_q_traits<T>;
+
+    const int    num_blocks_per_row = ncols / QK_K;
+    const size_t nb = (size_t) nrows * num_blocks_per_row;
+    const auto   ps = traits::make_ptrs(vx, nb);
+
+    const int  tid      = it.get_local_id(0);
+    const int  row_pair = it.get_group(0);
+    const int  row0     = row_pair * 2; // two consecutive output rows
+    const bool has_row1 = row0 + 1 < nrows;
+
+    // one 32-wide accumulator per output row (small footprint, no spill)
+    simd<float, 32> acc0 = 0.0f;
+    simd<float, 32> acc1 = 0.0f;
+
+    for (int ib = tid; ib < num_blocks_per_row; ib += GGML_SYCL_DMMV_ESIMD_WG_SIZE) {
+        simd<float, 256> y_vec = block_load<float, 256>(y + (size_t) ib * QK_K);
+
+        const size_t bi0 = (size_t) (row0 + 0) * num_blocks_per_row + ib;
+        const size_t bi1 = (size_t) (row0 + 1) * num_blocks_per_row + ib;
+
+        traits::mac_pair(ps, bi0, ps, bi1, has_row1, y_vec, acc0, acc1);
+    }
+
+    lmem[tid * 2 + 0] = reduce<float>(acc0, std::plus<>{});
+    lmem[tid * 2 + 1] = reduce<float>(acc1, std::plus<>{});
+    it.barrier(sycl::access::fence_space::local_space);
+
+    if (tid == 0) {
+        float sum0 = 0.0f;
+        float sum1 = 0.0f;
+        for (int p = 0; p < GGML_SYCL_DMMV_ESIMD_WG_SIZE; ++p) {
+            sum0 += lmem[p * 2 + 0];
+            sum1 += lmem[p * 2 + 1];
+        }
+        dst[row0 + 0] = sum0;
+        if (has_row1) {
+            dst[row0 + 1] = sum1;
+        }
+    }
+}
+
+static void dequantize_mul_mat_vec_q3_K_sycl_reorder_esimd(const void *vx, const float *y,
+                                                           float *dst, const int ncols,
+                                                           const int nrows,
+                                                           dpct::queue_ptr stream) {
+    GGML_ASSERT(ncols % QK_K == 0);
+    const int workgroups = (nrows + 1) / 2;
+    stream->submit([&](sycl::handler &h) {
+        sycl::local_accessor<float, 1> lmem(sycl::range<1>(GGML_SYCL_DMMV_ESIMD_WG_SIZE * 2), h);
+        h.parallel_for(
+            sycl::nd_range<1>(sycl::range<1>((size_t)workgroups * GGML_SYCL_DMMV_ESIMD_WG_SIZE), sycl::range<1>(GGML_SYCL_DMMV_ESIMD_WG_SIZE)),
+            [=](sycl::nd_item<1> it) [[intel::sycl_explicit_simd]] {
+                dequantize_mul_mat_vec_reorder_esimd<GGML_TYPE_Q3_K>(
+                    vx, y, dst, ncols, nrows, lmem, it);
+            });
+    });
+}
+
+static void dequantize_mul_mat_vec_q4_K_sycl_reorder_esimd(const void *vx, const float *y,
+                                                           float *dst, const int ncols,
+                                                           const int nrows,
+                                                           dpct::queue_ptr stream) {
+    GGML_ASSERT(ncols % QK_K == 0);
+    const int workgroups = (nrows + 1) / 2;
+    stream->submit([&](sycl::handler &h) {
+        sycl::local_accessor<float, 1> lmem(sycl::range<1>(GGML_SYCL_DMMV_ESIMD_WG_SIZE * 2), h);
+        h.parallel_for(
+            sycl::nd_range<1>(sycl::range<1>((size_t)workgroups * GGML_SYCL_DMMV_ESIMD_WG_SIZE), sycl::range<1>(GGML_SYCL_DMMV_ESIMD_WG_SIZE)),
+            [=](sycl::nd_item<1> it) [[intel::sycl_explicit_simd]] {
+                dequantize_mul_mat_vec_reorder_esimd<GGML_TYPE_Q4_K>(
+                    vx, y, dst, ncols, nrows, lmem, it);
+            });
+    });
+}
+
+static void dequantize_mul_mat_vec_q6_K_sycl_reorder_esimd(const void *vx, const float *y,
+                                                           float *dst, const int ncols,
+                                                           const int nrows,
+                                                           dpct::queue_ptr stream) {
+    GGML_ASSERT(ncols % QK_K == 0);
+    const int workgroups = (nrows + 1) / 2;
+    stream->submit([&](sycl::handler &h) {
+        sycl::local_accessor<float, 1> lmem(sycl::range<1>(GGML_SYCL_DMMV_ESIMD_WG_SIZE * 2), h);
+        h.parallel_for(
+            sycl::nd_range<1>(sycl::range<1>((size_t)workgroups * GGML_SYCL_DMMV_ESIMD_WG_SIZE), sycl::range<1>(GGML_SYCL_DMMV_ESIMD_WG_SIZE)),
+            [=](sycl::nd_item<1> it) [[intel::sycl_explicit_simd]] {
+                dequantize_mul_mat_vec_reorder_esimd<GGML_TYPE_Q6_K>(
+                    vx, y, dst, ncols, nrows, lmem, it);
+            });
+    });
+}
+
+#endif // GGML_SYCL_DMMV_HAS_ESIMD
+
 static void dequantize_mul_mat_vec_q4_K_sycl_reorder(const void *vx, const float *y,
                                                      float *dst, const int ncols,
                                                      const int nrows,
@@ -1992,7 +2102,15 @@ void ggml_sycl_op_dequantize_mul_mat_vec(
         case GGML_TYPE_Q3_K:
             if ((ggml_tensor_extra_gpu *) dst->src[0]->extra &&
                 ((ggml_tensor_extra_gpu *) dst->src[0]->extra)->optimized_feature.reorder) {
-                dequantize_mul_mat_vec_q3_K_sycl_reorder(src0_dd_i, src1_ddf_i, dst_dd_i, ne00, row_diff, stream);
+#ifdef GGML_SYCL_DMMV_HAS_ESIMD
+                if (g_ggml_sycl_enable_esimd) {
+                    dequantize_mul_mat_vec_q3_K_sycl_reorder_esimd(src0_dd_i, src1_ddf_i, dst_dd_i, ne00, row_diff, stream);
+                }
+                else
+#endif
+                {
+                    dequantize_mul_mat_vec_q3_K_sycl_reorder(src0_dd_i, src1_ddf_i, dst_dd_i, ne00, row_diff, stream);
+                }
             } else {
                 dequantize_mul_mat_vec_q3_K_sycl(src0_dd_i, src1_ddf_i, dst_dd_i, ne00, row_diff, stream);
             }
@@ -2000,7 +2118,15 @@ void ggml_sycl_op_dequantize_mul_mat_vec(
         case GGML_TYPE_Q4_K:
             if ((ggml_tensor_extra_gpu *) dst->src[0]->extra &&
                 ((ggml_tensor_extra_gpu *) dst->src[0]->extra)->optimized_feature.reorder) {
-                dequantize_mul_mat_vec_q4_K_sycl_reorder(src0_dd_i, src1_ddf_i, dst_dd_i, ne00, row_diff, stream);
+#ifdef GGML_SYCL_DMMV_HAS_ESIMD
+                if (g_ggml_sycl_enable_esimd) {
+                    dequantize_mul_mat_vec_q4_K_sycl_reorder_esimd(src0_dd_i, src1_ddf_i, dst_dd_i, ne00, row_diff, stream);
+                }
+                else
+#endif
+                {
+                    dequantize_mul_mat_vec_q4_K_sycl_reorder(src0_dd_i, src1_ddf_i, dst_dd_i, ne00, row_diff, stream);
+                }
             } else {
                 dequantize_mul_mat_vec_q4_K_sycl(src0_dd_i, src1_ddf_i, dst_dd_i, ne00, row_diff, stream);
             }
@@ -2016,7 +2142,15 @@ void ggml_sycl_op_dequantize_mul_mat_vec(
         case GGML_TYPE_Q6_K:
             if ((ggml_tensor_extra_gpu *) dst->src[0]->extra &&
                 ((ggml_tensor_extra_gpu *) dst->src[0]->extra)->optimized_feature.reorder) {
-                dequantize_mul_mat_vec_q6_K_sycl_reorder(src0_dd_i, src1_ddf_i, dst_dd_i, ne00, row_diff, stream);
+#ifdef GGML_SYCL_DMMV_HAS_ESIMD
+                if (g_ggml_sycl_enable_esimd) {
+                    dequantize_mul_mat_vec_q6_K_sycl_reorder_esimd(src0_dd_i, src1_ddf_i, dst_dd_i, ne00, row_diff, stream);
+                }
+                else
+#endif
+                {
+                    dequantize_mul_mat_vec_q6_K_sycl_reorder(src0_dd_i, src1_ddf_i, dst_dd_i, ne00, row_diff, stream);
+                }
             } else {
                 dequantize_mul_mat_vec_q6_K_sycl(src0_dd_i, src1_ddf_i, dst_dd_i, ne00, row_diff, stream);
             }

ggml/src/ggml-sycl/element_wise.cpp

@@ -81,43 +81,6 @@ static __dpct_inline__ T op_elu(T x) {
     return (x > static_cast<T>(0.f)) ? x : op_expm1(x);
 }
 
-template<typename T>
-static __dpct_inline__ T op_tanh(T x) {
-    if constexpr (std::is_same_v<T, sycl::ext::oneapi::bfloat16>) {
-        constexpr int ver = __INTEL_LLVM_COMPILER;
-#if defined(__INTEL_LLVM_COMPILER) && (__INTEL_LLVM_COMPILER >= 20260000)
-            return sycl::ext::oneapi::experimental::tanh(x);
-#else
-            return static_cast<T>(sycl::tanh(static_cast<float>(x)));
-#endif
-    } else {
-        return sycl::tanh(x);
-    }
-}
-
-template<typename T>
-static __dpct_inline__ T op_gelu(T x) {
-    const T GELU_COEF_A    = static_cast<T>(0.044715f);
-    const T SQRT_2_OVER_PI = static_cast<T>(0.79788456080286535587989211986876f);
-    return static_cast<T>(0.5f) * x *
-           (static_cast<T>(1.0f) +
-            op_tanh(SQRT_2_OVER_PI * x * (static_cast<T>(1.0f) + GELU_COEF_A * x * x)));
-}
-
-template<typename T>
-static __dpct_inline__ T op_exp(T x) {
-    if constexpr (std::is_same_v<T, sycl::ext::oneapi::bfloat16>) {
-        return sycl::ext::oneapi::experimental::exp(x);
-    } else {
-        return sycl::exp(x);
-    }
-}
-
-template<typename T>
-static __dpct_inline__ T op_silu(T x) {
-    return x / (static_cast<T>(1.0f) + op_exp(-x));
-}
-
 template<typename T>
 static __dpct_inline__ T op_erf(T x) {
     if constexpr (std::is_same_v<T, sycl::ext::oneapi::bfloat16>) {
@@ -448,6 +411,47 @@ static void unary_gated_op_generic_kernel(
     }
 }
 
+// Fused UNARY + MUL. Unlike the gated ops above, `x` and `g` are separate tensors of the
+// same shape; `o0`/`o1` are their row strides in elements, so a half-view needs no repack.
+// `dst` is contiguous and indexed flat. Math is done in f32, as the CPU and CUDA references do.
+template<typename T, typename F>
+static void unary_mul_flat_kernel(const T * x, const T * g, T * dst, const int64_t k, const sycl::nd_item<1> &item_ct1, F op) {
+    SYCL_GLOBAL_ID_LOOP(k, item_ct1) {
+        dst[i] = (T) (op((float) x[i]) * (float) g[i]);
+    }
+}
+
+template<typename T, typename F>
+static void unary_mul_strided_kernel(const T * x, const T * g, T * dst, const int64_t k, const sycl::uint3 n_fd, const int64_t o0, const int64_t o1, const sycl::nd_item<1> &item_ct1, F op) {
+    SYCL_GLOBAL_ID_LOOP(k, item_ct1) {
+        const sycl::uint2 rc = fast_div_modulo((uint32_t) i, n_fd);
+        const int64_t j0 = rc.x() * o0 + rc.y();
+        const int64_t j1 = o0 == o1 ? j0 : rc.x() * o1 + rc.y();
+        dst[i] = (T) (op((float) x[j0]) * (float) g[j1]);
+    }
+}
+
+template<typename T, typename F>
+static void unary_mul_sycl(const T * x, const T * g, T * dst, const int64_t k, const int64_t n, const int64_t o0, const int64_t o1, queue_ptr main_stream, F op) {
+    const size_t            num_blocks = ceil_div((size_t) k, (size_t) SYCL_GLU_BLOCK_SIZE);
+    const sycl::nd_range<1> range(num_blocks * sycl::range<1>(SYCL_GLU_BLOCK_SIZE), sycl::range<1>(SYCL_GLU_BLOCK_SIZE));
+
+    // o0 == o1 == n makes (i/n)*o0 + (i%n) == i, so the strided kernel degenerates to the flat one
+    if (o0 == n && o1 == n) {
+        main_stream->parallel_for(range, [=](sycl::nd_item<1> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] {
+            unary_mul_flat_kernel(x, g, dst, k, item_ct1, op);
+        });
+        return;
+    }
+
+    // 32-bit fastdiv, exact only below 2^31; ggml_sycl_can_fuse() already declined past that
+    GGML_ASSERT(k < ((int64_t) 1 << 31));
+    const sycl::uint3 n_fd = init_fastdiv_values((uint32_t) n);
+    main_stream->parallel_for(range, [=](sycl::nd_item<1> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] {
+        unary_mul_strided_kernel(x, g, dst, k, n_fd, o0, o1, item_ct1, op);
+    });
+}
+
 namespace ggml_sycl_detail {
 static void acc_f32_sycl(const char *x, const char *y, float *dst,
                          const int64_t n_elements,
@@ -991,6 +995,52 @@ static inline void ggml_sycl_op_swiglu(ggml_backend_sycl_context & ctx, ggml_ten
     });
 }
 
+// dst = op(unary_node->src[0]) * other, written straight to the MUL output, saving the
+// standalone unary launch. Preconditions come from ggml_sycl_can_fuse(); re-asserted here.
+void ggml_sycl_op_unary_mul_fused(ggml_backend_sycl_context & ctx, ggml_tensor * unary_node, ggml_tensor * mul_node) {
+    scope_op_debug_print scope_dbg_print(__func__, mul_node, /*num_src=*/2);
+
+    const ggml_tensor * x = unary_node->src[0];
+    const ggml_tensor * g = (mul_node->src[0] == unary_node) ? mul_node->src[1] : mul_node->src[0];
+
+    // g is picked by elimination; ggml_can_fuse()'s single-use rule rules out MUL(unary, unary)
+    GGML_ASSERT(g != unary_node);
+    GGML_ASSERT(x->type == g->type && x->type == mul_node->type);
+    GGML_ASSERT(ggml_are_same_shape(x, g) && ggml_are_same_shape(x, mul_node));
+    GGML_ASSERT(ggml_is_contiguous_1(x) && ggml_is_contiguous_1(g));
+    // dst is indexed flat
+    GGML_ASSERT(ggml_is_contiguous(mul_node));
+
+    queue_ptr main_stream = ctx.stream();
+    SYCL_CHECK(ggml_sycl_set_device(ctx.device));
+
+    const int64_t k = ggml_nelements(mul_node);
+    const int64_t n = mul_node->ne[0];
+
+    const auto dispatch_type = [&](auto op) {
+        switch (mul_node->type) {
+            case GGML_TYPE_F32:
+                unary_mul_sycl((const float *) x->data, (const float *) g->data, (float *) mul_node->data,
+                               k, n, x->nb[1] / sizeof(float), g->nb[1] / sizeof(float), main_stream, op);
+                break;
+            case GGML_TYPE_F16:
+                unary_mul_sycl((const sycl::half *) x->data, (const sycl::half *) g->data, (sycl::half *) mul_node->data,
+                               k, n, x->nb[1] / sizeof(sycl::half), g->nb[1] / sizeof(sycl::half), main_stream, op);
+                break;
+            default:
+                GGML_ABORT("fused unary+mul: unsupported type %s", ggml_type_name(mul_node->type));
+        }
+    };
+
+    switch (ggml_get_unary_op(unary_node)) {
+        case GGML_UNARY_OP_SILU:     dispatch_type([](float v) { return op_silu(v); });     break;
+        case GGML_UNARY_OP_SIGMOID:  dispatch_type([](float v) { return op_sigmoid(v); });  break;
+        case GGML_UNARY_OP_SOFTPLUS: dispatch_type([](float v) { return op_softplus(v); }); break;
+        default:
+            GGML_ABORT("fused unary+mul: unsupported unary op %s", ggml_unary_op_name(ggml_get_unary_op(unary_node)));
+    }
+}
+
 __dpct_inline__ float ggml_sycl_op_swiglu_oai_single(float x, float g, float alpha = 1.702f, float limit = 7.0f) {
     x = sycl::fmin(x, limit);
     g = sycl::fmax(sycl::fmin(g, limit), -limit);

ggml/src/ggml-sycl/element_wise.hpp

@@ -28,6 +28,39 @@ typed_data<T_Dst, T_Src> cast_data(ggml_tensor * dst) {
 
 const float GELU_QUICK_COEF = -1.702f;
 
+// Single-element activations, shared with the mat-vec kernels that fuse a GLU epilogue
+// (mmvq.cpp), so both apply the same formula.
+template <typename T> static __dpct_inline__ T op_tanh(T x) {
+    if constexpr (std::is_same_v<T, sycl::ext::oneapi::bfloat16>) {
+#if defined(__INTEL_LLVM_COMPILER) && (__INTEL_LLVM_COMPILER >= 20260000)
+        return sycl::ext::oneapi::experimental::tanh(x);
+#else
+        return static_cast<T>(sycl::tanh(static_cast<float>(x)));
+#endif
+    } else {
+        return sycl::tanh(x);
+    }
+}
+
+template <typename T> static __dpct_inline__ T op_gelu(T x) {
+    const T GELU_COEF_A    = static_cast<T>(0.044715f);
+    const T SQRT_2_OVER_PI = static_cast<T>(0.79788456080286535587989211986876f);
+    return static_cast<T>(0.5f) * x *
+           (static_cast<T>(1.0f) +
+            op_tanh(SQRT_2_OVER_PI * x * (static_cast<T>(1.0f) + GELU_COEF_A * x * x)));
+}
+
+template <typename T> static __dpct_inline__ T op_exp(T x) {
+    if constexpr (std::is_same_v<T, sycl::ext::oneapi::bfloat16>) {
+        return sycl::ext::oneapi::experimental::exp(x);
+    } else {
+        return sycl::exp(x);
+    }
+}
+
+template <typename T> static __dpct_inline__ T op_silu(T x) {
+    return x / (static_cast<T>(1.0f) + op_exp(-x));
+}
 
 void ggml_sycl_sqrt(ggml_backend_sycl_context & ctx, ggml_tensor * dst);
 
@@ -95,4 +128,7 @@ void ggml_sycl_trunc(ggml_backend_sycl_context & ctx, ggml_tensor * dst);
 
 void ggml_sycl_arange(ggml_backend_sycl_context & ctx, ggml_tensor * dst);
 
+// fused UNARY(silu|sigmoid|softplus) + MUL; see ggml_sycl_can_fuse() for the accepted shapes
+void ggml_sycl_op_unary_mul_fused(ggml_backend_sycl_context & ctx, ggml_tensor * unary_node, ggml_tensor * mul_node);
+
 #endif // GGML_SYCL_ELEMENTWISE_HPP

ggml/src/ggml-sycl/esimd.hpp

@@ -0,0 +1,392 @@
+//
+// MIT license
+// Copyright (C) 2026 Intel Corporation
+// SPDX-License-Identifier: MIT
+//
+
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+
+#ifndef GGML_SYCL_ESIMD_HPP
+#define GGML_SYCL_ESIMD_HPP
+
+#include <sycl/ext/intel/esimd.hpp>
+
+#include "common.hpp"
+
+namespace ggml_sycl_esimd {
+
+constexpr int GGML_SYCL_DMMV_ESIMD_WG_SIZE = 4;
+
+//
+// Shared ESIMD building blocks for the reordered K-quant dequantize-matvec
+// kernels.
+//
+// The reordered K-quant ESIMD matvec kernels share one skeleton: per super-block,
+// load a 256-float activation slice, load one weight block, dequantize it into 8
+// chunks of 32 and MAC each chunk against the matching activation slice, then
+// reduce and run a lane-0 epilogue.
+//
+// Each K-quant kernel emits exactly 8 chunks of 32 mapping to activation slices
+// 0..7, so the per-block work is captured by esimd_reorder_q_traits<T>::mac_pair,
+// which dequantizes two weight blocks and MACs both against a shared activation
+// vector with the two FMA chains interleaved (co-scheduled to hide FMA latency).
+// The "pair" is the (row0,row1) row pair owned by one work-group, so the
+// layout+dequant is written once per quant type here.
+//
+
+template <ggml_type T> struct esimd_reorder_q_traits;
+
+// build a 32-lane vector whose low 16 lanes are `lo` and high 16 are `hi`
+// (a super-chunk splits into two 16-wide halves with distinct scale/min codes).
+static ESIMD_INLINE sycl::ext::intel::esimd::simd<float, 32> splat_lo_hi(float lo, float hi) {
+    using namespace sycl::ext::intel::esimd;
+    simd<float, 32> v;
+    v.select<16, 1>(0)  = lo;
+    v.select<16, 1>(16) = hi;
+    return v;
+}
+
+// unpack one block of Q4_K/Q5_K scale/min codes (get_scale_min_k4 layout) into 8
+// float scales (dall * sc) and 8 float mins (-dmin * m); the min carries the
+// negation so the dequant epilogue adds.
+static ESIMD_INLINE void unpack_scale_min_k4(
+        sycl::ext::intel::esimd::simd<uint8_t, 12> scales, float dall, float dmin,
+        sycl::ext::intel::esimd::simd<float, 8> & scale_f,
+        sycl::ext::intel::esimd::simd<float, 8> & min_f) {
+    using namespace sycl::ext::intel::esimd;
+    simd<uint8_t, 8> sc = 0;
+    simd<uint8_t, 8> m  = 0;
+    simd<uint8_t, 4> scale_lo = scales.select<4, 1>(0);
+    simd<uint8_t, 4> min_lo   = scales.select<4, 1>(4);
+    simd<uint8_t, 4> hi_bits  = scales.select<4, 1>(8);
+    sc.select<4, 1>(0) = scale_lo & simd<uint8_t, 4>(0x3F);
+    sc.select<4, 1>(4) = (hi_bits & simd<uint8_t, 4>(0x0F)) |
+                         ((scale_lo >> simd<uint8_t, 4>(6)) << simd<uint8_t, 4>(4));
+    m.select<4, 1>(0)  = min_lo & simd<uint8_t, 4>(0x3F);
+    m.select<4, 1>(4)  = (hi_bits >> simd<uint8_t, 4>(4)) |
+                         ((min_lo >> simd<uint8_t, 4>(6)) << simd<uint8_t, 4>(4));
+    scale_f = convert<float>(sc) * dall;
+    min_f   = convert<float>(m) * (-dmin);
+}
+
+// ---------------------------------------------------------------------------
+// Q3_K, SOA reorder layout produced by reorder_qw_q3_k:
+//   [qs: nb*(QK_K/4)] [hmask: nb*(QK_K/8)] [scales: nb*12] [d: nb*sizeof(half)]
+// with nb = nrows*num_blocks_per_row. Single super-block scale d, no dmin.
+//
+// 3 bits per weight: 2 low bits in qs, 1 high bit in hmask. The 8 output chunks
+// of 32 (matching dequantize_row_q3_K) map to super-chunk s (0..7): byte base
+// 32*(s/4) into the 64-byte qs array, bit shift 2*(s%4); the low 16 lanes use
+// scale code 2s, the high 16 use 2s+1. hmask is a 32-byte array (like Q5_K's
+// qh) where chunk s uses bit s of the same 32 bytes, but INVERTED: the value is
+// (q & 3) - (hmask_bit_set ? 0 : 4), i.e. (q & 3) + 4*bit - 4.
+//
+// The 16 6-bit scale codes are packed into 12 bytes (get_scale_min layout for
+// Q3_K): low nibbles from bytes 0..7, high 2 bits from bytes 8..11 shifted by
+// 0/2/4/6; the dequant scale is d * (code - 32).
+// ---------------------------------------------------------------------------
+template <> struct esimd_reorder_q_traits<GGML_TYPE_Q3_K> {
+    struct ptrs {
+        const uint8_t *    qs;
+        const uint8_t *    hmask;
+        const uint8_t *    scales;
+        const sycl::half * d;
+    };
+
+    static ESIMD_INLINE ptrs make_ptrs(const void * vx, size_t nb) {
+        const uint8_t * qs     = (const uint8_t *) vx;
+        const uint8_t * hmask  = qs + nb * (QK_K / 4);
+        const uint8_t * scales = hmask + nb * (QK_K / 8);
+        const sycl::half * d   = (const sycl::half *) (scales + nb * 12);
+        return { qs, hmask, scales, d };
+    }
+
+    // unpack the 12 packed bytes into 16 6-bit scale codes (dequantize_row_q3_K
+    // aux layout), returned as float scale = d * (code - 32).
+    // done with wide (8/16-lane) ops rather than four 4-lane groups.
+    static ESIMD_INLINE sycl::ext::intel::esimd::simd<float, 16> unpack_scales(
+            sycl::ext::intel::esimd::simd<uint8_t, 12> in, float d) {
+        using namespace sycl::ext::intel::esimd;
+
+        // low 6-bit part: codes 0..7 = low nibble of bytes 0..7,
+        //                 codes 8..15 = high nibble of bytes 0..7
+        simd<uint8_t, 8>  lo8 = in.select<8, 1>(0);
+        simd<uint8_t, 16> code;
+        code.select<8, 1>(0) = lo8 & simd<uint8_t, 8>(0x0F);
+        code.select<8, 1>(8) = lo8 >> simd<uint8_t, 8>(4);
+
+        // high 2-bit part: bytes 8..11 replicated 4x, group g (0..3) shifted 2*g
+        simd<uint8_t, 16> hib;
+        hib.select<4, 1>(0)  = in.select<4, 1>(8);
+        hib.select<4, 1>(4)  = in.select<4, 1>(8);
+        hib.select<4, 1>(8)  = in.select<4, 1>(8);
+        hib.select<4, 1>(12) = in.select<4, 1>(8);
+        simd<uint8_t, 16> hshift;
+        hshift.select<4, 1>(0)  = 0;
+        hshift.select<4, 1>(4)  = 2;
+        hshift.select<4, 1>(8)  = 4;
+        hshift.select<4, 1>(12) = 6;
+        hib = (hib >> hshift) & simd<uint8_t, 16>(0x03);
+
+        code = code | (hib << simd<uint8_t, 16>(4));
+        return (convert<float>(code) - 32.0f) * d;
+    }
+
+    static ESIMD_INLINE void mac_pair(
+            const ptrs & pa, size_t bia,
+            const ptrs & pb, size_t bib, bool has_b,
+            sycl::ext::intel::esimd::simd<float, 256> & y_vec,
+            sycl::ext::intel::esimd::simd<float, 32> & acc_a,
+            sycl::ext::intel::esimd::simd<float, 32> & acc_b) {
+        using namespace sycl::ext::intel::esimd;
+
+        simd<uint8_t, 64> qs_a     = block_load<uint8_t, 64>(pa.qs + bia * (QK_K / 4));
+        simd<uint8_t, 64> qs_b     = 0;
+        simd<uint8_t, 32> hmask_a  = block_load<uint8_t, 32>(pa.hmask + bia * (QK_K / 8));
+        simd<uint8_t, 32> hmask_b  = 0;
+        simd<uint8_t, 12> scales_a = block_load<uint8_t, 12>(pa.scales + bia * 12);
+        simd<uint8_t, 12> scales_b = 0;
+
+        const float d_a = (float) pa.d[bia];
+        float d_b = 0.0f;
+        if (has_b) {
+            qs_b     = block_load<uint8_t, 64>(pb.qs + bib * (QK_K / 4));
+            hmask_b  = block_load<uint8_t, 32>(pb.hmask + bib * (QK_K / 8));
+            scales_b = block_load<uint8_t, 12>(pb.scales + bib * 12);
+            d_b = (float) pb.d[bib];
+        }
+
+        simd<float, 16> scale_f_a = unpack_scales(scales_a, d_a);
+        simd<float, 16> scale_f_b = unpack_scales(scales_b, d_b);
+
+#pragma unroll
+        for (int s = 0; s < 8; ++s) {
+            const int     byte_base = 32 * (s / 4);
+            const uint8_t shift     = (uint8_t) (2 * (s % 4));
+            simd<float, 32> y_s = y_vec.select<32, 1>(s * 32);
+
+            // 2 low bits from qs, high bit from hmask (bit s of the same 32 bytes);
+            // value = (q & 3) + 4*bit - 4  (inverted hmask: subtract 4 when bit clear).
+            // merge in the integer domain: q3 = (q & 3) | (bit << 2) in {0..7},
+            // then a single convert + subtract yields q3 - 4 (one convert, not two)
+            simd<uint16_t, 32> q3_a = convert<uint16_t>(
+                    (qs_a.select<32, 1>(byte_base) >> shift) & simd<uint8_t, 32>(3));
+            q3_a |= convert<uint16_t>(
+                    ((hmask_a >> simd<uint8_t, 32>((uint8_t) s)) & simd<uint8_t, 32>(1)) << simd<uint8_t, 32>(2));
+            simd<uint16_t, 32> q3_b = convert<uint16_t>(
+                    (qs_b.select<32, 1>(byte_base) >> shift) & simd<uint8_t, 32>(3));
+            q3_b |= convert<uint16_t>(
+                    ((hmask_b >> simd<uint8_t, 32>((uint8_t) s)) & simd<uint8_t, 32>(1)) << simd<uint8_t, 32>(2));
+
+            simd<float, 32> qf_a = convert<float>(q3_a) - 4.0f;
+            simd<float, 32> qf_b = convert<float>(q3_b) - 4.0f;
+
+            const float scale_a_lo = scale_f_a[2 * s + 0];
+            const float scale_a_hi = scale_f_a[2 * s + 1];
+            const float scale_b_lo = scale_f_b[2 * s + 0];
+            const float scale_b_hi = scale_f_b[2 * s + 1];
+
+            simd<float, 32> scale_vec_a = splat_lo_hi(scale_a_lo, scale_a_hi);
+            simd<float, 32> scale_vec_b = splat_lo_hi(scale_b_lo, scale_b_hi);
+
+            simd<float, 32> deq_a = qf_a * scale_vec_a;
+            simd<float, 32> deq_b = qf_b * scale_vec_b;
+
+            acc_a += y_s * deq_a;
+            acc_b += y_s * deq_b;
+        }
+    }
+};
+
+// ---------------------------------------------------------------------------
+// Q4_K, SOA reorder layout produced by reorder_qw_q4_k:
+//   [qs: nb*(QK_K/2)] [scales: nb*K_SCALE_SIZE] [dm: nb*sizeof(half2)]
+// with nb = nrows*num_blocks_per_row.
+// ---------------------------------------------------------------------------
+template <> struct esimd_reorder_q_traits<GGML_TYPE_Q4_K> {
+    struct ptrs {
+        const uint8_t *    qs;
+        const uint8_t *    scales;
+        const sycl::half * dm;
+    };
+
+    static ESIMD_INLINE ptrs make_ptrs(const void * vx, size_t nb) {
+        const uint8_t * qs     = (const uint8_t *) vx;
+        const uint8_t * scales = qs + nb * (QK_K / 2);
+        const sycl::half * dm  = (const sycl::half *) (scales + nb * K_SCALE_SIZE);
+        return { qs, scales, dm };
+    }
+
+    static ESIMD_INLINE void mac_pair(
+            const ptrs & pa, size_t bia,
+            const ptrs & pb, size_t bib, bool has_b,
+            sycl::ext::intel::esimd::simd<float, 256> & y_vec,
+            sycl::ext::intel::esimd::simd<float, 32> & acc_a,
+            sycl::ext::intel::esimd::simd<float, 32> & acc_b) {
+        using namespace sycl::ext::intel::esimd;
+
+        simd<uint8_t, 128> qs_a     = block_load<uint8_t, 128>(pa.qs + bia * (QK_K / 2));
+        simd<uint8_t, 128> qs_b     = 0;
+        simd<uint8_t, 12>  scales_a = block_load<uint8_t, 12>(pa.scales + bia * K_SCALE_SIZE);
+        simd<uint8_t, 12>  scales_b = 0;
+
+        const float dall_a = (float) pa.dm[bia * 2 + 0];
+        const float dmin_a = (float) pa.dm[bia * 2 + 1];
+        float dall_b = 0.0f;
+        float dmin_b = 0.0f;
+        if (has_b) {
+            qs_b     = block_load<uint8_t, 128>(pb.qs + bib * (QK_K / 2));
+            scales_b = block_load<uint8_t, 12>(pb.scales + bib * K_SCALE_SIZE);
+            dall_b = (float) pb.dm[bib * 2 + 0];
+            dmin_b = (float) pb.dm[bib * 2 + 1];
+        }
+
+        simd<float, 8> scale_f_a, min_f_a, scale_f_b, min_f_b;
+        unpack_scale_min_k4(scales_a, dall_a, dmin_a, scale_f_a, min_f_a);
+        unpack_scale_min_k4(scales_b, dall_b, dmin_b, scale_f_b, min_f_b);
+
+        simd<uint8_t, 128> qs_lo_a = qs_a & simd<uint8_t, 128>(0x0F);
+        simd<uint8_t, 128> qs_hi_a = qs_a >> simd<uint8_t, 128>(4);
+        simd<uint8_t, 128> qs_lo_b = qs_b & simd<uint8_t, 128>(0x0F);
+        simd<uint8_t, 128> qs_hi_b = qs_b >> simd<uint8_t, 128>(4);
+
+#pragma unroll
+        for (int sb = 0; sb < 8; sb += 2) {
+            const int q_offset = sb * 16;
+            simd<float, 32> y_lo = y_vec.select<32, 1>(sb * 32);
+            simd<float, 32> y_hi = y_vec.select<32, 1>((sb + 1) * 32);
+
+            const float scale_a_lo = scale_f_a[sb];
+            const float scale_a_hi = scale_f_a[sb + 1];
+            const float min_a_lo   = min_f_a[sb];
+            const float min_a_hi   = min_f_a[sb + 1];
+            const float scale_b_lo = scale_f_b[sb];
+            const float scale_b_hi = scale_f_b[sb + 1];
+            const float min_b_lo   = min_f_b[sb];
+            const float min_b_hi   = min_f_b[sb + 1];
+
+            simd<uint8_t, 32> qa_lo = qs_lo_a.select<32, 1>(q_offset);
+            simd<uint8_t, 32> qa_hi = qs_hi_a.select<32, 1>(q_offset);
+            simd<uint8_t, 32> qb_lo = qs_lo_b.select<32, 1>(q_offset);
+            simd<uint8_t, 32> qb_hi = qs_hi_b.select<32, 1>(q_offset);
+
+            simd<float, 32> deq_a_lo = convert<float>(qa_lo) * scale_a_lo + min_a_lo;
+            simd<float, 32> deq_a_hi = convert<float>(qa_hi) * scale_a_hi + min_a_hi;
+            simd<float, 32> deq_b_lo = convert<float>(qb_lo) * scale_b_lo + min_b_lo;
+            simd<float, 32> deq_b_hi = convert<float>(qb_hi) * scale_b_hi + min_b_hi;
+
+            acc_a += y_lo * deq_a_lo;
+            acc_b += y_lo * deq_b_lo;
+            acc_a += y_hi * deq_a_hi;
+            acc_b += y_hi * deq_b_hi;
+        }
+    }
+};
+
+// ---------------------------------------------------------------------------
+// Q6_K, SOA reorder layout:
+//   [ql: nb*(QK_K/2)] [qh: nb*(QK_K/4)] [scales(int8): nb*(QK_K/16)] [d: nb*half]
+// ---------------------------------------------------------------------------
+template <> struct esimd_reorder_q_traits<GGML_TYPE_Q6_K> {
+    struct ptrs {
+        const uint8_t *    ql;
+        const uint8_t *    qh;
+        const int8_t *     scales;
+        const sycl::half * d;
+    };
+
+    static ESIMD_INLINE ptrs make_ptrs(const void * vx, size_t nb) {
+        const uint8_t *    ql     = (const uint8_t *) vx;
+        const uint8_t *    qh     = ql + nb * (QK_K / 2);
+        const int8_t *     scales = (const int8_t *) (qh + nb * (QK_K / 4));
+        const sycl::half * d      = (const sycl::half *) (scales + nb * (QK_K / 16));
+        return { ql, qh, scales, d };
+    }
+
+    static ESIMD_INLINE void mac_pair(
+            const ptrs & pa, size_t bia,
+            const ptrs & pb, size_t bib, bool has_b,
+            sycl::ext::intel::esimd::simd<float, 256> & y_vec,
+            sycl::ext::intel::esimd::simd<float, 32> & acc_a,
+            sycl::ext::intel::esimd::simd<float, 32> & acc_b) {
+        using namespace sycl::ext::intel::esimd;
+
+        simd<uint8_t, 128> ql_a     = block_load<uint8_t, 128>(pa.ql + bia * (QK_K / 2));
+        simd<uint8_t, 128> ql_b     = 0;
+        simd<uint8_t, 64>  qh_a     = block_load<uint8_t, 64>(pa.qh + bia * (QK_K / 4));
+        simd<uint8_t, 64>  qh_b     = 0;
+        simd<int8_t, 16>   scales_a = block_load<int8_t, 16>(pa.scales + bia * (QK_K / 16));
+        simd<int8_t, 16>   scales_b = 0;
+
+        const float d_a = (float) pa.d[bia];
+        float d_b = 0.0f;
+        if (has_b) {
+            ql_b     = block_load<uint8_t, 128>(pb.ql + bib * (QK_K / 2));
+            qh_b     = block_load<uint8_t, 64>(pb.qh + bib * (QK_K / 4));
+            scales_b = block_load<int8_t, 16>(pb.scales + bib * (QK_K / 16));
+            d_b = (float) pb.d[bib];
+        }
+
+        simd<float, 16> sc_a = convert<float>(scales_a);
+        simd<float, 16> sc_b = convert<float>(scales_b);
+
+#pragma unroll
+        for (int im = 0; im < 2; ++im) {
+            simd<uint8_t, 32> ql_lo_a   = ql_a.select<32, 1>(64 * im);
+            simd<uint8_t, 32> ql_hi_a   = ql_a.select<32, 1>(64 * im + 32);
+            simd<uint8_t, 32> qh_bits_a = qh_a.select<32, 1>(32 * im);
+            simd<uint8_t, 32> ql_lo_b   = ql_b.select<32, 1>(64 * im);
+            simd<uint8_t, 32> ql_hi_b   = ql_b.select<32, 1>(64 * im + 32);
+            simd<uint8_t, 32> qh_bits_b = qh_b.select<32, 1>(32 * im);
+
+            // reconstruct each 32-wide 6-bit group (matches dequantize_row_q6_K)
+#pragma unroll
+            for (int g = 0; g < 4; ++g) {
+                simd<float, 32> y_g = y_vec.select<32, 1>(32 * (4 * im + g));
+
+                const float scale_a_lo = sc_a[8 * im + 2 * g + 0] * d_a;
+                const float scale_a_hi = sc_a[8 * im + 2 * g + 1] * d_a;
+                const float scale_b_lo = sc_b[8 * im + 2 * g + 0] * d_b;
+                const float scale_b_hi = sc_b[8 * im + 2 * g + 1] * d_b;
+
+                simd<float, 32> scale_vec_a = splat_lo_hi(scale_a_lo, scale_a_hi);
+                simd<float, 32> scale_vec_b = splat_lo_hi(scale_b_lo, scale_b_hi);
+
+                simd<uint8_t, 32> qa;
+                simd<uint8_t, 32> qb;
+                switch (g) {
+                    case 0:
+                        qa = (ql_lo_a & simd<uint8_t, 32>(0x0F)) | ((qh_bits_a & simd<uint8_t, 32>(0x03)) << simd<uint8_t, 32>(4));
+                        qb = (ql_lo_b & simd<uint8_t, 32>(0x0F)) | ((qh_bits_b & simd<uint8_t, 32>(0x03)) << simd<uint8_t, 32>(4));
+                        break;
+                    case 1:
+                        qa = (ql_hi_a & simd<uint8_t, 32>(0x0F)) | ((qh_bits_a & simd<uint8_t, 32>(0x0C)) << simd<uint8_t, 32>(2));
+                        qb = (ql_hi_b & simd<uint8_t, 32>(0x0F)) | ((qh_bits_b & simd<uint8_t, 32>(0x0C)) << simd<uint8_t, 32>(2));
+                        break;
+                    case 2:
+                        qa = (ql_lo_a >> simd<uint8_t, 32>(4)) | (qh_bits_a & simd<uint8_t, 32>(0x30));
+                        qb = (ql_lo_b >> simd<uint8_t, 32>(4)) | (qh_bits_b & simd<uint8_t, 32>(0x30));
+                        break;
+                    default:
+                        qa = (ql_hi_a >> simd<uint8_t, 32>(4)) | ((qh_bits_a & simd<uint8_t, 32>(0xC0)) >> simd<uint8_t, 32>(2));
+                        qb = (ql_hi_b >> simd<uint8_t, 32>(4)) | ((qh_bits_b & simd<uint8_t, 32>(0xC0)) >> simd<uint8_t, 32>(2));
+                        break;
+                }
+
+                simd<float, 32> deq_a = (convert<float>(qa) - 32.0f) * scale_vec_a;
+                simd<float, 32> deq_b = (convert<float>(qb) - 32.0f) * scale_vec_b;
+
+                acc_a += y_g * deq_a;
+                acc_b += y_g * deq_b;
+            }
+        }
+    }
+};
+
+} // namespace ggml_sycl_esimd
+
+#endif // GGML_SYCL_ESIMD_HPP