rm json-shim

.github/actions/ccache-clear/action.yml

@@ -1,88 +1,22 @@
-# note: place this as the last step of the job, so the new cache is saved by "Post ccache" right after the old one is cleared
 name: "ccache-clear"
-description: "Delete GitHub Actions caches matching a key prefix, oldest first"
+description: "Delete all GitHub Actions caches matching a key prefix"
 inputs:
   key:
     description: "Cache key prefix to match and delete"
     required: true
-  older:
-    description: "Only delete caches created more than this long ago (e.g. 90m, 1h, 1d). By default all matching caches are deleted"
-    required: false
-    default: ""
-  min:
-    description: "Stop deleting if fewer than this many caches would remain (e.g. 1). By default there is no minimum"
-    required: false
-    default: "0"
-  dry-run:
-    description: "Only print the caches that would be deleted, without deleting them"
-    required: false
-    default: "false"
 
 runs:
   using: "composite"
   steps:
     - name: Clear caches
       shell: bash
-      env:
-        CLEAR_KEY: ${{ inputs.key }}
-        CLEAR_OLDER: ${{ inputs.older }}
-        CLEAR_MIN: ${{ inputs.min }}
-        CLEAR_DRY_RUN: ${{ inputs.dry-run }}
       run: |
-        # Convert a duration (e.g. 90m, 1h, 1d, plain seconds) to seconds
-        to_seconds() {
-          local val="$1"
-          [[ "$val" =~ ^[0-9]+$ ]] && { echo "$val"; return 0; }
-          local num="${val%?}" unit="${val: -1}" mult
-          [[ "$num" =~ ^[0-9]+$ ]] || return 1
-          case "$unit" in
-            s) mult=1 ;;
-            m) mult=60 ;;
-            h) mult=3600 ;;
-            d) mult=86400 ;;
-            *) return 1 ;;
-          esac
-          echo $((num * mult))
-        }
-
-        [[ "$CLEAR_MIN" =~ ^[0-9]+$ ]] || { echo "Invalid min value: $CLEAR_MIN" >&2; exit 1; }
-        [[ "$CLEAR_DRY_RUN" =~ ^(true|false)$ ]] || { echo "Invalid dry-run value: $CLEAR_DRY_RUN" >&2; exit 1; }
-
-        CACHES=$(gh cache list --key "ccache-$CLEAR_KEY" --json id,key,createdAt --jq '.[] | [.createdAt, .id, .key] | @tsv' 2>/dev/null | LC_ALL=C sort)
+        CACHES=$(gh cache list --key "ccache-${{ inputs.key }}" --json id,key --jq '.[] | "\(.id) \(.key)"' 2>/dev/null)
         if [ -z "$CACHES" ]; then
-          echo "No caches found with key prefix: $CLEAR_KEY"
+          echo "No caches found with key prefix: ${{ inputs.key }}"
           exit 0
         fi
-
-        TOTAL=$(( $(wc -l <<< "$CACHES") ))
-
-        echo "Found $TOTAL cache(s) with key prefix: $CLEAR_KEY (oldest first):"
-        while IFS=$'\t' read -r CREATED ID KEY; do
-          printf '  %s  %s  %s\n' "$CREATED" "$ID" "$KEY"
-        done <<< "$CACHES"
-
-        CUTOFF=""
-        if [ -n "$CLEAR_OLDER" ]; then
-          OLDER_SECONDS=$(to_seconds "$CLEAR_OLDER") || { echo "Invalid older value: $CLEAR_OLDER (expected e.g. 90m, 1h, 1d)" >&2; exit 1; }
-          CUTOFF=$(( $(date +%s) - OLDER_SECONDS ))
-        fi
-
-        # Caches are sorted oldest first
-        DELETED=0
-        while IFS=$'\t' read -r CREATED ID KEY; do
-          if [ -n "$CUTOFF" ] && [ "$(date -d "$CREATED" +%s)" -ge "$CUTOFF" ]; then
-            echo "Rest are not older than $CLEAR_OLDER, stopping"
-            break
-          fi
-          if [ $((TOTAL - DELETED - 1)) -lt "$CLEAR_MIN" ]; then
-            echo "Keeping at least $CLEAR_MIN cache(s), stopping"
-            break
-          fi
-          if [ "$CLEAR_DRY_RUN" = "true" ]; then
-            echo "Would delete cache: $ID ($KEY)"
-          else
-            echo "Deleting cache: $ID ($KEY)"
-            gh cache delete "$ID"
-          fi
-          DELETED=$((DELETED + 1))
+        while read -r id key; do
+          echo "Deleting cache: $id ($key)"
+          gh cache delete "$id"
         done <<< "$CACHES"

.github/workflows/build-cmake-pkg.yml

@@ -27,26 +27,30 @@ jobs:
           cmake --install build --prefix "$PREFIX" --config Release
 
           export LLAMA_CONFIG="$PREFIX"/lib/cmake/llama/llama-config.cmake
-          build_commit=$(git rev-parse --short HEAD | xargs)
-          build_number=$(git rev-list --count HEAD | xargs)
+          tclsh <<'EOF'
+          set build(commit)  [string trim [exec git rev-parse --short HEAD]]
+          set build(number)  [string trim [exec git rev-list  --count HEAD]]
 
-          major=$(grep -oE "set\(LLAMA_VERSION_MAJOR[[:space:]]+[0-9]+" CMakeLists.txt | grep -oE "[0-9]+$")
-          minor=$(grep -oE "set\(LLAMA_VERSION_MINOR[[:space:]]+[0-9]+" CMakeLists.txt | grep -oE "[0-9]+$")
-          patch=$(grep -oE "set\(LLAMA_VERSION_PATCH[[:space:]]+[0-9]+" CMakeLists.txt | grep -oE "[0-9]+$")
-          build_version="$major.$minor.$patch"
+          set cmakelists [read [open "CMakeLists.txt" r]]
+          regexp {set\(LLAMA_VERSION_MAJOR\s+(\d+)\)} $cmakelists -> major
+          regexp {set\(LLAMA_VERSION_MINOR\s+(\d+)\)} $cmakelists -> minor
+          regexp {set\(LLAMA_VERSION_PATCH\s+(\d+)\)} $cmakelists -> patch
+          set build(version) "$major.$minor.$patch"
 
-          checks=("set\(LLAMA_VERSION[[:space:]]+$build_version\)"
-                  "set\(LLAMA_BUILD_COMMIT[[:space:]]+$build_commit\)"
-                  "set\(LLAMA_BUILD_NUMBER[[:space:]]+$build_number\)")
+          set llamaconfig [read [open "$env(LLAMA_CONFIG)" r]]
+          set checks [list "set\\(LLAMA_VERSION     \\s+$build(version)\\)" \
+                           "set\\(LLAMA_BUILD_COMMIT\\s+$build(commit)\\)" \
+                           "set\\(LLAMA_BUILD_NUMBER\\s+$build(number)\\)"]
 
-          for check in "${checks[@]}"; do
-              if ! grep -qE "$check" "$LLAMA_CONFIG"; then
-                  echo "Checking llama-config.cmake version... \"$check\" failed!"
+          puts -nonewline "Checking llama-config.cmake version... "
+          foreach check $checks {
+              if {![regexp -expanded -- $check $llamaconfig]} {
+                  puts "\"$check\" failed!"
                   exit 1
-              fi
-          done
-
-          echo "Checking llama-config.cmake version... success."
+              }
+          }
+          puts "success."
+          EOF
 
           cd examples/simple-cmake-pkg
           cmake -S . -B build -DCMAKE_PREFIX_PATH="$PREFIX"/lib/cmake

.github/workflows/build-cpu.yml

@@ -97,7 +97,8 @@ jobs:
           cmake -B build \
             -DGGML_NATIVE=OFF \
             -DLLAMA_FATAL_WARNINGS=ON \
-            -DGGML_RPC=ON
+            -DGGML_RPC=ON \
+            -DGGML_NATIVE=OFF
           time cmake --build build --config Release -j $(nproc)
 
       - name: Test
@@ -117,18 +118,6 @@ jobs:
           ./bin/llama-convert-llama2c-to-ggml --copy-vocab-from-model ./tok512.bin --llama2c-model stories260K.bin --llama2c-output-model stories260K.gguf
           ./bin/llama-completion -m stories260K.gguf -p "One day, Lily met a Shoggoth" -n 500 -c 256
 
-      # note: real deletion only on push to master (same condition as the ccache save),
-      #       dry-run otherwise (the token is read-only on PRs from forks)
-      - name: ccache-clear
-        uses: ./.github/actions/ccache-clear
-        env:
-          GH_TOKEN: ${{ github.token }}
-        with:
-          key: cpu-${{ matrix.os }}
-          older: 1h
-          min: 1
-          dry-run: ${{ github.event_name != 'push' || github.ref != 'refs/heads/master' }}
-
   windows:
     name: windows / ${{ matrix.build }}
     runs-on: windows-2025

.github/workflows/make-release.yml

@@ -55,71 +55,33 @@ jobs:
         env:
           GITHUB_REPOSITORY: ${{ github.repository }}
 
-      - name: Create nightly-tag.txt
-        id: nightly_tag_file
-        run: |
-          NIGHTLY_TAG="${{ steps.desc.outputs.nightly_tag }}"
-          if [[ -z "${NIGHTLY_TAG}" ]]; then
-            echo "Warning: no nightly tag found for the release commit - nightly-tag.txt will not be created"
-            echo "create=false" >> "$GITHUB_OUTPUT"
-            exit 0
-          fi
-          echo "${NIGHTLY_TAG}" > nightly-tag.txt
-          echo "create=true" >> "$GITHUB_OUTPUT"
-          echo "nightly-tag.txt:"
-          cat nightly-tag.txt
-
       - name: Create release
-        id: create_release
         if: ${{ github.event.inputs.dry_run == 'false' }}
         uses: ggml-org/action-create-release@v1
         env:
           GITHUB_TOKEN: ${{ github.token }}
         with:
           tag_name: ${{ steps.checks.outputs.version }}
-          prerelease: false
-          # TODO: enrich the body of the release with more information
+          # TODO: remove the prerelease flag once the semantic versioning workflow is ready
+          # ref: https://github.com/ggml-org/ggml/discussions/1579
+          prerelease: true
           body: |
-            ## Overview
-
-            New version has been released.
+            > [!NOTE]
+            > Semantic versioning is still work in progress.
+            > More info can be found in https://github.com/ggml-org/ggml/discussions/1579
 
             ${{ steps.desc.outputs.nightly }}
 
-            **Web UI:** the `nightly-tag.txt` asset contains the tag of the corresponding nightly release
-
-            **More info:** [dist : releases and versioning of ggml-org projects](https://github.com/ggml-org/ggml/discussions/1579)
-
             ## ${{ steps.desc.outputs.changelog_title }}
 
             ${{ steps.desc.outputs.changelog }}
 
-      - name: Upload nightly-tag.txt
-        if: ${{ github.event.inputs.dry_run == 'false' && steps.nightly_tag_file.outputs.create == 'true' }}
-        uses: actions/github-script@v8
-        with:
-          github-token: ${{secrets.GITHUB_TOKEN}}
-          script: |
-            const fs = require('fs');
-            const release_id = '${{ steps.create_release.outputs.id }}';
-            console.log('uploadReleaseAsset', 'nightly-tag.txt');
-            await github.rest.repos.uploadReleaseAsset({
-              owner: context.repo.owner,
-              repo: context.repo.repo,
-              release_id: release_id,
-              name: 'nightly-tag.txt',
-              data: await fs.readFileSync('./nightly-tag.txt')
-            });
-
       - name: Dry run summary
         if: ${{ github.event.inputs.dry_run == 'true' }}
         run: |
           if [[ "${{ steps.checks.outputs.checks_passed }}" == "true" ]]; then
             echo "Dry run complete - all checks passed."
             echo "Would have created tag: ${{ steps.checks.outputs.version }}"
-            if [[ -n "${{ steps.desc.outputs.nightly_tag }}" ]]; then
-              echo "Would have uploaded nightly-tag.txt: ${{ steps.desc.outputs.nightly_tag }}"
-            fi
           else
             echo "::error::Dry run found release check failures. A release tag would not be created."
             exit 1

.github/workflows/release.yml

@@ -145,6 +145,11 @@ jobs:
             ${{ env.CMAKE_ARGS }}
           cmake --build build --config Release -j $(sysctl -n hw.logicalcpu)
 
+      - name: ccache-clear
+        uses: ./.github/actions/ccache-clear
+        with:
+          key: release-${{ matrix.os }}-${{ matrix.arch }}
+
       - name: Determine tag name
         id: tag
         uses: ./.github/actions/get-tag-name
@@ -161,11 +166,6 @@ jobs:
           path: llama-${{ steps.tag.outputs.name }}-bin-macos-${{ matrix.build }}.tar.gz
           name: llama-bin-macos-${{ matrix.build }}.tar.gz
 
-      - name: ccache-clear
-        uses: ./.github/actions/ccache-clear
-        with:
-          key: release-${{ matrix.os }}-${{ matrix.arch }}
-
   ubuntu-cpu:
     needs: [check-release, get-version]
     if: ${{ needs.check-release.outputs.should_release == 'true' }}
@@ -231,6 +231,12 @@ jobs:
             ${{ env.CMAKE_ARGS }}
           cmake --build build --config Release -j $(nproc)
 
+      - name: ccache-clear
+        if: ${{ matrix.build != 's390x' }}
+        uses: ./.github/actions/ccache-clear
+        with:
+          key: release-${{ matrix.os }}-cpu
+
       - name: Determine tag name
         id: tag
         uses: ./.github/actions/get-tag-name
@@ -247,12 +253,6 @@ jobs:
           path: llama-${{ steps.tag.outputs.name }}-bin-ubuntu-${{ matrix.build }}.tar.gz
           name: llama-bin-ubuntu-${{ matrix.build }}.tar.gz
 
-      - name: ccache-clear
-        if: ${{ matrix.build != 's390x' }}
-        uses: ./.github/actions/ccache-clear
-        with:
-          key: release-${{ matrix.os }}-cpu
-
   ubuntu-vulkan:
     needs: [check-release, get-version]
     if: ${{ needs.check-release.outputs.should_release == 'true' }}
@@ -318,6 +318,11 @@ jobs:
             ${{ env.CMAKE_ARGS }}
           cmake --build build --config Release -j $(nproc)
 
+      - name: ccache-clear
+        uses: ./.github/actions/ccache-clear
+        with:
+          key: release-${{ matrix.os }}-vulkan
+
       - name: Determine tag name
         id: tag
         uses: ./.github/actions/get-tag-name
@@ -334,11 +339,6 @@ jobs:
           path: llama-${{ steps.tag.outputs.name }}-bin-ubuntu-vulkan-${{ matrix.build }}.tar.gz
           name: llama-bin-ubuntu-vulkan-${{ matrix.build }}.tar.gz
 
-      - name: ccache-clear
-        uses: ./.github/actions/ccache-clear
-        with:
-          key: release-${{ matrix.os }}-vulkan
-
   android-arm64:
     needs: [check-release, get-version]
     if: ${{ needs.check-release.outputs.should_release == 'true' }}
@@ -512,6 +512,11 @@ jobs:
             ${{ env.CMAKE_ARGS }}
           cmake --build build/ReleaseOV --config Release --parallel
 
+      - name: ccache-clear
+        uses: ./.github/actions/ccache-clear
+        with:
+          key: release-ubuntu-24.04-openvino-release-no-preset-v1
+
       - name: Determine tag name
         id: tag
         uses: ./.github/actions/get-tag-name
@@ -546,11 +551,6 @@ jobs:
           path: llama-${{ steps.tag.outputs.name }}-bin-ubuntu-openvino-${{ env.OPENVINO_VERSION_MAJOR }}-x64.tar.gz
           name: llama-bin-ubuntu-openvino-${{ env.OPENVINO_VERSION_MAJOR }}-x64.tar.gz
 
-      - name: ccache-clear
-        uses: ./.github/actions/ccache-clear
-        with:
-          key: release-ubuntu-24.04-openvino-release-no-preset-v1
-
   windows-openvino:
     needs: [check-release]
     if: ${{ needs.check-release.outputs.should_release == 'true' }}
@@ -637,6 +637,11 @@ jobs:
 
           cmake --build build\ReleaseOV --config Release -- /m
 
+      - name: ccache-clear
+        uses: ./.github/actions/ccache-clear
+        with:
+          key: release-windows-2022-openvino
+
       - name: Determine tag name
         id: tag
         uses: ./.github/actions/get-tag-name
@@ -675,11 +680,6 @@ jobs:
           path: llama-${{ steps.tag.outputs.name }}-bin-win-openvino-${{ env.OPENVINO_VERSION_MAJOR }}-x64.zip
           name: llama-bin-win-openvino-${{ env.OPENVINO_VERSION_MAJOR }}-x64.zip
 
-      - name: ccache-clear
-        uses: ./.github/actions/ccache-clear
-        with:
-          key: release-windows-2022-openvino
-
   windows-cpu:
     name: windows-cpu / ${{ matrix.arch }}
     needs: [check-release]
@@ -733,6 +733,11 @@ jobs:
             ${{ env.CMAKE_ARGS }}
           cmake --build build --config Release
 
+      - name: ccache-clear
+        uses: ./.github/actions/ccache-clear
+        with:
+          key: release-windows-2025-vs2026-${{ matrix.arch }}-cpu
+
       - name: Pack artifacts
         id: pack_artifacts
         run: |
@@ -744,11 +749,6 @@ jobs:
           path: llama-bin-win-cpu-${{ matrix.arch }}.zip
           name: llama-bin-win-cpu-${{ matrix.arch }}.zip
 
-      - name: ccache-clear
-        uses: ./.github/actions/ccache-clear
-        with:
-          key: release-windows-2025-vs2026-${{ matrix.arch }}-cpu
-
   windows-rocm:
     needs: [check-release]
     if: ${{ needs.check-release.outputs.should_release == 'true' }}
@@ -774,7 +774,6 @@ jobs:
         with:
           key: windows-rocm-${{ matrix.ROCM_VERSION }}-${{ matrix.build }}
           evict-old-files: 1d
-          max-size: "1G"
 
       # - name: Cache ROCm Installation
       #   id: cache-rocm
@@ -842,6 +841,11 @@ jobs:
             -DAMDGPU_TARGETS="${{ matrix.gpu_targets }}"
           cmake --build . --config Release --parallel ${env:NUMBER_OF_PROCESSORS}
 
+      - name: ccache-clear
+        uses: ./.github/actions/ccache-clear
+        with:
+          key: windows-rocm-${{ matrix.ROCM_VERSION }}-${{ matrix.build }}
+
       - name: Verify HIP backend was built
         run: |
           $hipDll = Get-ChildItem -Path build\bin -Filter "ggml-hip*.dll" -ErrorAction SilentlyContinue
@@ -874,11 +878,6 @@ jobs:
           path: llama-bin-win-rocm-${{ env.ROCM_VERSION_SHORT }}-${{ matrix.build }}.zip
           name: llama-bin-win-rocm-${{ env.ROCM_VERSION_SHORT }}-${{ matrix.build }}.zip
 
-      - name: ccache-clear
-        uses: ./.github/actions/ccache-clear
-        with:
-          key: windows-rocm-${{ matrix.ROCM_VERSION }}-${{ matrix.build }}
-
   windows:
     needs: [check-release]
     if: ${{ needs.check-release.outputs.should_release == 'true' }}
@@ -1044,6 +1043,11 @@ jobs:
           set /A NINJA_JOBS=%NUMBER_OF_PROCESSORS%-1
           cmake --build build --config Release -j %NINJA_JOBS% --target ggml-cuda
 
+      - name: ccache-clear
+        uses: ./.github/actions/ccache-clear
+        with:
+          key: release-windows-2022-${{ matrix.arch }}-cuda-${{ matrix.cuda }}
+
       - name: Pack artifacts
         id: pack_artifacts
         run: |
@@ -1079,11 +1083,6 @@ jobs:
           path: cudart-llama-bin-win-cuda-${{ matrix.cuda }}-${{ matrix.arch }}.zip
           name: cudart-llama-bin-win-cuda-${{ matrix.cuda }}-${{ matrix.arch }}.zip
 
-      - name: ccache-clear
-        uses: ./.github/actions/ccache-clear
-        with:
-          key: release-windows-2022-${{ matrix.arch }}-cuda-${{ matrix.cuda }}
-
   windows-sycl:
     needs: [check-release]
     if: ${{ needs.check-release.outputs.should_release == 'true' }}
@@ -1143,6 +1142,11 @@ jobs:
             -DLLAMA_BUILD_BORINGSSL=ON
           cmake --build build --target ggml-sycl -j %NUMBER_OF_PROCESSORS%
 
+      - name: ccache-clear
+        uses: ./.github/actions/ccache-clear
+        with:
+          key: release-windows-2022-x64-sycl
+
       - name: Build the release package
         id: pack_artifacts
         run: |
@@ -1189,11 +1193,6 @@ jobs:
           path: llama-bin-win-sycl-x64.zip
           name: llama-bin-win-sycl-x64.zip
 
-      - name: ccache-clear
-        uses: ./.github/actions/ccache-clear
-        with:
-          key: release-windows-2022-x64-sycl
-
   ubuntu-24-sycl:
     needs: [check-release]
     if: ${{ needs.check-release.outputs.should_release == 'true' }}
@@ -1266,6 +1265,11 @@ jobs:
             -DGGML_SYCL_F16=${{ matrix.fp16 }}
           time cmake --build build --config Release -j $(nproc)
 
+      - name: ccache-clear
+        uses: ./.github/actions/ccache-clear
+        with:
+          key: release-ubuntu-24.04-sycl-${{ matrix.build }}
+
       - name: Determine tag name
         id: tag
         uses: ./.github/actions/get-tag-name
@@ -1282,139 +1286,123 @@ jobs:
           path: llama-${{ steps.tag.outputs.name }}-bin-ubuntu-sycl-${{ matrix.build }}-x64.tar.gz
           name: llama-bin-ubuntu-sycl-${{ matrix.build }}-x64.tar.gz
 
-      - name: ccache-clear
-        uses: ./.github/actions/ccache-clear
-        with:
-          key: release-ubuntu-24.04-sycl-${{ matrix.build }}
+#   ubuntu-22-rocm:
+#     needs: [check-release, get-version]
+#     if: ${{ needs.check-release.outputs.should_release == 'true' }}
 
-  ubuntu-22-rocm:
-    needs: [check-release, get-version]
-    if: ${{ needs.check-release.outputs.should_release == 'true' }}
+#     runs-on: ubuntu-22.04
 
-    runs-on: ubuntu-22.04
+#     permissions:
+#       actions: write
 
-    permissions:
-      actions: write
+#     strategy:
+#       matrix:
+#         include:
+#           - ROCM_VERSION: "7.14.0"
+#             gpu_targets: "gfx908;gfx90a;gfx942;gfx950;gfx1010;gfx1011;gfx1012;gfx1030;gfx1031;gfx1032;gfx1033;gfx1034;gfx1035;gfx1036;gfx1100;gfx1101;gfx1102;gfx1150;gfx1151;gfx1152;gfx1200;gfx1201"
+#             build: 'x64'
 
-    strategy:
-      matrix:
-        include:
-          - ROCM_VERSION: "7.14.0"
-            gpu_targets: "gfx908;gfx90a;gfx942;gfx950;gfx1010;gfx1011;gfx1012;gfx1030;gfx1031;gfx1032;gfx1033;gfx1034;gfx1035;gfx1036;gfx1100;gfx1101;gfx1102;gfx1150;gfx1151;gfx1152;gfx1200;gfx1201"
-            build: 'x64'
+#     steps:
+#       - name: Clone
+#         id: checkout
+#         uses: actions/checkout@v6
+#         with:
+#           fetch-depth: 0
 
-    steps:
-      - name: Clone
-        id: checkout
-        uses: actions/checkout@v6
-        with:
-          fetch-depth: 0
+#       - name: Setup Node.js
+#         uses: actions/setup-node@v6
+#         with:
+#           node-version: "24"
+#           cache: "npm"
+#           cache-dependency-path: "tools/ui/package-lock.json"
 
-      - name: Setup Node.js
-        uses: actions/setup-node@v6
-        with:
-          node-version: "24"
-          cache: "npm"
-          cache-dependency-path: "tools/ui/package-lock.json"
+#       - name: Free up disk space
+#         uses: ggml-org/free-disk-space@v1.3.1
+#         with:
+#           tool-cache: true
 
-      - name: Free up disk space
-        uses: ggml-org/free-disk-space@v1.3.1
-        with:
-          tool-cache: true
+#       # - name: ccache
+#       #   uses: ggml-org/ccache-action@v1.2.21
+#       #   with:
+#       #     key: release-ubuntu-22.04-rocm-${{ matrix.ROCM_VERSION }}
 
-      - name: ccache
-        uses: ggml-org/ccache-action@v1.2.21
-        with:
-          key: release-ubuntu-rocm-${{ matrix.ROCM_VERSION }}-${{ matrix.build }}
-          evict-old-files: 1d
-          max-size: "1G"
+#       - name: Dependencies
+#         id: depends
+#         run: |
+#           sudo apt install -y build-essential git cmake wget
 
-      - name: Tune ccache for reinstalled ROCm toolchain
-        run: |
-          # ROCm is pip-installed fresh each run, so the clang binary's mtime
-          # changes every time. With the default compiler_check=mtime that
-          # invalidates the cache; hash compiler contents instead so warm
-          # builds hit.
-          ccache --set-config=compiler_check=content
-          ccache --set-config=sloppiness=time_macros,include_file_mtime,include_file_ctime
+#       - name: Setup TheRock with Wheels
+#         id: therock_env
+#         run: |
+#           # Create Python virtual environment
+#           python3 -m venv .venv
+#           source .venv/bin/activate
 
-      - name: Dependencies
-        id: depends
-        run: |
-          sudo apt install -y build-essential git cmake wget
+#           # Install ROCm wheels for build
+#           # libraries = HIP runtime and CMake configs needed for linking
+#           # devel = compilers, headers, static libs
+#           python -m pip install --upgrade pip
+#           python -m pip install --index-url https://repo.amd.com/rocm/whl-multi-arch/ "rocm[libraries,devel]==${{ matrix.ROCM_VERSION }}"
 
-      - name: Setup TheRock with Wheels
-        id: therock_env
-        run: |
-          # Create Python virtual environment
-          python3 -m venv .venv
-          source .venv/bin/activate
+#           # Get ROCm installation paths using the rocm-sdk CLI tool
+#           ROCM_PATH=$(rocm-sdk path --root)
+#           CMAKE_PATH=$(rocm-sdk path --cmake)
+#           BIN_PATH=$(rocm-sdk path --bin)
+#           echo "ROCM_PATH=$ROCM_PATH"
+#           echo "CMAKE_PATH=$CMAKE_PATH"
+#           echo "BIN_PATH=$BIN_PATH"
 
-          # Install ROCm wheels for build
-          # libraries = HIP runtime and CMake configs needed for linking
-          # devel = compilers, headers, static libs
-          python -m pip install --upgrade pip
-          python -m pip install --index-url https://repo.amd.com/rocm/whl-multi-arch/ "rocm[libraries,devel]==${{ matrix.ROCM_VERSION }}"
+#           # Set environment variables
+#           echo "ROCM_PATH=$ROCM_PATH" >> $GITHUB_ENV
+#           echo "CMAKE_PREFIX_PATH=$CMAKE_PATH" >> $GITHUB_ENV
+#           echo "HIP_PATH=$ROCM_PATH" >> $GITHUB_ENV
+#           echo "PATH=$BIN_PATH:${PATH}" >> $GITHUB_ENV
+#           echo "LD_LIBRARY_PATH=$ROCM_PATH/lib:${LD_LIBRARY_PATH:-}" >> $GITHUB_ENV
 
-          # Get ROCm installation paths using the rocm-sdk CLI tool
-          ROCM_PATH=$(rocm-sdk path --root)
-          CMAKE_PATH=$(rocm-sdk path --cmake)
-          BIN_PATH=$(rocm-sdk path --bin)
-          echo "ROCM_PATH=$ROCM_PATH"
-          echo "CMAKE_PATH=$CMAKE_PATH"
-          echo "BIN_PATH=$BIN_PATH"
+#           # Keep venv activated for subsequent steps
+#           echo "$(pwd)/.venv/bin" >> $GITHUB_PATH
 
-          # Set environment variables
-          echo "ROCM_PATH=$ROCM_PATH" >> $GITHUB_ENV
-          echo "CMAKE_PREFIX_PATH=$CMAKE_PATH" >> $GITHUB_ENV
-          echo "HIP_PATH=$ROCM_PATH" >> $GITHUB_ENV
-          echo "PATH=$BIN_PATH:${PATH}" >> $GITHUB_ENV
-          echo "LD_LIBRARY_PATH=$ROCM_PATH/lib:${LD_LIBRARY_PATH:-}" >> $GITHUB_ENV
+#       - name: Build with native CMake HIP support
+#         id: cmake_build
+#         run: |
+#           cmake -B build -S . \
+#             -DCMAKE_HIP_COMPILER="$(hipconfig -l)/clang" \
+#             -DCMAKE_BUILD_TYPE=Release \
+#             -DGGML_BACKEND_DL=ON \
+#             -DGGML_NATIVE=OFF \
+#             -DCMAKE_INSTALL_RPATH='$ORIGIN' \
+#             -DCMAKE_BUILD_WITH_INSTALL_RPATH=ON \
+#             -DGGML_CPU_ALL_VARIANTS=ON \
+#             -DGPU_TARGETS="${{ matrix.gpu_targets }}" \
+#             -DGGML_HIP=ON \
+#             -DHIP_PLATFORM=amd \
+#             -DHF_UI_VERSION=${{ needs.get-version.outputs.ui_version }} \
+#             ${{ env.CMAKE_ARGS }}
+#           cmake --build build --config Release -j $(nproc)
 
-          # Keep venv activated for subsequent steps
-          echo "$(pwd)/.venv/bin" >> $GITHUB_PATH
+#       # - name: ccache-clear
+#       #   uses: ./.github/actions/ccache-clear
+#       #   with:
+#       #     key: release-ubuntu-22.04-rocm-${{ matrix.ROCM_VERSION }}
 
-      - name: Build with native CMake HIP support
-        id: cmake_build
-        run: |
-          cmake -B build -S . \
-            -DCMAKE_HIP_COMPILER="$(hipconfig -l)/clang" \
-            -DCMAKE_BUILD_TYPE=Release \
-            -DGGML_BACKEND_DL=ON \
-            -DGGML_NATIVE=OFF \
-            -DCMAKE_INSTALL_RPATH='$ORIGIN' \
-            -DCMAKE_BUILD_WITH_INSTALL_RPATH=ON \
-            -DGGML_CPU_ALL_VARIANTS=ON \
-            -DGPU_TARGETS="${{ matrix.gpu_targets }}" \
-            -DGGML_HIP=ON \
-            -DHIP_PLATFORM=amd \
-            -DHF_UI_VERSION=${{ needs.get-version.outputs.ui_version }} \
-            ${{ env.CMAKE_ARGS }}
-          cmake --build build --config Release -j $(nproc)
+#       - name: Determine tag name
+#         id: tag
+#         uses: ./.github/actions/get-tag-name
 
-      - name: Determine tag name
-        id: tag
-        uses: ./.github/actions/get-tag-name
+#       - name: Get ROCm short version
+#         run: echo "ROCM_VERSION_SHORT=$(echo '${{ matrix.ROCM_VERSION }}' | cut -d '.' -f 1,2)" >> $GITHUB_ENV
 
-      - name: Get ROCm short version
-        run: echo "ROCM_VERSION_SHORT=$(echo '${{ matrix.ROCM_VERSION }}' | cut -d '.' -f 1,2)" >> $GITHUB_ENV
+#       - name: Pack artifacts
+#         id: pack_artifacts
+#         run: |
+#           cp LICENSE ./build/bin/
+#           tar -czvf llama-${{ steps.tag.outputs.name }}-bin-ubuntu-rocm-${{ env.ROCM_VERSION_SHORT }}-${{ matrix.build }}.tar.gz --transform "s,^\.,llama-${{ steps.tag.outputs.name }}," -C ./build/bin .
 
-      - name: Pack artifacts
-        id: pack_artifacts
-        run: |
-          cp LICENSE ./build/bin/
-          tar -czvf llama-${{ steps.tag.outputs.name }}-bin-ubuntu-rocm-${{ env.ROCM_VERSION_SHORT }}-${{ matrix.build }}.tar.gz --transform "s,^\.,llama-${{ steps.tag.outputs.name }}," -C ./build/bin .
-
-      - name: Upload artifacts
-        uses: actions/upload-artifact@v6
-        with:
-          path: llama-${{ steps.tag.outputs.name }}-bin-ubuntu-rocm-${{ env.ROCM_VERSION_SHORT }}-${{ matrix.build }}.tar.gz
-          name: llama-bin-ubuntu-rocm-${{ env.ROCM_VERSION_SHORT }}-${{ matrix.build }}.tar.gz
-
-      - name: ccache-clear
-        uses: ./.github/actions/ccache-clear
-        with:
-          key: release-ubuntu-rocm-${{ matrix.ROCM_VERSION }}-${{ matrix.build }}
+#       - name: Upload artifacts
+#         uses: actions/upload-artifact@v6
+#         with:
+#           path: llama-${{ steps.tag.outputs.name }}-bin-ubuntu-rocm-${{ env.ROCM_VERSION_SHORT }}-${{ matrix.build }}.tar.gz
+#           name: llama-bin-ubuntu-rocm-${{ env.ROCM_VERSION_SHORT }}-${{ matrix.build }}.tar.gz
 
   ios-xcode:
     needs: [check-release, get-version]
@@ -1595,7 +1583,7 @@ jobs:
       - windows-sycl
       - windows-rocm
       - windows-openvino
-      - ubuntu-22-rocm
+      #- ubuntu-22-rocm
       - ubuntu-cpu
       - ubuntu-vulkan
       - ubuntu-24-openvino
@@ -1700,7 +1688,6 @@ jobs:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         with:
           tag_name: ${{ steps.tag.outputs.name }}
-          prerelease: true
           body: |
             <details open>
 
@@ -1726,7 +1713,7 @@ jobs:
             - [Ubuntu s390x (CPU)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-s390x.tar.gz)
             - [Ubuntu x64 (Vulkan)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-vulkan-x64.tar.gz)
             - [Ubuntu arm64 (Vulkan)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-vulkan-arm64.tar.gz)
-            - [Ubuntu x64 (ROCm 7.14)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-rocm-7.14-x64.tar.gz)
+            - Ubuntu x64 (ROCm 7.14)[DISABLED](https://github.com/ggml-org/llama.cpp/pull/26969)
             - [Ubuntu x64 (OpenVINO)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-openvino-${{ needs.ubuntu-24-openvino.outputs.openvino_version }}-x64.tar.gz)
             - [Ubuntu x64 (SYCL FP32)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-sycl-fp32-x64.tar.gz)
             - [Ubuntu x64 (SYCL FP16)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-sycl-fp16-x64.tar.gz)

.pi/gg/SYSTEM.md

@@ -17,7 +17,6 @@ Coding:
 Pull requests (PRs):
 - New branch names are prefixed with "gg/"
 - Before opening a pull request, ask the user to confirm the description
-- Don't explicitly wrap lines in the PR description (each paragraph and bullet is a single line)
 - When creating a pull request, look for the repository's PR template and follow it
 - For the AI usage disclosure section, write "YES. pi:llama.cpp/[MODEL]"
 - Ask the user to tell you what model was used and write it in place of [MODEL]

CMakeLists.txt

@@ -4,8 +4,8 @@ include(CheckIncludeFileCXX)
 
 ### llama.cpp version
 set(LLAMA_VERSION_MAJOR 0)
-set(LLAMA_VERSION_MINOR 2)
-set(LLAMA_VERSION_PATCH 0)
+set(LLAMA_VERSION_MINOR 1)
+set(LLAMA_VERSION_PATCH 2)
 set(LLAMA_VERSION_BASE "${LLAMA_VERSION_MAJOR}.${LLAMA_VERSION_MINOR}.${LLAMA_VERSION_PATCH}")
 
 # whether this is a development/nightly build

README.md

@@ -7,9 +7,9 @@
 <b>LLM inference in C/C++</b>
 
 [![License: MIT](https://img.shields.io/badge/license-MIT-blue.svg)](https://opensource.org/licenses/MIT)
-[![Release](https://img.shields.io/github/v/release/ggml-org/llama.cpp?filter=v*&color=brightgreen)](https://github.com/ggml-org/llama.cpp/releases?q=tag:v0)
-[![Nightly](https://img.shields.io/github/v/release/ggml-org/llama.cpp?label=nightly&filter=b*&color=orange)](https://github.com/ggml-org/llama.cpp/releases?q=b)
-[![Server](https://img.shields.io/github/actions/workflow/status/ggml-org/llama.cpp/server.yml?label=Server)](https://github.com/ggml-org/llama.cpp/actions/workflows/server.yml)
+[![Release](https://img.shields.io/github/v/release/ggml-org/llama.cpp?filter=v*)](https://github.com/ggml-org/llama.cpp/releases?q=tag:v0)
+[![Nightly](https://img.shields.io/github/v/release/ggml-org/llama.cpp?label=nightly)](https://github.com/ggml-org/llama.cpp/releases)
+[![Server](https://github.com/ggml-org/llama.cpp/actions/workflows/server.yml/badge.svg)](https://github.com/ggml-org/llama.cpp/actions/workflows/server.yml)
 [![Docker](https://img.shields.io/github/actions/workflow/status/ggml-org/llama.cpp/docker.yml?label=Docker)](https://github.com/ggml-org/llama.cpp/actions/workflows/docker.yml)
 [![Winget](https://img.shields.io/github/actions/workflow/status/ggml-org/llama.cpp/winget.yml?label=Winget)](https://github.com/ggml-org/llama.cpp/actions/workflows/winget.yml)
 

common/CMakeLists.txt

@@ -81,6 +81,8 @@ add_library(${TARGET}
     imatrix-loader.cpp
     imatrix-loader.h
     json-schema-to-grammar.cpp
+    json.cpp
+    json.h
     llguidance.cpp
     log.cpp
     log.h

common/arg.cpp

@@ -5,6 +5,7 @@
 #include "common.h"
 #include "download.h"
 #include "json-schema-to-grammar.h"
+#include "json.h"
 #include "llama.h"
 #include "log.h"
 #include "sampling.h"
@@ -21,9 +22,6 @@
 #include <shellapi.h>
 #endif
 
-#define JSON_ASSERT GGML_ASSERT
-#include <nlohmann/json.hpp>
-
 #include <algorithm>
 #include <cinttypes>
 #include <climits>
@@ -32,6 +30,7 @@
 #include <filesystem>
 #include <fstream>
 #include <list>
+#include <numeric>
 #include <regex>
 #include <set>
 #include <string>
@@ -55,7 +54,7 @@
 
 #define LLAMA_MAX_URL_LENGTH 2084 // Maximum URL Length in Chrome: 2083
 
-using json = nlohmann::ordered_json;
+using json = common_json;
 using namespace common_arg_utils;
 
 static std::initializer_list<enum llama_example> mmproj_examples = {

common/chat-auto-parser-generator.cpp

@@ -5,13 +5,12 @@
 #include "common.h"
 #include "json-schema-to-grammar.h"
 #include "log.h"
-#include "nlohmann/json.hpp"
 #include "peg-parser.h"
 
 #include <stdexcept>
 #include <string>
 
-using json = nlohmann::ordered_json;
+using json = common_json;
 
 // Helper to iterate over tools/functions
 static void foreach_function(const json & tools, const std::function<void(const json &)> & fn) {
@@ -391,7 +390,7 @@ common_peg_parser analyze_tools::build_tool_parser_tag_tagged(parser_build_conte
 
         std::set<std::string> required;
         if (params.contains("required")) {
-            params.at("required").get_to(required);
+            required = params.at("required").get<std::set<std::string>>();
         }
 
         auto schema_info = common_schema_info();

common/chat-auto-parser-helpers.cpp

@@ -4,14 +4,11 @@
 #include "chat-peg-parser.h"
 #include "chat.h"
 #include "log.h"
-#include "nlohmann/json.hpp"
 #include "peg-parser.h"
 
 #include <cctype>
 #include <numeric>
 
-using json = nlohmann::ordered_json;
-
 std::string trim_whitespace(const std::string & str) {
     size_t start = 0;
     while (start < str.length() && std::isspace(static_cast<unsigned char>(str[start]))) {

common/chat-auto-parser.h

@@ -4,7 +4,7 @@
 #include "common.h"
 #include "jinja/caps.h"
 #include "peg-parser.h"
-#include "nlohmann/json.hpp"
+#include "json.h"
 
 #include <chrono>
 #include <optional>
@@ -12,7 +12,7 @@
 #include <utility>
 #include <vector>
 
-using json = nlohmann::ordered_json;
+using json = common_json;
 
 class common_chat_peg_builder;
 

common/chat-diff-analyzer.cpp

@@ -4,11 +4,11 @@
 #include "chat.h"
 #include "common.h"
 #include "log.h"
-#include "nlohmann/json.hpp"
 #include "peg-parser.h"
 
 #include <algorithm>
 #include <cctype>
+#include <numeric>
 #include <ostream>
 #include <sstream>
 
@@ -17,7 +17,7 @@
 #define ANSI_ORANGE "\033[1m\x1b[38;5;214m"
 #define ANSI_RED    "\033[1m\x1b[38;5;196m"
 
-using json = nlohmann::ordered_json;
+using json = common_json;
 
 namespace autoparser {
 
@@ -929,7 +929,7 @@ void analyze_tools::analyze_tool_call_format_json_native(const std::string & cle
     int  json_end       = clean_haystack.find_last_of('}');
     std::string cut     = clean_haystack.substr(json_start, json_end - json_start + 1);
     json call_struct    = json::parse(cut);
-    auto register_field = [&](const std::string & prefix, const nlohmann::detail::iteration_proxy_value<json::iterator> & subel) {
+    auto register_field = [&](const std::string & prefix, const common_json_entry & subel) {
         if (subel.value().is_string() && std::string(subel.value()).find("call0000") != std::string::npos) {
             format.id_field = !prefix.empty() ? prefix + "." + subel.key() : subel.key();
         } else if (subel.value().is_string() && std::string(subel.value()) == fun_name_needle) {

common/chat-peg-parser.cpp

@@ -4,12 +4,10 @@
 #include "ggml.h"
 #include "peg-parser.h"
 
-#include <nlohmann/json.hpp>
-
 #include <cstdint>
 #include <functional>
 
-using ordered_json = nlohmann::ordered_json;
+using ordered_json = common_json;
 
 static std::string_view trim_trailing_space(std::string_view sv, int max = -1) {
     int count = 0;

common/chat-peg-parser.h

@@ -128,7 +128,7 @@ class common_chat_peg_builder : public common_peg_parser_builder {
     // parameters_order: order in which JSON fields should be parsed
     common_peg_parser standard_json_tools(const std::string &              section_start,
                                           const std::string &              section_end,
-                                          const nlohmann::ordered_json &   tools,
+                                          const common_json &   tools,
                                           bool                             parallel_tool_calls,
                                           bool                             force_tool_calls,
                                           const std::string &              name_key = "",
@@ -143,13 +143,13 @@ class common_chat_peg_builder : public common_peg_parser_builder {
     // Legacy-compatible helper for building XML/tagged style tool calls
     // Used by tests and manual parsers
     common_peg_parser standard_constructed_tools(const std::map<std::string, std::string> & markers,
-                                                 const nlohmann::ordered_json &             tools,
+                                                 const common_json &             tools,
                                                  bool                                       parallel_tool_calls,
                                                  bool                                       force_tool_calls);
 
     // Helper for Python-style function call format: name(arg1="value1", arg2=123)
     // Used by LFM2 and similar templates
-    common_peg_parser python_style_tool_calls(const nlohmann::ordered_json & tools,
+    common_peg_parser python_style_tool_calls(const common_json & tools,
                                               bool                           parallel_tool_calls,
                                               bool                           allow_json_literals);
 
@@ -158,19 +158,19 @@ class common_chat_peg_builder : public common_peg_parser_builder {
     common_peg_parser python_or_json_value();
 
     // Implementation helpers for standard_json_tools — one per JSON tool call layout mode
-    common_peg_parser build_json_tools_function_is_key(const nlohmann::ordered_json & tools,
+    common_peg_parser build_json_tools_function_is_key(const common_json & tools,
                                                        const std::string &            args_key,
                                                        const std::string &            effective_args_key,
                                                        const std::string &            call_id_key,
                                                        const std::string &            gen_call_id_key);
 
-    common_peg_parser build_json_tools_nested_keys(const nlohmann::ordered_json & tools,
+    common_peg_parser build_json_tools_nested_keys(const common_json & tools,
                                                    const std::string &            effective_name_key,
                                                    const std::string &            effective_args_key,
                                                    const std::string &            call_id_key,
                                                    const std::string &            gen_call_id_key);
 
-    common_peg_parser build_json_tools_flat_keys(const nlohmann::ordered_json &   tools,
+    common_peg_parser build_json_tools_flat_keys(const common_json &   tools,
                                                  const std::string &              effective_name_key,
                                                  const std::string &              effective_args_key,
                                                  const std::string &              call_id_key,

common/chat.cpp

@@ -6,6 +6,7 @@
 #include "common.h"
 #include "ggml.h"
 #include "json-schema-to-grammar.h"
+#include "json.h"
 #include "log.h"
 
 #include "jinja/value.h"
@@ -13,14 +14,13 @@
 #include "jinja/caps.h"
 #include "peg-parser.h"
 
-#include "nlohmann/json.hpp"
-
 #include <algorithm>
 #include <cstdio>
 #include <cstdlib>
 #include <ctime>
 #include <exception>
 #include <functional>
+#include <iomanip>
 #include <map>
 
 #include <optional>
@@ -30,7 +30,7 @@
 #include <utility>
 #include <vector>
 
-using json = nlohmann::ordered_json;
+using json = common_json;
 
 static std::string format_time(const std::chrono::system_clock::time_point & now, const std::string & format) {
     auto               time       = std::chrono::system_clock::to_time_t(now);
@@ -48,7 +48,7 @@ static json safe_args_parse(const std::string & to_parse) {
     }
     try {
         return json::parse(stripped);
-    } catch (json::exception & e) {
+    } catch (const common_json_error & e) {
         return stripped;
     }
 }
@@ -488,17 +488,17 @@ struct messages_inp_normalizer {
         json normalized = json::array();
         for (const auto & msg : messages) {
             json copy = msg;
-            auto it = copy.find("content");
-            if (it != copy.end()) {
-                if (only_typed && it->is_string()) {
-                    *it = json::array({
+            if (copy.contains("content")) {
+                json & it = copy.at("content");
+                if (only_typed && it.is_string()) {
+                    it = json::array({
                         json{
                             {"type", "text"},
-                            {"text", it->get<std::string>()},
+                            {"text", it.get<std::string>()},
                         }
                     });
-                } else if (only_string && it->is_array()) {
-                    *it = concat_content_parts(*it);
+                } else if (only_string && it.is_array()) {
+                    it = concat_content_parts(it);
                 }
             }
             normalized.push_back(std::move(copy));
@@ -608,7 +608,7 @@ std::vector<common_chat_tool> common_chat_tools_parse_oaicompat(const json & too
     return result;
 }
 
-common_chat_continuation common_chat_continuation_parse(const nlohmann::ordered_json & value) {
+common_chat_continuation common_chat_continuation_parse(const common_json & value) {
     if (value.is_boolean() && value.get<bool>()) {
         return COMMON_CHAT_CONTINUATION_AUTO;
     }
@@ -920,7 +920,7 @@ static void foreach_parameter(const json &
     const auto &          props = params.at("properties");
     std::set<std::string> required;
     if (params.contains("required") && params.at("required").is_array()) {
-        params.at("required").get_to(required);
+        required = params.at("required").get<std::set<std::string>>();
     }
     for (const auto & [name, prop] : props.items()) {
         bool is_required = (required.find(name) != required.end());
@@ -937,7 +937,7 @@ static std::string common_chat_template_direct_apply_impl(
     jinja::context ctx(tmpl.source());
 
     // messages_override is already built for this template, do not touch its content parts
-    nlohmann::ordered_json inp = nlohmann::ordered_json{
+    json inp = json{
         {"messages", messages_override.has_value()
             ? *messages_override
             : messages_inp_normalizer(tmpl.original_caps()).normalize(inputs.messages)},
@@ -1058,7 +1058,7 @@ static common_chat_params common_chat_params_init_ministral_3(const common_chat_
                 });
             } else if (msg.at("content").is_array()) {
                 auto blocks = msg.at("content");
-                content.insert(content.end(), blocks.begin(), blocks.end());
+                content.insert(blocks);
             }
         }
 
@@ -2238,7 +2238,7 @@ static common_chat_params common_chat_params_init_deepseek_v3_2(const common_cha
 
                 std::set<std::string> required;
                 if (params.contains("required")) {
-                    params.at("required").get_to(required);
+                    required = params.at("required").get<std::set<std::string>>();
                 }
 
                 auto schema_info = common_schema_info();
@@ -2860,7 +2860,7 @@ static common_chat_params common_chat_params_init_minimax_m3(const common_chat_t
 
                 std::set<std::string> required;
                 if (schema.contains("required")) {
-                    schema.at("required").get_to(required);
+                    required = schema.at("required").get<std::set<std::string>>();
                 }
 
                 std::vector<common_peg_parser> required_elements;
@@ -2972,10 +2972,10 @@ static void system_message_not_supported(json & messages) {
             auto & second_msg = messages[1];
             second_msg["content"] = first_msg.at("content").get<std::string>()
                 + "\n" + second_msg.at("content").get<std::string>();
-            messages.erase(messages.begin());
+            messages.erase(0);
         } else {
             LOG_WRN("Removing system prompt due to template not supporting system role\n");
-            messages.erase(messages.begin());
+            messages.erase(0);
         }
     }
 }

common/chat.h

@@ -8,7 +8,7 @@
 #include "jinja/runtime.h"
 #include "jinja/caps.h"
 
-#include "nlohmann/json_fwd.hpp"
+#include "json.h"
 
 #include <chrono>
 #include <functional>
@@ -17,7 +17,6 @@
 #include <vector>
 
 using chat_template_caps = jinja::caps;
-using json = nlohmann::ordered_json;
 
 struct common_chat_templates;
 
@@ -87,7 +86,7 @@ struct common_chat_msg {
     std::string                               tool_name;
     std::string                               tool_call_id;
 
-    nlohmann::ordered_json to_json_oaicompat(bool concat_typed_text = false) const;
+    common_json to_json_oaicompat(bool concat_typed_text = false) const;
 
     std::string render_content(const std::string & delimiter = "\n\n") const;
 
@@ -211,7 +210,7 @@ struct common_chat_msg_delimiters {
     // split tokens into message spans. skips maps a start index to a length of a region to jump over without matching
     common_chat_msg_spans split(const llama_tokens & tokens, const std::map<size_t, size_t> & skips = {}) const;
 
-    nlohmann::ordered_json to_json() const;
+    common_json to_json() const;
 };
 
 struct common_chat_tool {
@@ -350,16 +349,16 @@ common_chat_tool_choice common_chat_tool_choice_parse_oaicompat(const std::strin
 bool common_chat_templates_support_enable_thinking(const common_chat_templates * chat_templates);
 
 // Parses a JSON array of messages in OpenAI's chat completion API format.
-std::vector<common_chat_msg> common_chat_msgs_parse_oaicompat(const nlohmann::ordered_json & messages);
+std::vector<common_chat_msg> common_chat_msgs_parse_oaicompat(const common_json & messages);
 
-std::vector<common_chat_tool> common_chat_tools_parse_oaicompat(const nlohmann::ordered_json & tools);
+std::vector<common_chat_tool> common_chat_tools_parse_oaicompat(const common_json & tools);
 
-common_chat_continuation common_chat_continuation_parse(const nlohmann::ordered_json & value);
+common_chat_continuation common_chat_continuation_parse(const common_json & value);
 
 // DEPRECATED: only used in tests
-nlohmann::ordered_json common_chat_msgs_to_json_oaicompat(const std::vector<common_chat_msg> & msgs, bool concat_typed_text = false);
+common_json common_chat_msgs_to_json_oaicompat(const std::vector<common_chat_msg> & msgs, bool concat_typed_text = false);
 
-nlohmann::ordered_json common_chat_tools_to_json_oaicompat(const std::vector<common_chat_tool> & tools);
+common_json common_chat_tools_to_json_oaicompat(const std::vector<common_chat_tool> & tools);
 
 // get template caps, useful for reporting to server /props endpoint
 std::map<std::string, bool> common_chat_templates_get_caps(const common_chat_templates * chat_templates);
@@ -386,4 +385,4 @@ struct common_chat_prompt_preset {
 
 common_chat_prompt_preset common_chat_get_asr_prompt(const common_chat_templates * chat_templates);
 
-common_chat_msg_delimiters common_chat_msg_delimiters_parse(const nlohmann::ordered_json & delimiters);
+common_chat_msg_delimiters common_chat_msg_delimiters_parse(const common_json & delimiters);

common/download.cpp

@@ -5,9 +5,7 @@
 #include "log.h"
 #include "download.h"
 #include "hf-cache.h"
-
-#define JSON_ASSERT GGML_ASSERT
-#include <nlohmann/json.hpp>
+#include "json.h"
 
 #include <algorithm>
 #include <filesystem>
@@ -44,8 +42,6 @@
 #include <unistd.h>
 #endif
 
-using json = nlohmann::ordered_json;
-
 //
 // downloader
 //
@@ -856,8 +852,8 @@ static std::string common_docker_get_token(const std::string & repo) {
         throw std::runtime_error("Failed to get Docker registry token, HTTP code: " + std::to_string(res.first));
     }
 
-    std::string            response_str(res.second.begin(), res.second.end());
-    nlohmann::ordered_json response = nlohmann::ordered_json::parse(response_str);
+    std::string response_str(res.second.begin(), res.second.end());
+    common_json response = common_json::parse(response_str);
 
     if (!response.contains("token")) {
         throw std::runtime_error("Docker registry token response missing 'token' field");
@@ -919,9 +915,9 @@ std::string common_docker_resolve_model(const std::string & docker) {
             throw std::runtime_error("Failed to get Docker manifest, HTTP code: " + std::to_string(manifest_res.first));
         }
 
-        std::string            manifest_str(manifest_res.second.begin(), manifest_res.second.end());
-        nlohmann::ordered_json manifest = nlohmann::ordered_json::parse(manifest_str);
-        std::string            gguf_digest;  // Find the GGUF layer
+        std::string manifest_str(manifest_res.second.begin(), manifest_res.second.end());
+        common_json manifest = common_json::parse(manifest_str);
+        std::string gguf_digest;  // Find the GGUF layer
         if (manifest.contains("layers")) {
             for (const auto & layer : manifest["layers"]) {
                 if (layer.contains("mediaType")) {

common/hf-cache.cpp

@@ -4,9 +4,7 @@
 #include "common.h"
 #include "log.h"
 #include "http.h"
-
-#define JSON_ASSERT GGML_ASSERT
-#include <nlohmann/json.hpp>
+#include "json.h"
 
 #include <filesystem>
 #include <fstream>
@@ -15,8 +13,6 @@
 #include <string_view>
 #include <stdexcept>
 
-namespace nl = nlohmann;
-
 #if defined(_WIN32)
 #define WIN32_LEAN_AND_MEAN
 #ifndef NOMINMAX
@@ -195,8 +191,8 @@ static void safe_write_file(const fs::path & path, const std::string & data) {
     }
 }
 
-static nl::json api_get(const std::string & url,
-                        const std::string & token) {
+static common_json api_get(const std::string & url,
+                           const std::string & token) {
     auto [cli, parts] = common_http_client(url);
 
     httplib::Headers headers = {
@@ -214,10 +210,10 @@ static nl::json api_get(const std::string & url,
         auto body = res->body;
 
         if (res->status == 200) {
-            return nl::json::parse(res->body);
+            return common_json::parse(res->body);
         }
         try {
-            body = nl::json::parse(res->body)["error"].get<std::string>();
+            body = common_json::parse(res->body)["error"].get<std::string>();
         } catch (...) { }
 
         throw std::runtime_error("GET failed (" + std::to_string(res->status) + "): " + body);
@@ -280,7 +276,7 @@ static std::string get_repo_commit(const std::string & repo_id,
         safe_write_file(refs_path / name, commit);
         return commit;
 
-    } catch (const nl::json::exception & e) {
+    } catch (const common_json_error & e) {
         LOG_ERR("%s: JSON error: %s\n", __func__, e.what());
     } catch (const std::exception & e) {
         LOG_ERR("%s: error: %s\n", __func__, e.what());
@@ -358,7 +354,7 @@ hf_files get_repo_files(const std::string & repo_id,
 
             files.push_back(file);
         }
-    } catch (const nl::json::exception & e) {
+    } catch (const common_json_error & e) {
         LOG_ERR("%s: JSON error: %s\n", __func__, e.what());
     } catch (const std::exception & e) {
         LOG_ERR("%s: error: %s\n", __func__, e.what());

common/jinja/README.md

@@ -7,7 +7,7 @@ The implementation can be found in the `common/jinja` directory.
 ## Key Features
 
 - Input marking: security against special token injection
-- Decoupled from `nlohmann::json`: this dependency is only used for JSON-to-internal type translation and is completely optional
+- Decoupled from the JSON library: `common_json` is only used for JSON-to-internal type translation and is completely optional
 - Minimal primitive types: int, float, bool, string, array, object, none, undefined
 - Detailed logging: allow source tracing on error
 - Clean architecture: workarounds are applied to input data before entering the runtime (see `common/chat.cpp`)

common/jinja/caps.cpp

@@ -4,14 +4,14 @@
 
 // note: the json dependency is only for defining input in a convenient way
 // we can remove it in the future when we figure out a better way to define inputs using jinja::value
-#include <nlohmann/json.hpp>
+#include "json.h"
 
 #include <functional>
 #include <sstream>
 
 #define FILENAME "jinja-caps"
 
-using json = nlohmann::ordered_json;
+using json = common_json;
 
 namespace jinja {
 

common/jinja/value.cpp

@@ -3,7 +3,7 @@
 #include "value.h"
 
 // for converting from JSON to jinja values
-#include <nlohmann/json.hpp>
+#include "json.h"
 
 #include <sstream>
 #include <string>
@@ -1355,7 +1355,7 @@ const func_builtins & value_undefined_t::get_builtins() const {
 //////////////////////////////////
 
 
-static value from_json(const nlohmann::ordered_json & j, bool mark_input) {
+static value from_json(const common_json & j, bool mark_input) {
     if (j.is_null()) {
         return mk_val<value_none>();
     } else if (j.is_boolean()) {
@@ -1452,7 +1452,7 @@ bool value_compare(const value & a, const value & b, value_compare_op op) {
 }
 
 template<>
-void global_from_json(context & ctx, const nlohmann::ordered_json & json_obj, bool mark_input) {
+void global_from_json(context & ctx, const common_json & json_obj, bool mark_input) {
     // printf("global_from_json: %s\n" , json_obj.dump(2).c_str());
     if (json_obj.is_null() || !json_obj.is_object()) {
         throw std::runtime_error("global_from_json: input JSON value must be an object");

common/jinja/value.h

@@ -86,7 +86,7 @@ struct context; // forward declaration
 // marking input can be useful for tracking data provenance
 // and preventing template injection attacks
 //
-// Note: T_JSON can be nlohmann::ordered_json
+// Note: T_JSON can be common_json
 template<typename T_JSON>
 void global_from_json(context & ctx, const T_JSON & json_obj, bool mark_input);
 

common/json-schema-to-grammar.cpp

@@ -1,9 +1,8 @@
 #include "json-schema-to-grammar.h"
 #include "common.h"
 
-#include <nlohmann/json.hpp>
-
 #include <algorithm>
+#include <limits>
 #include <map>
 #include <regex>
 #include <sstream>
@@ -12,7 +11,7 @@
 #include <unordered_set>
 #include <vector>
 
-using json = nlohmann::ordered_json;
+using json = common_json;
 
 static std::string build_repetition(const std::string & item_rule, int min_items, int max_items, const std::string & separator_rule = "") {
     auto has_max = max_items != std::numeric_limits<int>::max();
@@ -917,7 +916,11 @@ public:
             return _add_rule(rule_name, _resolve_ref(schema["$ref"]));
         }
         if (schema.contains("oneOf") || schema.contains("anyOf")) {
-            std::vector<json> alt_schemas = schema.contains("oneOf") ? schema["oneOf"].get<std::vector<json>>() : schema["anyOf"].get<std::vector<json>>();
+            const json & alts = schema.contains("oneOf") ? schema.at("oneOf") : schema.at("anyOf");
+            std::vector<json> alt_schemas;
+            for (const auto & alt : alts) {
+                alt_schemas.push_back(alt);
+            }
             return _add_rule(rule_name, _generate_union_rule(name, alt_schemas));
         }
         if (schema_type.is_array()) {
@@ -1111,7 +1114,7 @@ common_schema_info::~common_schema_info() = default;
 common_schema_info::common_schema_info(common_schema_info &&) noexcept = default;
 common_schema_info & common_schema_info::operator=(common_schema_info &&) noexcept = default;
 
-void common_schema_info::resolve_refs(nlohmann::ordered_json & schema) {
+void common_schema_info::resolve_refs(common_json & schema) {
     impl_->resolve_refs(schema, "");
 }
 
@@ -1119,7 +1122,7 @@ void common_schema_info::resolve_refs(nlohmann::ordered_json & schema) {
 // Some models emit raw string values rather than JSON-encoded strings for string parameters.
 // If any branch of the schema (via oneOf, anyOf, $ref, etc.) permits a string, this returns
 // true, allowing callers to handle the value as a raw string for simplicity.
-bool common_schema_info::resolves_to_string(const nlohmann::ordered_json & schema) {
+bool common_schema_info::resolves_to_string(const common_json & schema) {
     std::unordered_set<std::string> visited_refs;
 
     std::function<bool(const json &)> check = [&](const json & s) -> bool {
@@ -1227,7 +1230,7 @@ bool common_schema_info::resolves_to_string(const nlohmann::ordered_json & schem
     return check(schema);
 }
 
-std::string json_schema_to_grammar(const json & schema, bool force_gbnf) {
+std::string json_schema_to_grammar(const common_json & schema, bool force_gbnf) {
 #ifdef LLAMA_USE_LLGUIDANCE
     if (!force_gbnf) {
         return "%llguidance {}\nstart: %json " + schema.dump();
@@ -1248,10 +1251,10 @@ std::string build_grammar(const std::function<void(const common_grammar_builder
         /* .add_rule = */ [&](const std::string & name, const std::string & rule) {
             return converter._add_rule(name, rule);
         },
-        /* .add_schema = */ [&](const std::string & name, const nlohmann::ordered_json & schema) {
+        /* .add_schema = */ [&](const std::string & name, const common_json & schema) {
             return converter.visit(schema, name == "root" ? "" : name);
         },
-        /* .resolve_refs = */ [&](nlohmann::ordered_json & schema) {
+        /* .resolve_refs = */ [&](common_json & schema) {
             converter.resolve_refs(schema, "");
         }
     };

common/json-schema-to-grammar.h

@@ -1,12 +1,12 @@
 #pragma once
 
-#include <nlohmann/json_fwd.hpp>
+#include "json.h"
 
 #include <functional>
 #include <memory>
 #include <string>
 
-std::string json_schema_to_grammar(const nlohmann::ordered_json & schema,
+std::string json_schema_to_grammar(const common_json & schema,
                                    bool force_gbnf = false);
 
 class common_schema_converter;
@@ -24,14 +24,14 @@ class common_schema_info {
     common_schema_info(common_schema_info &&) noexcept;
     common_schema_info & operator=(common_schema_info &&) noexcept;
 
-    void resolve_refs(nlohmann::ordered_json & schema);
-    bool resolves_to_string(const nlohmann::ordered_json & schema);
+    void resolve_refs(common_json & schema);
+    bool resolves_to_string(const common_json & schema);
 };
 
 struct common_grammar_builder {
     std::function<std::string(const std::string &, const std::string &)> add_rule;
-    std::function<std::string(const std::string &, const nlohmann::ordered_json &)> add_schema;
-    std::function<void(nlohmann::ordered_json &)> resolve_refs;
+    std::function<std::string(const std::string &, const common_json &)> add_schema;
+    std::function<void(common_json &)> resolve_refs;
 };
 
 struct common_grammar_options {

common/json.cpp

@@ -0,0 +1,437 @@
+#include "json.h"
+
+#include "ggml.h"
+
+#define JSON_ASSERT GGML_ASSERT
+#include <nlohmann/json.hpp>
+
+#include <iterator>
+#include <new>
+#include <set>
+#include <unordered_map>
+#include <vector>
+
+using nlohmann::ordered_json;
+
+// a common_json is the backing value, so any value of a tree can be used as a common_json
+static_assert(sizeof(ordered_json)  <= sizeof(common_json),  "common_json storage is too small");
+static_assert(alignof(ordered_json) <= alignof(common_json), "common_json alignment is too weak");
+
+// runs fn and gives every error of the backing library as a common_json_error
+template <typename F>
+static decltype(auto) guard(F && fn) {
+    try {
+        return fn();
+    } catch (const ordered_json::exception & e) {
+        throw common_json_error(e.what());
+    }
+}
+
+static ordered_json & as_json(common_json * self) {
+    return *reinterpret_cast<ordered_json *>(self);
+}
+
+static const ordered_json & as_json(const common_json * self) {
+    return *reinterpret_cast<const ordered_json *>(self);
+}
+
+static common_json & as_common(ordered_json & json) {
+    return *reinterpret_cast<common_json *>(&json);
+}
+
+static const common_json & as_common(const ordered_json & json) {
+    return *reinterpret_cast<const common_json *>(&json);
+}
+
+static ordered_json to_json(const common_json_value & val) {
+    switch (val.type) {
+        case common_json_value::VAL_NULL:   return nullptr;
+        case common_json_value::VAL_BOOL:   return val.val_bool;
+        case common_json_value::VAL_INT:    return val.val_int;
+        case common_json_value::VAL_UINT:   return val.val_uint;
+        case common_json_value::VAL_DOUBLE: return val.val_double;
+        case common_json_value::VAL_STRING: return val.val_string;
+        case common_json_value::VAL_JSON:
+            // one owner means no one else can see this tree, so it is safe to move it out
+            // note: this makes a value single use, same as the json_ref of the backing library
+            if (val.val_json.use_count() == 1) {
+                return std::move(as_json(val.val_json.get()));
+            }
+            return as_json(val.val_json.get());
+    }
+
+    return nullptr;
+}
+
+common_json_value::common_json_value(const char * val) {
+    if (val) {
+        type       = VAL_STRING;
+        val_string = val;
+    } else {
+        type = VAL_NULL;
+    }
+}
+
+common_json_value::common_json_value(const common_json & val) :
+    type(VAL_JSON), val_json(std::make_shared<common_json>(val)) {}
+
+common_json_value::common_json_value(common_json && val) :
+    type(VAL_JSON), val_json(std::make_shared<common_json>(std::move(val))) {}
+
+template <typename T>
+common_json_value::common_json_value(const std::set<T> & vals) : type(VAL_JSON) {
+    common_json out = common_json::array();
+
+    for (const auto & val : vals) {
+        out.push_back(val);
+    }
+
+    val_json = std::make_shared<common_json>(std::move(out));
+}
+
+// a set value is usable only for the types below
+#define COMMON_JSON_SET(...) template common_json_value::common_json_value(const std::set<__VA_ARGS__> &);
+
+COMMON_JSON_SET(int)
+COMMON_JSON_SET(std::string)
+
+#undef COMMON_JSON_SET
+
+template <typename T>
+common_json_value::common_json_value(const std::map<std::string, T> & vals) : type(VAL_JSON) {
+    common_json out = common_json::object();
+
+    for (const auto & val : vals) {
+        out.set({ val.first, val.second });
+    }
+
+    val_json = std::make_shared<common_json>(std::move(out));
+}
+
+// a map value is usable only for the types below
+#define COMMON_JSON_MAP(...) template common_json_value::common_json_value(const std::map<std::string, __VA_ARGS__> &);
+
+COMMON_JSON_MAP(bool)
+COMMON_JSON_MAP(std::string)
+
+#undef COMMON_JSON_MAP
+
+template <typename T>
+common_json_value::common_json_value(const std::unordered_map<std::string, T> & vals) : type(VAL_JSON) {
+    common_json out = common_json::object();
+
+    for (const auto & val : vals) {
+        out.set({ val.first, val.second });
+    }
+
+    val_json = std::make_shared<common_json>(std::move(out));
+}
+
+// an unordered map value is usable only for the types below
+#define COMMON_JSON_UMAP(...) template common_json_value::common_json_value(const std::unordered_map<std::string, __VA_ARGS__> &);
+
+COMMON_JSON_UMAP(size_t)
+
+#undef COMMON_JSON_UMAP
+
+template <typename T>
+common_json_value::common_json_value(const std::vector<T> & vals) : type(VAL_JSON) {
+    common_json out = common_json::array();
+
+    for (const auto & val : vals) {
+        out.push_back(val);
+    }
+
+    val_json = std::make_shared<common_json>(std::move(out));
+}
+
+// a vector value is usable only for the types below
+// note: std::vector<bool> is not here, its proxy reference does not convert
+#define COMMON_JSON_VEC(...) template common_json_value::common_json_value(const std::vector<__VA_ARGS__> &);
+
+COMMON_JSON_VEC(int)
+COMMON_JSON_VEC(unsigned char)
+COMMON_JSON_VEC(unsigned int)
+COMMON_JSON_VEC(long)
+COMMON_JSON_VEC(unsigned long)
+COMMON_JSON_VEC(long long)
+COMMON_JSON_VEC(unsigned long long)
+COMMON_JSON_VEC(float)
+COMMON_JSON_VEC(double)
+COMMON_JSON_VEC(std::string)
+COMMON_JSON_VEC(std::vector<float>)
+COMMON_JSON_VEC(common_json)
+
+#undef COMMON_JSON_VEC
+
+common_json_value::common_json_value(std::initializer_list<common_json_item> items) :
+    type(VAL_JSON), val_json(std::make_shared<common_json>(items)) {}
+
+// null, same as the backing library
+// operator[] turns it into an object, push_back() into an array
+common_json::common_json() {
+    new (storage) ordered_json();
+}
+
+common_json::common_json(const common_json & other) {
+    new (storage) ordered_json(as_json(&other));
+}
+
+common_json::common_json(common_json && other) noexcept {
+    new (storage) ordered_json(std::move(as_json(&other)));
+}
+
+common_json::common_json(std::initializer_list<common_json_item> items) {
+    new (storage) ordered_json(ordered_json::object());
+
+    for (const auto & item : items) {
+        set(item);
+    }
+}
+
+common_json::common_json(const common_json_value & val) {
+    new (storage) ordered_json(to_json(val));
+}
+
+common_json::common_json(std::nullptr_t) {
+    new (storage) ordered_json(nullptr);
+}
+
+common_json & common_json::operator=(common_json other) noexcept {
+    as_json(this).swap(as_json(&other));
+
+    return *this;
+}
+
+common_json::~common_json() {
+    as_json(this).~basic_json();
+}
+
+common_json common_json::parse(const std::string & text) {
+    try {
+        // the assignment moves the parsed tree in, it does not copy
+        common_json out;
+        as_json(&out) = ordered_json::parse(text);
+        return out;
+    } catch (const std::exception & e) {
+        throw common_json_error(e.what());
+    }
+}
+
+common_json common_json::parse_no_throw(const std::string & text) {
+    common_json out;
+    as_json(&out) = ordered_json::parse(text, nullptr, false);
+    return out;
+}
+
+bool common_json::is_discarded() const {
+    return as_json(this).is_discarded();
+}
+
+common_json common_json::array() {
+    common_json out;
+    as_json(&out) = ordered_json::array();
+    return out;
+}
+
+common_json common_json::array(std::initializer_list<common_json_value> vals) {
+    common_json out;
+    ordered_json & arr = as_json(&out);
+    arr = ordered_json::array();
+
+    for (const auto & val : vals) {
+        arr.push_back(to_json(val));
+    }
+
+    return out;
+}
+
+common_json common_json::object() {
+    common_json out;
+    as_json(&out) = ordered_json::object();
+    return out;
+}
+
+common_json common_json::object(std::initializer_list<common_json_item> items) {
+    return common_json(items);
+}
+
+common_json common_json::make(const common_json_value & val) {
+    return common_json(val);
+}
+
+bool common_json::is_null()           const { return as_json(this).is_null(); }
+bool common_json::is_object()         const { return as_json(this).is_object(); }
+bool common_json::is_array()          const { return as_json(this).is_array(); }
+bool common_json::is_string()         const { return as_json(this).is_string(); }
+bool common_json::is_boolean()        const { return as_json(this).is_boolean(); }
+bool common_json::is_number()         const { return as_json(this).is_number(); }
+bool common_json::is_number_integer() const { return as_json(this).is_number_integer(); }
+bool common_json::is_number_float()   const { return as_json(this).is_number_float(); }
+
+bool   common_json::empty() const { return as_json(this).empty(); }
+size_t common_json::size()  const { return as_json(this).size(); }
+
+bool common_json::contains(const std::string & key) const {
+    return as_json(this).contains(key);
+}
+
+bool common_json::operator==(const common_json_value & val) const {
+    // compare a tree in place, to_json() would copy it
+    if (val.type == common_json_value::VAL_JSON) {
+        return as_json(this) == as_json(val.val_json.get());
+    }
+    return as_json(this) == to_json(val);
+}
+
+bool common_json::operator!=(const common_json_value & val) const {
+    return !(*this == val);
+}
+
+common_json       & common_json::at(const std::string & key)       { return guard([&]() -> common_json       & { return as_common(as_json(this).at(key)); }); }
+const common_json & common_json::at(const std::string & key) const { return guard([&]() -> const common_json & { return as_common(as_json(this).at(key)); }); }
+common_json       & common_json::at(size_t idx)                    { return guard([&]() -> common_json       & { return as_common(as_json(this).at(idx)); }); }
+const common_json & common_json::at(size_t idx)              const { return guard([&]() -> const common_json & { return as_common(as_json(this).at(idx)); }); }
+
+common_json       & common_json::operator[](const std::string & key)       { return guard([&]() -> common_json       & { return as_common(as_json(this)[key]); }); }
+const common_json & common_json::operator[](const std::string & key) const { return guard([&]() -> const common_json & { return as_common(as_json(this).at(key)); }); }
+common_json       & common_json::operator[](size_t idx)                    { return guard([&]() -> common_json       & { return as_common(as_json(this)[idx]); }); }
+const common_json & common_json::operator[](size_t idx)              const { return guard([&]() -> const common_json & { return as_common(as_json(this).at(idx)); }); }
+
+common_json       & common_json::front()       { return as_common(as_json(this).front()); }
+const common_json & common_json::front() const { return as_common(as_json(this).front()); }
+common_json       & common_json::back()        { return as_common(as_json(this).back()); }
+const common_json & common_json::back()  const { return as_common(as_json(this).back()); }
+
+void common_json::clear() {
+    as_json(this).clear();
+}
+
+void common_json::erase(const std::string & key) {
+    guard([&] { as_json(this).erase(key); });
+}
+
+void common_json::erase(size_t idx) {
+    guard([&] { as_json(this).erase(idx); });
+}
+
+void common_json::assign(const common_json_value & val) {
+    as_json(this) = to_json(val);
+}
+
+void common_json::set(const common_json_item & item) {
+    guard([&] { as_json(this)[item.key] = to_json(item.val); });
+}
+
+void common_json::push_back(const common_json_value & val) {
+    guard([&] { as_json(this).push_back(to_json(val)); });
+}
+
+void common_json::push_back(std::initializer_list<common_json_item> items) {
+    common_json val(items);
+
+    guard([&] { as_json(this).push_back(std::move(as_json(&val))); });
+}
+
+size_t common_json::count(const std::string & key) const {
+    return as_json(this).count(key);
+}
+
+void common_json::insert(const common_json & vals) {
+    guard([&] {
+        ordered_json & self = as_json(this);
+
+        self.insert(self.end(), as_json(&vals).begin(), as_json(&vals).end());
+    });
+}
+
+std::string common_json::dump(int indent) const {
+    return guard([&] { return as_json(this).dump(indent); });
+}
+
+std::string common_json::dump_safe(int indent) const {
+    return as_json(this).dump(indent, ' ', false, ordered_json::error_handler_t::replace);
+}
+
+// an array is indexed directly, an object needs a walk from the start
+common_json & common_json::iterator::operator*() const {
+    return guard([&]() -> common_json & {
+        ordered_json & j = as_json(node);
+
+        if (j.is_object()) {
+            return as_common(std::next(j.begin(), idx).value());
+        }
+        if (j.is_array()) {
+            return as_common(j[idx]);
+        }
+
+        // a plain value gives itself once, same as the backing library
+        return *node;
+    });
+}
+
+std::string common_json::iterator::key() const {
+    return guard([&] { return std::next(as_json(node).begin(), idx).key(); });
+}
+
+common_json::iterator common_json::begin() const {
+    return iterator(const_cast<common_json *>(this), 0);
+}
+
+common_json::iterator common_json::end() const {
+    return iterator(const_cast<common_json *>(this), size());
+}
+
+// the keys follow the backing library: the index for an array, "" for a plain value
+common_json::items_view::entry common_json::items_view::iterator::operator*() const {
+    return guard([&]() -> entry {
+        ordered_json & j = as_json(node);
+
+        if (j.is_object()) {
+            auto it = std::next(j.begin(), idx);
+
+            return { it.key(), as_common(it.value()) };
+        }
+        if (j.is_array()) {
+            return { std::to_string(idx), as_common(j[idx]) };
+        }
+
+        return { std::string(), *node };
+    });
+}
+
+common_json::items_view common_json::items() const {
+    return items_view(const_cast<common_json *>(this), size());
+}
+
+template <typename T> T common_json::get() const {
+    return guard([&] { return as_json(this).get<T>(); });
+}
+
+// the backing library cannot build a common_json, so this one is just a copy
+template <> common_json common_json::get<common_json>() const {
+    return *this;
+}
+
+// get<T>() is usable only for the types below
+
+#define COMMON_JSON_GET(...) template __VA_ARGS__ common_json::get<__VA_ARGS__>() const;
+
+COMMON_JSON_GET(bool)
+COMMON_JSON_GET(int)
+COMMON_JSON_GET(unsigned int)
+COMMON_JSON_GET(long)
+COMMON_JSON_GET(unsigned long)
+COMMON_JSON_GET(long long)
+COMMON_JSON_GET(unsigned long long)
+COMMON_JSON_GET(float)
+COMMON_JSON_GET(double)
+COMMON_JSON_GET(std::string)
+COMMON_JSON_GET(std::vector<float>)
+COMMON_JSON_GET(std::vector<std::string>)
+COMMON_JSON_GET(std::set<std::string>)
+COMMON_JSON_GET(std::vector<int>)
+COMMON_JSON_GET(std::vector<size_t>)
+COMMON_JSON_GET(std::unordered_map<std::string, size_t>)
+
+#undef COMMON_JSON_GET

common/json.h

@@ -0,0 +1,350 @@
+#pragma once
+
+// JSON object, it works without the need to include a JSON library header
+// the underlay library is pimpl, it should never be exposed here
+// the backing value lives inside this object, so at() and the iterators give a real reference to it
+// note: object keys keep the order in which they are added
+// note: every JSON error comes out as a common_json_error
+
+#include <cstddef>
+#include <cstdint>
+#include <initializer_list>
+#include <iterator>
+#include <map>
+#include <memory>
+#include <set>
+#include <stdexcept>
+#include <string>
+#include <string_view>
+#include <type_traits>
+#include <unordered_map>
+#include <utility>
+#include <vector>
+
+class common_json;
+
+// common_json_value holds a list of these, and each of them holds a value, so one must come first
+struct common_json_item;
+
+struct common_json_error : std::runtime_error {
+    using std::runtime_error::runtime_error;
+};
+
+// one value, tagged so that this header stays free of the backing library
+// note: a value that holds a tree is single use, the second use gives null
+struct common_json_value {
+    enum value_type {
+        VAL_NULL,
+        VAL_BOOL,
+        VAL_INT,
+        VAL_UINT,
+        VAL_DOUBLE,
+        VAL_STRING,
+        VAL_JSON,
+    };
+
+    value_type type = VAL_NULL;
+
+    union {
+        bool     val_bool;
+        int64_t  val_int;
+        uint64_t val_uint = 0;
+        double   val_double;
+    };
+
+    std::string                  val_string;
+    std::shared_ptr<common_json> val_json;
+
+    common_json_value(std::nullptr_t = nullptr) : type(VAL_NULL) {}
+    common_json_value(bool val) : type(VAL_BOOL), val_bool(val) {}
+    common_json_value(std::string val) : type(VAL_STRING), val_string(std::move(val)) {}
+    // without this a string_view lands on the common_json ctor below and recurses
+    common_json_value(std::string_view val) : type(VAL_STRING), val_string(val) {}
+    common_json_value(const char * val);
+    common_json_value(const common_json & val);
+    common_json_value(common_json && val);
+    // only for the types instantiated in json.cpp, the rest fails at link time
+    template <typename T> common_json_value(const std::vector<T> & vals);
+    // a set becomes an array, in the set's own order
+    template <typename T> common_json_value(const std::set<T> & vals);
+    // a map becomes an object, keyed in the map's own order
+    template <typename T> common_json_value(const std::map<std::string, T> & vals);
+    template <typename T> common_json_value(const std::unordered_map<std::string, T> & vals);
+
+    // nested object, e.g. {"fn", {{"name", "x"}}}
+    // note: a nested pair {"a", "b"} becomes the object {"a": "b"}, not an array
+    // use common_json::array({"a", "b"}) to get an array
+    common_json_value(std::initializer_list<common_json_item> items);
+
+    template <typename T, typename std::enable_if<std::is_integral<T>::value && !std::is_same<T, bool>::value, int>::type = 0>
+    common_json_value(T val) : type(std::is_signed<T>::value ? VAL_INT : VAL_UINT) {
+        if (std::is_signed<T>::value) {
+            val_int = (int64_t) val;
+        } else {
+            val_uint = (uint64_t) val;
+        }
+    }
+
+    template <typename T, typename std::enable_if<std::is_floating_point<T>::value, int>::type = 0>
+    common_json_value(T val) : type(VAL_DOUBLE), val_double((double) val) {}
+};
+
+struct common_json_item {
+    std::string       key;
+    common_json_value val;
+
+    template <typename T>
+    common_json_item(std::string key, T && val) :
+        key(std::move(key)), val(std::forward<T>(val)) {}
+
+    // a braced list cannot deduce T, so it needs its own overload
+    common_json_item(std::string key, std::initializer_list<common_json_item> items) :
+        key(std::move(key)), val(items) {}
+};
+
+// the types common_json_value holds on its own
+// anything else reaches its common_json ctor and recurses forever
+template <typename T> struct common_json_is_value : std::integral_constant<bool,
+    std::is_arithmetic<T>::value ||
+    std::is_same<T, std::nullptr_t>::value ||
+    std::is_same<T, std::string>::value ||
+    std::is_same<T, std::string_view>::value ||
+    std::is_same<T, char *>::value ||
+    std::is_same<T, const char *>::value ||
+    std::is_same<T, common_json>::value> {};
+
+template <typename T, typename A>
+struct common_json_is_value<std::vector<T, A>> : std::true_type {};
+
+template <typename T, typename C, typename A>
+struct common_json_is_value<std::set<T, C, A>> : std::true_type {};
+
+template <typename V, typename C, typename A>
+struct common_json_is_value<std::map<std::string, V, C, A>> : std::true_type {};
+
+template <typename V, typename H, typename E, typename A>
+struct common_json_is_value<std::unordered_map<std::string, V, H, E, A>> : std::true_type {};
+
+class common_json {
+  public:
+    common_json();
+    common_json(const common_json & other);
+    common_json(common_json && other) noexcept;
+    common_json(std::initializer_list<common_json_item> items);
+    common_json(const common_json_value & val);
+
+    // direct, a value would need two conversions in a row
+    common_json(std::nullptr_t);
+
+    // one step, so that "abc" or a vector can go straight into a common_json
+    template <typename T, typename std::enable_if<!std::is_same<typename std::decay<T>::type, common_json>::value &&
+                                                  !std::is_same<typename std::decay<T>::type, common_json_value>::value, int>::type = 0>
+    common_json(T && val) : common_json(common_json_value(std::forward<T>(val))) {
+        static_assert(common_json_is_value<typename std::decay<T>::type>::value,
+                      "no common_json_value ctor holds this type, add one instead of letting it recurse");
+    }
+
+    // by value, same as the backing library
+    // the right side is copied before the left side can invalidate it, e.g. msg["a"] = msg.at("b")
+    common_json & operator=(common_json other) noexcept;
+
+    ~common_json();
+
+    // throws common_json_error if the text is not valid JSON
+    static common_json parse(const std::string & text);
+
+    // gives a discarded value instead of throwing, check it with is_discarded()
+    static common_json parse_no_throw(const std::string & text);
+
+    bool is_discarded() const;
+
+    static common_json array();
+    static common_json array(std::initializer_list<common_json_value> vals);
+    static common_json object();
+    static common_json object(std::initializer_list<common_json_item> items);
+
+    // holds a single value, e.g. make("abc").dump() gives "\"abc\""
+    static common_json make(const common_json_value & val);
+
+    bool is_null()    const;
+    bool is_object()  const;
+    bool is_array()   const;
+    bool is_string()  const;
+    bool is_boolean() const;
+    bool is_number()  const;
+    bool is_number_integer() const;
+    bool is_number_float()   const;
+
+    bool   empty() const;
+    size_t size()  const;
+
+    bool contains(const std::string & key) const;
+
+    bool operator==(const common_json_value & val) const;
+    bool operator!=(const common_json_value & val) const;
+
+    // at() throws common_json_error if the key is missing, operator[] adds a null value instead
+    common_json       & at(const std::string & key);
+    const common_json & at(const std::string & key) const;
+    common_json       & at(size_t idx);
+    const common_json & at(size_t idx) const;
+
+    common_json       & operator[](const std::string & key);
+    const common_json & operator[](const std::string & key) const;
+    common_json       & operator[](const char * key)       { return (*this)[std::string(key)]; }
+    const common_json & operator[](const char * key) const { return (*this)[std::string(key)]; }
+    common_json       & operator[](int idx)       { return (*this)[to_idx(idx)]; }
+    const common_json & operator[](int idx) const { return (*this)[to_idx(idx)]; }
+    common_json       & operator[](size_t idx);
+    const common_json & operator[](size_t idx) const;
+
+    common_json       & front();
+    const common_json & front() const;
+    common_json       & back();
+    const common_json & back()  const;
+
+    void clear();
+
+    void erase(const std::string & key);
+    void erase(size_t idx);
+
+    // only for the types instantiated in json.cpp, the rest fails at link time
+    template <typename T> T get() const;
+
+    // implicit get<T>() for plain values, so they can be assigned to their C++ type directly
+    // note: kept to this short list on purpose, a wider one makes j["key"] ambiguous
+    // note: a numeric one would make "str = json;" ambiguous, a number converts to char too
+    operator std::string() const { return get<std::string>(); }
+
+    template <typename T>
+    T value(const std::string & key, T def) const {
+        return contains(key) ? at(key).get<T>() : def;
+    }
+
+    std::string value(const std::string & key, const char * def) const {
+        return contains(key) ? at(key).get<std::string>() : std::string(def);
+    }
+
+    // a JSON default needs no get<T>(), it is already the right type
+    common_json value(const std::string & key, const common_json & def) const {
+        return contains(key) ? at(key) : def;
+    }
+
+    void assign(const common_json_value & val);
+    void set(const common_json_item & item);
+    void push_back(const common_json_value & val);
+
+    // appends one object, e.g. push_back({{"a", 1}})
+    void push_back(std::initializer_list<common_json_item> items);
+
+    // 1 if the key is there, 0 if not
+    size_t count(const std::string & key) const;
+
+    // appends every value of another array; inserting an array into itself throws
+    void insert(const common_json & vals);
+
+    // a common_json goes through the copy assignment above, everything else becomes a value
+    template <typename T, typename std::enable_if<!std::is_same<typename std::decay<T>::type, common_json>::value, int>::type = 0>
+    common_json & operator=(T && val) {
+        assign(common_json_value(std::forward<T>(val)));
+        return *this;
+    }
+
+    std::string dump(int indent = -1) const;
+
+    // same as dump(), but bad UTF-8 gets replaced instead of throwing
+    std::string dump_safe(int indent = -1) const;
+
+    // walks an array by index, or an object in insertion order
+    // a plain value gives itself once, same as the backing library
+    class iterator {
+      public:
+        using iterator_category = std::forward_iterator_tag;
+        using value_type        = common_json;
+        using difference_type   = std::ptrdiff_t;
+        using pointer           = common_json *;
+        using reference         = common_json &;
+
+        iterator(common_json * node, size_t idx) : node(node), idx(idx) {}
+
+        common_json & operator*() const;
+        common_json & value()     const { return **this; }
+        std::string   key()       const;
+
+        iterator & operator++() {
+            idx++;
+            return *this;
+        }
+
+        bool operator!=(const iterator & other) const { return idx != other.idx; }
+        bool operator==(const iterator & other) const { return idx == other.idx; }
+
+      private:
+        common_json * node;
+        size_t        idx;
+    };
+
+    iterator begin() const;
+    iterator end()   const;
+
+    // allows: for (const auto & [key, val] : obj.items())
+    class items_view {
+      public:
+        // the members are public, so an entry also works with structured bindings
+        struct entry {
+            std::string   k;
+            common_json & v;
+
+            const std::string & key()   const { return k; }
+            common_json &       value() const { return v; }
+        };
+
+        items_view(common_json * node, size_t n) : node(node), n(n) {}
+
+        class iterator {
+          public:
+            iterator(common_json * node, size_t idx) : node(node), idx(idx) {}
+
+            entry operator*() const;
+
+            iterator & operator++() {
+                idx++;
+                return *this;
+            }
+
+            bool operator!=(const iterator & other) const { return idx != other.idx; }
+
+          private:
+            common_json * node;
+            size_t        idx;
+        };
+
+        iterator begin() const { return iterator(node, 0); }
+        iterator end()   const { return iterator(node, n); }
+
+      private:
+        common_json * node;
+        size_t        n;
+    };
+
+    items_view items() const;
+
+  private:
+    // a negative index must not turn into a huge size_t
+    static size_t to_idx(int idx) {
+        if (idx < 0) {
+            throw common_json_error("negative array index");
+        }
+        return (size_t) idx;
+    }
+
+    // the backing value is built here, json.cpp checks that it fits
+    // it cannot be a pointer: a value inside a tree would then not be a common_json
+    // at() could then only give back a copy instead of a real reference
+    alignas(8) unsigned char storage[32];
+};
+
+// json.cpp defines this specialization, it must be declared before any use of it
+template <> common_json common_json::get<common_json>() const;
+
+using common_json_entry = common_json::items_view::entry;

common/peg-parser.cpp

@@ -10,7 +10,6 @@
 #include <initializer_list>
 #include <map>
 #include <memory>
-#include <nlohmann/json.hpp>
 #include <regex>
 #include <set>
 #include <stdexcept>
@@ -1120,8 +1119,8 @@ common_peg_parser common_peg_parser_builder::chars(const std::string & classes,
     return wrap(arena_.add_parser(common_peg_chars_parser{classes, ranges, negated, min, max}));
 }
 
-common_peg_parser common_peg_parser_builder::schema(const common_peg_parser & p, const std::string & name, const nlohmann::ordered_json & schema, bool raw) {
-    return wrap(arena_.add_parser(common_peg_schema_parser{p.id(), name, std::make_shared<nlohmann::ordered_json>(schema), raw}));
+common_peg_parser common_peg_parser_builder::schema(const common_peg_parser & p, const std::string & name, const common_json & schema, bool raw) {
+    return wrap(arena_.add_parser(common_peg_schema_parser{p.id(), name, std::make_shared<common_json>(schema), raw}));
 }
 
 common_peg_parser common_peg_parser_builder::rule(const std::string & name, const common_peg_parser & p, bool trigger) {
@@ -1805,8 +1804,8 @@ void common_peg_arena::build_grammar(const common_grammar_builder & builder, boo
     }
 }
 
-static nlohmann::json serialize_parser_variant(const common_peg_parser_variant & variant) {
-    using json = nlohmann::json;
+static common_json serialize_parser_variant(const common_peg_parser_variant & variant) {
+    using json = common_json;
 
     return std::visit([](const auto & p) -> json {
         using T = std::decay_t<decltype(p)>;
@@ -1860,7 +1859,7 @@ static nlohmann::json serialize_parser_variant(const common_peg_parser_variant &
                 {"type", "schema"},
                 {"child", p.child},
                 {"name", p.name},
-                {"schema", p.schema ? *p.schema : nullptr},
+                {"schema", p.schema ? *p.schema : json(nullptr)},
                 {"raw", p.raw}
             };
         } else if constexpr (std::is_same_v<T, common_peg_rule_parser>) {
@@ -1888,19 +1887,19 @@ static nlohmann::json serialize_parser_variant(const common_peg_parser_variant &
     }, variant);
 }
 
-nlohmann::json common_peg_arena::to_json() const {
-    auto parsers = nlohmann::json::array();
+common_json common_peg_arena::to_json() const {
+    auto parsers = common_json::array();
     for (const auto & parser : parsers_) {
         parsers.push_back(serialize_parser_variant(parser));
     }
-    return nlohmann::json{
+    return common_json{
         {"parsers", parsers},
         {"rules", rules_},
         {"root", root_}
     };
 }
 
-static common_peg_parser_variant deserialize_parser_variant(const nlohmann::json & j) {
+static common_peg_parser_variant deserialize_parser_variant(const common_json & j) {
     if (!j.contains("type") || !j["type"].is_string()) {
         throw std::runtime_error("Parser variant JSON missing or invalid 'type' field");
     }
@@ -1969,9 +1968,9 @@ static common_peg_parser_variant deserialize_parser_variant(const nlohmann::json
         }
         common_peg_chars_parser parser;
         parser.pattern = j["pattern"];
-        parser.negated = j["negated"];
-        parser.min_count = j["min_count"];
-        parser.max_count = j["max_count"];
+        parser.negated = j["negated"].get<bool>();
+        parser.min_count = j["min_count"].get<int>();
+        parser.max_count = j["max_count"].get<int>();
         for (const auto & range_json : j["ranges"]) {
             if (!range_json.contains("start") || !range_json.contains("end")) {
                 throw std::runtime_error("char_range missing 'start' or 'end' field");
@@ -2007,7 +2006,7 @@ static common_peg_parser_variant deserialize_parser_variant(const nlohmann::json
         parser.child = j["child"].get<common_peg_parser_id>();
         parser.name = j["name"];
         if (!j["schema"].is_null()) {
-            parser.schema = std::make_shared<nlohmann::ordered_json>(j["schema"]);
+            parser.schema = std::make_shared<common_json>(j["schema"]);
         }
         parser.raw = j["raw"].get<bool>();
         return parser;
@@ -2069,7 +2068,7 @@ static common_peg_parser_variant deserialize_parser_variant(const nlohmann::json
     throw std::runtime_error("Unknown parser type: " + type);
 }
 
-common_peg_arena common_peg_arena::from_json(const nlohmann::json & j) {
+common_peg_arena common_peg_arena::from_json(const common_json & j) {
     if (!j.contains("parsers") || !j["parsers"].is_array()) {
         throw std::runtime_error("JSON missing or invalid 'parsers' array");
     }
@@ -2109,7 +2108,7 @@ std::string common_peg_arena::save() const {
 }
 
 void common_peg_arena::load(const std::string & data) {
-    *this = from_json(nlohmann::json::parse(data));
+    *this = from_json(common_json::parse(data));
 }
 
 common_peg_arena build_peg_parser(const std::function<common_peg_parser(common_peg_parser_builder & builder)> & fn) {

common/peg-parser.h

@@ -1,6 +1,6 @@
 #pragma once
 
-#include <nlohmann/json_fwd.hpp>
+#include "json.h"
 
 #include <memory>
 #include <set>
@@ -245,7 +245,7 @@ struct common_peg_until_parser {
 struct common_peg_schema_parser {
     common_peg_parser_id child;
     std::string name;
-    std::shared_ptr<nlohmann::ordered_json> schema;
+    std::shared_ptr<common_json> schema;
 
     // Indicates if the GBNF should accept a raw string that matches the schema.
     bool raw;
@@ -332,8 +332,8 @@ class common_peg_arena {
 
     std::string dump(common_peg_parser_id id) const;
 
-    nlohmann::json to_json() const;
-    static common_peg_arena from_json(const nlohmann::json & j);
+    common_json to_json() const;
+    static common_peg_arena from_json(const common_json & j);
 
     std::string save() const;
     void load(const std::string & data);
@@ -490,7 +490,7 @@ class common_peg_parser_builder {
 
     // Wraps a parser with JSON schema metadata for grammar generation.
     // Used internally to convert JSON schemas to GBNF grammar rules.
-    common_peg_parser schema(const common_peg_parser & p, const std::string & name, const nlohmann::ordered_json & schema, bool raw = false);
+    common_peg_parser schema(const common_peg_parser & p, const std::string & name, const common_json & schema, bool raw = false);
 
     // Creates a named rule, stores it in the grammar, and returns a ref.
     // If trigger=true, marks this rule as an entry point for lazy grammar generation.

common/speculative.cpp

@@ -2322,9 +2322,6 @@ common_params common_base_params_to_speculative(const common_params & params) {
     const auto & params_spec = params.speculative.draft;
     common_params result = params;
 
-    result.embedding    = false;
-    result.pooling_type = LLAMA_POOLING_TYPE_UNSPECIFIED;
-
     if (has_draft) {
         result.devices               = params_spec.devices;
         result.model                 = params_spec.mparams;

conversion/__init__.py

@@ -58,16 +58,12 @@ TEXT_MODEL_MAP: dict[str, str] = {
     "DSparkDraftModel": "qwen",
     "DSparkSpeculator": "qwen",
     "Lfm2DSparkDraftModel": "qwen",
-    "LingDSparkModel": "qwen",
     "DeepseekV4ForCausalLM": "deepseek",
     "DeepseekV4DSparkModel": "deepseek",
     "DistilBertForMaskedLM": "bert",
     "DistilBertForSequenceClassification": "bert",
     "DistilBertModel": "bert",
     "Dots1ForCausalLM": "dots1",
-    "Dots3NoteForCausalLM": "dots3",
-    "Dots3NoteForConditionalGeneration": "dots3",
-    "Dots3NoteTextForCausalLM": "dots3",
     "DotsOCRForCausalLM": "qwen",
     "DreamModel": "dream",
     "Ernie4_5ForCausalLM": "ernie",
@@ -283,8 +279,6 @@ MMPROJ_MODEL_MAP: dict[str, str] = {
     "CogVLMForCausalLM": "cogvlm",
     "DeepseekOCR2ForCausalLM": "deepseek",
     "DeepseekOCRForCausalLM": "deepseek",
-    "Dots3NoteForCausalLM": "dots3",
-    "Dots3NoteForConditionalGeneration": "dots3",
     "DotsOCRForCausalLM": "dotsocr",
     "Exaone4_5_ForConditionalGeneration": "exaone",
     "Gemma3ForConditionalGeneration": "gemma",

conversion/dots3.py

@@ -1,323 +0,0 @@
-from __future__ import annotations
-
-import math
-import re
-
-import torch
-
-from typing import TYPE_CHECKING, Any, Callable, Iterable
-
-if TYPE_CHECKING:
-    from torch import Tensor
-
-from .base import MmprojModel, ModelBase, gguf
-
-from .deepseek import DeepseekV2Model
-
-
-@ModelBase.register("Dots3NoteForCausalLM", "Dots3NoteForConditionalGeneration", "Dots3NoteTextForCausalLM")
-class Dots3NoteModel(DeepseekV2Model):
-    model_arch = gguf.MODEL_ARCH.DOTS3NOTE
-    skip_mtp = False
-    supports_mtp_export = True
-
-    # trunk layer count, stashed before indexing for filter_tensors (mirrors DeepseekV32Model)
-    _n_main_layers: int | None = None
-
-    def index_tensors(self, remote_hf_model_id: str | None = None):
-        type(self)._n_main_layers = self.hparams["num_hidden_layers"]
-        return super().index_tensors(remote_hf_model_id=remote_hf_model_id)
-
-    def __init__(self, *args, **kwargs):
-        super().__init__(*args, **kwargs)
-
-        hparams = self.hparams
-
-        # config file doesn't specify MTP block, detect it from model weight
-        self.n_nextn = 1 if "model.mtp.embed_tokens.weight" in self.model_tensors else 0
-        if self.n_nextn:
-            self.block_count += self.n_nextn
-            self.tensor_map = gguf.get_tensor_name_map(self.model_arch, self.block_count)
-
-        self.layer_types = hparams["layer_types"]
-        if len(self.layer_types) < hparams["num_hidden_layers"]:
-            raise ValueError("layer_types is shorter than num_hidden_layers")
-
-        if hparams.get("use_dsa", True) is not True:
-            raise ValueError("dots3-note conversion requires use_dsa=true")
-        if hparams.get("normalization", "RMSNorm") != "RMSNorm" or hparams.get("final_norm", "RMSNorm") != "RMSNorm":
-            raise ValueError("dots3-note conversion only supports RMSNorm")
-        if hparams.get("k_rope_only_layernorm", True) is not True:
-            raise ValueError("dots3-note conversion requires k_rope_only_layernorm=true")
-        if hparams.get("topk_method", "noaux_tc") != "noaux_tc" or hparams.get("scoring_func") != "sigmoid":
-            raise ValueError("dots3-note conversion only supports noaux_tc/sigmoid expert gating")
-        if hparams.get("n_group", 1) != 1 or hparams.get("topk_group", 1) != 1:
-            raise ValueError("dots3-note conversion does not support grouped expert routing")
-        if hparams.get("use_dynamic_rsf", False) or hparams.get("moe_gating_fp32", False):
-            raise ValueError("dots3-note conversion does not support use_dynamic_rsf/moe_gating_fp32")
-        for key in ("attention_gate_type", "swa_attention_gate_type"):
-            if hparams.get(key, "headwise") != "headwise":
-                raise ValueError(f"dots3-note conversion only supports headwise attention gate, got {key}={hparams.get(key)!r}")
-        if hparams["swa_qk_nope_head_dim"] + hparams["swa_qk_rope_head_dim"] != hparams.get("swa_head_dim", 256):
-            raise ValueError("swa_head_dim must equal swa_qk_nope_head_dim + swa_qk_rope_head_dim")
-        if hparams["swa_qk_rope_head_dim"] != hparams["qk_rope_head_dim"]:
-            # both layer kinds share a single rope_dimension_count
-            raise ValueError("swa_qk_rope_head_dim must match qk_rope_head_dim")
-
-        self.apply_lora_rescale = hparams.get("apply_mla_qkv_lora_rescale", False)
-
-    def _is_swa_layer(self, bid: int) -> bool:
-        if bid >= self.hparams["num_hidden_layers"]:
-            # note: the NextN/MTP block uses the sliding-attention MLA
-            return True
-        return self.layer_types[bid] == "sliding_attention"
-
-    def set_vocab(self):
-        from transformers import AutoTokenizer
-        tokenizer = AutoTokenizer.from_pretrained(self.dir_model)
-        special_vocab = gguf.SpecialVocab(self.dir_model, load_merges=True)
-        tokens, toktypes, tokpre = self.get_vocab_base()
-        self.gguf_writer.add_tokenizer_model("gpt2")
-        self.gguf_writer.add_tokenizer_pre(tokpre)
-        self.gguf_writer.add_token_list(tokens)
-        self.gguf_writer.add_token_types(toktypes)
-        special_vocab._set_special_token("eot", tokenizer.get_added_vocab()["<|endofassistant|>"])  # ty: ignore[unresolved-attribute]
-        special_vocab.add_to_gguf(self.gguf_writer)
-
-    @classmethod
-    def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
-        if (titem := super().filter_tensors(item)) is None:
-            return None
-        name, gen = titem
-        if name.startswith(("vision_encoder.", "audio_encoder.")):
-            return None
-
-        assert cls._n_main_layers is not None
-        is_mtp = name.startswith("model.mtp.") or \
-            ((m := re.match(r"model\.layers\.(\d+)\.", name)) is not None and int(m.group(1)) >= cls._n_main_layers)
-
-        # --no-mtp: drop the NextN/MTP block; --mtp: keep only that block plus the shared embeddings/norm/lm_head
-        if is_mtp and cls.no_mtp:
-            return None
-        if cls.mtp_only and not is_mtp and name not in (
-            "model.embed_tokens.weight", "model.norm.weight", "lm_head.weight",
-        ):
-            return None
-
-        return name, gen
-
-    def set_gguf_parameters(self):
-        hparams = self.hparams
-
-        # head_count is a per-layer array because the two layer kinds have different head counts
-        n_layer = hparams["num_hidden_layers"]
-        hparams["num_attention_heads"] = [
-            hparams["swa_num_attention_heads"] if self._is_swa_layer(il) else hparams["num_attention_heads"]
-            for il in range(self.block_count)
-        ]
-
-        # prevent the base class from emitting key/value_length from the unused head_dim
-        hparams.pop("head_dim", None)
-
-        super().set_gguf_parameters()
-
-        # MLA geometry of the sliding-window layers (rope.freq_base_swa is emitted by the base class)
-        swa_kv_lora_rank = hparams["swa_kv_lora_rank"]
-        self.gguf_writer.add_sliding_window(hparams["sliding_window_size"])
-        self.gguf_writer.add_sliding_window_pattern([self._is_swa_layer(il) for il in range(n_layer)])
-        self.gguf_writer.add_kv_lora_rank_swa(swa_kv_lora_rank)
-        self.gguf_writer.add_key_length_swa(swa_kv_lora_rank + hparams["swa_qk_rope_head_dim"])
-        self.gguf_writer.add_value_length_swa(swa_kv_lora_rank)
-        self.gguf_writer.add_key_length_mla_swa(hparams["swa_qk_nope_head_dim"] + hparams["swa_qk_rope_head_dim"])
-        self.gguf_writer.add_value_length_mla_swa(hparams["swa_v_head_dim"])
-        if hparams["swa_q_lora_rank"] != hparams["q_lora_rank"]:
-            raise ValueError("dots3-note conversion assumes a shared q_lora_rank for both layer kinds")
-
-        if self.n_nextn:
-            self.gguf_writer.add_nextn_predict_layers(self.n_nextn)
-
-        # DSA indexer (full-attention layers only)
-        self.gguf_writer.add_indexer_head_count(hparams["index_n_heads"])
-        self.gguf_writer.add_indexer_key_length(hparams["index_head_dim"])
-        self.gguf_writer.add_indexer_top_k(hparams["index_topk"])
-        self.gguf_writer.add_indexer_types([not self._is_swa_layer(il) for il in range(n_layer)])
-
-    def prepare_metadata(self, vocab_only: bool):
-        from_dir = self.fname_out.is_dir()
-        super().prepare_metadata(vocab_only=vocab_only)
-
-        if not self.mtp_only or not from_dir:
-            return
-
-        output_type: str = self.ftype.name.partition("_")[2]
-        fname_default: str = gguf.naming_convention(
-            self.metadata.name, self.metadata.basename, self.metadata.finetune,
-            self.metadata.version, size_label=None, output_type=output_type, model_type=None)
-        self.fname_out = self.fname_out.parent / f"mtp-{fname_default}.gguf"
-
-    def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
-        # move the MTP token embedding into the NextN block so the standard nextn mapping picks it up
-        if name == "model.mtp.embed_tokens.weight":
-            name = f"model.layers.{self.hparams['num_hidden_layers']}.embed_tokens.weight"
-            bid = self.hparams["num_hidden_layers"]
-
-        # fold the activation rescale sqrt(n_embd/lora_rank) into the preceding RMSNorm weight
-        # this also covers the indexer wq_b, which reads the same rescaled q_lora activation
-        if self.apply_lora_rescale and bid is not None:
-            if name.endswith("q_a_layernorm.weight"):
-                data_torch = data_torch * math.sqrt(self.hparams["hidden_size"] / self.hparams["q_lora_rank"])
-            elif name.endswith("kv_a_layernorm.weight"):
-                rank = self.hparams["swa_kv_lora_rank"] if self._is_swa_layer(bid) else self.hparams["kv_lora_rank"]
-                data_torch = data_torch * math.sqrt(self.hparams["hidden_size"] / rank)
-
-        # MLA absorption: split kv_b_proj into k_b (transposed) and v_b, per-layer-kind geometry
-        if name.endswith("kv_b_proj.weight"):
-            assert bid is not None
-            if self._is_swa_layer(bid):
-                n_head = self.hparams["swa_num_attention_heads"]
-                qk_nope_head_dim = self.hparams["swa_qk_nope_head_dim"]
-                v_head_dim = self.hparams["swa_v_head_dim"]
-            else:
-                n_head = self.hparams["num_attention_heads"]
-                qk_nope_head_dim = self.hparams["qk_nope_head_dim"]
-                v_head_dim = self.hparams["v_head_dim"]
-            if isinstance(n_head, list):  # set_gguf_parameters turns this into a per-layer array
-                n_head = n_head[bid]
-
-            assert data_torch.shape[0] == n_head * (qk_nope_head_dim + v_head_dim)
-
-            kv_b = data_torch.view(n_head, qk_nope_head_dim + v_head_dim, data_torch.shape[-1])
-            k_b, v_b = kv_b.split([qk_nope_head_dim, v_head_dim], dim=1)
-            k_b = k_b.transpose(1, 2)
-
-            yield from ModelBase.modify_tensors(self, k_b, name.replace("kv_b_proj", "k_b_proj"), bid)
-            yield from ModelBase.modify_tensors(self, v_b, name.replace("kv_b_proj", "v_b_proj"), bid)
-            return
-
-        yield from super().modify_tensors(data_torch, name, bid)
-
-
-@ModelBase.register("Dots3NoteForCausalLM", "Dots3NoteForConditionalGeneration")
-class Dots3NoteMmprojModel(MmprojModel):
-    has_vision_encoder = True
-    has_audio_encoder = True
-
-    def __init__(self, *args, **kwargs):
-        super().__init__(*args, **kwargs)
-        assert self.hparams_vision is not None
-        assert self.hparams_audio is not None
-
-        # preprocessor_config.json nests the image params under vision_config
-        self.preprocessor_config = {**self.preprocessor_config, **self.preprocessor_config.get("vision_config", {})}
-
-        vis = self.hparams_vision
-        # in this config, hidden_size is the adapter output width; embed_dim is the tower width
-        vis["hidden_size"] = vis["embed_dim"]
-        vis["image_size"] = 0  # dynamic resolution
-        self.pyramid = [max(0, n) for n in vis["pyramid_num_routed"]]
-
-        if vis.get("adapter_type") != "patch_merger" or not vis.get("pre_pixel_shuffle"):
-            raise ValueError("dots3-note vision conversion requires adapter_type=patch_merger and pre_pixel_shuffle")
-        if vis.get("router_scoring_func", "sigmoid") != "sigmoid" or vis.get("router_scale", 1.0) != 1.0:
-            raise ValueError("dots3-note vision conversion only supports sigmoid routing with router_scale=1.0")
-        if vis.get("temporal_patch_size", 1) != 1 or vis.get("use_bias") or not vis.get("use_qk_norm"):
-            raise ValueError("unsupported dots3-note vision config variant")
-
-        aud = self.hparams_audio
-        if not aud.get("use_conv2d_stem") or not aud.get("use_rope") or not aud.get("use_rms_norm") or aud.get("use_causal"):
-            raise ValueError("unsupported dots3-note audio config variant")
-        if aud["whisper_config"].get("activation_function") != "swiglu":
-            raise ValueError("dots3-note audio conversion requires the swiglu activation")
-        if aud.get("merge_factor", 1) != 1 or aud.get("chunk_seconds") != 60:
-            raise ValueError("unsupported dots3-note audio chunking config")
-        # the graph hard-codes these rope parameters
-        rope = aud.get("rope_parameters", {})
-        if rope.get("partial_rotary_factor") != 0.5 or rope.get("rope_theta") != 10000.0:
-            raise ValueError("unsupported dots3-note audio rope config")
-
-    def get_audio_config(self) -> dict[str, Any] | None:
-        cfg = self.global_config.get("audio_config")
-        if cfg is not None:
-            # aliases so MmprojModel.find_aparam() / n_block_keys can resolve them
-            whisper = cfg["whisper_config"]
-            cfg["hidden_size"] = whisper["d_model"]
-            cfg["intermediate_size"] = whisper["encoder_ffn_dim"]
-            cfg["num_attention_heads"] = whisper["encoder_attention_heads"]
-            cfg["num_hidden_layers"] = whisper["encoder_layers"]
-        return cfg
-
-    def set_gguf_parameters(self):
-        super().set_gguf_parameters()
-        assert self.hparams_vision is not None
-        assert self.hparams_audio is not None
-
-        self.gguf_writer.add_clip_vision_projector_type(gguf.VisionProjectorType.DOTS3NOTE_V)
-        self.gguf_writer.add_vision_use_silu(True)
-        self.gguf_writer.add_vision_attention_layernorm_eps(self.hparams_vision["rms_norm_eps"])
-        self.gguf_writer.add_vision_spatial_merge_size(self.hparams_vision["spatial_merge_size"])
-        self.gguf_writer.add_vision_min_pixels(self.preprocessor_config["min_pixels"])
-        self.gguf_writer.add_vision_max_pixels(self.preprocessor_config["max_pixels"])
-        # pyramid MoE: per-block routed expert count, 0 = dense block
-        self.gguf_writer.add_vision_expert_count_per_layer(self.pyramid)
-        self.gguf_writer.add_vision_expert_used_count(int(self.hparams_vision["capacity_factor"]))
-
-        self.gguf_writer.add_clip_audio_projector_type(gguf.VisionProjectorType.DOTS3NOTE_A)
-        self.gguf_writer.add_audio_num_mel_bins(self.hparams_audio["whisper_config"]["num_mel_bins"])
-        self.gguf_writer.add_audio_attention_layernorm_eps(1e-6)  # Dots3NoteAudioRMSNorm default
-
-    @classmethod
-    def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
-        name, _ = item
-        if not name.startswith(("vision_encoder.", "audio_encoder.")):
-            return None
-        return super().filter_tensors(item)
-
-    _vis_experts: dict[int, dict[str, Tensor]] | None = None
-
-    def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
-        # router params have no .weight suffix in the checkpoint, but gguf tools expect one
-        if name.endswith((".gate_weight", ".router_bias")):
-            name += ".weight"
-
-        # audio fc1 fuses gate and up for swiglu; split it
-        if ".speech_encoder.layers." in name and ".fc1." in name:
-            gate, up = data_torch.chunk(2, dim=0)
-            yield from super().modify_tensors(gate, name.replace(".fc1.", ".fc1_gate."), bid)
-            yield from super().modify_tensors(up, name.replace(".fc1.", ".fc1_up."), bid)
-            return
-
-        # vision MoE: stack per-expert weights into a single 3D tensor per block
-        if ".mlp.experts." in name:
-            assert bid is not None
-            n_expert = self.pyramid[bid]
-            if self._vis_experts is None:
-                self._vis_experts = {}
-            buf = self._vis_experts.setdefault(bid, {})
-            buf[name] = data_torch
-
-            if len(buf) >= n_expert * 3:
-                for w_name in ("fc1", "fc2", "fc3"):
-                    datas: list[Tensor] = []
-                    for xid in range(n_expert):
-                        ename = f"vision_encoder.blocks.{bid}.mlp.experts.{xid}.{w_name}.weight"
-                        datas.append(buf.pop(ename))
-                    merged = torch.stack(datas, dim=0)
-                    yield from super().modify_tensors(merged, f"vision_encoder.blocks.{bid}.mlp.experts.{w_name}.weight", bid)
-            return
-
-        yield from super().modify_tensors(data_torch, name, bid)
-
-    def prepare_tensors(self):
-        super().prepare_tensors()
-        if self._vis_experts is not None:
-            leftover = [k for d in self._vis_experts.values() for k in d.keys()]
-            if leftover:
-                raise ValueError(f"unprocessed vision experts: {leftover}")
-
-    def tensor_force_quant(self, name, new_name, bid, n_dims):
-        # FP32 routing is load-bearing for the vision MoE (near-tied expert scores)
-        if ".ffn_gate_inp." in new_name or ".exp_probs_b." in new_name:
-            return gguf.GGMLQuantizationType.F32
-        if ".conv2d" in new_name or "a.conv_out" in new_name:
-            return gguf.GGMLQuantizationType.F32
-        return super().tensor_force_quant(name, new_name, bid, n_dims)

conversion/qwen.py

@@ -709,13 +709,7 @@ class DFlashModel(Qwen3Model):
         yield from super().modify_tensors(data_torch, name, bid)
 
 
-@ModelBase.register(
-    "Qwen3DSparkModel",
-    "DSparkDraftModel",
-    "DSparkSpeculator",
-    "Lfm2DSparkDraftModel",
-    "LingDSparkModel",
-)
+@ModelBase.register("Qwen3DSparkModel", "DSparkDraftModel", "DSparkSpeculator", "Lfm2DSparkDraftModel")
 @ModelBase.example("satgeze/Qwen3.6-27B-DSpark")
 class DSparkModel(DFlashModel):
     # DSpark = DFlash + a semi-autoregressive Markov head.

docs/backend/ET.md

@@ -116,7 +116,7 @@ in inline assembler.
 Most kernels are very naive with lots of low hanging fruits left:
 
 > [!IMPORTANT]
-> Several assembly instructions emitted by the compiler are not implemented
+> Several assembly instructions emmited by the compiler are not implemented
 > in hardware and software emulation in firmware is not ready yet.
 > Eventually firmware will transparently trap unimplemented instructions
 > and will emulate them inside exception handler. Until then, kernel
@@ -138,12 +138,12 @@ Most kernels are very naive with lots of low hanging fruits left:
 > kernel build process. Feel free to take ideas/code from there or try linking
 > it in.
 
-Before committing any changes to operations and/or kernels, don't forget
+Before commiting any changes to operations and/or kernels, don't forget
 to update supported ops reports (instructions at `docs/ops.md`).
 
 When logging is enabled (e.g. by setting `--log-file` cli param),
 each compute kernel run outputs a line with
-pipe-delimited key-value pairs containing kernel level performance information.
+pipe-delimited key-value pairs containing kernel level performance infomation.
 Line is prefixed with `ET_PERF`:
 
 ```
@@ -160,7 +160,7 @@ to `GGML_ET_PROFILE/et_runtime_trace.json` and `GGML_ET_PROFILE/kernel_map` on e
 
 ### Uberkernel
 
-The in-kernel implementation of device dispatch/kernel fusion. The ET SDK has a non-trivial op-to-op gap. `Uberkernel` (name taken from the original Esperanto AI's compiler)
+The in-knernel implementaiton of device dispatch/kernel fusion. The ET SDK has a non-trivial op-to-op gap. `Uberkernel` (name taken from the original Esperanto AI's compiler)
 dispatches multiple already existing kernel implementations with device side synchronization. Due to the processor's design, there is no natural memory visibility
 horizon between sub-kernel invocations. This makes uberkernel much more difficult to develop and debug. Currently Uberkerel is hidden begind the
 `GGML_ET_UBERKERNEL` environment variable and is disabled by default. Setting it to 1 enables it and provides significant performance improvements but is only

docs/build.md

@@ -70,23 +70,18 @@ cmake --build build --config Release
     - Tab Workload: Desktop-development with C++
     - Tab Components (select quickly via search): C++-_CMake_ Tools for Windows, _Git_ for Windows, C++-_Clang_ Compiler for Windows, MS-Build Support for LLVM-Toolset (clang)
     - Please remember to always use a Developer Command Prompt / PowerShell for VS2022 for git, build, test
-    - For Windows on ARM (arm64, WoA), build with:
+    - For Windows on ARM (arm64, WoA) build with:
+    ```bash
+    cmake --preset arm64-windows-llvm-release -D GGML_OPENMP_FETCH=ON
+    cmake --build build-arm64-windows-llvm-release
+    ```
+    `GGML_OPENMP_FETCH` downloads the official LLVM OpenMP runtime and requires Clang, 7-Zip and network access during configuration. CMake selects the runtime from the target architecture, so this also works when cross-compiling for WoA from x64. The extracted header, import library, DLL and OpenMP license are placed under `build/_deps`. The build copies `libomp.dll` and `LICENSE-LLVM-OpenMP` to the runtime output directory and installs them together. Omit the option to use CMake's normal OpenMP detection, or pass `-D GGML_OPENMP=OFF` to disable OpenMP.
+    For building with ninja generator and clang compiler as default:
+      -set path:set LIB=C:\Program Files (x86)\Windows Kits\10\Lib\10.0.22621.0\um\x64;C:\Program Files\Microsoft Visual Studio\2022\Community\VC\Tools\MSVC\14.41.34120\lib\x64\uwp;C:\Program Files (x86)\Windows Kits\10\Lib\10.0.22621.0\ucrt\x64
       ```bash
-      cmake --preset arm64-windows-llvm-release -D GGML_OPENMP_FETCH=ON
-      cmake --build build-arm64-windows-llvm-release
+      cmake --preset x64-windows-llvm-release
+      cmake --build build-x64-windows-llvm-release
       ```
-      - Use `ARM64 Native Tools Command Prompt for VS 2022` if you are building on an ARM64 machine.
-      - `GGML_OPENMP_FETCH` downloads the official LLVM OpenMP runtime and requires Clang, 7-Zip and network access during configuration. CMake selects the runtime from the target architecture, so this also works when cross-compiling for WoA from x64. The extracted header, import library, DLL and OpenMP license are placed under `build/_deps`. The build copies `libomp.dll` and `LICENSE-LLVM-OpenMP` to the runtime output directory and installs them together. Omit the option to use CMake's normal OpenMP detection, or pass `-D GGML_OPENMP=OFF` to disable OpenMP.
-    - For building with ninja generator and clang compiler as default:
-      - Set path:
-        ```
-        set LIB=C:\Program Files (x86)\Windows Kits\10\Lib\10.0.22621.0\um\x64;C:\Program Files\Microsoft Visual Studio\2022\Community\VC\Tools\MSVC\14.41.34120\lib\x64\uwp;C:\Program Files (x86)\Windows Kits\10\Lib\10.0.22621.0\ucrt\x64
-        ```
-      - Run:
-        ```bash
-        cmake --preset x64-windows-llvm-release
-        cmake --build build-x64-windows-llvm-release
-        ```
 - If you want HTTPS/TLS features, you may install OpenSSL development libraries. If not installed, the project will build and run without SSL support.
   - **Debian / Ubuntu:** `sudo apt-get install libssl-dev`
   - **Fedora / RHEL / Rocky / Alma:** `sudo dnf install openssl-devel`

docs/development/HOWTO-add-model.md

@@ -166,19 +166,6 @@ Examples:
 - Some models require scaling the input position. For example, `[0, 1, 2, ...]` becomes `[0, 0.5, 1, ...]`. In this case, you can provide the scaling via `freq_scale = 0.5f`.
 - Some models use learned RoPE frequencies instead of relying on `powf(freq_base, -2.0 * i / n_dims)`. In this case, you can provide the learned frequencies via the `rope_freqs` tensor (corresponding to the `c` argument in `ggml_rope_ext`), then set `freq_base = 1.0f`. An important note is that `rope_freqs` in GGML is the **inverse** (`theta = pos[i] / rope_freqs`), so you may need to invert `rope_freqs` during conversion.
 
-### Rotating only a part of the head
-
-Many models rotate only a part of each head and leave the rest untouched (often called the "nope" part). Do not build this with views plus `ggml_concat`, it's not efficient. Both layouts can be done with a single RoPE op:
-
-- `[rope|nope]`, rotated dims first: pass `n_dims` smaller than the head size to `ggml_rope_ext`. Dims from `n_dims` to the end are copied as-is.
-- `[nope|rope]`, rotated dims last: call `ggml_rope_set_offset(cur, n_offs)` on the result of the RoPE, where `n_offs` is the size of the leading untouched part. Dims outside `[n_offs, n_offs + n_dims)` are copied as-is.
-
-`n_offs` must be even, `n_offs + n_dims` must fit in the row, and vision RoPE is not supported. Note that the frequencies are computed relative to the rotated window.
-
-Example: DeepSeek-V4 uses `[nope|rope]` for its query, key and compressed KV tensors, so `src/models/deepseek4.cpp` ropes the whole tensor and then calls `ggml_rope_set_offset(cur, n_embd_head_nope)`.
-
-Exception: some models apply an extra op to the `nope` part, for example `deepseek32.cpp`, and may not use this optimization. While RoPE can be applied selectively to a part of the head, the extra op may not, so these models still need views plus `ggml_concat`.
-
 ## GGUF specification
 
 https://github.com/ggml-org/ggml/blob/master/docs/gguf.md

ggml/CMakeLists.txt

@@ -4,8 +4,8 @@ project("ggml" C CXX ASM)
 
 ### GGML Version
 set(GGML_VERSION_MAJOR 0)
-set(GGML_VERSION_MINOR 21)
-set(GGML_VERSION_PATCH 0)
+set(GGML_VERSION_MINOR 20)
+set(GGML_VERSION_PATCH 2)
 set(GGML_VERSION_BASE "${GGML_VERSION_MAJOR}.${GGML_VERSION_MINOR}.${GGML_VERSION_PATCH}")
 
 list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake/")

ggml/src/ggml-cpu/CMakeLists.txt

@@ -639,7 +639,6 @@ function(ggml_add_cpu_backend_variant_impl tag_name)
             ${KLEIDIAI_SRC}/kai/ukernels/matmul/matmul_clamp_fp32_bf16p_bf16p/
             ${KLEIDIAI_SRC}/kai/ukernels/matmul/matmul_clamp_f32_f16p_qsi4c32p/
             ${KLEIDIAI_SRC}/kai/ukernels/matmul/matmul_clamp_f32_f32p_f32p/
-            ${KLEIDIAI_SRC}/kai/ukernels/matmul/matmul_clamp_f32_f32_f32p/
             ${KLEIDIAI_SRC}/kai/ukernels/matmul/pack/)
 
         set(ARCH_FLAGS_TEMP "${ARCH_FLAGS}")
@@ -702,8 +701,6 @@ function(ggml_add_cpu_backend_variant_impl tag_name)
                 ${KLEIDIAI_SRC}/kai/ukernels/matmul/matmul_clamp_f32_f16p_qsi4c32p/kai_matmul_clamp_f32_f16p1vlx2_qsi4c32p4vlx2_1vlx4vl_sme2_mopa_asm.S
                 ${KLEIDIAI_SRC}/kai/ukernels/matmul/matmul_clamp_f32_f32p_f32p/kai_matmul_clamp_f32_f32p2vlx1_f32p2vlx1biasf32_sme2_mopa.c
                 ${KLEIDIAI_SRC}/kai/ukernels/matmul/matmul_clamp_f32_f32p_f32p/kai_matmul_clamp_f32_f32p2vlx1_f32p2vlx1biasf32_sme2_mopa_asm.S
-                ${KLEIDIAI_SRC}/kai/ukernels/matmul/matmul_clamp_f32_f32_f32p/kai_matmul_clamp_f32_f32_f32p2vlx1b_1x16vl_sme2_mla.c
-                ${KLEIDIAI_SRC}/kai/ukernels/matmul/matmul_clamp_f32_f32_f32p/kai_matmul_clamp_f32_f32_f32p2vlx1b_1x16vl_sme2_mla_asm.S
                 ${KLEIDIAI_SRC}/kai/ukernels/matmul/pack/kai_lhs_pack_bf16p2vlx2_f32_sme.c
                 ${KLEIDIAI_SRC}/kai/ukernels/matmul/pack/kai_rhs_pack_kxn_bf16p2vlx2b_f32_x32_sme.c
                 ${KLEIDIAI_SRC}/kai/ukernels/matmul/pack/kai_lhs_pack_f16pmrx2_f32_neon.c
@@ -740,9 +737,8 @@ function(ggml_add_cpu_backend_variant_impl tag_name)
         set_target_properties(${GGML_CPU_NAME} PROPERTIES COMPILE_FLAGS "-msimd128")
     endif()
 
-  if (CMAKE_C_COMPILER_ID STREQUAL "IntelLLVM" OR CMAKE_CXX_COMPILER_ID STREQUAL "IntelLLVM")
-      # The compiler automatically enables "-ffast-math" which can cause NaNs in tests due to "-fassociative-math"
-      target_compile_options(${GGML_CPU_NAME} PRIVATE "$<$<OR:$<COMPILE_LANG_AND_ID:C,IntelLLVM>,$<COMPILE_LANG_AND_ID:CXX,IntelLLVM>>:$<$<BOOL:${WIN32}>:/clang:>-fno-associative-math>")
-  endif()
-
+    if (CMAKE_CXX_COMPILER_ID STREQUAL "IntelLLVM")
+        # The compiler automatically enables "-ffast-math" which can cause NaNs in tests due to "-fassociative-math"
+        target_compile_options(${GGML_CPU_NAME} PRIVATE "-fno-associative-math")
+    endif()
 endfunction()

ggml/src/ggml-cpu/kleidiai/kernels.cpp

@@ -23,7 +23,6 @@
 #include "kai_matmul_clamp_f32_qsi8d32p1x8_qsi4c32p8x8_1x8_sve_dotprod.h"
 #include "kai_matmul_clamp_f32_f16p1vlx2_qsi4c32p4vlx2_1vlx4vl_sme2_mopa.h"
 #include "kai_matmul_clamp_f32_f32p2vlx1_f32p2vlx1biasf32_sme2_mopa.h"
-#include "kai_matmul_clamp_f32_f32_f32p2vlx1b_1x16vl_sme2_mla.h"
 #include "kai_matmul_clamp_f32_f32p2vlx1_f32p2vlx1b_2vlx2vl_sme_mopa.h"
 
 #include "kai_lhs_pack_bf16p2vlx2_f32_sme.h"
@@ -77,21 +76,6 @@ static inline void kernel_run_fn10(size_t m, size_t n, size_t k, size_t /*bl*/,
     Fn(m, n, k, lhs, rhs, dst, dst_stride_row, dst_stride_col, clamp_min, clamp_max);
 }
 
-template <void (*Fn)(size_t, size_t, size_t, const void *, size_t, const void *, void *, size_t, size_t, float, float)>
-static inline void kernel_run_lhs_stride_fn10(size_t       m,
-                                              size_t       n,
-                                              size_t       k,
-                                              size_t       lhs_stride,
-                                              const void * lhs,
-                                              const void * rhs,
-                                              void *       dst,
-                                              size_t       dst_stride_row,
-                                              size_t       dst_stride_col,
-                                              float        clamp_min,
-                                              float        clamp_max) {
-    Fn(m, n, k, lhs, lhs_stride, rhs, dst, dst_stride_row, dst_stride_col, clamp_min, clamp_max);
-}
-
 template<void(*Fn)(size_t,size_t,size_t,const void*,const void*,float*,size_t,size_t,float,float)>
 static inline void kernel_run_float_fn10(size_t m, size_t n, size_t k, size_t /*bl*/,
                                          const void* lhs, const void* rhs, void* dst,
@@ -963,25 +947,25 @@ static ggml_kleidiai_kernels ggml_kleidiai_kernels_f32[] = {
             /* .packed_size_ex        = */ &lhs_ps_fn5<kai_get_lhs_packed_size_lhs_pack_f32p2vlx1_f32_sme>,
             /* .pack_func_ex          = */ &lhs_pack_void_fn9<kai_run_lhs_pack_f32p2vlx1_f32_sme>,
         },
-        /* SME2 GEMV */
+        /* SME GEMV */
         {
-            /* .get_m_step            = */ kai_get_m_step_matmul_clamp_f32_f32_f32p2vlx1b_1x16vl_sme2_mla,
-            /* .get_n_step            = */ kai_get_n_step_matmul_clamp_f32_f32_f32p2vlx1b_1x16vl_sme2_mla,
-            /* .get_mr                = */ kai_get_m_step_matmul_clamp_f32_f32_f32p2vlx1b_1x16vl_sme2_mla,
-            /* .get_nr                = */ kai_get_nr_matmul_clamp_f32_f32_f32p2vlx1b_1x16vl_sme2_mla,
-            /* .get_kr                = */ kai_get_kr_matmul_clamp_f32_f32_f32p2vlx1b_1x16vl_sme2_mla,
-            /* .get_sr                = */ kai_get_sr_matmul_clamp_f32_f32_f32p2vlx1b_1x16vl_sme2_mla,
-            /* .get_dst_offset        = */ kai_get_dst_offset_matmul_clamp_f32_f32_f32p2vlx1b_1x16vl_sme2_mla,
-            /* .get_dst_size          = */ kai_get_dst_size_matmul_clamp_f32_f32_f32p2vlx1b_1x16vl_sme2_mla,
-            /* .get_lhs_offset_ex     = */ &kernel_offs_fn2<kai_get_lhs_offset_matmul_clamp_f32_f32_f32p2vlx1b_1x16vl_sme2_mla>,
-            /* .get_rhs_packed_offset_ex = */ &kernel_offs_fn2<kai_get_rhs_packed_offset_matmul_clamp_f32_f32_f32p2vlx1b_1x16vl_sme2_mla>,
-            /* .run_kernel_ex         = */ &kernel_run_lhs_stride_fn10<kai_run_matmul_clamp_f32_f32_f32p2vlx1b_1x16vl_sme2_mla>,
+            /* .get_m_step            = */ kai_get_m_step_matmul_clamp_f32_f32p2vlx1_f32p2vlx1biasf32_sme2_mopa,
+            /* .get_n_step            = */ kai_get_n_step_matmul_clamp_f32_f32p2vlx1_f32p2vlx1biasf32_sme2_mopa,
+            /* .get_mr                = */ kai_get_mr_matmul_clamp_f32_f32p2vlx1_f32p2vlx1biasf32_sme2_mopa,
+            /* .get_nr                = */ kai_get_nr_matmul_clamp_f32_f32p2vlx1_f32p2vlx1biasf32_sme2_mopa,
+            /* .get_kr                = */ kai_get_kr_matmul_clamp_f32_f32p2vlx1_f32p2vlx1biasf32_sme2_mopa,
+            /* .get_sr                = */ kai_get_sr_matmul_clamp_f32_f32p2vlx1_f32p2vlx1biasf32_sme2_mopa,
+            /* .get_dst_offset        = */ kai_get_dst_offset_matmul_clamp_f32_f32p2vlx1_f32p2vlx1biasf32_sme2_mopa,
+            /* .get_dst_size          = */ kai_get_dst_size_matmul_clamp_f32_f32p2vlx1_f32p2vlx1biasf32_sme2_mopa,
+            /* .get_lhs_offset_ex     = */ nullptr,
+            /* .get_rhs_packed_offset_ex = */ nullptr,
+            /* .run_kernel_ex         = */ nullptr,
         },
         /* .gemv_lhs_info = */ {
-            /* .get_offset            = */ nullptr,
-            /* .get_packed_offset_ex  = */ nullptr,
-            /* .packed_size_ex        = */ nullptr,
-            /* .pack_func_ex          = */ nullptr,
+            /* .get_offset            = */ kai_get_lhs_offset_lhs_pack_f32p2vlx1_f32_sme,
+            /* .get_packed_offset_ex  = */ &lhs_offs_fn5<kai_get_lhs_packed_offset_lhs_pack_f32p2vlx1_f32_sme>,
+            /* .packed_size_ex        = */ &lhs_ps_fn5<kai_get_lhs_packed_size_lhs_pack_f32p2vlx1_f32_sme>,
+            /* .pack_func_ex          = */ &lhs_pack_void_fn9<kai_run_lhs_pack_f32p2vlx1_f32_sme>,
         },
         /* .rhs_info = */ {
             /* .packed_stride         = */ nullptr,

ggml/src/ggml-cpu/kleidiai/kleidiai.cpp

@@ -696,15 +696,6 @@ class tensor_traits : public ggml::cpu::tensor_traits {
         }
 
         if (op->src[0]->type == GGML_TYPE_F32) {
-            ggml_kleidiai_kernels * primary     = kernel_chain[0];
-            kernel_info *           gemv_kernel = primary ? &primary->gemv : nullptr;
-            if (is_gemv && op->src[1]->nb[0] == (int64_t) sizeof(float) && gemv_kernel &&
-                gemv_kernel->get_lhs_offset_ex && gemv_kernel->get_rhs_packed_offset_ex &&
-                gemv_kernel->run_kernel_ex && gemv_kernel->get_dst_offset) {
-                size = 0;
-                return true;
-            }
-
             size_t cursor = 0;
             bool any_slot = false;
 
@@ -820,28 +811,15 @@ class tensor_traits : public ggml::cpu::tensor_traits {
             return false;
         }
 
-        const size_t k = ne00;
-        const size_t m = ne11;
-        const size_t n = ne01;
-        const bool use_gemv = m == 1 && src1->nb[0] == (int64_t) sizeof(float) &&
-                              kernels->gemv.get_lhs_offset_ex &&
-                              kernels->gemv.get_rhs_packed_offset_ex &&
-                              kernels->gemv.run_kernel_ex &&
-                              kernels->gemv.get_dst_offset;
-
-        kernel_info * kernel        = use_gemv ? &kernels->gemv : &kernels->gemm;
+        kernel_info * kernel        = &kernels->gemm;
         lhs_packing_info * lhs_info = &kernels->gemm_lhs_info;
 
-        if (!kernel || !kernel->get_lhs_offset_ex ||
+        if (!kernel || !lhs_info || !lhs_info->get_offset || !lhs_info->get_packed_offset_ex ||
+            !lhs_info->packed_size_ex || !lhs_info->pack_func_ex ||
             !kernel->get_rhs_packed_offset_ex || !kernel->run_kernel_ex || !kernel->get_dst_offset) {
             return false;
         }
 
-        if (!use_gemv && (!lhs_info || !lhs_info->get_offset || !lhs_info->get_packed_offset_ex ||
-                          !lhs_info->packed_size_ex || !lhs_info->pack_func_ex)) {
-            return false;
-        }
-
         const kleidiai_weight_header * header = kleidiai_weight_header_from_ptr(src0->data);
         const bool has_header = kleidiai_is_weight_header_valid(header);
 
@@ -854,14 +832,16 @@ class tensor_traits : public ggml::cpu::tensor_traits {
         const int nth = params->nth > 0 ? params->nth : 1;
         const int ith = params->ith;
 
+        const size_t k = ne00;
+        const size_t m = ne11;
+        const size_t n = ne01;
+
         const size_t mr = kernel->get_mr();
         const size_t kr = kernel->get_kr();
         const size_t sr = kernel->get_sr();
 
-        const size_t lhs_packed_size = use_gemv ? 0 : lhs_info->packed_size_ex(m, k, 0, mr, kr, sr);
-        if (!use_gemv) {
-            GGML_ASSERT(lhs_packed_size <= params->wsize);
-        }
+        const size_t lhs_packed_size = lhs_info->packed_size_ex(m, k, 0, mr, kr, sr);
+        GGML_ASSERT(lhs_packed_size <= params->wsize);
 
         uint8_t * lhs_packed   = static_cast<uint8_t *>(params->wdata);
         const size_t dst_stride = dst->nb[1];
@@ -873,7 +853,7 @@ class tensor_traits : public ggml::cpu::tensor_traits {
             const uint8_t * lhs_batch_base = static_cast<const uint8_t *>(src1->data) + batch_idx * src1->nb[2];
             uint8_t * dst_batch_base = static_cast<uint8_t *>(dst->data) + batch_idx * dst->nb[2];
 
-            if (!use_gemv) {
+            {
                 const int64_t m_roundup_mr = kai_roundup((int64_t)m, (int64_t)mr);
                 int64_t max_threads = mr ? (m_roundup_mr / (int64_t)mr) : nth;
                 max_threads = std::max<int64_t>(1, max_threads);
@@ -923,17 +903,15 @@ class tensor_traits : public ggml::cpu::tensor_traits {
                 const size_t n_to_process = std::min(chunk_cols, n - n_start);
 
                 if (n_to_process > 0) {
-                    const size_t lhs_offset = use_gemv ? kernel->get_lhs_offset_ex(0, k, 0)
-                                                       : lhs_info->get_packed_offset_ex(0, k, 0, mr, kr, sr);
+                    const size_t lhs_packed_offset = lhs_info->get_packed_offset_ex(0, k, 0, mr, kr, sr);
                     const size_t rhs_packed_offset = kernel->get_rhs_packed_offset_ex(n_start, k, 0);
                     const size_t dst_offset        = kernel->get_dst_offset(0, n_start, dst_stride);
 
-                    const void * lhs_ptr = use_gemv ? lhs_batch_base + lhs_offset
-                                                    : lhs_packed + lhs_offset;
+                    const void * lhs_ptr = lhs_packed + lhs_packed_offset;
                     const void * rhs_ptr = rhs_base + rhs_packed_offset;
                     float * dst_ptr      = reinterpret_cast<float *>(dst_batch_base + dst_offset);
 
-                    kernel->run_kernel_ex(m, n_to_process, k, use_gemv ? src1->nb[1] : 0,
+                    kernel->run_kernel_ex(m, n_to_process, k, 0,
                                           lhs_ptr,
                                           rhs_ptr,
                                           dst_ptr,

ggml/src/ggml-cpu/ops.cpp

@@ -1896,6 +1896,7 @@ void ggml_compute_forward_repeat_back(
 }
 
 // ggml_compute_forward_concat
+
 static void ggml_compute_forward_concat_any(
     const ggml_compute_params * params,
     ggml_tensor * dst) {
@@ -1903,6 +1904,8 @@ static void ggml_compute_forward_concat_any(
     const ggml_tensor * src0 = dst->src[0];
     const ggml_tensor * src1 = dst->src[1];
 
+    const size_t len = ggml_type_size(src0->type);
+
     const int ith = params->ith;
     const int nth = params->nth;
 
@@ -1911,38 +1914,31 @@ static void ggml_compute_forward_concat_any(
     const int32_t dim = ggml_get_op_params_i32(dst, 0);
 
     GGML_ASSERT(dim >= 0 && dim < 4);
-    GGML_ASSERT(ggml_is_contiguous_rows(src0));
-    GGML_ASSERT(ggml_is_contiguous_rows(src1));
 
     int64_t o[4] = {0, 0, 0, 0};
-
     if (dim == 0) {
-        GGML_ASSERT(src0->ne[0] % ggml_blck_size(src0->type) == 0);
-        GGML_ASSERT(src1->ne[0] % ggml_blck_size(src1->type) == 0);
-
         o[dim] = src0->ne[dim]/ggml_blck_size(src0->type);
     } else {
         o[dim] = src0->ne[dim];
     }
 
-    // Region 1: copy rows from src0
-    for (int i3 = 0; i3 < ne03; i3++) {
-        for (int i2 = ith; i2 < ne02; i2 += nth) {
-            for (int i1 = 0; i1 < ne01; i1++) {
-                const char * x = (const char *) src0->data + i1*nb01 + i2*nb02 + i3*nb03;
-                      char * y = (      char *) dst->data  + i1*nb1  + i2*nb2  + i3*nb3;
-                memcpy(y, x, ggml_row_size(src0->type, ne00));
-            }
-        }
-    }
+    const char * x;
 
-    // Region 2: copy rows from src1, offset into dst by o[]
-    for (int i3 = 0; i3 < ne13; i3++) {
-        for (int i2 = ith; i2 < ne12; i2 += nth) {
-            for (int i1 = 0; i1 < ne11; i1++) {
-                const char * x = (const char *) src1->data + i1*nb11         + i2*nb12         + i3*nb13;
-                      char * y = (      char *)  dst->data + (i1 + o[1])*nb1 + (i2 + o[2])*nb2 + (i3 + o[3])*nb3 + o[0]*nb0;
-                memcpy(y, x, ggml_row_size(src1->type, ne10));
+    // TODO: smarter multi-theading
+    for (int i3 = 0; i3 < ne3; i3++) {
+        for (int i2 = ith; i2 < ne2; i2 += nth) {
+            for (int i1 = 0; i1 < ne1; i1++) {
+                for (int i0 = 0; i0 < ne0/ggml_blck_size(dst->type); i0++) {
+                    if (i0 < ne00/ggml_blck_size(src0->type) && i1 < ne01 && i2 < ne02 && i3 < ne03) {
+                        x = (const char *)src0->data + (i0       )*nb00 + (i1       )*nb01 + (i2       )*nb02 + (i3       )*nb03;
+                    } else {
+                        x = (const char *)src1->data + (i0 - o[0])*nb10 + (i1 - o[1])*nb11 + (i2 - o[2])*nb12 + (i3 - o[3])*nb13;
+                    }
+
+                    char * y = (char *)dst->data + i0*nb0 + i1*nb1 + i2*nb2 + i3*nb3;
+
+                    memcpy(y, x, len);
+                }
             }
         }
     }
@@ -2082,6 +2078,14 @@ void ggml_compute_forward_concat(
     ggml_tensor * dst) {
 
     const ggml_tensor * src0 = dst->src[0];
+    const ggml_tensor * src1 = dst->src[1];
+
+    if (ggml_is_quantized(src0->type)) {
+        GGML_ASSERT(ggml_is_contiguous_rows(src0));
+        GGML_ASSERT(ggml_is_contiguous_rows(src1));
+        GGML_ASSERT(src0->ne[0] % ggml_blck_size(src0->type) == 0);
+        GGML_ASSERT(src1->ne[0] % ggml_blck_size(src1->type) == 0);
+    }
 
     switch (src0->type) {
         case GGML_TYPE_F16:

ggml/src/ggml-hexagon/ggml-hexagon.cpp

@@ -3180,9 +3180,8 @@ static bool ggml_hexagon_supported_argsort(const struct ggml_hexagon_session * s
 static bool ggml_hexagon_supported_rope(const struct ggml_hexagon_session * sess, const struct ggml_tensor * op) {
     const int32_t * op_params = &op->op_params[0];
 
-    // ggml_rope_set_offset: HVX kernels need a VLEN-aligned window start (32 f32 elems)
-    if (op_params[15] % 32 != 0) {
-        return false;
+    if (op_params[15] != 0) {
+        return false; // FIXME: support ggml_rope_set_offset
     }
 
     int mode = op_params[2];

ggml/src/ggml-hexagon/htp/rope-ops.c

@@ -53,7 +53,6 @@
 
 struct htp_rope_context {
     int32_t n_dims;
-    int32_t n_offs;
     int32_t mode;
     int32_t n_ctx_orig;
     int32_t sections[4];
@@ -406,40 +405,32 @@ static inline void hvx_rope_f32_aa(float * restrict dst, const float * restrict
 
 static void inline rope_basic_f32(struct htp_rope_context * rctx, uint8_t * restrict dst, uint8_t * restrict src,
                    uint32_t nr, uint32_t ne0, const float * restrict theta_cache) {
-    const uint32_t n_offs = rctx->n_offs; // VLEN-aligned (enforced by supports_op)
     #pragma unroll(4)
     for (uint32_t i = 0; i < nr; i++) {
         float * d = (float *) (dst + i * rctx->dst_row_size_aligned);
         float * s = (float *) (src + i * rctx->src0_row_size_aligned);
 
-        hvx_rope_f32_aa(d + n_offs, s + n_offs, rctx->n_dims, theta_cache);
+        hvx_rope_f32_aa(d, s, rctx->n_dims, theta_cache);
 
         // fill the remain channels with data from src tensor
-        if (n_offs > 0) {
-            hvx_copy_f32_uu((uint8_t *) d, (uint8_t *) s, n_offs);
-        }
-        if (n_offs + rctx->n_dims < ne0) {
-            hvx_copy_f32_uu((uint8_t *)(d + n_offs + rctx->n_dims), (uint8_t *)(s + n_offs + rctx->n_dims), ne0 - n_offs - rctx->n_dims);
+        if (rctx->n_dims < ne0) {
+            hvx_copy_f32_uu((uint8_t *)(d + rctx->n_dims), (uint8_t *)(s + rctx->n_dims), ne0 - rctx->n_dims);
         }
     }
 }
 
 static void inline rope_neox_f32(struct htp_rope_context * rctx, uint8_t * restrict dst, uint8_t * restrict src,
                    uint32_t nr, uint32_t ne0, const float * restrict theta_cache) {
-    const uint32_t n_offs = rctx->n_offs; // VLEN-aligned (enforced by supports_op)
     #pragma unroll(4)
     for (uint32_t i = 0; i < nr; i++) {
         float * d = (float *) (dst + i * rctx->dst_row_size_aligned);
         float * s = (float *) (src + i * rctx->src0_row_size_aligned);
 
-        hvx_rope_neox_f32_aa(d + n_offs, s + n_offs, rctx->n_dims, theta_cache);
+        hvx_rope_neox_f32_aa(d, s, rctx->n_dims, theta_cache);
 
         // fill the remain channels with data from src tensor
-        if (n_offs > 0) {
-            hvx_copy_f32_uu((uint8_t *) d, (uint8_t *) s, n_offs);
-        }
-        if (n_offs + rctx->n_dims < ne0) {
-            hvx_copy_f32_uu((uint8_t *)(d + n_offs + rctx->n_dims), (uint8_t *)(s + n_offs + rctx->n_dims), ne0 - n_offs - rctx->n_dims);
+        if (rctx->n_dims < ne0) {
+            hvx_copy_f32_uu((uint8_t *)(d + rctx->n_dims), (uint8_t *)(s + rctx->n_dims), ne0 - rctx->n_dims);
         }
     }
 }
@@ -682,7 +673,6 @@ static int execute_op_rope_f32(struct htp_ops_context * octx) {
     rctx.n_dims     = ((const int32_t *) op_params)[1];
     rctx.mode       = ((const int32_t *) op_params)[2];
     rctx.n_ctx_orig = ((const int32_t *) op_params)[4];
-    rctx.n_offs     = ((const int32_t *) op_params)[15];
 
     memcpy(&rctx.freq_base,   (int32_t *) op_params + 5,  sizeof(float));
     memcpy(&rctx.freq_scale,  (int32_t *) op_params + 6,  sizeof(float));

ggml/src/ggml-opencl/CMakeLists.txt

@@ -63,7 +63,6 @@ endfunction()
 set(GGML_OPENCL_KERNELS
     add
     add_id
-    moe_add_id_glu
     argsort
     tri
     fill

ggml/src/ggml-opencl/ggml-opencl.cpp

@@ -577,12 +577,6 @@ struct ggml_backend_opencl_context {
     // whether fuse moe combine
     cl_uint fuse_moe_combine;
 
-    // whether to fold the MoE bias adds into swiglu_oai
-    cl_uint fuse_moe_bias_glu;
-
-    // whether to fold the MoE down-projection bias add into the combine
-    cl_uint fuse_moe_bias_combine;
-
     bool adreno_has_large_buffer;
     bool adreno_use_large_buffer;
     bool adreno_use_bin_kernels;
@@ -664,7 +658,6 @@ struct ggml_backend_opencl_context {
 
     cl_program program_add;
     cl_program program_add_id;
-    cl_program program_moe_add_id_glu;
     cl_program program_clamp;
     cl_program program_cvt;
     cl_program program_diag_mask_inf;
@@ -730,7 +723,6 @@ struct ggml_backend_opencl_context {
     cl_kernel kernel_div, kernel_div_row, kernel_div_f16, kernel_div_row_f16;
     cl_kernel kernel_sub, kernel_sub_row, kernel_sub_f16, kernel_sub_row_f16;
     cl_kernel kernel_add_id;
-    cl_kernel kernel_add_id_add_id_swiglu_oai;
     cl_kernel kernel_scale_f32, kernel_scale_f32_4;
     cl_kernel kernel_sqr_cont_f32, kernel_sqr_cont_f32_4, kernel_sqr_cont_f16, kernel_sqr_cont_f16_4;
     cl_kernel kernel_sqrt_cont_f32, kernel_sqrt_cont_f32_4, kernel_sqrt_cont_f16, kernel_sqrt_cont_f16_4;
@@ -907,7 +899,6 @@ struct ggml_backend_opencl_context {
     cl_kernel kernel_moe_histogram, kernel_moe_scan, kernel_moe_fill, kernel_moe_scatter;
     cl_kernel kernel_moe_scatter_stable = nullptr;   // deterministic slot assignment
     cl_kernel kernel_moe_combine_f32 = nullptr;   // fused router-weight mul + cross-expert sum
-    cl_kernel kernel_moe_combine_bias_f32 = nullptr;  // same, with the down-projection bias add folded in
     cl_kernel kernel_mul_mv_id_q4_0_f32_8x_flat;
     cl_kernel kernel_mul_mv_id_q8_0_f32, kernel_mul_mv_id_q8_0_f32_flat;
     cl_kernel kernel_mul_mv_id_mxfp4_f32;
@@ -1355,23 +1346,6 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) {
         GGML_LOG_CONT(".");
     }
 
-    // moe_add_id_glu
-    {
-#ifdef GGML_OPENCL_EMBED_KERNELS
-        const std::string kernel_src {
-            #include "moe_add_id_glu.cl.h"
-        };
-#else
-        const std::string kernel_src = read_file("moe_add_id_glu.cl");
-#endif
-        backend_ctx->program_moe_add_id_glu =
-            build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts);
-
-        CL_CHECK((backend_ctx->kernel_add_id_add_id_swiglu_oai =
-            clCreateKernel(backend_ctx->program_moe_add_id_glu, "kernel_add_id_add_id_swiglu_oai", &err), err));
-        GGML_LOG_CONT(".");
-    }
-
     // tri
     {
 #ifdef GGML_OPENCL_EMBED_KERNELS
@@ -3302,8 +3276,6 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) {
             backend_ctx, kernel_src.c_str(), compile_opts);
         CL_CHECK((backend_ctx->kernel_moe_combine_f32 =
                     clCreateKernel(prog, "kernel_moe_combine_f32", &err), err));
-        CL_CHECK((backend_ctx->kernel_moe_combine_bias_f32 =
-                    clCreateKernel(prog, "kernel_moe_combine_bias_f32", &err), err));
         CL_CHECK(clReleaseProgram(prog));
         GGML_LOG_CONT(".");
     }
@@ -6040,12 +6012,6 @@ static ggml_backend_opencl_context * ggml_cl_init(ggml_backend_dev_t dev) {
     backend_ctx->adreno_moe_ragged_skip_gran = (ragged_gran_env != NULL) ? atoi(ragged_gran_env) : 8;
 
     // whether fuse moe combine
-    static const char * fuse_moe_bias_glu_env = getenv("GGML_OPENCL_FUSE_MOE_BIAS_GLU");
-    backend_ctx->fuse_moe_bias_glu = fuse_moe_bias_glu_env == NULL ? 1 : (atoi(fuse_moe_bias_glu_env) != 0);
-
-    static const char * fuse_moe_bias_combine_env = getenv("GGML_OPENCL_FUSE_MOE_BIAS_COMBINE");
-    backend_ctx->fuse_moe_bias_combine = fuse_moe_bias_combine_env == NULL ? 1 : (atoi(fuse_moe_bias_combine_env) != 0);
-
     static const char * fuse_moe_combine_env = getenv("GGML_OPENCL_FUSE_MOE_COMBINE");
     backend_ctx->fuse_moe_combine = fuse_moe_combine_env == NULL ? 1 : (atoi(fuse_moe_combine_env) != 0);
 
@@ -6914,300 +6880,6 @@ static bool ggml_opencl_can_fuse_moe_combine(const struct ggml_cgraph * cgraph,
     return true;
 }
 
-// Detect the gpt-oss MoE bias+activation epilogue on the PREFILL path:
-// {MUL_MAT_ID(gate), ADD_ID(gate_bias), MUL_MAT_ID(up), ADD_ID(up_bias), GLU(swiglu_oai)}.
-// The two matmuls still run as their own dispatches (the prefill GEMM is the vendor's);
-// what collapses is the epilogue — both add_id passes are in-place read-modify-writes of a
-// tensor the GLU immediately reads again, so they are three full passes over the same
-// [n_ff, n_expert_used, n_tokens] f32 tensor where one suffices.
-//
-// The decode counterpart is handled by the mxfp4 fused GEMV arm in ggml_opencl_can_fuse,
-// which folds the matmul too; this one deliberately fires only when that cannot (ne[2] > 1).
-static bool ggml_opencl_can_fuse_moe_bias_glu(const struct ggml_cgraph * cgraph, int node_idx) {
-    if (node_idx + 4 >= cgraph->n_nodes) {
-        return false;
-    }
-
-    const enum ggml_op mg_ops[] = { GGML_OP_MUL_MAT_ID, GGML_OP_ADD_ID, GGML_OP_MUL_MAT_ID, GGML_OP_ADD_ID, GGML_OP_GLU };
-    const int          mg_out[] = { node_idx + 4 };
-    if (!ggml_can_fuse_subgraph(cgraph, node_idx, 5, mg_ops, mg_out, 1)) {
-        return false;
-    }
-
-    const ggml_tensor * gmm = cgraph->nodes[node_idx];
-    const ggml_tensor * gad = cgraph->nodes[node_idx+1];
-    const ggml_tensor * umm = cgraph->nodes[node_idx+2];
-    const ggml_tensor * uad = cgraph->nodes[node_idx+3];
-    const ggml_tensor * glu = cgraph->nodes[node_idx+4];
-
-    if (ggml_get_glu_op(glu) != GGML_GLU_OP_SWIGLU_OAI) {
-        return false;
-    }
-    // Prefill only — at one token the mxfp4 arm above folds the matmul as well.
-    if (gmm->src[1]->ne[2] == 1) {
-        return false;
-    }
-    // Wiring: both matmuls share the activation and the expert selection, each add_id
-    // biases its own matmul, and the GLU consumes the two biased results as separate
-    // operands (so the same-buffer ne00_off/ne10_off split path is not in play).
-    if (gad->src[0] != gmm || uad->src[0] != umm ||
-        glu->src[0] != gad || glu->src[1] != uad ||
-        umm->src[1] != gmm->src[1] || umm->src[2] != gmm->src[2]) {
-        return false;
-    }
-    // A swapped GLU would exchange the gate/up roles the fused kernel hard-codes.
-    if (ggml_get_op_params_i32(glu, 1)) {
-        return false;
-    }
-    if (gad->type != GGML_TYPE_F32 || uad->type != GGML_TYPE_F32 || glu->type != GGML_TYPE_F32) {
-        return false;
-    }
-    if (!gad->src[1] || gad->src[1]->type != GGML_TYPE_F32 ||
-        !uad->src[1] || uad->src[1]->type != GGML_TYPE_F32) {
-        return false;
-    }
-    if (!gad->src[2] || gad->src[2]->type != GGML_TYPE_I32 || uad->src[2] != gad->src[2]) {
-        return false;
-    }
-    // Full width on both operands: the kernel writes one output element per input pair.
-    if (!ggml_are_same_shape(gad, uad) || glu->ne[0] != gad->ne[0] ||
-        glu->ne[1] != gad->ne[1] || glu->ne[2] != gad->ne[2] || glu->ne[3] != gad->ne[3]) {
-        return false;
-    }
-    if (gad->ne[3] != 1) {
-        return false;
-    }
-    // The destination is addressed by (expert slot, token) rather than the GLU's flat row
-    // walk; those agree only for a contiguous destination.
-    if (!ggml_is_contiguous(glu) || !ggml_is_contiguous(gmm) || !ggml_is_contiguous(umm)) {
-        return false;
-    }
-    return true;
-}
-
-static void ggml_cl_mul_mat_id(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst);
-
-// Runs the gate and up matmuls unchanged, then one kernel in place of
-// add_id(gate) + add_id(up) + swiglu_oai. See ggml_opencl_can_fuse_moe_bias_glu.
-static void ggml_cl_moe_bias_glu_fused(ggml_backend_t backend, ggml_tensor * gate_mm, const ggml_tensor * gate_add,
-                                       ggml_tensor * up_mm, const ggml_tensor * up_add, const ggml_tensor * glu) {
-    ggml_backend_opencl_context * backend_ctx = (ggml_backend_opencl_context *)backend->context;
-
-    ggml_cl_mul_mat_id(backend, gate_mm->src[0], gate_mm->src[1], gate_mm);
-    ggml_cl_mul_mat_id(backend, up_mm->src[0], up_mm->src[1], up_mm);
-
-    const ggml_tensor * gbias = gate_add->src[1];
-    const ggml_tensor * ubias = up_add->src[1];
-    const ggml_tensor * ids   = gate_add->src[2];
-
-    ggml_tensor_extra_cl * eg  = (ggml_tensor_extra_cl *)gate_mm->extra;
-    ggml_tensor_extra_cl * egb = (ggml_tensor_extra_cl *)gbias->extra;
-    ggml_tensor_extra_cl * eu  = (ggml_tensor_extra_cl *)up_mm->extra;
-    ggml_tensor_extra_cl * eub = (ggml_tensor_extra_cl *)ubias->extra;
-    ggml_tensor_extra_cl * ei  = (ggml_tensor_extra_cl *)ids->extra;
-    ggml_tensor_extra_cl * ed  = (ggml_tensor_extra_cl *)glu->extra;
-
-    cl_ulong off_g  = eg->offset  + gate_mm->view_offs;
-    cl_ulong off_gb = egb->offset + gbias->view_offs;
-    cl_ulong off_u  = eu->offset  + up_mm->view_offs;
-    cl_ulong off_ub = eub->offset + ubias->view_offs;
-    cl_ulong off_i  = ei->offset  + ids->view_offs;
-    cl_ulong off_d  = ed->offset  + glu->view_offs;
-
-    const cl_ulong nb01_g = gate_mm->nb[1];
-    const cl_ulong nb02_g = gate_mm->nb[2];
-    const cl_ulong nb01_u = up_mm->nb[1];
-    const cl_ulong nb02_u = up_mm->nb[2];
-    const cl_ulong nb11_g = gbias->nb[1];
-    const cl_ulong nb11_u = ubias->nb[1];
-    const cl_ulong nb21   = ids->nb[1];
-    const cl_ulong nbd1   = glu->nb[1];
-    const cl_ulong nbd2   = glu->nb[2];
-
-    const int   ne0   = (int)glu->ne[0];
-    const float alpha = ggml_get_op_params_f32(glu, 2);
-    const float limit = ggml_get_op_params_f32(glu, 3);
-
-    cl_kernel kernel = backend_ctx->kernel_add_id_add_id_swiglu_oai;
-
-    int i = 0;
-    CL_CHECK(clSetKernelArg(kernel, i++, sizeof(cl_mem),   &eg->data_device));
-    CL_CHECK(clSetKernelArg(kernel, i++, sizeof(cl_ulong), &off_g));
-    CL_CHECK(clSetKernelArg(kernel, i++, sizeof(cl_mem),   &egb->data_device));
-    CL_CHECK(clSetKernelArg(kernel, i++, sizeof(cl_ulong), &off_gb));
-    CL_CHECK(clSetKernelArg(kernel, i++, sizeof(cl_mem),   &eu->data_device));
-    CL_CHECK(clSetKernelArg(kernel, i++, sizeof(cl_ulong), &off_u));
-    CL_CHECK(clSetKernelArg(kernel, i++, sizeof(cl_mem),   &eub->data_device));
-    CL_CHECK(clSetKernelArg(kernel, i++, sizeof(cl_ulong), &off_ub));
-    CL_CHECK(clSetKernelArg(kernel, i++, sizeof(cl_mem),   &ei->data_device));
-    CL_CHECK(clSetKernelArg(kernel, i++, sizeof(cl_ulong), &off_i));
-    CL_CHECK(clSetKernelArg(kernel, i++, sizeof(cl_mem),   &ed->data_device));
-    CL_CHECK(clSetKernelArg(kernel, i++, sizeof(cl_ulong), &off_d));
-    CL_CHECK(clSetKernelArg(kernel, i++, sizeof(cl_ulong), &nb01_g));
-    CL_CHECK(clSetKernelArg(kernel, i++, sizeof(cl_ulong), &nb02_g));
-    CL_CHECK(clSetKernelArg(kernel, i++, sizeof(cl_ulong), &nb01_u));
-    CL_CHECK(clSetKernelArg(kernel, i++, sizeof(cl_ulong), &nb02_u));
-    CL_CHECK(clSetKernelArg(kernel, i++, sizeof(cl_ulong), &nb11_g));
-    CL_CHECK(clSetKernelArg(kernel, i++, sizeof(cl_ulong), &nb11_u));
-    CL_CHECK(clSetKernelArg(kernel, i++, sizeof(cl_ulong), &nb21));
-    CL_CHECK(clSetKernelArg(kernel, i++, sizeof(cl_ulong), &nbd1));
-    CL_CHECK(clSetKernelArg(kernel, i++, sizeof(cl_ulong), &nbd2));
-    CL_CHECK(clSetKernelArg(kernel, i++, sizeof(int),      &ne0));
-    CL_CHECK(clSetKernelArg(kernel, i++, sizeof(float),    &limit));
-    CL_CHECK(clSetKernelArg(kernel, i++, sizeof(float),    &alpha));
-
-    const int nth = MIN(ne0, (int) backend_ctx->get_kernel_workgroup_size(kernel));
-    size_t global_work_size[] = { (size_t)glu->ne[1]*nth, (size_t)glu->ne[2], 1 };
-    size_t local_work_size[]  = { (size_t)nth, 1, 1 };
-
-    backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, (ggml_tensor *)glu);
-}
-
-// Fusion B: the MoE down-projection bias add feeding the combine.
-//
-// The graph runs ADD_ID(down_bias) and then immediately the combine subgraph
-// {MUL(router weights), k VIEWs, k-1 ADDs}, and the ADD_ID's only consumer is that
-// MUL. Since the ADD_ID is an in-place read-modify-write of a tensor the combine
-// reads once more, the bias can be added inside the combine instead, dropping a
-// full pass over [n_embd, k, n_tokens].
-//
-// Shape checks for the combine tail are delegated to ggml_opencl_can_fuse_moe_combine
-// (which also owns the n_nodes >= 32 bail and the experts/dst aliasing bail); what is
-// added here is the ADD_ID wiring plus a subgraph check over the WHOLE run, so that
-// the intermediate bias result is confirmed not to escape.
-static bool ggml_opencl_can_fuse_moe_bias_combine(const struct ggml_cgraph * cgraph, int node_idx,
-                                                  const ggml_tensor ** out_final_add) {
-    if (node_idx + 1 >= cgraph->n_nodes) {
-        return false;
-    }
-    const ggml_tensor * add = cgraph->nodes[node_idx];
-    if (add->op != GGML_OP_ADD_ID) {
-        return false;
-    }
-    const ggml_tensor * mul = cgraph->nodes[node_idx+1];
-    if (mul->op != GGML_OP_MUL || mul->src[0] != add) {
-        return false;
-    }
-
-    const ggml_tensor * final_add = NULL;
-    if (!ggml_opencl_can_fuse_moe_combine(cgraph, node_idx+1, &final_add)) {
-        return false;
-    }
-
-    const ggml_tensor * raw  = add->src[0];
-    const ggml_tensor * bias = add->src[1];
-    const ggml_tensor * ids  = add->src[2];
-    if (!raw || !bias || !ids) {
-        return false;
-    }
-    if (raw->type != GGML_TYPE_F32 || bias->type != GGML_TYPE_F32 ||
-        ids->type != GGML_TYPE_I32 || add->type != GGML_TYPE_F32) {
-        return false;
-    }
-    // The combine reads the raw matmul output with the strides it computed from the
-    // add_id result, so the two must have the same layout.
-    if (!ggml_are_same_shape(raw, add) || !ggml_is_contiguous(raw)) {
-        return false;
-    }
-    if (raw->nb[1] != add->nb[1] || raw->nb[2] != add->nb[2]) {
-        return false;
-    }
-    // ids is indexed as [expert slot, token]; the combine walks the same two axes.
-    if (ids->ne[0] < add->ne[1] || ids->ne[1] < add->ne[2]) {
-        return false;
-    }
-
-    // Whole-run escape check: ADD_ID + MUL + k VIEWs + (k-1) ADDs, only the last node escapes.
-    const int k       = (int)add->ne[1];
-    const int n_nodes = 2 + k + (k - 1);
-    if (n_nodes >= 32 || node_idx + n_nodes > cgraph->n_nodes) {
-        return false;
-    }
-    enum ggml_op ops[32];
-    int n = 0;
-    ops[n++] = GGML_OP_ADD_ID;
-    ops[n++] = GGML_OP_MUL;
-    for (int j = 0; j < k;     ++j) ops[n++] = GGML_OP_VIEW;
-    for (int j = 0; j < k - 1; ++j) ops[n++] = GGML_OP_ADD;
-    const int outs[] = { node_idx + n_nodes - 1 };
-    if (!ggml_can_fuse_subgraph(cgraph, node_idx, n_nodes, ops, outs, 1)) {
-        return false;
-    }
-
-    *out_final_add = final_add;
-    return true;
-}
-
-
-// Fusion B dispatch: the combine, reading the RAW matmul output and adding the
-// per-expert bias row inline. See ggml_opencl_can_fuse_moe_bias_combine.
-static void ggml_cl_moe_bias_combine_fused(ggml_backend_t backend, const ggml_tensor * add,
-                                           const ggml_tensor * mul, const ggml_tensor * dst) {
-    ggml_backend_opencl_context * backend_ctx = (ggml_backend_opencl_context *)backend->context;
-
-    const ggml_tensor * experts = add->src[0];   // raw matmul output, bias not yet applied
-    const ggml_tensor * bias    = add->src[1];
-    const ggml_tensor * ids     = add->src[2];
-    const ggml_tensor * weights = mul->src[1];
-
-    ggml_tensor_extra_cl * ee = (ggml_tensor_extra_cl *)experts->extra;
-    ggml_tensor_extra_cl * eb = (ggml_tensor_extra_cl *)bias->extra;
-    ggml_tensor_extra_cl * ei = (ggml_tensor_extra_cl *)ids->extra;
-    ggml_tensor_extra_cl * ew = (ggml_tensor_extra_cl *)weights->extra;
-    ggml_tensor_extra_cl * ed = (ggml_tensor_extra_cl *)dst->extra;
-    cl_ulong off_e = ee->offset + experts->view_offs;
-    cl_ulong off_b = eb->offset + bias->view_offs;
-    cl_ulong off_i = ei->offset + ids->view_offs;
-    cl_ulong off_w = ew->offset + weights->view_offs;
-    cl_ulong off_d = ed->offset + dst->view_offs;
-
-    const int n_embd4 = (int)(experts->ne[0] / 4);
-    const int k       = (int)experts->ne[1];
-    const int nt      = (int)experts->ne[2];
-    const cl_uint e1 = (cl_uint)(experts->nb[1] / sizeof(float));
-    const cl_uint e2 = (cl_uint)(experts->nb[2] / sizeof(float));
-    const cl_uint w1 = (cl_uint)(weights->nb[1] / sizeof(float));
-    const cl_uint w2 = (cl_uint)(weights->nb[2] / sizeof(float));
-    const cl_uint d1 = (cl_uint)(dst->nb[1] / sizeof(float));
-    const cl_ulong nb_b1 = bias->nb[1];
-    const cl_ulong nb_i1 = ids->nb[1];
-
-    const size_t w_bytes = ggml_nbytes(weights);
-    backend_ctx->prealloc_moe_combine_w.allocate(backend_ctx->context, w_bytes);
-    CL_CHECK(clEnqueueCopyBuffer(backend_ctx->queue, ew->data_device, backend_ctx->prealloc_moe_combine_w.buffer,
-                                 off_w, 0, w_bytes, 0, NULL, NULL));
-    cl_mem   w_dev = backend_ctx->prealloc_moe_combine_w.buffer;
-    cl_ulong w_off = 0;
-
-    cl_kernel kernel = backend_ctx->kernel_moe_combine_bias_f32;
-    int a = 0;
-    CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem),   &ee->data_device));
-    CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_ulong), &off_e));
-    CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem),   &w_dev));
-    CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_ulong), &w_off));
-    CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem),   &eb->data_device));
-    CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_ulong), &off_b));
-    CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem),   &ei->data_device));
-    CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_ulong), &off_i));
-    CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem),   &ed->data_device));
-    CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_ulong), &off_d));
-    CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int),      &n_embd4));
-    CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int),      &k));
-    CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int),      &nt));
-    CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_uint),  &e1));
-    CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_uint),  &e2));
-    CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_uint),  &w1));
-    CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_uint),  &w2));
-    CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_uint),  &d1));
-    CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_ulong), &nb_b1));
-    CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_ulong), &nb_i1));
-
-    size_t lws[2] = { 64, 1 };
-    size_t gws[2] = { (size_t)(((n_embd4 + 63) / 64) * 64), (size_t)nt };
-    backend_ctx->enqueue_ndrange_kernel(kernel, 2, gws, lws, (ggml_tensor *)dst);
-}
-
-
 static void ggml_cl_moe_combine_fused(ggml_backend_t backend, const ggml_tensor * mul, const ggml_tensor * dst) {
     ggml_backend_opencl_context * backend_ctx = (ggml_backend_opencl_context *)backend->context;
     const ggml_tensor * experts = mul->src[0];
@@ -7362,31 +7034,6 @@ static ggml_status ggml_backend_opencl_graph_compute(ggml_backend_t backend, ggm
         }
         // Fuse the MoE combine: router-weight mul + cross-expert add chain ->
         // one weighted-sum-across-experts kernel.
-        // Fold the gpt-oss MoE bias epilogue: add_id(gate_bias) + add_id(up_bias) +
-        // glu(swiglu_oai) -> one kernel, leaving the two matmuls as their own dispatches.
-        // Both add_ids are in-place passes over a tensor the GLU reads again, so this
-        // drops two full read+write passes per layer. Opt out GGML_OPENCL_FUSE_MOE_BIAS_GLU=0.
-        if (backend_ctx->fuse_moe_bias_glu && !backend_ctx->disable_fusion &&
-            ggml_opencl_can_fuse_moe_bias_glu(cgraph, i)) {
-            ggml_cl_moe_bias_glu_fused(backend, node, cgraph->nodes[i+1], cgraph->nodes[i+2],
-                                       cgraph->nodes[i+3], cgraph->nodes[i+4]);
-            i += 4;
-            continue;
-        }
-
-        // Fold the MoE down-projection bias into the combine: add_id(down_bias) + the whole
-        // combine subgraph -> one kernel. Checked before the plain combine arm so the longer
-        // pattern wins. Opt out GGML_OPENCL_FUSE_MOE_BIAS_COMBINE=0.
-        if (backend_ctx->fuse_moe_bias_combine && backend_ctx->fuse_moe_combine &&
-            !backend_ctx->disable_fusion) {
-            const ggml_tensor * bias_combine_out = nullptr;
-            if (ggml_opencl_can_fuse_moe_bias_combine(cgraph, i, &bias_combine_out)) {
-                ggml_cl_moe_bias_combine_fused(backend, node, cgraph->nodes[i+1], bias_combine_out);
-                i += 2 * (int)node->ne[1];   // ADD_ID + MUL + k VIEWs + (k-1) ADDs
-                continue;
-            }
-        }
-
         if (backend_ctx->fuse_moe_combine && !backend_ctx->disable_fusion) {
             const ggml_tensor * combine_out = nullptr;
             if (ggml_opencl_can_fuse_moe_combine(cgraph, i, &combine_out)) {
@@ -7746,19 +7393,6 @@ static bool ggml_opencl_supports_op(ggml_backend_dev_t dev, const struct ggml_te
                        op->src[0]->type == GGML_TYPE_Q4_K  ||
                        op->src[0]->type == GGML_TYPE_Q5_K  ||
                        op->src[0]->type == GGML_TYPE_Q6_K) {
-                // The E031.41 compiler (usually with A7x) miscompiles the flat K-quant
-                // GEMV kernels (kernel_mul_mv_q*_K_f32_flat) and makes lm_head run much
-                // slower than it should. So, make it fallback to CPU to preserve performance
-                // for this compiler series.
-                static const char * a7x_lmhead_env = getenv("GGML_OPENCL_A7X_LMHEAD_CPU");
-                static const bool   a7x_lmhead_cpu = (a7x_lmhead_env == nullptr || a7x_lmhead_env[0] != '0');
-                if (a7x_lmhead_cpu &&
-                    backend_ctx->adreno_gen == ADRENO_GPU_GEN::A7X &&
-                    (op->src[0]->type == GGML_TYPE_Q4_K || op->src[0]->type == GGML_TYPE_Q5_K ||
-                     op->src[0]->type == GGML_TYPE_Q6_K) &&
-                    op->src[0]->ne[1] >= 32768) {   // vocab-scale weight; no FFN/attn weight is this tall
-                    return false;
-                }
                 return op->src[1]->type == GGML_TYPE_F32 && ggml_is_contiguous(op->src[0]) && ggml_is_contiguous(op->src[1]);
             } else if (op->src[0]->type == GGML_TYPE_Q8_0) {
                 return op->src[1]->type == GGML_TYPE_F32;
@@ -7800,6 +7434,9 @@ static bool ggml_opencl_supports_op(ggml_backend_dev_t dev, const struct ggml_te
         case GGML_OP_DIAG_MASK_INF:
             return op->ne[3] == 1;
         case GGML_OP_ROPE: {
+            if (((const int32_t *) op->op_params)[15] != 0) {
+                return false; // FIXME: support ggml_rope_set_offset
+            }
             const int mode = ((const int32_t *) op->op_params)[2];
             const bool is_mrope = mode & GGML_ROPE_TYPE_MROPE;
             const bool is_vision = mode == GGML_ROPE_TYPE_VISION;
@@ -24273,7 +23910,6 @@ static void ggml_cl_rope(ggml_backend_t backend, const ggml_tensor * src0, const
     const int n_dims     = ((int *) dst->op_params)[1];
     const int mode       = ((int *) dst->op_params)[2];
     const int n_ctx_orig = ((int32_t *) dst->op_params)[4];
-    const int n_offs     = ((int32_t *) dst->op_params)[15];
 
     float freq_base;
     float freq_scale;
@@ -24302,7 +23938,6 @@ static void ggml_cl_rope(ggml_backend_t backend, const ggml_tensor * src0, const
 
     if (is_vision) {
         GGML_ASSERT(n_dims == ne00/2);
-        GGML_ASSERT(n_offs == 0); // offset not supported for vision, as the rotated pairs span the whole row
     }
 
     cl_kernel kernel;
@@ -24394,12 +24029,6 @@ static void ggml_cl_rope(ggml_backend_t backend, const ggml_tensor * src0, const
     if (is_mrope && !is_vision) {
         CL_CHECK(clSetKernelArg(kernel, 34, sizeof(int), &is_imrope));
     }
-    // norm and neox have n_offs after beta_slow, mrope has it after is_imrope
-    if (!is_mrope && !is_vision) {
-        CL_CHECK(clSetKernelArg(kernel, 33, sizeof(int), &n_offs));
-    } else if (is_mrope && !is_vision) {
-        CL_CHECK(clSetKernelArg(kernel, 35, sizeof(int), &n_offs));
-    }
 
     size_t global_work_size[] = {(size_t)ne01*nth, (size_t)ne02, (size_t)ne03};
     size_t local_work_size[] = {(size_t)nth, 1, 1};

ggml/src/ggml-opencl/kernels/moe_add_id_glu.cl

@@ -1,76 +0,0 @@
-#pragma OPENCL EXTENSION cl_khr_fp16 : enable
-
-//------------------------------------------------------------------------------
-// add_id(gate) + add_id(up) + swiglu_oai, fused
-//
-// gpt-oss-class MoE FFNs run three full passes over the same
-// [n_ff, n_expert_used, n_tokens] f32 tensor: a per-expert bias add on the gate
-// matmul output, the same on the up matmul output, then swiglu_oai over the
-// two. Both bias adds are in-place, so each costs a full read plus a full write
-// of a tensor that is only read once more. Folding them into the swiglu pass
-// leaves two reads and one write instead of six passes.
-//
-// Grouping matches kernel_add_id: group 0 = expert slot (i1), group 1 = token
-// (i2). For a contiguous destination that addressing is identical to the flat
-// row walk kernel_swiglu_oai uses, since row i1 + i2*ne1 sits at
-// i1*nb1 + i2*ne1*nb1.
-//------------------------------------------------------------------------------
-kernel void kernel_add_id_add_id_swiglu_oai(
-    global char * src_g,
-    ulong         offset_g,
-    global char * src_gb,
-    ulong         offset_gb,
-    global char * src_u,
-    ulong         offset_u,
-    global char * src_ub,
-    ulong         offset_ub,
-    global char * src_ids,
-    ulong         offset_ids,
-    global char * dst,
-    ulong         offsetd,
-    ulong         nb01_g,
-    ulong         nb02_g,
-    ulong         nb01_u,
-    ulong         nb02_u,
-    ulong         nb11_g,
-    ulong         nb11_u,
-    ulong         nb21,
-    ulong         nbd1,
-    ulong         nbd2,
-    int           ne0,
-    float         limit,
-    float         alpha
-) {
-    src_g   = (global char *)(src_g   + offset_g);
-    src_gb  = (global char *)(src_gb  + offset_gb);
-    src_u   = (global char *)(src_u   + offset_u);
-    src_ub  = (global char *)(src_ub  + offset_ub);
-    src_ids = (global char *)(src_ids + offset_ids);
-    dst     = (global char *)(dst     + offsetd);
-
-    const int i1 = get_group_id(0);
-    const int i2 = get_group_id(1);
-
-    // The ids tensor is a view into a [n_expert, n_tokens] buffer, so its row
-    // stride is nb21 and the k selected ids are NOT contiguous per token.
-    const int i11 = *((global const int *) (src_ids + i1*sizeof(int) + i2*nb21));
-
-    global const float * g_row  = (global const float *)(src_g  + i1*nb01_g + i2*nb02_g);
-    global const float * u_row  = (global const float *)(src_u  + i1*nb01_u + i2*nb02_u);
-    global const float * gb_row = (global const float *)(src_gb + i11*nb11_g);
-    global const float * ub_row = (global const float *)(src_ub + i11*nb11_u);
-    global       float * d_row  = (global       float *)(dst    + i1*nbd1   + i2*nbd2);
-
-    for (int i0 = get_local_id(0); i0 < ne0; i0 += get_local_size(0)) {
-        float x0 = g_row[i0] + gb_row[i0];
-        float x1 = u_row[i0] + ub_row[i0];
-
-        x0 = min(x0, limit);
-        x1 = max(min(x1, limit), -limit);
-
-        float out_glu = x0 / (1.0f + exp(-x0 * alpha));
-        out_glu = out_glu * (1.0f + x1);
-
-        d_row[i0] = out_glu;
-    }
-}

ggml/src/ggml-opencl/kernels/moe_combine.cl

@@ -8,49 +8,6 @@
 // buffer and the k-1 elementwise add round-trips). Vectorized float4 over rows.
 // strides e1/e2/w1/w2/d1 are in ELEMENTS (floats).
 
-// Same weighted sum, with the per-expert bias add folded in.
-//
-// The MoE down projection's bias is applied by an in-place add_id whose only
-// consumer is this combine, so it costs a full read plus a full write of a
-// tensor that is read once more immediately afterwards. Reading the raw matmul
-// output here and adding the bias row while it is already in registers removes
-// that pass. Kept as a separate kernel so the unfused path is untouched.
-__kernel void kernel_moe_combine_bias_f32(
-        __global const char * e_buf, ulong off_e,
-        __global const char * w_buf, ulong off_w,
-        __global const char * b_buf, ulong off_b,   // per-expert bias rows
-        __global const char * i_buf, ulong off_i,   // expert ids
-        __global       char * d_buf, ulong off_d,
-        int  n_embd4,            // n_embd / 4
-        int  k,                  // n_expert_used
-        int  n_tokens,
-        uint e1, uint e2,        // experts strides (elements): per-expert, per-token
-        uint w1, uint w2,        // weights strides (elements)
-        uint d1,                 // dst per-token stride (elements)
-        ulong nb_b1,             // bias row stride (bytes)
-        ulong nb_i1)             // ids row stride (bytes) - ids is a view, not packed
-{
-    const uint r4  = get_global_id(0);
-    const uint tok = get_global_id(1);
-    if (r4 >= (uint)n_embd4 || tok >= (uint)n_tokens) return;
-
-    __global const float * E = (__global const float *)(e_buf + off_e) + tok*e2 + r4*4u;
-    __global const float * W = (__global const float *)(w_buf + off_w) + tok*w2;
-    __global const char  * B = b_buf + off_b;
-    __global const char  * I = i_buf + off_i + (ulong)tok*nb_i1;
-
-    float4 acc = (float4)(0.0f);
-    for (int e = 0; e < k; ++e) {
-        const int i11 = *((__global const int *)(I + (ulong)e*sizeof(int)));
-        __global const float * Brow = (__global const float *)(B + (ulong)i11*nb_b1) + r4*4u;
-        const float4 v = vload4(0, E + (uint)e*e1) + vload4(0, Brow);
-        acc = mad(v, (float4)(W[(uint)e*w1]), acc);
-    }
-
-    __global float * D = (__global float *)(d_buf + off_d) + tok*d1 + r4*4u;
-    vstore4(acc, 0, D);
-}
-
 __kernel void kernel_moe_combine_f32(
         __global const char * e_buf, ulong off_e,
         __global const char * w_buf, ulong off_w,

ggml/src/ggml-opencl/kernels/rope.cl

@@ -75,8 +75,7 @@ kernel void kernel_rope_norm_f32(
         float ext_factor,
         float attn_factor,
         float beta_fast,
-        float beta_slow,
-        int n_offs
+        float beta_slow
 ) {
     src0 = (global void*)((global char*)src0 + offset0);
     src1 = (global int*)((global char*)src1 + offset1);
@@ -95,15 +94,14 @@ kernel void kernel_rope_norm_f32(
     float inv_ndims = -1.f/n_dims;
 
     for (int i0 = 2*get_local_id(0); i0 < ne0; i0 += 2*get_local_size(0)) {
-        if (i0 >= n_offs && i0 < n_offs + n_dims) {
-            int iw = i0 - n_offs; // relative idx
-            int ic = iw/2;
+        if (i0 < n_dims) {
+            int ic = i0/2;
 
-            float theta = theta_base * pow(freq_base, inv_ndims*iw);
+            float theta = theta_base * pow(freq_base, inv_ndims*i0);
 
             float freq_factor = src2 != src0 ? src2[ic] : 1.0f;
 
-            float2 cos_sin_theta = rope_yarn(theta/freq_factor, freq_scale, corr_dims, iw, ext_factor, attn_factor);
+            float2 cos_sin_theta = rope_yarn(theta/freq_factor, freq_scale, corr_dims, i0, ext_factor, attn_factor);
 
             global float * src       = (global float *)((global char *) src0 + i3*nb03 + i2*nb02 + i1*nb01 + i0*nb00);
             global float * dst_data  = (global float *)((global char *)  dst + i3*nb3  + i2*nb2  + i1*nb1  + i0*nb0);
@@ -156,8 +154,7 @@ kernel void kernel_rope_norm_f16(
         float ext_factor,
         float attn_factor,
         float beta_fast,
-        float beta_slow,
-        int n_offs
+        float beta_slow
 ) {
     src0 = (global void*)((global char*)src0 + offset0);
     src1 = (global int*)((global char*)src1 + offset1);
@@ -176,15 +173,14 @@ kernel void kernel_rope_norm_f16(
     float inv_ndims = -1.f/n_dims;
 
     for (int i0 = 2*get_local_id(0); i0 < ne0; i0 += 2*get_local_size(0)) {
-        if (i0 >= n_offs && i0 < n_offs + n_dims) {
-            int iw = i0 - n_offs; // relative idx
-            int ic = iw/2;
+        if (i0 < n_dims) {
+            int ic = i0/2;
 
-            float theta = theta_base * pow(freq_base, inv_ndims*iw);
+            float theta = theta_base * pow(freq_base, inv_ndims*i0);
 
             float freq_factor = src2 != src0 ? src2[ic] : 1.0f;
 
-            float2 cos_sin_theta = rope_yarn(theta/freq_factor, freq_scale, corr_dims, iw, ext_factor, attn_factor);
+            float2 cos_sin_theta = rope_yarn(theta/freq_factor, freq_scale, corr_dims, i0, ext_factor, attn_factor);
 
             global half * src       = (global half *)((global char *) src0 + i3*nb03 + i2*nb02 + i1*nb01 + i0*nb00);
             global half * dst_data  = (global half *)((global char *)  dst + i3*nb3  + i2*nb2  + i1*nb1  + i0*nb0);
@@ -237,8 +233,7 @@ kernel void kernel_rope_neox_f32(
         float ext_factor,
         float attn_factor,
         float beta_fast,
-        float beta_slow,
-        int n_offs
+        float beta_slow
 ) {
     src0 = (global void*)((global char*)src0 + offset0);
     src1 = (global int*)((global char*)src1 + offset1);
@@ -257,18 +252,17 @@ kernel void kernel_rope_neox_f32(
     float inv_ndims = -1.f/n_dims;
 
     for (int i0 = 2*get_local_id(0); i0 < ne0; i0 += 2*get_local_size(0)) {
-        if (i0 >= n_offs && i0 < n_offs + n_dims) {
-            int iw = i0 - n_offs; // relative idx
-            int ic = iw/2;
+        if (i0 < n_dims) {
+            int ic = i0/2;
 
-            const float theta = theta_base * pow(freq_base, inv_ndims*iw);
+            const float theta = theta_base * pow(freq_base, inv_ndims*i0);
 
             const float freq_factor = src2 != src0 ? src2[ic] : 1.0f;
 
-            float2 cos_sin_theta = rope_yarn(theta/freq_factor, freq_scale, corr_dims, iw, ext_factor, attn_factor);
+            float2 cos_sin_theta = rope_yarn(theta/freq_factor, freq_scale, corr_dims, i0, ext_factor, attn_factor);
 
-            global float * src      = (global float *)((global char *) src0 + i3*nb03 + i2*nb02 + i1*nb01 + (n_offs + ic)*nb00);
-            global float * dst_data = (global float *)((global char *)  dst + i3*nb3  + i2*nb2  + i1*nb1  + (n_offs + ic)*nb0);
+            global float * src      = (global float *)((global char *) src0 + i3*nb03 + i2*nb02 + i1*nb01 + ic*nb00);
+            global float * dst_data = (global float *)((global char *)  dst + i3*nb3  + i2*nb2  + i1*nb1  + ic*nb0);
 
             const float x0 = src[0];
             const float x1 = src[n_dims/2];
@@ -318,8 +312,7 @@ kernel void kernel_rope_neox_f16(
         float ext_factor,
         float attn_factor,
         float beta_fast,
-        float beta_slow,
-        int n_offs
+        float beta_slow
 ) {
     src0 = (global void*)((global char*)src0 + offset0);
     src1 = (global int*)((global char*)src1 + offset1);
@@ -338,18 +331,17 @@ kernel void kernel_rope_neox_f16(
     float inv_ndims = -1.f/n_dims;
 
     for (int i0 = 2*get_local_id(0); i0 < ne0; i0 += 2*get_local_size(0)) {
-        if (i0 >= n_offs && i0 < n_offs + n_dims) {
-            int iw = i0 - n_offs; // relative idx
-            int ic = iw/2;
+        if (i0 < n_dims) {
+            int ic = i0/2;
 
-            const float theta = theta_base * pow(freq_base, inv_ndims*iw);
+            const float theta = theta_base * pow(freq_base, inv_ndims*i0);
 
             const float freq_factor = src2 != src0 ? src2[ic] : 1.0f;
 
-            float2 cos_sin_theta = rope_yarn(theta/freq_factor, freq_scale, corr_dims, iw, ext_factor, attn_factor);
+            float2 cos_sin_theta = rope_yarn(theta/freq_factor, freq_scale, corr_dims, i0, ext_factor, attn_factor);
 
-            global half * src       = (global half *)((global char *) src0 + i3*nb03 + i2*nb02 + i1*nb01 + (n_offs + ic)*nb00);
-            global half * dst_data  = (global half *)((global char *)  dst + i3*nb3  + i2*nb2  + i1*nb1  + (n_offs + ic)*nb0);
+            global half * src       = (global half *)((global char *) src0 + i3*nb03 + i2*nb02 + i1*nb01 + ic*nb00);
+            global half * dst_data  = (global half *)((global char *)  dst + i3*nb3  + i2*nb2  + i1*nb1  + ic*nb0);
 
             const float x0 = src[0];
             const float x1 = src[n_dims/2];
@@ -401,8 +393,7 @@ kernel void kernel_rope_multi_f32(
         float beta_fast,
         float beta_slow,
         int4 sections,
-        int  is_imrope,
-        int n_offs
+        int  is_imrope
 ) {
     src0 = (global void*)((global char*)src0 + offset0);
     src1 = (global int*)((global char*)src1 + offset1);
@@ -423,11 +414,10 @@ kernel void kernel_rope_multi_f32(
     float inv_ndims = -1.f/n_dims;
 
     for (int i0 = 2*get_local_id(0); i0 < ne0; i0 += 2*get_local_size(0)) {
-        if (i0 >= n_offs && i0 < n_offs + n_dims) {
-            int iw = i0 - n_offs; // relative idx
-            int ic = iw/2;
+        if (i0 < n_dims) {
+            int ic = i0/2;
 
-            const int sector = ic % sect_dims;
+            const int sector = (i0 / 2) % sect_dims;
             float theta_base = 0.0f;
 
             if (is_imrope) {
@@ -455,14 +445,14 @@ kernel void kernel_rope_multi_f32(
                 }
             }
 
-            const float theta = theta_base * pow(freq_base, inv_ndims*iw);
+            const float theta = theta_base * pow(freq_base, inv_ndims*i0);
 
             const float freq_factor = src2 != src0 ? src2[ic] : 1.0f;
 
-            float2 cos_sin_theta = rope_yarn(theta/freq_factor, freq_scale, corr_dims, iw, ext_factor, attn_factor);
+            float2 cos_sin_theta = rope_yarn(theta/freq_factor, freq_scale, corr_dims, i0, ext_factor, attn_factor);
 
-            global float * src      = (global float *)((global char *) src0 + i3*nb03 + i2*nb02 + i1*nb01 + (n_offs + ic)*nb00);
-            global float * dst_data = (global float *)((global char *)  dst + i3*nb3  + i2*nb2  + i1*nb1  + (n_offs + ic)*nb0);
+            global float * src      = (global float *)((global char *) src0 + i3*nb03 + i2*nb02 + i1*nb01 + ic*nb00);
+            global float * dst_data = (global float *)((global char *)  dst + i3*nb3  + i2*nb2  + i1*nb1  + ic*nb0);
 
             const float x0 = src[0];
             const float x1 = src[n_dims/2];
@@ -514,8 +504,7 @@ kernel void kernel_rope_multi_f16(
         float beta_fast,
         float beta_slow,
         int4 sections,
-        int  is_imrope,
-        int n_offs
+        int  is_imrope
 ) {
     src0 = (global void*)((global char*)src0 + offset0);
     src1 = (global int*)((global char*)src1 + offset1);
@@ -536,11 +525,10 @@ kernel void kernel_rope_multi_f16(
     float inv_ndims = -1.f/n_dims;
 
     for (int i0 = 2*get_local_id(0); i0 < ne0; i0 += 2*get_local_size(0)) {
-        if (i0 >= n_offs && i0 < n_offs + n_dims) {
-            int iw = i0 - n_offs; // relative idx
-            int ic = iw/2;
+        if (i0 < n_dims) {
+            int ic = i0/2;
 
-            const int sector = ic % sect_dims;
+            const int sector = (i0 / 2) % sect_dims;
             float theta_base = 0.0f;
 
             if (is_imrope) {
@@ -568,14 +556,14 @@ kernel void kernel_rope_multi_f16(
                 }
             }
 
-            const float theta = theta_base * pow(freq_base, inv_ndims*iw);
+            const float theta = theta_base * pow(freq_base, inv_ndims*i0);
 
             const float freq_factor = src2 != src0 ? src2[ic] : 1.0f;
 
-            float2 cos_sin_theta = rope_yarn(theta/freq_factor, freq_scale, corr_dims, iw, ext_factor, attn_factor);
+            float2 cos_sin_theta = rope_yarn(theta/freq_factor, freq_scale, corr_dims, i0, ext_factor, attn_factor);
 
-            global half * src      = (global half *)((global char *) src0 + i3*nb03 + i2*nb02 + i1*nb01 + (n_offs + ic)*nb00);
-            global half * dst_data = (global half *)((global char *)  dst + i3*nb3  + i2*nb2  + i1*nb1  + (n_offs + ic)*nb0);
+            global half * src      = (global half *)((global char *) src0 + i3*nb03 + i2*nb02 + i1*nb01 + ic*nb00);
+            global half * dst_data = (global half *)((global char *)  dst + i3*nb3  + i2*nb2  + i1*nb1  + ic*nb0);
 
             const float x0 = src[0];
             const float x1 = src[n_dims/2];

ggml/src/ggml-sycl/convert.cpp

@@ -76,19 +76,6 @@ static void dequantize_row_q2_K_sycl(const void *vx, dst_t *y, const int64_t k,
 #endif
 }
 
-template <typename dst_t>
-static void dequantize_row_q2_K_sycl_reorder(const void *vx, dst_t *y, const int64_t k,
-                                             dpct::queue_ptr stream) {
-    const int64_t nb = k / QK_K;
-
-    dpct::has_capability_or_fail(stream->get_device(), { sycl::aspect::fp16 });
-    stream->parallel_for(
-        sycl::nd_range<3>(sycl::range<3>(1, 1, nb) * sycl::range<3>(1, 1, 64), sycl::range<3>(1, 1, 64)),
-        [=](sycl::nd_item<3> item_ct1) {
-            dequantize_block_q2_K_reorder(vx, y, item_ct1, nb);
-        });
-}
-
 template <typename dst_t>
 static void dequantize_row_q3_K_sycl(const void *vx, dst_t *y, const int64_t k,
                                      dpct::queue_ptr stream) {
@@ -680,11 +667,7 @@ to_fp16_sycl_t ggml_get_to_fp16_sycl(ggml_type type, ggml_tensor * dst) {
                 return dequantize_block_sycl<QK8_0, QR8_0, dequantize_q8_0>;
             }
         case GGML_TYPE_Q2_K:
-            if (dst->src[0]->extra && ((ggml_tensor_extra_gpu *) dst->src[0]->extra)->optimized_feature.reorder) {
-                return dequantize_row_q2_K_sycl_reorder;
-            } else {
-                return dequantize_row_q2_K_sycl;
-            }
+            return dequantize_row_q2_K_sycl;
         case GGML_TYPE_Q3_K:
             if (dst->src[0]->extra && ((ggml_tensor_extra_gpu *) dst->src[0]->extra)->optimized_feature.reorder) {
                 return dequantize_row_q3_K_sycl_reorder;
@@ -770,11 +753,7 @@ to_fp32_sycl_t ggml_get_to_fp32_sycl(ggml_type type, ggml_tensor *dst) {
                 return dequantize_block_sycl<QK8_0, QR8_0, dequantize_q8_0>;
             }
         case GGML_TYPE_Q2_K:
-            if (dst->src[0]->extra && ((ggml_tensor_extra_gpu *) dst->src[0]->extra)->optimized_feature.reorder) {
-                return dequantize_row_q2_K_sycl_reorder;
-            } else {
-                return dequantize_row_q2_K_sycl;
-            }
+            return dequantize_row_q2_K_sycl;
         case GGML_TYPE_Q3_K:
             if (dst->src[0]->extra && ((ggml_tensor_extra_gpu *) dst->src[0]->extra)->optimized_feature.reorder) {
                 return dequantize_row_q3_K_sycl_reorder;

ggml/src/ggml-sycl/dequantize.hpp

@@ -943,47 +943,6 @@ static void dequantize_block_q2_K(const void * __restrict__ vx, dst_t * __restri
 
 }
 
-template<typename dst_t>
-static void dequantize_block_q2_K_reorder(const void * __restrict__ vx, dst_t * __restrict__ yy,
-                                          const sycl::nd_item<3> & item_ct1, int64_t n_blocks) {
-#if QK_K == 256
-    const int64_t i = item_ct1.get_group(2);
-    if (i >= n_blocks) {
-        return;
-    }
-
-    const uint8_t * base          = static_cast<const uint8_t *>(vx);
-    const size_t    qs_offset     = i * (QK_K / 4);
-    const size_t    scales_offset = n_blocks * (QK_K / 4) + i * (QK_K / 16);
-    const size_t    dm_offset     = n_blocks * (QK_K / 4) + n_blocks * (QK_K / 16) + i * sizeof(ggml_half2);
-
-    const uint8_t *     qs     = base + qs_offset;
-    const uint8_t *     scales = base + scales_offset;
-    const ggml_half2 * dm     = reinterpret_cast<const ggml_half2 *>(base + dm_offset);
-
-    const int64_t tid = item_ct1.get_local_id(2);
-    const int64_t n    = tid / 32;
-    const int64_t l    = tid - 32 * n;
-    const int64_t is   = 8 * n + l / 16;
-
-    const uint8_t q = qs[32 * n + l];
-    dst_t * y = yy + i * QK_K + 128 * n;
-
-    const float dall = (*dm)[0];
-    const float dmin = (*dm)[1];
-    y[l+ 0] = dall * (scales[is+0] & 0xF) * ((q >> 0) & 3) - dmin * (scales[is+0] >> 4);
-    y[l+32] = dall * (scales[is+2] & 0xF) * ((q >> 2) & 3) - dmin * (scales[is+2] >> 4);
-    y[l+64] = dall * (scales[is+4] & 0xF) * ((q >> 4) & 3) - dmin * (scales[is+4] >> 4);
-    y[l+96] = dall * (scales[is+6] & 0xF) * ((q >> 6) & 3) - dmin * (scales[is+6] >> 4);
-#else
-    GGML_UNUSED(vx);
-    GGML_UNUSED(yy);
-    GGML_UNUSED(item_ct1);
-    GGML_UNUSED(n_blocks);
-    GGML_ABORT("Q2_K reorder dequantize not supported for QK_K != 256");
-#endif
-}
-
 template<typename dst_t>
 static void dequantize_block_q3_K(const void * __restrict__ vx, dst_t * __restrict__ yy,
                                   const sycl::nd_item<3> &item_ct1) {

ggml/src/ggml-sycl/dmmv.cpp

@@ -1921,23 +1921,6 @@ ESIMD_INLINE void dequantize_mul_mat_vec_reorder_esimd(
     }
 }
 
-static void dequantize_mul_mat_vec_q2_K_sycl_reorder_esimd(const void *vx, const float *y,
-                                                           float *dst, const int ncols,
-                                                           const int nrows,
-                                                           dpct::queue_ptr stream) {
-    GGML_ASSERT(ncols % QK_K == 0);
-    const int workgroups = (nrows + 1) / 2;
-    stream->submit([&](sycl::handler &h) {
-        sycl::local_accessor<float, 1> lmem(sycl::range<1>(GGML_SYCL_DMMV_ESIMD_WG_SIZE * 2), h);
-        h.parallel_for(
-            sycl::nd_range<1>(sycl::range<1>((size_t)workgroups * GGML_SYCL_DMMV_ESIMD_WG_SIZE), sycl::range<1>(GGML_SYCL_DMMV_ESIMD_WG_SIZE)),
-            [=](sycl::nd_item<1> it) [[intel::sycl_explicit_simd]] {
-                dequantize_mul_mat_vec_reorder_esimd<GGML_TYPE_Q2_K>(
-                    vx, y, dst, ncols, nrows, lmem, it);
-            });
-    });
-}
-
 static void dequantize_mul_mat_vec_q3_K_sycl_reorder_esimd(const void *vx, const float *y,
                                                            float *dst, const int ncols,
                                                            const int nrows,
@@ -1972,23 +1955,6 @@ static void dequantize_mul_mat_vec_q4_K_sycl_reorder_esimd(const void *vx, const
     });
 }
 
-static void dequantize_mul_mat_vec_q5_K_sycl_reorder_esimd(const void *vx, const float *y,
-                                                           float *dst, const int ncols,
-                                                           const int nrows,
-                                                           dpct::queue_ptr stream) {
-    GGML_ASSERT(ncols % QK_K == 0);
-    const int workgroups = (nrows + 1) / 2;
-    stream->submit([&](sycl::handler &h) {
-        sycl::local_accessor<float, 1> lmem(sycl::range<1>(GGML_SYCL_DMMV_ESIMD_WG_SIZE * 2), h);
-        h.parallel_for(
-            sycl::nd_range<1>(sycl::range<1>((size_t)workgroups * GGML_SYCL_DMMV_ESIMD_WG_SIZE), sycl::range<1>(GGML_SYCL_DMMV_ESIMD_WG_SIZE)),
-            [=](sycl::nd_item<1> it) [[intel::sycl_explicit_simd]] {
-                dequantize_mul_mat_vec_reorder_esimd<GGML_TYPE_Q5_K>(
-                    vx, y, dst, ncols, nrows, lmem, it);
-            });
-    });
-}
-
 static void dequantize_mul_mat_vec_q6_K_sycl_reorder_esimd(const void *vx, const float *y,
                                                            float *dst, const int ncols,
                                                            const int nrows,
@@ -2128,15 +2094,7 @@ void ggml_sycl_op_dequantize_mul_mat_vec(
         case GGML_TYPE_Q2_K:
             if ((ggml_tensor_extra_gpu *) dst->src[0]->extra &&
                 ((ggml_tensor_extra_gpu *) dst->src[0]->extra)->optimized_feature.reorder) {
-#ifdef GGML_SYCL_DMMV_HAS_ESIMD
-                if (g_ggml_sycl_enable_esimd) {
-                    dequantize_mul_mat_vec_q2_K_sycl_reorder_esimd(src0_dd_i, src1_ddf_i, dst_dd_i, ne00, row_diff, stream);
-                }
-                else
-#endif
-                {
-                    dequantize_mul_mat_vec_q2_K_sycl_reorder(src0_dd_i, src1_ddf_i, dst_dd_i, ne00, row_diff, stream);
-                }
+                dequantize_mul_mat_vec_q2_K_sycl_reorder(src0_dd_i, src1_ddf_i, dst_dd_i, ne00, row_diff, stream);
             } else {
                 dequantize_mul_mat_vec_q2_K_sycl(src0_dd_i, src1_ddf_i, dst_dd_i, ne00, row_diff, stream);
             }
@@ -2176,15 +2134,7 @@ void ggml_sycl_op_dequantize_mul_mat_vec(
         case GGML_TYPE_Q5_K:
             if ((ggml_tensor_extra_gpu *) dst->src[0]->extra &&
                 ((ggml_tensor_extra_gpu *) dst->src[0]->extra)->optimized_feature.reorder) {
-#ifdef GGML_SYCL_DMMV_HAS_ESIMD
-                if (g_ggml_sycl_enable_esimd) {
-                    dequantize_mul_mat_vec_q5_K_sycl_reorder_esimd(src0_dd_i, src1_ddf_i, dst_dd_i, ne00, row_diff, stream);
-                }
-                else
-#endif
-                {
-                    dequantize_mul_mat_vec_q5_K_sycl_reorder(src0_dd_i, src1_ddf_i, dst_dd_i, ne00, row_diff, stream);
-                }
+                dequantize_mul_mat_vec_q5_K_sycl_reorder(src0_dd_i, src1_ddf_i, dst_dd_i, ne00, row_diff, stream);
             } else {
                 dequantize_mul_mat_vec_q5_K_sycl(src0_dd_i, src1_ddf_i, dst_dd_i, ne00, row_diff, stream);
             }

ggml/src/ggml-sycl/dpct/helper.hpp

@@ -62,7 +62,7 @@
 
 #define DPCT_UNUSED(x) (void)(x)
 
-[[noreturn]] inline void _abort(const char * str) {
+inline void _abort(const char * str) {
     std::cerr << str << std::endl;
     std::abort();
 }

ggml/src/ggml-sycl/element_wise.cpp

@@ -10,7 +10,7 @@
     (ITEM.get_local_range(IDX) * ITEM.get_group(IDX) + ITEM.get_local_id(IDX))
 
 static void acc_f32(const char * x, const char * y, float * dst, const int64_t ne,
-    const int64_t ne0, const int64_t ne1, const int64_t ne2,
+    const int64_t ne0, const int64_t ne1, const int64_t ne2, const int64_t ne3,
     const int64_t nb00, const int64_t nb01, const int64_t nb02, const int64_t nb03,
     const int64_t ne10, const int64_t ne11, const int64_t ne12, const int64_t ne13,
     const int64_t nb10, const int64_t nb11, const int64_t nb12, const int64_t nb13,
@@ -455,7 +455,7 @@ static void unary_mul_sycl(const T * x, const T * g, T * dst, const int64_t k, c
 namespace ggml_sycl_detail {
 static void acc_f32_sycl(const char *x, const char *y, float *dst,
                          const int64_t n_elements,
-                         const int64_t ne0, const int64_t ne1, const int64_t ne2,
+                         const int64_t ne0, const int64_t ne1, const int64_t ne2, const int64_t ne3,
                          const int64_t nb00, const int64_t nb01, const int64_t nb02, const int64_t nb03,
                          const int64_t ne10, const int64_t ne11, const int64_t ne12, const int64_t ne13,
                          const int64_t nb10, const int64_t nb11, const int64_t nb12, const int64_t nb13,
@@ -466,7 +466,7 @@ static void acc_f32_sycl(const char *x, const char *y, float *dst,
                                            sycl::range<3>(1, 1, SYCL_ACC_BLOCK_SIZE)),
                          [=](sycl::nd_item<3> /*item_ct1*/) [[sycl::reqd_sub_group_size(WARP_SIZE)]] {
                             acc_f32(x, y, dst, n_elements,
-                                ne0, ne1, ne2,
+                                ne0, ne1, ne2, ne3,
                                 nb00, nb01, nb02, nb03,
                                 ne10, ne11, ne12, ne13,
                                 nb10, nb11, nb12, nb13,
@@ -970,7 +970,7 @@ static inline void ggml_sycl_op_acc(ggml_backend_sycl_context & ctx, ggml_tensor
     const int64_t offset = (int64_t) ((const int32_t *) dst->op_params)[3] / (int64_t) sizeof(float);
 
     ggml_sycl_detail::acc_f32_sycl(src0_d, src1_d, dst_d, ggml_nelements(dst),
-        dst->ne[0], dst->ne[1], dst->ne[2],
+        dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3],
         src0->nb[0], src0->nb[1], src0->nb[2], src0->nb[3],
         src1->ne[0], src1->ne[1], src1->ne[2], src1->ne[3],
         src1->nb[0], src1->nb[1], src1->nb[2], src1->nb[3],

ggml/src/ggml-sycl/esimd.hpp

@@ -1,3 +1,15 @@
+//
+// MIT license
+// Copyright (C) 2026 Intel Corporation
+// SPDX-License-Identifier: MIT
+//
+
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+
 #ifndef GGML_SYCL_ESIMD_HPP
 #define GGML_SYCL_ESIMD_HPP
 
@@ -61,93 +73,6 @@ static ESIMD_INLINE void unpack_scale_min_k4(
     min_f   = convert<float>(m) * (-dmin);
 }
 
-// ---------------------------------------------------------------------------
-// Q2_K, SOA reorder layout produced by reorder_qw_q2_k:
-//   [qs: nb*(QK_K/4)] [scales: nb*(QK_K/16)] [dm: nb*sizeof(half2)]
-// with nb = nrows*num_blocks_per_row.
-//
-// 2 bits per weight. The 8 output chunks of 32 (matching dequantize_row_q2_K)
-// map to super-chunk s (0..7): byte base 32*(s/4) into the 64-byte qs array,
-// bit shift 2*(s%4); the low 16 lanes use scales[2s], the high 16 use
-// scales[2s+1], with dl = d*(sc & 0xF), ml = dmin*(sc >> 4), deq = dl*q - ml.
-// ---------------------------------------------------------------------------
-template <> struct esimd_reorder_q_traits<GGML_TYPE_Q2_K> {
-    struct ptrs {
-        const uint8_t *    qs;
-        const uint8_t *    scales;
-        const sycl::half * dm;
-    };
-
-    static ESIMD_INLINE ptrs make_ptrs(const void * vx, size_t nb) {
-        const uint8_t * qs     = (const uint8_t *) vx;
-        const uint8_t * scales = qs + nb * (QK_K / 4);
-        const sycl::half * dm  = (const sycl::half *) (scales + nb * (QK_K / 16));
-        return { qs, scales, dm };
-    }
-
-    static ESIMD_INLINE void mac_pair(
-            const ptrs & pa, size_t bia,
-            const ptrs & pb, size_t bib, bool has_b,
-            sycl::ext::intel::esimd::simd<float, 256> & y_vec,
-            sycl::ext::intel::esimd::simd<float, 32> & acc_a,
-            sycl::ext::intel::esimd::simd<float, 32> & acc_b) {
-        using namespace sycl::ext::intel::esimd;
-
-        simd<uint8_t, 64> qs_a     = block_load<uint8_t, 64>(pa.qs + bia * (QK_K / 4));
-        simd<uint8_t, 64> qs_b     = 0;
-        simd<uint8_t, 16> scales_a = block_load<uint8_t, 16>(pa.scales + bia * (QK_K / 16));
-        simd<uint8_t, 16> scales_b = 0;
-
-        const float dall_a = (float) pa.dm[bia * 2 + 0];
-        const float dmin_a = (float) pa.dm[bia * 2 + 1];
-        float dall_b = 0.0f;
-        float dmin_b = 0.0f;
-        if (has_b) {
-            qs_b     = block_load<uint8_t, 64>(pb.qs + bib * (QK_K / 4));
-            scales_b = block_load<uint8_t, 16>(pb.scales + bib * (QK_K / 16));
-            dall_b = (float) pb.dm[bib * 2 + 0];
-            dmin_b = (float) pb.dm[bib * 2 + 1];
-        }
-
-        // per-chunk scale (d * (sc & 0xF)) and min (-dmin * (sc >> 4)), all 16 codes;
-        // min carries the negation so the dequant epilogue adds (matches Q4_K/Q5_K)
-        simd<float, 16> scale_f_a = convert<float>(scales_a & simd<uint8_t, 16>(0x0F)) * dall_a;
-        simd<float, 16> min_f_a   = convert<float>(scales_a >> simd<uint8_t, 16>(4))  * (-dmin_a);
-        simd<float, 16> scale_f_b = convert<float>(scales_b & simd<uint8_t, 16>(0x0F)) * dall_b;
-        simd<float, 16> min_f_b   = convert<float>(scales_b >> simd<uint8_t, 16>(4))  * (-dmin_b);
-
-#pragma unroll
-        for (int s = 0; s < 8; ++s) {
-            const int     byte_base = 32 * (s / 4);
-            const uint8_t shift     = (uint8_t) (2 * (s % 4));
-            simd<float, 32> y_s = y_vec.select<32, 1>(s * 32);
-
-            simd<uint8_t, 32> qa = (qs_a.select<32, 1>(byte_base) >> shift) & simd<uint8_t, 32>(3);
-            simd<uint8_t, 32> qb = (qs_b.select<32, 1>(byte_base) >> shift) & simd<uint8_t, 32>(3);
-
-            const float scale_a_lo = scale_f_a[2 * s + 0];
-            const float scale_a_hi = scale_f_a[2 * s + 1];
-            const float min_a_lo   = min_f_a[2 * s + 0];
-            const float min_a_hi   = min_f_a[2 * s + 1];
-            const float scale_b_lo = scale_f_b[2 * s + 0];
-            const float scale_b_hi = scale_f_b[2 * s + 1];
-            const float min_b_lo   = min_f_b[2 * s + 0];
-            const float min_b_hi   = min_f_b[2 * s + 1];
-
-            simd<float, 32> scale_vec_a = splat_lo_hi(scale_a_lo, scale_a_hi);
-            simd<float, 32> min_vec_a   = splat_lo_hi(min_a_lo, min_a_hi);
-            simd<float, 32> scale_vec_b = splat_lo_hi(scale_b_lo, scale_b_hi);
-            simd<float, 32> min_vec_b   = splat_lo_hi(min_b_lo, min_b_hi);
-
-            simd<float, 32> deq_a = convert<float>(qa) * scale_vec_a + min_vec_a;
-            simd<float, 32> deq_b = convert<float>(qb) * scale_vec_b + min_vec_b;
-
-            acc_a += y_s * deq_a;
-            acc_b += y_s * deq_b;
-        }
-    }
-};
-
 // ---------------------------------------------------------------------------
 // Q3_K, SOA reorder layout produced by reorder_qw_q3_k:
 //   [qs: nb*(QK_K/4)] [hmask: nb*(QK_K/8)] [scales: nb*12] [d: nb*sizeof(half)]
@@ -362,128 +287,6 @@ template <> struct esimd_reorder_q_traits<GGML_TYPE_Q4_K> {
     }
 };
 
-// ---------------------------------------------------------------------------
-// Q5_K, SOA reorder layout produced by reorder_qw_q5_k:
-//   [qs: nb*(QK_K/2)] [qh: nb*(QK_K/8)] [scales: nb*K_SCALE_SIZE] [dm: nb*sizeof(half2)]
-// with nb = nrows*num_blocks_per_row.
-//
-// Identical to Q4_K except each 4-bit quant gains a 5th (high) bit from qh:
-// output chunk c (0..7) adds 16 when bit c of qh[l] is set, where qh[l] indexes
-// the same 32 bytes for every chunk (matches dequantize_row_q5_K).
-// ---------------------------------------------------------------------------
-template <> struct esimd_reorder_q_traits<GGML_TYPE_Q5_K> {
-    struct ptrs {
-        const uint8_t *    qs;
-        const uint8_t *    qh;
-        const uint8_t *    scales;
-        const sycl::half * dm;
-    };
-
-    static ESIMD_INLINE ptrs make_ptrs(const void * vx, size_t nb) {
-        const uint8_t * qs     = (const uint8_t *) vx;
-        const uint8_t * qh     = qs + nb * (QK_K / 2);
-        const uint8_t * scales = qh + nb * (QK_K / 8);
-        const sycl::half * dm  = (const sycl::half *) (scales + nb * K_SCALE_SIZE);
-        return { qs, qh, scales, dm };
-    }
-
-    // extract bit `bit` (0..7) of each lane and move it to bit position 4,
-    // e.g. for the 4-bit base quant's 5th (high) bit. `bit` is always a
-    // compile-time-known unrolled loop constant at call sites, so this folds
-    // to a single mask (bit==4), mask+left-shift (bit<4), or mask+right-shift
-    // (bit>4) instead of the shift+mask+shift a naive `(qh>>bit & 1) << 4` emits.
-    static ESIMD_INLINE sycl::ext::intel::esimd::simd<uint16_t, 32> extract_bit_to_pos4(
-            sycl::ext::intel::esimd::simd<uint8_t, 32> qh, int bit) {
-        using namespace sycl::ext::intel::esimd;
-        simd<uint16_t, 32> masked = convert<uint16_t>(qh & simd<uint8_t, 32>((uint8_t) (1u << bit)));
-        if (bit < 4) {
-            return masked << simd<uint16_t, 32>((uint16_t) (4 - bit));
-        } else if (bit > 4) {
-            return masked >> simd<uint16_t, 32>((uint16_t) (bit - 4));
-        }
-        return masked;
-    }
-
-    static ESIMD_INLINE void mac_pair(
-            const ptrs & pa, size_t bia,
-            const ptrs & pb, size_t bib, bool has_b,
-            sycl::ext::intel::esimd::simd<float, 256> & y_vec,
-            sycl::ext::intel::esimd::simd<float, 32> & acc_a,
-            sycl::ext::intel::esimd::simd<float, 32> & acc_b) {
-        using namespace sycl::ext::intel::esimd;
-
-        simd<uint8_t, 128> qs_a     = block_load<uint8_t, 128>(pa.qs + bia * (QK_K / 2));
-        simd<uint8_t, 128> qs_b     = 0;
-        simd<uint8_t, 32>  qh_a     = block_load<uint8_t, 32>(pa.qh + bia * (QK_K / 8));
-        simd<uint8_t, 32>  qh_b     = 0;
-        simd<uint8_t, 12>  scales_a = block_load<uint8_t, 12>(pa.scales + bia * K_SCALE_SIZE);
-        simd<uint8_t, 12>  scales_b = 0;
-
-        const float dall_a = (float) pa.dm[bia * 2 + 0];
-        const float dmin_a = (float) pa.dm[bia * 2 + 1];
-        float dall_b = 0.0f;
-        float dmin_b = 0.0f;
-        if (has_b) {
-            qs_b     = block_load<uint8_t, 128>(pb.qs + bib * (QK_K / 2));
-            qh_b     = block_load<uint8_t, 32>(pb.qh + bib * (QK_K / 8));
-            scales_b = block_load<uint8_t, 12>(pb.scales + bib * K_SCALE_SIZE);
-            dall_b = (float) pb.dm[bib * 2 + 0];
-            dmin_b = (float) pb.dm[bib * 2 + 1];
-        }
-
-        simd<float, 8> scale_f_a, min_f_a, scale_f_b, min_f_b;
-        unpack_scale_min_k4(scales_a, dall_a, dmin_a, scale_f_a, min_f_a);
-        unpack_scale_min_k4(scales_b, dall_b, dmin_b, scale_f_b, min_f_b);
-
-        simd<uint8_t, 128> qs_lo_a = qs_a & simd<uint8_t, 128>(0x0F);
-        simd<uint8_t, 128> qs_hi_a = qs_a >> simd<uint8_t, 128>(4);
-        simd<uint8_t, 128> qs_lo_b = qs_b & simd<uint8_t, 128>(0x0F);
-        simd<uint8_t, 128> qs_hi_b = qs_b >> simd<uint8_t, 128>(4);
-
-#pragma unroll
-        for (int sb = 0; sb < 8; sb += 2) {
-            const int q_offset = sb * 16;
-            simd<float, 32> y_lo = y_vec.select<32, 1>(sb * 32);
-            simd<float, 32> y_hi = y_vec.select<32, 1>((sb + 1) * 32);
-
-            const float scale_a_lo = scale_f_a[sb];
-            const float scale_a_hi = scale_f_a[sb + 1];
-            const float min_a_lo   = min_f_a[sb];
-            const float min_a_hi   = min_f_a[sb + 1];
-            const float scale_b_lo = scale_f_b[sb];
-            const float scale_b_hi = scale_f_b[sb + 1];
-            const float min_b_lo   = min_f_b[sb];
-            const float min_b_hi   = min_f_b[sb + 1];
-
-            simd<uint8_t, 32> qa_lo_u8 = qs_lo_a.select<32, 1>(q_offset);
-            simd<uint8_t, 32> qa_hi_u8 = qs_hi_a.select<32, 1>(q_offset);
-            simd<uint8_t, 32> qb_lo_u8 = qs_lo_b.select<32, 1>(q_offset);
-            simd<uint8_t, 32> qb_hi_u8 = qs_hi_b.select<32, 1>(q_offset);
-            simd<uint16_t, 32> qa_lo = convert<uint16_t>(qa_lo_u8);
-            simd<uint16_t, 32> qa_hi = convert<uint16_t>(qa_hi_u8);
-            simd<uint16_t, 32> qb_lo = convert<uint16_t>(qb_lo_u8);
-            simd<uint16_t, 32> qb_hi = convert<uint16_t>(qb_hi_u8);
-
-            // add the 5th bit: chunk sb uses qh bit sb, chunk sb+1 uses qh bit sb+1;
-            // qh always indexes the same 32 bytes regardless of chunk
-            qa_lo += extract_bit_to_pos4(qh_a, sb);
-            qa_hi += extract_bit_to_pos4(qh_a, sb + 1);
-            qb_lo += extract_bit_to_pos4(qh_b, sb);
-            qb_hi += extract_bit_to_pos4(qh_b, sb + 1);
-
-            simd<float, 32> deq_a_lo = convert<float>(qa_lo) * scale_a_lo + min_a_lo;
-            simd<float, 32> deq_a_hi = convert<float>(qa_hi) * scale_a_hi + min_a_hi;
-            simd<float, 32> deq_b_lo = convert<float>(qb_lo) * scale_b_lo + min_b_lo;
-            simd<float, 32> deq_b_hi = convert<float>(qb_hi) * scale_b_hi + min_b_hi;
-
-            acc_a += y_lo * deq_a_lo;
-            acc_b += y_lo * deq_b_lo;
-            acc_a += y_hi * deq_a_hi;
-            acc_b += y_hi * deq_b_hi;
-        }
-    }
-};
-
 // ---------------------------------------------------------------------------
 // Q6_K, SOA reorder layout:
 //   [ql: nb*(QK_K/2)] [qh: nb*(QK_K/4)] [scales(int8): nb*(QK_K/16)] [d: nb*half]

ggml/src/ggml-sycl/fattn-mkl.cpp

@@ -43,7 +43,7 @@ static void mkl_fa_pack_q_fp16(
     dpct::queue_ptr stream,
     sycl::half * __restrict dst,
     const float * __restrict q_src,
-    int n_queries, int DKQ,
+    int n_queries, int n_query_rows, int DKQ,
     int gqa_ratio, int kvh_base_head,
     float q_scale, int64_t q_row_stride, int64_t q_head_stride,
     int64_t wg_size) {
@@ -121,7 +121,7 @@ static void mkl_fa_online_softmax_chunk(
     float * __restrict VKQ_accum,
     int q0, int q_rows, int n_queries, int DV,
     int chunk_size, int chunk_start,
-    int kvh_head,
+    int kvh_head, int gqa_ratio,
     const sycl::half * mask_data, int64_t mask_head_stride,
     int64_t mask_row_stride, int mask_n_heads,
     float logit_softcap, int64_t wg_size) {
@@ -473,6 +473,7 @@ void ggml_sycl_flash_attn_ext_mkl(ggml_backend_sycl_context & ctx, ggml_tensor *
     MKL_ACCUM(dequant_time_us, t_deq);
 
     // --- Resolve mask pointers ---
+    const sycl::half * mask_data = nullptr;
     int64_t mask_head_stride = 0;
     int64_t mask_row_stride  = 0;
     int     mask_n_heads     = 0;
@@ -546,7 +547,7 @@ void ggml_sycl_flash_attn_ext_mkl(ggml_backend_sycl_context & ctx, ggml_tensor *
             // 1. Pack all GQA Q heads into fp16 (full n_query_rows)
             mkl_fa_pack_q_fp16(stream,
                 Q_head_f16_ptr, Q_batch,
-                n_queries, DKQ,
+                n_queries, n_query_rows, DKQ,
                 gqa_ratio, kvh_base_head,
                 q_scale, q_row_stride, q_head_stride, wg_size);
 
@@ -604,7 +605,7 @@ void ggml_sycl_flash_attn_ext_mkl(ggml_backend_sycl_context & ctx, ggml_tensor *
                             KQ_max_ptr, KQ_sum_ptr, VKQ_accum_ptr,
                             q0, q_rows, n_queries, DV,
                             this_chunk, chunk_start,
-                            kvh_base_head,
+                            kvh_base_head, gqa_ratio,
                             mask_batch, mask_head_stride,
                             mask_row_stride, mask_n_heads,
                             logit_softcap, wg_size);

ggml/src/ggml-sycl/fattn-onednn.cpp

@@ -21,6 +21,14 @@ bool ggml_sycl_flash_attn_ext_onednn_supported(const ggml_tensor * dst) {
     if (!g_ggml_sycl_fa_onednn) {
         return false;
     }
+    // Battlemage (Xe2) only, for now. On other Intel archs oneDNN's fused SDPA returns wrong results
+    // for some shapes (e.g. head_dim=64 on Arc / xe_hpg) -- an oneDNN bug tracked upstream at
+    // https://github.com/uxlfoundation/oneDNN/issues/5510. Remove this hardware limitation once that
+    // is fixed; until then non-BMG archs fall back to the existing FA kernel.
+    const gpu_arch arch = ggml_sycl_info().devices[ggml_sycl_get_device()].hw_info.arch;
+    if (arch != gpu_arch::intel_gpu_bmg_g21 && arch != gpu_arch::intel_gpu_bmg_g31) {
+        return false;
+    }
     const ggml_tensor * Q     = dst->src[0];
     const ggml_tensor * K     = dst->src[1];
     const ggml_tensor * V     = dst->src[2];
@@ -52,17 +60,6 @@ bool ggml_sycl_flash_attn_ext_onednn_supported(const ggml_tensor * dst) {
             }
         }
     }
-    // This is the improved SPDA gate. Rather than gating Alchemist GPUs from all SPDA features, we instead target only the failing shapes.
-    // If the GPU being assessed isn't in the grouping below, it has full access to all SPDA shapes. Otherwise, if it's an Alchemist GPU, we block only the shapes with head sizes that fail.
-    // It is much easier to compare the device to a small list of failing cases than to define all the passing ones.
-    const gpu_arch arch = ggml_sycl_info().devices[ggml_sycl_get_device()].hw_info.arch;
-    bool support_spda = !(arch == gpu_arch::intel_gpu_dg2_g10 ||
-                         arch == gpu_arch::intel_gpu_dg2_g11 ||
-                         arch == gpu_arch::intel_gpu_dg2_g12);
-
-    if (!support_spda && K->ne[0] == 64) {
-        return false;
-    }
     // Optional KV-length ceiling (GGML_SYCL_FA_ONEDNN_MAX_KV, 0 = unlimited). Escape hatch:
     // very long sequences make the fused SDPA slow enough to risk the xe driver watchdog on
     // some stacks; past the cap we fall back to the native FA kernel instead.

ggml/src/ggml-sycl/ggml-sycl.cpp

@@ -921,16 +921,16 @@ ggml_backend_sycl_buffer_type_alloc_buffer(ggml_backend_buffer_type_t buft,
 
     void * dev_ptr;
     if (use_usm_system) {
-        GGML_SYCL_DEBUG("[SYCL] allocating %zu Bytes with USM system\n", size);
+        GGML_SYCL_DEBUG("[SYCL] allocating %lu Bytes with USM system\n", size);
         dev_ptr = (void *)aligned_malloc_host(alignment, aligned_size);
         if (!dev_ptr) {
-            GGML_LOG_ERROR("%s: can't allocate %zu Bytes of memory on host\n", __func__, size);
+            GGML_LOG_ERROR("%s: can't allocate %lu Bytes of memory on host\n", __func__, size);
             return nullptr;
         }
     } else {
         SYCL_CHECK(CHECK_TRY_ERROR(dev_ptr = (void *)ggml_sycl_malloc_device(size, *stream)));
         if (!dev_ptr) {
-          GGML_LOG_ERROR("%s: can't allocate %zu Bytes of memory on device\n", __func__, size);
+          GGML_LOG_ERROR("%s: can't allocate %lu Bytes of memory on device\n", __func__, size);
           return nullptr;
         }
     }
@@ -1177,7 +1177,7 @@ ggml_backend_sycl_split_buffer_init_tensor(ggml_backend_buffer_t buffer,
         SYCL_CHECK(CHECK_TRY_ERROR(buf = (char *)ggml_sycl_malloc_device(size, *stream)));
         if (!buf) {
             char err_buf[1024];
-            snprintf(err_buf, 1023, "%s: can't allocate %zu Bytes of memory on device\n", __func__, size);
+            snprintf(err_buf, 1023, "%s: can't allocate %lu Bytes of memory on device\n", __func__, size);
             throw std::runtime_error(err_buf);
         }
         // set padding to 0 to avoid possible NaN values
@@ -1517,13 +1517,8 @@ static ggml_backend_buffer_t ggml_backend_sycl_host_buffer_type_alloc_buffer(ggm
 }
 
 static size_t ggml_backend_sycl_host_buffer_type_get_max_size(ggml_backend_buffer_type_t buft) {
-
-    if (g_ggml_sycl_enable_host_pinned_mem) {
-        ggml_backend_sycl_device_context * dev_ctx = (ggml_backend_sycl_device_context *) buft->device->context;
-        return dpct::dev_mgr::instance().get_device(dev_ctx->device).get_max_mem_alloc_size();
-    } else {
-        return SIZE_MAX;
-    }
+    ggml_backend_sycl_device_context * dev_ctx = (ggml_backend_sycl_device_context *) buft->device->context;
+    return dpct::dev_mgr::instance().get_device(dev_ctx->device).get_max_mem_alloc_size();
 }
 
 ggml_backend_buffer_type_t ggml_backend_sycl_host_buffer_type() {
@@ -1651,7 +1646,7 @@ struct ggml_sycl_pool_leg : public ggml_sycl_pool {
 
         SYCL_CHECK(CHECK_TRY_ERROR(ptr = (void *)ggml_sycl_malloc_device(look_ahead_size, *qptr)));
         if (!ptr) {
-            GGML_LOG_ERROR("%s: can't allocate %zu Bytes of memory on device/GPU\n", __func__, look_ahead_size);
+            GGML_LOG_ERROR("%s: can't allocate %lu Bytes of memory on device/GPU\n", __func__, look_ahead_size);
             return nullptr;
         }
 
@@ -1663,7 +1658,7 @@ struct ggml_sycl_pool_leg : public ggml_sycl_pool {
                 (uint32_t)(max_size/1024/1024), (uint32_t)(g_sycl_pool_size[id]/1024/1024), (uint32_t)(size/1024/1024));
 #endif
 
-        // GGML_SYCL_DEBUG("ggml_sycl_pool_malloc_leg look_ahead_size=%zu, return %p\n", look_ahead_size, ptr);
+        // GGML_SYCL_DEBUG("ggml_sycl_pool_malloc_leg look_ahead_size=%lu, return %p\n", look_ahead_size, ptr);
         return ptr;
     }
 
@@ -1843,7 +1838,7 @@ struct ggml_sycl_pool_host : public ggml_sycl_pool {
 
             SYCL_CHECK(CHECK_TRY_ERROR(ptr = (void *) sycl::malloc_host(size, *qptr)));
             if (!ptr) {
-                GGML_LOG_ERROR("%s: can't allocate %zu Bytes of memory on host\n", __func__, size);
+                GGML_LOG_ERROR("%s: can't allocate %lu Bytes of memory on host\n", __func__, size);
                 return nullptr;
             }
             pool_size += size;
@@ -2779,9 +2774,9 @@ inline void ggml_sycl_op_mul_mat_sycl(
         const float * src1_ddf1_i = src1->type == GGML_TYPE_F32 ? (const float *) src1_ddf_i : src1_ddq_as_f32.get();
 
         {
-#if GGML_SYCL_DNNL
             const int64_t gemm_flops = (int64_t)row_diff * src1_ncols * ne10;
             const bool use_mkl_direct = gemm_flops < 256 * 256 * 256;
+#if GGML_SYCL_DNNL
             if (g_ggml_sycl_enable_dnn && !use_mkl_direct) {
                 DnnlGemmWrapper::row_gemm(ctx, row_diff, src1_ncols, ne10, src0_ddf_i,
                                           DnnlGemmWrapper::to_dt<float>(), src1_ddf1_i, DnnlGemmWrapper::to_dt<float>(),
@@ -3518,9 +3513,7 @@ static void ggml_sycl_mul_mat_batched_sycl(ggml_backend_sycl_context & ctx, cons
     float *            dst_ddf  = static_cast<float *>(dst->data);
 
     const sycl::half * src1_f16       = static_cast<const sycl::half *>(src1->data);
-#if GGML_SYCL_DNNL
     const size_t       type_size_src0 = ggml_type_size(src0->type);
-#endif
     const size_t       type_size_src1 = ggml_type_size(src1->type);
 
     bool is_src0_cont_2 = ggml_is_contiguous_2(src0);
@@ -3537,7 +3530,6 @@ static void ggml_sycl_mul_mat_batched_sycl(ggml_backend_sycl_context & ctx, cons
         scope_op_debug_print    scope_dbg_print(__func__, "/to_fp16_nc_sycl", dst, /*num_src=*/2,
                                                 " : converting src1 to fp16");
 
-#if GGML_SYCL_DNNL
         // iterate tensor dims and find the slowest moving dim and stride
         int last_dim=0;
         int last_str=0;
@@ -3557,6 +3549,7 @@ static void ggml_sycl_mul_mat_batched_sycl(ggml_backend_sycl_context & ctx, cons
             }
 
         }
+#if GGML_SYCL_DNNL
         // oneDNN handles strided data and does not need overhead of ggml_get_to_fp16_nc_sycl
         const int64_t ne_src1 = src1->nb[last_str] * src1->ne[last_dim] / type_size_src1;
         src1_f16_alloc.alloc(ne_src1);
@@ -3796,7 +3789,6 @@ inline bool ggml_sycl_supports_reorder_mmvq(enum ggml_type type) {
         case GGML_TYPE_Q1_0:
         case GGML_TYPE_Q4_0:
         case GGML_TYPE_Q8_0:
-        case GGML_TYPE_Q2_K:
         case GGML_TYPE_Q3_K:
         case GGML_TYPE_Q4_K:
         case GGML_TYPE_Q5_K:
@@ -3810,10 +3802,8 @@ inline bool ggml_sycl_supports_reorder_mmvq(enum ggml_type type) {
 static bool ggml_sycl_supports_reorder_esimd(enum ggml_type type) {
 #ifdef GGML_SYCL_DMMV_HAS_ESIMD
     switch (type) {
-        case GGML_TYPE_Q2_K:
         case GGML_TYPE_Q3_K:
         case GGML_TYPE_Q4_K:
-        case GGML_TYPE_Q5_K:
         case GGML_TYPE_Q6_K:
             return true;
         default:
@@ -6252,6 +6242,8 @@ static bool do_ggml_backend_sycl_device_supports_op(ggml_backend_dev_t dev, cons
         }
         case GGML_OP_ROPE:
         case GGML_OP_ROPE_BACK:
+            // FIXME: support ggml_rope_set_offset
+            return ((const int32_t *) op->op_params)[15] == 0;
         case GGML_OP_IM2COL:
         case GGML_OP_IM2COL_3D:
         case GGML_OP_UPSCALE:

ggml/src/ggml-sycl/im2col.cpp

@@ -85,7 +85,7 @@ static void im2col_sycl(const float *   x,
     */
     stream->parallel_for(sycl::nd_range<3>(block_nums * sycl::range<3>(1, 1, MIN(IC_KH_KW, SYCL_IM2COL_BLOCK_SIZE)),
                                            sycl::range<3>(1, 1, MIN(IC_KH_KW, SYCL_IM2COL_BLOCK_SIZE))),
-                         [=](sycl::nd_item<3>) {
+                         [=](sycl::nd_item<3> item_ct1) {
                              im2col_kernel(x, dst, IC, IW, IH, OH, OW, KW, KH, IC_IH_IW, IH_IW, N_OH, KH_KW, IC_KH_KW,
                                            s0, s1, p0, p1, d0, d1);
                          });
@@ -271,7 +271,7 @@ static void im2col_3d_sycl(const float *   src,
     */
     stream->parallel_for(sycl::nd_range<3>(block_nums * sycl::range<3>(1, 1, MIN(IC_KD_KH_KW, SYCL_IM2COL_BLOCK_SIZE)),
                                            sycl::range<3>(1, 1, MIN(IC_KD_KH_KW, SYCL_IM2COL_BLOCK_SIZE))),
-                         [=](sycl::nd_item<3>) {
+                         [=](sycl::nd_item<3> item_ct1) {
                              im2col_3d_kernel(src, dst, N, IC, ID, IH, IW, OC, KD, KH, KW, OD, OH, OW, OH_OW, KD_KH_KW,
                                               ID_IH_IW, KH_KW, IH_IW, IC_ID_IH_IW, IC_KD_KH_KW, OW_KD_KH_KW,
                                               OD_OH_OW_IC_KD_KH_KW, OH_OW_IC_KD_KH_KW, OW_IC_KD_KH_KW, N_OD_OH, OD_OH,

ggml/src/ggml-sycl/mmvq.cpp

@@ -1401,65 +1401,6 @@ static void mul_mat_vec_q2_K_q8_1_sycl_switch_ncols(
     }
 }
 
-static void reorder_mul_mat_vec_q2_k_q8_1_sycl(const void * vx, const void * vy, float * dst, const int ncols,
-                                               const int nrows, dpct::queue_ptr stream) {
-    GGML_ASSERT(ncols % QK_K == 0);
-
-    // Round up to a whole number of subgroup-sized workgroups; out-of-range rows are skipped inside the kernel.
-    constexpr size_t num_subgroups = WARP_SIZE;
-    const int block_num_y = ceil_div(nrows, GGML_SYCL_MMV_Y * (int) num_subgroups);
-    const sycl::range<3> block_nums(1, 1, block_num_y);
-    const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, num_subgroups * WARP_SIZE);
-
-    stream->submit([&](sycl::handler & cgh) {
-        cgh.parallel_for(sycl::nd_range<3>(block_nums * block_dims, block_dims),
-                         [=](sycl::nd_item<3> nd_item) [[sycl::reqd_sub_group_size(WARP_SIZE)]] {
-                             mul_mat_vec_q_reorder<reorder_vec_dot_q_sycl<GGML_TYPE_Q2_K>>(vx, vy, dst, ncols, nrows,
-                                                                                           nd_item);
-                         });
-    });
-}
-
-template <int ncols_dst>
-static void reorder_mul_mat_vec_q2_k_q8_1_sycl_ncols(
-        const void * vx, const void * vy, float * dst,
-        const int ncols, const int nrows,
-        const int stride_col_y_bytes, const int stride_col_dst,
-        dpct::queue_ptr stream) {
-    GGML_ASSERT(ncols % QK_K == 0);
-    constexpr size_t num_subgroups = WARP_SIZE;
-    const int block_num_y = ceil_div(nrows, GGML_SYCL_MMV_Y * (int) num_subgroups);
-    const sycl::range<3> block_nums(1, 1, block_num_y);
-    const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, num_subgroups * WARP_SIZE);
-
-    stream->submit([&](sycl::handler & cgh) {
-        cgh.parallel_for(sycl::nd_range<3>(block_nums * block_dims, block_dims),
-                         [=](sycl::nd_item<3> nd_item) [[sycl::reqd_sub_group_size(WARP_SIZE)]] {
-                             mul_mat_vec_q_reorder_ncols<reorder_vec_dot_q_sycl<GGML_TYPE_Q2_K>, ncols_dst>(
-                                 vx, /*vgate=*/ nullptr, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst,
-                                 /*glu_op=*/ GGML_GLU_OP_SWIGLU, nd_item);
-                         });
-    });
-}
-
-static void reorder_mul_mat_vec_q2_k_q8_1_sycl_switch_ncols(
-        const void * vx, const void * vy, float * dst,
-        const int ncols, const int nrows, const int ncols_dst,
-        const int stride_col_y_bytes, const int stride_col_dst,
-        dpct::queue_ptr stream) {
-    switch (ncols_dst) {
-        case 1: reorder_mul_mat_vec_q2_k_q8_1_sycl(vx, vy, dst, ncols, nrows, stream); break;
-        case 2: reorder_mul_mat_vec_q2_k_q8_1_sycl_ncols<2>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break;
-        case 3: reorder_mul_mat_vec_q2_k_q8_1_sycl_ncols<3>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break;
-        case 4: reorder_mul_mat_vec_q2_k_q8_1_sycl_ncols<4>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break;
-        case 5: reorder_mul_mat_vec_q2_k_q8_1_sycl_ncols<5>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break;
-        case 6: reorder_mul_mat_vec_q2_k_q8_1_sycl_ncols<6>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break;
-        case 7: reorder_mul_mat_vec_q2_k_q8_1_sycl_ncols<7>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break;
-        case 8: reorder_mul_mat_vec_q2_k_q8_1_sycl_ncols<8>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break;
-        default: GGML_ABORT("unsupported ncols_dst=%d for Q2_K reorder multi-col MMVQ", ncols_dst);
-    }
-}
-
 static void mul_mat_vec_q3_K_q8_1_sycl(const void *vx, const void *vy,
                                        float *dst, const int ncols,
                                        const int nrows,
@@ -2356,21 +2297,7 @@ void ggml_sycl_op_mul_mat_vec_q(ggml_backend_sycl_context & ctx, const ggml_tens
                 }
                 break;
             case GGML_TYPE_Q2_K:
-                if ((ggml_tensor_extra_gpu *) dst->src[0]->extra &&
-                    ((ggml_tensor_extra_gpu *) dst->src[0]->extra)->optimized_feature.reorder) {
-                    if (i == 0 && src1_ncols > 1 && src1_ncols <= 8) {
-                        const int stride_col_y_bytes = src1_padded_col_size * q8_1_ts / q8_1_bs;
-                        const int stride_col_dst     = dst->ne[0];
-                        GGML_SYCL_DEBUG("Calling reorder_mul_mat_vec_q2_k_q8_1_sycl_switch_ncols ncols=%d\n", (int)src1_ncols);
-                        reorder_mul_mat_vec_q2_k_q8_1_sycl_switch_ncols(
-                            src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff,
-                            src1_ncols, stride_col_y_bytes, stride_col_dst, stream);
-                        return;
-                    } else {
-                        GGML_SYCL_DEBUG("Calling reorder_mul_mat_vec_q2_k_q8_1_sycl\n");
-                        reorder_mul_mat_vec_q2_k_q8_1_sycl(src0_dd_i, src1_ddq_i_bs, dst_dd_i_bs, ne00, row_diff, stream);
-                    }
-                } else if (i == 0 && src1_ncols > 1 && src1_ncols <= 8) {
+                if (i == 0 && src1_ncols > 1 && src1_ncols <= 8) {
                     const int stride_col_y   = src1_padded_col_size / QK8_1;
                     const int stride_col_dst = dst->ne[0];
                     GGML_SYCL_DEBUG("Calling mul_mat_vec_q2_K_q8_1_sycl_switch_ncols ncols=%d\n", (int)src1_ncols);
@@ -2379,7 +2306,6 @@ void ggml_sycl_op_mul_mat_vec_q(ggml_backend_sycl_context & ctx, const ggml_tens
                         src1_ncols, stride_col_y, stride_col_dst, stream);
                     return;
                 } else if (i == 0 || src1_ncols == 1) {
-                    GGML_SYCL_DEBUG("Calling mul_mat_vec_q2_K_q8_1_sycl\n");
                     mul_mat_vec_q2_K_q8_1_sycl(src0_dd_i, src1_ddq_i_bs, dst_dd_i_bs, ne00, row_diff, stream);
                 }
                 break;

ggml/src/ggml-sycl/norm.cpp

@@ -7,6 +7,9 @@ static void norm_f32(const float* x, float* dst, const int ncols,
     const int64_t dst_stride_col, const int64_t dst_stride_row, const int64_t dst_stride_channel, const int64_t dst_stride_sample,
     const float eps, const sycl::nd_item<3>& item_ct1, sycl::float2* s_sum, int block_size) {
 
+    const int nrows = item_ct1.get_group_range(2);
+    const int nchannels = item_ct1.get_group_range(1);
+
     const int nthreads = item_ct1.get_local_range(2);
     const int sample  = item_ct1.get_group(0);
     const int channel = item_ct1.get_group(1);
@@ -152,6 +155,9 @@ static void rms_norm_f32(const float* x, float* dst, const int ncols,
     const float* mul = nullptr, const int64_t mul_stride_row = 0, const int64_t mul_stride_channel = 0,
     const int64_t mul_stride_sample = 0, const int mul_nrows = 0, const int mul_nchannels = 0, const int mul_nsamples = 0) {
 
+    const int nrows = item_ct1.get_group_range(2);
+    const int nchannels = item_ct1.get_group_range(1);
+
     const int sample  = item_ct1.get_group(0);
     const int channel = item_ct1.get_group(1);
     const int row     = item_ct1.get_group(2);
@@ -219,6 +225,8 @@ static void l2_norm_f32(const float * x, float * dst, const int ncols,
     const int64_t src_stride_sample, const int64_t dst_stride_col, const int64_t dst_stride_row,
     const int64_t dst_stride_channel, const int64_t dst_stride_sample, const float eps,
     const sycl::nd_item<3>& item_ct1, float* s_sum, const int block_size) {
+    const int nrows     = item_ct1.get_group_range(2);
+    const int nchannels = item_ct1.get_group_range(1);
 
     const int row     = item_ct1.get_group(2);
     const int channel = item_ct1.get_group(1);

ggml/src/ggml-sycl/quants.hpp

@@ -58,29 +58,6 @@ template <> struct block_q_t<GGML_TYPE_Q4_0> {
     static constexpr int block_to_q8_1_ratio() { return traits::qk / QK8_1; }
 };
 
-template <> struct block_q_t<GGML_TYPE_Q2_K> {
-    struct traits {
-        static constexpr uint32_t qk       = QK_K;
-        static constexpr uint32_t qi       = QI2_K;
-        static constexpr uint32_t qr       = QR2_K;
-        static constexpr uint32_t vdr_mmvq = 1;
-    };
-
-    // Reordered layout: [qs (QK_K/4 per block)] [scales (QK_K/16 per block)] [dm]
-    static constexpr std::pair<int, int> get_block_offset(const int block_index, const int /* n_blocks */) {
-        return { block_index * (QK_K / 4), 0 };
-    }
-
-    static constexpr std::pair<int, int> get_d_offset(int nrows, int ncols, const int block_index) {
-        auto nblocks        = (nrows * (ncols / QK_K));
-        auto total_qs_bytes = nblocks * (QK_K / 4);
-        return { total_qs_bytes + block_index * (QK_K / 16),
-                 total_qs_bytes + nblocks * (QK_K / 16) + block_index * sizeof(ggml_half2) };
-    }
-
-    static constexpr int block_to_q8_1_ratio() { return traits::qk / QK8_1; }
-};
-
 template <> struct block_q_t<GGML_TYPE_Q3_K> {
     struct traits {
         static constexpr uint32_t qk       = QK_K;

ggml/src/ggml-sycl/rope.cpp

@@ -41,7 +41,7 @@ template <bool forward, bool has_ff, typename T, typename D>
 static void rope_norm(const T *x, D *dst, const int ne00, const int ne01,
                       const int ne02, const int s01, const int s02,
                       const int s03, const int s1, const int s2, const int s3,
-                      const int n_dims, const int n_offs, const int32_t *pos,
+                      const int n_dims, const int32_t *pos,
                       const float freq_scale, const float ext_factor,
                       const float attn_factor, const rope_corr_dims corr_dims,
                       const float theta_scale, const float *freq_factors,
@@ -78,21 +78,19 @@ static void rope_norm(const T *x, D *dst, const int ne00, const int ne01,
             ggml_sycl_memcpy_1<4>(dst + idst, &v);
         }
     };
-    if (i0 < n_offs || i0 >= n_offs + n_dims) {
+    if (i0 >= n_dims) {
         store_coaelsced(x[ix + 0], x[ix + 1]);
         return;
     }
 
-    const int iw = i0 - n_offs; // relative idx
+    const float theta_base = pos[i2] * dpct::pow(theta_scale, i0 / 2.0f);
 
-    const float theta_base = pos[i2] * dpct::pow(theta_scale, iw / 2.0f);
-
-    const float freq_factor = has_ff ? freq_factors[iw / 2] : 1.0f;
+    const float freq_factor = has_ff ? freq_factors[i0 / 2] : 1.0f;
 
     float cos_theta;
     float sin_theta;
 
-    rope_yarn<forward>(theta_base / freq_factor, freq_scale, corr_dims, iw,
+    rope_yarn<forward>(theta_base / freq_factor, freq_scale, corr_dims, i0,
                        ext_factor, attn_factor, cos_theta, sin_theta);
 
     const float x0 = x[ix + 0];
@@ -106,7 +104,7 @@ template <bool forward, bool has_ff, typename T, typename D>
 static void rope_neox(const T *x, D *dst, const int ne00, const int ne01,
                       const int ne02, const int s01, const int s02,
                       const int s03, const int s1, const int s2, const int s3,
-                      const int n_dims, const int n_offs, const int32_t *pos,
+                      const int n_dims, const int32_t *pos,
                       const float freq_scale, const float ext_factor,
                       const float attn_factor, const rope_corr_dims corr_dims,
                       const float theta_scale, const float *freq_factors,
@@ -134,38 +132,35 @@ static void rope_neox(const T *x, D *dst, const int ne00, const int ne01,
         idst += row_indices[i2] * set_rows_stride;
     }
 
-    if (i0 < n_offs || i0 >= n_offs + n_dims) {
+    if (i0 >= n_dims) {
         dst[idst + i0 / 2 + 0] = ggml_sycl_cast<D>(x[ix + i0 / 2 + 0]);
         dst[idst + i0 / 2 + 1] = ggml_sycl_cast<D>(x[ix + i0 / 2 + 1]);
 
         return;
     }
 
-    const int iw = i0 - n_offs; // relative idx
+    const float theta_base = pos[i2] * dpct::pow(theta_scale, i0 / 2.0f);
 
-    const float theta_base = pos[i2] * dpct::pow(theta_scale, iw / 2.0f);
-
-    const float freq_factor = has_ff ? freq_factors[iw / 2] : 1.0f;
+    const float freq_factor = has_ff ? freq_factors[i0 / 2] : 1.0f;
 
     float cos_theta;
     float sin_theta;
 
-    rope_yarn<forward>(theta_base / freq_factor, freq_scale, corr_dims, iw,
+    rope_yarn<forward>(theta_base / freq_factor, freq_scale, corr_dims, i0,
                        ext_factor, attn_factor, cos_theta, sin_theta);
 
-    // idst/ix point at channel i0/2; the first channel of the rotated pair is n_offs + iw/2 = i0/2 + n_offs/2
-    const float x0 = x[ix + n_offs / 2 + 0];
-    const float x1 = x[ix + n_offs / 2 + n_dims / 2];
+    const float x0 = x[ix + 0];
+    const float x1 = x[ix + n_dims / 2];
 
-    dst[idst + n_offs / 2 + 0] = ggml_sycl_cast<D>(x0 * cos_theta - x1 * sin_theta);
-    dst[idst + n_offs / 2 + n_dims / 2] = ggml_sycl_cast<D>(x0 * sin_theta + x1 * cos_theta);
+    dst[idst + 0] = ggml_sycl_cast<D>(x0 * cos_theta - x1 * sin_theta);
+    dst[idst + n_dims / 2] = ggml_sycl_cast<D>(x0 * sin_theta + x1 * cos_theta);
 }
 
 template <bool forward, bool has_ff, typename T>
 static void rope_multi(const T *x, T *dst, const int ne00, const int ne01,
                        const int ne02, const int s01, const int s02,
                        const int s03, const int s1, const int s2, const int s3,
-                       const int n_dims, const int n_offs, const int32_t *pos,
+                       const int n_dims, const int32_t *pos,
                        const float freq_scale, const float ext_factor,
                        const float attn_factor, const rope_corr_dims corr_dims,
                        const float theta_scale, const float *freq_factors,
@@ -188,57 +183,54 @@ static void rope_multi(const T *x, T *dst, const int ne00, const int ne01,
     int idst = i0 / 2 + i1 * s1 + i2 * s2 + i3 * s3;
     const int ix = i0 / 2 + i1 * s01 + i2 * s02 + i3 * s03;
 
-    if (i0 < n_offs || i0 >= n_offs + n_dims) {
+    if (i0 >= n_dims) {
         dst[idst + i0 / 2 + 0] = x[ix + i0 / 2 + 0];
         dst[idst + i0 / 2 + 1] = x[ix + i0 / 2 + 1];
 
         return;
     }
 
-    const int iw = i0 - n_offs; // relative idx
-
     const int sect_dims =
         sections.v[0] + sections.v[1] + sections.v[2] + sections.v[3];
     const int sec_w = sections.v[1] + sections.v[0];
-    const int sector = (iw / 2) % sect_dims;
+    const int sector = (i0 / 2) % sect_dims;
 
     float theta_base = 0.0;
     if (is_imrope) {
         if (sector % 3 == 1 && sector < 3 * sections.v[1]) { // h
-            theta_base = pos[i2 + ne02 * 1] * dpct::pow(theta_scale, iw / 2.0f);
+            theta_base = pos[i2 + ne02 * 1] * dpct::pow(theta_scale, i0 / 2.0f);
         } else if (sector % 3 == 2 && sector < 3 * sections.v[2]) { // w
-            theta_base = pos[i2 + ne02 * 2] * dpct::pow(theta_scale, iw / 2.0f);
+            theta_base = pos[i2 + ne02 * 2] * dpct::pow(theta_scale, i0 / 2.0f);
         } else if (sector % 3 == 0 && sector < 3 * sections.v[0]) { // t
-            theta_base = pos[i2] * dpct::pow(theta_scale, iw / 2.0f);
+            theta_base = pos[i2] * dpct::pow(theta_scale, i0 / 2.0f);
         } else {
-            theta_base = pos[i2 + ne02 * 3] * dpct::pow(theta_scale, iw / 2.0f);
+            theta_base = pos[i2 + ne02 * 3] * dpct::pow(theta_scale, i0 / 2.0f);
         }
     } else {
         if (sector < sections.v[0]) {
-            theta_base = pos[i2] * dpct::pow(theta_scale, iw / 2.0f);
+            theta_base = pos[i2] * dpct::pow(theta_scale, i0 / 2.0f);
         } else if (sector >= sections.v[0] && sector < sec_w) {
-            theta_base = pos[i2 + ne02 * 1] * dpct::pow(theta_scale, iw / 2.0f);
+            theta_base = pos[i2 + ne02 * 1] * dpct::pow(theta_scale, i0 / 2.0f);
         } else if (sector >= sec_w && sector < sec_w + sections.v[2]) {
-            theta_base = pos[i2 + ne02 * 2] * dpct::pow(theta_scale, iw / 2.0f);
+            theta_base = pos[i2 + ne02 * 2] * dpct::pow(theta_scale, i0 / 2.0f);
         } else if (sector >= sec_w + sections.v[2]) {
-            theta_base = pos[i2 + ne02 * 3] * dpct::pow(theta_scale, iw / 2.0f);
+            theta_base = pos[i2 + ne02 * 3] * dpct::pow(theta_scale, i0 / 2.0f);
         }
     }
 
-    const float freq_factor = has_ff ? freq_factors[iw / 2] : 1.0f;
+    const float freq_factor = has_ff ? freq_factors[i0 / 2] : 1.0f;
 
     float cos_theta;
     float sin_theta;
 
-    rope_yarn<forward>(theta_base / freq_factor, freq_scale, corr_dims, iw,
+    rope_yarn<forward>(theta_base / freq_factor, freq_scale, corr_dims, i0,
                        ext_factor, attn_factor, cos_theta, sin_theta);
 
-    // idst/ix point at channel i0/2; the first channel of the rotated pair is n_offs + iw/2 = i0/2 + n_offs/2
-    const float x0 = x[ix + n_offs / 2 + 0];
-    const float x1 = x[ix + n_offs / 2 + n_dims / 2];
+    const float x0 = x[ix + 0];
+    const float x1 = x[ix + n_dims / 2];
 
-    dst[idst + n_offs / 2 + 0] = x0 * cos_theta - x1 * sin_theta;
-    dst[idst + n_offs / 2 + n_dims / 2] = x0 * sin_theta + x1 * cos_theta;
+    dst[idst + 0] = x0 * cos_theta - x1 * sin_theta;
+    dst[idst + n_dims / 2] = x0 * sin_theta + x1 * cos_theta;
 }
 
 template <bool forward, bool has_ff, typename T>
@@ -301,7 +293,7 @@ static void
 rope_norm_sycl(const T *x, D *dst, const int ne00, const int ne01,
                const int ne02, const int s01, const int s02, const int s03,
                const int s1, const int s2, const int s3, const int n_dims,
-               const int n_offs, const int nr, const int32_t *pos, const float freq_scale,
+               const int nr, const int32_t *pos, const float freq_scale,
                const float freq_base, const float ext_factor,
                const float attn_factor, const rope_corr_dims corr_dims,
                const float *freq_factors, const int64_t *row_indices,
@@ -321,7 +313,7 @@ rope_norm_sycl(const T *x, D *dst, const int ne00, const int ne01,
                 GGML_UNUSED(item_ct1);
                 rope_norm<forward, false>(
                     x, dst, ne00, ne01, ne02, s01, s02, s03, s1, s2, s3, n_dims,
-                    n_offs, pos, freq_scale, ext_factor, attn_factor, corr_dims,
+                    pos, freq_scale, ext_factor, attn_factor, corr_dims,
                     theta_scale, freq_factors, row_indices, set_rows_stride);
             });
     } else {
@@ -331,7 +323,7 @@ rope_norm_sycl(const T *x, D *dst, const int ne00, const int ne01,
                 GGML_UNUSED(item_ct1);
                 rope_norm<forward, true>(
                     x, dst, ne00, ne01, ne02, s01, s02, s03, s1, s2, s3, n_dims,
-                    n_offs, pos, freq_scale, ext_factor, attn_factor, corr_dims,
+                    pos, freq_scale, ext_factor, attn_factor, corr_dims,
                     theta_scale, freq_factors, row_indices, set_rows_stride);
             });
     }
@@ -342,7 +334,7 @@ static void
 rope_neox_sycl(const T *x, D *dst, const int ne00, const int ne01,
                const int ne02, const int s01, const int s02, const int s03,
                const int s1, const int s2, const int s3, const int n_dims,
-               const int n_offs, const int nr, const int32_t *pos, const float freq_scale,
+               const int nr, const int32_t *pos, const float freq_scale,
                const float freq_base, const float ext_factor,
                const float attn_factor, const rope_corr_dims corr_dims,
                const float *freq_factors, const int64_t *row_indices,
@@ -362,7 +354,7 @@ rope_neox_sycl(const T *x, D *dst, const int ne00, const int ne01,
                 GGML_UNUSED(item_ct1);
                 rope_neox<forward, false>(
                     x, dst, ne00, ne01, ne02, s01, s02, s03, s1, s2, s3, n_dims,
-                    n_offs, pos, freq_scale, ext_factor, attn_factor, corr_dims,
+                    pos, freq_scale, ext_factor, attn_factor, corr_dims,
                     theta_scale, freq_factors, row_indices, set_rows_stride);
             });
     } else {
@@ -372,7 +364,7 @@ rope_neox_sycl(const T *x, D *dst, const int ne00, const int ne01,
                 GGML_UNUSED(item_ct1);
                 rope_neox<forward, true>(
                     x, dst, ne00, ne01, ne02, s01, s02, s03, s1, s2, s3, n_dims,
-                    n_offs, pos, freq_scale, ext_factor, attn_factor, corr_dims,
+                    pos, freq_scale, ext_factor, attn_factor, corr_dims,
                     theta_scale, freq_factors, row_indices, set_rows_stride);
             });
     }
@@ -383,7 +375,7 @@ static void
 rope_multi_sycl(const T *x, T *dst, const int ne00, const int ne01,
                 const int ne02, const int s01, const int s02, const int s03,
                 const int s1, const int s2, const int s3, const int n_dims,
-                const int n_offs, const int nr, const int32_t *pos, const float freq_scale,
+                const int nr, const int32_t *pos, const float freq_scale,
                 const float freq_base, const float ext_factor,
                 const float attn_factor, const rope_corr_dims corr_dims,
                 const float *freq_factors, const mrope_sections sections,
@@ -403,7 +395,7 @@ rope_multi_sycl(const T *x, T *dst, const int ne00, const int ne01,
                 GGML_UNUSED(item_ct1);
                 rope_multi<forward, false, T>(
                     x, dst, ne00, ne01, ne02, s01, s02, s03, s1, s2, s3, n_dims,
-                    n_offs, pos, freq_scale, ext_factor, attn_factor, corr_dims,
+                    pos, freq_scale, ext_factor, attn_factor, corr_dims,
                     theta_scale, freq_factors, sections, is_imrope);
             });
     } else {
@@ -413,7 +405,7 @@ rope_multi_sycl(const T *x, T *dst, const int ne00, const int ne01,
                 GGML_UNUSED(item_ct1);
                 rope_multi<forward, true, T>(
                     x, dst, ne00, ne01, ne02, s01, s02, s03, s1, s2, s3, n_dims,
-                    n_offs, pos, freq_scale, ext_factor, attn_factor, corr_dims,
+                    pos, freq_scale, ext_factor, attn_factor, corr_dims,
                     theta_scale, freq_factors, sections, is_imrope);
             });
     }
@@ -505,7 +497,6 @@ void ggml_sycl_op_rope_impl(ggml_backend_sycl_context &ctx, ggml_tensor *dst,
     const int n_dims = ((int32_t *)dst->op_params)[1];
     const int mode = ((int32_t *)dst->op_params)[2];
     const int n_ctx_orig = ((int32_t *)dst->op_params)[4];
-    const int n_offs = ((int32_t *)dst->op_params)[15];
     mrope_sections sections;
 
     float freq_base;
@@ -535,7 +526,6 @@ void ggml_sycl_op_rope_impl(ggml_backend_sycl_context &ctx, ggml_tensor *dst,
 
     if (is_vision) {
         GGML_ASSERT(n_dims == ne00 / 2);
-        GGML_ASSERT(n_offs == 0); // offset not supported for vision, as the rotated pairs span the whole row
     }
 
     const int32_t *pos = (const int32_t *)src1_d;
@@ -555,19 +545,19 @@ void ggml_sycl_op_rope_impl(ggml_backend_sycl_context &ctx, ggml_tensor *dst,
         if (src0->type == GGML_TYPE_F32 && dst_type == GGML_TYPE_F32) {
             rope_neox_sycl<forward, float, float>(
                 (const float *)src0_d, (float *)dst_d, ne00, ne01, ne02, s01,
-                s02, s03, s1, s2, s3, n_dims, n_offs, nr, pos, freq_scale, freq_base,
+                s02, s03, s1, s2, s3, n_dims, nr, pos, freq_scale, freq_base,
                 ext_factor, attn_factor, corr_dims, freq_factors, row_indices,
                 set_rows_stride, stream);
         } else if (src0->type == GGML_TYPE_F32 && dst_type == GGML_TYPE_F16) {
             rope_neox_sycl<forward, float, sycl::half>(
                 (const float *)src0_d, (sycl::half *)dst_d, ne00, ne01, ne02,
-                s01, s02, s03, s1, s2, s3, n_dims, n_offs, nr, pos, freq_scale,
+                s01, s02, s03, s1, s2, s3, n_dims, nr, pos, freq_scale,
                 freq_base, ext_factor, attn_factor, corr_dims, freq_factors,
                 row_indices, set_rows_stride, stream);
         } else if (src0->type == GGML_TYPE_F16 && dst_type == GGML_TYPE_F16) {
             rope_neox_sycl<forward, sycl::half, sycl::half>(
                 (const sycl::half *)src0_d, (sycl::half *)dst_d, ne00, ne01,
-                ne02, s01, s02, s03, s1, s2, s3, n_dims, n_offs, nr, pos, freq_scale,
+                ne02, s01, s02, s03, s1, s2, s3, n_dims, nr, pos, freq_scale,
                 freq_base, ext_factor, attn_factor, corr_dims, freq_factors,
                 row_indices, set_rows_stride, stream);
         } else {
@@ -578,13 +568,13 @@ void ggml_sycl_op_rope_impl(ggml_backend_sycl_context &ctx, ggml_tensor *dst,
         if (src0->type == GGML_TYPE_F32) {
             rope_multi_sycl<forward>((const float *)src0_d, (float *)dst_d,
                                      ne00, ne01, ne02, s01, s02, s03, s1, s2,
-                                     s3, n_dims, n_offs, nr, pos, freq_scale, freq_base,
+                                     s3, n_dims, nr, pos, freq_scale, freq_base,
                                      ext_factor, attn_factor, corr_dims,
                                      freq_factors, sections, is_imrope, stream);
         } else if (src0->type == GGML_TYPE_F16) {
             rope_multi_sycl<forward>(
                 (const sycl::half *)src0_d, (sycl::half *)dst_d, ne00, ne01,
-                ne02, s01, s02, s03, s1, s2, s3, n_dims, n_offs, nr, pos, freq_scale,
+                ne02, s01, s02, s03, s1, s2, s3, n_dims, nr, pos, freq_scale,
                 freq_base, ext_factor, attn_factor, corr_dims, freq_factors,
                 sections, is_imrope, stream);
         } else {
@@ -612,19 +602,19 @@ void ggml_sycl_op_rope_impl(ggml_backend_sycl_context &ctx, ggml_tensor *dst,
         if (src0->type == GGML_TYPE_F32 && dst_type == GGML_TYPE_F32) {
             rope_norm_sycl<forward, float, float>(
                 (const float *)src0_d, (float *)dst_d, ne00, ne01, ne02, s01,
-                s02, s03, s1, s2, s3, n_dims, n_offs, nr, pos, freq_scale, freq_base,
+                s02, s03, s1, s2, s3, n_dims, nr, pos, freq_scale, freq_base,
                 ext_factor, attn_factor, corr_dims, freq_factors, row_indices,
                 set_rows_stride, stream);
         } else if (src0->type == GGML_TYPE_F32 && dst_type == GGML_TYPE_F16) {
             rope_norm_sycl<forward, float, sycl::half>(
                 (const float *)src0_d, (sycl::half *)dst_d, ne00, ne01, ne02,
-                s01, s02, s03, s1, s2, s3, n_dims, n_offs, nr, pos, freq_scale,
+                s01, s02, s03, s1, s2, s3, n_dims, nr, pos, freq_scale,
                 freq_base, ext_factor, attn_factor, corr_dims, freq_factors,
                 row_indices, set_rows_stride, stream);
         } else if (src0->type == GGML_TYPE_F16 && dst_type == GGML_TYPE_F16) {
             rope_norm_sycl<forward, sycl::half, sycl::half>(
                 (const sycl::half *)src0_d, (sycl::half *)dst_d, ne00, ne01,
-                ne02, s01, s02, s03, s1, s2, s3, n_dims, n_offs, nr, pos, freq_scale,
+                ne02, s01, s02, s03, s1, s2, s3, n_dims, nr, pos, freq_scale,
                 freq_base, ext_factor, attn_factor, corr_dims, freq_factors,
                 row_indices, set_rows_stride, stream);
         } else {

ggml/src/ggml-sycl/set_rows.cpp

@@ -291,7 +291,7 @@ static void set_rows_sycl(
 
     stream->parallel_for(
         sycl::nd_range<1>(grid_size * block_size, block_size),
-        [=](sycl::nd_item<1> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] {
+        [=](sycl::nd_item<1> item_ct1) [[intel::reqd_sub_group_size(WARP_SIZE)]] {
             k_set_rows<TIn, TIdx, TOut>(
                 src0_d, src1_d, dst_d,
                 ne00, ne01, ne02,

ggml/src/ggml-sycl/vecdotq.hpp

@@ -429,39 +429,6 @@ template <> struct reorder_vec_dot_q_sycl<GGML_TYPE_Q8_0> {
     }
 };
 
-template <> struct reorder_vec_dot_q_sycl<GGML_TYPE_Q2_K> {
-    static constexpr ggml_type gtype = GGML_TYPE_Q2_K;
-
-    using q2_k_block  = ggml_sycl_reordered::block_q_t<GGML_TYPE_Q2_K>;
-    using q2_k_traits = typename q2_k_block::traits;
-
-    __dpct_inline__ float operator()(const void * __restrict__ vbq, const std::pair<int, int> ibx_offset,
-                                     const std::pair<int, int> d_offset, const int8_t * q8_1_quant_ptr,
-                                     const sycl::half2 * q8_1_ds, const int & iqs) {
-        const uint8_t *    base   = static_cast<const uint8_t *>(vbq);
-        const uint8_t *    qs     = base + ibx_offset.first;
-        const uint8_t *    scales = base + d_offset.first;
-        const ggml_half2 * dm     = reinterpret_cast<const ggml_half2 *>(base + d_offset.second);
-
-        const int bq8_offset   = QR2_K * (iqs / QI8_1);
-        const int scale_offset = iqs - iqs % QI8_1 + (iqs % QI8_1) / (QI8_1 / 2);
-
-        const int v = get_int_from_uint8_aligned(qs, iqs);
-
-        int   u[QR2_K];
-        float d8[QR2_K];
-
-#pragma unroll
-        for (int i = 0; i < QR2_K; ++i) {
-            const int8_t * quant_base_ptr = q8_1_quant_ptr + (bq8_offset + i) * QK8_1;
-            u[i]                          = get_int_from_int8_aligned(quant_base_ptr, iqs % QI8_1);
-            d8[i]                         = (*(q8_1_ds + bq8_offset + i))[0];
-        }
-
-        return vec_dot_q2_K_q8_1_impl_mmvq(v, u, scales + scale_offset, *dm, d8);
-    }
-};
-
 template <> struct reorder_vec_dot_q_sycl<GGML_TYPE_Q3_K> {
     static constexpr ggml_type gtype = GGML_TYPE_Q3_K;
 

ggml/src/ggml-webgpu/ggml-webgpu.cpp

@@ -2714,7 +2714,6 @@ static webgpu_encoded_op ggml_webgpu_rope(webgpu_context & ctx,
     const int n_dims     = ((int32_t *) dst->op_params)[1];
     const int mode       = ((int32_t *) dst->op_params)[2];
     const int n_ctx_orig = ((int32_t *) dst->op_params)[4];
-    const int n_offs     = ((int32_t *) dst->op_params)[15];
 
     float freq_base;
     float freq_scale;
@@ -2763,8 +2762,7 @@ static webgpu_encoded_op ggml_webgpu_rope(webgpu_context & ctx,
         (uint32_t) sections[0],
         (uint32_t) sections[1],
         (uint32_t) sections[2],
-        (uint32_t) sections[3],
-        (uint32_t) n_offs
+        (uint32_t) sections[3]
     };
 
     std::vector<wgpu::BindGroupEntry> entries     = { ggml_webgpu_make_tensor_bind_group_entry(ctx, 0, src0),
@@ -4474,7 +4472,9 @@ static bool ggml_backend_webgpu_device_supports_op(ggml_backend_dev_t dev, const
             supports_op = (op->type == GGML_TYPE_F32 && src0->type == GGML_TYPE_F32) && ggml_is_contiguous_rows(src0);
             break;
         case GGML_OP_ROPE:
-            supports_op = op->type == GGML_TYPE_F32 || op->type == GGML_TYPE_F16;
+            // FIXME: support ggml_rope_set_offset
+            supports_op =
+                (op->type == GGML_TYPE_F32 || op->type == GGML_TYPE_F16) && ((const int32_t *) op->op_params)[15] == 0;
             break;
         case GGML_OP_GLU:
             switch (ggml_get_glu_op(op)) {

ggml/src/ggml-webgpu/wgsl-shaders/rope.wgsl

@@ -38,8 +38,7 @@ struct Params {
     sections0: u32,
     sections1: u32,
     sections2: u32,
-    sections3: u32,
-    n_offs: u32
+    sections3: u32
 };
 
 @group(0) @binding(0)
@@ -127,8 +126,7 @@ fn rope_yarn(theta_extrap: f32, i: u32) -> vec2<f32> {
 
 fn pair_base(i0: u32, div_2: bool) -> u32 {
     if (div_2) {
-        // first channel of the rotated pair: n_offs + (i0 - n_offs)/2
-        return i0 / 2 + params.n_offs / 2;
+        return i0 / 2;
     } else {
         return i0;
     }
@@ -167,22 +165,20 @@ fn main(@builtin(global_invocation_id) gid: vec3<u32>) {
     let i_src_row = params.offset_src0 + i3 * params.stride_src03 + i2 * params.stride_src02 + i1 * params.stride_src01;
     let i_dst_row = params.offset_dst + i3 * params.stride_dst3 + i2 * params.stride_dst2 + i1 * params.stride_dst1;
 
-    if ((i0 < params.n_offs || i0 >= params.n_offs + params.n_dims) && !is_vision) {
+    if (i0 >= params.n_dims && !is_vision) {
         let i_src = i_src_row + i0;
         let i_dst = i_dst_row + i0;
         rotate(i_dst, i_dst + 1, f32(src0[i_src]), f32(src0[i_src + 1]));
         return;
     }
 
-    let iw = i0 - params.n_offs; // relative idx
-
     var theta_base_mult: u32 = 0;
-    var theta_scale_pwr: u32 = iw / 2;
+    var theta_scale_pwr: u32 = i0 / 2;
     if (is_mrope) {
         let sect_dims = params.sections0 + params.sections1 + params.sections2 + params.sections3;
         let sec_w = params.sections1 + params.sections0;
         let sec_e = params.sections2 + sec_w;
-        let sector = (iw / 2) % sect_dims;
+        let sector = (i0 / 2) % sect_dims;
         if (is_imrope) {
           if (sector % 3 == 1 && sector < 3 * params.sections1) {
               theta_base_mult = 1;
@@ -207,7 +203,7 @@ fn main(@builtin(global_invocation_id) gid: vec3<u32>) {
           } else if (sector >= sec_e) {
               if (is_vision) {
                   theta_scale_pwr = sector - sec_e;
-                  theta_scale_pwr = (iw / 2) % sec_e;
+                  theta_scale_pwr = (i0 / 2) % sec_e;
               }
               theta_base_mult = 3;
           } else if (is_vision) {
@@ -216,7 +212,7 @@ fn main(@builtin(global_invocation_id) gid: vec3<u32>) {
         }
     }
     let theta_base = f32(src1[params.offset_src1 + i2 + params.ne2 * theta_base_mult]) * pow(params.theta_scale, f32(theta_scale_pwr));
-    let thetas = rope_yarn(theta_base/freq_factor(iw), iw);
+    let thetas = rope_yarn(theta_base/freq_factor(i0), i0);
 
     let i_src = i_src_row + pair_base(i0, is_neox || is_mrope || is_vision);
     let i_dst = i_dst_row + pair_base(i0, is_neox || is_mrope || is_vision);

gguf-py/gguf/constants.py

@@ -205,9 +205,6 @@ class Keys:
         VALUE_LENGTH_MLA             = "{arch}.attention.value_length_mla"
         KEY_LENGTH_SWA               = "{arch}.attention.key_length_swa"
         VALUE_LENGTH_SWA             = "{arch}.attention.value_length_swa"
-        KEY_LENGTH_MLA_SWA           = "{arch}.attention.key_length_mla_swa"
-        VALUE_LENGTH_MLA_SWA         = "{arch}.attention.value_length_mla_swa"
-        KV_LORA_RANK_SWA             = "{arch}.attention.kv_lora_rank_swa"
         SHARED_KV_LAYERS             = "{arch}.attention.shared_kv_layers"
         SLIDING_WINDOW_PATTERN       = "{arch}.attention.sliding_window_pattern"
         TEMPERATURE_SCALE            = "{arch}.attention.temperature_scale"
@@ -364,8 +361,6 @@ class Keys:
         IMAGE_MEAN            = "clip.vision.image_mean"
         IMAGE_STD             = "clip.vision.image_std"
         SPATIAL_MERGE_SIZE    = "clip.vision.spatial_merge_size"
-        EXPERT_COUNT_PER_LAYER = "clip.vision.expert_count_per_layer" # dots3note pyramid MoE, 0 = dense layer
-        EXPERT_USED_COUNT     = "clip.vision.expert_used_count"
         USE_GELU              = "clip.use_gelu"
         USE_SILU              = "clip.use_silu"
         N_WA_PATTERN          = "clip.vision.n_wa_pattern"     # used by qwen2.5vl
@@ -563,7 +558,6 @@ class MODEL_ARCH(IntEnum):
     BAILINGMOE2      = auto()
     BAILINGMOE3      = auto()
     DOTS1            = auto()
-    DOTS3NOTE        = auto()
     ARCEE            = auto()
     AFMOE            = auto()
     LAGUNA           = auto()
@@ -876,11 +870,6 @@ class MODEL_TENSOR(IntEnum):
     V_ENC_FFN_UP         = auto()
     V_ENC_FFN_GATE       = auto()
     V_ENC_FFN_DOWN       = auto()
-    V_ENC_FFN_GATE_INP   = auto() # dots3note vision MoE router
-    V_ENC_FFN_GATE_EXPS  = auto()
-    V_ENC_FFN_UP_EXPS    = auto()
-    V_ENC_FFN_DOWN_EXPS  = auto()
-    V_ENC_FFN_EXP_PROBS_B = auto()
     V_ENC_ATTN_POST_NORM = auto() # gemma4
     V_ENC_FFN_POST_NORM  = auto()
     V_LAYER_SCALE_1      = auto()
@@ -1286,7 +1275,6 @@ MODEL_ARCH_NAMES: dict[MODEL_ARCH, str] = {
     MODEL_ARCH.BAILINGMOE2:      "bailingmoe2",
     MODEL_ARCH.BAILINGMOE3:      "bailingmoe3",
     MODEL_ARCH.DOTS1:            "dots1",
-    MODEL_ARCH.DOTS3NOTE:        "dots3note",
     MODEL_ARCH.ARCEE:            "arcee",
     MODEL_ARCH.AFMOE:            "afmoe",
     MODEL_ARCH.LAGUNA:           "laguna",
@@ -1598,11 +1586,6 @@ TENSOR_NAMES: dict[MODEL_TENSOR, str] = {
     MODEL_TENSOR.V_ENC_FFN_UP:              "v.blk.{bid}.ffn_up",
     MODEL_TENSOR.V_ENC_FFN_GATE:            "v.blk.{bid}.ffn_gate",
     MODEL_TENSOR.V_ENC_FFN_DOWN:            "v.blk.{bid}.ffn_down",
-    MODEL_TENSOR.V_ENC_FFN_GATE_INP:        "v.blk.{bid}.ffn_gate_inp",
-    MODEL_TENSOR.V_ENC_FFN_GATE_EXPS:       "v.blk.{bid}.ffn_gate_exps",
-    MODEL_TENSOR.V_ENC_FFN_UP_EXPS:         "v.blk.{bid}.ffn_up_exps",
-    MODEL_TENSOR.V_ENC_FFN_DOWN_EXPS:       "v.blk.{bid}.ffn_down_exps",
-    MODEL_TENSOR.V_ENC_FFN_EXP_PROBS_B:     "v.blk.{bid}.exp_probs_b",
     MODEL_TENSOR.V_ENC_ATTN_POST_NORM:      "v.blk.{bid}.attn_post_norm",
     MODEL_TENSOR.V_ENC_FFN_POST_NORM:       "v.blk.{bid}.ffn_post_norm",
     MODEL_TENSOR.V_LAYER_SCALE_1:           "v.blk.{bid}.ls1",
@@ -1925,11 +1908,6 @@ MODEL_TENSORS: dict[MODEL_ARCH, list[MODEL_TENSOR]] = {
         MODEL_TENSOR.V_ENC_FFN_UP,
         MODEL_TENSOR.V_ENC_FFN_GATE,
         MODEL_TENSOR.V_ENC_FFN_DOWN,
-        MODEL_TENSOR.V_ENC_FFN_GATE_INP,
-        MODEL_TENSOR.V_ENC_FFN_GATE_EXPS,
-        MODEL_TENSOR.V_ENC_FFN_UP_EXPS,
-        MODEL_TENSOR.V_ENC_FFN_DOWN_EXPS,
-        MODEL_TENSOR.V_ENC_FFN_EXP_PROBS_B,
         MODEL_TENSOR.V_ENC_ATTN_POST_NORM,
         MODEL_TENSOR.V_ENC_FFN_POST_NORM,
         MODEL_TENSOR.V_LAYER_SCALE_1,
@@ -4356,44 +4334,6 @@ MODEL_TENSORS: dict[MODEL_ARCH, list[MODEL_TENSOR]] = {
         MODEL_TENSOR.FFN_UP_EXP,
         MODEL_TENSOR.FFN_UP_SHEXP,
     ],
-    MODEL_ARCH.DOTS3NOTE: [
-        MODEL_TENSOR.TOKEN_EMBD,
-        MODEL_TENSOR.OUTPUT_NORM,
-        MODEL_TENSOR.OUTPUT,
-        MODEL_TENSOR.ATTN_NORM,
-        MODEL_TENSOR.ATTN_Q_A,
-        MODEL_TENSOR.ATTN_Q_B,
-        MODEL_TENSOR.ATTN_KV_A_MQA,
-        MODEL_TENSOR.ATTN_K_B,
-        MODEL_TENSOR.ATTN_V_B,
-        MODEL_TENSOR.ATTN_Q_A_NORM,
-        MODEL_TENSOR.ATTN_KV_A_NORM,
-        MODEL_TENSOR.ATTN_K_NORM,
-        MODEL_TENSOR.ATTN_GATE,
-        MODEL_TENSOR.ATTN_OUT,
-        MODEL_TENSOR.FFN_NORM,
-        MODEL_TENSOR.FFN_GATE,
-        MODEL_TENSOR.FFN_DOWN,
-        MODEL_TENSOR.FFN_UP,
-        MODEL_TENSOR.FFN_GATE_INP,
-        MODEL_TENSOR.FFN_EXP_PROBS_B,
-        MODEL_TENSOR.FFN_GATE_EXP,
-        MODEL_TENSOR.FFN_DOWN_EXP,
-        MODEL_TENSOR.FFN_UP_EXP,
-        MODEL_TENSOR.FFN_GATE_SHEXP,
-        MODEL_TENSOR.FFN_DOWN_SHEXP,
-        MODEL_TENSOR.FFN_UP_SHEXP,
-        MODEL_TENSOR.INDEXER_K_NORM,
-        MODEL_TENSOR.INDEXER_PROJ,
-        MODEL_TENSOR.INDEXER_ATTN_K,
-        MODEL_TENSOR.INDEXER_ATTN_Q_B,
-        # NextN/MTP tensors - preserved but unused
-        MODEL_TENSOR.NEXTN_EH_PROJ,
-        MODEL_TENSOR.NEXTN_EMBED_TOKENS,
-        MODEL_TENSOR.NEXTN_ENORM,
-        MODEL_TENSOR.NEXTN_HNORM,
-        MODEL_TENSOR.NEXTN_SHARED_HEAD_NORM,
-    ],
     MODEL_ARCH.ARCEE: [
         MODEL_TENSOR.TOKEN_EMBD,
         MODEL_TENSOR.OUTPUT_NORM,
@@ -5514,8 +5454,6 @@ class VisionProjectorType:
     COGVLM = "cogvlm"
     JANUS_PRO = "janus_pro"
     DOTSOCR = "dots_ocr"
-    DOTS3NOTE_V = "dots3note_v"
-    DOTS3NOTE_A = "dots3note_a" # audio
     DEEPSEEKOCR = "deepseekocr"
     DEEPSEEKOCR2 = "deepseekocr2"
     LFM2A = "lfm2a" # audio

gguf-py/gguf/gguf_writer.py

@@ -785,15 +785,6 @@ class GGUFWriter:
     def add_key_length_swa(self, length: int) -> None:
         self.add_uint32(Keys.Attention.KEY_LENGTH_SWA.format(arch=self.arch), length)
 
-    def add_key_length_mla_swa(self, length: int) -> None:
-        self.add_uint32(Keys.Attention.KEY_LENGTH_MLA_SWA.format(arch=self.arch), length)
-
-    def add_value_length_mla_swa(self, length: int) -> None:
-        self.add_uint32(Keys.Attention.VALUE_LENGTH_MLA_SWA.format(arch=self.arch), length)
-
-    def add_kv_lora_rank_swa(self, length: int) -> None:
-        self.add_uint32(Keys.Attention.KV_LORA_RANK_SWA.format(arch=self.arch), length)
-
     def add_value_length_swa(self, length: int) -> None:
         self.add_uint32(Keys.Attention.VALUE_LENGTH_SWA.format(arch=self.arch), length)
 
@@ -1327,12 +1318,6 @@ class GGUFWriter:
     def add_vision_spatial_merge_size(self, value: int) -> None:
         self.add_uint32(Keys.ClipVision.SPATIAL_MERGE_SIZE, value)
 
-    def add_vision_expert_count_per_layer(self, value: Sequence[int]) -> None:
-        self.add_array(Keys.ClipVision.EXPERT_COUNT_PER_LAYER, value)
-
-    def add_vision_expert_used_count(self, value: int) -> None:
-        self.add_uint32(Keys.ClipVision.EXPERT_USED_COUNT, value)
-
     def add_vision_use_gelu(self, value: bool) -> None:
         self.add_bool(Keys.ClipVision.USE_GELU, value)
 

gguf-py/gguf/tensor_mapping.py

@@ -723,7 +723,6 @@ class TensorNameMap:
             "model.layers.layers.{bid}.mixer.k",                              # plamo2
             "model.layers.layers.{bid}.mixer.k_norm",                         # plamo3
             "layers.{bid}.self_attn.k_norm",                                  # qwen3-embedding
-            "model.layers.{bid}.self_attn.k_rope_only_layernorm",             # dots3note
             "model.layers.{bid}.attention.key_layernorm",                     # apertus
         ),
 
@@ -1454,7 +1453,6 @@ class TensorNameMap:
             "mlp_AR.linear_{bid}", # PaddleOCR-VL
             "merger.mlp.{bid}",
             "vision_tower.merger.mlp.{bid}", # dots.ocr
-            "vision_encoder.adapter.mlp.{bid}", # dots3note
             "vit.perceive.proj.{bid}", # HunyuanVL (proj.0 = conv1, proj.2 = conv2)
         ),
 
@@ -1505,7 +1503,6 @@ class TensorNameMap:
             "vision_model.radio_model.model.patch_generator.embedder", # Nemotron Nano v2 VL
             "model.vision_tower.patch_embedder.input_proj", # gemma4
             "vision_tower.patch_embed.patchifier.proj", # dots.ocr
-            "vision_encoder.patch_embed.proj", # dots3note
             "vision_model.conv1", # Step3-VL
             "model.vision_embedder.patch_dense", # gemma4 unified
             "model.vision_tower.patch_embedder.patch_embedding", # muse-glimmer
@@ -1514,7 +1511,6 @@ class TensorNameMap:
         MODEL_TENSOR.V_ENC_EMBD_NORM: (
             "visual.post_conv_layernorm", # glm4v
             "vision_tower.patch_embed.patchifier.norm", # dots.ocr
-            "vision_encoder.patch_embed.norm", # dots3note
         ),
 
         MODEL_TENSOR.V_ENC_EMBD_PATCH_NORM: (
@@ -1554,7 +1550,6 @@ class TensorNameMap:
         MODEL_TENSOR.V_ENC_ATTN_QKV: (
             "visual.blocks.{bid}.attn.qkv", # qwen3vl
             "vision_tower.blocks.{bid}.attn.qkv", # dots.ocr
-            "vision_encoder.blocks.{bid}.attn.qkv", # dots3note
             "model.vision.transformer.layers.{bid}.attention.query_key_value", # cogvlm
             "model.vision_model.transformer.layers.{bid}.self_attn.qkv_proj", # Deepseek-OCR CLIP
             "vision_tower.encoder.blocks.{bid}.wqkv", # Kimi-K2.5
@@ -1583,7 +1578,6 @@ class TensorNameMap:
         ),
 
         MODEL_TENSOR.V_ENC_ATTN_Q_NORM: (
-            "vision_encoder.blocks.{bid}.attn.q_norm", # dots3note
             "vision_tower.vision_model.encoder.layers.{bid}.attn.q_norm", # InternVL
             "model.vision_tower.encoder.layer.{bid}.attention.q_norm", # Intern-S1
             "visual.blocks.{bid}.attn.q_norm", # GLM-OCR
@@ -1611,7 +1605,6 @@ class TensorNameMap:
         ),
 
         MODEL_TENSOR.V_ENC_ATTN_K_NORM: (
-            "vision_encoder.blocks.{bid}.attn.k_norm", # dots3note
             "vision_tower.vision_model.encoder.layers.{bid}.attn.k_norm", # InternVL
             "model.vision_tower.encoder.layer.{bid}.attention.k_norm", # Intern-S1
             "visual.blocks.{bid}.attn.k_norm", # GLM-OCR
@@ -1657,7 +1650,6 @@ class TensorNameMap:
             "siglip2.vision_model.encoder.layers.{bid}.layer_norm1",
             "vision_model.radio_model.model.blocks.{bid}.norm1", # Nemotron Nano v2 VL
             "vision_tower.blocks.{bid}.norm1", # dots.ocr
-            "vision_encoder.blocks.{bid}.norm_1", # dots3note
             "vision_model.transformer.resblocks.{bid}.ln_1", # Step3-VL
             "model.qwen2_model.model.model.layers.{bid}.input_layernorm", # Deepseek-OCR-2 qwen2
             "model.vision_tower.layers.{bid}.norm1", # muse-glimmer
@@ -1685,7 +1677,6 @@ class TensorNameMap:
             "model.qwen2_model.model.model.layers.{bid}.self_attn.o_proj", # Deepseek-OCR-2 qwen2
             "vision_model.model.layers.{bid}.self_attn.o_proj.linear", # gemma4
             "vision_tower.blocks.{bid}.attn.proj", # dots.ocr
-            "vision_encoder.blocks.{bid}.attn.proj", # dots3note
             "vision_model.transformer.resblocks.{bid}.attn.out_proj", # Step3-VL
             "model.vision_tower.layers.{bid}.attn.proj", # muse-glimmer
         ),
@@ -1714,14 +1705,12 @@ class TensorNameMap:
             "vision_model.radio_model.model.blocks.{bid}.norm2", # Nemotron Nano v2 VL
             "vision_model.model.layers.{bid}.pre_feedforward_layernorm", # gemma4
             "vision_tower.blocks.{bid}.norm2", # dots.ocr
-            "vision_encoder.blocks.{bid}.norm_2", # dots3note
             "vision_model.transformer.resblocks.{bid}.ln_2", # Step3-VL
             "model.qwen2_model.model.model.layers.{bid}.post_attention_layernorm", # Deepseek-OCR-2 qwen2
             "model.vision_tower.layers.{bid}.norm2", # muse-glimmer
         ),
 
         MODEL_TENSOR.V_ENC_FFN_UP: (
-            "vision_encoder.blocks.{bid}.mlp.fc3", # dots3note
             "model.vision_tower.vision_model.encoder.layers.{bid}.mlp.fc1", # Granite4Vision
             "vision_tower.vision_model.encoder.layers.{bid}.mlp.fc1",
             "model.vision_tower.encoder.layers.{bid}.mlp.fc1", # minicpmv4_6
@@ -1747,7 +1736,6 @@ class TensorNameMap:
         ),
 
         MODEL_TENSOR.V_ENC_FFN_GATE: (
-            "vision_encoder.blocks.{bid}.mlp.fc1", # dots3note
             "vision_tower.transformer.layers.{bid}.feed_forward.gate_proj", # pixtral-hf
             "vision_encoder.transformer.layers.{bid}.feed_forward.w1", # pixtral
             "visual.blocks.{bid}.mlp.gate_proj", # qwen2.5vl
@@ -1756,7 +1744,6 @@ class TensorNameMap:
         ),
 
         MODEL_TENSOR.V_ENC_FFN_DOWN: (
-            "vision_encoder.blocks.{bid}.mlp.fc2", # dots3note
             "model.vision_tower.vision_model.encoder.layers.{bid}.mlp.fc2", # Granite4Vision
             "vision_tower.vision_model.encoder.layers.{bid}.mlp.fc2",
             "model.vision_tower.encoder.layers.{bid}.mlp.fc2", # minicpmv4_6
@@ -1781,29 +1768,6 @@ class TensorNameMap:
             "model.vision_tower.layers.{bid}.mlp.fc2", # muse-glimmer
         ),
 
-
-        MODEL_TENSOR.V_ENC_FFN_GATE_INP: (
-            "vision_encoder.blocks.{bid}.mlp.gate_weight", # dots3note
-        ),
-
-        MODEL_TENSOR.V_ENC_FFN_EXP_PROBS_B: (
-            "vision_encoder.blocks.{bid}.mlp.router_bias", # dots3note
-        ),
-
-        # note: expert weights are stacked into a single 3D tensor in conversion code,
-        # which emits the pseudo-names below
-        MODEL_TENSOR.V_ENC_FFN_GATE_EXPS: (
-            "vision_encoder.blocks.{bid}.mlp.experts.fc1", # dots3note
-        ),
-
-        MODEL_TENSOR.V_ENC_FFN_UP_EXPS: (
-            "vision_encoder.blocks.{bid}.mlp.experts.fc3", # dots3note
-        ),
-
-        MODEL_TENSOR.V_ENC_FFN_DOWN_EXPS: (
-            "vision_encoder.blocks.{bid}.mlp.experts.fc2", # dots3note
-        ),
-
         MODEL_TENSOR.V_ENC_ATTN_POST_NORM: (
             "vision_model.model.layers.{bid}.post_attention_layernorm", # gemma4
         ),
@@ -1835,7 +1799,6 @@ class TensorNameMap:
             "vision_model.layernorm_pre", # llama4
             "model.vision_model.pre_layrnorm", # Deepseek-OCR CLIP
             "vision_tower.patch_embed.patchifier.norm", # dots.ocr
-            "vision_encoder.patch_embed.norm", # dots3note
             "vision_model.ln_pre", # Step3-VL
             "model.vision_tower.ln_pre", # muse-glimmer
         ),
@@ -1857,7 +1820,6 @@ class TensorNameMap:
         MODEL_TENSOR.V_MM_POST_NORM: (
             "visual.merger.post_projection_norm", # glm4v
             "vision_tower.post_trunk_norm", # dots.ocr
-            "vision_encoder.post_trunk_norm", # dots3note
             "vit.perceive.after_rms", # HunyuanVL
         ),
 
@@ -1875,7 +1837,6 @@ class TensorNameMap:
             "mlp_AR.pre_norm", # PaddleOCR-VL
             "merger.ln_q",
             "vision_tower.merger.ln_q", # dots.ocr
-            "vision_encoder.adapter.ln_q", # dots3note
             "model.merger.mlp.0.pre_norm", # minicpmv4_6
         ),
 
@@ -2211,12 +2172,10 @@ class TensorNameMap:
 
         MODEL_TENSOR.A_ENC_CONV2D: (
             "audio_tower.conv2d{bid}", # qwen3omni
-            "audio_encoder.dots_encoder.speech_encoder.conv2d{bid}", # dots3note
         ),
 
         MODEL_TENSOR.A_ENC_CONV_OUT: (
             "audio_tower.conv_out", # qwen3omni
-            "audio_encoder.dots_encoder.speech_encoder.conv_out", # dots3note
             "speaker_encoder.mfa.conv", # qwen3tts speaker encoder: multi-layer feature aggregation
         ),
 
@@ -2224,14 +2183,12 @@ class TensorNameMap:
 
         MODEL_TENSOR.A_POST_NORM: (
             "audio_tower.layer_norm", # ultravox
-            "audio_encoder.dots_encoder.speech_encoder.layer_norm", # dots3note
             "audio_tower.ln_post", # qwen2omni
             "encoder.layer_norm", # mimo-audio-tokenizer
         ),
 
         MODEL_TENSOR.A_ENC_ATTN_Q: (
             "audio_tower.layers.{bid}.self_attn.q_proj", # ultravox
-            "audio_encoder.dots_encoder.speech_encoder.layers.{bid}.self_attn.q_proj", # dots3note
             "conformer.layers.{bid}.self_attn.linear_q", # lfm2
             "conformer.layers.{bid}.attention.attn.q_proj", # gemma3n
             "conformer.layers.{bid}.self_attn.q_proj", # gemma4
@@ -2242,7 +2199,6 @@ class TensorNameMap:
 
         MODEL_TENSOR.A_ENC_ATTN_K: (
             "audio_tower.layers.{bid}.self_attn.k_proj", # ultravox
-            "audio_encoder.dots_encoder.speech_encoder.layers.{bid}.self_attn.k_proj", # dots3note
             "conformer.layers.{bid}.self_attn.linear_k", # lfm2
             "conformer.layers.{bid}.attention.attn.k_proj", # gemma3n
             "conformer.layers.{bid}.self_attn.k_proj", # gemma4
@@ -2253,7 +2209,6 @@ class TensorNameMap:
 
         MODEL_TENSOR.A_ENC_ATTN_V: (
             "audio_tower.layers.{bid}.self_attn.v_proj", # ultravox
-            "audio_encoder.dots_encoder.speech_encoder.layers.{bid}.self_attn.v_proj", # dots3note
             "conformer.layers.{bid}.self_attn.linear_v", # lfm2
             "conformer.layers.{bid}.attention.attn.v_proj", # gemma3n
             "conformer.layers.{bid}.self_attn.v_proj", # gemma4
@@ -2285,7 +2240,6 @@ class TensorNameMap:
 
         MODEL_TENSOR.A_ENC_INPUT_NORM: (
             "audio_tower.layers.{bid}.self_attn_layer_norm", # ultravox
-            "audio_encoder.dots_encoder.speech_encoder.layers.{bid}.self_attn_layer_norm", # dots3note
             "conformer.layers.{bid}.norm_self_att", # lfm2
             "conformer.layers.{bid}.attention.pre_attn_norm", # gemma3n
             "sound_encoder.encoder.layers.{bid}.norm_self_att", # parakeet
@@ -2295,7 +2249,6 @@ class TensorNameMap:
 
         MODEL_TENSOR.A_ENC_OUTPUT: (
             "audio_tower.layers.{bid}.self_attn.out_proj", # ultravox
-            "audio_encoder.dots_encoder.speech_encoder.layers.{bid}.self_attn.out_proj", # dots3note
             "conformer.layers.{bid}.self_attn.linear_out", # lfm2
             "conformer.layers.{bid}.attention.post", # gemma3n
             "conformer.layers.{bid}.self_attn.post", # gemma4
@@ -2306,7 +2259,6 @@ class TensorNameMap:
 
         MODEL_TENSOR.A_ENC_OUTPUT_NORM: (
             "audio_tower.layers.{bid}.final_layer_norm", # ultravox
-            "audio_encoder.dots_encoder.speech_encoder.layers.{bid}.final_layer_norm", # dots3note
             "conformer.layers.{bid}.norm_out", # lfm2
             "conformer.layers.{bid}.attention.post_norm", # gemma3n
             "sound_encoder.encoder.layers.{bid}.norm_out", # parakeet
@@ -2332,7 +2284,6 @@ class TensorNameMap:
         ),
 
         MODEL_TENSOR.A_ENC_FFN_UP: (
-            "audio_encoder.dots_encoder.speech_encoder.layers.{bid}.fc1_up", # dots3note (split from fc1 in conversion code)
             "audio_tower.layers.{bid}.fc1", # ultravox
             "conformer.layers.{bid}.feed_forward1.linear1", # lfm2
             "conformer.layers.{bid}.ffw_layer_start.ffw_layer_1", # gemma3n
@@ -2342,12 +2293,9 @@ class TensorNameMap:
             "encoder.layers.{bid}.fc1", # mimo-audio-tokenizer
         ),
 
-        MODEL_TENSOR.A_ENC_FFN_GATE: (
-            "audio_encoder.dots_encoder.speech_encoder.layers.{bid}.fc1_gate", # dots3note (split from fc1 in conversion code)
-        ),
+        MODEL_TENSOR.A_ENC_FFN_GATE: (),
 
         MODEL_TENSOR.A_ENC_FFN_DOWN: (
-            "audio_encoder.dots_encoder.speech_encoder.layers.{bid}.fc2", # dots3note
             "audio_tower.layers.{bid}.fc2", # ultravox
             "conformer.layers.{bid}.feed_forward1.linear2", # lfm2
             "conformer.layers.{bid}.ffw_layer_start.ffw_layer_2", # gemma3n
@@ -2431,7 +2379,6 @@ class TensorNameMap:
 
         MODEL_TENSOR.A_MMPROJ: (
             "audio.multi_modal_projector.linear_{bid}", # ultravox, meralion
-            "audio_encoder.audio_adapter.proj.{bid}", # dots3note (proj.1, proj.3)
             "audio_adapter.model.{bid}", # lfm2
             "audio_tower.proj{bid}", # qwen3omni
             "sound_projection.linear{bid}", # parakeet (linear1, linear2)
@@ -2446,7 +2393,6 @@ class TensorNameMap:
 
         MODEL_TENSOR.A_MM_NORM_PRE: (
             "audio.multi_modal_projector.ln_pre", # ultravox
-            "audio_encoder.audio_adapter.proj.0", # dots3note
             "sound_projection.norm", # parakeet
         ),
 

scripts/make-release-desc.sh

@@ -15,8 +15,7 @@
 # tag exists.
 #
 # Env (when running in GitHub Actions):
-#   GITHUB_OUTPUT: previous_tag, changelog_title, changelog, nightly and nightly_tag
-#     are written here
+#   GITHUB_OUTPUT: previous_tag, changelog_title, changelog and nightly are written here
 #   GITHUB_REPOSITORY: owner/repo, used to build the nightly release URL (skipped when unset)
 set -euo pipefail
 
@@ -53,10 +52,10 @@ PREV="$( { git tag --list; echo "${VERSION}"; } \
 
 if [[ -n "${PREV}" ]]; then
     CHANGELOG="$(git log --oneline "${PREV}..${RELEASE_COMMIT}")"
-    CHANGELOG_TITLE="Changelog since ${PREV}"
+    CHANGELOG_TITLE="Change log since ${PREV}"
 else
     CHANGELOG="(no previous release tag found)"
-    CHANGELOG_TITLE="Changelog"
+    CHANGELOG_TITLE="Change log"
 fi
 
 # Nightly release: the b* tag pointing at the release commit (|| true: no match is not an error)
@@ -81,7 +80,6 @@ if [[ -n "${GITHUB_OUTPUT:-}" ]]; then
         echo "previous_tag=${PREV}"
         echo "changelog_title=${CHANGELOG_TITLE}"
         echo "nightly=${NIGHTLY}"
-        echo "nightly_tag=${NIGHTLY_TAG}"
         echo "changelog<<CHANGELOG_EOF"
         echo "${CHANGELOG}"
         echo "CHANGELOG_EOF"

scripts/release.sh

@@ -1,204 +0,0 @@
-#!/bin/bash
-#
-# Release preparation script for llama.cpp.
-#
-# Bumps the version in CMakeLists.txt on a release candidate branch.
-# The branch should then be pushed and a PR created, reviewed, and
-# merged. After the PR is merged and the build-cpu workflow has
-# completed successfully, the release is finalized by the make-release
-# workflow (.github/workflows/make-release.yml), which creates the tag.
-#
-# Usage:
-#   ./scripts/release.sh [major|minor|patch] [--dry-run]
-#
-# Example:
-#   $ ./scripts/release.sh minor
-#
-# The script:
-# 1. Creates a release candidate branch (llama-rc-v<major>.<minor>.<patch>)
-# 2. Bumps the version in CMakeLists.txt
-# 3. Commits the version bump
-#
-
-set -e
-
-if [ ! -f "CMakeLists.txt" ] || [ ! -d "scripts" ]; then
-    echo "Error: Must be run from llama.cpp root directory"
-    exit 1
-fi
-
-# Parse command line arguments
-VERSION_TYPE=""
-DRY_RUN=false
-
-for arg in "$@"; do
-    case $arg in
-        --dry-run)
-            DRY_RUN=true
-            ;;
-        major|minor|patch)
-            VERSION_TYPE="$arg"
-            ;;
-        *)
-            echo "Error: Unknown argument '$arg'"
-            echo "Usage: $0 [major|minor|patch] [--dry-run]"
-            exit 1
-            ;;
-    esac
-done
-
-# Default to patch if no version type specified
-VERSION_TYPE="${VERSION_TYPE:-patch}"
-
-# Common validation functions
-check_git_status() {
-    # Check for uncommitted changes (skip in dry-run)
-    if [ "$DRY_RUN" = false ] && ! git diff-index --quiet HEAD --; then
-        echo "Error: You have uncommitted changes. Please commit or stash them first."
-        exit 1
-    fi
-}
-
-check_master_branch() {
-    # Ensure we're on master branch
-    CURRENT_BRANCH=$(git branch --show-current)
-    if [ "$CURRENT_BRANCH" != "master" ]; then
-        if [ "$DRY_RUN" = true ]; then
-            echo "[dry run] Warning: Not on master branch (currently on: $CURRENT_BRANCH). Continuing with dry-run..."
-            echo ""
-        else
-            echo "Error: Must be on master branch. Currently on: $CURRENT_BRANCH"
-            exit 1
-        fi
-    fi
-}
-
-check_master_up_to_date() {
-    # Check if we have the latest from master (skip in dry-run)
-    if [ "$DRY_RUN" = false ]; then
-        echo "Checking if local master is up-to-date with remote..."
-        git fetch origin master
-        LOCAL=$(git rev-parse HEAD)
-        REMOTE=$(git rev-parse origin/master)
-
-        if [ "$LOCAL" != "$REMOTE" ]; then
-            echo "Error: Your local master branch is not up-to-date with origin/master."
-            echo "Please run 'git pull origin master' first."
-            exit 1
-        fi
-        echo "✓ Local master is up-to-date with remote"
-        echo ""
-    elif [ "$(git branch --show-current)" = "master" ]; then
-        echo "[dry run] Warning: Dry-run mode - not checking if master is up-to-date with remote"
-        echo ""
-    fi
-}
-
-# In-place sed that works on both GNU (Linux) and BSD (macOS) sed
-sed_inplace() {
-    if sed --version >/dev/null 2>&1; then
-        sed -i "$@"
-    else
-        sed -i '' "$@"
-    fi
-}
-
-prepare_release() {
-    if [ "$DRY_RUN" = true ]; then
-        echo "[dry-run] Preparing release (no changes will be made)"
-    else
-        echo "Starting release preparation..."
-    fi
-    echo ""
-
-    check_git_status
-    check_master_branch
-    check_master_up_to_date
-
-    # Extract current version from CMakeLists.txt
-    echo "Step 1: Reading current version..."
-    MAJOR=$(grep "set(LLAMA_VERSION_MAJOR" CMakeLists.txt | sed 's/.*MAJOR \([0-9]*\).*/\1/')
-    MINOR=$(grep "set(LLAMA_VERSION_MINOR" CMakeLists.txt | sed 's/.*MINOR \([0-9]*\).*/\1/')
-    PATCH=$(grep "set(LLAMA_VERSION_PATCH" CMakeLists.txt | sed 's/.*PATCH \([0-9]*\).*/\1/')
-
-    echo "Current version: $MAJOR.$MINOR.$PATCH"
-
-    # Calculate new version
-    case $VERSION_TYPE in
-        major)
-            NEW_MAJOR=$((MAJOR + 1))
-            NEW_MINOR=0
-            NEW_PATCH=0
-            ;;
-        minor)
-            NEW_MAJOR=$MAJOR
-            NEW_MINOR=$((MINOR + 1))
-            NEW_PATCH=0
-            ;;
-        patch)
-            NEW_MAJOR=$MAJOR
-            NEW_MINOR=$MINOR
-            NEW_PATCH=$((PATCH + 1))
-            ;;
-    esac
-
-    NEW_VERSION="$NEW_MAJOR.$NEW_MINOR.$NEW_PATCH"
-    RC_BRANCH="llama-rc-v$NEW_VERSION"
-    echo "New release version: $NEW_VERSION"
-    echo "Release candidate branch: $RC_BRANCH"
-    echo ""
-
-    # Create release candidate branch
-    echo "Step 2: Creating release candidate branch..."
-    if [ "$DRY_RUN" = true ]; then
-        echo "  [dry-run] Would create branch: $RC_BRANCH"
-    else
-        git checkout -b "$RC_BRANCH"
-        echo "✓ Created and switched to branch: $RC_BRANCH"
-    fi
-    echo ""
-
-    # Update CMakeLists.txt for release
-    echo "Step 3: Updating version in CMakeLists.txt..."
-    if [ "$DRY_RUN" = true ]; then
-        echo "  [dry-run] Would update LLAMA_VERSION_MAJOR to $NEW_MAJOR"
-        echo "  [dry-run] Would update LLAMA_VERSION_MINOR to $NEW_MINOR"
-        echo "  [dry-run] Would update LLAMA_VERSION_PATCH to $NEW_PATCH"
-    else
-        sed_inplace -e "s/set(LLAMA_VERSION_MAJOR [0-9]*)/set(LLAMA_VERSION_MAJOR $NEW_MAJOR)/" CMakeLists.txt
-        sed_inplace -e "s/set(LLAMA_VERSION_MINOR [0-9]*)/set(LLAMA_VERSION_MINOR $NEW_MINOR)/" CMakeLists.txt
-        sed_inplace -e "s/set(LLAMA_VERSION_PATCH [0-9]*)/set(LLAMA_VERSION_PATCH $NEW_PATCH)/" CMakeLists.txt
-    fi
-    echo ""
-
-    # Commit version bump
-    echo "Step 4: Committing version bump..."
-    if [ "$DRY_RUN" = true ]; then
-        echo "  [dry-run] Would commit: 'llama.cpp : bump version to $NEW_VERSION'"
-    else
-        git add CMakeLists.txt
-        git commit -m "llama.cpp : bump version to $NEW_VERSION"
-    fi
-    echo ""
-
-    echo ""
-    if [ "$DRY_RUN" = true ]; then
-        echo "[dry-run] Summary (no changes were made):"
-        echo "  • Would have created branch: $RC_BRANCH"
-        echo "  • Would have updated version to: $NEW_VERSION"
-    else
-        echo "Release preparation completed!"
-        echo "Summary:"
-        echo "  • Created branch: $RC_BRANCH"
-        echo "  • Updated version to: $NEW_VERSION"
-        echo ""
-        echo "Next steps:"
-        echo "  • Push branch to remote: git push origin $RC_BRANCH"
-        echo "  • Create a Pull Request from $RC_BRANCH to master"
-        echo "  • After the PR is merged and the build-cpu workflow has passed,"
-        echo "    create the release with the make-release workflow"
-        echo "    (.github/workflows/make-release.yml)"
-    fi
-}
-
-prepare_release

scripts/sync-ggml.last

@@ -1 +1 @@
-8599e0ea3756c4bac4ef813af2241cb1a8bbfb0b
+8c63e70982c95ceb862e3a1073a2c1beef75d60a

src/CMakeLists.txt

@@ -25,7 +25,6 @@ add_library(llama
             llama-kv-cache.cpp
             llama-kv-cache-iswa.cpp
             llama-kv-cache-dsa.cpp
-            llama-kv-cache-dsa-iswa.cpp
             llama-kv-cache-msa.cpp
             llama-kv-cache-dsv4.cpp
             llama-memory.cpp

src/llama-arch.cpp

@@ -110,7 +110,6 @@ static const std::map<llm_arch, const char *> LLM_ARCH_NAMES = {
     { LLM_ARCH_BAILINGMOE2,      "bailingmoe2"      },
     { LLM_ARCH_BAILINGMOE3,      "bailingmoe3"      },
     { LLM_ARCH_DOTS1,            "dots1"            },
-    { LLM_ARCH_DOTS3NOTE,        "dots3note"        },
     { LLM_ARCH_ARCEE,            "arcee"            },
     { LLM_ARCH_AFMOE,            "afmoe"            },
     { LLM_ARCH_LAGUNA,           "laguna"           },
@@ -274,9 +273,6 @@ static const std::map<llm_kv, const char *> LLM_KV_NAMES = {
     { LLM_KV_ATTENTION_VALUE_LENGTH_MLA,             "%s.attention.value_length_mla"             },
     { LLM_KV_ATTENTION_KEY_LENGTH_SWA,               "%s.attention.key_length_swa"               },
     { LLM_KV_ATTENTION_VALUE_LENGTH_SWA,             "%s.attention.value_length_swa"             },
-    { LLM_KV_ATTENTION_KEY_LENGTH_MLA_SWA,           "%s.attention.key_length_mla_swa"           },
-    { LLM_KV_ATTENTION_VALUE_LENGTH_MLA_SWA,         "%s.attention.value_length_mla_swa"         },
-    { LLM_KV_ATTENTION_KV_LORA_RANK_SWA,             "%s.attention.kv_lora_rank_swa"             },
     { LLM_KV_ATTENTION_INDEXER_HEAD_COUNT,           "%s.attention.indexer.head_count"           },
     { LLM_KV_ATTENTION_INDEXER_KEY_LENGTH,           "%s.attention.indexer.key_length"           },
     { LLM_KV_ATTENTION_INDEXER_TOP_K,                "%s.attention.indexer.top_k"                },
@@ -1038,7 +1034,6 @@ bool llm_arch_supports_rs_rollback(const llm_arch & arch) {
         case LLM_ARCH_NEMOTRON_H_MOE:
         case LLM_ARCH_LFM2:
         case LLM_ARCH_LFM2MOE:
-        case LLM_ARCH_BAILINGMOE3:
             return true;
         default:
             return false;
@@ -1061,13 +1056,14 @@ bool llm_arch_supports_sm_tensor(const llm_arch & arch) {
         case LLM_ARCH_DEEPSEEK2:
         case LLM_ARCH_DEEPSEEK32:
         case LLM_ARCH_DEEPSEEK4:
-        case LLM_ARCH_DOTS3NOTE:
         case LLM_ARCH_GLM_DSA:
         case LLM_ARCH_BITNET:
         case LLM_ARCH_T5:
         case LLM_ARCH_NEMOTRON_H:
         case LLM_ARCH_NEMOTRON_H_MOE:
         case LLM_ARCH_GRANITE_HYBRID:
+        case LLM_ARCH_LFM2:
+        case LLM_ARCH_LFM2MOE:
         case LLM_ARCH_MINIMAX_01:
         case LLM_ARCH_MINIMAX_M2:
         case LLM_ARCH_MINIMAX_M3:

src/llama-arch.h

@@ -115,7 +115,6 @@ enum llm_arch {
     LLM_ARCH_BAILINGMOE2,
     LLM_ARCH_BAILINGMOE3,
     LLM_ARCH_DOTS1,
-    LLM_ARCH_DOTS3NOTE,
     LLM_ARCH_ARCEE,
     LLM_ARCH_AFMOE,
     LLM_ARCH_LAGUNA,
@@ -279,9 +278,6 @@ enum llm_kv {
     LLM_KV_ATTENTION_VALUE_LENGTH_MLA,
     LLM_KV_ATTENTION_KEY_LENGTH_SWA,
     LLM_KV_ATTENTION_VALUE_LENGTH_SWA,
-    LLM_KV_ATTENTION_KEY_LENGTH_MLA_SWA,
-    LLM_KV_ATTENTION_VALUE_LENGTH_MLA_SWA,
-    LLM_KV_ATTENTION_KV_LORA_RANK_SWA,
     LLM_KV_ATTENTION_INDEXER_HEAD_COUNT,
     LLM_KV_ATTENTION_INDEXER_KEY_LENGTH,
     LLM_KV_ATTENTION_INDEXER_TOP_K,

src/llama-graph.cpp

@@ -9,7 +9,6 @@
 #include "llama-kv-cache.h"
 #include "llama-kv-cache-iswa.h"
 #include "llama-kv-cache-dsa.h"
-#include "llama-kv-cache-dsa-iswa.h"
 #include "llama-kv-cache-msa.h"
 #include "llama-kv-cache-dsv4.h"
 #include "llama-memory-hybrid.h"
@@ -508,12 +507,10 @@ void llm_graph_input_attn_k::set_input(const llama_ubatch * ubatch) {
 }
 
 bool llm_graph_input_attn_k::can_reuse(const llm_graph_params & params) {
-    mctx = static_cast<const llama_kv_cache_context *>(params.mctx);
+    const auto * mctx = static_cast<const llama_kv_cache_context *>(params.mctx);
 
-    return can_reuse_impl(params);
-}
+    this->mctx = mctx;
 
-bool llm_graph_input_attn_k::can_reuse_impl(const llm_graph_params & params) {
     bool res = true;
 
     res &= self_k_idxs->ne[0] == params.ubatch.n_tokens;
@@ -570,12 +567,10 @@ void llm_graph_input_attn_k_dsa::set_input(const llama_ubatch * ubatch) {
 }
 
 bool llm_graph_input_attn_k_dsa::can_reuse(const llm_graph_params & params) {
-    mctx = static_cast<const llama_kv_cache_dsa_context *>(params.mctx);
+    const auto * mctx = static_cast<const llama_kv_cache_dsa_context *>(params.mctx);
 
-    return can_reuse_impl(params);
-}
+    this->mctx = mctx;
 
-bool llm_graph_input_attn_k_dsa::can_reuse_impl(const llm_graph_params & params) {
     bool res = true;
 
     res &= self_k_idxs_mla->ne[0] == params.ubatch.n_tokens;
@@ -587,25 +582,6 @@ bool llm_graph_input_attn_k_dsa::can_reuse_impl(const llm_graph_params & params)
     return res;
 }
 
-void llm_graph_input_attn_k_dsa_iswa::set_input(const llama_ubatch * ubatch) {
-    inp_dsa->set_input(ubatch);
-    inp_swa->set_input(ubatch);
-}
-
-bool llm_graph_input_attn_k_dsa_iswa::can_reuse(const llm_graph_params & params) {
-    mctx = static_cast<const llama_kv_cache_dsa_iswa_context *>(params.mctx);
-
-    inp_dsa->mctx = mctx->get_dsa();
-    inp_swa->mctx = mctx->get_swa();
-
-    bool res = true;
-
-    res &= inp_dsa->can_reuse_impl(params);
-    res &= inp_swa->can_reuse_impl(params);
-
-    return res;
-}
-
 void llm_graph_input_attn_kv_iswa::set_input(const llama_ubatch * ubatch) {
     // base tensors may not be allocated if there are no non-SWA attention layers
     if (self_k_idxs && self_k_idxs->buffer) {
@@ -3234,12 +3210,8 @@ ggml_tensor * llm_graph_context::build_attn(
     return cur;
 }
 
-static std::unique_ptr<llm_graph_input_attn_k_dsa> build_attn_inp_k_dsa_impl(
-           ggml_context * ctx0,
-     const llama_ubatch & ubatch,
-    const llama_hparams & hparams,
-    const llama_cparams & cparams,
-    const llama_kv_cache_dsa_context * mctx_cur) {
+llm_graph_input_attn_k_dsa * llm_graph_context::build_attn_inp_k_dsa() const {
+    const auto * mctx_cur = static_cast<const llama_kv_cache_dsa_context *>(mctx);
 
     auto inp = std::make_unique<llm_graph_input_attn_k_dsa>(hparams, cparams, mctx_cur);
 
@@ -3263,35 +3235,9 @@ static std::unique_ptr<llm_graph_input_attn_k_dsa> build_attn_inp_k_dsa_impl(
         inp->self_k_rot_lid = mctx_cur->get_lid()->build_input_k_rot(ctx0);
     }
 
-    return inp;
-}
-
-llm_graph_input_attn_k_dsa * llm_graph_context::build_attn_inp_k_dsa() const {
-    const auto * mctx_cur = static_cast<const llama_kv_cache_dsa_context *>(mctx);
-
-    auto inp = build_attn_inp_k_dsa_impl(ctx0, ubatch, hparams, cparams, mctx_cur);
-
     return (llm_graph_input_attn_k_dsa *) res->add_input(std::move(inp));
 }
 
-llm_graph_input_attn_k_dsa_iswa * llm_graph_context::build_attn_inp_k_dsa_iswa() const {
-    const auto * mctx_cur = static_cast<const llama_kv_cache_dsa_iswa_context *>(mctx);
-
-    auto inp_dsa = build_attn_inp_k_dsa_impl(ctx0, ubatch, hparams, cparams, mctx_cur->get_dsa());
-
-    // build_attn_inp_k_impl rejects SWA caches, so construct the input directly
-    auto inp_swa = std::make_unique<llm_graph_input_attn_k>(hparams, cparams, mctx_cur->get_swa());
-
-    inp_swa->self_k_idxs = mctx_cur->get_swa()->build_input_k_idxs(ctx0, ubatch);
-
-    inp_swa->self_kq_mask = build_attn_inp_kq_mask(ctx0, mctx_cur->get_swa(), ubatch, cparams);
-    inp_swa->self_kq_mask_cnv = inp_swa->self_kq_mask;
-
-    auto inp = std::make_unique<llm_graph_input_attn_k_dsa_iswa>(std::move(inp_dsa), std::move(inp_swa), mctx_cur);
-
-    return (llm_graph_input_attn_k_dsa_iswa *) res->add_input(std::move(inp));
-}
-
 llm_graph_input_attn_kv_msa * llm_graph_context::build_attn_inp_kv_msa(bool msa_enabled) const {
     const auto * mctx_cur = static_cast<const llama_kv_cache_msa_context *>(mctx);
 

src/llama-graph.h

@@ -23,7 +23,6 @@ struct llama_memory_context_i;
 
 class llama_kv_cache_context;
 class llama_kv_cache_dsa_context;
-class llama_kv_cache_dsa_iswa_context;
 class llama_kv_cache_msa_context;
 class llama_kv_cache_dsv4_raw_context;
 class llama_kv_cache_dsv4_context;
@@ -375,9 +374,6 @@ public:
 
     bool can_reuse(const llm_graph_params & params) override;
 
-    // like can_reuse, but does not re-bind mctx
-    bool can_reuse_impl(const llm_graph_params & params);
-
     ggml_tensor * get_k_idxs() const { return self_k_idxs; }
 
     ggml_tensor * get_kq_mask() const { return self_kq_mask_cnv; }
@@ -409,9 +405,6 @@ public:
 
     bool can_reuse(const llm_graph_params & params) override;
 
-    // like can_reuse, but does not re-bind mctx
-    bool can_reuse_impl(const llm_graph_params & params);
-
     ggml_tensor * get_k_idxs_mla() const { return self_k_idxs_mla; }
     ggml_tensor * get_k_idxs_lid() const { return self_k_idxs_lid; }
 
@@ -434,32 +427,6 @@ public:
     const llama_kv_cache_dsa_context * mctx;
 };
 
-// DSA input (full-attention layers + indexer) with K-only input for the SWA layers
-class llm_graph_input_attn_k_dsa_iswa : public llm_graph_input_i {
-public:
-    llm_graph_input_attn_k_dsa_iswa(
-            std::unique_ptr<llm_graph_input_attn_k_dsa> inp_dsa,
-            std::unique_ptr<llm_graph_input_attn_k>     inp_swa,
-            const llama_kv_cache_dsa_iswa_context *     mctx) :
-        inp_dsa(std::move(inp_dsa)),
-        inp_swa(std::move(inp_swa)),
-        mctx(mctx) {
-    }
-    ~llm_graph_input_attn_k_dsa_iswa() = default;
-
-    void set_input(const llama_ubatch * ubatch) override;
-
-    bool can_reuse(const llm_graph_params & params) override;
-
-    llm_graph_input_attn_k_dsa * get_dsa() const { return inp_dsa.get(); }
-    llm_graph_input_attn_k     * get_swa() const { return inp_swa.get(); }
-
-    std::unique_ptr<llm_graph_input_attn_k_dsa> inp_dsa;
-    std::unique_ptr<llm_graph_input_attn_k>     inp_swa;
-
-    const llama_kv_cache_dsa_iswa_context * mctx;
-};
-
 // standard K/V attention input against the base cache, plus destination indices for the indexer key cache
 class llm_graph_input_attn_kv_msa : public llm_graph_input_attn_kv {
 public:
@@ -1224,8 +1191,6 @@ struct llm_graph_context {
 
     llm_graph_input_attn_k_dsa * build_attn_inp_k_dsa() const;
 
-    llm_graph_input_attn_k_dsa_iswa * build_attn_inp_k_dsa_iswa() const;
-
     llm_graph_input_attn_kv_msa * build_attn_inp_kv_msa(bool msa_enabled) const;
 
     ggml_tensor * build_attn(

src/llama-hparams.h

@@ -101,11 +101,6 @@ struct llama_hparams {
     uint32_t n_group_used       = 0;
     uint32_t n_group_experts    = 0;
 
-    // MLA + SWA (i.e. dots3note)
-    uint32_t n_lora_kv_swa           = 0;
-    uint32_t n_embd_head_k_mla_swa   = 0;
-    uint32_t n_embd_head_v_mla_swa   = 0;
-
     float    expert_group_scale   = 0.05f;
     float    expert_weights_scale = 0.0f;
     bool     expert_weights_norm  = false;

src/llama-kv-cache-dsa-iswa.cpp

@@ -1,341 +0,0 @@
-#include "llama-kv-cache-dsa-iswa.h"
-
-#include "llama-impl.h"
-#include "llama-batch.h"
-#include "llama-model.h"
-
-#include <algorithm>
-#include <cassert>
-
-//
-// llama_kv_cache_dsa_iswa
-//
-
-llama_kv_cache_dsa_iswa::llama_kv_cache_dsa_iswa(
-        const llama_model & model,
-                ggml_type   type_k,
-                ggml_type   type_v,
-                     bool   v_trans,
-                     bool   offload,
-                     bool   swa_full,
-                     bool   unified,
-                 uint32_t   kv_size,
-                 uint32_t   n_seq_max,
-                 uint32_t   n_ubatch,
-                 uint32_t   n_pad,
-    const layer_filter_cb & filter_mla,
-    const layer_filter_cb & filter_lid,
-    const  layer_reuse_cb & reuse) : unified(unified) {
-
-    const auto & hparams = model.hparams;
-
-    // chain filters
-    const layer_filter_cb filter_dsa = [&](int32_t il) {
-        if (filter_mla && !filter_mla(il)) {
-            return false;
-        }
-
-        return !hparams.is_swa(il);
-    };
-
-    const layer_filter_cb filter_swa = [&](int32_t il) {
-        if (filter_mla && !filter_mla(il)) {
-            return false;
-        }
-
-        return hparams.is_swa(il);
-    };
-
-    const uint32_t size_dsa = kv_size;
-
-    // note: the SWA cache is always padded to 256 for performance
-    //       https://github.com/ggml-org/llama.cpp/issues/17037
-    uint32_t size_swa = GGML_PAD(std::min(size_dsa, hparams.n_swa*(unified ? n_seq_max : 1) + n_ubatch), 256);
-
-    // when using full-size SWA cache, we set the SWA cache size to be equal to the base cache size
-    if (swa_full) {
-        LLAMA_LOG_WARN("%s: using full-size SWA cache (ref: %s)\n",
-                __func__, "https://github.com/ggml-org/llama.cpp/pull/13194#issuecomment-2868343055");
-
-        size_swa = size_dsa;
-    }
-
-    LLAMA_LOG_INFO("%s: creating DSA KV cache, size = %u cells\n", __func__, size_dsa);
-
-    kv_dsa = std::make_unique<llama_kv_cache_dsa>(
-            model, type_k, type_v,
-            v_trans, offload, unified, size_dsa, n_seq_max, n_pad,
-            0, LLAMA_SWA_TYPE_NONE, filter_dsa, filter_lid, reuse);
-
-    LLAMA_LOG_INFO("%s: creating SWA KV cache, size = %u cells\n", __func__, size_swa);
-
-    kv_swa = std::make_unique<llama_kv_cache>(
-            model, hparams, type_k, type_v,
-            v_trans, offload, unified, size_swa, n_seq_max, n_pad,
-            hparams.n_swa, hparams.swa_type, nullptr, filter_swa, reuse, nullptr);
-}
-
-void llama_kv_cache_dsa_iswa::clear(bool data) {
-    kv_dsa->clear(data);
-    kv_swa->clear(data);
-}
-
-bool llama_kv_cache_dsa_iswa::seq_rm(llama_seq_id seq_id, llama_pos p0, llama_pos p1) {
-    bool res = true;
-
-    res = res & kv_dsa->seq_rm(seq_id, p0, p1);
-    res = res & kv_swa->seq_rm(seq_id, p0, p1);
-
-    return res;
-}
-
-void llama_kv_cache_dsa_iswa::seq_cp(llama_seq_id seq_id_src, llama_seq_id seq_id_dst, llama_pos p0, llama_pos p1) {
-    kv_dsa->seq_cp(seq_id_src, seq_id_dst, p0, p1);
-    kv_swa->seq_cp(seq_id_src, seq_id_dst, p0, p1);
-}
-
-void llama_kv_cache_dsa_iswa::seq_keep(llama_seq_id seq_id) {
-    kv_dsa->seq_keep(seq_id);
-    kv_swa->seq_keep(seq_id);
-}
-
-void llama_kv_cache_dsa_iswa::seq_add(llama_seq_id seq_id, llama_pos p0, llama_pos p1, llama_pos shift) {
-    kv_dsa->seq_add(seq_id, p0, p1, shift);
-    kv_swa->seq_add(seq_id, p0, p1, shift);
-}
-
-void llama_kv_cache_dsa_iswa::seq_div(llama_seq_id seq_id, llama_pos p0, llama_pos p1, int d) {
-    kv_dsa->seq_div(seq_id, p0, p1, d);
-    kv_swa->seq_div(seq_id, p0, p1, d);
-}
-
-llama_pos llama_kv_cache_dsa_iswa::seq_pos_min(llama_seq_id seq_id) const {
-    // the DSA cache is a superset of the SWA cache, so we can just check the SWA cache
-    return kv_swa->seq_pos_min(seq_id);
-}
-
-llama_pos llama_kv_cache_dsa_iswa::seq_pos_max(llama_seq_id seq_id) const {
-    return kv_swa->seq_pos_max(seq_id);
-}
-
-std::map<ggml_backend_buffer_type_t, size_t> llama_kv_cache_dsa_iswa::memory_breakdown() const {
-    std::map<ggml_backend_buffer_type_t, size_t> mb = kv_dsa->memory_breakdown();
-    for (const auto & buft_size : kv_swa->memory_breakdown()) {
-        mb[buft_size.first] += buft_size.second;
-    }
-    return mb;
-}
-
-llama_memory_context_ptr llama_kv_cache_dsa_iswa::init_batch(llama_batch_allocr & balloc, uint32_t n_ubatch, bool embd_all) {
-    GGML_UNUSED(embd_all);
-
-    // first try simple split
-    do {
-        if (!unified) {
-            // requires equal splits, so we skip the simple split
-            break;
-        }
-
-        balloc.split_reset();
-
-        std::vector<llama_ubatch> ubatches;
-        while (true) {
-            auto ubatch = balloc.split_simple(n_ubatch);
-
-            if (ubatch.n_tokens == 0) {
-                break;
-            }
-
-            ubatches.push_back(std::move(ubatch)); // NOLINT
-        }
-
-        if (balloc.get_n_used() < balloc.get_n_tokens()) {
-            // failed to find a suitable split
-            break;
-        }
-
-        auto sinfos_mla = kv_dsa->get_mla()->prepare(ubatches);
-        if (sinfos_mla.empty()) {
-            break;
-        }
-
-        auto sinfos_lid = kv_dsa->get_lid()->prepare(ubatches);
-        if (sinfos_lid.empty()) {
-            break;
-        }
-
-        auto sinfos_swa = kv_swa->prepare(ubatches);
-        if (sinfos_swa.empty()) {
-            break;
-        }
-
-        assert(sinfos_mla.size() == sinfos_swa.size());
-
-        return std::make_unique<llama_kv_cache_dsa_iswa_context>(
-                this, std::move(sinfos_mla), std::move(sinfos_lid), std::move(sinfos_swa), std::move(ubatches));
-    } while (false);
-
-    // if it fails, try equal split
-    do {
-        balloc.split_reset();
-
-        std::vector<llama_ubatch> ubatches;
-        while (true) {
-            auto ubatch = balloc.split_equal(n_ubatch, !unified, 0);
-
-            if (ubatch.n_tokens == 0) {
-                break;
-            }
-
-            ubatches.push_back(std::move(ubatch)); // NOLINT
-        }
-
-        if (balloc.get_n_used() < balloc.get_n_tokens()) {
-            // failed to find a suitable split
-            break;
-        }
-
-        auto sinfos_mla = kv_dsa->get_mla()->prepare(ubatches);
-        if (sinfos_mla.empty()) {
-            break;
-        }
-
-        auto sinfos_lid = kv_dsa->get_lid()->prepare(ubatches);
-        if (sinfos_lid.empty()) {
-            break;
-        }
-
-        auto sinfos_swa = kv_swa->prepare(ubatches);
-        if (sinfos_swa.empty()) {
-            break;
-        }
-
-        assert(sinfos_mla.size() == sinfos_swa.size());
-
-        return std::make_unique<llama_kv_cache_dsa_iswa_context>(
-                this, std::move(sinfos_mla), std::move(sinfos_lid), std::move(sinfos_swa), std::move(ubatches));
-    } while (false);
-
-    return std::make_unique<llama_kv_cache_dsa_iswa_context>(LLAMA_MEMORY_STATUS_FAILED_PREPARE);
-}
-
-llama_memory_context_ptr llama_kv_cache_dsa_iswa::init_full() {
-    return std::make_unique<llama_kv_cache_dsa_iswa_context>(this);
-}
-
-llama_memory_context_ptr llama_kv_cache_dsa_iswa::init_update(llama_context * lctx, bool optimize) {
-    return std::make_unique<llama_kv_cache_dsa_iswa_context>(this, lctx, optimize);
-}
-
-bool llama_kv_cache_dsa_iswa::get_can_shift() const {
-    return kv_dsa->get_can_shift() &&
-           kv_swa->get_can_shift() &&
-           kv_dsa->get_mla()->get_size() == kv_swa->get_size();
-}
-
-void llama_kv_cache_dsa_iswa::state_write(llama_io_write_i & io, llama_seq_id seq_id, llama_state_seq_flags flags) const {
-    if ((flags & LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY) == 0) {
-        kv_dsa->state_write(io, seq_id, flags);
-    }
-
-    kv_swa->state_write(io, seq_id, flags);
-}
-
-void llama_kv_cache_dsa_iswa::state_read(llama_io_read_i & io, llama_seq_id seq_id, llama_state_seq_flags flags) {
-    if ((flags & LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY) == 0) {
-        kv_dsa->state_read(io, seq_id, flags);
-    }
-
-    kv_swa->state_read(io, seq_id, flags);
-}
-
-llama_kv_cache_dsa * llama_kv_cache_dsa_iswa::get_dsa() const {
-    return kv_dsa.get();
-}
-
-llama_kv_cache * llama_kv_cache_dsa_iswa::get_swa() const {
-    return kv_swa.get();
-}
-
-//
-// llama_kv_cache_dsa_iswa_context
-//
-
-llama_kv_cache_dsa_iswa_context::llama_kv_cache_dsa_iswa_context(llama_memory_status status) : status(status) {}
-
-llama_kv_cache_dsa_iswa_context::llama_kv_cache_dsa_iswa_context(
-        llama_kv_cache_dsa_iswa * kv) :
-    ctx_dsa(kv->get_dsa()->init_full()),
-    ctx_swa(kv->get_swa()->init_full()),
-    status(llama_memory_status_combine(ctx_dsa->get_status(), ctx_swa->get_status())) {
-}
-
-llama_kv_cache_dsa_iswa_context::llama_kv_cache_dsa_iswa_context(
-        llama_kv_cache_dsa_iswa * kv,
-        llama_context * lctx,
-        bool optimize) :
-    ctx_dsa(kv->get_dsa()->init_update(lctx, optimize)),
-    ctx_swa(kv->get_swa()->init_update(lctx, optimize)),
-    status(llama_memory_status_combine(ctx_dsa->get_status(), ctx_swa->get_status())) {
-}
-
-llama_kv_cache_dsa_iswa_context::llama_kv_cache_dsa_iswa_context(
-        llama_kv_cache_dsa_iswa * kv,
-        slot_info_vec_t sinfos_mla,
-        slot_info_vec_t sinfos_lid,
-        slot_info_vec_t sinfos_swa,
-        std::vector<llama_ubatch> ubatches) :
-    ubatches(std::move(ubatches)),
-    // note: here we copy the ubatches. not sure if this is ideal
-    ctx_dsa(new llama_kv_cache_dsa_context(kv->get_dsa(), std::move(sinfos_mla), std::move(sinfos_lid), this->ubatches)),
-    ctx_swa(new llama_kv_cache_context(kv->get_swa(), std::move(sinfos_swa), this->ubatches)),
-    status(llama_memory_status_combine(ctx_dsa->get_status(), ctx_swa->get_status())) {
-}
-
-llama_kv_cache_dsa_iswa_context:: ~llama_kv_cache_dsa_iswa_context() = default;
-
-bool llama_kv_cache_dsa_iswa_context::next() {
-    assert(status == LLAMA_MEMORY_STATUS_SUCCESS);
-
-    ctx_dsa->next();
-    ctx_swa->next();
-
-    if (++i_next >= ubatches.size()) {
-        return false;
-    }
-
-    return true;
-}
-
-bool llama_kv_cache_dsa_iswa_context::apply() {
-    assert(!llama_memory_status_is_fail(status));
-
-    bool res = true;
-
-    res = res & ctx_dsa->apply();
-    res = res & ctx_swa->apply();
-
-    return res;
-}
-
-llama_memory_status llama_kv_cache_dsa_iswa_context::get_status() const {
-    return status;
-}
-
-const llama_ubatch & llama_kv_cache_dsa_iswa_context::get_ubatch() const {
-    assert(status == LLAMA_MEMORY_STATUS_SUCCESS);
-
-    return ubatches[i_next];
-}
-
-const llama_kv_cache_dsa_context * llama_kv_cache_dsa_iswa_context::get_dsa() const {
-    assert(status == LLAMA_MEMORY_STATUS_SUCCESS);
-
-    return static_cast<const llama_kv_cache_dsa_context *>(ctx_dsa.get());
-}
-
-const llama_kv_cache_context * llama_kv_cache_dsa_iswa_context::get_swa() const {
-    assert(status == LLAMA_MEMORY_STATUS_SUCCESS);
-
-    return static_cast<const llama_kv_cache_context *>(ctx_swa.get());
-}

src/llama-kv-cache-dsa-iswa.h

@@ -1,134 +0,0 @@
-#pragma once
-
-#include "llama-kv-cache-dsa.h"
-
-#include <vector>
-
-//
-// llama_kv_cache_dsa_iswa
-//
-
-// utilizes two child memories: llama_kv_cache_dsa for the full-attention (DSA) layers and llama_kv_cache for the SWA layers
-
-class llama_kv_cache_dsa_iswa : public llama_memory_i {
-public:
-    llama_kv_cache_dsa_iswa(
-            const llama_model & model,
-                    ggml_type   type_k,
-                    ggml_type   type_v,
-                         bool   v_trans,
-                         bool   offload,
-                         bool   swa_full,
-                         bool   unified,
-                     uint32_t   kv_size,
-                     uint32_t   n_seq_max,
-                     uint32_t   n_ubatch,
-                     uint32_t   n_pad,
-        const layer_filter_cb & filter_mla,
-        const layer_filter_cb & filter_lid,
-        const  layer_reuse_cb & reuse);
-
-    ~llama_kv_cache_dsa_iswa() = default;
-
-    //
-    // llama_memory_i
-    //
-
-    llama_memory_context_ptr init_batch(
-            llama_batch_allocr & balloc,
-            uint32_t n_ubatch,
-            bool embd_all) override;
-
-    llama_memory_context_ptr init_full() override;
-
-    llama_memory_context_ptr init_update(llama_context * lctx, bool optimize) override;
-
-    bool get_can_shift() const override;
-
-    void clear(bool data) override;
-
-    bool seq_rm  (llama_seq_id seq_id,                              llama_pos p0, llama_pos p1) override;
-    void seq_cp  (llama_seq_id seq_id_src, llama_seq_id seq_id_dst, llama_pos p0, llama_pos p1) override;
-    void seq_keep(llama_seq_id seq_id)                                                          override;
-    void seq_add (llama_seq_id seq_id,                              llama_pos p0, llama_pos p1, llama_pos shift) override;
-    void seq_div (llama_seq_id seq_id,                              llama_pos p0, llama_pos p1, int d) override;
-
-    llama_pos seq_pos_min(llama_seq_id seq_id) const override;
-    llama_pos seq_pos_max(llama_seq_id seq_id) const override;
-
-    std::map<ggml_backend_buffer_type_t, size_t> memory_breakdown() const override;
-
-    // state write/load
-
-    void state_write(llama_io_write_i & io, llama_seq_id seq_id = -1, llama_state_seq_flags flags = 0) const override;
-    void state_read (llama_io_read_i  & io, llama_seq_id seq_id = -1, llama_state_seq_flags flags = 0) override;
-
-    //
-    // llama_kv_cache_dsa_iswa specific API
-    //
-
-    llama_kv_cache_dsa * get_dsa() const;
-    llama_kv_cache     * get_swa() const;
-
-private:
-    const bool unified;
-
-    std::unique_ptr<llama_kv_cache_dsa> kv_dsa;
-    std::unique_ptr<llama_kv_cache>     kv_swa;
-};
-
-class llama_kv_cache_dsa_iswa_context : public llama_memory_context_i {
-public:
-    using slot_info_vec_t = llama_kv_cache::slot_info_vec_t;
-
-    // used for errors
-    llama_kv_cache_dsa_iswa_context(llama_memory_status status);
-
-    // used to create a full-cache context
-    llama_kv_cache_dsa_iswa_context(
-            llama_kv_cache_dsa_iswa * kv);
-
-    // used to create an update context
-    llama_kv_cache_dsa_iswa_context(
-            llama_kv_cache_dsa_iswa * kv,
-            llama_context * lctx,
-            bool optimize);
-
-    // used to create a batch processing context from a batch
-    llama_kv_cache_dsa_iswa_context(
-            llama_kv_cache_dsa_iswa * kv,
-            slot_info_vec_t sinfos_mla,
-            slot_info_vec_t sinfos_lid,
-            slot_info_vec_t sinfos_swa,
-            std::vector<llama_ubatch> ubatches);
-
-    virtual ~llama_kv_cache_dsa_iswa_context();
-
-    //
-    // llama_memory_context_i
-    //
-
-    bool next()  override;
-    bool apply() override;
-
-    llama_memory_status  get_status() const override;
-    const llama_ubatch & get_ubatch() const override;
-
-    //
-    // llama_kv_cache_dsa_iswa_context specific API
-    //
-
-    const llama_kv_cache_dsa_context * get_dsa() const;
-    const llama_kv_cache_context     * get_swa() const;
-
-private:
-    // the index of the next ubatch to process
-    size_t i_next = 0;
-
-    std::vector<llama_ubatch> ubatches;
-
-    const llama_memory_context_ptr ctx_dsa;
-    const llama_memory_context_ptr ctx_swa;
-
-    const llama_memory_status status;
-};

src/llama-kv-cache.cpp

@@ -323,8 +323,7 @@ llama_kv_cache::llama_kv_cache(
             hparams.n_embd_head_k() % 64 == 0;
 
         // always create Hadamard rotation tensors for DeepSeek lightning indexers
-        if ((model.arch == LLM_ARCH_DEEPSEEK32 || model.arch == LLM_ARCH_DEEPSEEK4 ||
-                model.arch == LLM_ARCH_GLM_DSA || model.arch == LLM_ARCH_DOTS3NOTE) &&
+        if ((model.arch == LLM_ARCH_DEEPSEEK32 || model.arch == LLM_ARCH_DEEPSEEK4 || model.arch == LLM_ARCH_GLM_DSA) &&
                 hparams.n_embd_head_k_full == hparams.indexer_head_size) {
             attn_rot_k = true;
         }

src/llama-model-saver.cpp

@@ -31,7 +31,6 @@ bool llama_model_saver_supports_arch(llm_arch arch) {
         case LLM_ARCH_MELLUM:
         case LLM_ARCH_LAGUNA:
         case LLM_ARCH_GRANITE_SWA:
-        case LLM_ARCH_DOTS3NOTE: // TODO: need to handle SWA pattern and MLA+SWA config
             return false;
         default:
             return true;

src/llama-model.cpp

@@ -11,7 +11,6 @@
 #include "llama-kv-cache.h"
 #include "llama-kv-cache-iswa.h"
 #include "llama-kv-cache-dsa.h"
-#include "llama-kv-cache-dsa-iswa.h"
 #include "llama-kv-cache-msa.h"
 #include "llama-kv-cache-dsv4.h"
 #include "llama-memory-hybrid.h"
@@ -195,8 +194,6 @@ static llama_model * llama_model_mapping(llm_arch arch, const llama_model_params
             return new llama_model_deepseek2ocr(params);
         case LLM_ARCH_DEEPSEEK32:
             return new llama_model_deepseek32(params);
-        case LLM_ARCH_DOTS3NOTE:
-            return new llama_model_dots3note(params);
         case LLM_ARCH_DEEPSEEK4:
             return new llama_model_deepseek4(params);
         case LLM_ARCH_GLM_DSA:
@@ -490,10 +487,6 @@ struct ggml_backend_meta_split_state llama_meta_device_get_split_state(const str
             return get_tensor_config_impl(GGML_BACKEND_SPLIT_AXIS_1, "ssm_out.weight");
         }
         if (std::regex_match(tensor_name, pattern_r_cache) || std::regex_match(tensor_name, pattern_s_cache)) {
-            if (ud->model->arch == LLM_ARCH_LFM2 || ud->model->arch == LLM_ARCH_LFM2MOE) {
-                // the LFM2 shortconv block runs fully mirrored, so its conv state must be mirrored too
-                return get_tensor_config_impl(GGML_BACKEND_SPLIT_AXIS_MIRRORED, "");
-            }
             return get_tensor_config_impl(GGML_BACKEND_SPLIT_AXIS_0, "ssm_out.weight");
         }
         if (std::regex_match(tensor_name, pattern_ssm_conv1d)) {
@@ -854,7 +847,6 @@ const char * llm_type_name(llm_type type) {
         case LLM_TYPE_230B_A10B:     return "230B.A10B";
         case LLM_TYPE_428B_A23B:     return "428B.A23B";
         case LLM_TYPE_235B_A22B:     return "235B.A22B";
-        case LLM_TYPE_288B_A19B:     return "288B.A19B";
         case LLM_TYPE_300B_A47B:     return "300B.A47B";
         case LLM_TYPE_310B_A15B:     return "310B.A15B";
         case LLM_TYPE_355B_A32B:     return "355B.A32B";
@@ -1928,9 +1920,7 @@ void llama_model::print_info() const {
             LLAMA_LOG_INFO("%s: expert_weights_scale  = %.1f\n",   __func__, hparams.expert_weights_scale);
         }
 
-        if (arch == LLM_ARCH_DEEPSEEK2 || arch == LLM_ARCH_DEEPSEEK2OCR ||
-                arch == LLM_ARCH_DEEPSEEK32 || arch == LLM_ARCH_GLM_DSA ||
-                arch == LLM_ARCH_DOTS3NOTE || arch == LLM_ARCH_MISTRAL4) {
+        if (arch == LLM_ARCH_DEEPSEEK2 || arch == LLM_ARCH_DEEPSEEK2OCR || arch == LLM_ARCH_DEEPSEEK32 || arch == LLM_ARCH_GLM_DSA || arch == LLM_ARCH_MISTRAL4) {
             LLAMA_LOG_INFO("%s: n_layer_dense_lead    = %d\n",     __func__, hparams.n_layer_dense_lead);
             LLAMA_LOG_INFO("%s: n_lora_q              = %d\n",     __func__, hparams.n_lora_q);
             LLAMA_LOG_INFO("%s: n_lora_kv             = %d\n",     __func__, hparams.n_lora_kv);
@@ -2199,57 +2189,6 @@ llama_memory_i * llama_model::create_memory(const llama_memory_params & params,
                             nullptr);
                 }
             } break;
-        case LLM_ARCH_DOTS3NOTE:
-            {
-                GGML_ASSERT(hparams.swa_type != LLAMA_SWA_TYPE_NONE);
-
-                if (params.ctx_type == LLAMA_CONTEXT_TYPE_MTP && hparams.n_layer_nextn > 0) {
-                    // MTP draft context: plain attention KV cache holding only the nextn layer
-                    llama_kv_cache::layer_filter_cb filter =
-                        [&](uint32_t il) { return il >= hparams.n_layer(); };
-
-                    res = new llama_kv_cache(
-                            *this,
-                            hparams,
-                            params.type_k,
-                            params.type_v,
-                            !cparams.flash_attn,
-                            cparams.offload_kqv,
-                            cparams.kv_unified,
-                            cparams.n_ctx_seq,
-                            cparams.n_seq_max,
-                            1,
-                            hparams.n_swa,
-                            hparams.swa_type,
-                            nullptr,
-                            filter,
-                            nullptr,
-                            nullptr);
-                } else {
-                    // main context: DSA cache for the trunk full-attention layers plus a window-sized SWA cache
-                    llama_kv_cache::layer_filter_cb filter_mla = nullptr;
-                    if (hparams.n_layer_nextn > 0) {
-                        filter_mla = [&](uint32_t il) { return il < hparams.n_layer(); };
-                    }
-                    llama_kv_cache::layer_filter_cb filter_lid = [&](uint32_t il) { return il < hparams.n_layer() && hparams.is_indexer_full(il); };
-
-                    res = new llama_kv_cache_dsa_iswa(
-                            *this,
-                            params.type_k,
-                            params.type_v,
-                            !cparams.flash_attn,
-                            cparams.offload_kqv,
-                            params.swa_full,
-                            cparams.kv_unified,
-                            cparams.n_ctx_seq,
-                            cparams.n_seq_max,
-                            cparams.n_ubatch,
-                            1,
-                            filter_mla,
-                            filter_lid,
-                            nullptr);
-                }
-            } break;
         case LLM_ARCH_DEEPSEEK4:
             {
                 GGML_ASSERT(hparams.swa_type != LLAMA_SWA_TYPE_NONE);
@@ -2718,7 +2657,6 @@ llama_rope_type llama_model_rope_type(const llama_model * model) {
         case LLM_ARCH_LLAMA_EMBED:
         case LLM_ARCH_MAINCODER:
         case LLM_ARCH_GLM_DSA:
-        case LLM_ARCH_DOTS3NOTE:
         case LLM_ARCH_NANBEIGE:
         case LLM_ARCH_POCKETTTS:
             return LLAMA_ROPE_TYPE_NORM;

src/llama-model.h

@@ -140,7 +140,6 @@ enum llm_type {
     LLM_TYPE_230B_A10B, // Minimax M2
     LLM_TYPE_428B_A23B, // Minimax M3
     LLM_TYPE_235B_A22B,
-    LLM_TYPE_288B_A19B, // dots3-note
     LLM_TYPE_300B_A47B, // Ernie MoE big
     LLM_TYPE_310B_A15B, // /MiMo-V2-Flash
     LLM_TYPE_355B_A32B, // GLM-4.5

src/models/bailingmoe3.cpp

@@ -1,8 +1,6 @@
 #include "models.h"
 #include "llama-memory-recurrent.h"
 
-#include <algorithm>
-
 void llama_model_bailingmoe3::load_arch_hparams(llama_model_loader & ml) {
     ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS,      hparams.f_norm_rms_eps);
     ml.get_key(LLM_KV_ATTENTION_KEY_LENGTH_MLA,         hparams.n_embd_head_k_mla_impl);
@@ -181,9 +179,7 @@ static ggml_tensor * bailingmoe3_causal_conv1d(
         int64_t n_seq_tokens,
         int64_t n_seqs,
         int64_t n_tokens,
-        int64_t cache_head,
-        uint32_t mem_size,
-        uint32_t n_rs_seq) {
+        int64_t cache_head) {
     const int64_t d_inner = head_dim * n_head;
     const int64_t conv_state_size = (d_conv - 1) * d_inner;
     const int64_t total_state_size = 3 * conv_state_size;
@@ -197,18 +193,13 @@ static ggml_tensor * bailingmoe3_causal_conv1d(
     x_proj = ggml_reshape_3d(ctx0, x_proj, d_inner, n_seq_tokens, n_seqs);
     ggml_tensor * conv_x = ggml_concat(ctx0, conv_state, ggml_transpose(ctx0, x_proj), 0);
 
-    const int64_t K = (int64_t) n_rs_seq + 1;
-    const int64_t n_written = std::min<int64_t>(n_seq_tokens, K);
-
-    for (int64_t slot = 0; slot < n_written; ++slot) {
-        ggml_tensor * conv_snap = ggml_view_3d(ctx0, conv_x, d_conv - 1, d_inner, n_seqs,
-                conv_x->nb[1], conv_x->nb[2], (conv_x->ne[0] - (d_conv - 1) - slot) * conv_x->nb[0]);
-        ggml_build_forward_expand(gf, ggml_cpy(ctx0, conv_snap,
-                ggml_view_3d(ctx0, conv_states_all, d_conv - 1, d_inner, n_seqs,
-                    (d_conv - 1) * ggml_element_size(conv_states_all),
-                    total_state_size * ggml_element_size(conv_states_all),
-                    ((slot * mem_size + cache_head) * total_state_size + qkv * conv_state_size) * ggml_element_size(conv_states_all))));
-    }
+    ggml_tensor * last_conv_x = ggml_view_3d(ctx0, conv_x, d_conv - 1, d_inner, n_seqs,
+            conv_x->nb[1], conv_x->nb[2], n_seq_tokens * conv_x->nb[0]);
+    ggml_build_forward_expand(gf, ggml_cpy(ctx0, last_conv_x,
+            ggml_view_3d(ctx0, conv_states_all, d_conv - 1, d_inner, n_seqs,
+                (d_conv - 1) * ggml_element_size(conv_states_all),
+                total_state_size * ggml_element_size(conv_states_all),
+                (cache_head * total_state_size + qkv * conv_state_size) * ggml_element_size(conv_states_all))));
 
     ggml_tensor * conv_weight = ggml_reshape_2d(ctx0, conv_w, d_conv, d_inner);
     ggml_tensor * out = ggml_ssm_conv(ctx0, conv_x, conv_weight);
@@ -246,8 +237,6 @@ llama_model_bailingmoe3::graph::graph(const llama_model & model, const llm_graph
     GGML_ASSERT(ubatch.n_tokens == n_seq_tokens * n_seqs);
 
     for (int il = 0; il < n_layer; ++il) {
-        res->t_layer_inp[il] = inpL;
-
         const auto & layer = model.layers[il];
         ggml_tensor * inpSA = inpL;
         ggml_tensor * cur = build_norm(inpL, layer.attn_norm, nullptr, LLM_NORM_RMS, il);
@@ -256,19 +245,18 @@ llama_model_bailingmoe3::graph::graph(const llama_model & model, const llm_graph
         if (hparams.is_recr(il)) {
             const auto * mctx_cur = inp_rs->mctx;
             const auto cache_head = mctx_cur->get_head();
-            const auto mem_size = mctx_cur->get_size();
             ggml_tensor * conv_states_all = mctx_cur->get_r_l(il);
             ggml_tensor * conv_state_all = build_rs(inp_rs, conv_states_all, hparams.n_embd_r(), n_seqs);
 
             ggml_tensor * q = bailingmoe3_causal_conv1d(
                     gf, ctx0, conv_states_all, conv_state_all, 0, cur, layer.wq, layer.ssm_q_conv,
-                    d_conv, head_dim, n_head, n_seq_tokens, n_seqs, n_tokens, cache_head, mem_size, cparams.n_rs_seq);
+                    d_conv, head_dim, n_head, n_seq_tokens, n_seqs, n_tokens, cache_head);
             ggml_tensor * k = bailingmoe3_causal_conv1d(
                     gf, ctx0, conv_states_all, conv_state_all, 1, cur, layer.wk, layer.ssm_k_conv,
-                    d_conv, head_dim, n_head, n_seq_tokens, n_seqs, n_tokens, cache_head, mem_size, cparams.n_rs_seq);
+                    d_conv, head_dim, n_head, n_seq_tokens, n_seqs, n_tokens, cache_head);
             ggml_tensor * v = bailingmoe3_causal_conv1d(
                     gf, ctx0, conv_states_all, conv_state_all, 2, cur, layer.wv, layer.ssm_v_conv,
-                    d_conv, head_dim, n_head, n_seq_tokens, n_seqs, n_tokens, cache_head, mem_size, cparams.n_rs_seq);
+                    d_conv, head_dim, n_head, n_seq_tokens, n_seqs, n_tokens, cache_head);
 
             ggml_tensor * gate = ggml_mul_mat(ctx0, layer.ssm_f_a, cur);
             gate = ggml_add(ctx0, gate, layer.ssm_dt_b);
@@ -288,8 +276,11 @@ llama_model_bailingmoe3::graph::graph(const llama_model & model, const llm_graph
             ggml_tensor * state = build_rs(inp_rs, states_all, hparams.n_embd_s(), n_seqs);
             state = ggml_reshape_4d(ctx0, state, head_dim, head_dim, n_head, n_seqs);
 
-            ggml_tensor * out = ggml_cont(ctx0, build_recurrent_attn(
-                    inp_rs, states_all, q, k, v, gate, beta, state, il));
+            auto result = build_delta_net(q, k, v, gate, beta, state, il);
+            ggml_tensor * out = ggml_cont(ctx0, result.first);
+            ggml_build_forward_expand(gf, ggml_cpy(ctx0, result.second,
+                    ggml_view_1d(ctx0, states_all, hparams.n_embd_s() * n_seqs,
+                        cache_head * hparams.n_embd_s() * ggml_element_size(states_all))));
 
             ggml_tensor * out_gate = ggml_mul_mat(ctx0, layer.ssm_g_a, cur);
             out_gate = ggml_reshape_3d(ctx0, out_gate, head_dim, n_head, n_tokens);

src/models/deepseek2.cpp

@@ -524,9 +524,17 @@ llama_model_deepseek2::graph::graph(const llama_model & model, const llm_graph_p
                 q = ggml_mul_mat(ctx0, model.layers[il].wq, cur);
                 cb(q, "q", il);
             }
-            // {n_embd_head_k, n_head, n_tokens}
-            q = ggml_reshape_3d(ctx0, q, n_embd_head_k, n_head, n_tokens);
-            cb(q, "q", il);
+            // split into {n_embd_head_qk_nope, n_head, n_tokens}
+            ggml_tensor * q_nope =
+                ggml_view_3d(ctx0, q, n_embd_head_qk_nope, n_head, n_tokens, ggml_row_size(q->type, n_embd_head_k),
+                             ggml_row_size(q->type, n_embd_head_k) * n_head, 0);
+            cb(q_nope, "q_nope", il);
+
+            // and {n_embd_head_qk_rope, n_head, n_tokens}
+            ggml_tensor * q_pe = ggml_view_3d(
+                ctx0, q, n_embd_head_qk_rope, n_head, n_tokens, ggml_row_size(q->type, n_embd_head_k),
+                ggml_row_size(q->type, n_embd_head_k) * n_head, ggml_row_size(q->type, n_embd_head_qk_nope));
+            cb(q_pe, "q_pe", il);
 
             ggml_tensor * kv_cmpr_pe = ggml_mul_mat(ctx0, model.layers[il].wkv_a_mqa, cur);
             cb(kv_cmpr_pe, "kv_cmpr_pe", il);
@@ -544,6 +552,10 @@ llama_model_deepseek2::graph::graph(const llama_model & model, const llm_graph_p
                                               ggml_row_size(kv_cmpr_pe->type, kv_lora_rank));
             cb(k_pe, "k_pe", il);
 
+            q_pe = ggml_rope_ext(ctx0, q_pe, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
+                                 ext_factor, attn_factor, beta_fast, beta_slow);
+            cb(q_pe, "q_pe", il);
+
             k_pe = ggml_rope_ext(ctx0, k_pe, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
                                  ext_factor, attn_factor, beta_fast, beta_slow);
             cb(k_pe, "k_pe", il);
@@ -552,20 +564,6 @@ llama_model_deepseek2::graph::graph(const llama_model & model, const llm_graph_p
             cb(kv_cmpr, "kv_cmpr", il);
 
             if (is_mla) {
-                // split into {n_embd_head_qk_nope, n_head, n_tokens}
-                ggml_tensor * q_nope = ggml_view_3d(ctx0, q, n_embd_head_qk_nope, n_head, n_tokens,
-                                                    q->nb[1], q->nb[2], 0);
-                cb(q_nope, "q_nope", il);
-
-                // and {n_embd_head_qk_rope, n_head, n_tokens}
-                ggml_tensor * q_pe = ggml_view_3d(ctx0, q, n_embd_head_qk_rope, n_head, n_tokens,
-                                                  q->nb[1], q->nb[2], ggml_row_size(q->type, n_embd_head_qk_nope));
-                cb(q_pe, "q_pe", il);
-
-                q_pe = ggml_rope_ext(ctx0, q_pe, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
-                                     ext_factor, attn_factor, beta_fast, beta_slow);
-                cb(q_pe, "q_pe", il);
-
                 // {n_embd_head_qk_nope, n_tokens, n_head}
                 q_nope = ggml_permute(ctx0, q_nope, 0, 2, 1, 3);
                 cb(q_nope, "q_nope_perm", il);
@@ -625,14 +623,10 @@ llama_model_deepseek2::graph::graph(const llama_model & model, const llm_graph_p
                 Vcur = ggml_cont(ctx0, Vcur);
                 cb(Vcur, "Vcur_cont", il);
 
-                // RoPE is applied to the trailing dims only
-                ggml_tensor * Qcur = ggml_rope_ext(ctx0, q, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig,
-                                                   freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow);
-                Qcur = ggml_rope_set_offset(Qcur, n_embd_head_qk_nope);
+                ggml_tensor * Qcur = ggml_concat(ctx0, q_nope, q_pe, 0);
                 cb(Qcur, "Qcur", il);
 
-                ggml_tensor * Kcur = ggml_concat(ctx0, k_nope,
-                        ggml_repeat_4d(ctx0, k_pe, n_embd_head_qk_rope, n_head, n_tokens, 1), 0);
+                ggml_tensor * Kcur = ggml_concat(ctx0, k_nope, ggml_repeat(ctx0, k_pe, q_pe), 0);
                 cb(Kcur, "Kcur", il);
 
                 if (inp_attn_scale) {

src/models/deepseek4.cpp

@@ -501,10 +501,21 @@ ggml_tensor * llama_model_deepseek4::graph::build_hca_compressed_kv_from_state(
     comp = build_norm(comp, norm, nullptr, LLM_NORM_RMS, il);
     cb(comp, name, il);
 
-    comp = ggml_rope_ext(ctx0, comp, comp_pos, nullptr, n_embd_head_rope, rope_type, n_ctx_orig,
+    ggml_tensor * comp_nope = ggml_view_3d(ctx0, comp, n_embd_head_nope, 1, n_blocks,
+            ggml_row_size(comp->type, n_embd_head),
+            ggml_row_size(comp->type, n_embd_head),
+            0);
+    ggml_tensor * comp_pe = ggml_view_3d(ctx0, comp, n_embd_head_rope, 1, n_blocks,
+            ggml_row_size(comp->type, n_embd_head),
+            ggml_row_size(comp->type, n_embd_head),
+            ggml_row_size(comp->type, n_embd_head_nope));
+
+    comp_pe = ggml_rope_ext(ctx0, comp_pe, comp_pos, nullptr, n_embd_head_rope, rope_type, n_ctx_orig,
             hparams.dsv4_compress_rope_base, freq_scale, ext_factor,
             dsv4_rope_attn_factor(freq_scale, ext_factor), beta_fast, beta_slow);
-    comp = ggml_rope_set_offset(comp, n_embd_head_nope);
+    cb(comp_pe, name, il);
+
+    comp = ggml_concat(ctx0, comp_nope, comp_pe, 0);
     cb(comp, name, il);
 
     return comp;
@@ -574,10 +585,21 @@ ggml_tensor * llama_model_deepseek4::graph::build_overlap_compressed_kv_from_sta
     comp = build_norm(comp, norm, nullptr, LLM_NORM_RMS, il);
     cb(comp, name, il);
 
-    comp = ggml_rope_ext(ctx0, comp, comp_pos, nullptr, n_embd_head_rope, rope_type, n_ctx_orig,
+    ggml_tensor * comp_nope = ggml_view_3d(ctx0, comp, n_embd_head_nope, 1, n_blocks,
+            ggml_row_size(comp->type, n_embd_head),
+            ggml_row_size(comp->type, n_embd_head),
+            0);
+    ggml_tensor * comp_pe = ggml_view_3d(ctx0, comp, n_embd_head_rope, 1, n_blocks,
+            ggml_row_size(comp->type, n_embd_head),
+            ggml_row_size(comp->type, n_embd_head),
+            ggml_row_size(comp->type, n_embd_head_nope));
+
+    comp_pe = ggml_rope_ext(ctx0, comp_pe, comp_pos, nullptr, n_embd_head_rope, rope_type, n_ctx_orig,
             hparams.dsv4_compress_rope_base, freq_scale, ext_factor,
             dsv4_rope_attn_factor(freq_scale, ext_factor), beta_fast, beta_slow);
-    comp = ggml_rope_set_offset(comp, n_embd_head_nope);
+    cb(comp_pe, name, il);
+
+    comp = ggml_concat(ctx0, comp_nope, comp_pe, 0);
     cb(comp, name, il);
 
     return comp;
@@ -606,12 +628,21 @@ ggml_tensor * llama_model_deepseek4::graph::build_lid_top_k(
     indexer_q = ggml_reshape_3d(ctx0, indexer_q, n_embd_indexer_head, n_indexer_head, nt);
     cb(indexer_q, "lid_q", il);
 
-    indexer_q = ggml_rope_ext(ctx0, indexer_q, inp_pos, nullptr, n_embd_indexer_head_rope,
+    ggml_tensor * indexer_q_nope = ggml_view_3d(ctx0, indexer_q, n_embd_indexer_head_nope, n_indexer_head, nt,
+            ggml_row_size(indexer_q->type, n_embd_indexer_head),
+            ggml_row_size(indexer_q->type, n_embd_indexer_head)*n_indexer_head,
+            0);
+    ggml_tensor * indexer_q_pe = ggml_view_3d(ctx0, indexer_q, n_embd_indexer_head_rope, n_indexer_head, nt,
+            ggml_row_size(indexer_q->type, n_embd_indexer_head),
+            ggml_row_size(indexer_q->type, n_embd_indexer_head)*n_indexer_head,
+            ggml_row_size(indexer_q->type, n_embd_indexer_head_nope));
+
+    indexer_q_pe = ggml_rope_ext(ctx0, indexer_q_pe, inp_pos, nullptr, n_embd_indexer_head_rope,
             rope_type, n_ctx_orig, hparams.dsv4_compress_rope_base, freq_scale,
             ext_factor, dsv4_rope_attn_factor(freq_scale, ext_factor), beta_fast, beta_slow);
-    indexer_q = ggml_rope_set_offset(indexer_q, n_embd_indexer_head_nope);
-    cb(indexer_q, "lid_q_rope", il);
+    cb(indexer_q_pe, "lid_q_pe", il);
 
+    indexer_q = ggml_concat(ctx0, indexer_q_nope, indexer_q_pe, 0);
     indexer_q = llama_mul_mat_hadamard(ctx0, indexer_q, inp_lid.k_rot);
     cb(indexer_q, "lid_q_rot", il);
 
@@ -914,9 +945,18 @@ ggml_tensor * llama_model_deepseek4::graph::build_attention_impl(
     q = ggml_rms_norm(ctx0, q, norm_rms_eps);
     cb(q, "q_norm", il);
 
-    q = ggml_rope_ext(ctx0, q, inp_pos, nullptr, n_embd_head_rope, rope_type, n_ctx_orig_l,
+    ggml_tensor * q_nope = ggml_view_3d(ctx0, q, n_embd_head_nope, n_head, nt,
+            ggml_row_size(q->type, n_embd_head),
+            ggml_row_size(q->type, n_embd_head)*n_head,
+            0);
+    ggml_tensor * q_pe = ggml_view_3d(ctx0, q, n_embd_head_rope, n_head, nt,
+            ggml_row_size(q->type, n_embd_head),
+            ggml_row_size(q->type, n_embd_head)*n_head,
+            ggml_row_size(q->type, n_embd_head_nope));
+    q_pe = ggml_rope_ext(ctx0, q_pe, inp_pos, nullptr, n_embd_head_rope, rope_type, n_ctx_orig_l,
             freq_base_l, freq_scale_l, ext_factor_l, attn_factor_l, beta_fast_l, beta_slow_l);
-    q = ggml_rope_set_offset(q, n_embd_head_nope);
+    cb(q_pe, "q_pe", il);
+    q = ggml_concat(ctx0, q_nope, q_pe, 0);
     cb(q, "q", il);
 
     ggml_tensor * kv = build_lora_mm(layer.wkv, cur);
@@ -924,9 +964,18 @@ ggml_tensor * llama_model_deepseek4::graph::build_attention_impl(
     kv = ggml_reshape_3d(ctx0, kv, n_embd_head, 1, nt);
     cb(kv, "kv_norm", il);
 
-    kv = ggml_rope_ext(ctx0, kv, inp_pos, nullptr, n_embd_head_rope, rope_type, n_ctx_orig_l,
+    ggml_tensor * kv_nope = ggml_view_3d(ctx0, kv, n_embd_head_nope, 1, nt,
+            ggml_row_size(kv->type, n_embd_head),
+            ggml_row_size(kv->type, n_embd_head),
+            0);
+    ggml_tensor * kv_pe = ggml_view_3d(ctx0, kv, n_embd_head_rope, 1, nt,
+            ggml_row_size(kv->type, n_embd_head),
+            ggml_row_size(kv->type, n_embd_head),
+            ggml_row_size(kv->type, n_embd_head_nope));
+    kv_pe = ggml_rope_ext(ctx0, kv_pe, inp_pos, nullptr, n_embd_head_rope, rope_type, n_ctx_orig_l,
             freq_base_l, freq_scale_l, ext_factor_l, attn_factor_l, beta_fast_l, beta_slow_l);
-    kv = ggml_rope_set_offset(kv, n_embd_head_nope);
+    cb(kv_pe, "kv_pe", il);
+    kv = ggml_concat(ctx0, kv_nope, kv_pe, 0);
     cb(kv, "kv", il);
 
     const int64_t ratio = hparams.dsv4_compress_ratios[il];
@@ -1196,9 +1245,17 @@ ggml_tensor * llama_model_deepseek4::graph::build_attention_impl(
     }
 
     out = ggml_reshape_3d(ctx0, out, n_embd_head, n_head, nt);
-    out = ggml_rope_ext_back(ctx0, out, inp_pos, nullptr, n_embd_head_rope, rope_type, n_ctx_orig_l,
+    ggml_tensor * out_nope = ggml_view_3d(ctx0, out, n_embd_head_nope, n_head, nt,
+            ggml_row_size(out->type, n_embd_head),
+            ggml_row_size(out->type, n_embd_head)*n_head,
+            0);
+    ggml_tensor * out_pe = ggml_view_3d(ctx0, out, n_embd_head_rope, n_head, nt,
+            ggml_row_size(out->type, n_embd_head),
+            ggml_row_size(out->type, n_embd_head)*n_head,
+            ggml_row_size(out->type, n_embd_head_nope));
+    out_pe = ggml_rope_ext_back(ctx0, out_pe, inp_pos, nullptr, n_embd_head_rope, rope_type, n_ctx_orig_l,
             freq_base_l, freq_scale_l, ext_factor_l, attn_factor_l, beta_fast_l, beta_slow_l);
-    out = ggml_rope_set_offset(out, n_embd_head_nope);
+    out = ggml_concat(ctx0, out_nope, out_pe, 0);
     cb(out, "attn_derope", il);
 
     out = ggml_reshape_3d(ctx0, out, o_group_dim, n_groups, nt);

src/models/dflash.cpp

@@ -591,9 +591,17 @@ llama_model_dflash::graph_dsv4::graph_dsv4(const llama_model & model, const llm_
             kv = build_norm(kv, layer.attn_kv_norm, nullptr, LLM_NORM_RMS, il);
             kv = ggml_reshape_3d(ctx0, kv, n_embd_head, 1, n_tokens);
 
-            kv = ggml_rope_ext(ctx0, kv, inp_pos, nullptr, n_embd_head_rope, rope_type, 0,
+            ggml_tensor * kv_nope = ggml_view_3d(ctx0, kv, n_embd_head_nope, 1, n_tokens,
+                    ggml_row_size(kv->type, n_embd_head),
+                    ggml_row_size(kv->type, n_embd_head),
+                    0);
+            ggml_tensor * kv_pe = ggml_view_3d(ctx0, kv, n_embd_head_rope, 1, n_tokens,
+                    ggml_row_size(kv->type, n_embd_head),
+                    ggml_row_size(kv->type, n_embd_head),
+                    ggml_row_size(kv->type, n_embd_head_nope));
+            kv_pe = ggml_rope_ext(ctx0, kv_pe, inp_pos, nullptr, n_embd_head_rope, rope_type, 0,
                     freq_base, 1.0f, 0.0f, 1.0f, 0.0f, 0.0f);
-            kv = ggml_rope_set_offset(kv, n_embd_head_nope);
+            kv = ggml_concat(ctx0, kv_nope, kv_pe, 0);
             cb(kv, "kv_injected", il);
 
             if (inp_attn->self_k_rot_swa) {

src/models/dots3note.cpp

@@ -1,480 +0,0 @@
-#include "models.h"
-
-#include "llama-kv-cache.h"
-#include "llama-kv-cache-dsa.h"
-
-// note: code adapted from deepseek32.cpp (DSA indexer + absorbed MLA) and step35.cpp (head-wise output gate)
-
-void llama_model_dots3note::load_arch_hparams(llama_model_loader & ml) {
-    ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);
-    hparams.f_norm_eps = 1e-6;  // eps for the indexer k_norm layer norm
-
-    // TODO: use MTP layer
-    ml.get_key(LLM_KV_NEXTN_PREDICT_LAYERS, hparams.n_layer_nextn, false);
-    GGML_ASSERT(hparams.n_layer_nextn < hparams.n_layer_all && "n_layer_nextn must be < n_layer_all");
-
-    // MoE parameters
-    ml.get_key(LLM_KV_EXPERT_SHARED_COUNT,        hparams.n_expert_shared);
-    ml.get_key(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp);
-    ml.get_key(LLM_KV_LEADING_DENSE_BLOCK_COUNT,  hparams.n_layer_dense_lead);
-    ml.get_key(LLM_KV_EXPERT_WEIGHTS_SCALE,       hparams.expert_weights_scale, false);
-    ml.get_key(LLM_KV_EXPERT_WEIGHTS_NORM,        hparams.expert_weights_norm, false);
-    ml.get_key(LLM_KV_EXPERT_GATING_FUNC,         hparams.expert_gating_func);
-
-    // MLA parameters of the full-attention layers
-    ml.get_key(LLM_KV_ATTENTION_Q_LORA_RANK,      hparams.n_lora_q);
-    ml.get_key(LLM_KV_ATTENTION_KV_LORA_RANK,     hparams.n_lora_kv);
-    ml.get_key(LLM_KV_ATTENTION_KEY_LENGTH_MLA,   hparams.n_embd_head_k_mla_impl);
-    ml.get_key(LLM_KV_ATTENTION_VALUE_LENGTH_MLA, hparams.n_embd_head_v_mla_impl);
-
-    // MLA parameters of the sliding-window layers
-    ml.get_key(LLM_KV_ATTENTION_KV_LORA_RANK_SWA,     hparams.n_lora_kv_swa);
-    ml.get_key(LLM_KV_ATTENTION_KEY_LENGTH_MLA_SWA,   hparams.n_embd_head_k_mla_swa);
-    ml.get_key(LLM_KV_ATTENTION_VALUE_LENGTH_MLA_SWA, hparams.n_embd_head_v_mla_swa);
-
-    hparams.swa_type = LLAMA_SWA_TYPE_STANDARD;
-    ml.get_key(LLM_KV_ATTENTION_SLIDING_WINDOW, hparams.n_swa);
-    ml.get_key(LLM_KV_ROPE_FREQ_BASE_SWA,       hparams.rope_freq_base_train_swa);
-    ml.get_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, hparams.is_swa_impl);
-
-    // DSA parameters
-    ml.get_key(LLM_KV_ATTENTION_INDEXER_HEAD_COUNT, hparams.indexer_n_head);
-    ml.get_key(LLM_KV_ATTENTION_INDEXER_KEY_LENGTH, hparams.indexer_head_size);
-    ml.get_key(LLM_KV_ATTENTION_INDEXER_TOP_K,      hparams.indexer_top_k);
-    ml.get_arr(LLM_KV_ATTENTION_INDEXER_TYPES, hparams.is_indexer_full_impl);
-
-    switch (hparams.n_layer()) {
-        case 46: type = LLM_TYPE_288B_A19B; break;
-        default: type = LLM_TYPE_UNKNOWN;
-    }
-}
-
-void llama_model_dots3note::load_arch_tensors(llama_model_loader & ml) {
-    LLAMA_LOAD_LOCALS;
-    GGML_UNUSED(ml);
-
-    if (!hparams.is_mla()) {
-        throw std::runtime_error("DOTS3NOTE architecture requires MLA");
-    }
-
-    const int64_t n_embd_head_qk_rope = hparams.n_rot();
-
-    const int64_t q_lora_rank     = hparams.n_lora_q;
-    const int64_t n_ff_exp        = hparams.n_ff_exp;
-    const int64_t n_expert_shared = hparams.n_expert_shared;
-
-    tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, 0);
-
-    output_norm = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd}, 0);
-    output      = create_tensor(tn(LLM_TENSOR_OUTPUT,      "weight"), {n_embd, n_vocab}, TENSOR_NOT_REQUIRED);
-    if (!output) {
-        output = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, TENSOR_DUPLICATED);
-    }
-
-    for (int i = 0; i < n_layer_all; ++i) {
-        auto & layer = layers[i];
-
-        const bool is_mtp = i >= n_layer;
-        // the NextN/MTP block uses the sliding-attention geometry
-        const bool is_swa = is_mtp || hparams.is_swa(i);
-
-        // MTP tensors are preserved in the GGUF but there is no MTP graph yet
-        const int flags = is_mtp ? TENSOR_SKIP | TENSOR_NOT_REQUIRED : 0;
-
-        const int64_t n_head_l = hparams.n_head(i);
-
-        const int64_t kv_lora_rank       = is_swa ? hparams.n_lora_kv_swa         : hparams.n_lora_kv;
-        const int64_t n_embd_head_k_mla  = is_swa ? hparams.n_embd_head_k_mla_swa : hparams.n_embd_head_k_mla();
-        const int64_t n_embd_head_v_mla  = is_swa ? hparams.n_embd_head_v_mla_swa : hparams.n_embd_head_v_mla();
-        const int64_t n_embd_head_qk_nope = n_embd_head_k_mla - n_embd_head_qk_rope;
-
-        layer.attn_norm      = create_tensor(tn(LLM_TENSOR_ATTN_NORM,      "weight", i), {n_embd}, flags);
-        layer.attn_q_a_norm  = create_tensor(tn(LLM_TENSOR_ATTN_Q_A_NORM,  "weight", i), {q_lora_rank}, flags);
-        layer.attn_kv_a_norm = create_tensor(tn(LLM_TENSOR_ATTN_KV_A_NORM, "weight", i), {kv_lora_rank}, flags);
-        // norm applied on the shared rope key before rope
-        layer.attn_k_norm    = create_tensor(tn(LLM_TENSOR_ATTN_K_NORM,    "weight", i), {n_embd_head_qk_rope}, flags);
-
-        layer.wq_a = create_tensor(tn(LLM_TENSOR_ATTN_Q_A, "weight", i), {n_embd, q_lora_rank}, flags);
-        layer.wq_b = create_tensor(tn(LLM_TENSOR_ATTN_Q_B, "weight", i), {q_lora_rank, n_head_l * n_embd_head_k_mla}, flags);
-
-        layer.wkv_a_mqa = create_tensor(tn(LLM_TENSOR_ATTN_KV_A_MQA, "weight", i), {n_embd, kv_lora_rank + n_embd_head_qk_rope}, flags);
-
-        layer.wk_b = create_tensor(tn(LLM_TENSOR_ATTN_K_B, "weight", i), {n_embd_head_qk_nope, kv_lora_rank, n_head_l}, flags);
-        layer.wv_b = create_tensor(tn(LLM_TENSOR_ATTN_V_B, "weight", i), {kv_lora_rank, n_embd_head_v_mla, n_head_l}, flags);
-
-        layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_head_l * n_embd_head_v_mla, n_embd}, flags);
-
-        // head-wise sigmoid output gate
-        layer.wqkv_gate = create_tensor(tn(LLM_TENSOR_ATTN_GATE, "weight", i), {n_embd, n_head_l}, flags);
-
-        layer.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd}, flags);
-
-        // DSA indexer
-        if (!is_mtp && hparams.is_indexer_full(i)) {
-            layer.indexer_k_norm   = create_tensor(tn(LLM_TENSOR_INDEXER_K_NORM,   "weight", i), {hparams.indexer_head_size}, flags);
-            layer.indexer_k_norm_b = create_tensor(tn(LLM_TENSOR_INDEXER_K_NORM,   "bias",   i), {hparams.indexer_head_size}, flags);
-            layer.indexer_proj     = create_tensor(tn(LLM_TENSOR_INDEXER_PROJ,     "weight", i), {n_embd, hparams.indexer_n_head}, flags);
-            layer.indexer_attn_k   = create_tensor(tn(LLM_TENSOR_INDEXER_ATTN_K,   "weight", i), {n_embd, hparams.indexer_head_size}, flags);
-            layer.indexer_attn_q_b = create_tensor(tn(LLM_TENSOR_INDEXER_ATTN_Q_B, "weight", i), {q_lora_rank, hparams.indexer_n_head * hparams.indexer_head_size}, flags);
-        }
-
-        if (is_mtp || i < (int) hparams.n_layer_dense_lead) {
-            layer.ffn_gate = create_tensor(tn(LLM_TENSOR_FFN_GATE, "weight", i), {n_embd,   n_ff}, flags);
-            layer.ffn_down = create_tensor(tn(LLM_TENSOR_FFN_DOWN, "weight", i), {  n_ff, n_embd}, flags);
-            layer.ffn_up   = create_tensor(tn(LLM_TENSOR_FFN_UP,   "weight", i), {n_embd,   n_ff}, flags);
-        } else {
-            if (n_expert == 0 || n_expert_used == 0) {
-                throw std::runtime_error("n_expert and n_expert_used must be > 0");
-            }
-
-            layer.ffn_gate_inp    = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP,    "weight", i), {n_embd, n_expert}, flags);
-            layer.ffn_exp_probs_b = create_tensor(tn(LLM_TENSOR_FFN_EXP_PROBS_B, "bias",   i), {n_expert}, flags);
-
-            layer.ffn_gate_exps = create_tensor(tn(LLM_TENSOR_FFN_GATE_EXPS, "weight", i), {  n_embd, n_ff_exp, n_expert}, flags);
-            layer.ffn_down_exps = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS, "weight", i), {n_ff_exp,   n_embd, n_expert}, flags);
-            layer.ffn_up_exps   = create_tensor(tn(LLM_TENSOR_FFN_UP_EXPS,   "weight", i), {  n_embd, n_ff_exp, n_expert}, flags);
-
-            layer.ffn_gate_shexp = create_tensor(tn(LLM_TENSOR_FFN_GATE_SHEXP, "weight", i), {n_embd, n_ff_exp * n_expert_shared}, flags);
-            layer.ffn_down_shexp = create_tensor(tn(LLM_TENSOR_FFN_DOWN_SHEXP, "weight", i), {        n_ff_exp * n_expert_shared, n_embd}, flags);
-            layer.ffn_up_shexp   = create_tensor(tn(LLM_TENSOR_FFN_UP_SHEXP,   "weight", i), {n_embd, n_ff_exp * n_expert_shared}, flags);
-        }
-
-        if (is_mtp) {
-            layer.nextn.eh_proj          = create_tensor(tn(LLM_TENSOR_NEXTN_EH_PROJ,          "weight", i), { 2 * n_embd, n_embd }, flags);
-            layer.nextn.enorm            = create_tensor(tn(LLM_TENSOR_NEXTN_ENORM,            "weight", i), { n_embd }, flags);
-            layer.nextn.hnorm            = create_tensor(tn(LLM_TENSOR_NEXTN_HNORM,            "weight", i), { n_embd }, flags);
-            layer.nextn.embed_tokens     = create_tensor(tn(LLM_TENSOR_NEXTN_EMBED_TOKENS,     "weight", i), { n_embd, n_vocab }, flags);
-            layer.nextn.shared_head_norm = create_tensor(tn(LLM_TENSOR_NEXTN_SHARED_HEAD_NORM, "weight", i), { n_embd }, flags);
-        }
-    }
-}
-
-std::unique_ptr<llm_graph_context> llama_model_dots3note::build_arch_graph(const llm_graph_params & params) const {
-    return std::make_unique<graph>(*this, params);
-}
-
-llama_model_dots3note::graph::graph(const llama_model & model, const llm_graph_params & params) :
-    llm_graph_context(params) {
-    GGML_ASSERT(hparams.is_mla());
-
-    const int64_t n_embd_head_qk_rope = hparams.n_rot();
-
-    const int64_t n_indexer_head      = hparams.indexer_n_head;
-    const int64_t n_embd_indexer_head = hparams.indexer_head_size;
-    const uint32_t n_indexer_top_k = hparams.indexer_top_k;
-
-    // the indexer head layout is [rope | nope]
-    GGML_ASSERT(hparams.n_rot() <= n_embd_indexer_head);
-
-    ggml_tensor * cur;
-    ggml_tensor * inpL;
-
-    inpL = build_inp_embd(model.tok_embd);
-
-    ggml_tensor * inp_pos = build_inp_pos();
-
-    llm_graph_input_attn_k_dsa_iswa * inp_attn = build_attn_inp_k_dsa_iswa();
-
-    ggml_tensor * inp_out_ids = build_inp_out_ids();
-
-    for (int il = 0; il < n_layer; ++il) {
-        ggml_tensor * inpSA = inpL;
-
-        const bool is_swa = hparams.is_swa(il);
-
-        const int64_t n_head_l = hparams.n_head(il);
-
-        const int64_t kv_lora_rank        = is_swa ? hparams.n_lora_kv_swa         : hparams.n_lora_kv;
-        const int64_t n_embd_head_k_mla   = is_swa ? hparams.n_embd_head_k_mla_swa : hparams.n_embd_head_k_mla();
-        const int64_t n_embd_head_v_mla   = is_swa ? hparams.n_embd_head_v_mla_swa : hparams.n_embd_head_v_mla();
-        const int64_t n_embd_head_qk_nope = n_embd_head_k_mla - n_embd_head_qk_rope;
-
-        const float kq_scale    = 1.0f/sqrtf(float(n_embd_head_k_mla));
-        const float freq_base_l = model.get_rope_freq_base(cparams, il);
-
-        // norm
-        cur = build_norm(inpL, model.layers[il].attn_norm, NULL, LLM_NORM_RMS, il);
-        cb(cur, "attn_norm", il);
-
-        // self_attention
-        {
-            ggml_tensor * attn_inp = cur;
-
-            ggml_tensor * qr = ggml_mul_mat(ctx0, model.layers[il].wq_a, cur);
-            cb(qr, "qr", il);
-
-            qr = build_norm(qr, model.layers[il].attn_q_a_norm, nullptr, LLM_NORM_RMS, il);
-            cb(qr, "qr", il);
-
-            ggml_tensor * top_k = nullptr;
-
-            // lightning indexer (full-attention layers only)
-            if (!is_swa) {
-                ggml_tensor * indexer_q = ggml_mul_mat(ctx0, model.layers[il].indexer_attn_q_b, qr);
-                cb(indexer_q, "indexer_q", il);
-
-                // {n_embd_indexer_head, n_indexer_head, n_tokens}
-                indexer_q = ggml_reshape_3d(ctx0, indexer_q, n_embd_indexer_head, n_indexer_head, n_tokens);
-                indexer_q = ggml_rope_ext(ctx0, indexer_q, inp_pos, nullptr, n_rot,
-                                     LLAMA_ROPE_TYPE_NEOX, n_ctx_orig, freq_base, freq_scale,
-                                     ext_factor, attn_factor, beta_fast, beta_slow);
-                cb(indexer_q, "indexer_q", il);
-
-                ggml_tensor * indexer_k = ggml_mul_mat(ctx0, model.layers[il].indexer_attn_k, cur);
-                cb(indexer_k, "indexer_k", il);
-
-                indexer_k = build_norm(indexer_k, model.layers[il].indexer_k_norm, model.layers[il].indexer_k_norm_b, LLM_NORM, il);
-                cb(indexer_k, "indexer_k", il);
-
-                // {n_embd_indexer_head, 1, n_tokens}
-                indexer_k = ggml_reshape_3d(ctx0, indexer_k, n_embd_indexer_head, 1, n_tokens);
-                indexer_k = ggml_rope_ext(ctx0, indexer_k, inp_pos, nullptr, n_rot,
-                                     LLAMA_ROPE_TYPE_NEOX, n_ctx_orig, freq_base, freq_scale,
-                                     ext_factor, attn_factor, beta_fast, beta_slow);
-                cb(indexer_k, "indexer_k", il);
-
-                // perform Hadamard transform on indexer q and k
-                indexer_q = ggml_mul_mat(ctx0, inp_attn->get_dsa()->self_k_rot_lid, indexer_q);
-                cb(indexer_q, "indexer_q", il);
-                indexer_k = ggml_mul_mat(ctx0, inp_attn->get_dsa()->self_k_rot_lid, indexer_k);
-                cb(indexer_k, "indexer_k", il);
-
-                // store indexer keys to KV cache
-                const auto * mctx_lid = inp_attn->get_dsa()->mctx->get_lid();
-                const auto & k_idxs_lid = inp_attn->get_dsa()->get_k_idxs_lid();
-                ggml_build_forward_expand(gf, mctx_lid->cpy_k(ctx0, indexer_k, k_idxs_lid, il));
-
-                ggml_tensor * indexer_weights = ggml_mul_mat(ctx0, model.layers[il].indexer_proj, cur);
-                cb(indexer_weights, "indexer_weights", il);
-
-                indexer_k = mctx_lid->get_k(ctx0, il);
-
-                // split the batch into streams if needed
-                const auto n_stream = indexer_k->ne[3];
-                indexer_q = ggml_view_4d(ctx0, indexer_q, indexer_q->ne[0], indexer_q->ne[1], indexer_q->ne[2]/n_stream, n_stream, indexer_q->nb[1], indexer_q->nb[2], indexer_q->nb[3]/n_stream, 0);
-                indexer_weights = ggml_view_4d(ctx0, indexer_weights, indexer_weights->ne[0], indexer_weights->ne[1]/n_stream, indexer_weights->ne[2], n_stream, indexer_weights->nb[1], indexer_weights->nb[2]/n_stream, indexer_weights->nb[3]/n_stream, 0);
-
-                // pre-scale weights to avoid scaling operations on huge indexer_score tensor
-                indexer_weights = ggml_scale(ctx0, indexer_weights, 1.0f / sqrtf(float(n_embd_indexer_head * n_indexer_head)));
-                cb(indexer_weights, "indexer_weights", il);
-
-                ggml_tensor * indexer_score = nullptr;
-                if (cparams.fused_lid) {
-                    indexer_score = ggml_lightning_indexer(ctx0, indexer_q, indexer_k, indexer_weights, inp_attn->get_dsa()->get_kq_mask_lid());
-                    cb(indexer_score, "indexer_score", il);
-                    res->add_fused_node({LLM_FUSED_OP_LIGHTNING_INDEXER, indexer_score, il});
-                } else {
-                    indexer_q = ggml_permute(ctx0, indexer_q, 0, 2, 1, 3);
-                    cb(indexer_q, "indexer_q", il);
-                    indexer_k = ggml_permute(ctx0, indexer_k, 0, 2, 1, 3);
-                    cb(indexer_k, "indexer_k", il);
-
-                    ggml_tensor * indexer_kq = ggml_mul_mat(ctx0, indexer_k, indexer_q);
-                    cb(indexer_kq, "indexer_kq", il);
-
-                    // ReLU requires contiguous tensors
-                    indexer_kq = ggml_cont(ctx0, ggml_permute(ctx0, indexer_kq, 2, 1, 0, 3));
-                    cb(indexer_kq, "indexer_kq", il);
-
-                    indexer_score = ggml_relu(ctx0, indexer_kq);
-                    cb(indexer_score, "indexer_score", il);
-
-                    indexer_score = ggml_mul(ctx0, indexer_score, indexer_weights);
-                    cb(indexer_score, "indexer_score", il);
-
-                    // sum by q n_indexer_head dimension
-                    indexer_score = ggml_sum_rows(ctx0, indexer_score);
-                    cb(indexer_score, "indexer_score", il);
-
-                    // permute result to match KQ mask
-                    indexer_score = ggml_cont(ctx0, ggml_permute(ctx0, indexer_score, 2, 1, 0, 3));
-                    cb(indexer_score, "indexer_score", il);
-
-                    ggml_tensor * indexer_kq_mask = inp_attn->get_dsa()->get_kq_mask_lid();
-                    indexer_score = ggml_add(ctx0, indexer_score, indexer_kq_mask);
-                    cb(indexer_score, "indexer_score", il);
-                }
-
-                // get indices of top k indexer scores
-                uint32_t n_top_k = indexer_score->ne[0] < n_indexer_top_k ? indexer_score->ne[0] : n_indexer_top_k;
-                top_k = ggml_cont(ctx0, ggml_top_k(ctx0, indexer_score, n_top_k));
-                cb(top_k, "top_k", il);
-            }
-
-            ggml_tensor * q = ggml_mul_mat(ctx0, model.layers[il].wq_b, qr);
-            cb(q, "q", il);
-
-            // split into {n_embd_head_qk_nope, n_head_l, n_tokens}
-            ggml_tensor * q_nope =
-                ggml_view_3d(ctx0, q, n_embd_head_qk_nope, n_head_l, n_tokens, ggml_row_size(q->type, n_embd_head_k_mla),
-                             ggml_row_size(q->type, n_embd_head_k_mla) * n_head_l, 0);
-            cb(q_nope, "q_nope", il);
-
-            // and {n_embd_head_qk_rope, n_head_l, n_tokens}
-            ggml_tensor * q_pe = ggml_view_3d(
-                ctx0, q, n_embd_head_qk_rope, n_head_l, n_tokens, ggml_row_size(q->type, n_embd_head_k_mla),
-                ggml_row_size(q->type, n_embd_head_k_mla) * n_head_l, ggml_row_size(q->type, n_embd_head_qk_nope));
-            cb(q_pe, "q_pe", il);
-
-            ggml_tensor * kv_cmpr_pe = ggml_mul_mat(ctx0, model.layers[il].wkv_a_mqa, cur);
-            cb(kv_cmpr_pe, "kv_cmpr_pe", il);
-
-            // split into {kv_lora_rank, n_tokens}
-            ggml_tensor * kv_cmpr =
-                ggml_view_2d(ctx0, kv_cmpr_pe, kv_lora_rank, n_tokens,
-                             ggml_row_size(kv_cmpr_pe->type, kv_lora_rank + n_embd_head_qk_rope), 0);
-            cb(kv_cmpr, "kv_cmpr", il);
-
-            // and {n_embd_head_qk_rope, 1, n_tokens}
-            ggml_tensor * k_pe = ggml_view_3d(ctx0, kv_cmpr_pe, n_embd_head_qk_rope, 1, n_tokens,
-                                              ggml_row_size(kv_cmpr_pe->type, kv_lora_rank + n_embd_head_qk_rope),
-                                              ggml_row_size(kv_cmpr_pe->type, kv_lora_rank + n_embd_head_qk_rope),
-                                              ggml_row_size(kv_cmpr_pe->type, kv_lora_rank));
-            cb(k_pe, "k_pe", il);
-
-            // norm on the shared rope key, applied before rope
-            k_pe = build_norm(k_pe, model.layers[il].attn_k_norm, nullptr, LLM_NORM_RMS, il);
-            cb(k_pe, "k_pe", il);
-
-            q_pe = ggml_rope_ext(ctx0, q_pe, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base_l, freq_scale,
-                                 ext_factor, attn_factor, beta_fast, beta_slow);
-            cb(q_pe, "q_pe", il);
-
-            k_pe = ggml_rope_ext(ctx0, k_pe, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base_l, freq_scale,
-                                 ext_factor, attn_factor, beta_fast, beta_slow);
-            cb(k_pe, "k_pe", il);
-
-            kv_cmpr = build_norm(kv_cmpr, model.layers[il].attn_kv_a_norm, nullptr, LLM_NORM_RMS, il);
-            cb(kv_cmpr, "kv_cmpr", il);
-
-            // MLA attention with the absorption optimization
-            {
-                // {n_embd_head_qk_nope, n_tokens, n_head_l}
-                q_nope = ggml_permute(ctx0, q_nope, 0, 2, 1, 3);
-                cb(q_nope, "q_nope_perm", il);
-
-                // {n_embd_head_qk_nope, kv_lora_rank, n_head_l} x {n_embd_head_qk_nope, n_tokens, n_head_l}
-                ggml_tensor * q_nope_absorbed = ggml_mul_mat(ctx0, model.layers[il].wk_b, q_nope);
-                cb(q_nope_absorbed, "q_nope_absorbed", il);
-
-                // {kv_lora_rank, n_head_l, n_tokens}
-                q_nope_absorbed = ggml_permute(ctx0, q_nope_absorbed, 0, 2, 1, 3);
-                cb(q_nope_absorbed, "q_nope_absorbed_perm", il);
-
-                // {n_embd_head_qk_rope + kv_lora_rank, n_head_l, n_tokens}
-                ggml_tensor * Qcur = ggml_concat(ctx0, q_nope_absorbed, q_pe, 0);
-                cb(Qcur, "Qcur", il);
-
-                kv_cmpr = ggml_reshape_3d(ctx0, kv_cmpr, kv_lora_rank, 1, n_tokens);
-                cb(kv_cmpr, "kv_cmpr_reshape", il);
-
-                // {n_embd_head_qk_rope + kv_lora_rank, 1, n_tokens}
-                ggml_tensor * Kcur = ggml_concat(ctx0, kv_cmpr, k_pe, 0);
-                cb(Kcur, "Kcur", il);
-
-                // {kv_lora_rank, 1, n_tokens}
-                ggml_tensor * Vcur = kv_cmpr;
-                cb(Vcur, "Vcur", il);
-
-                // apply the head-wise output gate before o_proj, so wo stays out of build_attn
-                if (is_swa) {
-                    cur = build_attn(inp_attn->get_swa(),
-                            nullptr, nullptr, nullptr,
-                            Qcur, Kcur, Vcur, nullptr, nullptr, model.layers[il].wv_b, kq_scale, il);
-                } else {
-                    cur = build_attn(inp_attn->get_dsa(),
-                            nullptr, nullptr, nullptr,
-                            Qcur, Kcur, Vcur, nullptr, nullptr, model.layers[il].wv_b, top_k, kq_scale, il);
-                }
-                cb(cur, "attn_out", il);
-
-                ggml_tensor * gate = build_lora_mm(model.layers[il].wqkv_gate, attn_inp);
-                cb(gate, "attn_gate", il);
-
-                gate = ggml_sigmoid(ctx0, gate);
-                cb(gate, "attn_gate_sigmoid", il);
-
-                // broadcast the per-head gate over the head dimension
-                ggml_tensor * attn_3d = ggml_reshape_3d(ctx0, cur, n_embd_head_v_mla, n_head_l, n_tokens);
-                ggml_tensor * gate_3d = ggml_reshape_3d(ctx0, gate,                1, n_head_l, n_tokens);
-                attn_3d = ggml_mul(ctx0, attn_3d, gate_3d);
-                cb(attn_3d, "attn_gated", il);
-
-                cur = ggml_reshape_2d(ctx0, attn_3d, n_embd_head_v_mla * n_head_l, n_tokens);
-
-                cur = build_lora_mm(model.layers[il].wo, cur, model.layers[il].wo_s);
-                cb(cur, "attn_output", il);
-            }
-        }
-
-        if (il == n_layer - 1 && inp_out_ids) {
-            cur   = ggml_get_rows(ctx0, cur, inp_out_ids);
-            inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids);
-        }
-
-        ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA);
-        cb(ffn_inp, "ffn_inp", il);
-
-        cur = build_norm(ffn_inp, model.layers[il].ffn_norm, NULL, LLM_NORM_RMS, il);
-        cb(cur, "ffn_norm", il);
-
-        if ((uint32_t) il < hparams.n_layer_dense_lead) {
-            cur = build_ffn(cur,
-                model.layers[il].ffn_up, NULL, model.layers[il].ffn_up_s,
-                model.layers[il].ffn_gate, NULL, model.layers[il].ffn_gate_s,
-                model.layers[il].ffn_down, NULL, model.layers[il].ffn_down_s,
-                NULL, LLM_FFN_SILU, LLM_FFN_PAR, il);
-            cb(cur, "ffn_out", il);
-        } else {
-            ggml_tensor * moe_out = build_moe_ffn(cur,
-                model.layers[il].ffn_gate_inp,
-                model.layers[il].ffn_up_exps,
-                model.layers[il].ffn_gate_exps,
-                model.layers[il].ffn_down_exps,
-                model.layers[il].ffn_exp_probs_b,
-                n_expert, n_expert_used,
-                LLM_FFN_SILU, hparams.expert_weights_norm,
-                hparams.expert_weights_scale,
-                (llama_expert_gating_func_type) hparams.expert_gating_func,
-                il,
-                nullptr,
-                model.layers[il].ffn_gate_up_exps,
-                model.layers[il].ffn_up_exps_s,
-                model.layers[il].ffn_gate_exps_s,
-                model.layers[il].ffn_down_exps_s);
-            cb(moe_out, "ffn_moe_out", il);
-
-            ggml_tensor * ffn_shexp =
-                build_ffn(cur,
-                    model.layers[il].ffn_up_shexp, NULL, model.layers[il].ffn_up_shexp_s,
-                    model.layers[il].ffn_gate_shexp, NULL, model.layers[il].ffn_gate_shexp_s,
-                    model.layers[il].ffn_down_shexp, NULL, model.layers[il].ffn_down_shexp_s,
-                    NULL, LLM_FFN_SILU, LLM_FFN_PAR, il);
-            cb(ffn_shexp, "ffn_shexp", il);
-
-            cur = ggml_add(ctx0, moe_out, ffn_shexp);
-            cb(cur, "ffn_out", il);
-        }
-
-        cur = ggml_add(ctx0, cur, ffn_inp);
-
-        cur = build_cvec(cur, il);
-        cb(cur, "l_out", il);
-
-        inpL = cur;
-    }
-
-    cur = inpL;
-
-    cur = build_norm(cur, model.output_norm, NULL, LLM_NORM_RMS, -1);
-
-    cb(cur, "result_norm", -1);
-    res->t_embd = cur;
-
-    cur = ggml_mul_mat(ctx0, model.output, cur);
-
-    cb(cur, "result_output", -1);
-    res->t_logits = cur;
-
-    ggml_build_forward_expand(gf, cur);
-}

src/models/minicpm3.cpp

@@ -115,9 +115,19 @@ llama_model_minicpm3::graph::graph(const llama_model & model, const llm_graph_pa
             q = ggml_mul_mat(ctx0, model.layers[il].wq_b, q);
             cb(q, "q", il);
 
-            // {n_embd_head_k, n_head, n_tokens}, RoPE is applied to the trailing dims only
-            q = ggml_reshape_3d(ctx0, q, hparams.n_embd_head_k(), n_head, n_tokens);
-            cb(q, "q", il);
+            // split into {n_head * n_embd_head_qk_nope, n_tokens}
+            ggml_tensor * q_nope = ggml_view_3d(ctx0, q, n_embd_head_qk_nope, n_head, n_tokens,
+                    ggml_row_size(q->type, hparams.n_embd_head_k()),
+                    ggml_row_size(q->type, hparams.n_embd_head_k() * n_head),
+                    0);
+            cb(q_nope, "q_nope", il);
+
+            // and {n_head * n_embd_head_qk_rope, n_tokens}
+            ggml_tensor * q_pe = ggml_view_3d(ctx0, q, n_embd_head_qk_rope, n_head, n_tokens,
+                    ggml_row_size(q->type, hparams.n_embd_head_k()),
+                    ggml_row_size(q->type, hparams.n_embd_head_k() * n_head),
+                    ggml_row_size(q->type, n_embd_head_qk_nope));
+            cb(q_pe, "q_pe", il);
 
             // {n_embd, kv_lora_rank + n_embd_head_qk_rope} * {n_embd, n_tokens} -> {kv_lora_rank + n_embd_head_qk_rope, n_tokens}
             ggml_tensor * kv_pe_compresseed = ggml_mul_mat(ctx0, model.layers[il].wkv_a_mqa, cur);
@@ -162,13 +172,12 @@ llama_model_minicpm3::graph::graph(const llama_model & model, const llm_graph_pa
             v_states = ggml_cont(ctx0, v_states);
             cb(v_states, "v_states", il);
 
-            q = ggml_rope_ext(
-                    ctx0, q, inp_pos, rope_factors,
+            q_pe = ggml_rope_ext(
+                    ctx0, q_pe, inp_pos, rope_factors,
                     n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
                     ext_factor, attn_factor, beta_fast, beta_slow
                     );
-            q = ggml_rope_set_offset(q, n_embd_head_qk_nope);
-            cb(q, "q_rope", il);
+            cb(q_pe, "q_pe", il);
 
             // shared RoPE key
             k_pe = ggml_rope_ext(
@@ -178,11 +187,10 @@ llama_model_minicpm3::graph::graph(const llama_model & model, const llm_graph_pa
                     );
             cb(k_pe, "k_pe", il);
 
-            ggml_tensor * q_states = q;
+            ggml_tensor * q_states = ggml_concat(ctx0, q_nope, q_pe, 0);
             cb(q_states, "q_states", il);
 
-            ggml_tensor * k_states = ggml_concat(ctx0, k_nope,
-                    ggml_repeat_4d(ctx0, k_pe, n_embd_head_qk_rope, n_head, n_tokens, 1), 0);
+            ggml_tensor * k_states = ggml_concat(ctx0, k_nope, ggml_repeat(ctx0, k_pe, q_pe), 0);
             cb(k_states, "k_states", il);
 
             cur = build_attn(inp_attn,

src/models/models.h

@@ -1156,18 +1156,6 @@ struct llama_model_deepseek32 : public llama_model_base {
 };
 
 
-struct llama_model_dots3note : public llama_model_base {
-    llama_model_dots3note(const struct llama_model_params & params) : llama_model_base(params) {}
-    void load_arch_hparams(llama_model_loader & ml) override;
-    void load_arch_tensors(llama_model_loader & ml) override;
-
-    struct graph : public llm_graph_context {
-        graph(const llama_model & model, const llm_graph_params & params);
-    };
-
-    std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;
-};
-
 struct llama_model_deepseek4 : public llama_model_base {
     llama_model_deepseek4(const struct llama_model_params & params) : llama_model_base(params) {}
     void load_arch_hparams(llama_model_loader & ml) override;

src/models/plm.cpp

@@ -81,9 +81,19 @@ llama_model_plm::graph::graph(const llama_model & model, const llm_graph_params
             q = ggml_mul_mat(ctx0, model.layers[il].wq, cur);
             cb(q, "q", il);
 
-            // {n_embd_head_k, n_head, n_tokens}, RoPE is applied to the trailing dims only
-            q = ggml_reshape_3d(ctx0, q, hparams.n_embd_head_k(), n_head, n_tokens);
-            cb(q, "q", il);
+            // split into {n_head * n_embd_head_qk_nope, n_tokens}
+            ggml_tensor * q_nope = ggml_view_3d(ctx0, q, n_embd_head_qk_nope, n_head, n_tokens,
+                    ggml_row_size(q->type, hparams.n_embd_head_k()),
+                    ggml_row_size(q->type, hparams.n_embd_head_k() * n_head),
+                    0);
+            cb(q_nope, "q_nope", il);
+
+            // and {n_head * n_embd_head_qk_rope, n_tokens}
+            ggml_tensor * q_pe = ggml_view_3d(ctx0, q, n_embd_head_qk_rope, n_head, n_tokens,
+                    ggml_row_size(q->type, hparams.n_embd_head_k()),
+                    ggml_row_size(q->type, hparams.n_embd_head_k() * n_head),
+                    ggml_row_size(q->type, n_embd_head_qk_nope));
+            cb(q_pe, "q_pe", il);
 
             // {n_embd, kv_lora_rank + n_embd_head_qk_rope} * {n_embd, n_tokens} -> {kv_lora_rank + n_embd_head_qk_rope, n_tokens}
             ggml_tensor * kv_pe_compresseed = ggml_mul_mat(ctx0, model.layers[il].wkv_a_mqa, cur);
@@ -133,13 +143,12 @@ llama_model_plm::graph::graph(const llama_model & model, const llm_graph_params
                     0);
             cb(v_states, "v_states", il);
 
-            q = ggml_rope_ext(
-                    ctx0, q, inp_pos, nullptr,
+            q_pe = ggml_rope_ext(
+                    ctx0, q_pe, inp_pos, nullptr,
                     n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
                     ext_factor, attn_factor, beta_fast, beta_slow
                     );
-            q = ggml_rope_set_offset(q, n_embd_head_qk_nope);
-            cb(q, "q_rope", il);
+            cb(q_pe, "q_pe", il);
 
             // shared RoPE key
             k_pe = ggml_rope_ext(
@@ -149,11 +158,10 @@ llama_model_plm::graph::graph(const llama_model & model, const llm_graph_params
                     );
             cb(k_pe, "k_pe", il);
 
-            ggml_tensor * q_states = q;
+            ggml_tensor * q_states = ggml_concat(ctx0, q_nope, q_pe, 0);
             cb(q_states, "q_states", il);
 
-            ggml_tensor * k_states = ggml_concat(ctx0, k_nope,
-                    ggml_repeat_4d(ctx0, k_pe, n_embd_head_qk_rope, n_head, n_tokens, 1), 0);
+            ggml_tensor * k_states = ggml_concat(ctx0, k_nope, ggml_repeat(ctx0, k_pe, q_pe), 0);
             cb(k_states, "k_states", il);
 
             cur = build_attn(inp_attn,

tests/peg-parser/test-json-serialization.cpp

@@ -8,7 +8,7 @@ void test_json_serialization(testing &t) {
     auto json_serialized = original.to_json().dump();
 
     t.test("compare before/after", [&](testing &t) {
-        auto deserialized = common_peg_arena::from_json(nlohmann::json::parse(json_serialized));
+        auto deserialized = common_peg_arena::from_json(common_json::parse(json_serialized));
 
         // Test complex JSON
         std::string input = R"({"name": "test", "values": [1, 2, 3], "nested": {"a": true}})";
@@ -23,6 +23,6 @@ void test_json_serialization(testing &t) {
     });
 
     t.bench("deserialize", [&]() {
-        auto deserialized = common_peg_arena::from_json(nlohmann::json::parse(json_serialized));
+        auto deserialized = common_peg_arena::from_json(common_json::parse(json_serialized));
     }, 100);
 }

tests/peg-parser/tests.h

@@ -1,7 +1,7 @@
 #pragma once
 
 // Common includes for all test files
-#include <nlohmann/json.hpp>
+#include "json.h"
 #include <string>
 #include <vector>
 
@@ -11,9 +11,9 @@
 #include "simple-tokenize.h"
 
 struct bench_tool_call {
-    std::string            id;
-    std::string            name;
-    nlohmann::ordered_json args;
+    std::string id;
+    std::string name;
+    common_json args;
 };
 
 // Test function declarations

tests/test-backend-ops.cpp

@@ -7085,10 +7085,9 @@ struct test_flash_attn_ext : public test_case {
     const ggml_type type_V;
     std::array<int32_t, 4> permute;
     const bool kv_view; // create K/V as views of a larger buffer (like a KV cache)
-    const bool v_is_view_of_k;
 
     std::string vars() override {
-        return VARS_TO_STR16(hsk, hsv, nh, nr23, kv, nb, mask, sinks, max_bias, logit_softcap, prec, type_K, type_V, permute, kv_view, v_is_view_of_k);
+        return VARS_TO_STR15(hsk, hsv, nh, nr23, kv, nb, mask, sinks, max_bias, logit_softcap, prec, type_K, type_V, permute, kv_view);
     }
 
     double max_nmse_err() override {
@@ -7105,9 +7104,9 @@ struct test_flash_attn_ext : public test_case {
     test_flash_attn_ext(int64_t hsk = 128, int64_t hsv = 128, int64_t nh = 32, std::array<int64_t, 2> nr23 = {1, 1}, int64_t kv = 96, int64_t nb = 8,
                         bool mask = true, bool sinks = false, float max_bias = 0.0f, float logit_softcap = 0.0f, ggml_prec prec = GGML_PREC_F32,
                         ggml_type type_K = GGML_TYPE_F16, ggml_type type_V = GGML_TYPE_F16, std::array<int32_t, 4> permute = {0, 1, 2, 3},
-                        bool kv_view = true, bool v_is_view_of_k = false)
+                        bool kv_view = true)
         : hsk(hsk), hsv(hsv), nh(nh), nr23(nr23), kv(kv), nb(nb), mask(mask), sinks(sinks), max_bias(max_bias), logit_softcap(logit_softcap), prec(prec),
-          type_K(type_K), type_V(type_V), permute(permute), kv_view(kv_view), v_is_view_of_k(v_is_view_of_k) {}
+          type_K(type_K), type_V(type_V), permute(permute), kv_view(kv_view) {}
 
     ggml_tensor * build_graph(ggml_context * ctx) override {
         const int64_t hsk_padded = GGML_PAD(hsk, ggml_blck_size(type_K));
@@ -7139,14 +7138,14 @@ struct test_flash_attn_ext : public test_case {
         ggml_set_name(k, "k");
 
         ggml_tensor * v = nullptr;
-        if (v_is_view_of_k) {
-            // the V cache is a sub-view of the K cache. this is used by some MLA-based models
+        if (type_K == type_V && hsk_padded == 576 && hsv_padded == 512) {
+            // TODO: this branch should become a separate test case parameter instead of hardcoding this for these head shapes
+
+            // in this branch, the V cache is sub-view of the K cache. this is used by some MLA-based models
             // for more info:
             //   - https://github.com/ggml-org/llama.cpp/pull/13435
             //   - https://github.com/ggml-org/llama.cpp/pull/18953#issuecomment-3774948392
             //   - https://github.com/ggml-org/llama.cpp/pull/18986
-            GGML_ASSERT(type_K == type_V && hsv_padded <= hsk_padded);
-
             v = ggml_view_4d(ctx, k, hsv_padded, kv, nh, nr23[1], k->nb[1], k->nb[2], k->nb[3], 0);
         } else {
             v = create_permuted(type_V,        hsv_padded, kv, nh,         nr23[1], kv_view); // the V tensor is usually a view of the V cache
@@ -9907,14 +9906,12 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {
                                                     if (hsk != 128 && prec == GGML_PREC_DEFAULT) continue;
                                                     for (ggml_type type_KV : {GGML_TYPE_F32, GGML_TYPE_F16, GGML_TYPE_BF16, GGML_TYPE_Q8_0, GGML_TYPE_Q5_1, GGML_TYPE_Q5_0, GGML_TYPE_Q4_1, GGML_TYPE_Q4_0, GGML_TYPE_IQ4_NL}) {
                                                         if (type_KV != GGML_TYPE_F16 && hsk != 64 && hsk != 72) continue;
-                                                        // DeepSeek MLA: the V cache is a sub-view of the K cache
-                                                        const bool v_is_view_of_k = hsk == 576;
                                                         test_cases.emplace_back(new test_flash_attn_ext(
-                                                                    hsk, hsv, nh, {nr2, nr3}, kv, nb, mask, sinks, max_bias, logit_softcap, prec, type_KV, type_KV, {0, 1, 2, 3}, true, v_is_view_of_k));
+                                                                    hsk, hsv, nh, {nr2, nr3}, kv, nb, mask, sinks, max_bias, logit_softcap, prec, type_KV, type_KV));
                                                         // run fewer test cases permuted
                                                         if (mask == true && max_bias == 0.0f && logit_softcap == 0 && kv == 512) {
                                                             test_cases.emplace_back(new test_flash_attn_ext(
-                                                                        hsk, hsv, nh, {nr2, nr3}, kv, nb, mask, sinks, max_bias, logit_softcap, prec, type_KV, type_KV, {0, 2, 1, 3}, true, v_is_view_of_k));
+                                                                        hsk, hsv, nh, {nr2, nr3}, kv, nb, mask, sinks, max_bias, logit_softcap, prec, type_KV, type_KV, {0, 2, 1, 3}));
                                                         }
                                                     }
                                                 }
@@ -9953,16 +9950,11 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {
     test_cases.emplace_back(new test_flash_attn_ext(256, 256, 2, {16, 1},  1025,  64, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q8_0, GGML_TYPE_Q8_0, {0, 2, 1, 3}));
     test_cases.emplace_back(new test_flash_attn_ext(256, 256, 2, {16, 1}, 16384,   1, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q8_0, GGML_TYPE_Q8_0));
 
-    // MLA shape: the V cache is a sub-view of the K cache, with quantized KV
-    test_cases.emplace_back(new test_flash_attn_ext(576, 512, 1, {20, 1},  113,   1, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q8_0, GGML_TYPE_Q8_0, {0, 1, 2, 3}, true, true));
-    test_cases.emplace_back(new test_flash_attn_ext(576, 512, 1, {20, 1}, 1024,   1, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q8_0, GGML_TYPE_Q8_0, {0, 1, 2, 3}, true, true));
-    test_cases.emplace_back(new test_flash_attn_ext(576, 512, 1, {20, 1}, 1024,  64, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q8_0, GGML_TYPE_Q8_0, {0, 1, 2, 3}, true, true));
-
-    // more V-is-sub-view-of-K cases: other head shapes, and full views with equal head sizes
-    test_cases.emplace_back(new test_flash_attn_ext(320, 256, 1, {32, 1}, 512, 1, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16, {0, 1, 2, 3}, true, true));
-    test_cases.emplace_back(new test_flash_attn_ext(192, 128, 4, {8, 1},  512, 1, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16, {0, 1, 2, 3}, true, true));
-    test_cases.emplace_back(new test_flash_attn_ext(128, 128, 8, {4, 1},  512, 8, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16, {0, 1, 2, 3}, true, true));
-    test_cases.emplace_back(new test_flash_attn_ext(64,  64,  4, {1, 1},  512, 1, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q8_0, GGML_TYPE_Q8_0, {0, 1, 2, 3}, true, true));
+    // MLA shape (V is a view of K) with quantized KV
+    // (the test harness builds V as a view of K for this shape; see build_graph)
+    test_cases.emplace_back(new test_flash_attn_ext(576, 512, 1, {20, 1},  113,   1, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q8_0, GGML_TYPE_Q8_0));
+    test_cases.emplace_back(new test_flash_attn_ext(576, 512, 1, {20, 1}, 1024,   1, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q8_0, GGML_TYPE_Q8_0));
+    test_cases.emplace_back(new test_flash_attn_ext(576, 512, 1, {20, 1}, 1024,  64, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q8_0, GGML_TYPE_Q8_0));
 
     // large-KV F16 cases (Qwen3.6-27B geometry and a llama-class control): the upstream matrix
     // stops at kv=1024, blind to long-context FA bugs (e.g. the oneDNN SDPA ordering race on BMG).

tests/test-chat-peg-parser.cpp

@@ -11,9 +11,9 @@
 #include <regex>
 #include <string>
 
-#include "nlohmann/json.hpp"
+#include "json.h"
 
-using json = nlohmann::ordered_json;
+using json = common_json;
 
 static json create_tools();
 static void test_example_native(testing & t);
@@ -63,10 +63,10 @@ static json create_tools() {
                           { { "type", "string" }, { "description", "The city and state, e.g. San Francisco, CA" } } },
                         { "unit",
                           { { "type", "string" },
-                            { "enum", { "celsius", "fahrenheit" } },
+                            { "enum", json::array({ "celsius", "fahrenheit" }) },
                             { "description",
                               "The temperature unit to use. Infer this from the users location." } } } } },
-                    { "required", { "location", "unit" } },
+                    { "required", json::array({ "location", "unit" }) },
                 } },
           }                      }
     };
@@ -86,14 +86,14 @@ static json create_tools() {
                           { { "type", "string" }, { "description", "The city and state, e.g. San Francisco, CA" } } },
                         { "unit",
                           { { "type", "string" },
-                            { "enum", { "celsius", "fahrenheit" } },
+                            { "enum", json::array({ "celsius", "fahrenheit" }) },
                             { "description", "The temperature unit to use. Infer this from the users location." } } },
                         { "days",
                           { { "type", "integer" },
                             { "description", "Number of days to forecast (1-10)" },
                             { "minimum", 1 },
                             { "maximum", 10 } } } } },
-                    { "required", { "location", "unit" } },
+                    { "required", json::array({ "location", "unit" }) },
                 } },
           }                      }
     };
@@ -114,9 +114,9 @@ static json create_tools() {
                         { "default", 5 } } },
                     { "category",
                       { { "type", "string" },
-                        { "enum", { "api", "troubleshooting", "billing", "general" } },
+                        { "enum", json::array({ "api", "troubleshooting", "billing", "general" }) },
                         { "description", "Filter search by specific category." } } } } },
-                { "required", { "query", "category" } },
+                { "required", json::array({ "query", "category" }) },
                 { "additionalProperties", false } } },
             { "strict", true } } }
     };
@@ -341,7 +341,7 @@ static void test_example_native(testing & t) {
                 { { "invoice_number", { { "type", "string" } } },
                   { "amount", { { "type", "number" } } },
                   { "due_date", { { "type", "string" } } } } },
-              { "required", { "invoice_number", "amount", "due_date" } } },
+              { "required", json::array({ "invoice_number", "amount", "due_date" }) } },
          /* .parallel_tool_calls =  */ false,
          /* .generation_prompt =    */ "<think>",
          /* .input =                */
@@ -406,7 +406,7 @@ static void test_example_qwen3_coder(testing & t) {
 
             std::set<std::string> required_properties;
             if (function.contains("required")) {
-                function.at("required").get_to(required_properties);
+                required_properties = function.at("required").get<std::set<std::string>>();
             }
 
             std::vector<common_peg_parser> arg_parsers;
@@ -661,8 +661,8 @@ void test_command7_parser_compare(testing & t) {
         "5. Provide a detailed cost breakdown that includes accommodation, transportation, meals, and entry fees "
         "to attractions.";
 
-    std::vector<std::tuple<std::string, std::string, nlohmann::json>> tool_calls = {
-        { "call_0", "plan_trip", nlohmann::json::parse(R"({
+    std::vector<std::tuple<std::string, std::string, common_json>> tool_calls = {
+        { "call_0", "plan_trip", common_json::parse(R"({
             "destination": "Japan",
             "duration": 14,
             "budget": 4000,
@@ -686,16 +686,16 @@ void test_command7_parser_compare(testing & t) {
     if (!tool_calls.empty()) {
         tokens.emplace_back("<|START_ACTION|>");
 
-        auto json = nlohmann::json::array();
+        auto json = common_json::array();
         for (const auto & tc : tool_calls) {
-            auto tc_json            = nlohmann::json::object();
+            auto tc_json            = common_json::object();
             tc_json["tool_call_id"] = std::get<0>(tc);
             tc_json["tool_name"]    = std::get<1>(tc);
             tc_json["parameters"]   = std::get<2>(tc);
             json.push_back(tc_json);
         }
 
-        auto tokenized = simple_tokenize(json.dump(-1, ' ', true));
+        auto tokenized = simple_tokenize(json.dump(-1));
         tokens.insert(tokens.end(), tokenized.begin(), tokenized.end());
 
         tokens.emplace_back("<|END_ACTION|>");
@@ -737,7 +737,7 @@ static void test_prefix_tool_names(testing & t) {
                       {
                           { "arg1", { { "type", "integer" } } },
                       } },
-                    { "required", { "arg1" } },
+                    { "required", json::array({ "arg1" }) },
                 } },
           } }
     };
@@ -757,7 +757,7 @@ static void test_prefix_tool_names(testing & t) {
                           { "arg1", { { "type", "integer" } } },
                           { "arg2", { { "type", "integer" } } },
                       } },
-                    { "required", { "arg1" } },
+                    { "required", json::array({ "arg1" }) },
                 } },
           } }
     };

tests/test-chat-template.cpp

@@ -7,7 +7,7 @@
 #include <fstream>
 #include <filesystem>
 
-#include <nlohmann/json.hpp>
+#include "json.h"
 
 #undef NDEBUG
 #include <cassert>
@@ -20,7 +20,7 @@
 #include "jinja/lexer.h"
 #include "jinja/caps.h"
 
-using json = nlohmann::ordered_json;
+using json = common_json;
 
 static int main_automated_tests(void);
 
@@ -304,8 +304,8 @@ void run_single(const std::string& contents, json input, bool use_common, bool d
         if (input.contains("eos_token")) {
             eos_token = input["eos_token"].get<std::string>();
         }
-        nlohmann::ordered_json msgs_json = input["messages"];
-        nlohmann::ordered_json tools_json = input["tools"];
+        common_json msgs_json = input["messages"];
+        common_json tools_json = input["tools"];
         auto messages = common_chat_msgs_parse_oaicompat(msgs_json);
         auto tools = common_chat_tools_parse_oaicompat(tools_json);
         auto output = format_using_common(contents, bos_token, eos_token, messages, tools);

tests/test-chat.cpp

@@ -19,12 +19,12 @@
 #include <fstream>
 #include <functional>
 #include <iostream>
-#include <nlohmann/json.hpp>
+#include "json.h"
 #include <set>
 #include <stdexcept>
 #include <string>
 
-using json = nlohmann::ordered_json;
+using json = common_json;
 
 static std::ostream & operator<<(std::ostream & os, const common_chat_msg_diff & diff) {
     os << "{ content_delta: " << diff.content_delta << "; ";

tests/test-grammar-integration.cpp

@@ -7,13 +7,13 @@
 #include "../src/unicode.h"
 #include "../src/llama-grammar.h"
 
-#include <nlohmann/json.hpp>
+#include "json.h"
 
 #include <cassert>
 #include <string>
 #include <vector>
 
-using json = nlohmann::ordered_json;
+using json = common_json;
 
 static llama_grammar * build_grammar_with_root(const std::string & grammar_str, const char * grammar_root) {
     return llama_grammar_init_impl(nullptr, grammar_str.c_str(), grammar_root, false, nullptr, 0, nullptr, 0);