model: add dots3-note (#27060)

* text: conversion

* init impl

* address review comments

* fix rope

* move to a new llama_kv_cache_dsa_iswa

.github/actions/ccache-clear/action.yml

@@ -1,3 +1,4 @@
+# note: place this as the last step of the job, so the new cache is saved by "Post ccache" right after the old one is cleared
 name: "ccache-clear"
 description: "Delete all GitHub Actions caches matching a key prefix"
 inputs:

.github/workflows/build-cmake-pkg.yml

@@ -27,30 +27,26 @@ jobs:
           cmake --install build --prefix "$PREFIX" --config Release
 
           export LLAMA_CONFIG="$PREFIX"/lib/cmake/llama/llama-config.cmake
-          tclsh <<'EOF'
-          set build(commit)  [string trim [exec git rev-parse --short HEAD]]
-          set build(number)  [string trim [exec git rev-list  --count HEAD]]
+          build_commit=$(git rev-parse --short HEAD | xargs)
+          build_number=$(git rev-list --count HEAD | xargs)
 
-          set cmakelists [read [open "CMakeLists.txt" r]]
-          regexp {set\(LLAMA_VERSION_MAJOR\s+(\d+)\)} $cmakelists -> major
-          regexp {set\(LLAMA_VERSION_MINOR\s+(\d+)\)} $cmakelists -> minor
-          regexp {set\(LLAMA_VERSION_PATCH\s+(\d+)\)} $cmakelists -> patch
-          set build(version) "$major.$minor.$patch"
+          major=$(grep -oE "set\(LLAMA_VERSION_MAJOR[[:space:]]+[0-9]+" CMakeLists.txt | grep -oE "[0-9]+$")
+          minor=$(grep -oE "set\(LLAMA_VERSION_MINOR[[:space:]]+[0-9]+" CMakeLists.txt | grep -oE "[0-9]+$")
+          patch=$(grep -oE "set\(LLAMA_VERSION_PATCH[[:space:]]+[0-9]+" CMakeLists.txt | grep -oE "[0-9]+$")
+          build_version="$major.$minor.$patch"
 
-          set llamaconfig [read [open "$env(LLAMA_CONFIG)" r]]
-          set checks [list "set\\(LLAMA_VERSION     \\s+$build(version)\\)" \
-                           "set\\(LLAMA_BUILD_COMMIT\\s+$build(commit)\\)" \
-                           "set\\(LLAMA_BUILD_NUMBER\\s+$build(number)\\)"]
+          checks=("set\(LLAMA_VERSION[[:space:]]+$build_version\)"
+                  "set\(LLAMA_BUILD_COMMIT[[:space:]]+$build_commit\)"
+                  "set\(LLAMA_BUILD_NUMBER[[:space:]]+$build_number\)")
 
-          puts -nonewline "Checking llama-config.cmake version... "
-          foreach check $checks {
-              if {![regexp -expanded -- $check $llamaconfig]} {
-                  puts "\"$check\" failed!"
+          for check in "${checks[@]}"; do
+              if ! grep -qE "$check" "$LLAMA_CONFIG"; then
+                  echo "Checking llama-config.cmake version... \"$check\" failed!"
                   exit 1
-              }
-          }
-          puts "success."
-          EOF
+              fi
+          done
+
+          echo "Checking llama-config.cmake version... success."
 
           cd examples/simple-cmake-pkg
           cmake -S . -B build -DCMAKE_PREFIX_PATH="$PREFIX"/lib/cmake

.github/workflows/build-cpu.yml

@@ -97,8 +97,7 @@ jobs:
           cmake -B build \
             -DGGML_NATIVE=OFF \
             -DLLAMA_FATAL_WARNINGS=ON \
-            -DGGML_RPC=ON \
-            -DGGML_NATIVE=OFF
+            -DGGML_RPC=ON
           time cmake --build build --config Release -j $(nproc)
 
       - name: Test
@@ -119,6 +118,7 @@ jobs:
           ./bin/llama-completion -m stories260K.gguf -p "One day, Lily met a Shoggoth" -n 500 -c 256
 
   windows:
+    name: windows / ${{ matrix.build }}
     runs-on: windows-2025
 
     env:
@@ -130,13 +130,13 @@ jobs:
         include:
           - build: 'x64-cpu-static'
             arch: 'x64'
-            defines: '-G "Ninja Multi-Config" -D CMAKE_TOOLCHAIN_FILE=cmake/x64-windows-llvm.cmake -DGGML_NATIVE=OFF -DLLAMA_BUILD_SERVER=ON -DGGML_RPC=ON -DBUILD_SHARED_LIBS=OFF'
+            defines: '-G "Ninja Multi-Config" -D CMAKE_TOOLCHAIN_FILE=cmake/x64-windows-llvm.cmake -DGGML_NATIVE=OFF -DGGML_OPENMP_FETCH=ON -DLLAMA_BUILD_SERVER=ON -DGGML_RPC=ON -DBUILD_SHARED_LIBS=OFF'
           - build: 'x64-openblas'
             arch: 'x64'
             defines: '-G "Ninja Multi-Config" -D CMAKE_TOOLCHAIN_FILE=cmake/x64-windows-llvm.cmake -DGGML_NATIVE=OFF -DLLAMA_BUILD_SERVER=ON -DGGML_RPC=ON -DGGML_BACKEND_DL=ON -DGGML_CPU_ALL_VARIANTS=ON -DGGML_OPENMP=OFF -DGGML_BLAS=ON -DGGML_BLAS_VENDOR=OpenBLAS -DBLAS_INCLUDE_DIRS="$env:RUNNER_TEMP/openblas/include" -DBLAS_LIBRARIES="$env:RUNNER_TEMP/openblas/lib/openblas.lib"'
           - build: 'arm64'
             arch: 'arm64'
-            defines: '-G "Ninja Multi-Config" -D CMAKE_TOOLCHAIN_FILE=cmake/arm64-windows-llvm.cmake -DGGML_NATIVE=OFF -DLLAMA_BUILD_SERVER=ON'
+            defines: '-G "Ninja Multi-Config" -D CMAKE_TOOLCHAIN_FILE=cmake/arm64-windows-llvm.cmake -DGGML_NATIVE=OFF -DGGML_OPENMP_FETCH=ON -DLLAMA_BUILD_SERVER=ON'
 
     steps:
       - name: Clone

.github/workflows/make-release.yml

@@ -55,6 +55,20 @@ jobs:
         env:
           GITHUB_REPOSITORY: ${{ github.repository }}
 
+      - name: Create nightly-tag.txt
+        id: nightly_tag_file
+        run: |
+          NIGHTLY_TAG="${{ steps.desc.outputs.nightly_tag }}"
+          if [[ -z "${NIGHTLY_TAG}" ]]; then
+            echo "Warning: no nightly tag found for the release commit - nightly-tag.txt will not be created"
+            echo "create=false" >> "$GITHUB_OUTPUT"
+            exit 0
+          fi
+          echo "${NIGHTLY_TAG}" > nightly-tag.txt
+          echo "create=true" >> "$GITHUB_OUTPUT"
+          echo "nightly-tag.txt:"
+          cat nightly-tag.txt
+
       - name: Create release
         if: ${{ github.event.inputs.dry_run == 'false' }}
         uses: ggml-org/action-create-release@v1
@@ -62,26 +76,49 @@ jobs:
           GITHUB_TOKEN: ${{ github.token }}
         with:
           tag_name: ${{ steps.checks.outputs.version }}
-          # TODO: remove the prerelease flag once the semantic versioning workflow is ready
-          # ref: https://github.com/ggml-org/ggml/discussions/1579
-          prerelease: true
+          prerelease: false
+          # TODO: enrich the body of the release with more information
           body: |
-            > [!NOTE]
-            > Semantic versioning is still work in progress.
-            > More info can be found in https://github.com/ggml-org/ggml/discussions/1579
+            ## Overview
+
+            New version has been released.
 
             ${{ steps.desc.outputs.nightly }}
 
+            **Web UI:** the `nightly-tag.txt` asset contains the tag of the corresponding nightly release
+
+            **More info:** [dist : releases and versioning of ggml-org projects](https://github.com/ggml-org/ggml/discussions/1579)
+
             ## ${{ steps.desc.outputs.changelog_title }}
 
             ${{ steps.desc.outputs.changelog }}
 
+      - name: Upload nightly-tag.txt
+        if: ${{ github.event.inputs.dry_run == 'false' && steps.nightly_tag_file.outputs.create == 'true' }}
+        uses: actions/github-script@v8
+        with:
+          github-token: ${{secrets.GITHUB_TOKEN}}
+          script: |
+            const fs = require('fs');
+            const release_id = '${{ steps.create_release.outputs.id }}';
+            console.log('uploadReleaseAsset', 'nightly-tag.txt');
+            await github.rest.repos.uploadReleaseAsset({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              release_id: release_id,
+              name: 'nightly-tag.txt',
+              data: await fs.readFileSync('./nightly-tag.txt')
+            });
+
       - name: Dry run summary
         if: ${{ github.event.inputs.dry_run == 'true' }}
         run: |
           if [[ "${{ steps.checks.outputs.checks_passed }}" == "true" ]]; then
             echo "Dry run complete - all checks passed."
             echo "Would have created tag: ${{ steps.checks.outputs.version }}"
+            if [[ -n "${{ steps.desc.outputs.nightly_tag }}" ]]; then
+              echo "Would have uploaded nightly-tag.txt: ${{ steps.desc.outputs.nightly_tag }}"
+            fi
           else
             echo "::error::Dry run found release check failures. A release tag would not be created."
             exit 1

.github/workflows/release.yml

@@ -145,11 +145,6 @@ jobs:
             ${{ env.CMAKE_ARGS }}
           cmake --build build --config Release -j $(sysctl -n hw.logicalcpu)
 
-      - name: ccache-clear
-        uses: ./.github/actions/ccache-clear
-        with:
-          key: release-${{ matrix.os }}-${{ matrix.arch }}
-
       - name: Determine tag name
         id: tag
         uses: ./.github/actions/get-tag-name
@@ -166,6 +161,11 @@ jobs:
           path: llama-${{ steps.tag.outputs.name }}-bin-macos-${{ matrix.build }}.tar.gz
           name: llama-bin-macos-${{ matrix.build }}.tar.gz
 
+      - name: ccache-clear
+        uses: ./.github/actions/ccache-clear
+        with:
+          key: release-${{ matrix.os }}-${{ matrix.arch }}
+
   ubuntu-cpu:
     needs: [check-release, get-version]
     if: ${{ needs.check-release.outputs.should_release == 'true' }}
@@ -231,12 +231,6 @@ jobs:
             ${{ env.CMAKE_ARGS }}
           cmake --build build --config Release -j $(nproc)
 
-      - name: ccache-clear
-        if: ${{ matrix.build != 's390x' }}
-        uses: ./.github/actions/ccache-clear
-        with:
-          key: release-${{ matrix.os }}-cpu
-
       - name: Determine tag name
         id: tag
         uses: ./.github/actions/get-tag-name
@@ -253,6 +247,12 @@ jobs:
           path: llama-${{ steps.tag.outputs.name }}-bin-ubuntu-${{ matrix.build }}.tar.gz
           name: llama-bin-ubuntu-${{ matrix.build }}.tar.gz
 
+      - name: ccache-clear
+        if: ${{ matrix.build != 's390x' }}
+        uses: ./.github/actions/ccache-clear
+        with:
+          key: release-${{ matrix.os }}-cpu
+
   ubuntu-vulkan:
     needs: [check-release, get-version]
     if: ${{ needs.check-release.outputs.should_release == 'true' }}
@@ -318,11 +318,6 @@ jobs:
             ${{ env.CMAKE_ARGS }}
           cmake --build build --config Release -j $(nproc)
 
-      - name: ccache-clear
-        uses: ./.github/actions/ccache-clear
-        with:
-          key: release-${{ matrix.os }}-vulkan
-
       - name: Determine tag name
         id: tag
         uses: ./.github/actions/get-tag-name
@@ -339,6 +334,11 @@ jobs:
           path: llama-${{ steps.tag.outputs.name }}-bin-ubuntu-vulkan-${{ matrix.build }}.tar.gz
           name: llama-bin-ubuntu-vulkan-${{ matrix.build }}.tar.gz
 
+      - name: ccache-clear
+        uses: ./.github/actions/ccache-clear
+        with:
+          key: release-${{ matrix.os }}-vulkan
+
   android-arm64:
     needs: [check-release, get-version]
     if: ${{ needs.check-release.outputs.should_release == 'true' }}
@@ -512,11 +512,6 @@ jobs:
             ${{ env.CMAKE_ARGS }}
           cmake --build build/ReleaseOV --config Release --parallel
 
-      - name: ccache-clear
-        uses: ./.github/actions/ccache-clear
-        with:
-          key: release-ubuntu-24.04-openvino-release-no-preset-v1
-
       - name: Determine tag name
         id: tag
         uses: ./.github/actions/get-tag-name
@@ -551,6 +546,11 @@ jobs:
           path: llama-${{ steps.tag.outputs.name }}-bin-ubuntu-openvino-${{ env.OPENVINO_VERSION_MAJOR }}-x64.tar.gz
           name: llama-bin-ubuntu-openvino-${{ env.OPENVINO_VERSION_MAJOR }}-x64.tar.gz
 
+      - name: ccache-clear
+        uses: ./.github/actions/ccache-clear
+        with:
+          key: release-ubuntu-24.04-openvino-release-no-preset-v1
+
   windows-openvino:
     needs: [check-release]
     if: ${{ needs.check-release.outputs.should_release == 'true' }}
@@ -637,11 +637,6 @@ jobs:
 
           cmake --build build\ReleaseOV --config Release -- /m
 
-      - name: ccache-clear
-        uses: ./.github/actions/ccache-clear
-        with:
-          key: release-windows-2022-openvino
-
       - name: Determine tag name
         id: tag
         uses: ./.github/actions/get-tag-name
@@ -680,7 +675,13 @@ jobs:
           path: llama-${{ steps.tag.outputs.name }}-bin-win-openvino-${{ env.OPENVINO_VERSION_MAJOR }}-x64.zip
           name: llama-bin-win-openvino-${{ env.OPENVINO_VERSION_MAJOR }}-x64.zip
 
+      - name: ccache-clear
+        uses: ./.github/actions/ccache-clear
+        with:
+          key: release-windows-2022-openvino
+
   windows-cpu:
+    name: windows-cpu / ${{ matrix.arch }}
     needs: [check-release]
     if: ${{ needs.check-release.outputs.should_release == 'true' }}
 
@@ -728,18 +729,13 @@ jobs:
             -DGGML_BACKEND_DL=ON ^
             -DGGML_CPU_ALL_VARIANTS=${{ matrix.arch == 'x64' && 'ON' || 'OFF' }} ^
             -DGGML_OPENMP=ON ^
+            -DGGML_OPENMP_FETCH=ON ^
             ${{ env.CMAKE_ARGS }}
           cmake --build build --config Release
 
-      - name: ccache-clear
-        uses: ./.github/actions/ccache-clear
-        with:
-          key: release-windows-2025-vs2026-${{ matrix.arch }}-cpu
-
       - name: Pack artifacts
         id: pack_artifacts
         run: |
-          Copy-Item "C:\Program Files\Microsoft Visual Studio\18\Enterprise\VC\Redist\MSVC\14.51.36231\debug_nonredist\${{ matrix.arch }}\Microsoft.VC145.OpenMP.LLVM\libomp140.${{ matrix.arch == 'x64' && 'x86_64' || 'aarch64' }}.dll" .\build\bin\Release\
           7z a -snl llama-bin-win-cpu-${{ matrix.arch }}.zip .\build\bin\Release\*
 
       - name: Upload artifacts
@@ -748,6 +744,11 @@ jobs:
           path: llama-bin-win-cpu-${{ matrix.arch }}.zip
           name: llama-bin-win-cpu-${{ matrix.arch }}.zip
 
+      - name: ccache-clear
+        uses: ./.github/actions/ccache-clear
+        with:
+          key: release-windows-2025-vs2026-${{ matrix.arch }}-cpu
+
   windows-rocm:
     needs: [check-release]
     if: ${{ needs.check-release.outputs.should_release == 'true' }}
@@ -840,11 +841,6 @@ jobs:
             -DAMDGPU_TARGETS="${{ matrix.gpu_targets }}"
           cmake --build . --config Release --parallel ${env:NUMBER_OF_PROCESSORS}
 
-      - name: ccache-clear
-        uses: ./.github/actions/ccache-clear
-        with:
-          key: windows-rocm-${{ matrix.ROCM_VERSION }}-${{ matrix.build }}
-
       - name: Verify HIP backend was built
         run: |
           $hipDll = Get-ChildItem -Path build\bin -Filter "ggml-hip*.dll" -ErrorAction SilentlyContinue
@@ -877,6 +873,11 @@ jobs:
           path: llama-bin-win-rocm-${{ env.ROCM_VERSION_SHORT }}-${{ matrix.build }}.zip
           name: llama-bin-win-rocm-${{ env.ROCM_VERSION_SHORT }}-${{ matrix.build }}.zip
 
+      - name: ccache-clear
+        uses: ./.github/actions/ccache-clear
+        with:
+          key: windows-rocm-${{ matrix.ROCM_VERSION }}-${{ matrix.build }}
+
   windows:
     needs: [check-release]
     if: ${{ needs.check-release.outputs.should_release == 'true' }}
@@ -1042,11 +1043,6 @@ jobs:
           set /A NINJA_JOBS=%NUMBER_OF_PROCESSORS%-1
           cmake --build build --config Release -j %NINJA_JOBS% --target ggml-cuda
 
-      - name: ccache-clear
-        uses: ./.github/actions/ccache-clear
-        with:
-          key: release-windows-2022-${{ matrix.arch }}-cuda-${{ matrix.cuda }}
-
       - name: Pack artifacts
         id: pack_artifacts
         run: |
@@ -1082,6 +1078,11 @@ jobs:
           path: cudart-llama-bin-win-cuda-${{ matrix.cuda }}-${{ matrix.arch }}.zip
           name: cudart-llama-bin-win-cuda-${{ matrix.cuda }}-${{ matrix.arch }}.zip
 
+      - name: ccache-clear
+        uses: ./.github/actions/ccache-clear
+        with:
+          key: release-windows-2022-${{ matrix.arch }}-cuda-${{ matrix.cuda }}
+
   windows-sycl:
     needs: [check-release]
     if: ${{ needs.check-release.outputs.should_release == 'true' }}
@@ -1141,11 +1142,6 @@ jobs:
             -DLLAMA_BUILD_BORINGSSL=ON
           cmake --build build --target ggml-sycl -j %NUMBER_OF_PROCESSORS%
 
-      - name: ccache-clear
-        uses: ./.github/actions/ccache-clear
-        with:
-          key: release-windows-2022-x64-sycl
-
       - name: Build the release package
         id: pack_artifacts
         run: |
@@ -1192,6 +1188,11 @@ jobs:
           path: llama-bin-win-sycl-x64.zip
           name: llama-bin-win-sycl-x64.zip
 
+      - name: ccache-clear
+        uses: ./.github/actions/ccache-clear
+        with:
+          key: release-windows-2022-x64-sycl
+
   ubuntu-24-sycl:
     needs: [check-release]
     if: ${{ needs.check-release.outputs.should_release == 'true' }}
@@ -1264,11 +1265,6 @@ jobs:
             -DGGML_SYCL_F16=${{ matrix.fp16 }}
           time cmake --build build --config Release -j $(nproc)
 
-      - name: ccache-clear
-        uses: ./.github/actions/ccache-clear
-        with:
-          key: release-ubuntu-24.04-sycl-${{ matrix.build }}
-
       - name: Determine tag name
         id: tag
         uses: ./.github/actions/get-tag-name
@@ -1285,6 +1281,11 @@ jobs:
           path: llama-${{ steps.tag.outputs.name }}-bin-ubuntu-sycl-${{ matrix.build }}-x64.tar.gz
           name: llama-bin-ubuntu-sycl-${{ matrix.build }}-x64.tar.gz
 
+      - name: ccache-clear
+        uses: ./.github/actions/ccache-clear
+        with:
+          key: release-ubuntu-24.04-sycl-${{ matrix.build }}
+
 #   ubuntu-22-rocm:
 #     needs: [check-release, get-version]
 #     if: ${{ needs.check-release.outputs.should_release == 'true' }}
@@ -1379,11 +1380,6 @@ jobs:
 #             ${{ env.CMAKE_ARGS }}
 #           cmake --build build --config Release -j $(nproc)
 
-#       # - name: ccache-clear
-#       #   uses: ./.github/actions/ccache-clear
-#       #   with:
-#       #     key: release-ubuntu-22.04-rocm-${{ matrix.ROCM_VERSION }}
-
 #       - name: Determine tag name
 #         id: tag
 #         uses: ./.github/actions/get-tag-name
@@ -1403,6 +1399,11 @@ jobs:
 #           path: llama-${{ steps.tag.outputs.name }}-bin-ubuntu-rocm-${{ env.ROCM_VERSION_SHORT }}-${{ matrix.build }}.tar.gz
 #           name: llama-bin-ubuntu-rocm-${{ env.ROCM_VERSION_SHORT }}-${{ matrix.build }}.tar.gz
 
+#       # - name: ccache-clear
+#       #   uses: ./.github/actions/ccache-clear
+#       #   with:
+#       #     key: release-ubuntu-22.04-rocm-${{ matrix.ROCM_VERSION }}
+
   ios-xcode:
     needs: [check-release, get-version]
     if: ${{ needs.check-release.outputs.should_release == 'true' }}
@@ -1687,6 +1688,7 @@ jobs:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         with:
           tag_name: ${{ steps.tag.outputs.name }}
+          prerelease: true
           body: |
             <details open>
 

CMakeLists.txt

@@ -4,8 +4,8 @@ include(CheckIncludeFileCXX)
 
 ### llama.cpp version
 set(LLAMA_VERSION_MAJOR 0)
-set(LLAMA_VERSION_MINOR 1)
-set(LLAMA_VERSION_PATCH 2)
+set(LLAMA_VERSION_MINOR 2)
+set(LLAMA_VERSION_PATCH 0)
 set(LLAMA_VERSION_BASE "${LLAMA_VERSION_MAJOR}.${LLAMA_VERSION_MINOR}.${LLAMA_VERSION_PATCH}")
 
 # whether this is a development/nightly build

cmake/arm64-windows-llvm.cmake

@@ -8,6 +8,7 @@ set( CMAKE_CXX_COMPILER  clang++ )
 
 set( CMAKE_C_COMPILER_TARGET   ${target} )
 set( CMAKE_CXX_COMPILER_TARGET ${target} )
+set( CMAKE_ASM_COMPILER_TARGET ${target} )
 
 set( arch_c_flags "-march=armv8.7-a -fvectorize -ffp-model=fast -fno-finite-math-only" )
 set( warn_c_flags "-Wno-format -Wno-unused-variable -Wno-unused-function -Wno-gnu-zero-variadic-macro-arguments" )

common/arg.cpp

@@ -2595,6 +2595,26 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
             params.mmproj_use_gpu = value;
         }
     ).set_examples(mmproj_examples).set_env("LLAMA_ARG_MMPROJ_OFFLOAD"));
+    add_opt(common_arg(
+        // note: "-mmdev" must sort after "--rpc" in the preset map, else RPC devices are not registered yet
+        {"-mmdev", "--mmproj-device"}, "DEVICE",
+        "device to use for multimodal projector (none = don't offload, default: auto)\n"
+        "use --list-devices to see a list of available devices",
+        [](common_params & params, const std::string & value) {
+            if (value == "none") {
+                params.mmproj_use_gpu = false;
+                params.mmproj_device  = nullptr;
+                return;
+            }
+            auto devices = parse_device_list(value);
+            // parse_device_list pushes nullptr at back so devices is length 2 for single device.
+            if (devices.size() > 2) {
+                throw std::invalid_argument("only one device may be specified for mmproj");
+            }
+            params.mmproj_use_gpu = true;
+            params.mmproj_device  = devices.front();
+        }
+    ).set_examples(mmproj_examples).set_env("MTMD_BACKEND_DEVICE")); // no LLAMA_ARG_ prefix for backward compatibility reason
     add_opt(common_arg(
         {"--image", "--audio", "--video"}, "FILE",
         "path to an image, audio, or video file. use with multimodal models, use comma-separated values for multiple files\n",

common/common.h

@@ -581,9 +581,10 @@ struct common_params {
 
     // multimodal models (see tools/mtmd)
     struct common_params_model mmproj;
-    bool mmproj_use_gpu = true;     // use GPU for multimodal model
-    bool no_mmproj = false;         // explicitly disable multimodal model
-    std::vector<std::string> image; // path to image file(s) ; TODO: change the name to "media"
+    bool mmproj_use_gpu = true;                 // use GPU for multimodal model
+    ggml_backend_dev_t mmproj_device = nullptr; // GPU device to use for multimodal model
+    bool no_mmproj = false;                     // explicitly disable multimodal model
+    std::vector<std::string> image;             // path to image file(s) ; TODO: change the name to "media"
     int image_min_tokens = -1;
     int image_max_tokens = -1;
     int mtmd_batch_max_tokens = 1024;

conversion/__init__.py

@@ -57,12 +57,16 @@ TEXT_MODEL_MAP: dict[str, str] = {
     "Qwen3DSparkModel": "qwen",
     "DSparkDraftModel": "qwen",
     "DSparkSpeculator": "qwen",
+    "Lfm2DSparkDraftModel": "qwen",
     "DeepseekV4ForCausalLM": "deepseek",
     "DeepseekV4DSparkModel": "deepseek",
     "DistilBertForMaskedLM": "bert",
     "DistilBertForSequenceClassification": "bert",
     "DistilBertModel": "bert",
     "Dots1ForCausalLM": "dots1",
+    "Dots3NoteForCausalLM": "dots3",
+    "Dots3NoteForConditionalGeneration": "dots3",
+    "Dots3NoteTextForCausalLM": "dots3",
     "DotsOCRForCausalLM": "qwen",
     "DreamModel": "dream",
     "Ernie4_5ForCausalLM": "ernie",

conversion/dots3.py

@@ -0,0 +1,195 @@
+from __future__ import annotations
+
+import math
+import re
+
+from typing import TYPE_CHECKING, Callable, Iterable
+
+if TYPE_CHECKING:
+    from torch import Tensor
+
+from .base import ModelBase, gguf
+
+from .deepseek import DeepseekV2Model
+
+
+@ModelBase.register("Dots3NoteForCausalLM", "Dots3NoteForConditionalGeneration", "Dots3NoteTextForCausalLM")
+class Dots3NoteModel(DeepseekV2Model):
+    model_arch = gguf.MODEL_ARCH.DOTS3NOTE
+    skip_mtp = False
+    supports_mtp_export = True
+
+    # trunk layer count, stashed before indexing for filter_tensors (mirrors DeepseekV32Model)
+    _n_main_layers: int | None = None
+
+    def index_tensors(self, remote_hf_model_id: str | None = None):
+        type(self)._n_main_layers = self.hparams["num_hidden_layers"]
+        return super().index_tensors(remote_hf_model_id=remote_hf_model_id)
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+
+        hparams = self.hparams
+
+        # config file doesn't specify MTP block, detect it from model weight
+        self.n_nextn = 1 if "model.mtp.embed_tokens.weight" in self.model_tensors else 0
+        if self.n_nextn:
+            self.block_count += self.n_nextn
+            self.tensor_map = gguf.get_tensor_name_map(self.model_arch, self.block_count)
+
+        self.layer_types = hparams["layer_types"]
+        if len(self.layer_types) < hparams["num_hidden_layers"]:
+            raise ValueError("layer_types is shorter than num_hidden_layers")
+
+        if hparams.get("use_dsa", True) is not True:
+            raise ValueError("dots3-note conversion requires use_dsa=true")
+        if hparams.get("normalization", "RMSNorm") != "RMSNorm" or hparams.get("final_norm", "RMSNorm") != "RMSNorm":
+            raise ValueError("dots3-note conversion only supports RMSNorm")
+        if hparams.get("k_rope_only_layernorm", True) is not True:
+            raise ValueError("dots3-note conversion requires k_rope_only_layernorm=true")
+        if hparams.get("topk_method", "noaux_tc") != "noaux_tc" or hparams.get("scoring_func") != "sigmoid":
+            raise ValueError("dots3-note conversion only supports noaux_tc/sigmoid expert gating")
+        if hparams.get("n_group", 1) != 1 or hparams.get("topk_group", 1) != 1:
+            raise ValueError("dots3-note conversion does not support grouped expert routing")
+        if hparams.get("use_dynamic_rsf", False) or hparams.get("moe_gating_fp32", False):
+            raise ValueError("dots3-note conversion does not support use_dynamic_rsf/moe_gating_fp32")
+        for key in ("attention_gate_type", "swa_attention_gate_type"):
+            if hparams.get(key, "headwise") != "headwise":
+                raise ValueError(f"dots3-note conversion only supports headwise attention gate, got {key}={hparams.get(key)!r}")
+        if hparams["swa_qk_nope_head_dim"] + hparams["swa_qk_rope_head_dim"] != hparams.get("swa_head_dim", 256):
+            raise ValueError("swa_head_dim must equal swa_qk_nope_head_dim + swa_qk_rope_head_dim")
+        if hparams["swa_qk_rope_head_dim"] != hparams["qk_rope_head_dim"]:
+            # both layer kinds share a single rope_dimension_count
+            raise ValueError("swa_qk_rope_head_dim must match qk_rope_head_dim")
+
+        self.apply_lora_rescale = hparams.get("apply_mla_qkv_lora_rescale", False)
+
+    def _is_swa_layer(self, bid: int) -> bool:
+        if bid >= self.hparams["num_hidden_layers"]:
+            # note: the NextN/MTP block uses the sliding-attention MLA
+            return True
+        return self.layer_types[bid] == "sliding_attention"
+
+    def set_vocab(self):
+        from transformers import AutoTokenizer
+        tokenizer = AutoTokenizer.from_pretrained(self.dir_model)
+        special_vocab = gguf.SpecialVocab(self.dir_model, load_merges=True)
+        tokens, toktypes, tokpre = self.get_vocab_base()
+        self.gguf_writer.add_tokenizer_model("gpt2")
+        self.gguf_writer.add_tokenizer_pre(tokpre)
+        self.gguf_writer.add_token_list(tokens)
+        self.gguf_writer.add_token_types(toktypes)
+        special_vocab._set_special_token("eot", tokenizer.get_added_vocab()["<|endofassistant|>"])  # ty: ignore[unresolved-attribute]
+        special_vocab.add_to_gguf(self.gguf_writer)
+
+    @classmethod
+    def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
+        if (titem := super().filter_tensors(item)) is None:
+            return None
+        name, gen = titem
+        if name.startswith(("vision_encoder.", "audio_encoder.")):
+            return None
+
+        assert cls._n_main_layers is not None
+        is_mtp = name.startswith("model.mtp.") or \
+            ((m := re.match(r"model\.layers\.(\d+)\.", name)) is not None and int(m.group(1)) >= cls._n_main_layers)
+
+        # --no-mtp: drop the NextN/MTP block; --mtp: keep only that block plus the shared embeddings/norm/lm_head
+        if is_mtp and cls.no_mtp:
+            return None
+        if cls.mtp_only and not is_mtp and name not in (
+            "model.embed_tokens.weight", "model.norm.weight", "lm_head.weight",
+        ):
+            return None
+
+        return name, gen
+
+    def set_gguf_parameters(self):
+        hparams = self.hparams
+
+        # head_count is a per-layer array because the two layer kinds have different head counts
+        n_layer = hparams["num_hidden_layers"]
+        hparams["num_attention_heads"] = [
+            hparams["swa_num_attention_heads"] if self._is_swa_layer(il) else hparams["num_attention_heads"]
+            for il in range(self.block_count)
+        ]
+
+        # prevent the base class from emitting key/value_length from the unused head_dim
+        hparams.pop("head_dim", None)
+
+        super().set_gguf_parameters()
+
+        # MLA geometry of the sliding-window layers (rope.freq_base_swa is emitted by the base class)
+        swa_kv_lora_rank = hparams["swa_kv_lora_rank"]
+        self.gguf_writer.add_sliding_window(hparams["sliding_window_size"])
+        self.gguf_writer.add_sliding_window_pattern([self._is_swa_layer(il) for il in range(n_layer)])
+        self.gguf_writer.add_kv_lora_rank_swa(swa_kv_lora_rank)
+        self.gguf_writer.add_key_length_swa(swa_kv_lora_rank + hparams["swa_qk_rope_head_dim"])
+        self.gguf_writer.add_value_length_swa(swa_kv_lora_rank)
+        self.gguf_writer.add_key_length_mla_swa(hparams["swa_qk_nope_head_dim"] + hparams["swa_qk_rope_head_dim"])
+        self.gguf_writer.add_value_length_mla_swa(hparams["swa_v_head_dim"])
+        if hparams["swa_q_lora_rank"] != hparams["q_lora_rank"]:
+            raise ValueError("dots3-note conversion assumes a shared q_lora_rank for both layer kinds")
+
+        if self.n_nextn:
+            self.gguf_writer.add_nextn_predict_layers(self.n_nextn)
+
+        # DSA indexer (full-attention layers only)
+        self.gguf_writer.add_indexer_head_count(hparams["index_n_heads"])
+        self.gguf_writer.add_indexer_key_length(hparams["index_head_dim"])
+        self.gguf_writer.add_indexer_top_k(hparams["index_topk"])
+        self.gguf_writer.add_indexer_types([not self._is_swa_layer(il) for il in range(n_layer)])
+
+    def prepare_metadata(self, vocab_only: bool):
+        from_dir = self.fname_out.is_dir()
+        super().prepare_metadata(vocab_only=vocab_only)
+
+        if not self.mtp_only or not from_dir:
+            return
+
+        output_type: str = self.ftype.name.partition("_")[2]
+        fname_default: str = gguf.naming_convention(
+            self.metadata.name, self.metadata.basename, self.metadata.finetune,
+            self.metadata.version, size_label=None, output_type=output_type, model_type=None)
+        self.fname_out = self.fname_out.parent / f"mtp-{fname_default}.gguf"
+
+    def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
+        # move the MTP token embedding into the NextN block so the standard nextn mapping picks it up
+        if name == "model.mtp.embed_tokens.weight":
+            name = f"model.layers.{self.hparams['num_hidden_layers']}.embed_tokens.weight"
+            bid = self.hparams["num_hidden_layers"]
+
+        # fold the activation rescale sqrt(n_embd/lora_rank) into the preceding RMSNorm weight
+        # this also covers the indexer wq_b, which reads the same rescaled q_lora activation
+        if self.apply_lora_rescale and bid is not None:
+            if name.endswith("q_a_layernorm.weight"):
+                data_torch = data_torch * math.sqrt(self.hparams["hidden_size"] / self.hparams["q_lora_rank"])
+            elif name.endswith("kv_a_layernorm.weight"):
+                rank = self.hparams["swa_kv_lora_rank"] if self._is_swa_layer(bid) else self.hparams["kv_lora_rank"]
+                data_torch = data_torch * math.sqrt(self.hparams["hidden_size"] / rank)
+
+        # MLA absorption: split kv_b_proj into k_b (transposed) and v_b, per-layer-kind geometry
+        if name.endswith("kv_b_proj.weight"):
+            assert bid is not None
+            if self._is_swa_layer(bid):
+                n_head = self.hparams["swa_num_attention_heads"]
+                qk_nope_head_dim = self.hparams["swa_qk_nope_head_dim"]
+                v_head_dim = self.hparams["swa_v_head_dim"]
+            else:
+                n_head = self.hparams["num_attention_heads"]
+                qk_nope_head_dim = self.hparams["qk_nope_head_dim"]
+                v_head_dim = self.hparams["v_head_dim"]
+            if isinstance(n_head, list):  # set_gguf_parameters turns this into a per-layer array
+                n_head = n_head[bid]
+
+            assert data_torch.shape[0] == n_head * (qk_nope_head_dim + v_head_dim)
+
+            kv_b = data_torch.view(n_head, qk_nope_head_dim + v_head_dim, data_torch.shape[-1])
+            k_b, v_b = kv_b.split([qk_nope_head_dim, v_head_dim], dim=1)
+            k_b = k_b.transpose(1, 2)
+
+            yield from ModelBase.modify_tensors(self, k_b, name.replace("kv_b_proj", "k_b_proj"), bid)
+            yield from ModelBase.modify_tensors(self, v_b, name.replace("kv_b_proj", "v_b_proj"), bid)
+            return
+
+        yield from super().modify_tensors(data_torch, name, bid)

conversion/qwen.py

@@ -709,7 +709,7 @@ class DFlashModel(Qwen3Model):
         yield from super().modify_tensors(data_torch, name, bid)
 
 
-@ModelBase.register("Qwen3DSparkModel", "DSparkDraftModel", "DSparkSpeculator")
+@ModelBase.register("Qwen3DSparkModel", "DSparkDraftModel", "DSparkSpeculator", "Lfm2DSparkDraftModel")
 @ModelBase.example("satgeze/Qwen3.6-27B-DSpark")
 class DSparkModel(DFlashModel):
     # DSpark = DFlash + a semi-autoregressive Markov head.
@@ -759,6 +759,13 @@ class DSparkModel(DFlashModel):
             return None
         return super().filter_tensors(item)
 
+    _ROPE_PERMUTE_SUFFIXES = (
+        "self_attn.q_proj.weight",
+        "self_attn.k_proj.weight",
+        "self_attn.q_norm.weight",
+        "self_attn.k_norm.weight",
+    )
+
     def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
         if name == "model.d2t":
             self._d2t = data_torch
@@ -767,6 +774,12 @@ class DSparkModel(DFlashModel):
         if self._n_vocab_draft == self.hparams["vocab_size"] and name.endswith(("embed_tokens.weight", "lm_head.weight")):
             return
 
+        # interleaved-rope checkpoints (rope_is_neox_style = false) -> NeoX layout: per head, even dims first then odd
+        if not self.hparams.get("rope_is_neox_style", True) and name.endswith(self._ROPE_PERMUTE_SUFFIXES):
+            head_dim = self.hparams["head_dim"]
+            shape = data_torch.shape
+            data_torch = data_torch.reshape(-1, head_dim // 2, 2, *shape[1:]).transpose(1, 2).reshape(shape)
+
         yield from super().modify_tensors(data_torch, name, bid)
 
     def prepare_tensors(self):

docs/backend/ET.md

@@ -116,7 +116,7 @@ in inline assembler.
 Most kernels are very naive with lots of low hanging fruits left:
 
 > [!IMPORTANT]
-> Several assembly instructions emmited by the compiler are not implemented
+> Several assembly instructions emitted by the compiler are not implemented
 > in hardware and software emulation in firmware is not ready yet.
 > Eventually firmware will transparently trap unimplemented instructions
 > and will emulate them inside exception handler. Until then, kernel
@@ -138,12 +138,12 @@ Most kernels are very naive with lots of low hanging fruits left:
 > kernel build process. Feel free to take ideas/code from there or try linking
 > it in.
 
-Before commiting any changes to operations and/or kernels, don't forget
+Before committing any changes to operations and/or kernels, don't forget
 to update supported ops reports (instructions at `docs/ops.md`).
 
 When logging is enabled (e.g. by setting `--log-file` cli param),
 each compute kernel run outputs a line with
-pipe-delimited key-value pairs containing kernel level performance infomation.
+pipe-delimited key-value pairs containing kernel level performance information.
 Line is prefixed with `ET_PERF`:
 
 ```
@@ -160,7 +160,7 @@ to `GGML_ET_PROFILE/et_runtime_trace.json` and `GGML_ET_PROFILE/kernel_map` on e
 
 ### Uberkernel
 
-The in-knernel implementaiton of device dispatch/kernel fusion. The ET SDK has a non-trivial op-to-op gap. `Uberkernel` (name taken from the original Esperanto AI's compiler)
+The in-kernel implementation of device dispatch/kernel fusion. The ET SDK has a non-trivial op-to-op gap. `Uberkernel` (name taken from the original Esperanto AI's compiler)
 dispatches multiple already existing kernel implementations with device side synchronization. Due to the processor's design, there is no natural memory visibility
 horizon between sub-kernel invocations. This makes uberkernel much more difficult to develop and debug. Currently Uberkerel is hidden begind the
 `GGML_ET_UBERKERNEL` environment variable and is disabled by default. Setting it to 1 enables it and provides significant performance improvements but is only

docs/build.md

@@ -72,9 +72,10 @@ cmake --build build --config Release
     - Please remember to always use a Developer Command Prompt / PowerShell for VS2022 for git, build, test
     - For Windows on ARM (arm64, WoA) build with:
     ```bash
-    cmake --preset arm64-windows-llvm-release -D GGML_OPENMP=OFF
+    cmake --preset arm64-windows-llvm-release -D GGML_OPENMP_FETCH=ON
     cmake --build build-arm64-windows-llvm-release
     ```
+    `GGML_OPENMP_FETCH` downloads the official LLVM OpenMP runtime and requires Clang, 7-Zip and network access during configuration. CMake selects the runtime from the target architecture, so this also works when cross-compiling for WoA from x64. The extracted header, import library, DLL and OpenMP license are placed under `build/_deps`. The build copies `libomp.dll` and `LICENSE-LLVM-OpenMP` to the runtime output directory and installs them together. Omit the option to use CMake's normal OpenMP detection, or pass `-D GGML_OPENMP=OFF` to disable OpenMP.
     For building with ninja generator and clang compiler as default:
       -set path:set LIB=C:\Program Files (x86)\Windows Kits\10\Lib\10.0.22621.0\um\x64;C:\Program Files\Microsoft Visual Studio\2022\Community\VC\Tools\MSVC\14.41.34120\lib\x64\uwp;C:\Program Files (x86)\Windows Kits\10\Lib\10.0.22621.0\ucrt\x64
       ```bash

docs/development/HOWTO-add-model.md

@@ -166,6 +166,19 @@ Examples:
 - Some models require scaling the input position. For example, `[0, 1, 2, ...]` becomes `[0, 0.5, 1, ...]`. In this case, you can provide the scaling via `freq_scale = 0.5f`.
 - Some models use learned RoPE frequencies instead of relying on `powf(freq_base, -2.0 * i / n_dims)`. In this case, you can provide the learned frequencies via the `rope_freqs` tensor (corresponding to the `c` argument in `ggml_rope_ext`), then set `freq_base = 1.0f`. An important note is that `rope_freqs` in GGML is the **inverse** (`theta = pos[i] / rope_freqs`), so you may need to invert `rope_freqs` during conversion.
 
+### Rotating only a part of the head
+
+Many models rotate only a part of each head and leave the rest untouched (often called the "nope" part). Do not build this with views plus `ggml_concat`, it's not efficient. Both layouts can be done with a single RoPE op:
+
+- `[rope|nope]`, rotated dims first: pass `n_dims` smaller than the head size to `ggml_rope_ext`. Dims from `n_dims` to the end are copied as-is.
+- `[nope|rope]`, rotated dims last: call `ggml_rope_set_offset(cur, n_offs)` on the result of the RoPE, where `n_offs` is the size of the leading untouched part. Dims outside `[n_offs, n_offs + n_dims)` are copied as-is.
+
+`n_offs` must be even, `n_offs + n_dims` must fit in the row, and vision RoPE is not supported. Note that the frequencies are computed relative to the rotated window.
+
+Example: DeepSeek-V4 uses `[nope|rope]` for its query, key and compressed KV tensors, so `src/models/deepseek4.cpp` ropes the whole tensor and then calls `ggml_rope_set_offset(cur, n_embd_head_nope)`.
+
+Exception: some models apply an extra op to the `nope` part, for example `deepseek32.cpp`, and may not use this optimization. While RoPE can be applied selectively to a part of the head, the extra op may not, so these models still need views plus `ggml_concat`.
+
 ## GGUF specification
 
 https://github.com/ggml-org/ggml/blob/master/docs/gguf.md

ggml/CMakeLists.txt

@@ -4,8 +4,8 @@ project("ggml" C CXX ASM)
 
 ### GGML Version
 set(GGML_VERSION_MAJOR 0)
-set(GGML_VERSION_MINOR 20)
-set(GGML_VERSION_PATCH 2)
+set(GGML_VERSION_MINOR 21)
+set(GGML_VERSION_PATCH 0)
 set(GGML_VERSION_BASE "${GGML_VERSION_MAJOR}.${GGML_VERSION_MINOR}.${GGML_VERSION_PATCH}")
 
 list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake/")
@@ -243,6 +243,7 @@ set   (GGML_METAL_MACOSX_VERSION_MIN "" CACHE STRING
                                             "ggml: metal minimum macOS version")
 set   (GGML_METAL_STD "" CACHE STRING       "ggml: metal standard version (-std flag)")
 option(GGML_OPENMP                          "ggml: use OpenMP"                                ON)
+option(GGML_OPENMP_FETCH                    "ggml: fetch LLVM OpenMP"                         OFF)
 option(GGML_RPC                             "ggml: use RPC"                                   OFF)
 option(GGML_SYCL                            "ggml: use SYCL"                                  OFF)
 option(GGML_SYCL_F16                        "ggml: use 16 bit floats for sycl calculations"   OFF)

ggml/src/CMakeLists.txt

@@ -222,9 +222,123 @@ if (GGML_SCHED_NO_REALLOC)
     target_compile_definitions(ggml-base PUBLIC GGML_SCHED_NO_REALLOC)
 endif()
 
-if (GGML_OPENMP)
+if (GGML_OPENMP_FETCH)
+    if (NOT GGML_OPENMP)
+        message(FATAL_ERROR "GGML_OPENMP_FETCH requires GGML_OPENMP")
+    elseif (NOT WIN32 OR NOT (CMAKE_C_COMPILER_ID MATCHES "Clang"))
+        message(FATAL_ERROR "GGML_OPENMP_FETCH currently requires Clang on Windows")
+    endif()
+
+    set(GGML_OPENMP_LLVM_VERSION "20.1.8")
+    string(REGEX MATCH "^[0-9]+" GGML_OPENMP_LLVM_VERSION_MAJOR "${GGML_OPENMP_LLVM_VERSION}")
+    string(REGEX MATCH "^[0-9]+" GGML_OPENMP_COMPILER_VERSION_MAJOR "${CMAKE_C_COMPILER_VERSION}")
+    if (NOT GGML_OPENMP_COMPILER_VERSION_MAJOR STREQUAL GGML_OPENMP_LLVM_VERSION_MAJOR)
+        message(FATAL_ERROR "LLVM OpenMP ${GGML_OPENMP_LLVM_VERSION} requires Clang ${GGML_OPENMP_LLVM_VERSION_MAJOR}.x")
+    endif()
+
+    string(TOLOWER "${CMAKE_SYSTEM_PROCESSOR}" GGML_OPENMP_SYSTEM_PROCESSOR)
+    if (GGML_OPENMP_SYSTEM_PROCESSOR MATCHES "^(amd64|x86_64)$")
+        set(GGML_OPENMP_ARCH "x64")
+        set(GGML_OPENMP_INSTALLER_SUFFIX "win64")
+        set(GGML_OPENMP_INSTALLER_SHA256 "3197846a2b19063687dd56e93e34cd941e3548d907f23a6131571321bdf9fe7b")
+    elseif (GGML_OPENMP_SYSTEM_PROCESSOR MATCHES "^(aarch64|arm64)$")
+        set(GGML_OPENMP_ARCH "arm64")
+        set(GGML_OPENMP_INSTALLER_SUFFIX "woa64")
+        set(GGML_OPENMP_INSTALLER_SHA256 "7c4ac97eb2ae6b960ca5f9caf3ff6124c8d2a18cc07a7840a4d2ea15537bad8e")
+    else()
+        message(FATAL_ERROR "GGML_OPENMP_FETCH does not support ${CMAKE_SYSTEM_PROCESSOR}")
+    endif()
+
+    set(GGML_OPENMP_CACHE_DIR "${CMAKE_BINARY_DIR}/_deps")
+    set(GGML_OPENMP_ROOT "${GGML_OPENMP_CACHE_DIR}/llvm-openmp-${GGML_OPENMP_LLVM_VERSION}-${GGML_OPENMP_ARCH}")
+    set(GGML_OPENMP_LIBRARY "${GGML_OPENMP_ROOT}/lib/libomp.lib")
+    set(GGML_OPENMP_RUNTIME "${GGML_OPENMP_ROOT}/bin/libomp.dll")
+    set(GGML_OPENMP_HEADER "${GGML_OPENMP_ROOT}/include/omp.h")
+    set(GGML_OPENMP_LICENSE "${GGML_OPENMP_ROOT}/LICENSE.TXT")
+    set(GGML_OPENMP_LICENSE_SHA256 "fdad1758a9e1f9d5a81e18879b3406772115edc92c24bfa36b70c654f325e8e4")
+
+    if (NOT EXISTS "${GGML_OPENMP_LIBRARY}" OR NOT EXISTS "${GGML_OPENMP_RUNTIME}" OR NOT EXISTS "${GGML_OPENMP_HEADER}")
+        find_program(GGML_OPENMP_7Z NAMES 7z 7zz 7za)
+        if (NOT GGML_OPENMP_7Z)
+            message(FATAL_ERROR "GGML_OPENMP_FETCH requires 7-Zip to extract the LLVM installer")
+        endif()
+
+        set(GGML_OPENMP_INSTALLER "${GGML_OPENMP_ROOT}/LLVM-${GGML_OPENMP_LLVM_VERSION}-${GGML_OPENMP_INSTALLER_SUFFIX}.exe")
+        set(GGML_OPENMP_EXTRACT_DIR "${GGML_OPENMP_ROOT}/extract")
+        set(GGML_OPENMP_INSTALLER_URL "https://github.com/llvm/llvm-project/releases/download/llvmorg-${GGML_OPENMP_LLVM_VERSION}/LLVM-${GGML_OPENMP_LLVM_VERSION}-${GGML_OPENMP_INSTALLER_SUFFIX}.exe")
+
+        file(MAKE_DIRECTORY "${GGML_OPENMP_EXTRACT_DIR}")
+        file(DOWNLOAD "${GGML_OPENMP_INSTALLER_URL}" "${GGML_OPENMP_INSTALLER}"
+            EXPECTED_HASH "SHA256=${GGML_OPENMP_INSTALLER_SHA256}"
+            SHOW_PROGRESS
+            STATUS GGML_OPENMP_DOWNLOAD_STATUS)
+        list(GET GGML_OPENMP_DOWNLOAD_STATUS 0 GGML_OPENMP_DOWNLOAD_RESULT)
+        if (NOT GGML_OPENMP_DOWNLOAD_RESULT EQUAL 0)
+            list(GET GGML_OPENMP_DOWNLOAD_STATUS 1 GGML_OPENMP_DOWNLOAD_ERROR)
+            message(FATAL_ERROR "Failed to download LLVM OpenMP: ${GGML_OPENMP_DOWNLOAD_ERROR}")
+        endif()
+
+        execute_process(
+            COMMAND "${GGML_OPENMP_7Z}" e -y "-o${GGML_OPENMP_EXTRACT_DIR}" "${GGML_OPENMP_INSTALLER}" -r libomp.lib libomp.dll omp.h
+            RESULT_VARIABLE GGML_OPENMP_EXTRACT_RESULT
+            OUTPUT_QUIET)
+        if (NOT GGML_OPENMP_EXTRACT_RESULT EQUAL 0 OR
+            NOT EXISTS "${GGML_OPENMP_EXTRACT_DIR}/libomp.lib" OR
+            NOT EXISTS "${GGML_OPENMP_EXTRACT_DIR}/libomp.dll" OR
+            NOT EXISTS "${GGML_OPENMP_EXTRACT_DIR}/omp.h")
+            message(FATAL_ERROR "Failed to extract libomp from ${GGML_OPENMP_INSTALLER}")
+        endif()
+
+        file(MAKE_DIRECTORY "${GGML_OPENMP_ROOT}/lib" "${GGML_OPENMP_ROOT}/bin" "${GGML_OPENMP_ROOT}/include")
+        file(COPY "${GGML_OPENMP_EXTRACT_DIR}/libomp.lib" DESTINATION "${GGML_OPENMP_ROOT}/lib")
+        file(COPY "${GGML_OPENMP_EXTRACT_DIR}/libomp.dll" DESTINATION "${GGML_OPENMP_ROOT}/bin")
+        file(COPY "${GGML_OPENMP_EXTRACT_DIR}/omp.h" DESTINATION "${GGML_OPENMP_ROOT}/include")
+        file(REMOVE_RECURSE "${GGML_OPENMP_INSTALLER}" "${GGML_OPENMP_EXTRACT_DIR}")
+    endif()
+
+    # The NSIS installer embeds LLVM's general license in its UI but does not install it as a file; use OpenMP's license to include its additional notices.
+    if (EXISTS "${GGML_OPENMP_LICENSE}")
+        file(SHA256 "${GGML_OPENMP_LICENSE}" GGML_OPENMP_LICENSE_ACTUAL_SHA256)
+    endif()
+    if (NOT GGML_OPENMP_LICENSE_ACTUAL_SHA256 STREQUAL GGML_OPENMP_LICENSE_SHA256)
+        file(DOWNLOAD "https://raw.githubusercontent.com/llvm/llvm-project/llvmorg-${GGML_OPENMP_LLVM_VERSION}/openmp/LICENSE.TXT" "${GGML_OPENMP_LICENSE}"
+            EXPECTED_HASH "SHA256=${GGML_OPENMP_LICENSE_SHA256}")
+    endif()
+
+    if (COMMAND license_add_file)
+        license_add_file("LLVM OpenMP" "${GGML_OPENMP_LICENSE}")
+    endif()
+
+    add_library(ggml-openmp-c INTERFACE)
+    target_compile_options(ggml-openmp-c INTERFACE "$<$<COMPILE_LANGUAGE:C>:-fopenmp=libomp>")
+    target_include_directories(ggml-openmp-c SYSTEM INTERFACE "${GGML_OPENMP_ROOT}/include")
+    target_link_libraries(ggml-openmp-c INTERFACE "${GGML_OPENMP_LIBRARY}")
+
+    add_library(ggml-openmp-cxx INTERFACE)
+    target_compile_options(ggml-openmp-cxx INTERFACE "$<$<COMPILE_LANGUAGE:CXX>:-fopenmp=libomp>")
+    target_include_directories(ggml-openmp-cxx SYSTEM INTERFACE "${GGML_OPENMP_ROOT}/include")
+    target_link_libraries(ggml-openmp-cxx INTERFACE "${GGML_OPENMP_LIBRARY}")
+
+    set(GGML_OPENMP_RUNTIME_OUTPUT_DIR "${CMAKE_RUNTIME_OUTPUT_DIRECTORY}")
+    if (CMAKE_CONFIGURATION_TYPES)
+        string(APPEND GGML_OPENMP_RUNTIME_OUTPUT_DIR "/$<CONFIG>")
+    endif()
+    add_custom_target(ggml-openmp-runtime ALL
+        COMMAND ${CMAKE_COMMAND} -E make_directory "${GGML_OPENMP_RUNTIME_OUTPUT_DIR}"
+        COMMAND ${CMAKE_COMMAND} -E copy_if_different "${GGML_OPENMP_RUNTIME}" "${GGML_OPENMP_RUNTIME_OUTPUT_DIR}/libomp.dll"
+        COMMAND ${CMAKE_COMMAND} -E copy_if_different "${GGML_OPENMP_LICENSE}" "${GGML_OPENMP_RUNTIME_OUTPUT_DIR}/LICENSE-LLVM-OpenMP")
+    add_dependencies(ggml-base ggml-openmp-runtime)
+    install(FILES "${GGML_OPENMP_RUNTIME}" DESTINATION ${CMAKE_INSTALL_BINDIR})
+    install(FILES "${GGML_OPENMP_LICENSE}" DESTINATION ${CMAKE_INSTALL_BINDIR} RENAME LICENSE-LLVM-OpenMP)
+
+    set(GGML_OPENMP_TARGET_C ggml-openmp-c)
+    set(GGML_OPENMP_TARGET_CXX ggml-openmp-cxx)
+    set(GGML_OPENMP_ENABLED "ON" CACHE INTERNAL "")
+elseif (GGML_OPENMP)
     find_package(OpenMP)
     if (OpenMP_FOUND)
+        set(GGML_OPENMP_TARGET_C OpenMP::OpenMP_C)
+        set(GGML_OPENMP_TARGET_CXX OpenMP::OpenMP_CXX)
         set(GGML_OPENMP_ENABLED "ON" CACHE INTERNAL "")
     else()
         set(GGML_OPENMP_ENABLED "OFF" CACHE INTERNAL "")
@@ -236,7 +350,7 @@ endif()
 
 if (GGML_OPENMP_ENABLED)
     target_compile_definitions(ggml-base PRIVATE GGML_USE_OPENMP)
-    target_link_libraries(ggml-base PRIVATE OpenMP::OpenMP_C OpenMP::OpenMP_CXX)
+    target_link_libraries(ggml-base PRIVATE ${GGML_OPENMP_TARGET_C} ${GGML_OPENMP_TARGET_CXX})
 endif()
 
 add_library(ggml

ggml/src/ggml-cpu/CMakeLists.txt

@@ -74,7 +74,7 @@ function(ggml_add_cpu_backend_variant_impl tag_name)
 
     if (GGML_OPENMP_ENABLED)
         target_compile_definitions(${GGML_CPU_NAME} PRIVATE GGML_USE_OPENMP)
-        target_link_libraries(${GGML_CPU_NAME} PRIVATE OpenMP::OpenMP_C OpenMP::OpenMP_CXX)
+        target_link_libraries(${GGML_CPU_NAME} PRIVATE ${GGML_OPENMP_TARGET_C} ${GGML_OPENMP_TARGET_CXX})
     endif()
 
     if (GGML_LLAMAFILE)
@@ -639,6 +639,7 @@ function(ggml_add_cpu_backend_variant_impl tag_name)
             ${KLEIDIAI_SRC}/kai/ukernels/matmul/matmul_clamp_fp32_bf16p_bf16p/
             ${KLEIDIAI_SRC}/kai/ukernels/matmul/matmul_clamp_f32_f16p_qsi4c32p/
             ${KLEIDIAI_SRC}/kai/ukernels/matmul/matmul_clamp_f32_f32p_f32p/
+            ${KLEIDIAI_SRC}/kai/ukernels/matmul/matmul_clamp_f32_f32_f32p/
             ${KLEIDIAI_SRC}/kai/ukernels/matmul/pack/)
 
         set(ARCH_FLAGS_TEMP "${ARCH_FLAGS}")
@@ -701,6 +702,8 @@ function(ggml_add_cpu_backend_variant_impl tag_name)
                 ${KLEIDIAI_SRC}/kai/ukernels/matmul/matmul_clamp_f32_f16p_qsi4c32p/kai_matmul_clamp_f32_f16p1vlx2_qsi4c32p4vlx2_1vlx4vl_sme2_mopa_asm.S
                 ${KLEIDIAI_SRC}/kai/ukernels/matmul/matmul_clamp_f32_f32p_f32p/kai_matmul_clamp_f32_f32p2vlx1_f32p2vlx1biasf32_sme2_mopa.c
                 ${KLEIDIAI_SRC}/kai/ukernels/matmul/matmul_clamp_f32_f32p_f32p/kai_matmul_clamp_f32_f32p2vlx1_f32p2vlx1biasf32_sme2_mopa_asm.S
+                ${KLEIDIAI_SRC}/kai/ukernels/matmul/matmul_clamp_f32_f32_f32p/kai_matmul_clamp_f32_f32_f32p2vlx1b_1x16vl_sme2_mla.c
+                ${KLEIDIAI_SRC}/kai/ukernels/matmul/matmul_clamp_f32_f32_f32p/kai_matmul_clamp_f32_f32_f32p2vlx1b_1x16vl_sme2_mla_asm.S
                 ${KLEIDIAI_SRC}/kai/ukernels/matmul/pack/kai_lhs_pack_bf16p2vlx2_f32_sme.c
                 ${KLEIDIAI_SRC}/kai/ukernels/matmul/pack/kai_rhs_pack_kxn_bf16p2vlx2b_f32_x32_sme.c
                 ${KLEIDIAI_SRC}/kai/ukernels/matmul/pack/kai_lhs_pack_f16pmrx2_f32_neon.c
@@ -737,8 +740,9 @@ function(ggml_add_cpu_backend_variant_impl tag_name)
         set_target_properties(${GGML_CPU_NAME} PROPERTIES COMPILE_FLAGS "-msimd128")
     endif()
 
-    if (CMAKE_CXX_COMPILER_ID STREQUAL "IntelLLVM")
-        # The compiler automatically enables "-ffast-math" which can cause NaNs in tests due to "-fassociative-math"
-        target_compile_options(${GGML_CPU_NAME} PRIVATE "-fno-associative-math")
-    endif()
+  if (CMAKE_C_COMPILER_ID STREQUAL "IntelLLVM" OR CMAKE_CXX_COMPILER_ID STREQUAL "IntelLLVM")
+      # The compiler automatically enables "-ffast-math" which can cause NaNs in tests due to "-fassociative-math"
+      target_compile_options(${GGML_CPU_NAME} PRIVATE "$<$<OR:$<COMPILE_LANG_AND_ID:C,IntelLLVM>,$<COMPILE_LANG_AND_ID:CXX,IntelLLVM>>:$<$<BOOL:${WIN32}>:/clang:>-fno-associative-math>")
+  endif()
+
 endfunction()

ggml/src/ggml-cpu/kleidiai/kernels.cpp

@@ -23,6 +23,7 @@
 #include "kai_matmul_clamp_f32_qsi8d32p1x8_qsi4c32p8x8_1x8_sve_dotprod.h"
 #include "kai_matmul_clamp_f32_f16p1vlx2_qsi4c32p4vlx2_1vlx4vl_sme2_mopa.h"
 #include "kai_matmul_clamp_f32_f32p2vlx1_f32p2vlx1biasf32_sme2_mopa.h"
+#include "kai_matmul_clamp_f32_f32_f32p2vlx1b_1x16vl_sme2_mla.h"
 #include "kai_matmul_clamp_f32_f32p2vlx1_f32p2vlx1b_2vlx2vl_sme_mopa.h"
 
 #include "kai_lhs_pack_bf16p2vlx2_f32_sme.h"
@@ -76,6 +77,21 @@ static inline void kernel_run_fn10(size_t m, size_t n, size_t k, size_t /*bl*/,
     Fn(m, n, k, lhs, rhs, dst, dst_stride_row, dst_stride_col, clamp_min, clamp_max);
 }
 
+template <void (*Fn)(size_t, size_t, size_t, const void *, size_t, const void *, void *, size_t, size_t, float, float)>
+static inline void kernel_run_lhs_stride_fn10(size_t       m,
+                                              size_t       n,
+                                              size_t       k,
+                                              size_t       lhs_stride,
+                                              const void * lhs,
+                                              const void * rhs,
+                                              void *       dst,
+                                              size_t       dst_stride_row,
+                                              size_t       dst_stride_col,
+                                              float        clamp_min,
+                                              float        clamp_max) {
+    Fn(m, n, k, lhs, lhs_stride, rhs, dst, dst_stride_row, dst_stride_col, clamp_min, clamp_max);
+}
+
 template<void(*Fn)(size_t,size_t,size_t,const void*,const void*,float*,size_t,size_t,float,float)>
 static inline void kernel_run_float_fn10(size_t m, size_t n, size_t k, size_t /*bl*/,
                                          const void* lhs, const void* rhs, void* dst,
@@ -947,25 +963,25 @@ static ggml_kleidiai_kernels ggml_kleidiai_kernels_f32[] = {
             /* .packed_size_ex        = */ &lhs_ps_fn5<kai_get_lhs_packed_size_lhs_pack_f32p2vlx1_f32_sme>,
             /* .pack_func_ex          = */ &lhs_pack_void_fn9<kai_run_lhs_pack_f32p2vlx1_f32_sme>,
         },
-        /* SME GEMV */
+        /* SME2 GEMV */
         {
-            /* .get_m_step            = */ kai_get_m_step_matmul_clamp_f32_f32p2vlx1_f32p2vlx1biasf32_sme2_mopa,
-            /* .get_n_step            = */ kai_get_n_step_matmul_clamp_f32_f32p2vlx1_f32p2vlx1biasf32_sme2_mopa,
-            /* .get_mr                = */ kai_get_mr_matmul_clamp_f32_f32p2vlx1_f32p2vlx1biasf32_sme2_mopa,
-            /* .get_nr                = */ kai_get_nr_matmul_clamp_f32_f32p2vlx1_f32p2vlx1biasf32_sme2_mopa,
-            /* .get_kr                = */ kai_get_kr_matmul_clamp_f32_f32p2vlx1_f32p2vlx1biasf32_sme2_mopa,
-            /* .get_sr                = */ kai_get_sr_matmul_clamp_f32_f32p2vlx1_f32p2vlx1biasf32_sme2_mopa,
-            /* .get_dst_offset        = */ kai_get_dst_offset_matmul_clamp_f32_f32p2vlx1_f32p2vlx1biasf32_sme2_mopa,
-            /* .get_dst_size          = */ kai_get_dst_size_matmul_clamp_f32_f32p2vlx1_f32p2vlx1biasf32_sme2_mopa,
-            /* .get_lhs_offset_ex     = */ nullptr,
-            /* .get_rhs_packed_offset_ex = */ nullptr,
-            /* .run_kernel_ex         = */ nullptr,
+            /* .get_m_step            = */ kai_get_m_step_matmul_clamp_f32_f32_f32p2vlx1b_1x16vl_sme2_mla,
+            /* .get_n_step            = */ kai_get_n_step_matmul_clamp_f32_f32_f32p2vlx1b_1x16vl_sme2_mla,
+            /* .get_mr                = */ kai_get_m_step_matmul_clamp_f32_f32_f32p2vlx1b_1x16vl_sme2_mla,
+            /* .get_nr                = */ kai_get_nr_matmul_clamp_f32_f32_f32p2vlx1b_1x16vl_sme2_mla,
+            /* .get_kr                = */ kai_get_kr_matmul_clamp_f32_f32_f32p2vlx1b_1x16vl_sme2_mla,
+            /* .get_sr                = */ kai_get_sr_matmul_clamp_f32_f32_f32p2vlx1b_1x16vl_sme2_mla,
+            /* .get_dst_offset        = */ kai_get_dst_offset_matmul_clamp_f32_f32_f32p2vlx1b_1x16vl_sme2_mla,
+            /* .get_dst_size          = */ kai_get_dst_size_matmul_clamp_f32_f32_f32p2vlx1b_1x16vl_sme2_mla,
+            /* .get_lhs_offset_ex     = */ &kernel_offs_fn2<kai_get_lhs_offset_matmul_clamp_f32_f32_f32p2vlx1b_1x16vl_sme2_mla>,
+            /* .get_rhs_packed_offset_ex = */ &kernel_offs_fn2<kai_get_rhs_packed_offset_matmul_clamp_f32_f32_f32p2vlx1b_1x16vl_sme2_mla>,
+            /* .run_kernel_ex         = */ &kernel_run_lhs_stride_fn10<kai_run_matmul_clamp_f32_f32_f32p2vlx1b_1x16vl_sme2_mla>,
         },
         /* .gemv_lhs_info = */ {
-            /* .get_offset            = */ kai_get_lhs_offset_lhs_pack_f32p2vlx1_f32_sme,
-            /* .get_packed_offset_ex  = */ &lhs_offs_fn5<kai_get_lhs_packed_offset_lhs_pack_f32p2vlx1_f32_sme>,
-            /* .packed_size_ex        = */ &lhs_ps_fn5<kai_get_lhs_packed_size_lhs_pack_f32p2vlx1_f32_sme>,
-            /* .pack_func_ex          = */ &lhs_pack_void_fn9<kai_run_lhs_pack_f32p2vlx1_f32_sme>,
+            /* .get_offset            = */ nullptr,
+            /* .get_packed_offset_ex  = */ nullptr,
+            /* .packed_size_ex        = */ nullptr,
+            /* .pack_func_ex          = */ nullptr,
         },
         /* .rhs_info = */ {
             /* .packed_stride         = */ nullptr,

ggml/src/ggml-cpu/kleidiai/kleidiai.cpp

@@ -696,6 +696,15 @@ class tensor_traits : public ggml::cpu::tensor_traits {
         }
 
         if (op->src[0]->type == GGML_TYPE_F32) {
+            ggml_kleidiai_kernels * primary     = kernel_chain[0];
+            kernel_info *           gemv_kernel = primary ? &primary->gemv : nullptr;
+            if (is_gemv && op->src[1]->nb[0] == (int64_t) sizeof(float) && gemv_kernel &&
+                gemv_kernel->get_lhs_offset_ex && gemv_kernel->get_rhs_packed_offset_ex &&
+                gemv_kernel->run_kernel_ex && gemv_kernel->get_dst_offset) {
+                size = 0;
+                return true;
+            }
+
             size_t cursor = 0;
             bool any_slot = false;
 
@@ -811,15 +820,28 @@ class tensor_traits : public ggml::cpu::tensor_traits {
             return false;
         }
 
-        kernel_info * kernel        = &kernels->gemm;
+        const size_t k = ne00;
+        const size_t m = ne11;
+        const size_t n = ne01;
+        const bool use_gemv = m == 1 && src1->nb[0] == (int64_t) sizeof(float) &&
+                              kernels->gemv.get_lhs_offset_ex &&
+                              kernels->gemv.get_rhs_packed_offset_ex &&
+                              kernels->gemv.run_kernel_ex &&
+                              kernels->gemv.get_dst_offset;
+
+        kernel_info * kernel        = use_gemv ? &kernels->gemv : &kernels->gemm;
         lhs_packing_info * lhs_info = &kernels->gemm_lhs_info;
 
-        if (!kernel || !lhs_info || !lhs_info->get_offset || !lhs_info->get_packed_offset_ex ||
-            !lhs_info->packed_size_ex || !lhs_info->pack_func_ex ||
+        if (!kernel || !kernel->get_lhs_offset_ex ||
             !kernel->get_rhs_packed_offset_ex || !kernel->run_kernel_ex || !kernel->get_dst_offset) {
             return false;
         }
 
+        if (!use_gemv && (!lhs_info || !lhs_info->get_offset || !lhs_info->get_packed_offset_ex ||
+                          !lhs_info->packed_size_ex || !lhs_info->pack_func_ex)) {
+            return false;
+        }
+
         const kleidiai_weight_header * header = kleidiai_weight_header_from_ptr(src0->data);
         const bool has_header = kleidiai_is_weight_header_valid(header);
 
@@ -832,16 +854,14 @@ class tensor_traits : public ggml::cpu::tensor_traits {
         const int nth = params->nth > 0 ? params->nth : 1;
         const int ith = params->ith;
 
-        const size_t k = ne00;
-        const size_t m = ne11;
-        const size_t n = ne01;
-
         const size_t mr = kernel->get_mr();
         const size_t kr = kernel->get_kr();
         const size_t sr = kernel->get_sr();
 
-        const size_t lhs_packed_size = lhs_info->packed_size_ex(m, k, 0, mr, kr, sr);
-        GGML_ASSERT(lhs_packed_size <= params->wsize);
+        const size_t lhs_packed_size = use_gemv ? 0 : lhs_info->packed_size_ex(m, k, 0, mr, kr, sr);
+        if (!use_gemv) {
+            GGML_ASSERT(lhs_packed_size <= params->wsize);
+        }
 
         uint8_t * lhs_packed   = static_cast<uint8_t *>(params->wdata);
         const size_t dst_stride = dst->nb[1];
@@ -853,7 +873,7 @@ class tensor_traits : public ggml::cpu::tensor_traits {
             const uint8_t * lhs_batch_base = static_cast<const uint8_t *>(src1->data) + batch_idx * src1->nb[2];
             uint8_t * dst_batch_base = static_cast<uint8_t *>(dst->data) + batch_idx * dst->nb[2];
 
-            {
+            if (!use_gemv) {
                 const int64_t m_roundup_mr = kai_roundup((int64_t)m, (int64_t)mr);
                 int64_t max_threads = mr ? (m_roundup_mr / (int64_t)mr) : nth;
                 max_threads = std::max<int64_t>(1, max_threads);
@@ -903,15 +923,17 @@ class tensor_traits : public ggml::cpu::tensor_traits {
                 const size_t n_to_process = std::min(chunk_cols, n - n_start);
 
                 if (n_to_process > 0) {
-                    const size_t lhs_packed_offset = lhs_info->get_packed_offset_ex(0, k, 0, mr, kr, sr);
+                    const size_t lhs_offset = use_gemv ? kernel->get_lhs_offset_ex(0, k, 0)
+                                                       : lhs_info->get_packed_offset_ex(0, k, 0, mr, kr, sr);
                     const size_t rhs_packed_offset = kernel->get_rhs_packed_offset_ex(n_start, k, 0);
                     const size_t dst_offset        = kernel->get_dst_offset(0, n_start, dst_stride);
 
-                    const void * lhs_ptr = lhs_packed + lhs_packed_offset;
+                    const void * lhs_ptr = use_gemv ? lhs_batch_base + lhs_offset
+                                                    : lhs_packed + lhs_offset;
                     const void * rhs_ptr = rhs_base + rhs_packed_offset;
                     float * dst_ptr      = reinterpret_cast<float *>(dst_batch_base + dst_offset);
 
-                    kernel->run_kernel_ex(m, n_to_process, k, 0,
+                    kernel->run_kernel_ex(m, n_to_process, k, use_gemv ? src1->nb[1] : 0,
                                           lhs_ptr,
                                           rhs_ptr,
                                           dst_ptr,

ggml/src/ggml-cuda/mmvq.cu

@@ -290,6 +290,42 @@ bool ggml_cuda_should_use_mmvq(enum ggml_type type, int cc, int64_t ne11) {
     if (!ggml_is_quantized(type)) {
         return false;
     }
+    // k-quants cost more to decode and mvq redoes that per column, so MMQ wins sooner.
+    // Only list quant-types MMQ supports, others would fall back to cuBLAS.
+    if (GGML_CUDA_CC_IS_NVIDIA(cc) && cc == GGML_CUDA_CC_ADA_LOVELACE) {
+        switch (type) { // tuned on RTX 4090
+            case GGML_TYPE_Q2_K:
+                return ne11 <= 4;
+            case GGML_TYPE_Q3_K:
+                return ne11 <= 6;
+            case GGML_TYPE_Q4_K:
+            case GGML_TYPE_Q5_K:
+                return ne11 <= 7;
+            default:
+                return ne11 <= MMVQ_MAX_BATCH_SIZE;
+        }
+    }
+    if (GGML_CUDA_CC_IS_NVIDIA(cc) && cc == GGML_CUDA_CC_BLACKWELL) {
+        switch (type) { // tuned on RTX 5090
+            case GGML_TYPE_Q2_K:
+            case GGML_TYPE_Q3_K:
+            case GGML_TYPE_Q4_K:
+            case GGML_TYPE_Q5_K:
+                return ne11 <= 5;
+            case GGML_TYPE_Q6_K:
+                return ne11 <= 7;
+            default:
+                return ne11 <= MMVQ_MAX_BATCH_SIZE;
+        }
+    }
+    if (GGML_CUDA_CC_IS_NVIDIA(cc) && cc == GGML_CUDA_CC_DGX_SPARK) {
+        switch (type) { // tuned on DGX Spark GB10
+            case GGML_TYPE_Q2_K:
+                return ne11 <= 6;
+            default:
+                return ne11 <= MMVQ_MAX_BATCH_SIZE;
+        }
+    }
     if (GGML_CUDA_CC_IS_CDNA(cc)) {
         if (GGML_CUDA_CC_IS_CDNA1(cc)) {
             switch (type) {

ggml/src/ggml-hexagon/ggml-hexagon.cpp

@@ -3180,8 +3180,9 @@ static bool ggml_hexagon_supported_argsort(const struct ggml_hexagon_session * s
 static bool ggml_hexagon_supported_rope(const struct ggml_hexagon_session * sess, const struct ggml_tensor * op) {
     const int32_t * op_params = &op->op_params[0];
 
-    if (op_params[15] != 0) {
-        return false; // FIXME: support ggml_rope_set_offset
+    // ggml_rope_set_offset: HVX kernels need a VLEN-aligned window start (32 f32 elems)
+    if (op_params[15] % 32 != 0) {
+        return false;
     }
 
     int mode = op_params[2];

ggml/src/ggml-hexagon/htp/rope-ops.c

@@ -53,6 +53,7 @@
 
 struct htp_rope_context {
     int32_t n_dims;
+    int32_t n_offs;
     int32_t mode;
     int32_t n_ctx_orig;
     int32_t sections[4];
@@ -405,32 +406,40 @@ static inline void hvx_rope_f32_aa(float * restrict dst, const float * restrict
 
 static void inline rope_basic_f32(struct htp_rope_context * rctx, uint8_t * restrict dst, uint8_t * restrict src,
                    uint32_t nr, uint32_t ne0, const float * restrict theta_cache) {
+    const uint32_t n_offs = rctx->n_offs; // VLEN-aligned (enforced by supports_op)
     #pragma unroll(4)
     for (uint32_t i = 0; i < nr; i++) {
         float * d = (float *) (dst + i * rctx->dst_row_size_aligned);
         float * s = (float *) (src + i * rctx->src0_row_size_aligned);
 
-        hvx_rope_f32_aa(d, s, rctx->n_dims, theta_cache);
+        hvx_rope_f32_aa(d + n_offs, s + n_offs, rctx->n_dims, theta_cache);
 
         // fill the remain channels with data from src tensor
-        if (rctx->n_dims < ne0) {
-            hvx_copy_f32_uu((uint8_t *)(d + rctx->n_dims), (uint8_t *)(s + rctx->n_dims), ne0 - rctx->n_dims);
+        if (n_offs > 0) {
+            hvx_copy_f32_uu((uint8_t *) d, (uint8_t *) s, n_offs);
+        }
+        if (n_offs + rctx->n_dims < ne0) {
+            hvx_copy_f32_uu((uint8_t *)(d + n_offs + rctx->n_dims), (uint8_t *)(s + n_offs + rctx->n_dims), ne0 - n_offs - rctx->n_dims);
         }
     }
 }
 
 static void inline rope_neox_f32(struct htp_rope_context * rctx, uint8_t * restrict dst, uint8_t * restrict src,
                    uint32_t nr, uint32_t ne0, const float * restrict theta_cache) {
+    const uint32_t n_offs = rctx->n_offs; // VLEN-aligned (enforced by supports_op)
     #pragma unroll(4)
     for (uint32_t i = 0; i < nr; i++) {
         float * d = (float *) (dst + i * rctx->dst_row_size_aligned);
         float * s = (float *) (src + i * rctx->src0_row_size_aligned);
 
-        hvx_rope_neox_f32_aa(d, s, rctx->n_dims, theta_cache);
+        hvx_rope_neox_f32_aa(d + n_offs, s + n_offs, rctx->n_dims, theta_cache);
 
         // fill the remain channels with data from src tensor
-        if (rctx->n_dims < ne0) {
-            hvx_copy_f32_uu((uint8_t *)(d + rctx->n_dims), (uint8_t *)(s + rctx->n_dims), ne0 - rctx->n_dims);
+        if (n_offs > 0) {
+            hvx_copy_f32_uu((uint8_t *) d, (uint8_t *) s, n_offs);
+        }
+        if (n_offs + rctx->n_dims < ne0) {
+            hvx_copy_f32_uu((uint8_t *)(d + n_offs + rctx->n_dims), (uint8_t *)(s + n_offs + rctx->n_dims), ne0 - n_offs - rctx->n_dims);
         }
     }
 }
@@ -673,6 +682,7 @@ static int execute_op_rope_f32(struct htp_ops_context * octx) {
     rctx.n_dims     = ((const int32_t *) op_params)[1];
     rctx.mode       = ((const int32_t *) op_params)[2];
     rctx.n_ctx_orig = ((const int32_t *) op_params)[4];
+    rctx.n_offs     = ((const int32_t *) op_params)[15];
 
     memcpy(&rctx.freq_base,   (int32_t *) op_params + 5,  sizeof(float));
     memcpy(&rctx.freq_scale,  (int32_t *) op_params + 6,  sizeof(float));

ggml/src/ggml-metal/ggml-metal-ops.cpp

@@ -2806,6 +2806,13 @@ bool ggml_metal_op_flash_attn_ext_use_vec(const ggml_tensor * op) {
 static bool ggml_metal_op_flash_attn_ext_use_kv_f16(const ggml_tensor * op) {
     assert(op->op == GGML_OP_FLASH_ATTN_EXT);
 
+    // depending on compute/bandwidth ratio, dequant to f16 kv is not always beneficial
+    // ref: https://github.com/ggml-org/llama.cpp/pull/27390#issuecomment-5355152767
+    // TODO: tune per device
+    if (op->src[0]->ne[1] < 32) {
+        return false;
+    }
+
     switch (op->src[1]->type) {
         case GGML_TYPE_Q4_0:
         case GGML_TYPE_Q4_1:
@@ -2968,9 +2975,10 @@ size_t ggml_metal_op_flash_attn_ext_extra_tmp(const ggml_tensor * op) {
 size_t ggml_metal_op_flash_attn_ext_extra_kv_f16(const ggml_tensor * op) {
     assert(op->op == GGML_OP_FLASH_ATTN_EXT);
 
-    if (!ggml_metal_op_flash_attn_ext_use_kv_f16(op)) {
-        return 0;
-    }
+    // note: always reserve the temp buffer to avoid graph reallocations
+    //if (!ggml_metal_op_flash_attn_ext_use_kv_f16(op)) {
+    //    return 0;
+    //}
 
     GGML_TENSOR_LOCALS( int32_t, ne2, op->src[2], ne);
 

ggml/src/ggml-metal/ggml-metal.metal

@@ -10365,9 +10365,12 @@ kernel void kernel_mul_mm(
     auto tB = tensor(ptrB, dextents<int32_t, 2>(K, N), array<int, 2>({1, strideB}));
 
     // Configure matmul operation
+    // note: K is dynamic_extent (clamped to the valid range in PHASE 2), since a static
+    //       N_MM_NK_TOTAL K tile would read src1 out of bounds when K % N_MM_NK_TOTAL != 0
+    // ref: https://github.com/ggml-org/llama.cpp/pull/27064
     mpp::tensor_ops::matmul2d<
         mpp::tensor_ops::matmul2d_descriptor(
-            NRB, NRA, N_MM_NK_TOTAL, false, true, true,
+            NRB, NRA, static_cast<int>(dynamic_extent), false, true, true,
             mpp::tensor_ops::matmul2d_descriptor::mode::multiply_accumulate),
         execution_simdgroups<N_MM_SIMD_GROUP_X * N_MM_SIMD_GROUP_Y>> mm;
 
@@ -10419,10 +10422,14 @@ kernel void kernel_mul_mm(
         threadgroup_barrier(mem_flags::mem_threadgroup);
 
         // === PHASE 2: Tensor matmul ===
-        auto mA = tA.slice(0, 0);
-        auto mB = tB.slice(loop_k, rb);
+        // Clamp the K extent of both operand tensors to the remaining valid K range so
+        // the dynamic-K op never reads past the K extent of src1 (or the staged A tile).
+        const int kExt = min(N_MM_NK_TOTAL, K - loop_k);
 
-        mm.run(mB, mA, cT);
+        auto tAv = tensor(sa, dextents<int32_t, 2>(kExt, NRA), array<int, 2>({1, N_MM_NK_TOTAL}));
+        auto tBv = tensor(ptrB + loop_k + rb * strideB, dextents<int32_t, 2>(kExt, N - rb), array<int, 2>({1, strideB}));
+
+        mm.run(tBv, tAv, cT);
 
         threadgroup_barrier(mem_flags::mem_threadgroup);
     }

ggml/src/ggml-opencl/ggml-opencl.cpp

@@ -582,6 +582,8 @@ struct ggml_backend_opencl_context {
     bool adreno_use_bin_kernels;
     get_adreno_bin_kernel_func_t get_adreno_bin_kernel_func = nullptr;
     ggml_cl_compiler_version adreno_cl_compiler_version;
+    // The q6_K flat mul_mat codegen workarounds are needed by old E031 compilers only.
+    bool q6_k_flat_old_compiler;
 
     std::string kernel_compile_opts;  // cached for lazy-compiled kernels.
 
@@ -1931,8 +1933,14 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) {
 #else
         const std::string kernel_src = read_file("mul_mv_q6_k_f32_flat.cl");
 #endif
+        // The codegen workarounds in this kernel are a measured 13-20% loss on
+        // compilers that do not need them, so only the affected ones build them;
+        // everyone else gets the original source.
+        const std::string q6k_opts = backend_ctx->q6_k_flat_old_compiler
+            ? compile_opts + " -DADRENO_OLD_COMPILER=1"
+            : compile_opts;
         cl_program prog =
-            build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts);
+            build_program_from_source(backend_ctx, kernel_src.c_str(), q6k_opts);
 
         CL_CHECK((backend_ctx->kernel_mul_mv_q6_K_f32_flat = clCreateKernel(prog, "kernel_mul_mv_q6_K_f32_flat", &err), err));
         CL_CHECK(clReleaseProgram(prog));
@@ -5917,6 +5925,16 @@ static ggml_backend_opencl_context * ggml_cl_init(ggml_backend_dev_t dev) {
         (backend_ctx->adreno_cl_compiler_version.type == E031 && backend_ctx->adreno_cl_compiler_version.major >= 47) ||
         (backend_ctx->adreno_cl_compiler_version.type == DX   && backend_ctx->adreno_cl_compiler_version.major >= 17);
 
+    // The q6_K flat mul_mat miscompile is a defect of the older E031 compilers, not a
+    // property of any GPU generation: it reproduces on E031.38 (Adreno 642L) and E031.41
+    // (Adreno 740) and is fixed by E031.45 (Adreno 619). Gate on the compiler so parts
+    // that do not need the workarounds do not pay for them. The explicit type check is
+    // required: newer_than_or_same() is false for every non-E031 compiler, so negating it
+    // alone would enable the workarounds on E17/DX.
+    backend_ctx->q6_k_flat_old_compiler =
+        backend_ctx->adreno_cl_compiler_version.type == E031 &&
+        !backend_ctx->adreno_cl_compiler_version.newer_than_or_same(E031, 45, 0, 0);
+
     size_t ext_str_size;
     clGetDeviceInfo(device, CL_DEVICE_EXTENSIONS, 0, NULL, &ext_str_size);
     char *ext_buffer = (char *)alloca(ext_str_size + 1);
@@ -7375,6 +7393,19 @@ static bool ggml_opencl_supports_op(ggml_backend_dev_t dev, const struct ggml_te
                        op->src[0]->type == GGML_TYPE_Q4_K  ||
                        op->src[0]->type == GGML_TYPE_Q5_K  ||
                        op->src[0]->type == GGML_TYPE_Q6_K) {
+                // The E031.41 compiler (usually with A7x) miscompiles the flat K-quant
+                // GEMV kernels (kernel_mul_mv_q*_K_f32_flat) and makes lm_head run much
+                // slower than it should. So, make it fallback to CPU to preserve performance
+                // for this compiler series.
+                static const char * a7x_lmhead_env = getenv("GGML_OPENCL_A7X_LMHEAD_CPU");
+                static const bool   a7x_lmhead_cpu = (a7x_lmhead_env == nullptr || a7x_lmhead_env[0] != '0');
+                if (a7x_lmhead_cpu &&
+                    backend_ctx->adreno_gen == ADRENO_GPU_GEN::A7X &&
+                    (op->src[0]->type == GGML_TYPE_Q4_K || op->src[0]->type == GGML_TYPE_Q5_K ||
+                     op->src[0]->type == GGML_TYPE_Q6_K) &&
+                    op->src[0]->ne[1] >= 32768) {   // vocab-scale weight; no FFN/attn weight is this tall
+                    return false;
+                }
                 return op->src[1]->type == GGML_TYPE_F32 && ggml_is_contiguous(op->src[0]) && ggml_is_contiguous(op->src[1]);
             } else if (op->src[0]->type == GGML_TYPE_Q8_0) {
                 return op->src[1]->type == GGML_TYPE_F32;
@@ -7416,9 +7447,6 @@ static bool ggml_opencl_supports_op(ggml_backend_dev_t dev, const struct ggml_te
         case GGML_OP_DIAG_MASK_INF:
             return op->ne[3] == 1;
         case GGML_OP_ROPE: {
-            if (((const int32_t *) op->op_params)[15] != 0) {
-                return false; // FIXME: support ggml_rope_set_offset
-            }
             const int mode = ((const int32_t *) op->op_params)[2];
             const bool is_mrope = mode & GGML_ROPE_TYPE_MROPE;
             const bool is_vision = mode == GGML_ROPE_TYPE_VISION;
@@ -7496,6 +7524,7 @@ static bool ggml_opencl_supports_op(ggml_backend_dev_t dev, const struct ggml_te
                                      v->type == GGML_TYPE_F16 && op->type == GGML_TYPE_F16;
             const bool is_f32_f16  = q->type == GGML_TYPE_F32 && k->type == GGML_TYPE_F16 &&
                                      v->type == GGML_TYPE_F16 && op->type == GGML_TYPE_F32;
+
             const bool is_f32_q8_0 = q->type == GGML_TYPE_F32 && k->type == GGML_TYPE_Q8_0 &&
                                      v->type == GGML_TYPE_Q8_0 && op->type == GGML_TYPE_F32 &&
                                      dk % 32 == 0 && dv % 32 == 0;
@@ -7503,6 +7532,21 @@ static bool ggml_opencl_supports_op(ggml_backend_dev_t dev, const struct ggml_te
                                      v->type == GGML_TYPE_Q4_0 && op->type == GGML_TYPE_F32 &&
                                      dk % 32 == 0 && dv % 32 == 0;
 
+            // A7X (Adreno 740, compiler E031.41) SIGSEGVs inside clBuildProgram
+            // building the flash_attn programs whose KV path is mixed-type or
+            // dequantized — f32_f16, q8_0, q4_0 (reproduced at DK=40 and DK=64; it
+            // is DK-independent). It is a driver crash, not codegen-wrong-output, so
+            // it cannot be caught in-process (fatal=false only handles clean compile
+            // errors). The uniform f16_f16 / f32_f32 programs compile fine on this
+            // compiler, so decline only the KV-convert variants; ggml then runs
+            // those (f16-KV / quant-KV) attention layers on the CPU backend.
+            // Negative compiler carve-out, same idiom as the Intel DK=512 decline
+            // below and the X1E driver-quirk guards.
+            if (backend_ctx && backend_ctx->adreno_gen == ADRENO_GPU_GEN::A7X &&
+                (is_f32_f16 || is_f32_q8_0 || is_f32_q4_0)) {
+                return false;
+            }
+
             // Asymmetric KV: host-dequants both sides to F32, uses f32 kernel.
             auto is_kv_type_ok = [](ggml_type t) {
                 return t == GGML_TYPE_F16 || t == GGML_TYPE_F32 ||
@@ -12830,7 +12874,10 @@ static void ggml_cl_norm(ggml_backend_t backend, const ggml_tensor * src0, const
     GGML_TENSOR_LOCALS(int,      ne0, src0, ne);
     GGML_TENSOR_LOCALS(cl_ulong, nb0, src0, nb);
 
-    const int nth = MIN(64, ne00);
+    int nth = 1;
+    while (nth < ne00 && nth < 64) {
+        nth *= 2;
+    }
 
     cl_kernel kernel = backend_ctx->kernel_norm;
 
@@ -20580,6 +20627,12 @@ static void ggml_cl_mul_mat(ggml_backend_t backend, const ggml_tensor * src0, co
             CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int),      &ne1));
             CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int),      &r2));
             CL_CHECK(clSetKernelArg(kernel, 16, sizeof(int),      &r3));
+            // The optimizer-barrier arg exists only in the ADRENO_OLD_COMPILER build of
+            // this kernel; conformant compilers get the original 17-arg signature.
+            if (backend_ctx->q6_k_flat_old_compiler) {
+                cl_uchar q6k_mask = 0xFF;   // never 0xFE in prod; see the kernel note
+                CL_CHECK(clSetKernelArg(kernel, 17, sizeof(cl_uchar), &q6k_mask));
+            }
 #else
             kernel = backend_ctx->kernel_mul_mv_q6_K_f32;
 
@@ -23867,6 +23920,7 @@ static void ggml_cl_rope(ggml_backend_t backend, const ggml_tensor * src0, const
     const int n_dims     = ((int *) dst->op_params)[1];
     const int mode       = ((int *) dst->op_params)[2];
     const int n_ctx_orig = ((int32_t *) dst->op_params)[4];
+    const int n_offs     = ((int32_t *) dst->op_params)[15];
 
     float freq_base;
     float freq_scale;
@@ -23895,6 +23949,7 @@ static void ggml_cl_rope(ggml_backend_t backend, const ggml_tensor * src0, const
 
     if (is_vision) {
         GGML_ASSERT(n_dims == ne00/2);
+        GGML_ASSERT(n_offs == 0); // offset not supported for vision, as the rotated pairs span the whole row
     }
 
     cl_kernel kernel;
@@ -23986,6 +24041,12 @@ static void ggml_cl_rope(ggml_backend_t backend, const ggml_tensor * src0, const
     if (is_mrope && !is_vision) {
         CL_CHECK(clSetKernelArg(kernel, 34, sizeof(int), &is_imrope));
     }
+    // norm and neox have n_offs after beta_slow, mrope has it after is_imrope
+    if (!is_mrope && !is_vision) {
+        CL_CHECK(clSetKernelArg(kernel, 33, sizeof(int), &n_offs));
+    } else if (is_mrope && !is_vision) {
+        CL_CHECK(clSetKernelArg(kernel, 35, sizeof(int), &n_offs));
+    }
 
     size_t global_work_size[] = {(size_t)ne01*nth, (size_t)ne02, (size_t)ne03};
     size_t local_work_size[] = {(size_t)nth, 1, 1};

ggml/src/ggml-opencl/kernels/mul_mv_q6_k_f32_flat.cl

@@ -28,6 +28,13 @@
 
 #define QK_K       256
 
+// ADRENO_OLD_COMPILER is defined by the host (-D) only for the Adreno E031
+// compilers older than E031.45, which miscompile several constructs this kernel
+// used (confirmed on E031.38 and E031.41; E031.45 is clean). Every other
+// compiler -- newer E031, E17, DX, Intel, and every non-Adreno device that
+// builds this program -- takes the #else branches, which are the original
+// source: the workarounds below cost ~13% on the q6_K flat n=1 GEMV where they
+// are not needed.
 inline float block_q_6_K_dot_y_flat(
     global uchar * blk_ql,
     global uchar * blk_qh,
@@ -37,6 +44,9 @@ inline float block_q_6_K_dot_y_flat(
     int ip,
     int is,
     int l0,
+#if defined(ADRENO_OLD_COMPILER)
+    int dbg,
+#endif
     float4 y0,
     float4 y1,
     float4 y2,
@@ -48,10 +58,40 @@ inline float block_q_6_K_dot_y_flat(
     global uchar * q1 = blk_ql     + ib*128 + q_offset_l;
     global uchar * q2 = q1         + QK_K/8;
     global uchar * qh = blk_qh     + ib*64 + q_offset_h;
-    global char  * sc = blk_scales + ib*16 + is;
 
     float dall = blk_d[ib];
 
+#if defined(ADRENO_OLD_COMPILER)
+    // The vectorized dequant (int4/float4 bit-ops, convert_*4, dot()) and vload4
+    // are miscompiled here -> garbage weights. Reconstruct the 6-bit weights and
+    // take the dot product scalar. q4_K/q5_K flat already use scalar paths, which
+    // is why q6_K was the only flat GEMV that failed.
+    // Scales are SIGNED int8; read as uchar and sign-extend arithmetically so the
+    // result does not depend on whether the compiler treats `char` as signed.
+    global uchar * sc = (global uchar *)(blk_scales + ib*16 + is);
+
+    int s0 = (int)sc[0] - 256*(sc[0] >> 7);
+    int s2 = (int)sc[2] - 256*(sc[2] >> 7);
+    int s4 = (int)sc[4] - 256*(sc[4] >> 7);
+    int s6 = (int)sc[6] - 256*(sc[6] >> 7);
+
+    // one 6-bit weight: low/high nibble of a ql byte OR'd with a 2-bit qh plane
+    // (plane p in {0,1,2,3} selects qh bits 2p..2p+1) placed at bits 4-5, minus 32.
+    #define Q6W(qb, sh, hb, p) ((float)((((int)(qb) >> (sh)) & 15) | ((((int)(hb) >> (2*(p))) & 3) << 4)) - 32.f)
+
+    float d0 = y0.s0*Q6W(q1[0],0,qh[0],0) + y0.s1*Q6W(q1[1],0,qh[1],0) + y0.s2*Q6W(q1[2],0,qh[2],0) + y0.s3*Q6W(q1[3],0,qh[3],0);
+    float d1 = y1.s0*Q6W(q2[0],0,qh[0],1) + y1.s1*Q6W(q2[1],0,qh[1],1) + y1.s2*Q6W(q2[2],0,qh[2],1) + y1.s3*Q6W(q2[3],0,qh[3],1);
+    float d2 = y2.s0*Q6W(q1[0],4,qh[0],2) + y2.s1*Q6W(q1[1],4,qh[1],2) + y2.s2*Q6W(q1[2],4,qh[2],2) + y2.s3*Q6W(q1[3],4,qh[3],2);
+    float d3 = y3.s0*Q6W(q2[0],4,qh[0],3) + y3.s1*Q6W(q2[1],4,qh[1],3) + y3.s2*Q6W(q2[2],4,qh[2],3) + y3.s3*Q6W(q2[3],4,qh[3],3);
+    #undef Q6W
+
+    if (dbg) printf("HELPER dall=%f s=[%d %d %d %d] d=[%f %f %f %f] ql0=%d qh0=%d y00=%f\n",
+        dall, s0, s2, s4, s6, d0, d1, d2, d3, (int)q1[0], (int)qh[0], y0.s0);
+
+    return dall * (d0 * s0 + d1 * s2 + d2 * s4 + d3 * s6);
+#else
+    global char * sc = blk_scales + ib*16 + is;
+
     // Vectorized loads: 3 uchar4 weight loads instead of 12 scalar byte reads.
     // q_offset_l/h are 4-aligned, so these are aligned vector loads.
     uchar4 q1v = vload4(0, q1);
@@ -72,6 +112,7 @@ inline float block_q_6_K_dot_y_flat(
 
     return dall * (dot(y0, w0) * sc[0] + dot(y1, w1) * sc[2] +
                    dot(y2, w2) * sc[4] + dot(y3, w3) * sc[6]);
+#endif
 }
 
 #undef N_DST
@@ -113,6 +154,11 @@ kernel void kernel_mul_mv_q6_K_f32_flat(
         int ne1,
         int r2,
         int r3
+#if defined(ADRENO_OLD_COMPILER)
+        ,
+        uchar q6k_mask   // runtime 0xFF; the host passes it so the compiler cannot
+                         // constant-fold the printf guards below into nothing
+#endif
 ) {
     src1 = (global float*)((global char*)src1 + offset1);
     dst = (global float*)((global char*)dst + offsetd);
@@ -128,6 +174,22 @@ kernel void kernel_mul_mv_q6_K_f32_flat(
 
     int first_row = (N_SIMDGROUP * r0 + get_sub_group_id()) * N_DST;
 
+#if defined(ADRENO_OLD_COMPILER)
+    // 64-bit `ulong` integer arithmetic is miscompiled here -> the base-pointer byte
+    // offsets came out wrong, so EVERY weight/scale read hit the wrong address. This
+    // was the primary cause of the q6_K flat failure (q5_K uses int offsets and is
+    // unaffected). Compute the block index in `int` and widen to `ulong` only inside
+    // the pointer expression: the byte offset stays 64-bit, but there is no ulong
+    // arithmetic chain to miscompile. The int index would overflow past ~2^31 blocks,
+    // which no realistic weight reaches -- but that is a narrowing, so keep it off the
+    // conformant path, which retains full ulong arithmetic.
+    int offset_src0 = first_row*nb + (i12/r2)*(nb*ne01) + (i13/r3)*(nb*ne01*ne02);
+
+    global uchar * blk_ql     = (global uchar *) src0_ql + (ulong)offset_src0 * 128;
+    global uchar * blk_qh     = (global uchar *) src0_qh + (ulong)offset_src0 * 64;
+    global char  * blk_scales = (global char  *) src0_s  + (ulong)offset_src0 * 16;
+    global half  * blk_d      = (global half  *) src0_d  + offset_src0;
+#else
     ulong offset_src0    = first_row*nb + (i12/r2)*(nb*ne01) + (i13/r3)*(nb*ne01*ne02);
     ulong offset_src0_ql = offset_src0 * 128;
     ulong offset_src0_qh = offset_src0 * 64;
@@ -138,6 +200,7 @@ kernel void kernel_mul_mv_q6_K_f32_flat(
     global uchar * blk_qh     = (global uchar *) src0_qh + offset_src0_qh;
     global char  * blk_scales = (global char  *) src0_s  + offset_src0_s;
     global half  * blk_d      = (global half  *) src0_d  + offset_src0_d;
+#endif
     global float * yy         = (global float *) src1    + r1*ne10 + im*ne00*ne1;
 
     int tid = get_sub_group_local_id()%(N_SIMDWIDTH/BLOCK_STRIDE); // within-super-block part, 0..15
@@ -155,24 +218,55 @@ kernel void kernel_mul_mv_q6_K_f32_flat(
 
     for (int ib = ix; ib < nb; ib += BLOCK_STRIDE) {
         global float * y = yy + ib * QK_K + 128*ip + l0;
+#if defined(ADRENO_OLD_COMPILER)
+        // vload4 of f32 is miscompiled here; index the lanes scalar instead.
+        float4 y0 = (float4)(y[ 0], y[ 1], y[ 2], y[ 3]);
+        float4 y1 = (float4)(y[32], y[33], y[34], y[35]);
+        float4 y2 = (float4)(y[64], y[65], y[66], y[67]);
+        float4 y3 = (float4)(y[96], y[97], y[98], y[99]);
+#else
         float4 y0 = vload4(0, y +  0);
         float4 y1 = vload4(0, y + 32);
         float4 y2 = vload4(0, y + 64);
         float4 y3 = vload4(0, y + 96);
+#endif
 
         for (int row = 0; row < N_DST; row++) {
             if (first_row + row < ne01) {
+#if defined(ADRENO_OLD_COMPILER)
+                int dbg = (q6k_mask==0xFE && r0==0 && r1==0 && im==0 && row==0 && ib==0 &&
+                           ne00==256 && ne01==16 && get_sub_group_local_id()==0) ? 1 : 0;
+                sumf[row] += block_q_6_K_dot_y_flat(
+                    blk_ql + row*nb*128, blk_qh + row*nb*64, blk_scales + row*nb*16, blk_d + row*nb,
+                    ib, ip, is, l0, dbg, y0, y1, y2, y3);
+#else
                 sumf[row] += block_q_6_K_dot_y_flat(
                     blk_ql + row*nb*128, blk_qh + row*nb*64, blk_scales + row*nb*16, blk_d + row*nb,
                     ib, ip, is, l0, y0, y1, y2, y3);
+#endif
             }
         }
     }
 
+#if defined(ADRENO_OLD_COMPILER)
+    // Optimizer barrier. This compiler drops the sumf partials unless a side effect
+    // forces them to materialize. q6k_mask is a kernel arg the compiler cannot prove
+    // is never 0xFE (the host always passes 0xFF), so the printf survives compilation
+    // but never executes. FRAGILE: the exact set and placement of these guarded
+    // printfs is load-bearing on E031.41 -- removing any one re-breaks q6_K.
+    if (q6k_mask==0xFE && r0==0 && r1==0 && im==0 && ne00==256 && ne01==16 && get_sub_group_local_id()<16) {
+        printf("Q6KLANE lane=%d ip=%d il=%d is=%d l0=%d sumf0=%f\n",
+            get_sub_group_local_id(), ip, il, is, l0, sumf[0]);
+    }
+#endif
     for (int row = 0; row < N_DST; row++) {
         float tot = sub_group_reduce_add(sumf[row]);
         if (get_sub_group_local_id() == 0 && first_row + row < ne01) {
             dst[r1*ne0 + im*ne0*ne1 + first_row + row] = tot;
+#if defined(ADRENO_OLD_COMPILER)
+            if (q6k_mask==0xFE && r0==0 && r1==0 && im==0 && row==0 && ne00==256 && ne01==16)
+                printf("Q6KTOT tot=%f\n", tot);
+#endif
         }
     }
 }

ggml/src/ggml-opencl/kernels/rope.cl

@@ -75,7 +75,8 @@ kernel void kernel_rope_norm_f32(
         float ext_factor,
         float attn_factor,
         float beta_fast,
-        float beta_slow
+        float beta_slow,
+        int n_offs
 ) {
     src0 = (global void*)((global char*)src0 + offset0);
     src1 = (global int*)((global char*)src1 + offset1);
@@ -94,14 +95,15 @@ kernel void kernel_rope_norm_f32(
     float inv_ndims = -1.f/n_dims;
 
     for (int i0 = 2*get_local_id(0); i0 < ne0; i0 += 2*get_local_size(0)) {
-        if (i0 < n_dims) {
-            int ic = i0/2;
+        if (i0 >= n_offs && i0 < n_offs + n_dims) {
+            int iw = i0 - n_offs; // relative idx
+            int ic = iw/2;
 
-            float theta = theta_base * pow(freq_base, inv_ndims*i0);
+            float theta = theta_base * pow(freq_base, inv_ndims*iw);
 
             float freq_factor = src2 != src0 ? src2[ic] : 1.0f;
 
-            float2 cos_sin_theta = rope_yarn(theta/freq_factor, freq_scale, corr_dims, i0, ext_factor, attn_factor);
+            float2 cos_sin_theta = rope_yarn(theta/freq_factor, freq_scale, corr_dims, iw, ext_factor, attn_factor);
 
             global float * src       = (global float *)((global char *) src0 + i3*nb03 + i2*nb02 + i1*nb01 + i0*nb00);
             global float * dst_data  = (global float *)((global char *)  dst + i3*nb3  + i2*nb2  + i1*nb1  + i0*nb0);
@@ -154,7 +156,8 @@ kernel void kernel_rope_norm_f16(
         float ext_factor,
         float attn_factor,
         float beta_fast,
-        float beta_slow
+        float beta_slow,
+        int n_offs
 ) {
     src0 = (global void*)((global char*)src0 + offset0);
     src1 = (global int*)((global char*)src1 + offset1);
@@ -173,14 +176,15 @@ kernel void kernel_rope_norm_f16(
     float inv_ndims = -1.f/n_dims;
 
     for (int i0 = 2*get_local_id(0); i0 < ne0; i0 += 2*get_local_size(0)) {
-        if (i0 < n_dims) {
-            int ic = i0/2;
+        if (i0 >= n_offs && i0 < n_offs + n_dims) {
+            int iw = i0 - n_offs; // relative idx
+            int ic = iw/2;
 
-            float theta = theta_base * pow(freq_base, inv_ndims*i0);
+            float theta = theta_base * pow(freq_base, inv_ndims*iw);
 
             float freq_factor = src2 != src0 ? src2[ic] : 1.0f;
 
-            float2 cos_sin_theta = rope_yarn(theta/freq_factor, freq_scale, corr_dims, i0, ext_factor, attn_factor);
+            float2 cos_sin_theta = rope_yarn(theta/freq_factor, freq_scale, corr_dims, iw, ext_factor, attn_factor);
 
             global half * src       = (global half *)((global char *) src0 + i3*nb03 + i2*nb02 + i1*nb01 + i0*nb00);
             global half * dst_data  = (global half *)((global char *)  dst + i3*nb3  + i2*nb2  + i1*nb1  + i0*nb0);
@@ -233,7 +237,8 @@ kernel void kernel_rope_neox_f32(
         float ext_factor,
         float attn_factor,
         float beta_fast,
-        float beta_slow
+        float beta_slow,
+        int n_offs
 ) {
     src0 = (global void*)((global char*)src0 + offset0);
     src1 = (global int*)((global char*)src1 + offset1);
@@ -252,17 +257,18 @@ kernel void kernel_rope_neox_f32(
     float inv_ndims = -1.f/n_dims;
 
     for (int i0 = 2*get_local_id(0); i0 < ne0; i0 += 2*get_local_size(0)) {
-        if (i0 < n_dims) {
-            int ic = i0/2;
+        if (i0 >= n_offs && i0 < n_offs + n_dims) {
+            int iw = i0 - n_offs; // relative idx
+            int ic = iw/2;
 
-            const float theta = theta_base * pow(freq_base, inv_ndims*i0);
+            const float theta = theta_base * pow(freq_base, inv_ndims*iw);
 
             const float freq_factor = src2 != src0 ? src2[ic] : 1.0f;
 
-            float2 cos_sin_theta = rope_yarn(theta/freq_factor, freq_scale, corr_dims, i0, ext_factor, attn_factor);
+            float2 cos_sin_theta = rope_yarn(theta/freq_factor, freq_scale, corr_dims, iw, ext_factor, attn_factor);
 
-            global float * src      = (global float *)((global char *) src0 + i3*nb03 + i2*nb02 + i1*nb01 + ic*nb00);
-            global float * dst_data = (global float *)((global char *)  dst + i3*nb3  + i2*nb2  + i1*nb1  + ic*nb0);
+            global float * src      = (global float *)((global char *) src0 + i3*nb03 + i2*nb02 + i1*nb01 + (n_offs + ic)*nb00);
+            global float * dst_data = (global float *)((global char *)  dst + i3*nb3  + i2*nb2  + i1*nb1  + (n_offs + ic)*nb0);
 
             const float x0 = src[0];
             const float x1 = src[n_dims/2];
@@ -312,7 +318,8 @@ kernel void kernel_rope_neox_f16(
         float ext_factor,
         float attn_factor,
         float beta_fast,
-        float beta_slow
+        float beta_slow,
+        int n_offs
 ) {
     src0 = (global void*)((global char*)src0 + offset0);
     src1 = (global int*)((global char*)src1 + offset1);
@@ -331,17 +338,18 @@ kernel void kernel_rope_neox_f16(
     float inv_ndims = -1.f/n_dims;
 
     for (int i0 = 2*get_local_id(0); i0 < ne0; i0 += 2*get_local_size(0)) {
-        if (i0 < n_dims) {
-            int ic = i0/2;
+        if (i0 >= n_offs && i0 < n_offs + n_dims) {
+            int iw = i0 - n_offs; // relative idx
+            int ic = iw/2;
 
-            const float theta = theta_base * pow(freq_base, inv_ndims*i0);
+            const float theta = theta_base * pow(freq_base, inv_ndims*iw);
 
             const float freq_factor = src2 != src0 ? src2[ic] : 1.0f;
 
-            float2 cos_sin_theta = rope_yarn(theta/freq_factor, freq_scale, corr_dims, i0, ext_factor, attn_factor);
+            float2 cos_sin_theta = rope_yarn(theta/freq_factor, freq_scale, corr_dims, iw, ext_factor, attn_factor);
 
-            global half * src       = (global half *)((global char *) src0 + i3*nb03 + i2*nb02 + i1*nb01 + ic*nb00);
-            global half * dst_data  = (global half *)((global char *)  dst + i3*nb3  + i2*nb2  + i1*nb1  + ic*nb0);
+            global half * src       = (global half *)((global char *) src0 + i3*nb03 + i2*nb02 + i1*nb01 + (n_offs + ic)*nb00);
+            global half * dst_data  = (global half *)((global char *)  dst + i3*nb3  + i2*nb2  + i1*nb1  + (n_offs + ic)*nb0);
 
             const float x0 = src[0];
             const float x1 = src[n_dims/2];
@@ -393,7 +401,8 @@ kernel void kernel_rope_multi_f32(
         float beta_fast,
         float beta_slow,
         int4 sections,
-        int  is_imrope
+        int  is_imrope,
+        int n_offs
 ) {
     src0 = (global void*)((global char*)src0 + offset0);
     src1 = (global int*)((global char*)src1 + offset1);
@@ -414,10 +423,11 @@ kernel void kernel_rope_multi_f32(
     float inv_ndims = -1.f/n_dims;
 
     for (int i0 = 2*get_local_id(0); i0 < ne0; i0 += 2*get_local_size(0)) {
-        if (i0 < n_dims) {
-            int ic = i0/2;
+        if (i0 >= n_offs && i0 < n_offs + n_dims) {
+            int iw = i0 - n_offs; // relative idx
+            int ic = iw/2;
 
-            const int sector = (i0 / 2) % sect_dims;
+            const int sector = ic % sect_dims;
             float theta_base = 0.0f;
 
             if (is_imrope) {
@@ -445,14 +455,14 @@ kernel void kernel_rope_multi_f32(
                 }
             }
 
-            const float theta = theta_base * pow(freq_base, inv_ndims*i0);
+            const float theta = theta_base * pow(freq_base, inv_ndims*iw);
 
             const float freq_factor = src2 != src0 ? src2[ic] : 1.0f;
 
-            float2 cos_sin_theta = rope_yarn(theta/freq_factor, freq_scale, corr_dims, i0, ext_factor, attn_factor);
+            float2 cos_sin_theta = rope_yarn(theta/freq_factor, freq_scale, corr_dims, iw, ext_factor, attn_factor);
 
-            global float * src      = (global float *)((global char *) src0 + i3*nb03 + i2*nb02 + i1*nb01 + ic*nb00);
-            global float * dst_data = (global float *)((global char *)  dst + i3*nb3  + i2*nb2  + i1*nb1  + ic*nb0);
+            global float * src      = (global float *)((global char *) src0 + i3*nb03 + i2*nb02 + i1*nb01 + (n_offs + ic)*nb00);
+            global float * dst_data = (global float *)((global char *)  dst + i3*nb3  + i2*nb2  + i1*nb1  + (n_offs + ic)*nb0);
 
             const float x0 = src[0];
             const float x1 = src[n_dims/2];
@@ -504,7 +514,8 @@ kernel void kernel_rope_multi_f16(
         float beta_fast,
         float beta_slow,
         int4 sections,
-        int  is_imrope
+        int  is_imrope,
+        int n_offs
 ) {
     src0 = (global void*)((global char*)src0 + offset0);
     src1 = (global int*)((global char*)src1 + offset1);
@@ -525,10 +536,11 @@ kernel void kernel_rope_multi_f16(
     float inv_ndims = -1.f/n_dims;
 
     for (int i0 = 2*get_local_id(0); i0 < ne0; i0 += 2*get_local_size(0)) {
-        if (i0 < n_dims) {
-            int ic = i0/2;
+        if (i0 >= n_offs && i0 < n_offs + n_dims) {
+            int iw = i0 - n_offs; // relative idx
+            int ic = iw/2;
 
-            const int sector = (i0 / 2) % sect_dims;
+            const int sector = ic % sect_dims;
             float theta_base = 0.0f;
 
             if (is_imrope) {
@@ -556,14 +568,14 @@ kernel void kernel_rope_multi_f16(
                 }
             }
 
-            const float theta = theta_base * pow(freq_base, inv_ndims*i0);
+            const float theta = theta_base * pow(freq_base, inv_ndims*iw);
 
             const float freq_factor = src2 != src0 ? src2[ic] : 1.0f;
 
-            float2 cos_sin_theta = rope_yarn(theta/freq_factor, freq_scale, corr_dims, i0, ext_factor, attn_factor);
+            float2 cos_sin_theta = rope_yarn(theta/freq_factor, freq_scale, corr_dims, iw, ext_factor, attn_factor);
 
-            global half * src      = (global half *)((global char *) src0 + i3*nb03 + i2*nb02 + i1*nb01 + ic*nb00);
-            global half * dst_data = (global half *)((global char *)  dst + i3*nb3  + i2*nb2  + i1*nb1  + ic*nb0);
+            global half * src      = (global half *)((global char *) src0 + i3*nb03 + i2*nb02 + i1*nb01 + (n_offs + ic)*nb00);
+            global half * dst_data = (global half *)((global char *)  dst + i3*nb3  + i2*nb2  + i1*nb1  + (n_offs + ic)*nb0);
 
             const float x0 = src[0];
             const float x1 = src[n_dims/2];

ggml/src/ggml-sycl/dmmv.cpp

@@ -1955,6 +1955,23 @@ static void dequantize_mul_mat_vec_q4_K_sycl_reorder_esimd(const void *vx, const
     });
 }
 
+static void dequantize_mul_mat_vec_q5_K_sycl_reorder_esimd(const void *vx, const float *y,
+                                                           float *dst, const int ncols,
+                                                           const int nrows,
+                                                           dpct::queue_ptr stream) {
+    GGML_ASSERT(ncols % QK_K == 0);
+    const int workgroups = (nrows + 1) / 2;
+    stream->submit([&](sycl::handler &h) {
+        sycl::local_accessor<float, 1> lmem(sycl::range<1>(GGML_SYCL_DMMV_ESIMD_WG_SIZE * 2), h);
+        h.parallel_for(
+            sycl::nd_range<1>(sycl::range<1>((size_t)workgroups * GGML_SYCL_DMMV_ESIMD_WG_SIZE), sycl::range<1>(GGML_SYCL_DMMV_ESIMD_WG_SIZE)),
+            [=](sycl::nd_item<1> it) [[intel::sycl_explicit_simd]] {
+                dequantize_mul_mat_vec_reorder_esimd<GGML_TYPE_Q5_K>(
+                    vx, y, dst, ncols, nrows, lmem, it);
+            });
+    });
+}
+
 static void dequantize_mul_mat_vec_q6_K_sycl_reorder_esimd(const void *vx, const float *y,
                                                            float *dst, const int ncols,
                                                            const int nrows,
@@ -2134,7 +2151,15 @@ void ggml_sycl_op_dequantize_mul_mat_vec(
         case GGML_TYPE_Q5_K:
             if ((ggml_tensor_extra_gpu *) dst->src[0]->extra &&
                 ((ggml_tensor_extra_gpu *) dst->src[0]->extra)->optimized_feature.reorder) {
-                dequantize_mul_mat_vec_q5_K_sycl_reorder(src0_dd_i, src1_ddf_i, dst_dd_i, ne00, row_diff, stream);
+#ifdef GGML_SYCL_DMMV_HAS_ESIMD
+                if (g_ggml_sycl_enable_esimd) {
+                    dequantize_mul_mat_vec_q5_K_sycl_reorder_esimd(src0_dd_i, src1_ddf_i, dst_dd_i, ne00, row_diff, stream);
+                }
+                else
+#endif
+                {
+                    dequantize_mul_mat_vec_q5_K_sycl_reorder(src0_dd_i, src1_ddf_i, dst_dd_i, ne00, row_diff, stream);
+                }
             } else {
                 dequantize_mul_mat_vec_q5_K_sycl(src0_dd_i, src1_ddf_i, dst_dd_i, ne00, row_diff, stream);
             }

ggml/src/ggml-sycl/dpct/helper.hpp

@@ -62,7 +62,7 @@
 
 #define DPCT_UNUSED(x) (void)(x)
 
-inline void _abort(const char * str) {
+[[noreturn]] inline void _abort(const char * str) {
     std::cerr << str << std::endl;
     std::abort();
 }

ggml/src/ggml-sycl/element_wise.cpp

@@ -10,7 +10,7 @@
     (ITEM.get_local_range(IDX) * ITEM.get_group(IDX) + ITEM.get_local_id(IDX))
 
 static void acc_f32(const char * x, const char * y, float * dst, const int64_t ne,
-    const int64_t ne0, const int64_t ne1, const int64_t ne2, const int64_t ne3,
+    const int64_t ne0, const int64_t ne1, const int64_t ne2,
     const int64_t nb00, const int64_t nb01, const int64_t nb02, const int64_t nb03,
     const int64_t ne10, const int64_t ne11, const int64_t ne12, const int64_t ne13,
     const int64_t nb10, const int64_t nb11, const int64_t nb12, const int64_t nb13,
@@ -455,7 +455,7 @@ static void unary_mul_sycl(const T * x, const T * g, T * dst, const int64_t k, c
 namespace ggml_sycl_detail {
 static void acc_f32_sycl(const char *x, const char *y, float *dst,
                          const int64_t n_elements,
-                         const int64_t ne0, const int64_t ne1, const int64_t ne2, const int64_t ne3,
+                         const int64_t ne0, const int64_t ne1, const int64_t ne2,
                          const int64_t nb00, const int64_t nb01, const int64_t nb02, const int64_t nb03,
                          const int64_t ne10, const int64_t ne11, const int64_t ne12, const int64_t ne13,
                          const int64_t nb10, const int64_t nb11, const int64_t nb12, const int64_t nb13,
@@ -466,7 +466,7 @@ static void acc_f32_sycl(const char *x, const char *y, float *dst,
                                            sycl::range<3>(1, 1, SYCL_ACC_BLOCK_SIZE)),
                          [=](sycl::nd_item<3> /*item_ct1*/) [[sycl::reqd_sub_group_size(WARP_SIZE)]] {
                             acc_f32(x, y, dst, n_elements,
-                                ne0, ne1, ne2, ne3,
+                                ne0, ne1, ne2,
                                 nb00, nb01, nb02, nb03,
                                 ne10, ne11, ne12, ne13,
                                 nb10, nb11, nb12, nb13,
@@ -970,7 +970,7 @@ static inline void ggml_sycl_op_acc(ggml_backend_sycl_context & ctx, ggml_tensor
     const int64_t offset = (int64_t) ((const int32_t *) dst->op_params)[3] / (int64_t) sizeof(float);
 
     ggml_sycl_detail::acc_f32_sycl(src0_d, src1_d, dst_d, ggml_nelements(dst),
-        dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3],
+        dst->ne[0], dst->ne[1], dst->ne[2],
         src0->nb[0], src0->nb[1], src0->nb[2], src0->nb[3],
         src1->ne[0], src1->ne[1], src1->ne[2], src1->ne[3],
         src1->nb[0], src1->nb[1], src1->nb[2], src1->nb[3],

ggml/src/ggml-sycl/esimd.hpp

@@ -1,15 +1,3 @@
-//
-// MIT license
-// Copyright (C) 2026 Intel Corporation
-// SPDX-License-Identifier: MIT
-//
-
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-
 #ifndef GGML_SYCL_ESIMD_HPP
 #define GGML_SYCL_ESIMD_HPP
 
@@ -287,6 +275,128 @@ template <> struct esimd_reorder_q_traits<GGML_TYPE_Q4_K> {
     }
 };
 
+// ---------------------------------------------------------------------------
+// Q5_K, SOA reorder layout produced by reorder_qw_q5_k:
+//   [qs: nb*(QK_K/2)] [qh: nb*(QK_K/8)] [scales: nb*K_SCALE_SIZE] [dm: nb*sizeof(half2)]
+// with nb = nrows*num_blocks_per_row.
+//
+// Identical to Q4_K except each 4-bit quant gains a 5th (high) bit from qh:
+// output chunk c (0..7) adds 16 when bit c of qh[l] is set, where qh[l] indexes
+// the same 32 bytes for every chunk (matches dequantize_row_q5_K).
+// ---------------------------------------------------------------------------
+template <> struct esimd_reorder_q_traits<GGML_TYPE_Q5_K> {
+    struct ptrs {
+        const uint8_t *    qs;
+        const uint8_t *    qh;
+        const uint8_t *    scales;
+        const sycl::half * dm;
+    };
+
+    static ESIMD_INLINE ptrs make_ptrs(const void * vx, size_t nb) {
+        const uint8_t * qs     = (const uint8_t *) vx;
+        const uint8_t * qh     = qs + nb * (QK_K / 2);
+        const uint8_t * scales = qh + nb * (QK_K / 8);
+        const sycl::half * dm  = (const sycl::half *) (scales + nb * K_SCALE_SIZE);
+        return { qs, qh, scales, dm };
+    }
+
+    // extract bit `bit` (0..7) of each lane and move it to bit position 4,
+    // e.g. for the 4-bit base quant's 5th (high) bit. `bit` is always a
+    // compile-time-known unrolled loop constant at call sites, so this folds
+    // to a single mask (bit==4), mask+left-shift (bit<4), or mask+right-shift
+    // (bit>4) instead of the shift+mask+shift a naive `(qh>>bit & 1) << 4` emits.
+    static ESIMD_INLINE sycl::ext::intel::esimd::simd<uint16_t, 32> extract_bit_to_pos4(
+            sycl::ext::intel::esimd::simd<uint8_t, 32> qh, int bit) {
+        using namespace sycl::ext::intel::esimd;
+        simd<uint16_t, 32> masked = convert<uint16_t>(qh & simd<uint8_t, 32>((uint8_t) (1u << bit)));
+        if (bit < 4) {
+            return masked << simd<uint16_t, 32>((uint16_t) (4 - bit));
+        } else if (bit > 4) {
+            return masked >> simd<uint16_t, 32>((uint16_t) (bit - 4));
+        }
+        return masked;
+    }
+
+    static ESIMD_INLINE void mac_pair(
+            const ptrs & pa, size_t bia,
+            const ptrs & pb, size_t bib, bool has_b,
+            sycl::ext::intel::esimd::simd<float, 256> & y_vec,
+            sycl::ext::intel::esimd::simd<float, 32> & acc_a,
+            sycl::ext::intel::esimd::simd<float, 32> & acc_b) {
+        using namespace sycl::ext::intel::esimd;
+
+        simd<uint8_t, 128> qs_a     = block_load<uint8_t, 128>(pa.qs + bia * (QK_K / 2));
+        simd<uint8_t, 128> qs_b     = 0;
+        simd<uint8_t, 32>  qh_a     = block_load<uint8_t, 32>(pa.qh + bia * (QK_K / 8));
+        simd<uint8_t, 32>  qh_b     = 0;
+        simd<uint8_t, 12>  scales_a = block_load<uint8_t, 12>(pa.scales + bia * K_SCALE_SIZE);
+        simd<uint8_t, 12>  scales_b = 0;
+
+        const float dall_a = (float) pa.dm[bia * 2 + 0];
+        const float dmin_a = (float) pa.dm[bia * 2 + 1];
+        float dall_b = 0.0f;
+        float dmin_b = 0.0f;
+        if (has_b) {
+            qs_b     = block_load<uint8_t, 128>(pb.qs + bib * (QK_K / 2));
+            qh_b     = block_load<uint8_t, 32>(pb.qh + bib * (QK_K / 8));
+            scales_b = block_load<uint8_t, 12>(pb.scales + bib * K_SCALE_SIZE);
+            dall_b = (float) pb.dm[bib * 2 + 0];
+            dmin_b = (float) pb.dm[bib * 2 + 1];
+        }
+
+        simd<float, 8> scale_f_a, min_f_a, scale_f_b, min_f_b;
+        unpack_scale_min_k4(scales_a, dall_a, dmin_a, scale_f_a, min_f_a);
+        unpack_scale_min_k4(scales_b, dall_b, dmin_b, scale_f_b, min_f_b);
+
+        simd<uint8_t, 128> qs_lo_a = qs_a & simd<uint8_t, 128>(0x0F);
+        simd<uint8_t, 128> qs_hi_a = qs_a >> simd<uint8_t, 128>(4);
+        simd<uint8_t, 128> qs_lo_b = qs_b & simd<uint8_t, 128>(0x0F);
+        simd<uint8_t, 128> qs_hi_b = qs_b >> simd<uint8_t, 128>(4);
+
+#pragma unroll
+        for (int sb = 0; sb < 8; sb += 2) {
+            const int q_offset = sb * 16;
+            simd<float, 32> y_lo = y_vec.select<32, 1>(sb * 32);
+            simd<float, 32> y_hi = y_vec.select<32, 1>((sb + 1) * 32);
+
+            const float scale_a_lo = scale_f_a[sb];
+            const float scale_a_hi = scale_f_a[sb + 1];
+            const float min_a_lo   = min_f_a[sb];
+            const float min_a_hi   = min_f_a[sb + 1];
+            const float scale_b_lo = scale_f_b[sb];
+            const float scale_b_hi = scale_f_b[sb + 1];
+            const float min_b_lo   = min_f_b[sb];
+            const float min_b_hi   = min_f_b[sb + 1];
+
+            simd<uint8_t, 32> qa_lo_u8 = qs_lo_a.select<32, 1>(q_offset);
+            simd<uint8_t, 32> qa_hi_u8 = qs_hi_a.select<32, 1>(q_offset);
+            simd<uint8_t, 32> qb_lo_u8 = qs_lo_b.select<32, 1>(q_offset);
+            simd<uint8_t, 32> qb_hi_u8 = qs_hi_b.select<32, 1>(q_offset);
+            simd<uint16_t, 32> qa_lo = convert<uint16_t>(qa_lo_u8);
+            simd<uint16_t, 32> qa_hi = convert<uint16_t>(qa_hi_u8);
+            simd<uint16_t, 32> qb_lo = convert<uint16_t>(qb_lo_u8);
+            simd<uint16_t, 32> qb_hi = convert<uint16_t>(qb_hi_u8);
+
+            // add the 5th bit: chunk sb uses qh bit sb, chunk sb+1 uses qh bit sb+1;
+            // qh always indexes the same 32 bytes regardless of chunk
+            qa_lo += extract_bit_to_pos4(qh_a, sb);
+            qa_hi += extract_bit_to_pos4(qh_a, sb + 1);
+            qb_lo += extract_bit_to_pos4(qh_b, sb);
+            qb_hi += extract_bit_to_pos4(qh_b, sb + 1);
+
+            simd<float, 32> deq_a_lo = convert<float>(qa_lo) * scale_a_lo + min_a_lo;
+            simd<float, 32> deq_a_hi = convert<float>(qa_hi) * scale_a_hi + min_a_hi;
+            simd<float, 32> deq_b_lo = convert<float>(qb_lo) * scale_b_lo + min_b_lo;
+            simd<float, 32> deq_b_hi = convert<float>(qb_hi) * scale_b_hi + min_b_hi;
+
+            acc_a += y_lo * deq_a_lo;
+            acc_b += y_lo * deq_b_lo;
+            acc_a += y_hi * deq_a_hi;
+            acc_b += y_hi * deq_b_hi;
+        }
+    }
+};
+
 // ---------------------------------------------------------------------------
 // Q6_K, SOA reorder layout:
 //   [ql: nb*(QK_K/2)] [qh: nb*(QK_K/4)] [scales(int8): nb*(QK_K/16)] [d: nb*half]

ggml/src/ggml-sycl/fattn-mkl.cpp

@@ -43,7 +43,7 @@ static void mkl_fa_pack_q_fp16(
     dpct::queue_ptr stream,
     sycl::half * __restrict dst,
     const float * __restrict q_src,
-    int n_queries, int n_query_rows, int DKQ,
+    int n_queries, int DKQ,
     int gqa_ratio, int kvh_base_head,
     float q_scale, int64_t q_row_stride, int64_t q_head_stride,
     int64_t wg_size) {
@@ -121,7 +121,7 @@ static void mkl_fa_online_softmax_chunk(
     float * __restrict VKQ_accum,
     int q0, int q_rows, int n_queries, int DV,
     int chunk_size, int chunk_start,
-    int kvh_head, int gqa_ratio,
+    int kvh_head,
     const sycl::half * mask_data, int64_t mask_head_stride,
     int64_t mask_row_stride, int mask_n_heads,
     float logit_softcap, int64_t wg_size) {
@@ -473,7 +473,6 @@ void ggml_sycl_flash_attn_ext_mkl(ggml_backend_sycl_context & ctx, ggml_tensor *
     MKL_ACCUM(dequant_time_us, t_deq);
 
     // --- Resolve mask pointers ---
-    const sycl::half * mask_data = nullptr;
     int64_t mask_head_stride = 0;
     int64_t mask_row_stride  = 0;
     int     mask_n_heads     = 0;
@@ -547,7 +546,7 @@ void ggml_sycl_flash_attn_ext_mkl(ggml_backend_sycl_context & ctx, ggml_tensor *
             // 1. Pack all GQA Q heads into fp16 (full n_query_rows)
             mkl_fa_pack_q_fp16(stream,
                 Q_head_f16_ptr, Q_batch,
-                n_queries, n_query_rows, DKQ,
+                n_queries, DKQ,
                 gqa_ratio, kvh_base_head,
                 q_scale, q_row_stride, q_head_stride, wg_size);
 
@@ -605,7 +604,7 @@ void ggml_sycl_flash_attn_ext_mkl(ggml_backend_sycl_context & ctx, ggml_tensor *
                             KQ_max_ptr, KQ_sum_ptr, VKQ_accum_ptr,
                             q0, q_rows, n_queries, DV,
                             this_chunk, chunk_start,
-                            kvh_base_head, gqa_ratio,
+                            kvh_base_head,
                             mask_batch, mask_head_stride,
                             mask_row_stride, mask_n_heads,
                             logit_softcap, wg_size);

ggml/src/ggml-sycl/fattn-onednn.cpp

@@ -21,14 +21,6 @@ bool ggml_sycl_flash_attn_ext_onednn_supported(const ggml_tensor * dst) {
     if (!g_ggml_sycl_fa_onednn) {
         return false;
     }
-    // Battlemage (Xe2) only, for now. On other Intel archs oneDNN's fused SDPA returns wrong results
-    // for some shapes (e.g. head_dim=64 on Arc / xe_hpg) -- an oneDNN bug tracked upstream at
-    // https://github.com/uxlfoundation/oneDNN/issues/5510. Remove this hardware limitation once that
-    // is fixed; until then non-BMG archs fall back to the existing FA kernel.
-    const gpu_arch arch = ggml_sycl_info().devices[ggml_sycl_get_device()].hw_info.arch;
-    if (arch != gpu_arch::intel_gpu_bmg_g21 && arch != gpu_arch::intel_gpu_bmg_g31) {
-        return false;
-    }
     const ggml_tensor * Q     = dst->src[0];
     const ggml_tensor * K     = dst->src[1];
     const ggml_tensor * V     = dst->src[2];
@@ -60,6 +52,17 @@ bool ggml_sycl_flash_attn_ext_onednn_supported(const ggml_tensor * dst) {
             }
         }
     }
+    // This is the improved SPDA gate. Rather than gating Alchemist GPUs from all SPDA features, we instead target only the failing shapes.
+    // If the GPU being assessed isn't in the grouping below, it has full access to all SPDA shapes. Otherwise, if it's an Alchemist GPU, we block only the shapes with head sizes that fail.
+    // It is much easier to compare the device to a small list of failing cases than to define all the passing ones.
+    const gpu_arch arch = ggml_sycl_info().devices[ggml_sycl_get_device()].hw_info.arch;
+    bool support_spda = !(arch == gpu_arch::intel_gpu_dg2_g10 ||
+                         arch == gpu_arch::intel_gpu_dg2_g11 ||
+                         arch == gpu_arch::intel_gpu_dg2_g12);
+
+    if (!support_spda && K->ne[0] == 64) {
+        return false;
+    }
     // Optional KV-length ceiling (GGML_SYCL_FA_ONEDNN_MAX_KV, 0 = unlimited). Escape hatch:
     // very long sequences make the fused SDPA slow enough to risk the xe driver watchdog on
     // some stacks; past the cap we fall back to the native FA kernel instead.

ggml/src/ggml-sycl/ggml-sycl.cpp

@@ -921,16 +921,16 @@ ggml_backend_sycl_buffer_type_alloc_buffer(ggml_backend_buffer_type_t buft,
 
     void * dev_ptr;
     if (use_usm_system) {
-        GGML_SYCL_DEBUG("[SYCL] allocating %lu Bytes with USM system\n", size);
+        GGML_SYCL_DEBUG("[SYCL] allocating %zu Bytes with USM system\n", size);
         dev_ptr = (void *)aligned_malloc_host(alignment, aligned_size);
         if (!dev_ptr) {
-            GGML_LOG_ERROR("%s: can't allocate %lu Bytes of memory on host\n", __func__, size);
+            GGML_LOG_ERROR("%s: can't allocate %zu Bytes of memory on host\n", __func__, size);
             return nullptr;
         }
     } else {
         SYCL_CHECK(CHECK_TRY_ERROR(dev_ptr = (void *)ggml_sycl_malloc_device(size, *stream)));
         if (!dev_ptr) {
-          GGML_LOG_ERROR("%s: can't allocate %lu Bytes of memory on device\n", __func__, size);
+          GGML_LOG_ERROR("%s: can't allocate %zu Bytes of memory on device\n", __func__, size);
           return nullptr;
         }
     }
@@ -1177,7 +1177,7 @@ ggml_backend_sycl_split_buffer_init_tensor(ggml_backend_buffer_t buffer,
         SYCL_CHECK(CHECK_TRY_ERROR(buf = (char *)ggml_sycl_malloc_device(size, *stream)));
         if (!buf) {
             char err_buf[1024];
-            snprintf(err_buf, 1023, "%s: can't allocate %lu Bytes of memory on device\n", __func__, size);
+            snprintf(err_buf, 1023, "%s: can't allocate %zu Bytes of memory on device\n", __func__, size);
             throw std::runtime_error(err_buf);
         }
         // set padding to 0 to avoid possible NaN values
@@ -1517,8 +1517,13 @@ static ggml_backend_buffer_t ggml_backend_sycl_host_buffer_type_alloc_buffer(ggm
 }
 
 static size_t ggml_backend_sycl_host_buffer_type_get_max_size(ggml_backend_buffer_type_t buft) {
-    ggml_backend_sycl_device_context * dev_ctx = (ggml_backend_sycl_device_context *) buft->device->context;
-    return dpct::dev_mgr::instance().get_device(dev_ctx->device).get_max_mem_alloc_size();
+
+    if (g_ggml_sycl_enable_host_pinned_mem) {
+        ggml_backend_sycl_device_context * dev_ctx = (ggml_backend_sycl_device_context *) buft->device->context;
+        return dpct::dev_mgr::instance().get_device(dev_ctx->device).get_max_mem_alloc_size();
+    } else {
+        return SIZE_MAX;
+    }
 }
 
 ggml_backend_buffer_type_t ggml_backend_sycl_host_buffer_type() {
@@ -1646,7 +1651,7 @@ struct ggml_sycl_pool_leg : public ggml_sycl_pool {
 
         SYCL_CHECK(CHECK_TRY_ERROR(ptr = (void *)ggml_sycl_malloc_device(look_ahead_size, *qptr)));
         if (!ptr) {
-            GGML_LOG_ERROR("%s: can't allocate %lu Bytes of memory on device/GPU\n", __func__, look_ahead_size);
+            GGML_LOG_ERROR("%s: can't allocate %zu Bytes of memory on device/GPU\n", __func__, look_ahead_size);
             return nullptr;
         }
 
@@ -1658,7 +1663,7 @@ struct ggml_sycl_pool_leg : public ggml_sycl_pool {
                 (uint32_t)(max_size/1024/1024), (uint32_t)(g_sycl_pool_size[id]/1024/1024), (uint32_t)(size/1024/1024));
 #endif
 
-        // GGML_SYCL_DEBUG("ggml_sycl_pool_malloc_leg look_ahead_size=%lu, return %p\n", look_ahead_size, ptr);
+        // GGML_SYCL_DEBUG("ggml_sycl_pool_malloc_leg look_ahead_size=%zu, return %p\n", look_ahead_size, ptr);
         return ptr;
     }
 
@@ -1838,7 +1843,7 @@ struct ggml_sycl_pool_host : public ggml_sycl_pool {
 
             SYCL_CHECK(CHECK_TRY_ERROR(ptr = (void *) sycl::malloc_host(size, *qptr)));
             if (!ptr) {
-                GGML_LOG_ERROR("%s: can't allocate %lu Bytes of memory on host\n", __func__, size);
+                GGML_LOG_ERROR("%s: can't allocate %zu Bytes of memory on host\n", __func__, size);
                 return nullptr;
             }
             pool_size += size;
@@ -2774,9 +2779,9 @@ inline void ggml_sycl_op_mul_mat_sycl(
         const float * src1_ddf1_i = src1->type == GGML_TYPE_F32 ? (const float *) src1_ddf_i : src1_ddq_as_f32.get();
 
         {
+#if GGML_SYCL_DNNL
             const int64_t gemm_flops = (int64_t)row_diff * src1_ncols * ne10;
             const bool use_mkl_direct = gemm_flops < 256 * 256 * 256;
-#if GGML_SYCL_DNNL
             if (g_ggml_sycl_enable_dnn && !use_mkl_direct) {
                 DnnlGemmWrapper::row_gemm(ctx, row_diff, src1_ncols, ne10, src0_ddf_i,
                                           DnnlGemmWrapper::to_dt<float>(), src1_ddf1_i, DnnlGemmWrapper::to_dt<float>(),
@@ -3513,7 +3518,9 @@ static void ggml_sycl_mul_mat_batched_sycl(ggml_backend_sycl_context & ctx, cons
     float *            dst_ddf  = static_cast<float *>(dst->data);
 
     const sycl::half * src1_f16       = static_cast<const sycl::half *>(src1->data);
+#if GGML_SYCL_DNNL
     const size_t       type_size_src0 = ggml_type_size(src0->type);
+#endif
     const size_t       type_size_src1 = ggml_type_size(src1->type);
 
     bool is_src0_cont_2 = ggml_is_contiguous_2(src0);
@@ -3530,6 +3537,7 @@ static void ggml_sycl_mul_mat_batched_sycl(ggml_backend_sycl_context & ctx, cons
         scope_op_debug_print    scope_dbg_print(__func__, "/to_fp16_nc_sycl", dst, /*num_src=*/2,
                                                 " : converting src1 to fp16");
 
+#if GGML_SYCL_DNNL
         // iterate tensor dims and find the slowest moving dim and stride
         int last_dim=0;
         int last_str=0;
@@ -3549,7 +3557,6 @@ static void ggml_sycl_mul_mat_batched_sycl(ggml_backend_sycl_context & ctx, cons
             }
 
         }
-#if GGML_SYCL_DNNL
         // oneDNN handles strided data and does not need overhead of ggml_get_to_fp16_nc_sycl
         const int64_t ne_src1 = src1->nb[last_str] * src1->ne[last_dim] / type_size_src1;
         src1_f16_alloc.alloc(ne_src1);
@@ -3804,6 +3811,7 @@ static bool ggml_sycl_supports_reorder_esimd(enum ggml_type type) {
     switch (type) {
         case GGML_TYPE_Q3_K:
         case GGML_TYPE_Q4_K:
+        case GGML_TYPE_Q5_K:
         case GGML_TYPE_Q6_K:
             return true;
         default:
@@ -6242,8 +6250,6 @@ static bool do_ggml_backend_sycl_device_supports_op(ggml_backend_dev_t dev, cons
         }
         case GGML_OP_ROPE:
         case GGML_OP_ROPE_BACK:
-            // FIXME: support ggml_rope_set_offset
-            return ((const int32_t *) op->op_params)[15] == 0;
         case GGML_OP_IM2COL:
         case GGML_OP_IM2COL_3D:
         case GGML_OP_UPSCALE:

ggml/src/ggml-sycl/im2col.cpp

@@ -85,7 +85,7 @@ static void im2col_sycl(const float *   x,
     */
     stream->parallel_for(sycl::nd_range<3>(block_nums * sycl::range<3>(1, 1, MIN(IC_KH_KW, SYCL_IM2COL_BLOCK_SIZE)),
                                            sycl::range<3>(1, 1, MIN(IC_KH_KW, SYCL_IM2COL_BLOCK_SIZE))),
-                         [=](sycl::nd_item<3> item_ct1) {
+                         [=](sycl::nd_item<3>) {
                              im2col_kernel(x, dst, IC, IW, IH, OH, OW, KW, KH, IC_IH_IW, IH_IW, N_OH, KH_KW, IC_KH_KW,
                                            s0, s1, p0, p1, d0, d1);
                          });
@@ -271,7 +271,7 @@ static void im2col_3d_sycl(const float *   src,
     */
     stream->parallel_for(sycl::nd_range<3>(block_nums * sycl::range<3>(1, 1, MIN(IC_KD_KH_KW, SYCL_IM2COL_BLOCK_SIZE)),
                                            sycl::range<3>(1, 1, MIN(IC_KD_KH_KW, SYCL_IM2COL_BLOCK_SIZE))),
-                         [=](sycl::nd_item<3> item_ct1) {
+                         [=](sycl::nd_item<3>) {
                              im2col_3d_kernel(src, dst, N, IC, ID, IH, IW, OC, KD, KH, KW, OD, OH, OW, OH_OW, KD_KH_KW,
                                               ID_IH_IW, KH_KW, IH_IW, IC_ID_IH_IW, IC_KD_KH_KW, OW_KD_KH_KW,
                                               OD_OH_OW_IC_KD_KH_KW, OH_OW_IC_KD_KH_KW, OW_IC_KD_KH_KW, N_OD_OH, OD_OH,

ggml/src/ggml-sycl/norm.cpp

@@ -7,9 +7,6 @@ static void norm_f32(const float* x, float* dst, const int ncols,
     const int64_t dst_stride_col, const int64_t dst_stride_row, const int64_t dst_stride_channel, const int64_t dst_stride_sample,
     const float eps, const sycl::nd_item<3>& item_ct1, sycl::float2* s_sum, int block_size) {
 
-    const int nrows = item_ct1.get_group_range(2);
-    const int nchannels = item_ct1.get_group_range(1);
-
     const int nthreads = item_ct1.get_local_range(2);
     const int sample  = item_ct1.get_group(0);
     const int channel = item_ct1.get_group(1);
@@ -155,9 +152,6 @@ static void rms_norm_f32(const float* x, float* dst, const int ncols,
     const float* mul = nullptr, const int64_t mul_stride_row = 0, const int64_t mul_stride_channel = 0,
     const int64_t mul_stride_sample = 0, const int mul_nrows = 0, const int mul_nchannels = 0, const int mul_nsamples = 0) {
 
-    const int nrows = item_ct1.get_group_range(2);
-    const int nchannels = item_ct1.get_group_range(1);
-
     const int sample  = item_ct1.get_group(0);
     const int channel = item_ct1.get_group(1);
     const int row     = item_ct1.get_group(2);
@@ -225,8 +219,6 @@ static void l2_norm_f32(const float * x, float * dst, const int ncols,
     const int64_t src_stride_sample, const int64_t dst_stride_col, const int64_t dst_stride_row,
     const int64_t dst_stride_channel, const int64_t dst_stride_sample, const float eps,
     const sycl::nd_item<3>& item_ct1, float* s_sum, const int block_size) {
-    const int nrows     = item_ct1.get_group_range(2);
-    const int nchannels = item_ct1.get_group_range(1);
 
     const int row     = item_ct1.get_group(2);
     const int channel = item_ct1.get_group(1);

ggml/src/ggml-sycl/rope.cpp

@@ -41,7 +41,7 @@ template <bool forward, bool has_ff, typename T, typename D>
 static void rope_norm(const T *x, D *dst, const int ne00, const int ne01,
                       const int ne02, const int s01, const int s02,
                       const int s03, const int s1, const int s2, const int s3,
-                      const int n_dims, const int32_t *pos,
+                      const int n_dims, const int n_offs, const int32_t *pos,
                       const float freq_scale, const float ext_factor,
                       const float attn_factor, const rope_corr_dims corr_dims,
                       const float theta_scale, const float *freq_factors,
@@ -78,19 +78,21 @@ static void rope_norm(const T *x, D *dst, const int ne00, const int ne01,
             ggml_sycl_memcpy_1<4>(dst + idst, &v);
         }
     };
-    if (i0 >= n_dims) {
+    if (i0 < n_offs || i0 >= n_offs + n_dims) {
         store_coaelsced(x[ix + 0], x[ix + 1]);
         return;
     }
 
-    const float theta_base = pos[i2] * dpct::pow(theta_scale, i0 / 2.0f);
+    const int iw = i0 - n_offs; // relative idx
 
-    const float freq_factor = has_ff ? freq_factors[i0 / 2] : 1.0f;
+    const float theta_base = pos[i2] * dpct::pow(theta_scale, iw / 2.0f);
+
+    const float freq_factor = has_ff ? freq_factors[iw / 2] : 1.0f;
 
     float cos_theta;
     float sin_theta;
 
-    rope_yarn<forward>(theta_base / freq_factor, freq_scale, corr_dims, i0,
+    rope_yarn<forward>(theta_base / freq_factor, freq_scale, corr_dims, iw,
                        ext_factor, attn_factor, cos_theta, sin_theta);
 
     const float x0 = x[ix + 0];
@@ -104,7 +106,7 @@ template <bool forward, bool has_ff, typename T, typename D>
 static void rope_neox(const T *x, D *dst, const int ne00, const int ne01,
                       const int ne02, const int s01, const int s02,
                       const int s03, const int s1, const int s2, const int s3,
-                      const int n_dims, const int32_t *pos,
+                      const int n_dims, const int n_offs, const int32_t *pos,
                       const float freq_scale, const float ext_factor,
                       const float attn_factor, const rope_corr_dims corr_dims,
                       const float theta_scale, const float *freq_factors,
@@ -132,35 +134,38 @@ static void rope_neox(const T *x, D *dst, const int ne00, const int ne01,
         idst += row_indices[i2] * set_rows_stride;
     }
 
-    if (i0 >= n_dims) {
+    if (i0 < n_offs || i0 >= n_offs + n_dims) {
         dst[idst + i0 / 2 + 0] = ggml_sycl_cast<D>(x[ix + i0 / 2 + 0]);
         dst[idst + i0 / 2 + 1] = ggml_sycl_cast<D>(x[ix + i0 / 2 + 1]);
 
         return;
     }
 
-    const float theta_base = pos[i2] * dpct::pow(theta_scale, i0 / 2.0f);
+    const int iw = i0 - n_offs; // relative idx
 
-    const float freq_factor = has_ff ? freq_factors[i0 / 2] : 1.0f;
+    const float theta_base = pos[i2] * dpct::pow(theta_scale, iw / 2.0f);
+
+    const float freq_factor = has_ff ? freq_factors[iw / 2] : 1.0f;
 
     float cos_theta;
     float sin_theta;
 
-    rope_yarn<forward>(theta_base / freq_factor, freq_scale, corr_dims, i0,
+    rope_yarn<forward>(theta_base / freq_factor, freq_scale, corr_dims, iw,
                        ext_factor, attn_factor, cos_theta, sin_theta);
 
-    const float x0 = x[ix + 0];
-    const float x1 = x[ix + n_dims / 2];
+    // idst/ix point at channel i0/2; the first channel of the rotated pair is n_offs + iw/2 = i0/2 + n_offs/2
+    const float x0 = x[ix + n_offs / 2 + 0];
+    const float x1 = x[ix + n_offs / 2 + n_dims / 2];
 
-    dst[idst + 0] = ggml_sycl_cast<D>(x0 * cos_theta - x1 * sin_theta);
-    dst[idst + n_dims / 2] = ggml_sycl_cast<D>(x0 * sin_theta + x1 * cos_theta);
+    dst[idst + n_offs / 2 + 0] = ggml_sycl_cast<D>(x0 * cos_theta - x1 * sin_theta);
+    dst[idst + n_offs / 2 + n_dims / 2] = ggml_sycl_cast<D>(x0 * sin_theta + x1 * cos_theta);
 }
 
 template <bool forward, bool has_ff, typename T>
 static void rope_multi(const T *x, T *dst, const int ne00, const int ne01,
                        const int ne02, const int s01, const int s02,
                        const int s03, const int s1, const int s2, const int s3,
-                       const int n_dims, const int32_t *pos,
+                       const int n_dims, const int n_offs, const int32_t *pos,
                        const float freq_scale, const float ext_factor,
                        const float attn_factor, const rope_corr_dims corr_dims,
                        const float theta_scale, const float *freq_factors,
@@ -183,54 +188,57 @@ static void rope_multi(const T *x, T *dst, const int ne00, const int ne01,
     int idst = i0 / 2 + i1 * s1 + i2 * s2 + i3 * s3;
     const int ix = i0 / 2 + i1 * s01 + i2 * s02 + i3 * s03;
 
-    if (i0 >= n_dims) {
+    if (i0 < n_offs || i0 >= n_offs + n_dims) {
         dst[idst + i0 / 2 + 0] = x[ix + i0 / 2 + 0];
         dst[idst + i0 / 2 + 1] = x[ix + i0 / 2 + 1];
 
         return;
     }
 
+    const int iw = i0 - n_offs; // relative idx
+
     const int sect_dims =
         sections.v[0] + sections.v[1] + sections.v[2] + sections.v[3];
     const int sec_w = sections.v[1] + sections.v[0];
-    const int sector = (i0 / 2) % sect_dims;
+    const int sector = (iw / 2) % sect_dims;
 
     float theta_base = 0.0;
     if (is_imrope) {
         if (sector % 3 == 1 && sector < 3 * sections.v[1]) { // h
-            theta_base = pos[i2 + ne02 * 1] * dpct::pow(theta_scale, i0 / 2.0f);
+            theta_base = pos[i2 + ne02 * 1] * dpct::pow(theta_scale, iw / 2.0f);
         } else if (sector % 3 == 2 && sector < 3 * sections.v[2]) { // w
-            theta_base = pos[i2 + ne02 * 2] * dpct::pow(theta_scale, i0 / 2.0f);
+            theta_base = pos[i2 + ne02 * 2] * dpct::pow(theta_scale, iw / 2.0f);
         } else if (sector % 3 == 0 && sector < 3 * sections.v[0]) { // t
-            theta_base = pos[i2] * dpct::pow(theta_scale, i0 / 2.0f);
+            theta_base = pos[i2] * dpct::pow(theta_scale, iw / 2.0f);
         } else {
-            theta_base = pos[i2 + ne02 * 3] * dpct::pow(theta_scale, i0 / 2.0f);
+            theta_base = pos[i2 + ne02 * 3] * dpct::pow(theta_scale, iw / 2.0f);
         }
     } else {
         if (sector < sections.v[0]) {
-            theta_base = pos[i2] * dpct::pow(theta_scale, i0 / 2.0f);
+            theta_base = pos[i2] * dpct::pow(theta_scale, iw / 2.0f);
         } else if (sector >= sections.v[0] && sector < sec_w) {
-            theta_base = pos[i2 + ne02 * 1] * dpct::pow(theta_scale, i0 / 2.0f);
+            theta_base = pos[i2 + ne02 * 1] * dpct::pow(theta_scale, iw / 2.0f);
         } else if (sector >= sec_w && sector < sec_w + sections.v[2]) {
-            theta_base = pos[i2 + ne02 * 2] * dpct::pow(theta_scale, i0 / 2.0f);
+            theta_base = pos[i2 + ne02 * 2] * dpct::pow(theta_scale, iw / 2.0f);
         } else if (sector >= sec_w + sections.v[2]) {
-            theta_base = pos[i2 + ne02 * 3] * dpct::pow(theta_scale, i0 / 2.0f);
+            theta_base = pos[i2 + ne02 * 3] * dpct::pow(theta_scale, iw / 2.0f);
         }
     }
 
-    const float freq_factor = has_ff ? freq_factors[i0 / 2] : 1.0f;
+    const float freq_factor = has_ff ? freq_factors[iw / 2] : 1.0f;
 
     float cos_theta;
     float sin_theta;
 
-    rope_yarn<forward>(theta_base / freq_factor, freq_scale, corr_dims, i0,
+    rope_yarn<forward>(theta_base / freq_factor, freq_scale, corr_dims, iw,
                        ext_factor, attn_factor, cos_theta, sin_theta);
 
-    const float x0 = x[ix + 0];
-    const float x1 = x[ix + n_dims / 2];
+    // idst/ix point at channel i0/2; the first channel of the rotated pair is n_offs + iw/2 = i0/2 + n_offs/2
+    const float x0 = x[ix + n_offs / 2 + 0];
+    const float x1 = x[ix + n_offs / 2 + n_dims / 2];
 
-    dst[idst + 0] = x0 * cos_theta - x1 * sin_theta;
-    dst[idst + n_dims / 2] = x0 * sin_theta + x1 * cos_theta;
+    dst[idst + n_offs / 2 + 0] = x0 * cos_theta - x1 * sin_theta;
+    dst[idst + n_offs / 2 + n_dims / 2] = x0 * sin_theta + x1 * cos_theta;
 }
 
 template <bool forward, bool has_ff, typename T>
@@ -293,7 +301,7 @@ static void
 rope_norm_sycl(const T *x, D *dst, const int ne00, const int ne01,
                const int ne02, const int s01, const int s02, const int s03,
                const int s1, const int s2, const int s3, const int n_dims,
-               const int nr, const int32_t *pos, const float freq_scale,
+               const int n_offs, const int nr, const int32_t *pos, const float freq_scale,
                const float freq_base, const float ext_factor,
                const float attn_factor, const rope_corr_dims corr_dims,
                const float *freq_factors, const int64_t *row_indices,
@@ -313,7 +321,7 @@ rope_norm_sycl(const T *x, D *dst, const int ne00, const int ne01,
                 GGML_UNUSED(item_ct1);
                 rope_norm<forward, false>(
                     x, dst, ne00, ne01, ne02, s01, s02, s03, s1, s2, s3, n_dims,
-                    pos, freq_scale, ext_factor, attn_factor, corr_dims,
+                    n_offs, pos, freq_scale, ext_factor, attn_factor, corr_dims,
                     theta_scale, freq_factors, row_indices, set_rows_stride);
             });
     } else {
@@ -323,7 +331,7 @@ rope_norm_sycl(const T *x, D *dst, const int ne00, const int ne01,
                 GGML_UNUSED(item_ct1);
                 rope_norm<forward, true>(
                     x, dst, ne00, ne01, ne02, s01, s02, s03, s1, s2, s3, n_dims,
-                    pos, freq_scale, ext_factor, attn_factor, corr_dims,
+                    n_offs, pos, freq_scale, ext_factor, attn_factor, corr_dims,
                     theta_scale, freq_factors, row_indices, set_rows_stride);
             });
     }
@@ -334,7 +342,7 @@ static void
 rope_neox_sycl(const T *x, D *dst, const int ne00, const int ne01,
                const int ne02, const int s01, const int s02, const int s03,
                const int s1, const int s2, const int s3, const int n_dims,
-               const int nr, const int32_t *pos, const float freq_scale,
+               const int n_offs, const int nr, const int32_t *pos, const float freq_scale,
                const float freq_base, const float ext_factor,
                const float attn_factor, const rope_corr_dims corr_dims,
                const float *freq_factors, const int64_t *row_indices,
@@ -354,7 +362,7 @@ rope_neox_sycl(const T *x, D *dst, const int ne00, const int ne01,
                 GGML_UNUSED(item_ct1);
                 rope_neox<forward, false>(
                     x, dst, ne00, ne01, ne02, s01, s02, s03, s1, s2, s3, n_dims,
-                    pos, freq_scale, ext_factor, attn_factor, corr_dims,
+                    n_offs, pos, freq_scale, ext_factor, attn_factor, corr_dims,
                     theta_scale, freq_factors, row_indices, set_rows_stride);
             });
     } else {
@@ -364,7 +372,7 @@ rope_neox_sycl(const T *x, D *dst, const int ne00, const int ne01,
                 GGML_UNUSED(item_ct1);
                 rope_neox<forward, true>(
                     x, dst, ne00, ne01, ne02, s01, s02, s03, s1, s2, s3, n_dims,
-                    pos, freq_scale, ext_factor, attn_factor, corr_dims,
+                    n_offs, pos, freq_scale, ext_factor, attn_factor, corr_dims,
                     theta_scale, freq_factors, row_indices, set_rows_stride);
             });
     }
@@ -375,7 +383,7 @@ static void
 rope_multi_sycl(const T *x, T *dst, const int ne00, const int ne01,
                 const int ne02, const int s01, const int s02, const int s03,
                 const int s1, const int s2, const int s3, const int n_dims,
-                const int nr, const int32_t *pos, const float freq_scale,
+                const int n_offs, const int nr, const int32_t *pos, const float freq_scale,
                 const float freq_base, const float ext_factor,
                 const float attn_factor, const rope_corr_dims corr_dims,
                 const float *freq_factors, const mrope_sections sections,
@@ -395,7 +403,7 @@ rope_multi_sycl(const T *x, T *dst, const int ne00, const int ne01,
                 GGML_UNUSED(item_ct1);
                 rope_multi<forward, false, T>(
                     x, dst, ne00, ne01, ne02, s01, s02, s03, s1, s2, s3, n_dims,
-                    pos, freq_scale, ext_factor, attn_factor, corr_dims,
+                    n_offs, pos, freq_scale, ext_factor, attn_factor, corr_dims,
                     theta_scale, freq_factors, sections, is_imrope);
             });
     } else {
@@ -405,7 +413,7 @@ rope_multi_sycl(const T *x, T *dst, const int ne00, const int ne01,
                 GGML_UNUSED(item_ct1);
                 rope_multi<forward, true, T>(
                     x, dst, ne00, ne01, ne02, s01, s02, s03, s1, s2, s3, n_dims,
-                    pos, freq_scale, ext_factor, attn_factor, corr_dims,
+                    n_offs, pos, freq_scale, ext_factor, attn_factor, corr_dims,
                     theta_scale, freq_factors, sections, is_imrope);
             });
     }
@@ -497,6 +505,7 @@ void ggml_sycl_op_rope_impl(ggml_backend_sycl_context &ctx, ggml_tensor *dst,
     const int n_dims = ((int32_t *)dst->op_params)[1];
     const int mode = ((int32_t *)dst->op_params)[2];
     const int n_ctx_orig = ((int32_t *)dst->op_params)[4];
+    const int n_offs = ((int32_t *)dst->op_params)[15];
     mrope_sections sections;
 
     float freq_base;
@@ -526,6 +535,7 @@ void ggml_sycl_op_rope_impl(ggml_backend_sycl_context &ctx, ggml_tensor *dst,
 
     if (is_vision) {
         GGML_ASSERT(n_dims == ne00 / 2);
+        GGML_ASSERT(n_offs == 0); // offset not supported for vision, as the rotated pairs span the whole row
     }
 
     const int32_t *pos = (const int32_t *)src1_d;
@@ -545,19 +555,19 @@ void ggml_sycl_op_rope_impl(ggml_backend_sycl_context &ctx, ggml_tensor *dst,
         if (src0->type == GGML_TYPE_F32 && dst_type == GGML_TYPE_F32) {
             rope_neox_sycl<forward, float, float>(
                 (const float *)src0_d, (float *)dst_d, ne00, ne01, ne02, s01,
-                s02, s03, s1, s2, s3, n_dims, nr, pos, freq_scale, freq_base,
+                s02, s03, s1, s2, s3, n_dims, n_offs, nr, pos, freq_scale, freq_base,
                 ext_factor, attn_factor, corr_dims, freq_factors, row_indices,
                 set_rows_stride, stream);
         } else if (src0->type == GGML_TYPE_F32 && dst_type == GGML_TYPE_F16) {
             rope_neox_sycl<forward, float, sycl::half>(
                 (const float *)src0_d, (sycl::half *)dst_d, ne00, ne01, ne02,
-                s01, s02, s03, s1, s2, s3, n_dims, nr, pos, freq_scale,
+                s01, s02, s03, s1, s2, s3, n_dims, n_offs, nr, pos, freq_scale,
                 freq_base, ext_factor, attn_factor, corr_dims, freq_factors,
                 row_indices, set_rows_stride, stream);
         } else if (src0->type == GGML_TYPE_F16 && dst_type == GGML_TYPE_F16) {
             rope_neox_sycl<forward, sycl::half, sycl::half>(
                 (const sycl::half *)src0_d, (sycl::half *)dst_d, ne00, ne01,
-                ne02, s01, s02, s03, s1, s2, s3, n_dims, nr, pos, freq_scale,
+                ne02, s01, s02, s03, s1, s2, s3, n_dims, n_offs, nr, pos, freq_scale,
                 freq_base, ext_factor, attn_factor, corr_dims, freq_factors,
                 row_indices, set_rows_stride, stream);
         } else {
@@ -568,13 +578,13 @@ void ggml_sycl_op_rope_impl(ggml_backend_sycl_context &ctx, ggml_tensor *dst,
         if (src0->type == GGML_TYPE_F32) {
             rope_multi_sycl<forward>((const float *)src0_d, (float *)dst_d,
                                      ne00, ne01, ne02, s01, s02, s03, s1, s2,
-                                     s3, n_dims, nr, pos, freq_scale, freq_base,
+                                     s3, n_dims, n_offs, nr, pos, freq_scale, freq_base,
                                      ext_factor, attn_factor, corr_dims,
                                      freq_factors, sections, is_imrope, stream);
         } else if (src0->type == GGML_TYPE_F16) {
             rope_multi_sycl<forward>(
                 (const sycl::half *)src0_d, (sycl::half *)dst_d, ne00, ne01,
-                ne02, s01, s02, s03, s1, s2, s3, n_dims, nr, pos, freq_scale,
+                ne02, s01, s02, s03, s1, s2, s3, n_dims, n_offs, nr, pos, freq_scale,
                 freq_base, ext_factor, attn_factor, corr_dims, freq_factors,
                 sections, is_imrope, stream);
         } else {
@@ -602,19 +612,19 @@ void ggml_sycl_op_rope_impl(ggml_backend_sycl_context &ctx, ggml_tensor *dst,
         if (src0->type == GGML_TYPE_F32 && dst_type == GGML_TYPE_F32) {
             rope_norm_sycl<forward, float, float>(
                 (const float *)src0_d, (float *)dst_d, ne00, ne01, ne02, s01,
-                s02, s03, s1, s2, s3, n_dims, nr, pos, freq_scale, freq_base,
+                s02, s03, s1, s2, s3, n_dims, n_offs, nr, pos, freq_scale, freq_base,
                 ext_factor, attn_factor, corr_dims, freq_factors, row_indices,
                 set_rows_stride, stream);
         } else if (src0->type == GGML_TYPE_F32 && dst_type == GGML_TYPE_F16) {
             rope_norm_sycl<forward, float, sycl::half>(
                 (const float *)src0_d, (sycl::half *)dst_d, ne00, ne01, ne02,
-                s01, s02, s03, s1, s2, s3, n_dims, nr, pos, freq_scale,
+                s01, s02, s03, s1, s2, s3, n_dims, n_offs, nr, pos, freq_scale,
                 freq_base, ext_factor, attn_factor, corr_dims, freq_factors,
                 row_indices, set_rows_stride, stream);
         } else if (src0->type == GGML_TYPE_F16 && dst_type == GGML_TYPE_F16) {
             rope_norm_sycl<forward, sycl::half, sycl::half>(
                 (const sycl::half *)src0_d, (sycl::half *)dst_d, ne00, ne01,
-                ne02, s01, s02, s03, s1, s2, s3, n_dims, nr, pos, freq_scale,
+                ne02, s01, s02, s03, s1, s2, s3, n_dims, n_offs, nr, pos, freq_scale,
                 freq_base, ext_factor, attn_factor, corr_dims, freq_factors,
                 row_indices, set_rows_stride, stream);
         } else {

ggml/src/ggml-sycl/set_rows.cpp

@@ -291,7 +291,7 @@ static void set_rows_sycl(
 
     stream->parallel_for(
         sycl::nd_range<1>(grid_size * block_size, block_size),
-        [=](sycl::nd_item<1> item_ct1) [[intel::reqd_sub_group_size(WARP_SIZE)]] {
+        [=](sycl::nd_item<1> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] {
             k_set_rows<TIn, TIdx, TOut>(
                 src0_d, src1_d, dst_d,
                 ne00, ne01, ne02,

ggml/src/ggml-vulkan/vulkan-shaders/flash_attn.comp

@@ -121,13 +121,13 @@ void main() {
         const uint buf_ib = r * qf_stride + d / 8;
         const uint buf_iqs = d % 8;
 
-        FLOAT_TYPEV4 vals = is_in_bounds ? FLOAT_TYPEV4(data_qv4[q_offset / 4 + (i * Br + r) * q_stride / 4 + d] * p.scale) : FLOAT_TYPEV4(0.0f);
-        const FLOAT_TYPEV4 abs_vals = abs(vals);
+        vec4 vals = is_in_bounds ? data_qv4[q_offset / 4 + (i * Br + r) * q_stride / 4 + d] * p.scale : vec4(0.0f);
+        const vec4 abs_vals = abs(vals);
 
-        const FLOAT_TYPE thread_max = max(max(abs_vals.x, abs_vals.y), max(abs_vals.z, abs_vals.w));
-        const FLOAT_TYPE amax = subgroupClusteredMax(thread_max, 8);
-        const FLOAT_TYPE qd = amax / FLOAT_TYPE(127.0);
-        const FLOAT_TYPE qd_inv = qd != FLOAT_TYPE(0.0) ? FLOAT_TYPE(1.0) / qd : FLOAT_TYPE(0.0);
+        const float thread_max = max(max(abs_vals.x, abs_vals.y), max(abs_vals.z, abs_vals.w));
+        const float amax = subgroupClusteredMax(thread_max, 8);
+        const float qd = amax / 127.0f;
+        const float qd_inv = qd != 0.0f ? 1.0f / qd : 0.0f;
         vals = round(vals * qd_inv);
 
         Qf[buf_ib].qs[buf_iqs] = pack32(i8vec4(vals));
@@ -136,11 +136,11 @@ void main() {
         // the row-sum scaled by qd, used in k_dot_correction.
         if (FaTypeK == FA_TYPE_Q8_0) {
             if (buf_iqs == 0) {
-                Qf[buf_ib].ds = FLOAT_TYPEV2(qd, 0.0);
+                Qf[buf_ib].ds = FLOAT_TYPEV2(qd, 0.0f);
             }
         } else {
-            const FLOAT_TYPE thread_sum = vals.x + vals.y + vals.z + vals.w;
-            const FLOAT_TYPE sum = subgroupClusteredAdd(thread_sum, 8);
+            const float thread_sum = vals.x + vals.y + vals.z + vals.w;
+            const float sum = subgroupClusteredAdd(thread_sum, 8);
 
             if (buf_iqs == 0) {
                 Qf[buf_ib].ds = FLOAT_TYPEV2(qd, sum * qd);

ggml/src/ggml-webgpu/ggml-webgpu.cpp

@@ -2714,6 +2714,7 @@ static webgpu_encoded_op ggml_webgpu_rope(webgpu_context & ctx,
     const int n_dims     = ((int32_t *) dst->op_params)[1];
     const int mode       = ((int32_t *) dst->op_params)[2];
     const int n_ctx_orig = ((int32_t *) dst->op_params)[4];
+    const int n_offs     = ((int32_t *) dst->op_params)[15];
 
     float freq_base;
     float freq_scale;
@@ -2762,7 +2763,8 @@ static webgpu_encoded_op ggml_webgpu_rope(webgpu_context & ctx,
         (uint32_t) sections[0],
         (uint32_t) sections[1],
         (uint32_t) sections[2],
-        (uint32_t) sections[3]
+        (uint32_t) sections[3],
+        (uint32_t) n_offs
     };
 
     std::vector<wgpu::BindGroupEntry> entries     = { ggml_webgpu_make_tensor_bind_group_entry(ctx, 0, src0),
@@ -4472,9 +4474,7 @@ static bool ggml_backend_webgpu_device_supports_op(ggml_backend_dev_t dev, const
             supports_op = (op->type == GGML_TYPE_F32 && src0->type == GGML_TYPE_F32) && ggml_is_contiguous_rows(src0);
             break;
         case GGML_OP_ROPE:
-            // FIXME: support ggml_rope_set_offset
-            supports_op =
-                (op->type == GGML_TYPE_F32 || op->type == GGML_TYPE_F16) && ((const int32_t *) op->op_params)[15] == 0;
+            supports_op = op->type == GGML_TYPE_F32 || op->type == GGML_TYPE_F16;
             break;
         case GGML_OP_GLU:
             switch (ggml_get_glu_op(op)) {

ggml/src/ggml-webgpu/wgsl-shaders/rope.wgsl

@@ -38,7 +38,8 @@ struct Params {
     sections0: u32,
     sections1: u32,
     sections2: u32,
-    sections3: u32
+    sections3: u32,
+    n_offs: u32
 };
 
 @group(0) @binding(0)
@@ -126,7 +127,8 @@ fn rope_yarn(theta_extrap: f32, i: u32) -> vec2<f32> {
 
 fn pair_base(i0: u32, div_2: bool) -> u32 {
     if (div_2) {
-        return i0 / 2;
+        // first channel of the rotated pair: n_offs + (i0 - n_offs)/2
+        return i0 / 2 + params.n_offs / 2;
     } else {
         return i0;
     }
@@ -165,20 +167,22 @@ fn main(@builtin(global_invocation_id) gid: vec3<u32>) {
     let i_src_row = params.offset_src0 + i3 * params.stride_src03 + i2 * params.stride_src02 + i1 * params.stride_src01;
     let i_dst_row = params.offset_dst + i3 * params.stride_dst3 + i2 * params.stride_dst2 + i1 * params.stride_dst1;
 
-    if (i0 >= params.n_dims && !is_vision) {
+    if ((i0 < params.n_offs || i0 >= params.n_offs + params.n_dims) && !is_vision) {
         let i_src = i_src_row + i0;
         let i_dst = i_dst_row + i0;
         rotate(i_dst, i_dst + 1, f32(src0[i_src]), f32(src0[i_src + 1]));
         return;
     }
 
+    let iw = i0 - params.n_offs; // relative idx
+
     var theta_base_mult: u32 = 0;
-    var theta_scale_pwr: u32 = i0 / 2;
+    var theta_scale_pwr: u32 = iw / 2;
     if (is_mrope) {
         let sect_dims = params.sections0 + params.sections1 + params.sections2 + params.sections3;
         let sec_w = params.sections1 + params.sections0;
         let sec_e = params.sections2 + sec_w;
-        let sector = (i0 / 2) % sect_dims;
+        let sector = (iw / 2) % sect_dims;
         if (is_imrope) {
           if (sector % 3 == 1 && sector < 3 * params.sections1) {
               theta_base_mult = 1;
@@ -203,7 +207,7 @@ fn main(@builtin(global_invocation_id) gid: vec3<u32>) {
           } else if (sector >= sec_e) {
               if (is_vision) {
                   theta_scale_pwr = sector - sec_e;
-                  theta_scale_pwr = (i0 / 2) % sec_e;
+                  theta_scale_pwr = (iw / 2) % sec_e;
               }
               theta_base_mult = 3;
           } else if (is_vision) {
@@ -212,7 +216,7 @@ fn main(@builtin(global_invocation_id) gid: vec3<u32>) {
         }
     }
     let theta_base = f32(src1[params.offset_src1 + i2 + params.ne2 * theta_base_mult]) * pow(params.theta_scale, f32(theta_scale_pwr));
-    let thetas = rope_yarn(theta_base/freq_factor(i0), i0);
+    let thetas = rope_yarn(theta_base/freq_factor(iw), iw);
 
     let i_src = i_src_row + pair_base(i0, is_neox || is_mrope || is_vision);
     let i_dst = i_dst_row + pair_base(i0, is_neox || is_mrope || is_vision);

ggml/src/ggml-zendnn/CMakeLists.txt

@@ -86,6 +86,6 @@ endif()
 
 target_link_libraries(ggml-zendnn PRIVATE m pthread)
 
-if (GGML_OPENMP)
-    target_link_libraries(ggml-zendnn PRIVATE OpenMP::OpenMP_CXX)
+if (GGML_OPENMP_ENABLED)
+    target_link_libraries(ggml-zendnn PRIVATE ${GGML_OPENMP_TARGET_CXX})
 endif()

gguf-py/gguf/constants.py

@@ -205,6 +205,9 @@ class Keys:
         VALUE_LENGTH_MLA             = "{arch}.attention.value_length_mla"
         KEY_LENGTH_SWA               = "{arch}.attention.key_length_swa"
         VALUE_LENGTH_SWA             = "{arch}.attention.value_length_swa"
+        KEY_LENGTH_MLA_SWA           = "{arch}.attention.key_length_mla_swa"
+        VALUE_LENGTH_MLA_SWA         = "{arch}.attention.value_length_mla_swa"
+        KV_LORA_RANK_SWA             = "{arch}.attention.kv_lora_rank_swa"
         SHARED_KV_LAYERS             = "{arch}.attention.shared_kv_layers"
         SLIDING_WINDOW_PATTERN       = "{arch}.attention.sliding_window_pattern"
         TEMPERATURE_SCALE            = "{arch}.attention.temperature_scale"
@@ -558,6 +561,7 @@ class MODEL_ARCH(IntEnum):
     BAILINGMOE2      = auto()
     BAILINGMOE3      = auto()
     DOTS1            = auto()
+    DOTS3NOTE        = auto()
     ARCEE            = auto()
     AFMOE            = auto()
     LAGUNA           = auto()
@@ -1275,6 +1279,7 @@ MODEL_ARCH_NAMES: dict[MODEL_ARCH, str] = {
     MODEL_ARCH.BAILINGMOE2:      "bailingmoe2",
     MODEL_ARCH.BAILINGMOE3:      "bailingmoe3",
     MODEL_ARCH.DOTS1:            "dots1",
+    MODEL_ARCH.DOTS3NOTE:        "dots3note",
     MODEL_ARCH.ARCEE:            "arcee",
     MODEL_ARCH.AFMOE:            "afmoe",
     MODEL_ARCH.LAGUNA:           "laguna",
@@ -4334,6 +4339,44 @@ MODEL_TENSORS: dict[MODEL_ARCH, list[MODEL_TENSOR]] = {
         MODEL_TENSOR.FFN_UP_EXP,
         MODEL_TENSOR.FFN_UP_SHEXP,
     ],
+    MODEL_ARCH.DOTS3NOTE: [
+        MODEL_TENSOR.TOKEN_EMBD,
+        MODEL_TENSOR.OUTPUT_NORM,
+        MODEL_TENSOR.OUTPUT,
+        MODEL_TENSOR.ATTN_NORM,
+        MODEL_TENSOR.ATTN_Q_A,
+        MODEL_TENSOR.ATTN_Q_B,
+        MODEL_TENSOR.ATTN_KV_A_MQA,
+        MODEL_TENSOR.ATTN_K_B,
+        MODEL_TENSOR.ATTN_V_B,
+        MODEL_TENSOR.ATTN_Q_A_NORM,
+        MODEL_TENSOR.ATTN_KV_A_NORM,
+        MODEL_TENSOR.ATTN_K_NORM,
+        MODEL_TENSOR.ATTN_GATE,
+        MODEL_TENSOR.ATTN_OUT,
+        MODEL_TENSOR.FFN_NORM,
+        MODEL_TENSOR.FFN_GATE,
+        MODEL_TENSOR.FFN_DOWN,
+        MODEL_TENSOR.FFN_UP,
+        MODEL_TENSOR.FFN_GATE_INP,
+        MODEL_TENSOR.FFN_EXP_PROBS_B,
+        MODEL_TENSOR.FFN_GATE_EXP,
+        MODEL_TENSOR.FFN_DOWN_EXP,
+        MODEL_TENSOR.FFN_UP_EXP,
+        MODEL_TENSOR.FFN_GATE_SHEXP,
+        MODEL_TENSOR.FFN_DOWN_SHEXP,
+        MODEL_TENSOR.FFN_UP_SHEXP,
+        MODEL_TENSOR.INDEXER_K_NORM,
+        MODEL_TENSOR.INDEXER_PROJ,
+        MODEL_TENSOR.INDEXER_ATTN_K,
+        MODEL_TENSOR.INDEXER_ATTN_Q_B,
+        # NextN/MTP tensors - preserved but unused
+        MODEL_TENSOR.NEXTN_EH_PROJ,
+        MODEL_TENSOR.NEXTN_EMBED_TOKENS,
+        MODEL_TENSOR.NEXTN_ENORM,
+        MODEL_TENSOR.NEXTN_HNORM,
+        MODEL_TENSOR.NEXTN_SHARED_HEAD_NORM,
+    ],
     MODEL_ARCH.ARCEE: [
         MODEL_TENSOR.TOKEN_EMBD,
         MODEL_TENSOR.OUTPUT_NORM,

gguf-py/gguf/gguf_writer.py

@@ -785,6 +785,15 @@ class GGUFWriter:
     def add_key_length_swa(self, length: int) -> None:
         self.add_uint32(Keys.Attention.KEY_LENGTH_SWA.format(arch=self.arch), length)
 
+    def add_key_length_mla_swa(self, length: int) -> None:
+        self.add_uint32(Keys.Attention.KEY_LENGTH_MLA_SWA.format(arch=self.arch), length)
+
+    def add_value_length_mla_swa(self, length: int) -> None:
+        self.add_uint32(Keys.Attention.VALUE_LENGTH_MLA_SWA.format(arch=self.arch), length)
+
+    def add_kv_lora_rank_swa(self, length: int) -> None:
+        self.add_uint32(Keys.Attention.KV_LORA_RANK_SWA.format(arch=self.arch), length)
+
     def add_value_length_swa(self, length: int) -> None:
         self.add_uint32(Keys.Attention.VALUE_LENGTH_SWA.format(arch=self.arch), length)
 

gguf-py/gguf/tensor_mapping.py

@@ -723,6 +723,7 @@ class TensorNameMap:
             "model.layers.layers.{bid}.mixer.k",                              # plamo2
             "model.layers.layers.{bid}.mixer.k_norm",                         # plamo3
             "layers.{bid}.self_attn.k_norm",                                  # qwen3-embedding
+            "model.layers.{bid}.self_attn.k_rope_only_layernorm",             # dots3note
             "model.layers.{bid}.attention.key_layernorm",                     # apertus
         ),
 

scripts/make-release-desc.sh

@@ -15,7 +15,8 @@
 # tag exists.
 #
 # Env (when running in GitHub Actions):
-#   GITHUB_OUTPUT: previous_tag, changelog_title, changelog and nightly are written here
+#   GITHUB_OUTPUT: previous_tag, changelog_title, changelog, nightly and nightly_tag
+#     are written here
 #   GITHUB_REPOSITORY: owner/repo, used to build the nightly release URL (skipped when unset)
 set -euo pipefail
 
@@ -52,10 +53,10 @@ PREV="$( { git tag --list; echo "${VERSION}"; } \
 
 if [[ -n "${PREV}" ]]; then
     CHANGELOG="$(git log --oneline "${PREV}..${RELEASE_COMMIT}")"
-    CHANGELOG_TITLE="Change log since ${PREV}"
+    CHANGELOG_TITLE="Changelog since ${PREV}"
 else
     CHANGELOG="(no previous release tag found)"
-    CHANGELOG_TITLE="Change log"
+    CHANGELOG_TITLE="Changelog"
 fi
 
 # Nightly release: the b* tag pointing at the release commit (|| true: no match is not an error)
@@ -80,6 +81,7 @@ if [[ -n "${GITHUB_OUTPUT:-}" ]]; then
         echo "previous_tag=${PREV}"
         echo "changelog_title=${CHANGELOG_TITLE}"
         echo "nightly=${NIGHTLY}"
+        echo "nightly_tag=${NIGHTLY_TAG}"
         echo "changelog<<CHANGELOG_EOF"
         echo "${CHANGELOG}"
         echo "CHANGELOG_EOF"

scripts/release.sh

@@ -0,0 +1,204 @@
+#!/bin/bash
+#
+# Release preparation script for llama.cpp.
+#
+# Bumps the version in CMakeLists.txt on a release candidate branch.
+# The branch should then be pushed and a PR created, reviewed, and
+# merged. After the PR is merged and the build-cpu workflow has
+# completed successfully, the release is finalized by the make-release
+# workflow (.github/workflows/make-release.yml), which creates the tag.
+#
+# Usage:
+#   ./scripts/release.sh [major|minor|patch] [--dry-run]
+#
+# Example:
+#   $ ./scripts/release.sh minor
+#
+# The script:
+# 1. Creates a release candidate branch (llama-rc-v<major>.<minor>.<patch>)
+# 2. Bumps the version in CMakeLists.txt
+# 3. Commits the version bump
+#
+
+set -e
+
+if [ ! -f "CMakeLists.txt" ] || [ ! -d "scripts" ]; then
+    echo "Error: Must be run from llama.cpp root directory"
+    exit 1
+fi
+
+# Parse command line arguments
+VERSION_TYPE=""
+DRY_RUN=false
+
+for arg in "$@"; do
+    case $arg in
+        --dry-run)
+            DRY_RUN=true
+            ;;
+        major|minor|patch)
+            VERSION_TYPE="$arg"
+            ;;
+        *)
+            echo "Error: Unknown argument '$arg'"
+            echo "Usage: $0 [major|minor|patch] [--dry-run]"
+            exit 1
+            ;;
+    esac
+done
+
+# Default to patch if no version type specified
+VERSION_TYPE="${VERSION_TYPE:-patch}"
+
+# Common validation functions
+check_git_status() {
+    # Check for uncommitted changes (skip in dry-run)
+    if [ "$DRY_RUN" = false ] && ! git diff-index --quiet HEAD --; then
+        echo "Error: You have uncommitted changes. Please commit or stash them first."
+        exit 1
+    fi
+}
+
+check_master_branch() {
+    # Ensure we're on master branch
+    CURRENT_BRANCH=$(git branch --show-current)
+    if [ "$CURRENT_BRANCH" != "master" ]; then
+        if [ "$DRY_RUN" = true ]; then
+            echo "[dry run] Warning: Not on master branch (currently on: $CURRENT_BRANCH). Continuing with dry-run..."
+            echo ""
+        else
+            echo "Error: Must be on master branch. Currently on: $CURRENT_BRANCH"
+            exit 1
+        fi
+    fi
+}
+
+check_master_up_to_date() {
+    # Check if we have the latest from master (skip in dry-run)
+    if [ "$DRY_RUN" = false ]; then
+        echo "Checking if local master is up-to-date with remote..."
+        git fetch origin master
+        LOCAL=$(git rev-parse HEAD)
+        REMOTE=$(git rev-parse origin/master)
+
+        if [ "$LOCAL" != "$REMOTE" ]; then
+            echo "Error: Your local master branch is not up-to-date with origin/master."
+            echo "Please run 'git pull origin master' first."
+            exit 1
+        fi
+        echo "✓ Local master is up-to-date with remote"
+        echo ""
+    elif [ "$(git branch --show-current)" = "master" ]; then
+        echo "[dry run] Warning: Dry-run mode - not checking if master is up-to-date with remote"
+        echo ""
+    fi
+}
+
+# In-place sed that works on both GNU (Linux) and BSD (macOS) sed
+sed_inplace() {
+    if sed --version >/dev/null 2>&1; then
+        sed -i "$@"
+    else
+        sed -i '' "$@"
+    fi
+}
+
+prepare_release() {
+    if [ "$DRY_RUN" = true ]; then
+        echo "[dry-run] Preparing release (no changes will be made)"
+    else
+        echo "Starting release preparation..."
+    fi
+    echo ""
+
+    check_git_status
+    check_master_branch
+    check_master_up_to_date
+
+    # Extract current version from CMakeLists.txt
+    echo "Step 1: Reading current version..."
+    MAJOR=$(grep "set(LLAMA_VERSION_MAJOR" CMakeLists.txt | sed 's/.*MAJOR \([0-9]*\).*/\1/')
+    MINOR=$(grep "set(LLAMA_VERSION_MINOR" CMakeLists.txt | sed 's/.*MINOR \([0-9]*\).*/\1/')
+    PATCH=$(grep "set(LLAMA_VERSION_PATCH" CMakeLists.txt | sed 's/.*PATCH \([0-9]*\).*/\1/')
+
+    echo "Current version: $MAJOR.$MINOR.$PATCH"
+
+    # Calculate new version
+    case $VERSION_TYPE in
+        major)
+            NEW_MAJOR=$((MAJOR + 1))
+            NEW_MINOR=0
+            NEW_PATCH=0
+            ;;
+        minor)
+            NEW_MAJOR=$MAJOR
+            NEW_MINOR=$((MINOR + 1))
+            NEW_PATCH=0
+            ;;
+        patch)
+            NEW_MAJOR=$MAJOR
+            NEW_MINOR=$MINOR
+            NEW_PATCH=$((PATCH + 1))
+            ;;
+    esac
+
+    NEW_VERSION="$NEW_MAJOR.$NEW_MINOR.$NEW_PATCH"
+    RC_BRANCH="llama-rc-v$NEW_VERSION"
+    echo "New release version: $NEW_VERSION"
+    echo "Release candidate branch: $RC_BRANCH"
+    echo ""
+
+    # Create release candidate branch
+    echo "Step 2: Creating release candidate branch..."
+    if [ "$DRY_RUN" = true ]; then
+        echo "  [dry-run] Would create branch: $RC_BRANCH"
+    else
+        git checkout -b "$RC_BRANCH"
+        echo "✓ Created and switched to branch: $RC_BRANCH"
+    fi
+    echo ""
+
+    # Update CMakeLists.txt for release
+    echo "Step 3: Updating version in CMakeLists.txt..."
+    if [ "$DRY_RUN" = true ]; then
+        echo "  [dry-run] Would update LLAMA_VERSION_MAJOR to $NEW_MAJOR"
+        echo "  [dry-run] Would update LLAMA_VERSION_MINOR to $NEW_MINOR"
+        echo "  [dry-run] Would update LLAMA_VERSION_PATCH to $NEW_PATCH"
+    else
+        sed_inplace -e "s/set(LLAMA_VERSION_MAJOR [0-9]*)/set(LLAMA_VERSION_MAJOR $NEW_MAJOR)/" CMakeLists.txt
+        sed_inplace -e "s/set(LLAMA_VERSION_MINOR [0-9]*)/set(LLAMA_VERSION_MINOR $NEW_MINOR)/" CMakeLists.txt
+        sed_inplace -e "s/set(LLAMA_VERSION_PATCH [0-9]*)/set(LLAMA_VERSION_PATCH $NEW_PATCH)/" CMakeLists.txt
+    fi
+    echo ""
+
+    # Commit version bump
+    echo "Step 4: Committing version bump..."
+    if [ "$DRY_RUN" = true ]; then
+        echo "  [dry-run] Would commit: 'llama.cpp : bump version to $NEW_VERSION'"
+    else
+        git add CMakeLists.txt
+        git commit -m "llama.cpp : bump version to $NEW_VERSION"
+    fi
+    echo ""
+
+    echo ""
+    if [ "$DRY_RUN" = true ]; then
+        echo "[dry-run] Summary (no changes were made):"
+        echo "  • Would have created branch: $RC_BRANCH"
+        echo "  • Would have updated version to: $NEW_VERSION"
+    else
+        echo "Release preparation completed!"
+        echo "Summary:"
+        echo "  • Created branch: $RC_BRANCH"
+        echo "  • Updated version to: $NEW_VERSION"
+        echo ""
+        echo "Next steps:"
+        echo "  • Push branch to remote: git push origin $RC_BRANCH"
+        echo "  • Create a Pull Request from $RC_BRANCH to master"
+        echo "  • After the PR is merged and the build-cpu workflow has passed,"
+        echo "    create the release with the make-release workflow"
+        echo "    (.github/workflows/make-release.yml)"
+    fi
+}
+
+prepare_release

scripts/sync-ggml.last

@@ -1 +1 @@
-8c63e70982c95ceb862e3a1073a2c1beef75d60a
+8599e0ea3756c4bac4ef813af2241cb1a8bbfb0b

src/CMakeLists.txt

@@ -25,6 +25,7 @@ add_library(llama
             llama-kv-cache.cpp
             llama-kv-cache-iswa.cpp
             llama-kv-cache-dsa.cpp
+            llama-kv-cache-dsa-iswa.cpp
             llama-kv-cache-msa.cpp
             llama-kv-cache-dsv4.cpp
             llama-memory.cpp

src/llama-arch.cpp

@@ -110,6 +110,7 @@ static const std::map<llm_arch, const char *> LLM_ARCH_NAMES = {
     { LLM_ARCH_BAILINGMOE2,      "bailingmoe2"      },
     { LLM_ARCH_BAILINGMOE3,      "bailingmoe3"      },
     { LLM_ARCH_DOTS1,            "dots1"            },
+    { LLM_ARCH_DOTS3NOTE,        "dots3note"        },
     { LLM_ARCH_ARCEE,            "arcee"            },
     { LLM_ARCH_AFMOE,            "afmoe"            },
     { LLM_ARCH_LAGUNA,           "laguna"           },
@@ -273,6 +274,9 @@ static const std::map<llm_kv, const char *> LLM_KV_NAMES = {
     { LLM_KV_ATTENTION_VALUE_LENGTH_MLA,             "%s.attention.value_length_mla"             },
     { LLM_KV_ATTENTION_KEY_LENGTH_SWA,               "%s.attention.key_length_swa"               },
     { LLM_KV_ATTENTION_VALUE_LENGTH_SWA,             "%s.attention.value_length_swa"             },
+    { LLM_KV_ATTENTION_KEY_LENGTH_MLA_SWA,           "%s.attention.key_length_mla_swa"           },
+    { LLM_KV_ATTENTION_VALUE_LENGTH_MLA_SWA,         "%s.attention.value_length_mla_swa"         },
+    { LLM_KV_ATTENTION_KV_LORA_RANK_SWA,             "%s.attention.kv_lora_rank_swa"             },
     { LLM_KV_ATTENTION_INDEXER_HEAD_COUNT,           "%s.attention.indexer.head_count"           },
     { LLM_KV_ATTENTION_INDEXER_KEY_LENGTH,           "%s.attention.indexer.key_length"           },
     { LLM_KV_ATTENTION_INDEXER_TOP_K,                "%s.attention.indexer.top_k"                },
@@ -1032,6 +1036,8 @@ bool llm_arch_supports_rs_rollback(const llm_arch & arch) {
         case LLM_ARCH_DEEPSEEK4:
         case LLM_ARCH_NEMOTRON_H:
         case LLM_ARCH_NEMOTRON_H_MOE:
+        case LLM_ARCH_LFM2:
+        case LLM_ARCH_LFM2MOE:
             return true;
         default:
             return false;
@@ -1054,14 +1060,13 @@ bool llm_arch_supports_sm_tensor(const llm_arch & arch) {
         case LLM_ARCH_DEEPSEEK2:
         case LLM_ARCH_DEEPSEEK32:
         case LLM_ARCH_DEEPSEEK4:
+        case LLM_ARCH_DOTS3NOTE:
         case LLM_ARCH_GLM_DSA:
         case LLM_ARCH_BITNET:
         case LLM_ARCH_T5:
         case LLM_ARCH_NEMOTRON_H:
         case LLM_ARCH_NEMOTRON_H_MOE:
         case LLM_ARCH_GRANITE_HYBRID:
-        case LLM_ARCH_LFM2:
-        case LLM_ARCH_LFM2MOE:
         case LLM_ARCH_MINIMAX_01:
         case LLM_ARCH_MINIMAX_M2:
         case LLM_ARCH_MINIMAX_M3:

src/llama-arch.h

@@ -115,6 +115,7 @@ enum llm_arch {
     LLM_ARCH_BAILINGMOE2,
     LLM_ARCH_BAILINGMOE3,
     LLM_ARCH_DOTS1,
+    LLM_ARCH_DOTS3NOTE,
     LLM_ARCH_ARCEE,
     LLM_ARCH_AFMOE,
     LLM_ARCH_LAGUNA,
@@ -278,6 +279,9 @@ enum llm_kv {
     LLM_KV_ATTENTION_VALUE_LENGTH_MLA,
     LLM_KV_ATTENTION_KEY_LENGTH_SWA,
     LLM_KV_ATTENTION_VALUE_LENGTH_SWA,
+    LLM_KV_ATTENTION_KEY_LENGTH_MLA_SWA,
+    LLM_KV_ATTENTION_VALUE_LENGTH_MLA_SWA,
+    LLM_KV_ATTENTION_KV_LORA_RANK_SWA,
     LLM_KV_ATTENTION_INDEXER_HEAD_COUNT,
     LLM_KV_ATTENTION_INDEXER_KEY_LENGTH,
     LLM_KV_ATTENTION_INDEXER_TOP_K,

src/llama-graph.cpp

@@ -9,6 +9,7 @@
 #include "llama-kv-cache.h"
 #include "llama-kv-cache-iswa.h"
 #include "llama-kv-cache-dsa.h"
+#include "llama-kv-cache-dsa-iswa.h"
 #include "llama-kv-cache-msa.h"
 #include "llama-kv-cache-dsv4.h"
 #include "llama-memory-hybrid.h"
@@ -507,10 +508,12 @@ void llm_graph_input_attn_k::set_input(const llama_ubatch * ubatch) {
 }
 
 bool llm_graph_input_attn_k::can_reuse(const llm_graph_params & params) {
-    const auto * mctx = static_cast<const llama_kv_cache_context *>(params.mctx);
+    mctx = static_cast<const llama_kv_cache_context *>(params.mctx);
 
-    this->mctx = mctx;
+    return can_reuse_impl(params);
+}
 
+bool llm_graph_input_attn_k::can_reuse_impl(const llm_graph_params & params) {
     bool res = true;
 
     res &= self_k_idxs->ne[0] == params.ubatch.n_tokens;
@@ -567,10 +570,12 @@ void llm_graph_input_attn_k_dsa::set_input(const llama_ubatch * ubatch) {
 }
 
 bool llm_graph_input_attn_k_dsa::can_reuse(const llm_graph_params & params) {
-    const auto * mctx = static_cast<const llama_kv_cache_dsa_context *>(params.mctx);
+    mctx = static_cast<const llama_kv_cache_dsa_context *>(params.mctx);
 
-    this->mctx = mctx;
+    return can_reuse_impl(params);
+}
 
+bool llm_graph_input_attn_k_dsa::can_reuse_impl(const llm_graph_params & params) {
     bool res = true;
 
     res &= self_k_idxs_mla->ne[0] == params.ubatch.n_tokens;
@@ -582,6 +587,25 @@ bool llm_graph_input_attn_k_dsa::can_reuse(const llm_graph_params & params) {
     return res;
 }
 
+void llm_graph_input_attn_k_dsa_iswa::set_input(const llama_ubatch * ubatch) {
+    inp_dsa->set_input(ubatch);
+    inp_swa->set_input(ubatch);
+}
+
+bool llm_graph_input_attn_k_dsa_iswa::can_reuse(const llm_graph_params & params) {
+    mctx = static_cast<const llama_kv_cache_dsa_iswa_context *>(params.mctx);
+
+    inp_dsa->mctx = mctx->get_dsa();
+    inp_swa->mctx = mctx->get_swa();
+
+    bool res = true;
+
+    res &= inp_dsa->can_reuse_impl(params);
+    res &= inp_swa->can_reuse_impl(params);
+
+    return res;
+}
+
 void llm_graph_input_attn_kv_iswa::set_input(const llama_ubatch * ubatch) {
     // base tensors may not be allocated if there are no non-SWA attention layers
     if (self_k_idxs && self_k_idxs->buffer) {
@@ -3210,8 +3234,12 @@ ggml_tensor * llm_graph_context::build_attn(
     return cur;
 }
 
-llm_graph_input_attn_k_dsa * llm_graph_context::build_attn_inp_k_dsa() const {
-    const auto * mctx_cur = static_cast<const llama_kv_cache_dsa_context *>(mctx);
+static std::unique_ptr<llm_graph_input_attn_k_dsa> build_attn_inp_k_dsa_impl(
+           ggml_context * ctx0,
+     const llama_ubatch & ubatch,
+    const llama_hparams & hparams,
+    const llama_cparams & cparams,
+    const llama_kv_cache_dsa_context * mctx_cur) {
 
     auto inp = std::make_unique<llm_graph_input_attn_k_dsa>(hparams, cparams, mctx_cur);
 
@@ -3235,9 +3263,35 @@ llm_graph_input_attn_k_dsa * llm_graph_context::build_attn_inp_k_dsa() const {
         inp->self_k_rot_lid = mctx_cur->get_lid()->build_input_k_rot(ctx0);
     }
 
+    return inp;
+}
+
+llm_graph_input_attn_k_dsa * llm_graph_context::build_attn_inp_k_dsa() const {
+    const auto * mctx_cur = static_cast<const llama_kv_cache_dsa_context *>(mctx);
+
+    auto inp = build_attn_inp_k_dsa_impl(ctx0, ubatch, hparams, cparams, mctx_cur);
+
     return (llm_graph_input_attn_k_dsa *) res->add_input(std::move(inp));
 }
 
+llm_graph_input_attn_k_dsa_iswa * llm_graph_context::build_attn_inp_k_dsa_iswa() const {
+    const auto * mctx_cur = static_cast<const llama_kv_cache_dsa_iswa_context *>(mctx);
+
+    auto inp_dsa = build_attn_inp_k_dsa_impl(ctx0, ubatch, hparams, cparams, mctx_cur->get_dsa());
+
+    // build_attn_inp_k_impl rejects SWA caches, so construct the input directly
+    auto inp_swa = std::make_unique<llm_graph_input_attn_k>(hparams, cparams, mctx_cur->get_swa());
+
+    inp_swa->self_k_idxs = mctx_cur->get_swa()->build_input_k_idxs(ctx0, ubatch);
+
+    inp_swa->self_kq_mask = build_attn_inp_kq_mask(ctx0, mctx_cur->get_swa(), ubatch, cparams);
+    inp_swa->self_kq_mask_cnv = inp_swa->self_kq_mask;
+
+    auto inp = std::make_unique<llm_graph_input_attn_k_dsa_iswa>(std::move(inp_dsa), std::move(inp_swa), mctx_cur);
+
+    return (llm_graph_input_attn_k_dsa_iswa *) res->add_input(std::move(inp));
+}
+
 llm_graph_input_attn_kv_msa * llm_graph_context::build_attn_inp_kv_msa(bool msa_enabled) const {
     const auto * mctx_cur = static_cast<const llama_kv_cache_msa_context *>(mctx);
 

src/llama-graph.h

@@ -23,6 +23,7 @@ struct llama_memory_context_i;
 
 class llama_kv_cache_context;
 class llama_kv_cache_dsa_context;
+class llama_kv_cache_dsa_iswa_context;
 class llama_kv_cache_msa_context;
 class llama_kv_cache_dsv4_raw_context;
 class llama_kv_cache_dsv4_context;
@@ -374,6 +375,9 @@ public:
 
     bool can_reuse(const llm_graph_params & params) override;
 
+    // like can_reuse, but does not re-bind mctx
+    bool can_reuse_impl(const llm_graph_params & params);
+
     ggml_tensor * get_k_idxs() const { return self_k_idxs; }
 
     ggml_tensor * get_kq_mask() const { return self_kq_mask_cnv; }
@@ -405,6 +409,9 @@ public:
 
     bool can_reuse(const llm_graph_params & params) override;
 
+    // like can_reuse, but does not re-bind mctx
+    bool can_reuse_impl(const llm_graph_params & params);
+
     ggml_tensor * get_k_idxs_mla() const { return self_k_idxs_mla; }
     ggml_tensor * get_k_idxs_lid() const { return self_k_idxs_lid; }
 
@@ -427,6 +434,32 @@ public:
     const llama_kv_cache_dsa_context * mctx;
 };
 
+// DSA input (full-attention layers + indexer) with K-only input for the SWA layers
+class llm_graph_input_attn_k_dsa_iswa : public llm_graph_input_i {
+public:
+    llm_graph_input_attn_k_dsa_iswa(
+            std::unique_ptr<llm_graph_input_attn_k_dsa> inp_dsa,
+            std::unique_ptr<llm_graph_input_attn_k>     inp_swa,
+            const llama_kv_cache_dsa_iswa_context *     mctx) :
+        inp_dsa(std::move(inp_dsa)),
+        inp_swa(std::move(inp_swa)),
+        mctx(mctx) {
+    }
+    ~llm_graph_input_attn_k_dsa_iswa() = default;
+
+    void set_input(const llama_ubatch * ubatch) override;
+
+    bool can_reuse(const llm_graph_params & params) override;
+
+    llm_graph_input_attn_k_dsa * get_dsa() const { return inp_dsa.get(); }
+    llm_graph_input_attn_k     * get_swa() const { return inp_swa.get(); }
+
+    std::unique_ptr<llm_graph_input_attn_k_dsa> inp_dsa;
+    std::unique_ptr<llm_graph_input_attn_k>     inp_swa;
+
+    const llama_kv_cache_dsa_iswa_context * mctx;
+};
+
 // standard K/V attention input against the base cache, plus destination indices for the indexer key cache
 class llm_graph_input_attn_kv_msa : public llm_graph_input_attn_kv {
 public:
@@ -1191,6 +1224,8 @@ struct llm_graph_context {
 
     llm_graph_input_attn_k_dsa * build_attn_inp_k_dsa() const;
 
+    llm_graph_input_attn_k_dsa_iswa * build_attn_inp_k_dsa_iswa() const;
+
     llm_graph_input_attn_kv_msa * build_attn_inp_kv_msa(bool msa_enabled) const;
 
     ggml_tensor * build_attn(

src/llama-hparams.h

@@ -101,6 +101,11 @@ struct llama_hparams {
     uint32_t n_group_used       = 0;
     uint32_t n_group_experts    = 0;
 
+    // MLA + SWA (i.e. dots3note)
+    uint32_t n_lora_kv_swa           = 0;
+    uint32_t n_embd_head_k_mla_swa   = 0;
+    uint32_t n_embd_head_v_mla_swa   = 0;
+
     float    expert_group_scale   = 0.05f;
     float    expert_weights_scale = 0.0f;
     bool     expert_weights_norm  = false;

src/llama-kv-cache-dsa-iswa.cpp

@@ -0,0 +1,341 @@
+#include "llama-kv-cache-dsa-iswa.h"
+
+#include "llama-impl.h"
+#include "llama-batch.h"
+#include "llama-model.h"
+
+#include <algorithm>
+#include <cassert>
+
+//
+// llama_kv_cache_dsa_iswa
+//
+
+llama_kv_cache_dsa_iswa::llama_kv_cache_dsa_iswa(
+        const llama_model & model,
+                ggml_type   type_k,
+                ggml_type   type_v,
+                     bool   v_trans,
+                     bool   offload,
+                     bool   swa_full,
+                     bool   unified,
+                 uint32_t   kv_size,
+                 uint32_t   n_seq_max,
+                 uint32_t   n_ubatch,
+                 uint32_t   n_pad,
+    const layer_filter_cb & filter_mla,
+    const layer_filter_cb & filter_lid,
+    const  layer_reuse_cb & reuse) : unified(unified) {
+
+    const auto & hparams = model.hparams;
+
+    // chain filters
+    const layer_filter_cb filter_dsa = [&](int32_t il) {
+        if (filter_mla && !filter_mla(il)) {
+            return false;
+        }
+
+        return !hparams.is_swa(il);
+    };
+
+    const layer_filter_cb filter_swa = [&](int32_t il) {
+        if (filter_mla && !filter_mla(il)) {
+            return false;
+        }
+
+        return hparams.is_swa(il);
+    };
+
+    const uint32_t size_dsa = kv_size;
+
+    // note: the SWA cache is always padded to 256 for performance
+    //       https://github.com/ggml-org/llama.cpp/issues/17037
+    uint32_t size_swa = GGML_PAD(std::min(size_dsa, hparams.n_swa*(unified ? n_seq_max : 1) + n_ubatch), 256);
+
+    // when using full-size SWA cache, we set the SWA cache size to be equal to the base cache size
+    if (swa_full) {
+        LLAMA_LOG_WARN("%s: using full-size SWA cache (ref: %s)\n",
+                __func__, "https://github.com/ggml-org/llama.cpp/pull/13194#issuecomment-2868343055");
+
+        size_swa = size_dsa;
+    }
+
+    LLAMA_LOG_INFO("%s: creating DSA KV cache, size = %u cells\n", __func__, size_dsa);
+
+    kv_dsa = std::make_unique<llama_kv_cache_dsa>(
+            model, type_k, type_v,
+            v_trans, offload, unified, size_dsa, n_seq_max, n_pad,
+            0, LLAMA_SWA_TYPE_NONE, filter_dsa, filter_lid, reuse);
+
+    LLAMA_LOG_INFO("%s: creating SWA KV cache, size = %u cells\n", __func__, size_swa);
+
+    kv_swa = std::make_unique<llama_kv_cache>(
+            model, hparams, type_k, type_v,
+            v_trans, offload, unified, size_swa, n_seq_max, n_pad,
+            hparams.n_swa, hparams.swa_type, nullptr, filter_swa, reuse, nullptr);
+}
+
+void llama_kv_cache_dsa_iswa::clear(bool data) {
+    kv_dsa->clear(data);
+    kv_swa->clear(data);
+}
+
+bool llama_kv_cache_dsa_iswa::seq_rm(llama_seq_id seq_id, llama_pos p0, llama_pos p1) {
+    bool res = true;
+
+    res = res & kv_dsa->seq_rm(seq_id, p0, p1);
+    res = res & kv_swa->seq_rm(seq_id, p0, p1);
+
+    return res;
+}
+
+void llama_kv_cache_dsa_iswa::seq_cp(llama_seq_id seq_id_src, llama_seq_id seq_id_dst, llama_pos p0, llama_pos p1) {
+    kv_dsa->seq_cp(seq_id_src, seq_id_dst, p0, p1);
+    kv_swa->seq_cp(seq_id_src, seq_id_dst, p0, p1);
+}
+
+void llama_kv_cache_dsa_iswa::seq_keep(llama_seq_id seq_id) {
+    kv_dsa->seq_keep(seq_id);
+    kv_swa->seq_keep(seq_id);
+}
+
+void llama_kv_cache_dsa_iswa::seq_add(llama_seq_id seq_id, llama_pos p0, llama_pos p1, llama_pos shift) {
+    kv_dsa->seq_add(seq_id, p0, p1, shift);
+    kv_swa->seq_add(seq_id, p0, p1, shift);
+}
+
+void llama_kv_cache_dsa_iswa::seq_div(llama_seq_id seq_id, llama_pos p0, llama_pos p1, int d) {
+    kv_dsa->seq_div(seq_id, p0, p1, d);
+    kv_swa->seq_div(seq_id, p0, p1, d);
+}
+
+llama_pos llama_kv_cache_dsa_iswa::seq_pos_min(llama_seq_id seq_id) const {
+    // the DSA cache is a superset of the SWA cache, so we can just check the SWA cache
+    return kv_swa->seq_pos_min(seq_id);
+}
+
+llama_pos llama_kv_cache_dsa_iswa::seq_pos_max(llama_seq_id seq_id) const {
+    return kv_swa->seq_pos_max(seq_id);
+}
+
+std::map<ggml_backend_buffer_type_t, size_t> llama_kv_cache_dsa_iswa::memory_breakdown() const {
+    std::map<ggml_backend_buffer_type_t, size_t> mb = kv_dsa->memory_breakdown();
+    for (const auto & buft_size : kv_swa->memory_breakdown()) {
+        mb[buft_size.first] += buft_size.second;
+    }
+    return mb;
+}
+
+llama_memory_context_ptr llama_kv_cache_dsa_iswa::init_batch(llama_batch_allocr & balloc, uint32_t n_ubatch, bool embd_all) {
+    GGML_UNUSED(embd_all);
+
+    // first try simple split
+    do {
+        if (!unified) {
+            // requires equal splits, so we skip the simple split
+            break;
+        }
+
+        balloc.split_reset();
+
+        std::vector<llama_ubatch> ubatches;
+        while (true) {
+            auto ubatch = balloc.split_simple(n_ubatch);
+
+            if (ubatch.n_tokens == 0) {
+                break;
+            }
+
+            ubatches.push_back(std::move(ubatch)); // NOLINT
+        }
+
+        if (balloc.get_n_used() < balloc.get_n_tokens()) {
+            // failed to find a suitable split
+            break;
+        }
+
+        auto sinfos_mla = kv_dsa->get_mla()->prepare(ubatches);
+        if (sinfos_mla.empty()) {
+            break;
+        }
+
+        auto sinfos_lid = kv_dsa->get_lid()->prepare(ubatches);
+        if (sinfos_lid.empty()) {
+            break;
+        }
+
+        auto sinfos_swa = kv_swa->prepare(ubatches);
+        if (sinfos_swa.empty()) {
+            break;
+        }
+
+        assert(sinfos_mla.size() == sinfos_swa.size());
+
+        return std::make_unique<llama_kv_cache_dsa_iswa_context>(
+                this, std::move(sinfos_mla), std::move(sinfos_lid), std::move(sinfos_swa), std::move(ubatches));
+    } while (false);
+
+    // if it fails, try equal split
+    do {
+        balloc.split_reset();
+
+        std::vector<llama_ubatch> ubatches;
+        while (true) {
+            auto ubatch = balloc.split_equal(n_ubatch, !unified, 0);
+
+            if (ubatch.n_tokens == 0) {
+                break;
+            }
+
+            ubatches.push_back(std::move(ubatch)); // NOLINT
+        }
+
+        if (balloc.get_n_used() < balloc.get_n_tokens()) {
+            // failed to find a suitable split
+            break;
+        }
+
+        auto sinfos_mla = kv_dsa->get_mla()->prepare(ubatches);
+        if (sinfos_mla.empty()) {
+            break;
+        }
+
+        auto sinfos_lid = kv_dsa->get_lid()->prepare(ubatches);
+        if (sinfos_lid.empty()) {
+            break;
+        }
+
+        auto sinfos_swa = kv_swa->prepare(ubatches);
+        if (sinfos_swa.empty()) {
+            break;
+        }
+
+        assert(sinfos_mla.size() == sinfos_swa.size());
+
+        return std::make_unique<llama_kv_cache_dsa_iswa_context>(
+                this, std::move(sinfos_mla), std::move(sinfos_lid), std::move(sinfos_swa), std::move(ubatches));
+    } while (false);
+
+    return std::make_unique<llama_kv_cache_dsa_iswa_context>(LLAMA_MEMORY_STATUS_FAILED_PREPARE);
+}
+
+llama_memory_context_ptr llama_kv_cache_dsa_iswa::init_full() {
+    return std::make_unique<llama_kv_cache_dsa_iswa_context>(this);
+}
+
+llama_memory_context_ptr llama_kv_cache_dsa_iswa::init_update(llama_context * lctx, bool optimize) {
+    return std::make_unique<llama_kv_cache_dsa_iswa_context>(this, lctx, optimize);
+}
+
+bool llama_kv_cache_dsa_iswa::get_can_shift() const {
+    return kv_dsa->get_can_shift() &&
+           kv_swa->get_can_shift() &&
+           kv_dsa->get_mla()->get_size() == kv_swa->get_size();
+}
+
+void llama_kv_cache_dsa_iswa::state_write(llama_io_write_i & io, llama_seq_id seq_id, llama_state_seq_flags flags) const {
+    if ((flags & LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY) == 0) {
+        kv_dsa->state_write(io, seq_id, flags);
+    }
+
+    kv_swa->state_write(io, seq_id, flags);
+}
+
+void llama_kv_cache_dsa_iswa::state_read(llama_io_read_i & io, llama_seq_id seq_id, llama_state_seq_flags flags) {
+    if ((flags & LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY) == 0) {
+        kv_dsa->state_read(io, seq_id, flags);
+    }
+
+    kv_swa->state_read(io, seq_id, flags);
+}
+
+llama_kv_cache_dsa * llama_kv_cache_dsa_iswa::get_dsa() const {
+    return kv_dsa.get();
+}
+
+llama_kv_cache * llama_kv_cache_dsa_iswa::get_swa() const {
+    return kv_swa.get();
+}
+
+//
+// llama_kv_cache_dsa_iswa_context
+//
+
+llama_kv_cache_dsa_iswa_context::llama_kv_cache_dsa_iswa_context(llama_memory_status status) : status(status) {}
+
+llama_kv_cache_dsa_iswa_context::llama_kv_cache_dsa_iswa_context(
+        llama_kv_cache_dsa_iswa * kv) :
+    ctx_dsa(kv->get_dsa()->init_full()),
+    ctx_swa(kv->get_swa()->init_full()),
+    status(llama_memory_status_combine(ctx_dsa->get_status(), ctx_swa->get_status())) {
+}
+
+llama_kv_cache_dsa_iswa_context::llama_kv_cache_dsa_iswa_context(
+        llama_kv_cache_dsa_iswa * kv,
+        llama_context * lctx,
+        bool optimize) :
+    ctx_dsa(kv->get_dsa()->init_update(lctx, optimize)),
+    ctx_swa(kv->get_swa()->init_update(lctx, optimize)),
+    status(llama_memory_status_combine(ctx_dsa->get_status(), ctx_swa->get_status())) {
+}
+
+llama_kv_cache_dsa_iswa_context::llama_kv_cache_dsa_iswa_context(
+        llama_kv_cache_dsa_iswa * kv,
+        slot_info_vec_t sinfos_mla,
+        slot_info_vec_t sinfos_lid,
+        slot_info_vec_t sinfos_swa,
+        std::vector<llama_ubatch> ubatches) :
+    ubatches(std::move(ubatches)),
+    // note: here we copy the ubatches. not sure if this is ideal
+    ctx_dsa(new llama_kv_cache_dsa_context(kv->get_dsa(), std::move(sinfos_mla), std::move(sinfos_lid), this->ubatches)),
+    ctx_swa(new llama_kv_cache_context(kv->get_swa(), std::move(sinfos_swa), this->ubatches)),
+    status(llama_memory_status_combine(ctx_dsa->get_status(), ctx_swa->get_status())) {
+}
+
+llama_kv_cache_dsa_iswa_context:: ~llama_kv_cache_dsa_iswa_context() = default;
+
+bool llama_kv_cache_dsa_iswa_context::next() {
+    assert(status == LLAMA_MEMORY_STATUS_SUCCESS);
+
+    ctx_dsa->next();
+    ctx_swa->next();
+
+    if (++i_next >= ubatches.size()) {
+        return false;
+    }
+
+    return true;
+}
+
+bool llama_kv_cache_dsa_iswa_context::apply() {
+    assert(!llama_memory_status_is_fail(status));
+
+    bool res = true;
+
+    res = res & ctx_dsa->apply();
+    res = res & ctx_swa->apply();
+
+    return res;
+}
+
+llama_memory_status llama_kv_cache_dsa_iswa_context::get_status() const {
+    return status;
+}
+
+const llama_ubatch & llama_kv_cache_dsa_iswa_context::get_ubatch() const {
+    assert(status == LLAMA_MEMORY_STATUS_SUCCESS);
+
+    return ubatches[i_next];
+}
+
+const llama_kv_cache_dsa_context * llama_kv_cache_dsa_iswa_context::get_dsa() const {
+    assert(status == LLAMA_MEMORY_STATUS_SUCCESS);
+
+    return static_cast<const llama_kv_cache_dsa_context *>(ctx_dsa.get());
+}
+
+const llama_kv_cache_context * llama_kv_cache_dsa_iswa_context::get_swa() const {
+    assert(status == LLAMA_MEMORY_STATUS_SUCCESS);
+
+    return static_cast<const llama_kv_cache_context *>(ctx_swa.get());
+}

src/llama-kv-cache-dsa-iswa.h

@@ -0,0 +1,134 @@
+#pragma once
+
+#include "llama-kv-cache-dsa.h"
+
+#include <vector>
+
+//
+// llama_kv_cache_dsa_iswa
+//
+
+// utilizes two child memories: llama_kv_cache_dsa for the full-attention (DSA) layers and llama_kv_cache for the SWA layers
+
+class llama_kv_cache_dsa_iswa : public llama_memory_i {
+public:
+    llama_kv_cache_dsa_iswa(
+            const llama_model & model,
+                    ggml_type   type_k,
+                    ggml_type   type_v,
+                         bool   v_trans,
+                         bool   offload,
+                         bool   swa_full,
+                         bool   unified,
+                     uint32_t   kv_size,
+                     uint32_t   n_seq_max,
+                     uint32_t   n_ubatch,
+                     uint32_t   n_pad,
+        const layer_filter_cb & filter_mla,
+        const layer_filter_cb & filter_lid,
+        const  layer_reuse_cb & reuse);
+
+    ~llama_kv_cache_dsa_iswa() = default;
+
+    //
+    // llama_memory_i
+    //
+
+    llama_memory_context_ptr init_batch(
+            llama_batch_allocr & balloc,
+            uint32_t n_ubatch,
+            bool embd_all) override;
+
+    llama_memory_context_ptr init_full() override;
+
+    llama_memory_context_ptr init_update(llama_context * lctx, bool optimize) override;
+
+    bool get_can_shift() const override;
+
+    void clear(bool data) override;
+
+    bool seq_rm  (llama_seq_id seq_id,                              llama_pos p0, llama_pos p1) override;
+    void seq_cp  (llama_seq_id seq_id_src, llama_seq_id seq_id_dst, llama_pos p0, llama_pos p1) override;
+    void seq_keep(llama_seq_id seq_id)                                                          override;
+    void seq_add (llama_seq_id seq_id,                              llama_pos p0, llama_pos p1, llama_pos shift) override;
+    void seq_div (llama_seq_id seq_id,                              llama_pos p0, llama_pos p1, int d) override;
+
+    llama_pos seq_pos_min(llama_seq_id seq_id) const override;
+    llama_pos seq_pos_max(llama_seq_id seq_id) const override;
+
+    std::map<ggml_backend_buffer_type_t, size_t> memory_breakdown() const override;
+
+    // state write/load
+
+    void state_write(llama_io_write_i & io, llama_seq_id seq_id = -1, llama_state_seq_flags flags = 0) const override;
+    void state_read (llama_io_read_i  & io, llama_seq_id seq_id = -1, llama_state_seq_flags flags = 0) override;
+
+    //
+    // llama_kv_cache_dsa_iswa specific API
+    //
+
+    llama_kv_cache_dsa * get_dsa() const;
+    llama_kv_cache     * get_swa() const;
+
+private:
+    const bool unified;
+
+    std::unique_ptr<llama_kv_cache_dsa> kv_dsa;
+    std::unique_ptr<llama_kv_cache>     kv_swa;
+};
+
+class llama_kv_cache_dsa_iswa_context : public llama_memory_context_i {
+public:
+    using slot_info_vec_t = llama_kv_cache::slot_info_vec_t;
+
+    // used for errors
+    llama_kv_cache_dsa_iswa_context(llama_memory_status status);
+
+    // used to create a full-cache context
+    llama_kv_cache_dsa_iswa_context(
+            llama_kv_cache_dsa_iswa * kv);
+
+    // used to create an update context
+    llama_kv_cache_dsa_iswa_context(
+            llama_kv_cache_dsa_iswa * kv,
+            llama_context * lctx,
+            bool optimize);
+
+    // used to create a batch processing context from a batch
+    llama_kv_cache_dsa_iswa_context(
+            llama_kv_cache_dsa_iswa * kv,
+            slot_info_vec_t sinfos_mla,
+            slot_info_vec_t sinfos_lid,
+            slot_info_vec_t sinfos_swa,
+            std::vector<llama_ubatch> ubatches);
+
+    virtual ~llama_kv_cache_dsa_iswa_context();
+
+    //
+    // llama_memory_context_i
+    //
+
+    bool next()  override;
+    bool apply() override;
+
+    llama_memory_status  get_status() const override;
+    const llama_ubatch & get_ubatch() const override;
+
+    //
+    // llama_kv_cache_dsa_iswa_context specific API
+    //
+
+    const llama_kv_cache_dsa_context * get_dsa() const;
+    const llama_kv_cache_context     * get_swa() const;
+
+private:
+    // the index of the next ubatch to process
+    size_t i_next = 0;
+
+    std::vector<llama_ubatch> ubatches;
+
+    const llama_memory_context_ptr ctx_dsa;
+    const llama_memory_context_ptr ctx_swa;
+
+    const llama_memory_status status;
+};

src/llama-kv-cache.cpp

@@ -323,7 +323,8 @@ llama_kv_cache::llama_kv_cache(
             hparams.n_embd_head_k() % 64 == 0;
 
         // always create Hadamard rotation tensors for DeepSeek lightning indexers
-        if ((model.arch == LLM_ARCH_DEEPSEEK32 || model.arch == LLM_ARCH_DEEPSEEK4 || model.arch == LLM_ARCH_GLM_DSA) &&
+        if ((model.arch == LLM_ARCH_DEEPSEEK32 || model.arch == LLM_ARCH_DEEPSEEK4 ||
+                model.arch == LLM_ARCH_GLM_DSA || model.arch == LLM_ARCH_DOTS3NOTE) &&
                 hparams.n_embd_head_k_full == hparams.indexer_head_size) {
             attn_rot_k = true;
         }

src/llama-model-saver.cpp

@@ -31,6 +31,7 @@ bool llama_model_saver_supports_arch(llm_arch arch) {
         case LLM_ARCH_MELLUM:
         case LLM_ARCH_LAGUNA:
         case LLM_ARCH_GRANITE_SWA:
+        case LLM_ARCH_DOTS3NOTE: // TODO: need to handle SWA pattern and MLA+SWA config
             return false;
         default:
             return true;

src/llama-model.cpp

@@ -11,6 +11,7 @@
 #include "llama-kv-cache.h"
 #include "llama-kv-cache-iswa.h"
 #include "llama-kv-cache-dsa.h"
+#include "llama-kv-cache-dsa-iswa.h"
 #include "llama-kv-cache-msa.h"
 #include "llama-kv-cache-dsv4.h"
 #include "llama-memory-hybrid.h"
@@ -194,6 +195,8 @@ static llama_model * llama_model_mapping(llm_arch arch, const llama_model_params
             return new llama_model_deepseek2ocr(params);
         case LLM_ARCH_DEEPSEEK32:
             return new llama_model_deepseek32(params);
+        case LLM_ARCH_DOTS3NOTE:
+            return new llama_model_dots3note(params);
         case LLM_ARCH_DEEPSEEK4:
             return new llama_model_deepseek4(params);
         case LLM_ARCH_GLM_DSA:
@@ -487,6 +490,10 @@ struct ggml_backend_meta_split_state llama_meta_device_get_split_state(const str
             return get_tensor_config_impl(GGML_BACKEND_SPLIT_AXIS_1, "ssm_out.weight");
         }
         if (std::regex_match(tensor_name, pattern_r_cache) || std::regex_match(tensor_name, pattern_s_cache)) {
+            if (ud->model->arch == LLM_ARCH_LFM2 || ud->model->arch == LLM_ARCH_LFM2MOE) {
+                // the LFM2 shortconv block runs fully mirrored, so its conv state must be mirrored too
+                return get_tensor_config_impl(GGML_BACKEND_SPLIT_AXIS_MIRRORED, "");
+            }
             return get_tensor_config_impl(GGML_BACKEND_SPLIT_AXIS_0, "ssm_out.weight");
         }
         if (std::regex_match(tensor_name, pattern_ssm_conv1d)) {
@@ -847,6 +854,7 @@ const char * llm_type_name(llm_type type) {
         case LLM_TYPE_230B_A10B:     return "230B.A10B";
         case LLM_TYPE_428B_A23B:     return "428B.A23B";
         case LLM_TYPE_235B_A22B:     return "235B.A22B";
+        case LLM_TYPE_288B_A19B:     return "288B.A19B";
         case LLM_TYPE_300B_A47B:     return "300B.A47B";
         case LLM_TYPE_310B_A15B:     return "310B.A15B";
         case LLM_TYPE_355B_A32B:     return "355B.A32B";
@@ -1920,7 +1928,9 @@ void llama_model::print_info() const {
             LLAMA_LOG_INFO("%s: expert_weights_scale  = %.1f\n",   __func__, hparams.expert_weights_scale);
         }
 
-        if (arch == LLM_ARCH_DEEPSEEK2 || arch == LLM_ARCH_DEEPSEEK2OCR || arch == LLM_ARCH_DEEPSEEK32 || arch == LLM_ARCH_GLM_DSA || arch == LLM_ARCH_MISTRAL4) {
+        if (arch == LLM_ARCH_DEEPSEEK2 || arch == LLM_ARCH_DEEPSEEK2OCR ||
+                arch == LLM_ARCH_DEEPSEEK32 || arch == LLM_ARCH_GLM_DSA ||
+                arch == LLM_ARCH_DOTS3NOTE || arch == LLM_ARCH_MISTRAL4) {
             LLAMA_LOG_INFO("%s: n_layer_dense_lead    = %d\n",     __func__, hparams.n_layer_dense_lead);
             LLAMA_LOG_INFO("%s: n_lora_q              = %d\n",     __func__, hparams.n_lora_q);
             LLAMA_LOG_INFO("%s: n_lora_kv             = %d\n",     __func__, hparams.n_lora_kv);
@@ -2189,6 +2199,57 @@ llama_memory_i * llama_model::create_memory(const llama_memory_params & params,
                             nullptr);
                 }
             } break;
+        case LLM_ARCH_DOTS3NOTE:
+            {
+                GGML_ASSERT(hparams.swa_type != LLAMA_SWA_TYPE_NONE);
+
+                if (params.ctx_type == LLAMA_CONTEXT_TYPE_MTP && hparams.n_layer_nextn > 0) {
+                    // MTP draft context: plain attention KV cache holding only the nextn layer
+                    llama_kv_cache::layer_filter_cb filter =
+                        [&](uint32_t il) { return il >= hparams.n_layer(); };
+
+                    res = new llama_kv_cache(
+                            *this,
+                            hparams,
+                            params.type_k,
+                            params.type_v,
+                            !cparams.flash_attn,
+                            cparams.offload_kqv,
+                            cparams.kv_unified,
+                            cparams.n_ctx_seq,
+                            cparams.n_seq_max,
+                            1,
+                            hparams.n_swa,
+                            hparams.swa_type,
+                            nullptr,
+                            filter,
+                            nullptr,
+                            nullptr);
+                } else {
+                    // main context: DSA cache for the trunk full-attention layers plus a window-sized SWA cache
+                    llama_kv_cache::layer_filter_cb filter_mla = nullptr;
+                    if (hparams.n_layer_nextn > 0) {
+                        filter_mla = [&](uint32_t il) { return il < hparams.n_layer(); };
+                    }
+                    llama_kv_cache::layer_filter_cb filter_lid = [&](uint32_t il) { return il < hparams.n_layer() && hparams.is_indexer_full(il); };
+
+                    res = new llama_kv_cache_dsa_iswa(
+                            *this,
+                            params.type_k,
+                            params.type_v,
+                            !cparams.flash_attn,
+                            cparams.offload_kqv,
+                            params.swa_full,
+                            cparams.kv_unified,
+                            cparams.n_ctx_seq,
+                            cparams.n_seq_max,
+                            cparams.n_ubatch,
+                            1,
+                            filter_mla,
+                            filter_lid,
+                            nullptr);
+                }
+            } break;
         case LLM_ARCH_DEEPSEEK4:
             {
                 GGML_ASSERT(hparams.swa_type != LLAMA_SWA_TYPE_NONE);
@@ -2657,6 +2718,7 @@ llama_rope_type llama_model_rope_type(const llama_model * model) {
         case LLM_ARCH_LLAMA_EMBED:
         case LLM_ARCH_MAINCODER:
         case LLM_ARCH_GLM_DSA:
+        case LLM_ARCH_DOTS3NOTE:
         case LLM_ARCH_NANBEIGE:
         case LLM_ARCH_POCKETTTS:
             return LLAMA_ROPE_TYPE_NORM;

src/llama-model.h

@@ -140,6 +140,7 @@ enum llm_type {
     LLM_TYPE_230B_A10B, // Minimax M2
     LLM_TYPE_428B_A23B, // Minimax M3
     LLM_TYPE_235B_A22B,
+    LLM_TYPE_288B_A19B, // dots3-note
     LLM_TYPE_300B_A47B, // Ernie MoE big
     LLM_TYPE_310B_A15B, // /MiMo-V2-Flash
     LLM_TYPE_355B_A32B, // GLM-4.5

src/models/deepseek2.cpp

@@ -524,17 +524,9 @@ llama_model_deepseek2::graph::graph(const llama_model & model, const llm_graph_p
                 q = ggml_mul_mat(ctx0, model.layers[il].wq, cur);
                 cb(q, "q", il);
             }
-            // split into {n_embd_head_qk_nope, n_head, n_tokens}
-            ggml_tensor * q_nope =
-                ggml_view_3d(ctx0, q, n_embd_head_qk_nope, n_head, n_tokens, ggml_row_size(q->type, n_embd_head_k),
-                             ggml_row_size(q->type, n_embd_head_k) * n_head, 0);
-            cb(q_nope, "q_nope", il);
-
-            // and {n_embd_head_qk_rope, n_head, n_tokens}
-            ggml_tensor * q_pe = ggml_view_3d(
-                ctx0, q, n_embd_head_qk_rope, n_head, n_tokens, ggml_row_size(q->type, n_embd_head_k),
-                ggml_row_size(q->type, n_embd_head_k) * n_head, ggml_row_size(q->type, n_embd_head_qk_nope));
-            cb(q_pe, "q_pe", il);
+            // {n_embd_head_k, n_head, n_tokens}
+            q = ggml_reshape_3d(ctx0, q, n_embd_head_k, n_head, n_tokens);
+            cb(q, "q", il);
 
             ggml_tensor * kv_cmpr_pe = ggml_mul_mat(ctx0, model.layers[il].wkv_a_mqa, cur);
             cb(kv_cmpr_pe, "kv_cmpr_pe", il);
@@ -552,10 +544,6 @@ llama_model_deepseek2::graph::graph(const llama_model & model, const llm_graph_p
                                               ggml_row_size(kv_cmpr_pe->type, kv_lora_rank));
             cb(k_pe, "k_pe", il);
 
-            q_pe = ggml_rope_ext(ctx0, q_pe, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
-                                 ext_factor, attn_factor, beta_fast, beta_slow);
-            cb(q_pe, "q_pe", il);
-
             k_pe = ggml_rope_ext(ctx0, k_pe, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
                                  ext_factor, attn_factor, beta_fast, beta_slow);
             cb(k_pe, "k_pe", il);
@@ -564,6 +552,20 @@ llama_model_deepseek2::graph::graph(const llama_model & model, const llm_graph_p
             cb(kv_cmpr, "kv_cmpr", il);
 
             if (is_mla) {
+                // split into {n_embd_head_qk_nope, n_head, n_tokens}
+                ggml_tensor * q_nope = ggml_view_3d(ctx0, q, n_embd_head_qk_nope, n_head, n_tokens,
+                                                    q->nb[1], q->nb[2], 0);
+                cb(q_nope, "q_nope", il);
+
+                // and {n_embd_head_qk_rope, n_head, n_tokens}
+                ggml_tensor * q_pe = ggml_view_3d(ctx0, q, n_embd_head_qk_rope, n_head, n_tokens,
+                                                  q->nb[1], q->nb[2], ggml_row_size(q->type, n_embd_head_qk_nope));
+                cb(q_pe, "q_pe", il);
+
+                q_pe = ggml_rope_ext(ctx0, q_pe, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
+                                     ext_factor, attn_factor, beta_fast, beta_slow);
+                cb(q_pe, "q_pe", il);
+
                 // {n_embd_head_qk_nope, n_tokens, n_head}
                 q_nope = ggml_permute(ctx0, q_nope, 0, 2, 1, 3);
                 cb(q_nope, "q_nope_perm", il);
@@ -623,10 +625,14 @@ llama_model_deepseek2::graph::graph(const llama_model & model, const llm_graph_p
                 Vcur = ggml_cont(ctx0, Vcur);
                 cb(Vcur, "Vcur_cont", il);
 
-                ggml_tensor * Qcur = ggml_concat(ctx0, q_nope, q_pe, 0);
+                // RoPE is applied to the trailing dims only
+                ggml_tensor * Qcur = ggml_rope_ext(ctx0, q, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig,
+                                                   freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow);
+                Qcur = ggml_rope_set_offset(Qcur, n_embd_head_qk_nope);
                 cb(Qcur, "Qcur", il);
 
-                ggml_tensor * Kcur = ggml_concat(ctx0, k_nope, ggml_repeat(ctx0, k_pe, q_pe), 0);
+                ggml_tensor * Kcur = ggml_concat(ctx0, k_nope,
+                        ggml_repeat_4d(ctx0, k_pe, n_embd_head_qk_rope, n_head, n_tokens, 1), 0);
                 cb(Kcur, "Kcur", il);
 
                 if (inp_attn_scale) {

src/models/deepseek4.cpp

@@ -501,21 +501,10 @@ ggml_tensor * llama_model_deepseek4::graph::build_hca_compressed_kv_from_state(
     comp = build_norm(comp, norm, nullptr, LLM_NORM_RMS, il);
     cb(comp, name, il);
 
-    ggml_tensor * comp_nope = ggml_view_3d(ctx0, comp, n_embd_head_nope, 1, n_blocks,
-            ggml_row_size(comp->type, n_embd_head),
-            ggml_row_size(comp->type, n_embd_head),
-            0);
-    ggml_tensor * comp_pe = ggml_view_3d(ctx0, comp, n_embd_head_rope, 1, n_blocks,
-            ggml_row_size(comp->type, n_embd_head),
-            ggml_row_size(comp->type, n_embd_head),
-            ggml_row_size(comp->type, n_embd_head_nope));
-
-    comp_pe = ggml_rope_ext(ctx0, comp_pe, comp_pos, nullptr, n_embd_head_rope, rope_type, n_ctx_orig,
+    comp = ggml_rope_ext(ctx0, comp, comp_pos, nullptr, n_embd_head_rope, rope_type, n_ctx_orig,
             hparams.dsv4_compress_rope_base, freq_scale, ext_factor,
             dsv4_rope_attn_factor(freq_scale, ext_factor), beta_fast, beta_slow);
-    cb(comp_pe, name, il);
-
-    comp = ggml_concat(ctx0, comp_nope, comp_pe, 0);
+    comp = ggml_rope_set_offset(comp, n_embd_head_nope);
     cb(comp, name, il);
 
     return comp;
@@ -585,21 +574,10 @@ ggml_tensor * llama_model_deepseek4::graph::build_overlap_compressed_kv_from_sta
     comp = build_norm(comp, norm, nullptr, LLM_NORM_RMS, il);
     cb(comp, name, il);
 
-    ggml_tensor * comp_nope = ggml_view_3d(ctx0, comp, n_embd_head_nope, 1, n_blocks,
-            ggml_row_size(comp->type, n_embd_head),
-            ggml_row_size(comp->type, n_embd_head),
-            0);
-    ggml_tensor * comp_pe = ggml_view_3d(ctx0, comp, n_embd_head_rope, 1, n_blocks,
-            ggml_row_size(comp->type, n_embd_head),
-            ggml_row_size(comp->type, n_embd_head),
-            ggml_row_size(comp->type, n_embd_head_nope));
-
-    comp_pe = ggml_rope_ext(ctx0, comp_pe, comp_pos, nullptr, n_embd_head_rope, rope_type, n_ctx_orig,
+    comp = ggml_rope_ext(ctx0, comp, comp_pos, nullptr, n_embd_head_rope, rope_type, n_ctx_orig,
             hparams.dsv4_compress_rope_base, freq_scale, ext_factor,
             dsv4_rope_attn_factor(freq_scale, ext_factor), beta_fast, beta_slow);
-    cb(comp_pe, name, il);
-
-    comp = ggml_concat(ctx0, comp_nope, comp_pe, 0);
+    comp = ggml_rope_set_offset(comp, n_embd_head_nope);
     cb(comp, name, il);
 
     return comp;
@@ -628,21 +606,12 @@ ggml_tensor * llama_model_deepseek4::graph::build_lid_top_k(
     indexer_q = ggml_reshape_3d(ctx0, indexer_q, n_embd_indexer_head, n_indexer_head, nt);
     cb(indexer_q, "lid_q", il);
 
-    ggml_tensor * indexer_q_nope = ggml_view_3d(ctx0, indexer_q, n_embd_indexer_head_nope, n_indexer_head, nt,
-            ggml_row_size(indexer_q->type, n_embd_indexer_head),
-            ggml_row_size(indexer_q->type, n_embd_indexer_head)*n_indexer_head,
-            0);
-    ggml_tensor * indexer_q_pe = ggml_view_3d(ctx0, indexer_q, n_embd_indexer_head_rope, n_indexer_head, nt,
-            ggml_row_size(indexer_q->type, n_embd_indexer_head),
-            ggml_row_size(indexer_q->type, n_embd_indexer_head)*n_indexer_head,
-            ggml_row_size(indexer_q->type, n_embd_indexer_head_nope));
-
-    indexer_q_pe = ggml_rope_ext(ctx0, indexer_q_pe, inp_pos, nullptr, n_embd_indexer_head_rope,
+    indexer_q = ggml_rope_ext(ctx0, indexer_q, inp_pos, nullptr, n_embd_indexer_head_rope,
             rope_type, n_ctx_orig, hparams.dsv4_compress_rope_base, freq_scale,
             ext_factor, dsv4_rope_attn_factor(freq_scale, ext_factor), beta_fast, beta_slow);
-    cb(indexer_q_pe, "lid_q_pe", il);
+    indexer_q = ggml_rope_set_offset(indexer_q, n_embd_indexer_head_nope);
+    cb(indexer_q, "lid_q_rope", il);
 
-    indexer_q = ggml_concat(ctx0, indexer_q_nope, indexer_q_pe, 0);
     indexer_q = llama_mul_mat_hadamard(ctx0, indexer_q, inp_lid.k_rot);
     cb(indexer_q, "lid_q_rot", il);
 
@@ -945,18 +914,9 @@ ggml_tensor * llama_model_deepseek4::graph::build_attention_impl(
     q = ggml_rms_norm(ctx0, q, norm_rms_eps);
     cb(q, "q_norm", il);
 
-    ggml_tensor * q_nope = ggml_view_3d(ctx0, q, n_embd_head_nope, n_head, nt,
-            ggml_row_size(q->type, n_embd_head),
-            ggml_row_size(q->type, n_embd_head)*n_head,
-            0);
-    ggml_tensor * q_pe = ggml_view_3d(ctx0, q, n_embd_head_rope, n_head, nt,
-            ggml_row_size(q->type, n_embd_head),
-            ggml_row_size(q->type, n_embd_head)*n_head,
-            ggml_row_size(q->type, n_embd_head_nope));
-    q_pe = ggml_rope_ext(ctx0, q_pe, inp_pos, nullptr, n_embd_head_rope, rope_type, n_ctx_orig_l,
+    q = ggml_rope_ext(ctx0, q, inp_pos, nullptr, n_embd_head_rope, rope_type, n_ctx_orig_l,
             freq_base_l, freq_scale_l, ext_factor_l, attn_factor_l, beta_fast_l, beta_slow_l);
-    cb(q_pe, "q_pe", il);
-    q = ggml_concat(ctx0, q_nope, q_pe, 0);
+    q = ggml_rope_set_offset(q, n_embd_head_nope);
     cb(q, "q", il);
 
     ggml_tensor * kv = build_lora_mm(layer.wkv, cur);
@@ -964,18 +924,9 @@ ggml_tensor * llama_model_deepseek4::graph::build_attention_impl(
     kv = ggml_reshape_3d(ctx0, kv, n_embd_head, 1, nt);
     cb(kv, "kv_norm", il);
 
-    ggml_tensor * kv_nope = ggml_view_3d(ctx0, kv, n_embd_head_nope, 1, nt,
-            ggml_row_size(kv->type, n_embd_head),
-            ggml_row_size(kv->type, n_embd_head),
-            0);
-    ggml_tensor * kv_pe = ggml_view_3d(ctx0, kv, n_embd_head_rope, 1, nt,
-            ggml_row_size(kv->type, n_embd_head),
-            ggml_row_size(kv->type, n_embd_head),
-            ggml_row_size(kv->type, n_embd_head_nope));
-    kv_pe = ggml_rope_ext(ctx0, kv_pe, inp_pos, nullptr, n_embd_head_rope, rope_type, n_ctx_orig_l,
+    kv = ggml_rope_ext(ctx0, kv, inp_pos, nullptr, n_embd_head_rope, rope_type, n_ctx_orig_l,
             freq_base_l, freq_scale_l, ext_factor_l, attn_factor_l, beta_fast_l, beta_slow_l);
-    cb(kv_pe, "kv_pe", il);
-    kv = ggml_concat(ctx0, kv_nope, kv_pe, 0);
+    kv = ggml_rope_set_offset(kv, n_embd_head_nope);
     cb(kv, "kv", il);
 
     const int64_t ratio = hparams.dsv4_compress_ratios[il];
@@ -1245,17 +1196,9 @@ ggml_tensor * llama_model_deepseek4::graph::build_attention_impl(
     }
 
     out = ggml_reshape_3d(ctx0, out, n_embd_head, n_head, nt);
-    ggml_tensor * out_nope = ggml_view_3d(ctx0, out, n_embd_head_nope, n_head, nt,
-            ggml_row_size(out->type, n_embd_head),
-            ggml_row_size(out->type, n_embd_head)*n_head,
-            0);
-    ggml_tensor * out_pe = ggml_view_3d(ctx0, out, n_embd_head_rope, n_head, nt,
-            ggml_row_size(out->type, n_embd_head),
-            ggml_row_size(out->type, n_embd_head)*n_head,
-            ggml_row_size(out->type, n_embd_head_nope));
-    out_pe = ggml_rope_ext_back(ctx0, out_pe, inp_pos, nullptr, n_embd_head_rope, rope_type, n_ctx_orig_l,
+    out = ggml_rope_ext_back(ctx0, out, inp_pos, nullptr, n_embd_head_rope, rope_type, n_ctx_orig_l,
             freq_base_l, freq_scale_l, ext_factor_l, attn_factor_l, beta_fast_l, beta_slow_l);
-    out = ggml_concat(ctx0, out_nope, out_pe, 0);
+    out = ggml_rope_set_offset(out, n_embd_head_nope);
     cb(out, "attn_derope", il);
 
     out = ggml_reshape_3d(ctx0, out, o_group_dim, n_groups, nt);

src/models/dflash.cpp

@@ -591,17 +591,9 @@ llama_model_dflash::graph_dsv4::graph_dsv4(const llama_model & model, const llm_
             kv = build_norm(kv, layer.attn_kv_norm, nullptr, LLM_NORM_RMS, il);
             kv = ggml_reshape_3d(ctx0, kv, n_embd_head, 1, n_tokens);
 
-            ggml_tensor * kv_nope = ggml_view_3d(ctx0, kv, n_embd_head_nope, 1, n_tokens,
-                    ggml_row_size(kv->type, n_embd_head),
-                    ggml_row_size(kv->type, n_embd_head),
-                    0);
-            ggml_tensor * kv_pe = ggml_view_3d(ctx0, kv, n_embd_head_rope, 1, n_tokens,
-                    ggml_row_size(kv->type, n_embd_head),
-                    ggml_row_size(kv->type, n_embd_head),
-                    ggml_row_size(kv->type, n_embd_head_nope));
-            kv_pe = ggml_rope_ext(ctx0, kv_pe, inp_pos, nullptr, n_embd_head_rope, rope_type, 0,
+            kv = ggml_rope_ext(ctx0, kv, inp_pos, nullptr, n_embd_head_rope, rope_type, 0,
                     freq_base, 1.0f, 0.0f, 1.0f, 0.0f, 0.0f);
-            kv = ggml_concat(ctx0, kv_nope, kv_pe, 0);
+            kv = ggml_rope_set_offset(kv, n_embd_head_nope);
             cb(kv, "kv_injected", il);
 
             if (inp_attn->self_k_rot_swa) {

src/models/dots3note.cpp

@@ -0,0 +1,480 @@
+#include "models.h"
+
+#include "llama-kv-cache.h"
+#include "llama-kv-cache-dsa.h"
+
+// note: code adapted from deepseek32.cpp (DSA indexer + absorbed MLA) and step35.cpp (head-wise output gate)
+
+void llama_model_dots3note::load_arch_hparams(llama_model_loader & ml) {
+    ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);
+    hparams.f_norm_eps = 1e-6;  // eps for the indexer k_norm layer norm
+
+    // TODO: use MTP layer
+    ml.get_key(LLM_KV_NEXTN_PREDICT_LAYERS, hparams.n_layer_nextn, false);
+    GGML_ASSERT(hparams.n_layer_nextn < hparams.n_layer_all && "n_layer_nextn must be < n_layer_all");
+
+    // MoE parameters
+    ml.get_key(LLM_KV_EXPERT_SHARED_COUNT,        hparams.n_expert_shared);
+    ml.get_key(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp);
+    ml.get_key(LLM_KV_LEADING_DENSE_BLOCK_COUNT,  hparams.n_layer_dense_lead);
+    ml.get_key(LLM_KV_EXPERT_WEIGHTS_SCALE,       hparams.expert_weights_scale, false);
+    ml.get_key(LLM_KV_EXPERT_WEIGHTS_NORM,        hparams.expert_weights_norm, false);
+    ml.get_key(LLM_KV_EXPERT_GATING_FUNC,         hparams.expert_gating_func);
+
+    // MLA parameters of the full-attention layers
+    ml.get_key(LLM_KV_ATTENTION_Q_LORA_RANK,      hparams.n_lora_q);
+    ml.get_key(LLM_KV_ATTENTION_KV_LORA_RANK,     hparams.n_lora_kv);
+    ml.get_key(LLM_KV_ATTENTION_KEY_LENGTH_MLA,   hparams.n_embd_head_k_mla_impl);
+    ml.get_key(LLM_KV_ATTENTION_VALUE_LENGTH_MLA, hparams.n_embd_head_v_mla_impl);
+
+    // MLA parameters of the sliding-window layers
+    ml.get_key(LLM_KV_ATTENTION_KV_LORA_RANK_SWA,     hparams.n_lora_kv_swa);
+    ml.get_key(LLM_KV_ATTENTION_KEY_LENGTH_MLA_SWA,   hparams.n_embd_head_k_mla_swa);
+    ml.get_key(LLM_KV_ATTENTION_VALUE_LENGTH_MLA_SWA, hparams.n_embd_head_v_mla_swa);
+
+    hparams.swa_type = LLAMA_SWA_TYPE_STANDARD;
+    ml.get_key(LLM_KV_ATTENTION_SLIDING_WINDOW, hparams.n_swa);
+    ml.get_key(LLM_KV_ROPE_FREQ_BASE_SWA,       hparams.rope_freq_base_train_swa);
+    ml.get_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, hparams.is_swa_impl);
+
+    // DSA parameters
+    ml.get_key(LLM_KV_ATTENTION_INDEXER_HEAD_COUNT, hparams.indexer_n_head);
+    ml.get_key(LLM_KV_ATTENTION_INDEXER_KEY_LENGTH, hparams.indexer_head_size);
+    ml.get_key(LLM_KV_ATTENTION_INDEXER_TOP_K,      hparams.indexer_top_k);
+    ml.get_arr(LLM_KV_ATTENTION_INDEXER_TYPES, hparams.is_indexer_full_impl);
+
+    switch (hparams.n_layer()) {
+        case 46: type = LLM_TYPE_288B_A19B; break;
+        default: type = LLM_TYPE_UNKNOWN;
+    }
+}
+
+void llama_model_dots3note::load_arch_tensors(llama_model_loader & ml) {
+    LLAMA_LOAD_LOCALS;
+    GGML_UNUSED(ml);
+
+    if (!hparams.is_mla()) {
+        throw std::runtime_error("DOTS3NOTE architecture requires MLA");
+    }
+
+    const int64_t n_embd_head_qk_rope = hparams.n_rot();
+
+    const int64_t q_lora_rank     = hparams.n_lora_q;
+    const int64_t n_ff_exp        = hparams.n_ff_exp;
+    const int64_t n_expert_shared = hparams.n_expert_shared;
+
+    tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, 0);
+
+    output_norm = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd}, 0);
+    output      = create_tensor(tn(LLM_TENSOR_OUTPUT,      "weight"), {n_embd, n_vocab}, TENSOR_NOT_REQUIRED);
+    if (!output) {
+        output = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, TENSOR_DUPLICATED);
+    }
+
+    for (int i = 0; i < n_layer_all; ++i) {
+        auto & layer = layers[i];
+
+        const bool is_mtp = i >= n_layer;
+        // the NextN/MTP block uses the sliding-attention geometry
+        const bool is_swa = is_mtp || hparams.is_swa(i);
+
+        // MTP tensors are preserved in the GGUF but there is no MTP graph yet
+        const int flags = is_mtp ? TENSOR_SKIP | TENSOR_NOT_REQUIRED : 0;
+
+        const int64_t n_head_l = hparams.n_head(i);
+
+        const int64_t kv_lora_rank       = is_swa ? hparams.n_lora_kv_swa         : hparams.n_lora_kv;
+        const int64_t n_embd_head_k_mla  = is_swa ? hparams.n_embd_head_k_mla_swa : hparams.n_embd_head_k_mla();
+        const int64_t n_embd_head_v_mla  = is_swa ? hparams.n_embd_head_v_mla_swa : hparams.n_embd_head_v_mla();
+        const int64_t n_embd_head_qk_nope = n_embd_head_k_mla - n_embd_head_qk_rope;
+
+        layer.attn_norm      = create_tensor(tn(LLM_TENSOR_ATTN_NORM,      "weight", i), {n_embd}, flags);
+        layer.attn_q_a_norm  = create_tensor(tn(LLM_TENSOR_ATTN_Q_A_NORM,  "weight", i), {q_lora_rank}, flags);
+        layer.attn_kv_a_norm = create_tensor(tn(LLM_TENSOR_ATTN_KV_A_NORM, "weight", i), {kv_lora_rank}, flags);
+        // norm applied on the shared rope key before rope
+        layer.attn_k_norm    = create_tensor(tn(LLM_TENSOR_ATTN_K_NORM,    "weight", i), {n_embd_head_qk_rope}, flags);
+
+        layer.wq_a = create_tensor(tn(LLM_TENSOR_ATTN_Q_A, "weight", i), {n_embd, q_lora_rank}, flags);
+        layer.wq_b = create_tensor(tn(LLM_TENSOR_ATTN_Q_B, "weight", i), {q_lora_rank, n_head_l * n_embd_head_k_mla}, flags);
+
+        layer.wkv_a_mqa = create_tensor(tn(LLM_TENSOR_ATTN_KV_A_MQA, "weight", i), {n_embd, kv_lora_rank + n_embd_head_qk_rope}, flags);
+
+        layer.wk_b = create_tensor(tn(LLM_TENSOR_ATTN_K_B, "weight", i), {n_embd_head_qk_nope, kv_lora_rank, n_head_l}, flags);
+        layer.wv_b = create_tensor(tn(LLM_TENSOR_ATTN_V_B, "weight", i), {kv_lora_rank, n_embd_head_v_mla, n_head_l}, flags);
+
+        layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_head_l * n_embd_head_v_mla, n_embd}, flags);
+
+        // head-wise sigmoid output gate
+        layer.wqkv_gate = create_tensor(tn(LLM_TENSOR_ATTN_GATE, "weight", i), {n_embd, n_head_l}, flags);
+
+        layer.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd}, flags);
+
+        // DSA indexer
+        if (!is_mtp && hparams.is_indexer_full(i)) {
+            layer.indexer_k_norm   = create_tensor(tn(LLM_TENSOR_INDEXER_K_NORM,   "weight", i), {hparams.indexer_head_size}, flags);
+            layer.indexer_k_norm_b = create_tensor(tn(LLM_TENSOR_INDEXER_K_NORM,   "bias",   i), {hparams.indexer_head_size}, flags);
+            layer.indexer_proj     = create_tensor(tn(LLM_TENSOR_INDEXER_PROJ,     "weight", i), {n_embd, hparams.indexer_n_head}, flags);
+            layer.indexer_attn_k   = create_tensor(tn(LLM_TENSOR_INDEXER_ATTN_K,   "weight", i), {n_embd, hparams.indexer_head_size}, flags);
+            layer.indexer_attn_q_b = create_tensor(tn(LLM_TENSOR_INDEXER_ATTN_Q_B, "weight", i), {q_lora_rank, hparams.indexer_n_head * hparams.indexer_head_size}, flags);
+        }
+
+        if (is_mtp || i < (int) hparams.n_layer_dense_lead) {
+            layer.ffn_gate = create_tensor(tn(LLM_TENSOR_FFN_GATE, "weight", i), {n_embd,   n_ff}, flags);
+            layer.ffn_down = create_tensor(tn(LLM_TENSOR_FFN_DOWN, "weight", i), {  n_ff, n_embd}, flags);
+            layer.ffn_up   = create_tensor(tn(LLM_TENSOR_FFN_UP,   "weight", i), {n_embd,   n_ff}, flags);
+        } else {
+            if (n_expert == 0 || n_expert_used == 0) {
+                throw std::runtime_error("n_expert and n_expert_used must be > 0");
+            }
+
+            layer.ffn_gate_inp    = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP,    "weight", i), {n_embd, n_expert}, flags);
+            layer.ffn_exp_probs_b = create_tensor(tn(LLM_TENSOR_FFN_EXP_PROBS_B, "bias",   i), {n_expert}, flags);
+
+            layer.ffn_gate_exps = create_tensor(tn(LLM_TENSOR_FFN_GATE_EXPS, "weight", i), {  n_embd, n_ff_exp, n_expert}, flags);
+            layer.ffn_down_exps = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS, "weight", i), {n_ff_exp,   n_embd, n_expert}, flags);
+            layer.ffn_up_exps   = create_tensor(tn(LLM_TENSOR_FFN_UP_EXPS,   "weight", i), {  n_embd, n_ff_exp, n_expert}, flags);
+
+            layer.ffn_gate_shexp = create_tensor(tn(LLM_TENSOR_FFN_GATE_SHEXP, "weight", i), {n_embd, n_ff_exp * n_expert_shared}, flags);
+            layer.ffn_down_shexp = create_tensor(tn(LLM_TENSOR_FFN_DOWN_SHEXP, "weight", i), {        n_ff_exp * n_expert_shared, n_embd}, flags);
+            layer.ffn_up_shexp   = create_tensor(tn(LLM_TENSOR_FFN_UP_SHEXP,   "weight", i), {n_embd, n_ff_exp * n_expert_shared}, flags);
+        }
+
+        if (is_mtp) {
+            layer.nextn.eh_proj          = create_tensor(tn(LLM_TENSOR_NEXTN_EH_PROJ,          "weight", i), { 2 * n_embd, n_embd }, flags);
+            layer.nextn.enorm            = create_tensor(tn(LLM_TENSOR_NEXTN_ENORM,            "weight", i), { n_embd }, flags);
+            layer.nextn.hnorm            = create_tensor(tn(LLM_TENSOR_NEXTN_HNORM,            "weight", i), { n_embd }, flags);
+            layer.nextn.embed_tokens     = create_tensor(tn(LLM_TENSOR_NEXTN_EMBED_TOKENS,     "weight", i), { n_embd, n_vocab }, flags);
+            layer.nextn.shared_head_norm = create_tensor(tn(LLM_TENSOR_NEXTN_SHARED_HEAD_NORM, "weight", i), { n_embd }, flags);
+        }
+    }
+}
+
+std::unique_ptr<llm_graph_context> llama_model_dots3note::build_arch_graph(const llm_graph_params & params) const {
+    return std::make_unique<graph>(*this, params);
+}
+
+llama_model_dots3note::graph::graph(const llama_model & model, const llm_graph_params & params) :
+    llm_graph_context(params) {
+    GGML_ASSERT(hparams.is_mla());
+
+    const int64_t n_embd_head_qk_rope = hparams.n_rot();
+
+    const int64_t n_indexer_head      = hparams.indexer_n_head;
+    const int64_t n_embd_indexer_head = hparams.indexer_head_size;
+    const uint32_t n_indexer_top_k = hparams.indexer_top_k;
+
+    // the indexer head layout is [rope | nope]
+    GGML_ASSERT(hparams.n_rot() <= n_embd_indexer_head);
+
+    ggml_tensor * cur;
+    ggml_tensor * inpL;
+
+    inpL = build_inp_embd(model.tok_embd);
+
+    ggml_tensor * inp_pos = build_inp_pos();
+
+    llm_graph_input_attn_k_dsa_iswa * inp_attn = build_attn_inp_k_dsa_iswa();
+
+    ggml_tensor * inp_out_ids = build_inp_out_ids();
+
+    for (int il = 0; il < n_layer; ++il) {
+        ggml_tensor * inpSA = inpL;
+
+        const bool is_swa = hparams.is_swa(il);
+
+        const int64_t n_head_l = hparams.n_head(il);
+
+        const int64_t kv_lora_rank        = is_swa ? hparams.n_lora_kv_swa         : hparams.n_lora_kv;
+        const int64_t n_embd_head_k_mla   = is_swa ? hparams.n_embd_head_k_mla_swa : hparams.n_embd_head_k_mla();
+        const int64_t n_embd_head_v_mla   = is_swa ? hparams.n_embd_head_v_mla_swa : hparams.n_embd_head_v_mla();
+        const int64_t n_embd_head_qk_nope = n_embd_head_k_mla - n_embd_head_qk_rope;
+
+        const float kq_scale    = 1.0f/sqrtf(float(n_embd_head_k_mla));
+        const float freq_base_l = model.get_rope_freq_base(cparams, il);
+
+        // norm
+        cur = build_norm(inpL, model.layers[il].attn_norm, NULL, LLM_NORM_RMS, il);
+        cb(cur, "attn_norm", il);
+
+        // self_attention
+        {
+            ggml_tensor * attn_inp = cur;
+
+            ggml_tensor * qr = ggml_mul_mat(ctx0, model.layers[il].wq_a, cur);
+            cb(qr, "qr", il);
+
+            qr = build_norm(qr, model.layers[il].attn_q_a_norm, nullptr, LLM_NORM_RMS, il);
+            cb(qr, "qr", il);
+
+            ggml_tensor * top_k = nullptr;
+
+            // lightning indexer (full-attention layers only)
+            if (!is_swa) {
+                ggml_tensor * indexer_q = ggml_mul_mat(ctx0, model.layers[il].indexer_attn_q_b, qr);
+                cb(indexer_q, "indexer_q", il);
+
+                // {n_embd_indexer_head, n_indexer_head, n_tokens}
+                indexer_q = ggml_reshape_3d(ctx0, indexer_q, n_embd_indexer_head, n_indexer_head, n_tokens);
+                indexer_q = ggml_rope_ext(ctx0, indexer_q, inp_pos, nullptr, n_rot,
+                                     LLAMA_ROPE_TYPE_NEOX, n_ctx_orig, freq_base, freq_scale,
+                                     ext_factor, attn_factor, beta_fast, beta_slow);
+                cb(indexer_q, "indexer_q", il);
+
+                ggml_tensor * indexer_k = ggml_mul_mat(ctx0, model.layers[il].indexer_attn_k, cur);
+                cb(indexer_k, "indexer_k", il);
+
+                indexer_k = build_norm(indexer_k, model.layers[il].indexer_k_norm, model.layers[il].indexer_k_norm_b, LLM_NORM, il);
+                cb(indexer_k, "indexer_k", il);
+
+                // {n_embd_indexer_head, 1, n_tokens}
+                indexer_k = ggml_reshape_3d(ctx0, indexer_k, n_embd_indexer_head, 1, n_tokens);
+                indexer_k = ggml_rope_ext(ctx0, indexer_k, inp_pos, nullptr, n_rot,
+                                     LLAMA_ROPE_TYPE_NEOX, n_ctx_orig, freq_base, freq_scale,
+                                     ext_factor, attn_factor, beta_fast, beta_slow);
+                cb(indexer_k, "indexer_k", il);
+
+                // perform Hadamard transform on indexer q and k
+                indexer_q = ggml_mul_mat(ctx0, inp_attn->get_dsa()->self_k_rot_lid, indexer_q);
+                cb(indexer_q, "indexer_q", il);
+                indexer_k = ggml_mul_mat(ctx0, inp_attn->get_dsa()->self_k_rot_lid, indexer_k);
+                cb(indexer_k, "indexer_k", il);
+
+                // store indexer keys to KV cache
+                const auto * mctx_lid = inp_attn->get_dsa()->mctx->get_lid();
+                const auto & k_idxs_lid = inp_attn->get_dsa()->get_k_idxs_lid();
+                ggml_build_forward_expand(gf, mctx_lid->cpy_k(ctx0, indexer_k, k_idxs_lid, il));
+
+                ggml_tensor * indexer_weights = ggml_mul_mat(ctx0, model.layers[il].indexer_proj, cur);
+                cb(indexer_weights, "indexer_weights", il);
+
+                indexer_k = mctx_lid->get_k(ctx0, il);
+
+                // split the batch into streams if needed
+                const auto n_stream = indexer_k->ne[3];
+                indexer_q = ggml_view_4d(ctx0, indexer_q, indexer_q->ne[0], indexer_q->ne[1], indexer_q->ne[2]/n_stream, n_stream, indexer_q->nb[1], indexer_q->nb[2], indexer_q->nb[3]/n_stream, 0);
+                indexer_weights = ggml_view_4d(ctx0, indexer_weights, indexer_weights->ne[0], indexer_weights->ne[1]/n_stream, indexer_weights->ne[2], n_stream, indexer_weights->nb[1], indexer_weights->nb[2]/n_stream, indexer_weights->nb[3]/n_stream, 0);
+
+                // pre-scale weights to avoid scaling operations on huge indexer_score tensor
+                indexer_weights = ggml_scale(ctx0, indexer_weights, 1.0f / sqrtf(float(n_embd_indexer_head * n_indexer_head)));
+                cb(indexer_weights, "indexer_weights", il);
+
+                ggml_tensor * indexer_score = nullptr;
+                if (cparams.fused_lid) {
+                    indexer_score = ggml_lightning_indexer(ctx0, indexer_q, indexer_k, indexer_weights, inp_attn->get_dsa()->get_kq_mask_lid());
+                    cb(indexer_score, "indexer_score", il);
+                    res->add_fused_node({LLM_FUSED_OP_LIGHTNING_INDEXER, indexer_score, il});
+                } else {
+                    indexer_q = ggml_permute(ctx0, indexer_q, 0, 2, 1, 3);
+                    cb(indexer_q, "indexer_q", il);
+                    indexer_k = ggml_permute(ctx0, indexer_k, 0, 2, 1, 3);
+                    cb(indexer_k, "indexer_k", il);
+
+                    ggml_tensor * indexer_kq = ggml_mul_mat(ctx0, indexer_k, indexer_q);
+                    cb(indexer_kq, "indexer_kq", il);
+
+                    // ReLU requires contiguous tensors
+                    indexer_kq = ggml_cont(ctx0, ggml_permute(ctx0, indexer_kq, 2, 1, 0, 3));
+                    cb(indexer_kq, "indexer_kq", il);
+
+                    indexer_score = ggml_relu(ctx0, indexer_kq);
+                    cb(indexer_score, "indexer_score", il);
+
+                    indexer_score = ggml_mul(ctx0, indexer_score, indexer_weights);
+                    cb(indexer_score, "indexer_score", il);
+
+                    // sum by q n_indexer_head dimension
+                    indexer_score = ggml_sum_rows(ctx0, indexer_score);
+                    cb(indexer_score, "indexer_score", il);
+
+                    // permute result to match KQ mask
+                    indexer_score = ggml_cont(ctx0, ggml_permute(ctx0, indexer_score, 2, 1, 0, 3));
+                    cb(indexer_score, "indexer_score", il);
+
+                    ggml_tensor * indexer_kq_mask = inp_attn->get_dsa()->get_kq_mask_lid();
+                    indexer_score = ggml_add(ctx0, indexer_score, indexer_kq_mask);
+                    cb(indexer_score, "indexer_score", il);
+                }
+
+                // get indices of top k indexer scores
+                uint32_t n_top_k = indexer_score->ne[0] < n_indexer_top_k ? indexer_score->ne[0] : n_indexer_top_k;
+                top_k = ggml_cont(ctx0, ggml_top_k(ctx0, indexer_score, n_top_k));
+                cb(top_k, "top_k", il);
+            }
+
+            ggml_tensor * q = ggml_mul_mat(ctx0, model.layers[il].wq_b, qr);
+            cb(q, "q", il);
+
+            // split into {n_embd_head_qk_nope, n_head_l, n_tokens}
+            ggml_tensor * q_nope =
+                ggml_view_3d(ctx0, q, n_embd_head_qk_nope, n_head_l, n_tokens, ggml_row_size(q->type, n_embd_head_k_mla),
+                             ggml_row_size(q->type, n_embd_head_k_mla) * n_head_l, 0);
+            cb(q_nope, "q_nope", il);
+
+            // and {n_embd_head_qk_rope, n_head_l, n_tokens}
+            ggml_tensor * q_pe = ggml_view_3d(
+                ctx0, q, n_embd_head_qk_rope, n_head_l, n_tokens, ggml_row_size(q->type, n_embd_head_k_mla),
+                ggml_row_size(q->type, n_embd_head_k_mla) * n_head_l, ggml_row_size(q->type, n_embd_head_qk_nope));
+            cb(q_pe, "q_pe", il);
+
+            ggml_tensor * kv_cmpr_pe = ggml_mul_mat(ctx0, model.layers[il].wkv_a_mqa, cur);
+            cb(kv_cmpr_pe, "kv_cmpr_pe", il);
+
+            // split into {kv_lora_rank, n_tokens}
+            ggml_tensor * kv_cmpr =
+                ggml_view_2d(ctx0, kv_cmpr_pe, kv_lora_rank, n_tokens,
+                             ggml_row_size(kv_cmpr_pe->type, kv_lora_rank + n_embd_head_qk_rope), 0);
+            cb(kv_cmpr, "kv_cmpr", il);
+
+            // and {n_embd_head_qk_rope, 1, n_tokens}
+            ggml_tensor * k_pe = ggml_view_3d(ctx0, kv_cmpr_pe, n_embd_head_qk_rope, 1, n_tokens,
+                                              ggml_row_size(kv_cmpr_pe->type, kv_lora_rank + n_embd_head_qk_rope),
+                                              ggml_row_size(kv_cmpr_pe->type, kv_lora_rank + n_embd_head_qk_rope),
+                                              ggml_row_size(kv_cmpr_pe->type, kv_lora_rank));
+            cb(k_pe, "k_pe", il);
+
+            // norm on the shared rope key, applied before rope
+            k_pe = build_norm(k_pe, model.layers[il].attn_k_norm, nullptr, LLM_NORM_RMS, il);
+            cb(k_pe, "k_pe", il);
+
+            q_pe = ggml_rope_ext(ctx0, q_pe, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base_l, freq_scale,
+                                 ext_factor, attn_factor, beta_fast, beta_slow);
+            cb(q_pe, "q_pe", il);
+
+            k_pe = ggml_rope_ext(ctx0, k_pe, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base_l, freq_scale,
+                                 ext_factor, attn_factor, beta_fast, beta_slow);
+            cb(k_pe, "k_pe", il);
+
+            kv_cmpr = build_norm(kv_cmpr, model.layers[il].attn_kv_a_norm, nullptr, LLM_NORM_RMS, il);
+            cb(kv_cmpr, "kv_cmpr", il);
+
+            // MLA attention with the absorption optimization
+            {
+                // {n_embd_head_qk_nope, n_tokens, n_head_l}
+                q_nope = ggml_permute(ctx0, q_nope, 0, 2, 1, 3);
+                cb(q_nope, "q_nope_perm", il);
+
+                // {n_embd_head_qk_nope, kv_lora_rank, n_head_l} x {n_embd_head_qk_nope, n_tokens, n_head_l}
+                ggml_tensor * q_nope_absorbed = ggml_mul_mat(ctx0, model.layers[il].wk_b, q_nope);
+                cb(q_nope_absorbed, "q_nope_absorbed", il);
+
+                // {kv_lora_rank, n_head_l, n_tokens}
+                q_nope_absorbed = ggml_permute(ctx0, q_nope_absorbed, 0, 2, 1, 3);
+                cb(q_nope_absorbed, "q_nope_absorbed_perm", il);
+
+                // {n_embd_head_qk_rope + kv_lora_rank, n_head_l, n_tokens}
+                ggml_tensor * Qcur = ggml_concat(ctx0, q_nope_absorbed, q_pe, 0);
+                cb(Qcur, "Qcur", il);
+
+                kv_cmpr = ggml_reshape_3d(ctx0, kv_cmpr, kv_lora_rank, 1, n_tokens);
+                cb(kv_cmpr, "kv_cmpr_reshape", il);
+
+                // {n_embd_head_qk_rope + kv_lora_rank, 1, n_tokens}
+                ggml_tensor * Kcur = ggml_concat(ctx0, kv_cmpr, k_pe, 0);
+                cb(Kcur, "Kcur", il);
+
+                // {kv_lora_rank, 1, n_tokens}
+                ggml_tensor * Vcur = kv_cmpr;
+                cb(Vcur, "Vcur", il);
+
+                // apply the head-wise output gate before o_proj, so wo stays out of build_attn
+                if (is_swa) {
+                    cur = build_attn(inp_attn->get_swa(),
+                            nullptr, nullptr, nullptr,
+                            Qcur, Kcur, Vcur, nullptr, nullptr, model.layers[il].wv_b, kq_scale, il);
+                } else {
+                    cur = build_attn(inp_attn->get_dsa(),
+                            nullptr, nullptr, nullptr,
+                            Qcur, Kcur, Vcur, nullptr, nullptr, model.layers[il].wv_b, top_k, kq_scale, il);
+                }
+                cb(cur, "attn_out", il);
+
+                ggml_tensor * gate = build_lora_mm(model.layers[il].wqkv_gate, attn_inp);
+                cb(gate, "attn_gate", il);
+
+                gate = ggml_sigmoid(ctx0, gate);
+                cb(gate, "attn_gate_sigmoid", il);
+
+                // broadcast the per-head gate over the head dimension
+                ggml_tensor * attn_3d = ggml_reshape_3d(ctx0, cur, n_embd_head_v_mla, n_head_l, n_tokens);
+                ggml_tensor * gate_3d = ggml_reshape_3d(ctx0, gate,                1, n_head_l, n_tokens);
+                attn_3d = ggml_mul(ctx0, attn_3d, gate_3d);
+                cb(attn_3d, "attn_gated", il);
+
+                cur = ggml_reshape_2d(ctx0, attn_3d, n_embd_head_v_mla * n_head_l, n_tokens);
+
+                cur = build_lora_mm(model.layers[il].wo, cur, model.layers[il].wo_s);
+                cb(cur, "attn_output", il);
+            }
+        }
+
+        if (il == n_layer - 1 && inp_out_ids) {
+            cur   = ggml_get_rows(ctx0, cur, inp_out_ids);
+            inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids);
+        }
+
+        ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA);
+        cb(ffn_inp, "ffn_inp", il);
+
+        cur = build_norm(ffn_inp, model.layers[il].ffn_norm, NULL, LLM_NORM_RMS, il);
+        cb(cur, "ffn_norm", il);
+
+        if ((uint32_t) il < hparams.n_layer_dense_lead) {
+            cur = build_ffn(cur,
+                model.layers[il].ffn_up, NULL, model.layers[il].ffn_up_s,
+                model.layers[il].ffn_gate, NULL, model.layers[il].ffn_gate_s,
+                model.layers[il].ffn_down, NULL, model.layers[il].ffn_down_s,
+                NULL, LLM_FFN_SILU, LLM_FFN_PAR, il);
+            cb(cur, "ffn_out", il);
+        } else {
+            ggml_tensor * moe_out = build_moe_ffn(cur,
+                model.layers[il].ffn_gate_inp,
+                model.layers[il].ffn_up_exps,
+                model.layers[il].ffn_gate_exps,
+                model.layers[il].ffn_down_exps,
+                model.layers[il].ffn_exp_probs_b,
+                n_expert, n_expert_used,
+                LLM_FFN_SILU, hparams.expert_weights_norm,
+                hparams.expert_weights_scale,
+                (llama_expert_gating_func_type) hparams.expert_gating_func,
+                il,
+                nullptr,
+                model.layers[il].ffn_gate_up_exps,
+                model.layers[il].ffn_up_exps_s,
+                model.layers[il].ffn_gate_exps_s,
+                model.layers[il].ffn_down_exps_s);
+            cb(moe_out, "ffn_moe_out", il);
+
+            ggml_tensor * ffn_shexp =
+                build_ffn(cur,
+                    model.layers[il].ffn_up_shexp, NULL, model.layers[il].ffn_up_shexp_s,
+                    model.layers[il].ffn_gate_shexp, NULL, model.layers[il].ffn_gate_shexp_s,
+                    model.layers[il].ffn_down_shexp, NULL, model.layers[il].ffn_down_shexp_s,
+                    NULL, LLM_FFN_SILU, LLM_FFN_PAR, il);
+            cb(ffn_shexp, "ffn_shexp", il);
+
+            cur = ggml_add(ctx0, moe_out, ffn_shexp);
+            cb(cur, "ffn_out", il);
+        }
+
+        cur = ggml_add(ctx0, cur, ffn_inp);
+
+        cur = build_cvec(cur, il);
+        cb(cur, "l_out", il);
+
+        inpL = cur;
+    }
+
+    cur = inpL;
+
+    cur = build_norm(cur, model.output_norm, NULL, LLM_NORM_RMS, -1);
+
+    cb(cur, "result_norm", -1);
+    res->t_embd = cur;
+
+    cur = ggml_mul_mat(ctx0, model.output, cur);
+
+    cb(cur, "result_output", -1);
+    res->t_logits = cur;
+
+    ggml_build_forward_expand(gf, cur);
+}

src/models/lfm2.cpp

@@ -2,6 +2,8 @@
 #include "../llama-memory-hybrid-iswa.h"
 #include "../llama-memory-hybrid.h"
 
+#include <algorithm>
+
 void llama_model_lfm2::load_arch_hparams(llama_model_loader & ml) {
     ml.get_key(LLM_KV_SHORTCONV_L_CACHE,           hparams.n_shortconv_l_cache);
     ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);
@@ -202,15 +204,20 @@ llama_model_lfm2::graph<iswa>::graph(const llama_model & model, const llm_graph_
         }
         GGML_ASSERT(bx->ne[0] > conv->ne[0]);
 
-        // last d_conv columns is a new conv state
-        auto * new_conv = ggml_view_3d(ctx0, bx, conv->ne[0], bx->ne[1], bx->ne[2], bx->nb[1], bx->nb[2],
-                                       (bx->ne[0] - conv->ne[0]) * ggml_element_size(bx));
-        GGML_ASSERT(ggml_are_same_shape(conv, new_conv));
+        // write conv states: slot 0 = the final state, slot s = the state s tokens back (partial rollback)
+        const int64_t K         = hparams.causal_attn && cparams.n_rs_seq > 0 ? (int64_t) cparams.n_rs_seq + 1 : 1;
+        const int64_t n_written = std::min<int64_t>(n_seq_tokens, K);
+        const auto    mem_size  = mctx_cur->get_size();
+        const size_t  row_size  = ggml_row_size(conv_state->type, (int64_t) d_conv * n_embd);
 
-        // write new conv conv state
-        ggml_build_forward_expand(gf, ggml_cpy(ctx0, new_conv,
-                                               ggml_view_1d(ctx0, conv_state, ggml_nelements(new_conv),
-                                                            kv_head * d_conv * n_embd * ggml_element_size(new_conv))));
+        for (int64_t slot = 0; slot < n_written; ++slot) {
+            auto * conv_snap = ggml_view_3d(ctx0, bx, d_conv, bx->ne[1], bx->ne[2], bx->nb[1], bx->nb[2],
+                                            (bx->ne[0] - d_conv - slot) * ggml_element_size(bx));
+            ggml_build_forward_expand(gf, ggml_cpy(ctx0, conv_snap,
+                                                   ggml_view_2d(ctx0, conv_state, (int64_t) d_conv * n_embd, n_seqs,
+                                                                conv_state->nb[1],
+                                                                ((size_t) slot * mem_size + kv_head) * row_size)));
+        }
 
         auto * conv_kernel = model.layers[il].shortconv.conv;
         auto * conv_out    = ggml_ssm_conv(ctx0, bx, conv_kernel);
@@ -242,6 +249,8 @@ llama_model_lfm2::graph<iswa>::graph(const llama_model & model, const llm_graph_
     ggml_tensor * inp_out_ids = build_inp_out_ids();
 
     for (int il = 0; il < n_layer; ++il) {
+        res->t_layer_inp[il] = cur;
+
         const bool is_moe_layer = il >= static_cast<int>(hparams.n_layer_dense_lead);
 
         auto * prev_cur = cur;

src/models/minicpm3.cpp

@@ -115,19 +115,9 @@ llama_model_minicpm3::graph::graph(const llama_model & model, const llm_graph_pa
             q = ggml_mul_mat(ctx0, model.layers[il].wq_b, q);
             cb(q, "q", il);
 
-            // split into {n_head * n_embd_head_qk_nope, n_tokens}
-            ggml_tensor * q_nope = ggml_view_3d(ctx0, q, n_embd_head_qk_nope, n_head, n_tokens,
-                    ggml_row_size(q->type, hparams.n_embd_head_k()),
-                    ggml_row_size(q->type, hparams.n_embd_head_k() * n_head),
-                    0);
-            cb(q_nope, "q_nope", il);
-
-            // and {n_head * n_embd_head_qk_rope, n_tokens}
-            ggml_tensor * q_pe = ggml_view_3d(ctx0, q, n_embd_head_qk_rope, n_head, n_tokens,
-                    ggml_row_size(q->type, hparams.n_embd_head_k()),
-                    ggml_row_size(q->type, hparams.n_embd_head_k() * n_head),
-                    ggml_row_size(q->type, n_embd_head_qk_nope));
-            cb(q_pe, "q_pe", il);
+            // {n_embd_head_k, n_head, n_tokens}, RoPE is applied to the trailing dims only
+            q = ggml_reshape_3d(ctx0, q, hparams.n_embd_head_k(), n_head, n_tokens);
+            cb(q, "q", il);
 
             // {n_embd, kv_lora_rank + n_embd_head_qk_rope} * {n_embd, n_tokens} -> {kv_lora_rank + n_embd_head_qk_rope, n_tokens}
             ggml_tensor * kv_pe_compresseed = ggml_mul_mat(ctx0, model.layers[il].wkv_a_mqa, cur);
@@ -172,12 +162,13 @@ llama_model_minicpm3::graph::graph(const llama_model & model, const llm_graph_pa
             v_states = ggml_cont(ctx0, v_states);
             cb(v_states, "v_states", il);
 
-            q_pe = ggml_rope_ext(
-                    ctx0, q_pe, inp_pos, rope_factors,
+            q = ggml_rope_ext(
+                    ctx0, q, inp_pos, rope_factors,
                     n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
                     ext_factor, attn_factor, beta_fast, beta_slow
                     );
-            cb(q_pe, "q_pe", il);
+            q = ggml_rope_set_offset(q, n_embd_head_qk_nope);
+            cb(q, "q_rope", il);
 
             // shared RoPE key
             k_pe = ggml_rope_ext(
@@ -187,10 +178,11 @@ llama_model_minicpm3::graph::graph(const llama_model & model, const llm_graph_pa
                     );
             cb(k_pe, "k_pe", il);
 
-            ggml_tensor * q_states = ggml_concat(ctx0, q_nope, q_pe, 0);
+            ggml_tensor * q_states = q;
             cb(q_states, "q_states", il);
 
-            ggml_tensor * k_states = ggml_concat(ctx0, k_nope, ggml_repeat(ctx0, k_pe, q_pe), 0);
+            ggml_tensor * k_states = ggml_concat(ctx0, k_nope,
+                    ggml_repeat_4d(ctx0, k_pe, n_embd_head_qk_rope, n_head, n_tokens, 1), 0);
             cb(k_states, "k_states", il);
 
             cur = build_attn(inp_attn,

src/models/models.h

@@ -1156,6 +1156,18 @@ struct llama_model_deepseek32 : public llama_model_base {
 };
 
 
+struct llama_model_dots3note : public llama_model_base {
+    llama_model_dots3note(const struct llama_model_params & params) : llama_model_base(params) {}
+    void load_arch_hparams(llama_model_loader & ml) override;
+    void load_arch_tensors(llama_model_loader & ml) override;
+
+    struct graph : public llm_graph_context {
+        graph(const llama_model & model, const llm_graph_params & params);
+    };
+
+    std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;
+};
+
 struct llama_model_deepseek4 : public llama_model_base {
     llama_model_deepseek4(const struct llama_model_params & params) : llama_model_base(params) {}
     void load_arch_hparams(llama_model_loader & ml) override;

src/models/plm.cpp

@@ -81,19 +81,9 @@ llama_model_plm::graph::graph(const llama_model & model, const llm_graph_params
             q = ggml_mul_mat(ctx0, model.layers[il].wq, cur);
             cb(q, "q", il);
 
-            // split into {n_head * n_embd_head_qk_nope, n_tokens}
-            ggml_tensor * q_nope = ggml_view_3d(ctx0, q, n_embd_head_qk_nope, n_head, n_tokens,
-                    ggml_row_size(q->type, hparams.n_embd_head_k()),
-                    ggml_row_size(q->type, hparams.n_embd_head_k() * n_head),
-                    0);
-            cb(q_nope, "q_nope", il);
-
-            // and {n_head * n_embd_head_qk_rope, n_tokens}
-            ggml_tensor * q_pe = ggml_view_3d(ctx0, q, n_embd_head_qk_rope, n_head, n_tokens,
-                    ggml_row_size(q->type, hparams.n_embd_head_k()),
-                    ggml_row_size(q->type, hparams.n_embd_head_k() * n_head),
-                    ggml_row_size(q->type, n_embd_head_qk_nope));
-            cb(q_pe, "q_pe", il);
+            // {n_embd_head_k, n_head, n_tokens}, RoPE is applied to the trailing dims only
+            q = ggml_reshape_3d(ctx0, q, hparams.n_embd_head_k(), n_head, n_tokens);
+            cb(q, "q", il);
 
             // {n_embd, kv_lora_rank + n_embd_head_qk_rope} * {n_embd, n_tokens} -> {kv_lora_rank + n_embd_head_qk_rope, n_tokens}
             ggml_tensor * kv_pe_compresseed = ggml_mul_mat(ctx0, model.layers[il].wkv_a_mqa, cur);
@@ -143,12 +133,13 @@ llama_model_plm::graph::graph(const llama_model & model, const llm_graph_params
                     0);
             cb(v_states, "v_states", il);
 
-            q_pe = ggml_rope_ext(
-                    ctx0, q_pe, inp_pos, nullptr,
+            q = ggml_rope_ext(
+                    ctx0, q, inp_pos, nullptr,
                     n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
                     ext_factor, attn_factor, beta_fast, beta_slow
                     );
-            cb(q_pe, "q_pe", il);
+            q = ggml_rope_set_offset(q, n_embd_head_qk_nope);
+            cb(q, "q_rope", il);
 
             // shared RoPE key
             k_pe = ggml_rope_ext(
@@ -158,10 +149,11 @@ llama_model_plm::graph::graph(const llama_model & model, const llm_graph_params
                     );
             cb(k_pe, "k_pe", il);
 
-            ggml_tensor * q_states = ggml_concat(ctx0, q_nope, q_pe, 0);
+            ggml_tensor * q_states = q;
             cb(q_states, "q_states", il);
 
-            ggml_tensor * k_states = ggml_concat(ctx0, k_nope, ggml_repeat(ctx0, k_pe, q_pe), 0);
+            ggml_tensor * k_states = ggml_concat(ctx0, k_nope,
+                    ggml_repeat_4d(ctx0, k_pe, n_embd_head_qk_rope, n_head, n_tokens, 1), 0);
             cb(k_states, "k_states", il);
 
             cur = build_attn(inp_attn,

tests/test-backend-ops.cpp

@@ -7085,9 +7085,10 @@ struct test_flash_attn_ext : public test_case {
     const ggml_type type_V;
     std::array<int32_t, 4> permute;
     const bool kv_view; // create K/V as views of a larger buffer (like a KV cache)
+    const bool v_is_view_of_k;
 
     std::string vars() override {
-        return VARS_TO_STR15(hsk, hsv, nh, nr23, kv, nb, mask, sinks, max_bias, logit_softcap, prec, type_K, type_V, permute, kv_view);
+        return VARS_TO_STR16(hsk, hsv, nh, nr23, kv, nb, mask, sinks, max_bias, logit_softcap, prec, type_K, type_V, permute, kv_view, v_is_view_of_k);
     }
 
     double max_nmse_err() override {
@@ -7104,9 +7105,9 @@ struct test_flash_attn_ext : public test_case {
     test_flash_attn_ext(int64_t hsk = 128, int64_t hsv = 128, int64_t nh = 32, std::array<int64_t, 2> nr23 = {1, 1}, int64_t kv = 96, int64_t nb = 8,
                         bool mask = true, bool sinks = false, float max_bias = 0.0f, float logit_softcap = 0.0f, ggml_prec prec = GGML_PREC_F32,
                         ggml_type type_K = GGML_TYPE_F16, ggml_type type_V = GGML_TYPE_F16, std::array<int32_t, 4> permute = {0, 1, 2, 3},
-                        bool kv_view = true)
+                        bool kv_view = true, bool v_is_view_of_k = false)
         : hsk(hsk), hsv(hsv), nh(nh), nr23(nr23), kv(kv), nb(nb), mask(mask), sinks(sinks), max_bias(max_bias), logit_softcap(logit_softcap), prec(prec),
-          type_K(type_K), type_V(type_V), permute(permute), kv_view(kv_view) {}
+          type_K(type_K), type_V(type_V), permute(permute), kv_view(kv_view), v_is_view_of_k(v_is_view_of_k) {}
 
     ggml_tensor * build_graph(ggml_context * ctx) override {
         const int64_t hsk_padded = GGML_PAD(hsk, ggml_blck_size(type_K));
@@ -7138,14 +7139,14 @@ struct test_flash_attn_ext : public test_case {
         ggml_set_name(k, "k");
 
         ggml_tensor * v = nullptr;
-        if (type_K == type_V && hsk_padded == 576 && hsv_padded == 512) {
-            // TODO: this branch should become a separate test case parameter instead of hardcoding this for these head shapes
-
-            // in this branch, the V cache is sub-view of the K cache. this is used by some MLA-based models
+        if (v_is_view_of_k) {
+            // the V cache is a sub-view of the K cache. this is used by some MLA-based models
             // for more info:
             //   - https://github.com/ggml-org/llama.cpp/pull/13435
             //   - https://github.com/ggml-org/llama.cpp/pull/18953#issuecomment-3774948392
             //   - https://github.com/ggml-org/llama.cpp/pull/18986
+            GGML_ASSERT(type_K == type_V && hsv_padded <= hsk_padded);
+
             v = ggml_view_4d(ctx, k, hsv_padded, kv, nh, nr23[1], k->nb[1], k->nb[2], k->nb[3], 0);
         } else {
             v = create_permuted(type_V,        hsv_padded, kv, nh,         nr23[1], kv_view); // the V tensor is usually a view of the V cache
@@ -9298,6 +9299,14 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {
 
     test_cases.emplace_back(new test_mul_mat(GGML_TYPE_Q8_0, GGML_TYPE_F32, 6, 4096, 5120, {1, 1}, {1, 1}));
 
+    // K not a multiple of 32
+    test_cases.emplace_back(new test_mul_mat(GGML_TYPE_F16, GGML_TYPE_F16, 64, 32,  65, {1, 1}, {1, 1}));
+    test_cases.emplace_back(new test_mul_mat(GGML_TYPE_F16, GGML_TYPE_F16, 64, 32,  80, {1, 1}, {1, 1}));
+    test_cases.emplace_back(new test_mul_mat(GGML_TYPE_F16, GGML_TYPE_F32, 64, 32,  80, {1, 1}, {1, 1}));
+    test_cases.emplace_back(new test_mul_mat(GGML_TYPE_F32, GGML_TYPE_F32, 64, 32,  80, {1, 1}, {1, 1}));
+    test_cases.emplace_back(new test_mul_mat(GGML_TYPE_F16, GGML_TYPE_F16, 64, 32, 588, {1, 1}, {1, 1})); // 14*14*3, e.g. conv_2d im2col
+    test_cases.emplace_back(new test_mul_mat(GGML_TYPE_F16, GGML_TYPE_F16, 64, 32,  80, {4, 1}, {1, 1}));
+
 #if 0
     // test the mat-mat path for Metal
     for (int k = 1; k < 512; ++k) {
@@ -9898,12 +9907,14 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {
                                                     if (hsk != 128 && prec == GGML_PREC_DEFAULT) continue;
                                                     for (ggml_type type_KV : {GGML_TYPE_F32, GGML_TYPE_F16, GGML_TYPE_BF16, GGML_TYPE_Q8_0, GGML_TYPE_Q5_1, GGML_TYPE_Q5_0, GGML_TYPE_Q4_1, GGML_TYPE_Q4_0, GGML_TYPE_IQ4_NL}) {
                                                         if (type_KV != GGML_TYPE_F16 && hsk != 64 && hsk != 72) continue;
+                                                        // DeepSeek MLA: the V cache is a sub-view of the K cache
+                                                        const bool v_is_view_of_k = hsk == 576;
                                                         test_cases.emplace_back(new test_flash_attn_ext(
-                                                                    hsk, hsv, nh, {nr2, nr3}, kv, nb, mask, sinks, max_bias, logit_softcap, prec, type_KV, type_KV));
+                                                                    hsk, hsv, nh, {nr2, nr3}, kv, nb, mask, sinks, max_bias, logit_softcap, prec, type_KV, type_KV, {0, 1, 2, 3}, true, v_is_view_of_k));
                                                         // run fewer test cases permuted
                                                         if (mask == true && max_bias == 0.0f && logit_softcap == 0 && kv == 512) {
                                                             test_cases.emplace_back(new test_flash_attn_ext(
-                                                                        hsk, hsv, nh, {nr2, nr3}, kv, nb, mask, sinks, max_bias, logit_softcap, prec, type_KV, type_KV, {0, 2, 1, 3}));
+                                                                        hsk, hsv, nh, {nr2, nr3}, kv, nb, mask, sinks, max_bias, logit_softcap, prec, type_KV, type_KV, {0, 2, 1, 3}, true, v_is_view_of_k));
                                                         }
                                                     }
                                                 }
@@ -9942,11 +9953,16 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {
     test_cases.emplace_back(new test_flash_attn_ext(256, 256, 2, {16, 1},  1025,  64, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q8_0, GGML_TYPE_Q8_0, {0, 2, 1, 3}));
     test_cases.emplace_back(new test_flash_attn_ext(256, 256, 2, {16, 1}, 16384,   1, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q8_0, GGML_TYPE_Q8_0));
 
-    // MLA shape (V is a view of K) with quantized KV
-    // (the test harness builds V as a view of K for this shape; see build_graph)
-    test_cases.emplace_back(new test_flash_attn_ext(576, 512, 1, {20, 1},  113,   1, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q8_0, GGML_TYPE_Q8_0));
-    test_cases.emplace_back(new test_flash_attn_ext(576, 512, 1, {20, 1}, 1024,   1, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q8_0, GGML_TYPE_Q8_0));
-    test_cases.emplace_back(new test_flash_attn_ext(576, 512, 1, {20, 1}, 1024,  64, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q8_0, GGML_TYPE_Q8_0));
+    // MLA shape: the V cache is a sub-view of the K cache, with quantized KV
+    test_cases.emplace_back(new test_flash_attn_ext(576, 512, 1, {20, 1},  113,   1, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q8_0, GGML_TYPE_Q8_0, {0, 1, 2, 3}, true, true));
+    test_cases.emplace_back(new test_flash_attn_ext(576, 512, 1, {20, 1}, 1024,   1, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q8_0, GGML_TYPE_Q8_0, {0, 1, 2, 3}, true, true));
+    test_cases.emplace_back(new test_flash_attn_ext(576, 512, 1, {20, 1}, 1024,  64, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q8_0, GGML_TYPE_Q8_0, {0, 1, 2, 3}, true, true));
+
+    // more V-is-sub-view-of-K cases: other head shapes, and full views with equal head sizes
+    test_cases.emplace_back(new test_flash_attn_ext(320, 256, 1, {32, 1}, 512, 1, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16, {0, 1, 2, 3}, true, true));
+    test_cases.emplace_back(new test_flash_attn_ext(192, 128, 4, {8, 1},  512, 1, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16, {0, 1, 2, 3}, true, true));
+    test_cases.emplace_back(new test_flash_attn_ext(128, 128, 8, {4, 1},  512, 8, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16, {0, 1, 2, 3}, true, true));
+    test_cases.emplace_back(new test_flash_attn_ext(64,  64,  4, {1, 1},  512, 1, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q8_0, GGML_TYPE_Q8_0, {0, 1, 2, 3}, true, true));
 
     // large-KV F16 cases (Qwen3.6-27B geometry and a llama-class control): the upstream matrix
     // stops at kv=1024, blind to long-context FA bugs (e.g. the oneDNN SDPA ordering race on BMG).

tests/test-llama-archs.cpp

@@ -104,6 +104,7 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) {
     } else if (arch == LLM_ARCH_DEEPSEEK2
             || arch == LLM_ARCH_DEEPSEEK32
             || arch == LLM_ARCH_GLM_DSA
+            || arch == LLM_ARCH_DOTS3NOTE
             || arch == LLM_ARCH_KIMI_LINEAR
             || arch == LLM_ARCH_BAILINGMOE3
             || arch == LLM_ARCH_KIMI_K3
@@ -166,6 +167,7 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) {
     if (arch == LLM_ARCH_DEEPSEEK2
             || arch == LLM_ARCH_DEEPSEEK32
             || arch == LLM_ARCH_GLM_DSA
+            || arch == LLM_ARCH_DOTS3NOTE
             || arch == LLM_ARCH_KIMI_LINEAR
             || arch == LLM_ARCH_BAILINGMOE3
             || arch == LLM_ARCH_KIMI_K3
@@ -175,6 +177,22 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) {
         ms.add_kv(LLM_KV_ROPE_DIMENSION_COUNT,       uint32_t(64));
         ms.add_kv(LLM_KV_ATTENTION_KEY_LENGTH_MLA,   uint32_t(192));
         ms.add_kv(LLM_KV_ATTENTION_VALUE_LENGTH_MLA, uint32_t(128));
+        if (arch == LLM_ARCH_DOTS3NOTE) {
+            // SWA layers reuse the same MLA geometry as the full layers in this fixture
+            ms.add_kv(LLM_KV_ATTENTION_KV_LORA_RANK_SWA,     uint32_t(512));
+            ms.add_kv(LLM_KV_ATTENTION_KEY_LENGTH_SWA,       uint32_t(576));
+            ms.add_kv(LLM_KV_ATTENTION_VALUE_LENGTH_SWA,     uint32_t(512));
+            ms.add_kv(LLM_KV_ATTENTION_KEY_LENGTH_MLA_SWA,   uint32_t(192));
+            ms.add_kv(LLM_KV_ATTENTION_VALUE_LENGTH_MLA_SWA, uint32_t(128));
+            ms.add_kv(LLM_KV_ROPE_FREQ_BASE_SWA,             10000.0f);
+            // indexer on the full-attention layers (inverse of the swa pattern)
+            std::vector<uint32_t> indexer_types;
+            indexer_types.reserve(n_layer);
+            for (uint32_t il = 0; il < n_layer; il++) {
+                indexer_types.push_back(il % 2 ? 0 : 1);
+            }
+            ms.add_kv(LLM_KV_ATTENTION_INDEXER_TYPES, indexer_types);
+        }
     } else if (arch == LLM_ARCH_MINIMAX_M3) {
         // partial rotary: n_rot must not exceed the indexer key length (64)
         ms.add_kv(LLM_KV_ROPE_DIMENSION_COUNT,       uint32_t(64));
@@ -197,7 +215,8 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) {
         ms.add_kv(LLM_KV_ROPE_FREQ_BASE_SWA,              10000.0f);
         // SWA pattern: every 5th layer is full attention (matches E2B layer_types)
         ms.add_kv(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, uint32_t(5));
-    } else if (arch == LLM_ARCH_COHERE2MOE || arch == LLM_ARCH_MIMO2 || arch == LLM_ARCH_STEP35 || arch == LLM_ARCH_MUSE_GLIMMER || arch == LLM_ARCH_GRANITE_SWA) {
+    } else if (arch == LLM_ARCH_COHERE2MOE || arch == LLM_ARCH_MIMO2 || arch == LLM_ARCH_STEP35 ||
+            arch == LLM_ARCH_MUSE_GLIMMER || arch == LLM_ARCH_GRANITE_SWA || arch == LLM_ARCH_DOTS3NOTE) {
         std::vector<uint32_t> pattern;
         pattern.reserve(n_layer);
         for (uint32_t il = 0; il < n_layer; il++) {
@@ -365,6 +384,7 @@ static bool moe_mandatory(const llm_arch arch) {
         case LLM_ARCH_DEEPSEEK:
         case LLM_ARCH_DEEPSEEK2:
         case LLM_ARCH_DEEPSEEK32:
+        case LLM_ARCH_DOTS3NOTE:
         case LLM_ARCH_GLM4_MOE:
         case LLM_ARCH_GLM_DSA:
         case LLM_ARCH_EXAONE_MOE:

tools/cli/README.md

@@ -162,6 +162,7 @@
 | `-mmu, --mmproj-url URL` | URL to a multimodal projector file. see tools/mtmd/README.md<br/>(env: LLAMA_ARG_MMPROJ_URL) |
 | `--mmproj-auto, --no-mmproj, --no-mmproj-auto` | whether to use multimodal projector file (if available), useful when using -hf (default: enabled)<br/>(env: LLAMA_ARG_MMPROJ_AUTO) |
 | `--mmproj-offload, --no-mmproj-offload` | whether to enable GPU offloading for multimodal projector (default: enabled)<br/>(env: LLAMA_ARG_MMPROJ_OFFLOAD) |
+| `-mmdev, --mmproj-device DEVICE` | device to use for multimodal projector (none = don't offload, default: auto)<br/>use --list-devices to see a list of available devices<br/>(env: MTMD_BACKEND_DEVICE) |
 | `--image, --audio, --video FILE` | path to an image, audio, or video file. use with multimodal models, use comma-separated values for multiple files |
 | `--image-min-tokens N` | minimum number of tokens each image can take, only used by vision models with dynamic resolution (default: read from model)<br/>(env: LLAMA_ARG_IMAGE_MIN_TOKENS) |
 | `--image-max-tokens N` | maximum number of tokens each image can take, only used by vision models with dynamic resolution (default: read from model)<br/>(env: LLAMA_ARG_IMAGE_MAX_TOKENS) |

tools/mtmd/clip.cpp

@@ -186,14 +186,13 @@ struct clip_ctx {
             throw std::runtime_error("failed to initialize CPU backend");
         }
         if (ctx_params.use_gpu) {
-            auto * backend_name = std::getenv("MTMD_BACKEND_DEVICE");
-            if (backend_name != nullptr) {
-                backend = ggml_backend_init_by_name(backend_name, nullptr);
+            if (ctx_params.device != nullptr) {
+                backend = ggml_backend_dev_init(ctx_params.device, nullptr);
                 if (!backend) {
-                    LOG_WRN("%s: Warning: Failed to initialize \"%s\" backend, falling back to default GPU backend\n", __func__, backend_name);
+                    throw std::runtime_error(string_format("%s: failed to initialize \"%s\" backend\n",
+                                                           __func__, ggml_backend_dev_name(ctx_params.device)));
                 }
-            }
-            if (!backend) {
+            } else {
                 backend = ggml_backend_init_by_type(GGML_BACKEND_DEVICE_TYPE_GPU, nullptr);
                 backend = backend ? backend : ggml_backend_init_by_type(GGML_BACKEND_DEVICE_TYPE_IGPU, nullptr);
             }

tools/mtmd/clip.h

@@ -48,6 +48,7 @@ enum clip_flash_attn_type {
 
 struct clip_context_params {
     bool use_gpu;
+    ggml_backend_dev_t device;
     enum clip_flash_attn_type flash_attn_type;
     int image_min_tokens;
     int image_max_tokens;

tools/mtmd/debug/mtmd-debug.cpp

@@ -84,6 +84,7 @@ int main(int argc, char ** argv) {
         const char * clip_path = params.mmproj.path.c_str();
         mtmd_context_params mparams = mtmd_context_params_default();
         mparams.use_gpu          = params.mmproj_use_gpu;
+        mparams.device           = params.mmproj_device;
         mparams.print_timings    = true;
         mparams.n_threads        = params.cpuparams.n_threads;
         mparams.flash_attn_type  = params.flash_attn_type;

tools/mtmd/mtmd-cli.cpp

@@ -154,6 +154,7 @@ struct mtmd_cli_context {
         const char * clip_path = params.mmproj.path.c_str();
         mtmd_context_params mparams = mtmd_context_params_default();
         mparams.use_gpu          = params.mmproj_use_gpu;
+        mparams.device           = params.mmproj_device;
         mparams.print_timings    = true;
         mparams.n_threads        = params.cpuparams.n_threads;
         mparams.flash_attn_type  = params.flash_attn_type;

tools/mtmd/mtmd.cpp

@@ -456,6 +456,7 @@ static clip_flash_attn_type mtmd_get_clip_flash_attn_type(enum llama_flash_attn_
 mtmd_context_params mtmd_context_params_default() {
     mtmd_context_params params {
         /* use_gpu           */ true,
+        /* device            */ nullptr,
         /* print_timings     */ true,
         /* n_threads         */ 4,
         /* image_marker      */ nullptr,
@@ -564,6 +565,7 @@ struct mtmd_context {
 
         clip_context_params ctx_clip_params {
             /* use_gpu           */ ctx_params.use_gpu,
+            /* device            */ ctx_params.device,
             /* flash_attn_type   */ mtmd_get_clip_flash_attn_type(ctx_params.flash_attn_type),
             /* image_min_tokens  */ ctx_params.image_min_tokens,
             /* image_max_tokens  */ ctx_params.image_max_tokens,

tools/mtmd/mtmd.h

@@ -89,6 +89,7 @@ typedef bool (*mtmd_progress_callback)(float progress, void * user_data);
 
 struct mtmd_context_params {
     bool use_gpu;
+    ggml_backend_dev_t device;
     bool print_timings;
     int n_threads;
     const char * image_marker; // deprecated, use media_marker instead

tools/server/README.md

@@ -178,6 +178,7 @@ For the full list of features, please refer to [server's changelog](https://gith
 | `-mmu, --mmproj-url URL` | URL to a multimodal projector file. see tools/mtmd/README.md<br/>(env: LLAMA_ARG_MMPROJ_URL) |
 | `--mmproj-auto, --no-mmproj, --no-mmproj-auto` | whether to use multimodal projector file (if available), useful when using -hf (default: enabled)<br/>(env: LLAMA_ARG_MMPROJ_AUTO) |
 | `--mmproj-offload, --no-mmproj-offload` | whether to enable GPU offloading for multimodal projector (default: enabled)<br/>(env: LLAMA_ARG_MMPROJ_OFFLOAD) |
+| `-mmdev, --mmproj-device DEVICE` | device to use for multimodal projector (none = don't offload, default: auto)<br/>use --list-devices to see a list of available devices<br/>(env: MTMD_BACKEND_DEVICE) |
 | `--image-min-tokens N` | minimum number of tokens each image can take, only used by vision models with dynamic resolution (default: read from model)<br/>(env: LLAMA_ARG_IMAGE_MIN_TOKENS) |
 | `--image-max-tokens N` | maximum number of tokens each image can take, only used by vision models with dynamic resolution (default: read from model)<br/>(env: LLAMA_ARG_IMAGE_MAX_TOKENS) |
 | `--mtmd-batch-max-tokens N` | maximum number of image tokens per batch when encoding images (default: 1024)<br/>(env: LLAMA_ARG_MTMD_BATCH_MAX_TOKENS) |
@@ -196,11 +197,11 @@ For the full list of features, please refer to [server's changelog](https://gith
 | `--ui-config, --webui-config JSON` | JSON that provides default UI settings (overrides UI defaults)<br/>(env: LLAMA_ARG_UI_CONFIG) |
 | `--ui-config-file, --webui-config-file PATH` | JSON file that provides default UI settings (overrides UI defaults)<br/>(env: LLAMA_ARG_UI_CONFIG_FILE) |
 | `--ui-mcp-proxy, --webui-mcp-proxy, --no-ui-mcp-proxy, --no-webui-mcp-proxy` | experimental: whether to enable MCP CORS proxy - do not enable in untrusted environments (default: disabled)<br/>(env: LLAMA_ARG_UI_MCP_PROXY) |
-| `--tools TOOL1,TOOL2,...` | experimental: whether to enable server tools for AI agents - do not enable in untrusted environments (default: no tools)<br/>specify "all" to enable all tools<br/>available tools: read_file, file_glob_search, grep_search, exec_shell_command, write_file, edit_file, get_info<br/>note: for security reasons, this will limit --cors-origins to localhost by default<br/>(env: LLAMA_ARG_TOOLS) |
+| `--tools TOOL1,TOOL2,...` | experimental: whether to enable built-in tools for AI agents - do not enable in untrusted environments (default: no tools)<br/>specify "all" to enable all tools<br/>available tools: read_file, file_glob_search, grep_search, exec_shell_command, write_file, edit_file, get_info<br/>note: for security reasons, this will limit --cors-origins to localhost by default<br/>(env: LLAMA_ARG_TOOLS) |
 | `--tools-runtime OPTION` | experimental: run tools in a separate runtime environment (default: none, use host environment)<br/>available options:<br/>  'docker:<image>', 'podman:<image>': spin up a new container and reuse it for all invocations, clean up on server exit<br/>  'docker-container:<id>', 'podman-container:<id>': use an existing container by ID, won't stop on server exit<br/>  'ssh:<target>': run tools on a remote POSIX host over SSH, key-based auth and a trusted host key are required<br/><br/>(env: LLAMA_ARG_TOOLS_RUNTIME) |
 | `--mcp-servers-config PATH` | experimental: path to JSON file with MCP server definitions (Cursor-compatible format) - do not enable in untrusted environments (default: none)<br/>note: for security reasons, this will limit --cors-origins to localhost by default<br/>(env: LLAMA_ARG_MCP_SERVERS_CONFIG) |
 | `--mcp-servers-json JSON` | experimental: inline JSON with MCP server definitions (Cursor-compatible format) - do not enable in untrusted environments (default: none)<br/>note: for security reasons, this will limit --cors-origins to localhost by default<br/>(env: LLAMA_ARG_MCP_SERVERS_JSON) |
-| `-ag, --agent, -no-ag, --no-agent` | whether to enable CORS proxy and all server tools - do not enable in untrusted environments (default: disabled)<br/>note: for security reasons, this will limit --cors-origins to localhost by default<br/>(env: LLAMA_ARG_AGENT) |
+| `-ag, --agent, -no-ag, --no-agent` | whether to enable CORS proxy and all built-in tools - do not enable in untrusted environments (default: disabled)<br/>note: for security reasons, this will limit --cors-origins to localhost by default<br/>(env: LLAMA_ARG_AGENT) |
 | `--ui, --webui, --no-ui, --no-webui` | whether to enable the Web UI (default: enabled)<br/>(env: LLAMA_ARG_UI) |
 | `--embedding, --embeddings` | restrict to only support embedding use case; use only with dedicated embedding models (default: disabled)<br/>(env: LLAMA_ARG_EMBEDDINGS) |
 | `--rerank, --reranking` | enable reranking endpoint on server (default: disabled)<br/>(env: LLAMA_ARG_RERANKING) |
@@ -1757,7 +1758,7 @@ The precedence rule for preset options is as follows:
 3. **Global options** defined in the preset file (`[*]`)
 
 We also offer additional options that are exclusive to presets (these aren't treated as command-line arguments):
-- `load-on-startup` (boolean): Controls whether the model loads automatically when the server starts
+- `load-on-startup` (boolean): Controls whether the model loads automatically when the server starts. Only applies at startup: if the model list is reloaded later (for example after editing the preset file), a newly added model is listed but not loaded
 - `stop-timeout` (int, seconds): After requested unload, wait for this many seconds before forcing termination (default: 10)
 - `dedup-cache-models` (boolean): When the preset uses `hf-repo` pointing to a model that is already downloaded, hide the corresponding cached model entry from `GET /models` (the preset entry remains visible). Set it in the `[*]` section to apply to all presets.
 

tools/server/server-context.cpp

@@ -998,6 +998,7 @@ private:
         mtmd_context_params mparams = mtmd_context_params_default();
         if (has_mmproj) {
             mparams.use_gpu          = params_base.mmproj_use_gpu;
+            mparams.device           = params_base.mmproj_device;
             mparams.print_timings    = false;
             mparams.n_threads        = params_base.cpuparams.n_threads;
             mparams.flash_attn_type  = params_base.flash_attn_type;

tools/server/server-models.cpp

@@ -672,24 +672,26 @@ void server_models::load_models() {
         apply_hidden();
         log_available_models();
 
-        std::vector<std::string> models_to_load;
-        for (const auto & [name, inst] : mapping) {
-            std::string val;
-            if (inst.meta.preset.get_option(COMMON_ARG_PRESET_LOAD_ON_STARTUP, val) && common_arg_utils::is_truthy(val)) {
-                models_to_load.push_back(name);
+        // skipped on reload, see startup_models
+        if (startup_models.has_value()) {
+            std::vector<std::string> models_to_load;
+            for (const auto & [name, inst] : mapping) {
+                std::string val;
+                if (inst.meta.preset.get_option(COMMON_ARG_PRESET_LOAD_ON_STARTUP, val) && common_arg_utils::is_truthy(val)) {
+                    models_to_load.push_back(name);
+                }
             }
-        }
-        if ((int)models_to_load.size() > base_params.models_max) {
-            throw std::runtime_error(string_format(
-                "number of models to load on startup (%zu) exceeds models_max (%d)",
-                models_to_load.size(), base_params.models_max));
+            if ((int)models_to_load.size() > base_params.models_max) {
+                throw std::runtime_error(string_format(
+                    "number of models to load on startup (%zu) exceeds models_max (%d)",
+                    models_to_load.size(), base_params.models_max));
+            }
+
+            // to be lazy-loaded after main() setup phase is completed
+            startup_models = std::move(models_to_load);
         }
 
         lk.unlock();
-        for (const auto & name : models_to_load) {
-            SRV_INF("(startup) loading model %s\n", name.c_str());
-            load(name);
-        }
     } else {
         // RELOAD: diff the new preset list against the current mapping and reconcile
         is_reloading = true;
@@ -819,8 +821,8 @@ void server_models::load_models() {
             inst.meta.update_caps();
         }
 
-        // add models that are new in this reload
-        std::vector<std::string> newly_added;
+        // add models that are new in this reload, load-on-startup is not honored here since a
+        // reload never spawns an instance
         for (const auto & [name, preset] : final_presets) {
             if (mapping.find(name) == mapping.end()) {
                 server_model_meta meta{
@@ -841,42 +843,40 @@ void server_models::load_models() {
                     // /* need_download */ false,
                 };
                 add_model(std::move(meta));
-                newly_added.push_back(name);
             }
         }
 
         apply_stop_timeout();
         apply_hidden();
 
-        // clear reload flag before unlocking for autoload - load() blocks on !is_reloading,
-        // so clearing it here (while still locked) prevents a deadlock in the autoload calls below
+        // clear reload flag under the lock, this releases the load() calls waiting on !is_reloading
         is_reloading = false;
         cv.notify_all();
 
         log_available_models();
 
-        // collect autoload candidates while still under the lock
-        std::vector<std::string> to_autoload;
-        for (const auto & name : newly_added) {
-            auto it = mapping.find(name);
-            if (it != mapping.end()) {
-                std::string val;
-                if (it->second.meta.preset.get_option(COMMON_ARG_PRESET_LOAD_ON_STARTUP, val) && common_arg_utils::is_truthy(val)) {
-                    to_autoload.push_back(name);
-                }
-            }
-        }
-
         lk.unlock();
-        for (const auto & name : to_autoload) {
-            SRV_INF("(reload) loading new model %s\n", name.c_str());
-            load(name);
-        }
 
         notify_sse("models_reload", "*");
     }
 }
 
+void server_models::load_startup_models() {
+    std::vector<std::string> to_load;
+    {
+        std::lock_guard<std::mutex> lk(mutex);
+        if (!startup_models.has_value()) {
+            return; // already drained
+        }
+        to_load = std::move(*startup_models);
+        startup_models.reset();
+    }
+    for (const auto & name : to_load) {
+        SRV_INF("(startup) loading model %s\n", name.c_str());
+        load(name);
+    }
+}
+
 void server_models::update_meta(const std::string & name, const server_model_meta & meta) {
     std::lock_guard<std::mutex> lk(mutex);
     auto it = mapping.find(name);

tools/server/server-models.h

@@ -136,6 +136,10 @@ private:
     // if true, the next get_meta() will trigger a reload of model list
     bool need_reload = false;
 
+    // models marked with load-on-startup, unset once load_startup_models() drains it
+    // no value means the startup phase is over, so a reload must not queue anything
+    std::optional<std::vector<std::string>> startup_models{std::in_place};
+
     // conv_id -> model name that currently serves its stream session, lets the resumable stream
     // routes go straight to the owning child instead of polling every one. populated when
     // proxy_request forwards a POST carrying an X-Conversation-Id. best effort: a stale entry just
@@ -231,6 +235,9 @@ public:
     //   - if a model is not running, it will be added or updated according to the source
     void load_models();
 
+    // lazy-load startup_models, to be called after main() setup phase
+    void load_startup_models();
+
     // check if a model instance exists (thread-safe)
     bool has_model(const std::string & name);
 

tools/server/server.cpp

@@ -133,7 +133,8 @@ int llama_server(common_params & params, int argc, char ** argv) {
 
     // router server never loads a model and must not touch the GPU
     const bool is_router_server = params.model.path.empty()
-                               && params.model.hf_repo.empty();
+                               && params.model.hf_repo.empty()
+                               && params.model.docker_repo.empty();
 
     // skip device enumeration so the CUDA primary context stays uncreated
     common_params_print_info(params, !is_router_server);
@@ -423,6 +424,18 @@ int llama_server(common_params & params, int argc, char ** argv) {
             ctx_http.stop();
         };
 
+        try {
+            models_routes->models.load_startup_models();
+        } catch (const std::exception & e) {
+            SRV_ERR("failed to load models on startup: %s\n", e.what());
+            ctx_http.stop();
+            if (ctx_http.thread.joinable()) {
+                ctx_http.thread.join();
+            }
+            clean_up();
+            return 1;
+        }
+
     } else {
         // setup clean up function, to be called before exit
         clean_up = [&ctx_http, &ctx_server, &mcp_mgr]() {

tools/tts/tts.cpp

@@ -86,6 +86,7 @@ int main(int argc, char ** argv) {
 
     mtmd_context_params mtmd_params = mtmd_context_params_default();
     mtmd_params.use_gpu = params.mmproj_use_gpu;
+    mtmd_params.device  = params.mmproj_device;
     mtmd::context_ptr mctx(mtmd_init_from_file(params.mmproj.path.c_str(), model, mtmd_params));
     if (!mctx) {
         LOG_ERR("failed to load mmproj %s\n", params.mmproj.path.c_str());

tools/ui/README.md

@@ -239,31 +239,44 @@ Routes → Components → Hooks → Stores → Services → Storage/API
 
 ### High-Level Architecture
 
-See: [`docs/architecture/high-level-architecture-simplified.md`](docs/architecture/high-level-architecture-simplified.md)
-
 ```mermaid
 flowchart TB
     subgraph Routes["📍 Routes"]
         R1["/ (Welcome)"]
         R2["/chat/[id]"]
+        R3["/mcp-servers"]
+        R4["/search"]
+        R5["/settings"]
         RL["+layout.svelte"]
     end
 
     subgraph Components["🧩 Components"]
-        C_Sidebar["ChatSidebar"]
         C_Screen["ChatScreen"]
         C_Form["ChatForm"]
         C_Messages["ChatMessages"]
-        C_ModelsSelector["ModelsSelector"]
+        C_Sidebar["ChatSidebar"]
+        C_Models["ModelsSelector"]
         C_Settings["ChatSettings"]
+        C_Mcp["McpServers"]
+    end
+
+    subgraph Hooks["🔌 Hooks"]
+        H1["use-chat-screen-active-model"]
+        H2["use-processing-state"]
+        H3["use-context-gauge"]
+        H4["use-models-selector"]
+        H5["use-tools-panel"]
     end
 
     subgraph Stores["🗄️ Stores"]
         S1["chatStore"]
         S2["conversationsStore"]
         S3["modelsStore"]
-        S4["serverStore"]
-        S5["settingsStore"]
+        S4["mcpStore"]
+        S5["agenticStore"]
+        S6["serverStore"]
+        S7["settingsStore"]
+        S8["toolsStore"]
     end
 
     subgraph Services["⚙️ Services"]
@@ -271,6 +284,9 @@ flowchart TB
         SV2["ModelsService"]
         SV3["PropsService"]
         SV4["DatabaseService"]
+        SV5["MCPService"]
+        SV6["ToolsService"]
+        SV7["SandboxService"]
     end
 
     subgraph Storage["💾 Storage"]
@@ -282,19 +298,28 @@ flowchart TB
         API1["/v1/chat/completions"]
         API2["/props"]
         API3["/models/*"]
+        API4["/tools"]
     end
 
     R1 & R2 --> C_Screen
     RL --> C_Sidebar
     C_Screen --> C_Form & C_Messages & C_Settings
-    C_Screen --> S1 & S2
-    C_ModelsSelector --> S3 & S4
+    C_Screen --> H1 & H2 & H3
+    C_Models --> H4
+    C_Mcp --> S4
+    C_Screen --> S1 & S2 & S3
+    C_Models --> S3
+    H1 --> S3
     S1 --> SV1 & SV4
+    S2 --> SV4
     S3 --> SV2 & SV3
+    S4 --> SV5
+    S5 --> SV1 & SV5 & SV6 & SV7
     SV4 --> ST1
     SV1 --> API1
     SV2 --> API3
     SV3 --> API2
+    SV6 --> API4
 ```
 
 ### Layer Breakdown
@@ -303,6 +328,9 @@ flowchart TB
 
 - **`/`** - Welcome screen, creates new conversation
 - **`/chat/[id]`** - Active chat interface
+- **`/mcp-servers`** - MCP server management
+- **`/search`** - Conversation search
+- **`/settings`** - Settings (optional `[[section]]`)
 - **`+layout.svelte`** - Sidebar, navigation, global initialization
 
 #### Components (`src/lib/components/`)
@@ -348,28 +376,68 @@ Components are organized in `app/` (application-specific) and `ui/` (shadcn-svel
 
 #### Hooks (`src/lib/hooks/`)
 
-- **`useModelChangeValidation`** - Validates model switch against conversation modalities
-- **`useProcessingState`** - Tracks streaming progress and token generation
+Hooks are the thin view-layer between components and stores: they own UI concerns (scroll, drag-and-drop, keyboard shortcuts, pickers, selection) and translate store state into view state.
+
+| Hook                            | Responsibility                                                 |
+| ------------------------------- | -------------------------------------------------------------- |
+| `use-chat-screen-active-model`  | Active model resolution + modality capability detection        |
+| `use-processing-state`          | View over `chatStore.processing` for streaming progress/tokens |
+| `use-context-gauge`             | View over `contextStatsStore` for the context usage gauge      |
+| `use-models-selector`           | Model selector dropdown state (loaded/available groups)        |
+| `use-tools-panel`               | Tools panel state                                              |
+| `use-reasoning-menu`            | Reasoning-effort menu state                                    |
+| `use-attachment-menu`           | Attachment menu + modality flags                               |
+| `use-draft-messages`            | Per-chat draft message/files persistence                       |
+| `use-chat-form-pickers`         | Chat form pickers (commands, mentions)                         |
+| `use-debounced-search`          | Shared debounced async search for pickers                      |
+| `use-picker-navigation`         | Picker keyboard navigation                                     |
+| `use-chat-message-edit-context` | Message edit context (content + extras)                        |
+| `use-chat-screen-drag-and-drop` | Drag-and-drop state machine                                    |
+| `use-chat-screen-file-upload`   | File upload queue + capability validation                      |
+| `use-chat-screen-scroll`        | Scroll container binding + navigation guard                    |
+| `use-auto-scroll`               | Auto-scroll controller for streaming                           |
+| `use-marquee-selection`         | Shift+click / marquee range selection                          |
+| `use-keyboard-shortcuts`        | Global keyboard shortcuts                                      |
+| `use-settings-navigation`       | Settings section navigation                                    |
+| `use-pwa`                       | PWA install/update + version mismatch detection                |
 
 #### Stores (`src/lib/stores/`)
 
-| Store                | Responsibility                                            |
-| -------------------- | --------------------------------------------------------- |
-| `chatStore`          | Message sending, streaming, abort control, error handling |
-| `conversationsStore` | CRUD for conversations, message branching, navigation     |
-| `modelsStore`        | Model list, selection, loading/unloading (ROUTER)         |
-| `serverStore`        | Server properties, role detection, modalities             |
-| `settingsStore`      | User preferences, parameter sync with server defaults     |
+Stores own reactive application state as Svelte 5 runes. Larger stores are split into directories and compose focused sub-stores behind a narrow host interface (see Architectural Patterns).
+
+| Store                | Responsibility                                                                                                  |
+| -------------------- | --------------------------------------------------------------------------------------------------------------- |
+| `chatStore`          | Chat lifecycle, streaming, abort control, error handling; composes `processing`, `activity`, `streams`, `flows` |
+| `conversationsStore` | Conversation CRUD, message branching, navigation, import/export; composes `preferences`                         |
+| `modelsStore`        | Model list, selection, loading/unloading (ROUTER); composes `props`, `status`                                   |
+| `mcpStore`           | MCP host role: multi-server lifecycle, tool routing; composes `health`, `resources`                             |
+| `agenticStore`       | Multi-turn agentic loop orchestration, tool execution; composes `gates`                                         |
+| `serverStore`        | Server connection state, `/props`, role detection, modalities                                                   |
+| `settingsStore`      | User preferences, theme, parameter sync with server defaults                                                    |
+| `toolsStore`         | Tool registry: server + MCP tools, enabled set for the LLM                                                      |
+| `permissionsStore`   | Persisted tool permission grants                                                                                |
+| `contextStatsStore`  | Context window usage for the active conversation                                                                |
+| `draftMessagesStore` | Per-chat draft message/files                                                                                    |
+| `deviceStore`        | Browser environment signals (mobile, OS, theme)                                                                 |
+| `versionStore`       | Build version information                                                                                       |
 
 #### Services (`src/lib/services/`)
 
-| Service                | Responsibility                                  |
-| ---------------------- | ----------------------------------------------- |
-| `ChatService`          | API calls to`/v1/chat/completions`, SSE parsing |
-| `ModelsService`        | `/models`, `/models/load`, `/models/unload`     |
-| `PropsService`         | `/props`, `/props?model=`                       |
-| `DatabaseService`      | IndexedDB operations via Dexie                  |
-| `ParameterSyncService` | Syncs settings with server defaults             |
+Services are a stateless protocol layer: static methods, pure I/O, no reactive state. Stores consume them for all API and storage access.
+
+| Service                       | Responsibility                                                            |
+| ----------------------------- | ------------------------------------------------------------------------- |
+| `ChatService`                 | `/v1/chat/completions` streaming + SSE parsing, message format conversion |
+| `ModelsService`               | `/models`, `/models/load`, `/models/unload`                               |
+| `PropsService`                | `/props`, `/props?model=`                                                 |
+| `DatabaseService`             | IndexedDB operations via Dexie                                            |
+| `MCPService`                  | MCP protocol: transports, connect, list/execute tools, prompts, resources |
+| `ToolsService`                | Server tool list/execute/stream (`/tools`)                                |
+| `SandboxService`              | Browser JS execution in a sandboxed worker                                |
+| `ParameterSyncService`        | Syncs settings with server defaults                                       |
+| `ConversationTransferService` | Conversation import/export JSONL + ZIP format                             |
+| `MigrationService`            | Non-destructive localStorage/IndexedDB migrations                         |
+| `RouterService`               | Dynamic route URL construction                                            |
 
 ---
 
@@ -377,8 +445,6 @@ Components are organized in `app/` (application-specific) and `ui/` (shadcn-svel
 
 ### MODEL Mode (Single Model)
 
-See: [`docs/flows/data-flow-simplified-model-mode.md`](docs/flows/data-flow-simplified-model-mode.md)
-
 ```mermaid
 sequenceDiagram
     participant User
@@ -388,8 +454,9 @@ sequenceDiagram
     participant API as llama-server
 
     Note over User,API: Initialization
-    UI->>Stores: initialize()
-    Stores->>DB: load conversations
+    UI->>Stores: initStores() (awaited by route loads)
+    Stores->>Stores: run migrations
+    Stores->>DB: load conversations (background)
     Stores->>API: GET /props
     API-->>Stores: server config
     Stores->>API: GET /v1/models
@@ -408,8 +475,6 @@ sequenceDiagram
 
 ### ROUTER Mode (Multi-Model)
 
-See: [`docs/flows/data-flow-simplified-router-mode.md`](docs/flows/data-flow-simplified-router-mode.md)
-
 ```mermaid
 sequenceDiagram
     participant User
@@ -441,17 +506,6 @@ sequenceDiagram
     end
 ```
 
-### Detailed Flow Diagrams
-
-| Flow          | Description                                | File                                                        |
-| ------------- | ------------------------------------------ | ----------------------------------------------------------- |
-| Chat          | Message lifecycle, streaming, regeneration | [`chat-flow.md`](docs/flows/chat-flow.md)                   |
-| Models        | Loading, unloading, modality caching       | [`models-flow.md`](docs/flows/models-flow.md)               |
-| Server        | Props fetching, role detection             | [`server-flow.md`](docs/flows/server-flow.md)               |
-| Conversations | CRUD, branching, import/export             | [`conversations-flow.md`](docs/flows/conversations-flow.md) |
-| Database      | IndexedDB schema, operations               | [`database-flow.md`](docs/flows/database-flow.md)           |
-| Settings      | Parameter sync, user overrides             | [`settings-flow.md`](docs/flows/settings-flow.md)           |
-
 ---
 
 ## Architectural Patterns
@@ -505,13 +559,14 @@ Components dispatch actions to stores, stores coordinate with services for I/O,
 
 ### 3. Per-Conversation State
 
-Enables concurrent streaming across multiple conversations:
+Enables concurrent streaming across multiple conversations. Loading is tracked
+per conversation by the activity ledger (`chatStore.activity`), while streaming
+state and abort controllers live in per-conversation maps:
 
 ```typescript
 class ChatStore {
-	chatLoadingStates = new Map<string, boolean>();
-	chatStreamingStates = new Map<string, { response: string; messageId: string }>();
-	abortControllers = new Map<string, AbortController>();
+	chatStreamingStates = new SvelteMap<string, { response: string; messageId: string }>();
+	abortControllers = new SvelteMap<string, AbortController>();
 }
 ```
 
@@ -567,20 +622,14 @@ get isRouterMode() {
 
 ### 7. Modality Validation
 
-Prevents sending attachments to incompatible models:
+Prevents sending attachments to incompatible models. The
+`use-chat-screen-active-model` hook derives the active model's capabilities
+from `modelsStore.props`:
 
 ```typescript
-// useModelChangeValidation hook
-const validate = (modelId: string) => {
-	const modelModalities = modelsStore.getModelModalities(modelId);
-	const conversationModalities = conversationsStore.usedModalities;
-
-	// Check if model supports all used modalities
-	if (conversationModalities.hasImages && !modelModalities.vision) {
-		return { valid: false, reason: 'Model does not support images' };
-	}
-	// ...
-};
+// use-chat-screen-active-model hook
+const hasVisionModality = $derived.by(() => modelsStore.props.modelSupportsVision(activeModelId));
+const hasAudioModality = $derived.by(() => modelsStore.props.modelSupportsAudio(activeModelId));
 ```
 
 ### 8. Persistent Storage Strategy
@@ -673,9 +722,6 @@ tools/ui/
 │   └── styles/           # Global styles
 ├── static/               # Static assets
 ├── tests/                # Test files
-├── docs/                 # Architecture diagrams
-│   ├── architecture/     # High-level architecture
-│   └── flows/            # Feature-specific flows
 └── .storybook/           # Storybook configuration
 ```
 

tools/ui/docs/architecture/high-level-architecture-simplified.md

@@ -1,145 +0,0 @@
-```mermaid
-flowchart TB
-    subgraph Routes["📍 Routes"]
-        R1["/ (Welcome)"]
-        R2["/chat/[id]"]
-        RL["+layout.svelte"]
-    end
-
-    subgraph Components["🧩 Components"]
-        C_Sidebar["ChatSidebar"]
-        C_Screen["ChatScreen"]
-        C_Form["ChatForm"]
-        C_Messages["ChatMessages"]
-        C_Message["ChatMessage"]
-        C_ChatMessageAgenticContent["ChatMessageAgenticContent"]
-        C_MessageEditForm["ChatMessageEditForm"]
-        C_ModelsSelector["ModelsSelector"]
-        C_Settings["ChatSettings"]
-        C_McpSettings["McpServersSettings"]
-        C_McpResourceBrowser["McpResourceBrowser"]
-        C_McpServersSelector["McpServersSelector"]
-    end
-
-    subgraph Hooks["🪝 Hooks"]
-        H1["useModelChangeValidation"]
-        H2["useProcessingState"]
-    end
-
-    subgraph Stores["🗄️ Stores"]
-        S1["chatStore<br/><i>Chat interactions & streaming</i>"]
-        SA["agenticStore<br/><i>Multi-turn agentic loop orchestration</i>"]
-        S2["conversationsStore<br/><i>Conversation data, messages & MCP overrides</i>"]
-        S3["modelsStore<br/><i>Model selection & loading</i>"]
-        S4["serverStore<br/><i>Server props & role detection</i>"]
-        S5["settingsStore<br/><i>User configuration incl. MCP</i>"]
-        S6["mcpStore<br/><i>MCP servers, tools, prompts</i>"]
-        S7["mcpResourceStore<br/><i>MCP resources & attachments</i>"]
-    end
-
-    subgraph Services["⚙️ Services"]
-        SV1["ChatService"]
-        SV2["ModelsService"]
-        SV3["PropsService"]
-        SV4["DatabaseService"]
-        SV5["ParameterSyncService"]
-        SV6["MCPService<br/><i>protocol operations</i>"]
-    end
-
-    subgraph Storage["💾 Storage"]
-        ST1["IndexedDB<br/><i>conversations, messages</i>"]
-        ST2["LocalStorage<br/><i>config, userOverrides, mcpServers</i>"]
-    end
-
-    subgraph APIs["🌐 llama-server API"]
-        API1["/v1/chat/completions"]
-        API2["/props"]
-        API3["/models/*"]
-        API4["/v1/models"]
-    end
-
-    subgraph ExternalMCP["🔌 External MCP Servers"]
-        EXT1["MCP Server 1<br/><i>WebSocket/HTTP/SSE</i>"]
-        EXT2["MCP Server N"]
-    end
-
-    %% Routes → Components
-    R1 & R2 --> C_Screen
-    RL --> C_Sidebar
-
-    %% Layout runs MCP health checks
-    RL --> S6
-
-    %% Component hierarchy
-    C_Screen --> C_Form & C_Messages & C_Settings
-    C_Messages --> C_Message
-    C_Message --> C_ChatMessageAgenticContent
-    C_Message --> C_MessageEditForm
-    C_Form & C_MessageEditForm --> C_ModelsSelector
-    C_Form --> C_McpServersSelector
-    C_Settings --> C_McpSettings
-    C_McpSettings --> C_McpResourceBrowser
-
-    %% Components → Hooks → Stores
-    C_Form & C_Messages --> H1 & H2
-    H1 --> S3 & S4
-    H2 --> S1 & S5
-
-    %% Components → Stores
-    C_Screen --> S1 & S2
-    C_Sidebar --> S2
-    C_ModelsSelector --> S3 & S4
-    C_Settings --> S5
-    C_McpSettings --> S6
-    C_McpResourceBrowser --> S6 & S7
-    C_McpServersSelector --> S6
-    C_Form --> S6
-
-    %% chatStore → agenticStore → mcpStore (agentic loop)
-    S1 --> SA
-    SA --> SV1
-    SA --> S6
-
-    %% Stores → Services
-    S1 --> SV1 & SV4
-    S2 --> SV4
-    S3 --> SV2 & SV3
-    S4 --> SV3
-    S5 --> SV5
-    S6 --> SV6
-    S7 --> SV6
-
-    %% Services → Storage
-    SV4 --> ST1
-    SV5 --> ST2
-
-    %% Services → APIs
-    SV1 --> API1
-    SV2 --> API3 & API4
-    SV3 --> API2
-
-    %% MCP → External Servers
-    SV6 --> EXT1 & EXT2
-
-    %% Styling
-    classDef routeStyle fill:#e1f5fe,stroke:#01579b,stroke-width:2px
-    classDef componentStyle fill:#f3e5f5,stroke:#7b1fa2,stroke-width:2px
-    classDef hookStyle fill:#fff8e1,stroke:#ff8f00,stroke-width:2px
-    classDef storeStyle fill:#fff3e0,stroke:#e65100,stroke-width:2px
-    classDef serviceStyle fill:#e8f5e9,stroke:#2e7d32,stroke-width:2px
-    classDef storageStyle fill:#fce4ec,stroke:#c2185b,stroke-width:2px
-    classDef apiStyle fill:#e3f2fd,stroke:#1565c0,stroke-width:2px
-    classDef mcpStyle fill:#e0f2f1,stroke:#00695c,stroke-width:2px
-    classDef agenticStyle fill:#e8eaf6,stroke:#283593,stroke-width:2px
-    classDef externalStyle fill:#f3e5f5,stroke:#6a1b9a,stroke-width:2px,stroke-dasharray: 5 5
-
-    class R1,R2,RL routeStyle
-    class C_Sidebar,C_Screen,C_Form,C_Messages,C_Message,C_ChatMessageAgenticContent,C_MessageEditForm,C_ModelsSelector,C_Settings componentStyle
-    class C_McpSettings,C_McpResourceBrowser,C_McpServersSelector componentStyle
-    class H1,H2 hookStyle
-    class S1,S2,S3,S4,S5,SA,S6,S7 storeStyle
-    class SV1,SV2,SV3,SV4,SV5,SV6 serviceStyle
-    class ST1,ST2 storageStyle
-    class API1,API2,API3,API4 apiStyle
-    class EXT1,EXT2 externalStyle
-```

tools/ui/docs/architecture/high-level-architecture.md

@@ -1,373 +0,0 @@
-```mermaid
-flowchart TB
-subgraph Routes["📍 Routes"]
-R1["/ (+page.svelte)"]
-R2["/chat/[id]"]
-RL["+layout.svelte"]
-end
-
-    subgraph Components["🧩 Components"]
-        direction TB
-        subgraph LayoutComponents["Layout"]
-            C_Sidebar["ChatSidebar"]
-            C_Screen["ChatScreen"]
-        end
-        subgraph ChatUIComponents["Chat UI"]
-            C_Form["ChatForm"]
-            C_Messages["ChatMessages"]
-            C_Message["ChatMessage"]
-            C_MessageUser["ChatMessageUser"]
-            C_MessageEditForm["ChatMessageEditForm"]
-            C_Attach["ChatAttachments"]
-            C_ModelsSelector["ModelsSelector"]
-            C_Settings["ChatSettings"]
-        end
-        subgraph MCPComponents["MCP UI"]
-            C_McpSettings["McpServersSettings"]
-            C_McpServerCard["McpServerCard"]
-            C_McpResourceBrowser["McpResourceBrowser"]
-            C_McpResourcePreview["McpResourcePreview"]
-            C_McpServersSelector["McpServersSelector"]
-        end
-    end
-
-    subgraph Hooks["🪝 Hooks"]
-        H1["useModelChangeValidation"]
-        H2["useProcessingState"]
-        H3["isMobile"]
-    end
-
-    subgraph Stores["🗄️ Stores"]
-        direction TB
-        subgraph S1["chatStore"]
-            S1State["<b>State:</b><br/>isLoading, currentResponse<br/>errorDialogState<br/>activeProcessingState<br/>chatLoadingStates<br/>chatStreamingStates<br/>abortControllers<br/>processingStates<br/>activeConversationId<br/>isStreamingActive"]
-            S1LoadState["<b>Loading State:</b><br/>setChatLoading()<br/>isChatLoading()<br/>syncLoadingStateForChat()<br/>clearUIState()<br/>isChatLoadingPublic()<br/>getAllLoadingChats()<br/>getAllStreamingChats()"]
-            S1ProcState["<b>Processing State:</b><br/>setActiveProcessingConversation()<br/>getProcessingState()<br/>clearProcessingState()<br/>getActiveProcessingState()<br/>updateProcessingStateFromTimings()<br/>getCurrentProcessingStateSync()<br/>restoreProcessingStateFromMessages()"]
-            S1Stream["<b>Streaming:</b><br/>streamChatCompletion()<br/>startStreaming()<br/>stopStreaming()<br/>stopGeneration()<br/>isStreaming()"]
-            S1Error["<b>Error Handling:</b><br/>showErrorDialog()<br/>dismissErrorDialog()<br/>isAbortError()"]
-            S1Msg["<b>Message Operations:</b><br/>addMessage()<br/>sendMessage()<br/>updateMessage()<br/>deleteMessage()<br/>getDeletionInfo()"]
-            S1Regen["<b>Regeneration:</b><br/>regenerateMessage()<br/>regenerateMessageWithBranching()<br/>continueAssistantMessage()"]
-            S1Edit["<b>Editing:</b><br/>editAssistantMessage()<br/>editUserMessagePreserveResponses()<br/>editMessageWithBranching()<br/>clearEditMode()<br/>isEditModeActive()<br/>getAddFilesHandler()<br/>setEditModeActive()"]
-            S1Utils["<b>Utilities:</b><br/>getApiOptions()<br/>parseTimingData()<br/>getOrCreateAbortController()<br/>getConversationModel()"]
-        end
-        subgraph SA["agenticStore"]
-            SAState["<b>State:</b><br/>sessions (Map)<br/>isAnyRunning"]
-            SASession["<b>Session Management:</b><br/>getSession()<br/>updateSession()<br/>clearSession()<br/>getActiveSessions()<br/>isRunning()<br/>currentTurn()<br/>totalToolCalls()<br/>lastError()<br/>streamingToolCall()"]
-            SAConfig["<b>Configuration:</b><br/>getConfig()<br/>maxTurns, maxToolPreviewLines"]
-            SAFlow["<b>Agentic Loop:</b><br/>runAgenticFlow()<br/>executeAgenticLoop()<br/>normalizeToolCalls()<br/>emitToolCallResult()<br/>extractBase64Attachments()"]
-        end
-        subgraph S2["conversationsStore"]
-            S2State["<b>State:</b><br/>conversations<br/>activeConversation<br/>activeMessages<br/>isInitialized<br/>pendingMcpServerOverrides<br/>titleUpdateConfirmationCallback"]
-            S2Lifecycle["<b>Lifecycle:</b><br/>initialize()<br/>loadConversations()<br/>clearActiveConversation()"]
-            S2ConvCRUD["<b>Conversation CRUD:</b><br/>createConversation()<br/>loadConversation()<br/>deleteConversation()<br/>deleteAll()<br/>updateConversationName()<br/>updateConversationTitleWithConfirmation()"]
-            S2MsgMgmt["<b>Message Management:</b><br/>refreshActiveMessages()<br/>addMessageToActive()<br/>updateMessageAtIndex()<br/>findMessageIndex()<br/>sliceActiveMessages()<br/>removeMessageAtIndex()<br/>getConversationMessages()"]
-            S2Nav["<b>Navigation:</b><br/>navigateToSibling()<br/>updateCurrentNode()<br/>updateConversationTimestamp()"]
-            S2McpOverrides["<b>MCP Per-Chat Overrides:</b><br/>getMcpServerOverride()<br/>getAllMcpServerOverrides()<br/>setMcpServerOverride()<br/>toggleMcpServerForChat()<br/>removeMcpServerOverride()<br/>isMcpServerEnabledForChat()<br/>clearPendingMcpServerOverrides()"]
-            S2Export["<b>Import/Export:</b><br/>downloadConversation()<br/>exportAllConversations()<br/>importConversations()<br/>importConversationsData()<br/>triggerDownload()"]
-            S2Utils["<b>Utilities:</b><br/>setTitleUpdateConfirmationCallback()"]
-        end
-        subgraph S3["modelsStore"]
-            S3State["<b>State:</b><br/>models, routerModels<br/>selectedModelId<br/>selectedModelName<br/>loading, updating, error<br/>modelLoadingStates<br/>modelPropsCache<br/>modelPropsFetching<br/>propsCacheVersion"]
-            S3Getters["<b>Computed Getters:</b><br/>selectedModel<br/>loadedModelIds<br/>loadingModelIds<br/>singleModelName"]
-            S3Modal["<b>Modalities:</b><br/>getModelModalities()<br/>modelSupportsVision()<br/>modelSupportsAudio()<br/>getModelModalitiesArray()<br/>getModelProps()<br/>updateModelModalities()"]
-            S3Status["<b>Status Queries:</b><br/>isModelLoaded()<br/>isModelOperationInProgress()<br/>getModelStatus()<br/>isModelPropsFetching()"]
-            S3Fetch["<b>Data Fetching:</b><br/>fetch()<br/>fetchRouterModels()<br/>fetchModelProps()<br/>fetchModalitiesForLoadedModels()"]
-            S3Select["<b>Model Selection:</b><br/>selectModelById()<br/>selectModelByName()<br/>clearSelection()<br/>findModelByName()<br/>findModelById()<br/>hasModel()"]
-            S3LoadUnload["<b>Loading/Unloading Models:</b><br/>loadModel()<br/>unloadModel()<br/>ensureModelLoaded()<br/>waitForModelStatus()<br/>pollForModelStatus()"]
-            S3Utils["<b>Utilities:</b><br/>toDisplayName()<br/>clear()"]
-        end
-        subgraph S4["serverStore"]
-            S4State["<b>State:</b><br/>props<br/>loading, error<br/>role<br/>fetchPromise"]
-            S4Getters["<b>Getters:</b><br/>defaultParams<br/>contextSize<br/>isRouterMode<br/>isModelMode"]
-            S4Data["<b>Data Handling:</b><br/>fetch()<br/>getErrorMessage()<br/>clear()"]
-            S4Utils["<b>Utilities:</b><br/>detectRole()"]
-        end
-        subgraph S5["settingsStore"]
-            S5State["<b>State:</b><br/>config<br/>theme<br/>isInitialized<br/>userOverrides"]
-            S5Lifecycle["<b>Lifecycle:</b><br/>initialize()<br/>loadConfig()<br/>saveConfig()<br/>loadTheme()<br/>saveTheme()"]
-            S5Update["<b>Config Updates:</b><br/>updateConfig()<br/>updateMultipleConfig()<br/>updateTheme()"]
-            S5Reset["<b>Reset:</b><br/>resetConfig()<br/>resetTheme()<br/>resetAll()<br/>resetParameterToServerDefault()"]
-            S5Sync["<b>Server Sync:</b><br/>syncWithServerDefaults()<br/>forceSyncWithServerDefaults()"]
-            S5Utils["<b>Utilities:</b><br/>getConfig()<br/>getAllConfig()<br/>getParameterInfo()<br/>getParameterDiff()<br/>getServerDefaults()<br/>clearAllUserOverrides()"]
-        end
-        subgraph S6["mcpStore"]
-            S6State["<b>State:</b><br/>isInitializing, error<br/>toolCount, connectedServers<br/>healthChecks (Map)<br/>connections (Map)<br/>toolsIndex (Map)"]
-            S6Lifecycle["<b>Lifecycle:</b><br/>ensureInitialized()<br/>initialize()<br/>shutdown()<br/>acquireConnection()<br/>releaseConnection()"]
-            S6Health["<b>Health Checks:</b><br/>runHealthCheck()<br/>runHealthChecksForServers()<br/>updateHealthCheck()<br/>getHealthCheckState()<br/>clearHealthCheck()"]
-            S6Servers["<b>Server Management:</b><br/>getServers()<br/>addServer()<br/>updateServer()<br/>removeServer()<br/>getServerById()<br/>getServerDisplayName()"]
-            S6Tools["<b>Tool Operations:</b><br/>getToolDefinitionsForLLM()<br/>getToolNames()<br/>hasTool()<br/>getToolServer()<br/>executeTool()<br/>executeToolByName()"]
-            S6Prompts["<b>Prompt Operations:</b><br/>getAllPrompts()<br/>getPrompt()<br/>hasPromptsCapability()<br/>getPromptCompletions()"]
-        end
-        subgraph S7["mcpResourceStore"]
-            S7State["<b>State:</b><br/>serverResources (Map)<br/>cachedResources (Map)<br/>subscriptions (Map)<br/>attachments[]<br/>isLoading"]
-            S7Resources["<b>Resource Discovery:</b><br/>setServerResources()<br/>getServerResources()<br/>getAllResourceInfos()<br/>getAllTemplateInfos()<br/>clearServerResources()"]
-            S7Cache["<b>Caching:</b><br/>cacheResourceContent()<br/>getCachedContent()<br/>invalidateCache()<br/>clearCache()"]
-            S7Subs["<b>Subscriptions:</b><br/>addSubscription()<br/>removeSubscription()<br/>isSubscribed()<br/>handleResourceUpdate()"]
-            S7Attach["<b>Attachments:</b><br/>addAttachment()<br/>updateAttachmentContent()<br/>removeAttachment()<br/>clearAttachments()<br/>toMessageExtras()"]
-        end
-
-        subgraph ReactiveExports["⚡ Reactive Exports"]
-            direction LR
-            subgraph ChatExports["chatStore"]
-                RE1["isLoading()"]
-                RE2["currentResponse()"]
-                RE3["errorDialog()"]
-                RE4["activeProcessingState()"]
-                RE5["isChatStreaming()"]
-                RE6["isChatLoading()"]
-                RE7["getChatStreaming()"]
-                RE8["getAllLoadingChats()"]
-                RE9["getAllStreamingChats()"]
-                RE9a["isEditModeActive()"]
-                RE9b["getAddFilesHandler()"]
-                RE9c["setEditModeActive()"]
-                RE9d["clearEditMode()"]
-            end
-            subgraph AgenticExports["agenticStore"]
-                REA1["agenticIsRunning()"]
-                REA2["agenticCurrentTurn()"]
-                REA3["agenticTotalToolCalls()"]
-                REA4["agenticLastError()"]
-                REA5["agenticStreamingToolCall()"]
-                REA6["agenticIsAnyRunning()"]
-            end
-            subgraph ConvExports["conversationsStore"]
-                RE10["conversations()"]
-                RE11["activeConversation()"]
-                RE12["activeMessages()"]
-                RE13["isConversationsInitialized()"]
-            end
-            subgraph ModelsExports["modelsStore"]
-                RE15["modelOptions()"]
-                RE16["routerModels()"]
-                RE17["modelsLoading()"]
-                RE18["modelsUpdating()"]
-                RE19["modelsError()"]
-                RE20["selectedModelId()"]
-                RE21["selectedModelName()"]
-                RE22["selectedModelOption()"]
-                RE23["loadedModelIds()"]
-                RE24["loadingModelIds()"]
-                RE25["propsCacheVersion()"]
-                RE26["singleModelName()"]
-            end
-            subgraph ServerExports["serverStore"]
-                RE27["serverProps()"]
-                RE28["serverLoading()"]
-                RE29["serverError()"]
-                RE30["serverRole()"]
-                RE31["defaultParams()"]
-                RE32["contextSize()"]
-                RE33["isRouterMode()"]
-                RE34["isModelMode()"]
-            end
-            subgraph SettingsExports["settingsStore"]
-                RE35["config()"]
-                RE36["theme()"]
-                RE37["isInitialized()"]
-            end
-            subgraph MCPExports["mcpStore / mcpResourceStore"]
-                RE38["mcpResources()"]
-                RE39["mcpResourceAttachments()"]
-                RE40["mcpHasResourceAttachments()"]
-                RE41["mcpTotalResourceCount()"]
-                RE42["mcpResourcesLoading()"]
-            end
-        end
-    end
-
-    subgraph Services["⚙️ Services"]
-        direction TB
-        subgraph SV1["ChatService"]
-            SV1Msg["<b>Messaging:</b><br/>sendMessage()"]
-            SV1Stream["<b>Streaming:</b><br/>handleStreamResponse()<br/>handleNonStreamResponse()"]
-            SV1Convert["<b>Conversion:</b><br/>convertDbMessageToApiChatMessageData()<br/>mergeToolCallDeltas()"]
-            SV1Utils["<b>Utilities:</b><br/>stripReasoningContent()<br/>extractModelName()<br/>parseErrorResponse()"]
-        end
-        subgraph SV2["ModelsService"]
-            SV2List["<b>Listing:</b><br/>list()<br/>listRouter()"]
-            SV2LoadUnload["<b>Load/Unload:</b><br/>load()<br/>unload()"]
-            SV2Status["<b>Status:</b><br/>isModelLoaded()<br/>isModelLoading()"]
-        end
-        subgraph SV3["PropsService"]
-            SV3Fetch["<b>Fetching:</b><br/>fetch()<br/>fetchForModel()"]
-        end
-        subgraph SV4["DatabaseService"]
-            SV4Conv["<b>Conversations:</b><br/>createConversation()<br/>getConversation()<br/>getAllConversations()<br/>updateConversation()<br/>deleteConversation()"]
-            SV4Msg["<b>Messages:</b><br/>createMessageBranch()<br/>createRootMessage()<br/>createSystemMessage()<br/>getConversationMessages()<br/>updateMessage()<br/>deleteMessage()<br/>deleteMessageCascading()"]
-            SV4Node["<b>Navigation:</b><br/>updateCurrentNode()"]
-            SV4Import["<b>Import:</b><br/>importConversations()"]
-        end
-        subgraph SV5["ParameterSyncService"]
-            SV5Extract["<b>Extraction:</b><br/>extractServerDefaults()"]
-            SV5Merge["<b>Merging:</b><br/>mergeWithServerDefaults()"]
-            SV5Info["<b>Info:</b><br/>getParameterInfo()<br/>canSyncParameter()<br/>getSyncableParameterKeys()<br/>validateServerParameter()"]
-            SV5Diff["<b>Diff:</b><br/>createParameterDiff()"]
-        end
-        subgraph SV6["MCPService"]
-            SV6Transport["<b>Transport:</b><br/>createTransport()<br/>WebSocket / StreamableHTTP / SSE"]
-            SV6Conn["<b>Connection:</b><br/>connect()<br/>disconnect()"]
-            SV6Tools["<b>Tools:</b><br/>listTools()<br/>callTool()"]
-            SV6Prompts["<b>Prompts:</b><br/>listPrompts()<br/>getPrompt()"]
-            SV6Resources["<b>Resources:</b><br/>listResources()<br/>listResourceTemplates()<br/>readResource()<br/>subscribeResource()<br/>unsubscribeResource()"]
-            SV6Complete["<b>Completions:</b><br/>complete()"]
-        end
-    end
-
-    subgraph ExternalMCP["🔌 External MCP Servers"]
-        EXT1["MCP Server 1<br/>(WebSocket/StreamableHTTP/SSE)"]
-        EXT2["MCP Server N"]
-    end
-
-    subgraph Storage["💾 Storage"]
-        ST1["IndexedDB"]
-        ST2["conversations"]
-        ST3["messages"]
-        ST5["LocalStorage"]
-        ST6["config"]
-        ST7["userOverrides"]
-        ST8["mcpServers"]
-    end
-
-    subgraph APIs["🌐 llama-server API"]
-        API1["/v1/chat/completions"]
-        API2["/props<br/>/props?model="]
-        API3["/models<br/>/models/load<br/>/models/unload"]
-        API4["/v1/models"]
-    end
-
-    %% Routes render Components
-    R1 --> C_Screen
-    R2 --> C_Screen
-    RL --> C_Sidebar
-
-    %% Layout runs MCP health checks on startup
-    RL --> S6
-
-    %% Component hierarchy
-    C_Screen --> C_Form & C_Messages & C_Settings
-    C_Messages --> C_Message
-    C_Message --> C_MessageUser
-    C_MessageUser --> C_MessageEditForm
-    C_MessageEditForm --> C_ModelsSelector
-    C_MessageEditForm --> C_Attach
-    C_Form --> C_ModelsSelector
-    C_Form --> C_Attach
-    C_Form --> C_McpServersSelector
-    C_Message --> C_Attach
-
-    %% MCP Components hierarchy
-    C_Settings --> C_McpSettings
-    C_McpSettings --> C_McpServerCard
-    C_McpServerCard --> C_McpResourceBrowser
-    C_McpResourceBrowser --> C_McpResourcePreview
-
-    %% Components use Hooks
-    C_Form --> H1
-    C_Message --> H1 & H2
-    C_MessageEditForm --> H1
-    C_Screen --> H2
-
-    %% Hooks use Stores
-    H1 --> S3 & S4
-    H2 --> S1 & S5
-
-    %% Components use Stores
-    C_Screen --> S1 & S2
-    C_Messages --> S2
-    C_Message --> S1 & S2 & S3
-    C_Form --> S1 & S3 & S6
-    C_Sidebar --> S2
-    C_ModelsSelector --> S3 & S4
-    C_Settings --> S5
-    C_McpSettings --> S6
-    C_McpServerCard --> S6
-    C_McpResourceBrowser --> S6 & S7
-    C_McpServersSelector --> S6
-
-    %% Stores export Reactive State
-    S1 -. exports .-> ChatExports
-    SA -. exports .-> AgenticExports
-    S2 -. exports .-> ConvExports
-    S3 -. exports .-> ModelsExports
-    S4 -. exports .-> ServerExports
-    S5 -. exports .-> SettingsExports
-    S6 -. exports .-> MCPExports
-    S7 -. exports .-> MCPExports
-
-    %% chatStore → agenticStore (agentic loop orchestration)
-    S1 --> SA
-    SA --> SV1
-    SA --> S6
-
-    %% Stores use Services
-    S1 --> SV1 & SV4
-    S2 --> SV4
-    S3 --> SV2 & SV3
-    S4 --> SV3
-    S5 --> SV5
-    S6 --> SV6
-    S7 --> SV6
-
-    %% Services to Storage
-    SV4 --> ST1
-    ST1 --> ST2 & ST3
-    SV5 --> ST5
-    ST5 --> ST6 & ST7 & ST8
-
-    %% Services to APIs
-    SV1 --> API1
-    SV2 --> API3 & API4
-    SV3 --> API2
-
-    %% MCP → External Servers
-    SV6 --> EXT1 & EXT2
-
-    %% Styling
-    classDef routeStyle fill:#e1f5fe,stroke:#01579b,stroke-width:2px
-    classDef componentStyle fill:#f3e5f5,stroke:#7b1fa2,stroke-width:2px
-    classDef componentGroupStyle fill:#e1bee7,stroke:#7b1fa2,stroke-width:1px
-    classDef hookStyle fill:#fff8e1,stroke:#ff8f00,stroke-width:2px
-    classDef storeStyle fill:#fff3e0,stroke:#e65100,stroke-width:2px
-    classDef stateStyle fill:#ffe0b2,stroke:#e65100,stroke-width:1px
-    classDef methodStyle fill:#ffecb3,stroke:#e65100,stroke-width:1px
-    classDef reactiveStyle fill:#fffde7,stroke:#f9a825,stroke-width:1px
-    classDef serviceStyle fill:#e8f5e9,stroke:#2e7d32,stroke-width:2px
-    classDef serviceMStyle fill:#c8e6c9,stroke:#2e7d32,stroke-width:1px
-    classDef externalStyle fill:#f3e5f5,stroke:#6a1b9a,stroke-width:2px,stroke-dasharray: 5 5
-    classDef storageStyle fill:#fce4ec,stroke:#c2185b,stroke-width:2px
-    classDef apiStyle fill:#e3f2fd,stroke:#1565c0,stroke-width:2px
-
-    class R1,R2,RL routeStyle
-    class C_Sidebar,C_Screen,C_Form,C_Messages,C_Message,C_MessageUser,C_MessageEditForm componentStyle
-    class C_ModelsSelector,C_Settings componentStyle
-    class C_Attach componentStyle
-    class C_McpSettings,C_McpServerCard,C_McpResourceBrowser,C_McpResourcePreview,C_McpServersSelector componentStyle
-    class H1,H2,H3 hookStyle
-    class LayoutComponents,ChatUIComponents,MCPComponents componentGroupStyle
-    class Hooks hookStyle
-    classDef agenticStyle fill:#e8eaf6,stroke:#283593,stroke-width:2px
-    classDef agenticMethodStyle fill:#c5cae9,stroke:#283593,stroke-width:1px
-
-    class S1,S2,S3,S4,S5,SA,S6,S7 storeStyle
-    class S1State,S2State,S3State,S4State,S5State,SAState,S6State,S7State stateStyle
-    class S1Msg,S1Regen,S1Edit,S1Stream,S1LoadState,S1ProcState,S1Error,S1Utils methodStyle
-    class SASession,SAConfig,SAFlow methodStyle
-    class S2Lifecycle,S2ConvCRUD,S2MsgMgmt,S2Nav,S2McpOverrides,S2Export,S2Utils methodStyle
-    class S3Getters,S3Modal,S3Status,S3Fetch,S3Select,S3LoadUnload,S3Utils methodStyle
-    class S4Getters,S4Data,S4Utils methodStyle
-    class S5Lifecycle,S5Update,S5Reset,S5Sync,S5Utils methodStyle
-    class S6Lifecycle,S6Health,S6Servers,S6Tools,S6Prompts methodStyle
-    class S7Resources,S7Cache,S7Subs,S7Attach methodStyle
-    class ChatExports,AgenticExports,ConvExports,ModelsExports,ServerExports,SettingsExports,MCPExports reactiveStyle
-    class SV1,SV2,SV3,SV4,SV5,SV6 serviceStyle
-    class SV6Transport,SV6Conn,SV6Tools,SV6Prompts,SV6Resources,SV6Complete serviceMStyle
-    class EXT1,EXT2 externalStyle
-    class SV1Msg,SV1Stream,SV1Convert,SV1Utils serviceMStyle
-    class SV2List,SV2LoadUnload,SV2Status serviceMStyle
-    class SV3Fetch serviceMStyle
-    class SV4Conv,SV4Msg,SV4Node,SV4Import serviceMStyle
-    class SV5Extract,SV5Merge,SV5Info,SV5Diff serviceMStyle
-    class ST1,ST2,ST3,ST5,ST6,ST7,ST8 storageStyle
-    class API1,API2,API3,API4 apiStyle
-```

tools/ui/docs/flows/chat-flow.md

@@ -1,228 +0,0 @@
-```mermaid
-sequenceDiagram
-    participant UI as 🧩 ChatForm / ChatMessage
-    participant chatStore as 🗄️ chatStore
-    participant agenticStore as 🗄️ agenticStore
-    participant convStore as 🗄️ conversationsStore
-    participant settingsStore as 🗄️ settingsStore
-    participant mcpStore as 🗄️ mcpStore
-    participant ChatSvc as ⚙️ ChatService
-    participant DbSvc as ⚙️ DatabaseService
-    participant API as 🌐 /v1/chat/completions
-
-    Note over chatStore: State:<br/>isLoading, currentResponse<br/>errorDialogState, activeProcessingState<br/>chatLoadingStates (Map)<br/>chatStreamingStates (Map)<br/>abortControllers (Map)<br/>processingStates (Map)
-
-    %% ═══════════════════════════════════════════════════════════════════════════
-    Note over UI,API: 💬 SEND MESSAGE
-    %% ═══════════════════════════════════════════════════════════════════════════
-
-    UI->>chatStore: sendMessage(content, extras)
-    activate chatStore
-
-    chatStore->>chatStore: setChatLoading(convId, true)
-    chatStore->>chatStore: clearChatStreaming(convId)
-
-    alt no active conversation
-        chatStore->>convStore: createConversation()
-        Note over convStore: → see conversations-flow.mmd
-    end
-
-    chatStore->>mcpStore: consumeResourceAttachmentsAsExtras()
-    Note right of mcpStore: Converts pending MCP resource<br/>attachments into message extras
-
-    chatStore->>chatStore: addMessage("user", content, extras)
-    chatStore->>DbSvc: createMessageBranch(userMsg, parentId)
-    chatStore->>convStore: addMessageToActive(userMsg)
-    chatStore->>convStore: updateCurrentNode(userMsg.id)
-
-    chatStore->>chatStore: createAssistantMessage(userMsg.id)
-    chatStore->>DbSvc: createMessageBranch(assistantMsg, userMsg.id)
-    chatStore->>convStore: addMessageToActive(assistantMsg)
-
-    chatStore->>chatStore: streamChatCompletion(messages, assistantMsg)
-    deactivate chatStore
-
-    %% ═══════════════════════════════════════════════════════════════════════════
-    Note over UI,API: 🌊 STREAMING (with agentic flow detection)
-    %% ═══════════════════════════════════════════════════════════════════════════
-
-    activate chatStore
-    chatStore->>chatStore: startStreaming()
-    Note right of chatStore: isStreamingActive = true
-
-    chatStore->>chatStore: setActiveProcessingConversation(convId)
-    chatStore->>chatStore: getOrCreateAbortController(convId)
-    Note right of chatStore: abortControllers.set(convId, new AbortController())
-
-    chatStore->>chatStore: getApiOptions()
-    Note right of chatStore: Merge from settingsStore.config:<br/>temperature, max_tokens, top_p, etc.
-
-    alt agenticConfig.enabled && mcpStore has connected servers
-        chatStore->>agenticStore: runAgenticFlow(convId, messages, assistantMsg, options, signal)
-        Note over agenticStore: Multi-turn agentic loop:<br/>1. Call ChatService.sendMessage()<br/>2. If response has tool_calls → execute via mcpStore<br/>3. Append tool results as messages<br/>4. Loop until no more tool_calls or maxTurns<br/>→ see agentic flow details below
-        agenticStore-->>chatStore: final response with timings
-    else standard (non-agentic) flow
-        chatStore->>ChatSvc: sendMessage(messages, options, signal)
-    end
-
-    activate ChatSvc
-
-    ChatSvc->>ChatSvc: convertDbMessageToApiChatMessageData(messages)
-    Note right of ChatSvc: DatabaseMessage[] → ApiChatMessageData[]<br/>Process attachments (images, PDFs, audio)
-
-    ChatSvc->>API: POST /v1/chat/completions
-    Note right of API: {messages, model?, stream: true, ...params}
-
-    loop SSE chunks
-        API-->>ChatSvc: data: {"choices":[{"delta":{...}}]}
-        ChatSvc->>ChatSvc: handleStreamResponse(response)
-
-        alt content chunk
-            ChatSvc-->>chatStore: onChunk(content)
-            chatStore->>chatStore: setChatStreaming(convId, response, msgId)
-            Note right of chatStore: currentResponse = $state(accumulated)
-            chatStore->>convStore: updateMessageAtIndex(idx, {content})
-        end
-
-        alt reasoning chunk
-            ChatSvc-->>chatStore: onReasoningChunk(reasoning)
-            chatStore->>convStore: updateMessageAtIndex(idx, {thinking})
-        end
-
-        alt tool_calls chunk
-            ChatSvc-->>chatStore: onToolCallChunk(toolCalls)
-            chatStore->>convStore: updateMessageAtIndex(idx, {toolCalls})
-        end
-
-        alt model info
-            ChatSvc-->>chatStore: onModel(modelName)
-            chatStore->>chatStore: recordModel(modelName)
-            chatStore->>DbSvc: updateMessage(msgId, {model})
-        end
-
-        alt timings (during stream)
-            ChatSvc-->>chatStore: onTimings(timings, promptProgress)
-            chatStore->>chatStore: updateProcessingStateFromTimings()
-        end
-
-        chatStore-->>UI: reactive $state update
-    end
-
-    API-->>ChatSvc: data: [DONE]
-    ChatSvc-->>chatStore: onComplete(content, reasoning, timings, toolCalls)
-    deactivate ChatSvc
-
-    chatStore->>chatStore: stopStreaming()
-    chatStore->>DbSvc: updateMessage(msgId, {content, timings, model})
-    chatStore->>convStore: updateCurrentNode(msgId)
-    chatStore->>chatStore: setChatLoading(convId, false)
-    chatStore->>chatStore: clearChatStreaming(convId)
-    chatStore->>chatStore: clearProcessingState(convId)
-    deactivate chatStore
-
-    %% ═══════════════════════════════════════════════════════════════════════════
-    Note over UI,API: ⏹️ STOP GENERATION
-    %% ═══════════════════════════════════════════════════════════════════════════
-
-    UI->>chatStore: stopGeneration()
-    activate chatStore
-    chatStore->>chatStore: savePartialResponseIfNeeded(convId)
-    Note right of chatStore: Save currentResponse to DB if non-empty
-    chatStore->>chatStore: abortControllers.get(convId).abort()
-    Note right of chatStore: fetch throws AbortError → caught by isAbortError()
-    chatStore->>chatStore: stopStreaming()
-    chatStore->>chatStore: setChatLoading(convId, false)
-    chatStore->>chatStore: clearChatStreaming(convId)
-    chatStore->>chatStore: clearProcessingState(convId)
-    deactivate chatStore
-
-    %% ═══════════════════════════════════════════════════════════════════════════
-    Note over UI,API: 🔁 REGENERATE
-    %% ═══════════════════════════════════════════════════════════════════════════
-
-    UI->>chatStore: regenerateMessageWithBranching(msgId, model?)
-    activate chatStore
-    chatStore->>convStore: findMessageIndex(msgId)
-    chatStore->>chatStore: Get parent of target message
-    chatStore->>chatStore: createAssistantMessage(parentId)
-    chatStore->>DbSvc: createMessageBranch(newAssistantMsg, parentId)
-    chatStore->>convStore: refreshActiveMessages()
-    Note right of chatStore: Same streaming flow
-    chatStore->>chatStore: streamChatCompletion(...)
-    deactivate chatStore
-
-    %% ═══════════════════════════════════════════════════════════════════════════
-    Note over UI,API: ➡️ CONTINUE
-    %% ═══════════════════════════════════════════════════════════════════════════
-
-    UI->>chatStore: continueAssistantMessage(msgId)
-    activate chatStore
-    chatStore->>chatStore: Get existing content from message
-    chatStore->>chatStore: streamChatCompletion(..., existingContent)
-    Note right of chatStore: Appends to existing message content
-    deactivate chatStore
-
-    %% ═══════════════════════════════════════════════════════════════════════════
-    Note over UI,API: ✏️ EDIT USER MESSAGE
-    %% ═══════════════════════════════════════════════════════════════════════════
-
-    UI->>chatStore: editMessageWithBranching(msgId, newContent, extras)
-    activate chatStore
-    chatStore->>chatStore: Get parent of target message
-    chatStore->>DbSvc: createMessageBranch(editedMsg, parentId)
-    chatStore->>convStore: refreshActiveMessages()
-    Note right of chatStore: Creates new branch, original preserved
-    chatStore->>chatStore: createAssistantMessage(editedMsg.id)
-    chatStore->>chatStore: streamChatCompletion(...)
-    Note right of chatStore: Automatically regenerates response
-    deactivate chatStore
-
-    %% ═══════════════════════════════════════════════════════════════════════════
-    Note over UI,API: ❌ ERROR HANDLING
-    %% ═══════════════════════════════════════════════════════════════════════════
-
-    Note over chatStore: On stream error (non-abort):
-    chatStore->>chatStore: showErrorDialog(type, message)
-    Note right of chatStore: errorDialogState = {type: 'timeout'|'server', message}
-    chatStore->>convStore: removeMessageAtIndex(failedMsgIdx)
-    chatStore->>DbSvc: deleteMessage(failedMsgId)
-
-    %% ═══════════════════════════════════════════════════════════════════════════
-    Note over UI,API: 🤖 AGENTIC LOOP (when agenticConfig.enabled)
-    %% ═══════════════════════════════════════════════════════════════════════════
-
-    Note over agenticStore: agenticStore.runAgenticFlow(convId, messages, assistantMsg, options, signal)
-    activate agenticStore
-    agenticStore->>agenticStore: getSession(convId) or create new
-    agenticStore->>agenticStore: updateSession(turn: 0, running: true)
-
-    loop executeAgenticLoop (until no tool_calls or maxTurns)
-        agenticStore->>agenticStore: turn++
-        agenticStore->>ChatSvc: sendMessage(messages, options, signal)
-        ChatSvc->>API: POST /v1/chat/completions
-        API-->>ChatSvc: response with potential tool_calls
-        ChatSvc-->>agenticStore: onComplete(content, reasoning, timings, toolCalls)
-
-        alt response has tool_calls
-            agenticStore->>agenticStore: normalizeToolCalls(toolCalls)
-            loop for each tool_call
-                agenticStore->>agenticStore: updateSession(streamingToolCall)
-                agenticStore->>mcpStore: executeTool(mcpCall, signal)
-                mcpStore-->>agenticStore: tool result
-                agenticStore->>agenticStore: extractBase64Attachments(result)
-                agenticStore->>agenticStore: emitToolCallResult(convId, ...)
-                agenticStore->>convStore: addMessageToActive(toolResultMsg)
-                agenticStore->>DbSvc: createMessageBranch(toolResultMsg)
-            end
-            agenticStore->>agenticStore: Create new assistantMsg for next turn
-            Note right of agenticStore: Continue loop with updated messages
-        else no tool_calls (final response)
-            agenticStore->>agenticStore: buildFinalTimings(allTurns)
-            Note right of agenticStore: Break loop, return final response
-        end
-    end
-
-    agenticStore->>agenticStore: updateSession(running: false)
-    agenticStore-->>chatStore: final content, timings, model
-    deactivate agenticStore
-```

tools/ui/docs/flows/conversations-flow.md

@@ -1,183 +0,0 @@
-```mermaid
-sequenceDiagram
-    participant UI as 🧩 ChatSidebar / ChatScreen
-    participant convStore as 🗄️ conversationsStore
-    participant chatStore as 🗄️ chatStore
-    participant DbSvc as ⚙️ DatabaseService
-    participant IDB as 💾 IndexedDB
-
-    Note over convStore: State:<br/>conversations: DatabaseConversation[]<br/>activeConversation: DatabaseConversation | null<br/>activeMessages: DatabaseMessage[]<br/>isInitialized: boolean<br/>pendingMcpServerOverrides: Map&lt;string, McpServerOverride&gt;
-
-    %% ═══════════════════════════════════════════════════════════════════════════
-    Note over UI,IDB: 🚀 INITIALIZATION
-    %% ═══════════════════════════════════════════════════════════════════════════
-
-    Note over convStore: Auto-initialized in constructor (browser only)
-    convStore->>convStore: initialize()
-    activate convStore
-    convStore->>convStore: loadConversations()
-    convStore->>DbSvc: getAllConversations()
-    DbSvc->>IDB: SELECT * FROM conversations ORDER BY lastModified DESC
-    IDB-->>DbSvc: Conversation[]
-    DbSvc-->>convStore: conversations
-    convStore->>convStore: conversations = $state(data)
-    convStore->>convStore: isInitialized = true
-    deactivate convStore
-
-    %% ═══════════════════════════════════════════════════════════════════════════
-    Note over UI,IDB: ➕ CREATE CONVERSATION
-    %% ═══════════════════════════════════════════════════════════════════════════
-
-    UI->>convStore: createConversation(name?)
-    activate convStore
-    convStore->>DbSvc: createConversation(name || "New Chat")
-    DbSvc->>IDB: INSERT INTO conversations
-    IDB-->>DbSvc: conversation {id, name, lastModified, currNode: ""}
-    DbSvc-->>convStore: conversation
-    convStore->>convStore: conversations.unshift(conversation)
-    convStore->>convStore: activeConversation = $state(conversation)
-    convStore->>convStore: activeMessages = $state([])
-
-    alt pendingMcpServerOverrides has entries
-        loop each pending override
-            convStore->>DbSvc: Store MCP server override for new conversation
-        end
-        convStore->>convStore: clearPendingMcpServerOverrides()
-    end
-    deactivate convStore
-
-    %% ═══════════════════════════════════════════════════════════════════════════
-    Note over UI,IDB: 📂 LOAD CONVERSATION
-    %% ═══════════════════════════════════════════════════════════════════════════
-
-    UI->>convStore: loadConversation(convId)
-    activate convStore
-    convStore->>DbSvc: getConversation(convId)
-    DbSvc->>IDB: SELECT * FROM conversations WHERE id = ?
-    IDB-->>DbSvc: conversation
-    convStore->>convStore: activeConversation = $state(conversation)
-
-    convStore->>convStore: refreshActiveMessages()
-    convStore->>DbSvc: getConversationMessages(convId)
-    DbSvc->>IDB: SELECT * FROM messages WHERE convId = ?
-    IDB-->>DbSvc: allMessages[]
-    convStore->>convStore: filterByLeafNodeId(allMessages, currNode)
-    Note right of convStore: Filter to show only current branch path
-    convStore->>convStore: activeMessages = $state(filtered)
-
-    Note right of convStore: Route (+page.svelte) then calls:<br/>chatStore.syncLoadingStateForChat(convId)
-    deactivate convStore
-
-    %% ═══════════════════════════════════════════════════════════════════════════
-    Note over UI,IDB: 🌳 MESSAGE BRANCHING MODEL
-    %% ═══════════════════════════════════════════════════════════════════════════
-
-    Note over IDB: Message Tree Structure:<br/>- Each message has parent (null for root)<br/>- Each message has children[] array<br/>- Conversation.currNode points to active leaf<br/>- filterByLeafNodeId() traverses from root to currNode
-
-    rect rgb(240, 240, 255)
-        Note over convStore: Example Branch Structure:
-        Note over convStore: root → user1 → assistant1 → user2 → assistant2a (currNode)<br/>                                    ↘ assistant2b (alt branch)
-    end
-
-    %% ═══════════════════════════════════════════════════════════════════════════
-    Note over UI,IDB: ↔️ BRANCH NAVIGATION
-    %% ═══════════════════════════════════════════════════════════════════════════
-
-    UI->>convStore: navigateToSibling(msgId, direction)
-    activate convStore
-    convStore->>convStore: Find message in activeMessages
-    convStore->>convStore: Get parent message
-    convStore->>convStore: Find sibling in parent.children[]
-    convStore->>convStore: findLeafNode(siblingId, allMessages)
-    Note right of convStore: Navigate to leaf of sibling branch
-    convStore->>convStore: updateCurrentNode(leafId)
-    convStore->>DbSvc: updateCurrentNode(convId, leafId)
-    DbSvc->>IDB: UPDATE conversations SET currNode = ?
-    convStore->>convStore: refreshActiveMessages()
-    deactivate convStore
-
-    %% ═══════════════════════════════════════════════════════════════════════════
-    Note over UI,IDB: 📝 UPDATE CONVERSATION
-    %% ═══════════════════════════════════════════════════════════════════════════
-
-    UI->>convStore: updateConversationName(convId, newName)
-    activate convStore
-    convStore->>DbSvc: updateConversation(convId, {name: newName})
-    DbSvc->>IDB: UPDATE conversations SET name = ?
-    convStore->>convStore: Update in conversations array
-    deactivate convStore
-
-    Note over convStore: Auto-title update (after first response):
-    convStore->>convStore: updateConversationTitleWithConfirmation()
-    convStore->>convStore: titleUpdateConfirmationCallback?()
-    Note right of convStore: Shows dialog if title would change
-
-    %% ═══════════════════════════════════════════════════════════════════════════
-    Note over UI,IDB: 🗑️ DELETE CONVERSATION
-    %% ═══════════════════════════════════════════════════════════════════════════
-
-    UI->>convStore: deleteConversation(convId)
-    activate convStore
-    convStore->>DbSvc: deleteConversation(convId)
-    DbSvc->>IDB: DELETE FROM conversations WHERE id = ?
-    DbSvc->>IDB: DELETE FROM messages WHERE convId = ?
-    convStore->>convStore: conversations.filter(c => c.id !== convId)
-    alt deleted active conversation
-        convStore->>convStore: clearActiveConversation()
-    end
-    deactivate convStore
-
-    UI->>convStore: deleteAll()
-    activate convStore
-    convStore->>DbSvc: Delete all conversations and messages
-    convStore->>convStore: conversations = []
-    convStore->>convStore: clearActiveConversation()
-    deactivate convStore
-
-    %% ═══════════════════════════════════════════════════════════════════════════
-    Note over UI,IDB: � MCP SERVER PER-CHAT OVERRIDES
-    %% ═══════════════════════════════════════════════════════════════════════════
-
-    Note over convStore: Conversations can override which MCP servers are enabled.
-    Note over convStore: Uses pendingMcpServerOverrides before conversation<br/>is created, then persists to conversation metadata.
-
-    UI->>convStore: setMcpServerOverride(convId, serverName, override)
-    Note right of convStore: override = {enabled: boolean}
-
-    UI->>convStore: toggleMcpServerForChat(convId, serverName, enabled)
-    activate convStore
-    convStore->>convStore: setMcpServerOverride(convId, serverName, {enabled})
-    deactivate convStore
-
-    UI->>convStore: isMcpServerEnabledForChat(convId, serverName)
-    Note right of convStore: Check override → fall back to global MCP config
-
-    UI->>convStore: getAllMcpServerOverrides(convId)
-    Note right of convStore: Returns all overrides for a conversation
-
-    UI->>convStore: removeMcpServerOverride(convId, serverName)
-    UI->>convStore: getMcpServerOverride(convId, serverName)
-
-    %% ═══════════════════════════════════════════════════════════════════════════
-    Note over UI,IDB: 📤 EXPORT / 📥 IMPORT
-    %% ═══════════════════════════════════════════════════════════════════════════
-
-    UI->>convStore: exportAllConversations()
-    activate convStore
-    convStore->>DbSvc: getAllConversations()
-    loop each conversation
-        convStore->>DbSvc: getConversationMessages(convId)
-    end
-    convStore->>convStore: triggerDownload(JSON blob)
-    deactivate convStore
-
-    UI->>convStore: importConversations(file)
-    activate convStore
-    convStore->>convStore: Parse JSON file
-    convStore->>convStore: importConversationsData(parsed)
-    convStore->>DbSvc: importConversations(parsed)
-    Note right of DbSvc: Skips duplicate conversations<br/>(checks existing by ID)
-    DbSvc->>IDB: INSERT conversations + messages (skip existing)
-    convStore->>convStore: loadConversations()
-    deactivate convStore
-```

tools/ui/docs/flows/data-flow-simplified-model-mode.md

@@ -1,45 +0,0 @@
-```mermaid
-%% MODEL Mode Data Flow (single model)
-%% Detailed flows: ./flows/server-flow.mmd, ./flows/models-flow.mmd, ./flows/chat-flow.mmd
-
-sequenceDiagram
-    participant User as 👤 User
-    participant UI as 🧩 UI
-    participant Stores as 🗄️ Stores
-    participant DB as 💾 IndexedDB
-    participant API as 🌐 llama-server
-
-    Note over User,API: 🚀 Initialization (see: server-flow.mmd, models-flow.mmd)
-
-    UI->>Stores: initialize()
-    Stores->>DB: load conversations
-    Stores->>API: GET /props
-    API-->>Stores: server config + modalities
-    Stores->>API: GET /v1/models
-    API-->>Stores: single model (auto-selected)
-
-    Note over User,API: 💬 Chat Flow (see: chat-flow.mmd)
-
-    User->>UI: send message
-    UI->>Stores: sendMessage()
-    Stores->>DB: save user message
-    Stores->>API: POST /v1/chat/completions (stream)
-    loop streaming
-        API-->>Stores: SSE chunks
-        Stores-->>UI: reactive update
-    end
-    API-->>Stores: done + timings
-    Stores->>DB: save assistant message
-
-    Note over User,API: 🔁 Regenerate
-
-    User->>UI: regenerate
-    Stores->>DB: create message branch
-    Note right of Stores: same streaming flow
-
-    Note over User,API: ⏹️ Stop
-
-    User->>UI: stop
-    Stores->>Stores: abort stream
-    Stores->>DB: save partial response
-```

tools/ui/docs/flows/data-flow-simplified-router-mode.md

@@ -1,77 +0,0 @@
-```mermaid
-%% ROUTER Mode Data Flow (multi-model)
-%% Detailed flows: ./flows/server-flow.mmd, ./flows/models-flow.mmd, ./flows/chat-flow.mmd
-
-sequenceDiagram
-    participant User as 👤 User
-    participant UI as 🧩 UI
-    participant Stores as 🗄️ Stores
-    participant DB as 💾 IndexedDB
-    participant API as 🌐 llama-server
-
-    Note over User,API: 🚀 Initialization (see: server-flow.mmd, models-flow.mmd)
-
-    UI->>Stores: initialize()
-    Stores->>DB: load conversations
-    Stores->>API: GET /props
-    API-->>Stores: {role: "router"}
-    Stores->>API: GET /v1/models
-    API-->>Stores: models[] with status (loaded/available)
-    loop each loaded model
-        Stores->>API: GET /props?model=X
-        API-->>Stores: modalities (vision/audio)
-    end
-
-    Note over User,API: 🔄 Model Selection (see: models-flow.mmd)
-
-    User->>UI: select model
-    alt model not loaded
-        Stores->>API: POST /models/load
-        loop poll status
-            Stores->>API: GET /v1/models
-            API-->>Stores: check if loaded
-        end
-        Stores->>API: GET /props?model=X
-        API-->>Stores: cache modalities
-    end
-    Stores->>Stores: validate modalities vs conversation
-    alt valid
-        Stores->>Stores: select model
-    else invalid
-        Stores->>API: POST /models/unload
-        UI->>User: show error toast
-    end
-
-    Note over User,API: 💬 Chat Flow (see: chat-flow.mmd)
-
-    User->>UI: send message
-    UI->>Stores: sendMessage()
-    Stores->>DB: save user message
-    Stores->>API: POST /v1/chat/completions {model: X}
-    Note right of API: router forwards to model
-    loop streaming
-        API-->>Stores: SSE chunks + model info
-        Stores-->>UI: reactive update
-    end
-    API-->>Stores: done + timings
-    Stores->>DB: save assistant message + model used
-
-    Note over User,API: 🔁 Regenerate (optional: different model)
-
-    User->>UI: regenerate
-    Stores->>Stores: validate modalities up to this message
-    Stores->>DB: create message branch
-    Note right of Stores: same streaming flow
-
-    Note over User,API: ⏹️ Stop
-
-    User->>UI: stop
-    Stores->>Stores: abort stream
-    Stores->>DB: save partial response
-
-    Note over User,API: 🗑️ LRU Unloading
-
-    Note right of API: Server auto-unloads LRU models<br/>when cache full
-    User->>UI: select unloaded model
-    Note right of Stores: triggers load flow again
-```

tools/ui/docs/flows/database-flow.md

@@ -1,174 +0,0 @@
-```mermaid
-sequenceDiagram
-    participant Store as 🗄️ Stores
-    participant DbSvc as ⚙️ DatabaseService
-    participant Dexie as 📦 Dexie ORM
-    participant IDB as 💾 IndexedDB
-
-    Note over DbSvc: Stateless service - all methods static<br/>Database: "LlamacppWebui"
-
-    %% ═══════════════════════════════════════════════════════════════════════════
-    Note over Store,IDB: 📊 SCHEMA
-    %% ═══════════════════════════════════════════════════════════════════════════
-
-    rect rgb(240, 248, 255)
-        Note over IDB: conversations table:<br/>id (PK), lastModified, currNode, name
-    end
-
-    rect rgb(255, 248, 240)
-        Note over IDB: messages table:<br/>id (PK), convId (FK), type, role, timestamp,<br/>parent, children[], content, thinking,<br/>toolCalls, extra[], model, timings
-    end
-
-    %% ═══════════════════════════════════════════════════════════════════════════
-    Note over Store,IDB: 💬 CONVERSATIONS CRUD
-    %% ═══════════════════════════════════════════════════════════════════════════
-
-    Store->>DbSvc: createConversation(name)
-    activate DbSvc
-    DbSvc->>DbSvc: Generate UUID
-    DbSvc->>Dexie: db.conversations.add({id, name, lastModified, currNode: ""})
-    Dexie->>IDB: INSERT
-    IDB-->>Dexie: success
-    DbSvc-->>Store: DatabaseConversation
-    deactivate DbSvc
-
-    Store->>DbSvc: getConversation(convId)
-    DbSvc->>Dexie: db.conversations.get(convId)
-    Dexie->>IDB: SELECT WHERE id = ?
-    IDB-->>DbSvc: DatabaseConversation
-
-    Store->>DbSvc: getAllConversations()
-    DbSvc->>Dexie: db.conversations.orderBy('lastModified').reverse().toArray()
-    Dexie->>IDB: SELECT ORDER BY lastModified DESC
-    IDB-->>DbSvc: DatabaseConversation[]
-
-    Store->>DbSvc: updateConversation(convId, updates)
-    DbSvc->>Dexie: db.conversations.update(convId, {...updates, lastModified})
-    Dexie->>IDB: UPDATE
-
-    Store->>DbSvc: deleteConversation(convId)
-    activate DbSvc
-    DbSvc->>Dexie: db.conversations.delete(convId)
-    Dexie->>IDB: DELETE FROM conversations
-    DbSvc->>Dexie: db.messages.where('convId').equals(convId).delete()
-    Dexie->>IDB: DELETE FROM messages WHERE convId = ?
-    deactivate DbSvc
-
-    %% ═══════════════════════════════════════════════════════════════════════════
-    Note over Store,IDB: 📝 MESSAGES CRUD
-    %% ═══════════════════════════════════════════════════════════════════════════
-
-    Store->>DbSvc: createRootMessage(convId)
-    activate DbSvc
-    DbSvc->>DbSvc: Create root message {type: "root", parent: null}
-    DbSvc->>Dexie: db.messages.add(rootMsg)
-    Dexie->>IDB: INSERT
-    DbSvc-->>Store: rootMessageId
-    deactivate DbSvc
-
-    Store->>DbSvc: createSystemMessage(convId, content, parentId)
-    activate DbSvc
-    DbSvc->>DbSvc: Create message {role: "system", parent: parentId}
-    DbSvc->>Dexie: db.messages.add(systemMsg)
-    Dexie->>IDB: INSERT
-    DbSvc-->>Store: DatabaseMessage
-    deactivate DbSvc
-
-    Store->>DbSvc: createMessageBranch(message, parentId)
-    activate DbSvc
-    DbSvc->>DbSvc: Generate UUID for new message
-    DbSvc->>Dexie: db.messages.add({...message, id, parent: parentId})
-    Dexie->>IDB: INSERT message
-
-    alt parentId exists
-        DbSvc->>Dexie: db.messages.get(parentId)
-        Dexie->>IDB: SELECT parent
-        DbSvc->>DbSvc: parent.children.push(newId)
-        DbSvc->>Dexie: db.messages.update(parentId, {children})
-        Dexie->>IDB: UPDATE parent.children
-    end
-
-    DbSvc->>Dexie: db.conversations.update(convId, {currNode: newId})
-    Dexie->>IDB: UPDATE conversation.currNode
-    DbSvc-->>Store: DatabaseMessage
-    deactivate DbSvc
-
-    Store->>DbSvc: getConversationMessages(convId)
-    DbSvc->>Dexie: db.messages.where('convId').equals(convId).toArray()
-    Dexie->>IDB: SELECT WHERE convId = ?
-    IDB-->>DbSvc: DatabaseMessage[]
-
-    Store->>DbSvc: updateMessage(msgId, updates)
-    DbSvc->>Dexie: db.messages.update(msgId, updates)
-    Dexie->>IDB: UPDATE
-
-    Store->>DbSvc: deleteMessage(msgId)
-    DbSvc->>Dexie: db.messages.delete(msgId)
-    Dexie->>IDB: DELETE
-
-    %% ═══════════════════════════════════════════════════════════════════════════
-    Note over Store,IDB: 🌳 BRANCHING OPERATIONS
-    %% ═══════════════════════════════════════════════════════════════════════════
-
-    Store->>DbSvc: updateCurrentNode(convId, nodeId)
-    DbSvc->>Dexie: db.conversations.update(convId, {currNode: nodeId, lastModified})
-    Dexie->>IDB: UPDATE
-
-    Store->>DbSvc: deleteMessageCascading(msgId)
-    activate DbSvc
-    DbSvc->>DbSvc: findDescendantMessages(msgId, allMessages)
-    Note right of DbSvc: Recursively find all children
-    loop each descendant
-        DbSvc->>Dexie: db.messages.delete(descendantId)
-        Dexie->>IDB: DELETE
-    end
-    DbSvc->>Dexie: db.messages.delete(msgId)
-    Dexie->>IDB: DELETE target message
-
-    alt target message has a parent
-        DbSvc->>Dexie: db.messages.get(parentId)
-        DbSvc->>DbSvc: parent.children.filter(id !== msgId)
-        DbSvc->>Dexie: db.messages.update(parentId, {children})
-        Note right of DbSvc: Remove deleted message from parent's children[]
-    end
-    deactivate DbSvc
-
-    %% ═══════════════════════════════════════════════════════════════════════════
-    Note over Store,IDB: 📥 IMPORT
-    %% ═══════════════════════════════════════════════════════════════════════════
-
-    Store->>DbSvc: importConversations(data)
-    activate DbSvc
-    loop each conversation in data
-        DbSvc->>Dexie: db.conversations.get(conv.id)
-        alt conversation already exists
-            Note right of DbSvc: Skip duplicate (keep existing)
-        else conversation is new
-            DbSvc->>Dexie: db.conversations.add(conversation)
-            Dexie->>IDB: INSERT conversation
-            loop each message
-                DbSvc->>Dexie: db.messages.add(message)
-                Dexie->>IDB: INSERT message
-            end
-        end
-    end
-    deactivate DbSvc
-
-    %% ═══════════════════════════════════════════════════════════════════════════
-    Note over Store,IDB: 🔗 MESSAGE TREE UTILITIES
-    %% ═══════════════════════════════════════════════════════════════════════════
-
-    Note over DbSvc: Used by stores (imported from utils):
-
-    rect rgb(240, 255, 240)
-        Note over DbSvc: filterByLeafNodeId(messages, leafId)<br/>→ Returns path from root to leaf<br/>→ Used to display current branch
-    end
-
-    rect rgb(240, 255, 240)
-        Note over DbSvc: findLeafNode(startId, messages)<br/>→ Traverse to deepest child<br/>→ Used for branch navigation
-    end
-
-    rect rgb(240, 255, 240)
-        Note over DbSvc: findDescendantMessages(msgId, messages)<br/>→ Find all children recursively<br/>→ Used for cascading deletes
-    end
-```

tools/ui/docs/flows/mcp-flow.md

@@ -1,226 +0,0 @@
-```mermaid
-sequenceDiagram
-    participant UI as 🧩 McpServersSettings / ChatForm
-    participant chatStore as 🗄️ chatStore
-    participant mcpStore as 🗄️ mcpStore
-    participant mcpResStore as 🗄️ mcpResourceStore
-    participant convStore as 🗄️ conversationsStore
-    participant MCPSvc as ⚙️ MCPService
-    participant LS as 💾 LocalStorage
-    participant ExtMCP as 🔌 External MCP Server
-
-    Note over mcpStore: State:<br/>isInitializing, error<br/>toolCount, connectedServers<br/>healthChecks (Map)<br/>connections (Map)<br/>toolsIndex (Map)<br/>serverConfigs (Map)
-
-    Note over mcpResStore: State:<br/>serverResources (Map)<br/>cachedResources (Map)<br/>subscriptions (Map)<br/>attachments[]
-
-    %% ═══════════════════════════════════════════════════════════════════════════
-    Note over UI,ExtMCP: 🚀 INITIALIZATION (App Startup)
-    %% ═══════════════════════════════════════════════════════════════════════════
-
-    UI->>mcpStore: ensureInitialized()
-    activate mcpStore
-
-    mcpStore->>LS: get(MCP_SERVERS_LOCALSTORAGE_KEY)
-    LS-->>mcpStore: MCPServerSettingsEntry[]
-
-    mcpStore->>mcpStore: parseServerSettings(servers)
-    Note right of mcpStore: Filter enabled servers<br/>Build MCPServerConfig objects<br/>Per-chat overrides checked via convStore
-
-    loop For each enabled server
-        mcpStore->>mcpStore: runHealthCheck(serverId)
-        mcpStore->>mcpStore: updateHealthCheck(id, CONNECTING)
-
-        mcpStore->>MCPSvc: connect(serverName, config, clientInfo, capabilities, onPhase)
-        activate MCPSvc
-
-        MCPSvc->>MCPSvc: createTransport(config)
-        Note right of MCPSvc: WebSocket / StreamableHTTP / SSE<br/>with optional CORS proxy
-
-        MCPSvc->>ExtMCP: Transport handshake
-        ExtMCP-->>MCPSvc: Connection established
-
-        MCPSvc->>ExtMCP: Initialize request
-        Note right of ExtMCP: Exchange capabilities<br/>Server info, protocol version
-
-        ExtMCP-->>MCPSvc: InitializeResult (serverInfo, capabilities)
-
-        MCPSvc->>ExtMCP: listTools()
-        ExtMCP-->>MCPSvc: Tool[]
-
-        MCPSvc-->>mcpStore: MCPConnection
-        deactivate MCPSvc
-
-        mcpStore->>mcpStore: connections.set(serverName, connection)
-        mcpStore->>mcpStore: indexTools(connection.tools, serverName)
-        Note right of mcpStore: toolsIndex.set(toolName, serverName)<br/>Handle name conflicts with prefixes
-
-        mcpStore->>mcpStore: updateHealthCheck(id, SUCCESS)
-        mcpStore->>mcpStore: _connectedServers.push(serverName)
-
-        alt Server supports resources
-            mcpStore->>MCPSvc: listAllResources(connection)
-            MCPSvc->>ExtMCP: listResources()
-            ExtMCP-->>MCPSvc: MCPResource[]
-            MCPSvc-->>mcpStore: resources
-
-            mcpStore->>MCPSvc: listAllResourceTemplates(connection)
-            MCPSvc->>ExtMCP: listResourceTemplates()
-            ExtMCP-->>MCPSvc: MCPResourceTemplate[]
-            MCPSvc-->>mcpStore: templates
-
-            mcpStore->>mcpResStore: setServerResources(serverName, resources, templates)
-        end
-    end
-
-    mcpStore->>mcpStore: _isInitializing = false
-    deactivate mcpStore
-
-    %% ═══════════════════════════════════════════════════════════════════════════
-    Note over UI,ExtMCP: 🔧 TOOL EXECUTION (Chat with Tools)
-    %% ═══════════════════════════════════════════════════════════════════════════
-
-    UI->>mcpStore: executeTool(mcpCall: MCPToolCall, signal?)
-    activate mcpStore
-
-    mcpStore->>mcpStore: toolsIndex.get(mcpCall.function.name)
-    Note right of mcpStore: Resolve serverName from toolsIndex<br/>MCPToolCall = {id, type, function: {name, arguments}}
-
-    mcpStore->>mcpStore: acquireConnection()
-    Note right of mcpStore: activeFlowCount++<br/>Prevent shutdown during execution
-
-    mcpStore->>mcpStore: connection = connections.get(serverName)
-
-    mcpStore->>MCPSvc: callTool(connection, {name, arguments}, signal)
-    activate MCPSvc
-
-    MCPSvc->>MCPSvc: throwIfAborted(signal)
-    MCPSvc->>ExtMCP: callTool(name, arguments)
-
-    alt Tool execution success
-        ExtMCP-->>MCPSvc: ToolCallResult (content, isError)
-        MCPSvc->>MCPSvc: formatToolResult(result)
-        Note right of MCPSvc: Handle text, image (base64),<br/>embedded resource content
-        MCPSvc-->>mcpStore: ToolExecutionResult
-    else Tool execution error
-        ExtMCP-->>MCPSvc: Error
-        MCPSvc-->>mcpStore: throw Error
-    else Aborted
-        MCPSvc-->>mcpStore: throw AbortError
-    end
-
-    deactivate MCPSvc
-
-    mcpStore->>mcpStore: releaseConnection()
-    Note right of mcpStore: activeFlowCount--
-
-    mcpStore-->>UI: ToolExecutionResult
-    deactivate mcpStore
-
-    %% ═══════════════════════════════════════════════════════════════════════════
-    Note over UI,ExtMCP: � RESOURCE ATTACHMENT CONSUMPTION
-    %% ═══════════════════════════════════════════════════════════════════════════
-
-    chatStore->>mcpStore: consumeResourceAttachmentsAsExtras()
-    activate mcpStore
-    mcpStore->>mcpResStore: getAttachments()
-    mcpResStore-->>mcpStore: MCPResourceAttachment[]
-    mcpStore->>mcpStore: Convert attachments to message extras
-    mcpStore->>mcpResStore: clearAttachments()
-    mcpStore-->>chatStore: MessageExtra[] (for user message)
-    deactivate mcpStore
-
-    %% ═══════════════════════════════════════════════════════════════════════════
-    Note over UI,ExtMCP: �📝 PROMPT OPERATIONS
-    %% ═══════════════════════════════════════════════════════════════════════════
-
-    UI->>mcpStore: getAllPrompts()
-    activate mcpStore
-
-    loop For each connected server with prompts capability
-        mcpStore->>MCPSvc: listPrompts(connection)
-        MCPSvc->>ExtMCP: listPrompts()
-        ExtMCP-->>MCPSvc: Prompt[]
-        MCPSvc-->>mcpStore: prompts
-    end
-
-    mcpStore-->>UI: MCPPromptInfo[] (with serverName)
-    deactivate mcpStore
-
-    UI->>mcpStore: getPrompt(serverName, promptName, args?)
-    activate mcpStore
-
-    mcpStore->>MCPSvc: getPrompt(connection, name, args)
-    MCPSvc->>ExtMCP: getPrompt({name, arguments})
-    ExtMCP-->>MCPSvc: GetPromptResult (messages)
-    MCPSvc-->>mcpStore: GetPromptResult
-
-    mcpStore-->>UI: GetPromptResult
-    deactivate mcpStore
-
-    %% ═══════════════════════════════════════════════════════════════════════════
-    Note over UI,ExtMCP: 📁 RESOURCE OPERATIONS
-    %% ═══════════════════════════════════════════════════════════════════════════
-
-    UI->>mcpResStore: addAttachment(resourceInfo)
-    activate mcpResStore
-    mcpResStore->>mcpResStore: Create MCPResourceAttachment (loading: true)
-    mcpResStore-->>UI: attachment
-
-    UI->>mcpStore: readResource(serverName, uri)
-    activate mcpStore
-
-    mcpStore->>MCPSvc: readResource(connection, uri)
-    MCPSvc->>ExtMCP: readResource({uri})
-    ExtMCP-->>MCPSvc: MCPReadResourceResult (contents)
-    MCPSvc-->>mcpStore: contents
-
-    mcpStore-->>UI: MCPResourceContent[]
-    deactivate mcpStore
-
-    UI->>mcpResStore: updateAttachmentContent(attachmentId, content)
-    mcpResStore->>mcpResStore: cacheResourceContent(resource, content)
-    deactivate mcpResStore
-
-    %% ═══════════════════════════════════════════════════════════════════════════
-    Note over UI,ExtMCP: 🔄 AUTO-RECONNECTION
-    %% ═══════════════════════════════════════════════════════════════════════════
-
-    Note over mcpStore: On WebSocket close or connection error:
-    mcpStore->>mcpStore: autoReconnect(serverName, attempt)
-    activate mcpStore
-
-    mcpStore->>mcpStore: Calculate backoff delay
-    Note right of mcpStore: delay = min(30s, 1s * 2^attempt)
-
-    mcpStore->>mcpStore: Wait for delay
-    mcpStore->>mcpStore: reconnectServer(serverName)
-
-    alt Reconnection success
-        mcpStore->>mcpStore: updateHealthCheck(id, SUCCESS)
-    else Max attempts reached
-        mcpStore->>mcpStore: updateHealthCheck(id, ERROR)
-    end
-    deactivate mcpStore
-
-    %% ═══════════════════════════════════════════════════════════════════════════
-    Note over UI,ExtMCP: 🛑 SHUTDOWN
-    %% ═══════════════════════════════════════════════════════════════════════════
-
-    UI->>mcpStore: shutdown()
-    activate mcpStore
-
-    mcpStore->>mcpStore: Wait for activeFlowCount == 0
-
-    loop For each connection
-        mcpStore->>MCPSvc: disconnect(connection)
-        MCPSvc->>MCPSvc: transport.onclose = undefined
-        MCPSvc->>ExtMCP: close()
-    end
-
-    mcpStore->>mcpStore: connections.clear()
-    mcpStore->>mcpStore: toolsIndex.clear()
-    mcpStore->>mcpStore: _connectedServers = []
-
-    mcpStore->>mcpResStore: clear()
-    deactivate mcpStore
-```

tools/ui/docs/flows/models-flow.md

@@ -1,181 +0,0 @@
-```mermaid
-sequenceDiagram
-    participant UI as 🧩 ModelsSelector
-    participant Hooks as 🪝 useModelChangeValidation
-    participant modelsStore as 🗄️ modelsStore
-    participant serverStore as 🗄️ serverStore
-    participant convStore as 🗄️ conversationsStore
-    participant ModelsSvc as ⚙️ ModelsService
-    participant PropsSvc as ⚙️ PropsService
-    participant API as 🌐 llama-server
-
-    Note over modelsStore: State:<br/>models: ModelOption[]<br/>routerModels: ApiModelDataEntry[]<br/>selectedModelId, selectedModelName<br/>loading, updating, error<br/>modelLoadingStates (Map)<br/>modelPropsCache (Map)<br/>propsCacheVersion
-
-    %% ═══════════════════════════════════════════════════════════════════════════
-    Note over UI,API: 🚀 INITIALIZATION (MODEL mode)
-    %% ═══════════════════════════════════════════════════════════════════════════
-
-    UI->>modelsStore: fetch()
-    activate modelsStore
-    modelsStore->>modelsStore: loading = true
-
-    alt serverStore.props not loaded
-        modelsStore->>serverStore: fetch()
-        Note over serverStore: → see server-flow.mmd
-    end
-
-    modelsStore->>ModelsSvc: list()
-    ModelsSvc->>API: GET /v1/models
-    API-->>ModelsSvc: ApiModelListResponse {data: [model]}
-
-    modelsStore->>modelsStore: models = $state(mapped)
-    Note right of modelsStore: Map to ModelOption[]:<br/>{id, name, model, description, capabilities}
-
-    Note over modelsStore: MODEL mode: Get modalities from serverStore.props
-    modelsStore->>modelsStore: modelPropsCache.set(model.id, serverStore.props)
-    modelsStore->>modelsStore: models[0].modalities = props.modalities
-
-    modelsStore->>modelsStore: Auto-select single model
-    Note right of modelsStore: selectedModelId = models[0].id
-    modelsStore->>modelsStore: loading = false
-    deactivate modelsStore
-
-    %% ═══════════════════════════════════════════════════════════════════════════
-    Note over UI,API: 🚀 INITIALIZATION (ROUTER mode)
-    %% ═══════════════════════════════════════════════════════════════════════════
-
-    UI->>modelsStore: fetch()
-    activate modelsStore
-    modelsStore->>ModelsSvc: list()
-    ModelsSvc->>API: GET /v1/models
-    API-->>ModelsSvc: ApiModelListResponse
-    modelsStore->>modelsStore: models = $state(mapped)
-    deactivate modelsStore
-
-    Note over UI: After models loaded, layout triggers:
-    UI->>modelsStore: fetchRouterModels()
-    activate modelsStore
-    modelsStore->>ModelsSvc: listRouter()
-    ModelsSvc->>API: GET /v1/models
-    API-->>ModelsSvc: ApiRouterModelsListResponse
-    Note right of API: {data: [{id, status, path, in_cache}]}
-    modelsStore->>modelsStore: routerModels = $state(data)
-
-    modelsStore->>modelsStore: fetchModalitiesForLoadedModels()
-    loop each model where status === "loaded"
-        modelsStore->>PropsSvc: fetchForModel(modelId)
-        PropsSvc->>API: GET /props?model={modelId}
-        API-->>PropsSvc: ApiLlamaCppServerProps
-        modelsStore->>modelsStore: modelPropsCache.set(modelId, props)
-    end
-    modelsStore->>modelsStore: propsCacheVersion++
-    deactivate modelsStore
-
-    %% ═══════════════════════════════════════════════════════════════════════════
-    Note over UI,API: 🔄 MODEL SELECTION (ROUTER mode)
-    %% ═══════════════════════════════════════════════════════════════════════════
-
-    UI->>Hooks: useModelChangeValidation({getRequiredModalities, onSuccess?, onValidationFailure?})
-    Note over Hooks: Hook configured per-component:<br/>ChatForm: getRequiredModalities = usedModalities<br/>ChatMessage: getRequiredModalities = getModalitiesUpToMessage(msgId)
-
-    UI->>Hooks: handleModelChange(modelId, modelName)
-    activate Hooks
-    Hooks->>Hooks: previousSelectedModelId = modelsStore.selectedModelId
-    Hooks->>modelsStore: isModelLoaded(modelName)?
-
-    alt model NOT loaded
-        Hooks->>modelsStore: loadModel(modelName)
-        Note over modelsStore: → see LOAD MODEL section below
-    end
-
-    Note over Hooks: Always fetch props (from cache or API)
-    Hooks->>modelsStore: fetchModelProps(modelName)
-    modelsStore-->>Hooks: props
-
-    Hooks->>convStore: getRequiredModalities()
-    convStore-->>Hooks: {vision, audio}
-
-    Hooks->>Hooks: Validate: model.modalities ⊇ required?
-
-    alt validation PASSED
-        Hooks->>modelsStore: selectModelById(modelId)
-        Hooks-->>UI: return true
-    else validation FAILED
-        Hooks->>UI: toast.error("Model doesn't support required modalities")
-        alt model was just loaded
-            Hooks->>modelsStore: unloadModel(modelName)
-        end
-        alt onValidationFailure provided
-            Hooks->>modelsStore: selectModelById(previousSelectedModelId)
-        end
-        Hooks-->>UI: return false
-    end
-    deactivate Hooks
-
-    %% ═══════════════════════════════════════════════════════════════════════════
-    Note over UI,API: ⬆️ LOAD MODEL (ROUTER mode)
-    %% ═══════════════════════════════════════════════════════════════════════════
-
-    modelsStore->>modelsStore: loadModel(modelId)
-    activate modelsStore
-
-    alt already loaded
-        modelsStore-->>modelsStore: return (no-op)
-    end
-
-    modelsStore->>modelsStore: modelLoadingStates.set(modelId, true)
-    modelsStore->>ModelsSvc: load(modelId)
-    ModelsSvc->>API: POST /models/load {model: modelId}
-    API-->>ModelsSvc: {status: "loading"}
-
-    modelsStore->>modelsStore: pollForModelStatus(modelId, LOADED)
-    loop poll every 500ms (max 60 attempts)
-        modelsStore->>modelsStore: fetchRouterModels()
-        modelsStore->>ModelsSvc: listRouter()
-        ModelsSvc->>API: GET /v1/models
-        API-->>ModelsSvc: models[]
-        modelsStore->>modelsStore: getModelStatus(modelId)
-        alt status === LOADED
-            Note right of modelsStore: break loop
-        else status === LOADING
-            Note right of modelsStore: wait 500ms, continue
-        end
-    end
-
-    modelsStore->>modelsStore: updateModelModalities(modelId)
-    modelsStore->>PropsSvc: fetchForModel(modelId)
-    PropsSvc->>API: GET /props?model={modelId}
-    API-->>PropsSvc: props with modalities
-    modelsStore->>modelsStore: modelPropsCache.set(modelId, props)
-    modelsStore->>modelsStore: propsCacheVersion++
-
-    modelsStore->>modelsStore: modelLoadingStates.set(modelId, false)
-    deactivate modelsStore
-
-    %% ═══════════════════════════════════════════════════════════════════════════
-    Note over UI,API: ⬇️ UNLOAD MODEL (ROUTER mode)
-    %% ═══════════════════════════════════════════════════════════════════════════
-
-    modelsStore->>modelsStore: unloadModel(modelId)
-    activate modelsStore
-    modelsStore->>modelsStore: modelLoadingStates.set(modelId, true)
-    modelsStore->>ModelsSvc: unload(modelId)
-    ModelsSvc->>API: POST /models/unload {model: modelId}
-
-    modelsStore->>modelsStore: pollForModelStatus(modelId, UNLOADED)
-    loop poll until unloaded
-        modelsStore->>ModelsSvc: listRouter()
-        ModelsSvc->>API: GET /v1/models
-    end
-
-    modelsStore->>modelsStore: modelLoadingStates.set(modelId, false)
-    deactivate modelsStore
-
-    %% ═══════════════════════════════════════════════════════════════════════════
-    Note over UI,API: 📊 COMPUTED GETTERS
-    %% ═══════════════════════════════════════════════════════════════════════════
-
-    Note over modelsStore: Getters:<br/>- selectedModel: ModelOption | null<br/>- loadedModelIds: string[] (from routerModels)<br/>- loadingModelIds: string[] (from modelLoadingStates)<br/>- singleModelName: string | null (MODEL mode only)
-
-    Note over modelsStore: Modality helpers:<br/>- getModelModalities(modelId): {vision, audio}<br/>- modelSupportsVision(modelId): boolean<br/>- modelSupportsAudio(modelId): boolean
-```

tools/ui/docs/flows/server-flow.md

@@ -1,76 +0,0 @@
-```mermaid
-sequenceDiagram
-    participant UI as 🧩 +layout.svelte
-    participant serverStore as 🗄️ serverStore
-    participant PropsSvc as ⚙️ PropsService
-    participant API as 🌐 llama-server
-
-    Note over serverStore: State:<br/>props: ApiLlamaCppServerProps | null<br/>loading, error<br/>role: ServerRole | null (MODEL | ROUTER)<br/>fetchPromise (deduplication)
-
-    %% ═══════════════════════════════════════════════════════════════════════════
-    Note over UI,API: 🚀 INITIALIZATION
-    %% ═══════════════════════════════════════════════════════════════════════════
-
-    UI->>serverStore: fetch()
-    activate serverStore
-
-    alt fetchPromise exists (already fetching)
-        serverStore-->>UI: return fetchPromise
-        Note right of serverStore: Deduplicate concurrent calls
-    end
-
-    serverStore->>serverStore: loading = true
-    serverStore->>serverStore: fetchPromise = new Promise()
-
-    serverStore->>PropsSvc: fetch()
-    PropsSvc->>API: GET /props
-    API-->>PropsSvc: ApiLlamaCppServerProps
-    Note right of API: {role, model_path, model_alias,<br/>modalities, default_generation_settings, ...}
-
-    PropsSvc-->>serverStore: props
-    serverStore->>serverStore: props = $state(data)
-
-    serverStore->>serverStore: detectRole(props)
-    Note right of serverStore: role = props.role === "router"<br/>  ? ServerRole.ROUTER<br/>  : ServerRole.MODEL
-
-    serverStore->>serverStore: loading = false
-    serverStore->>serverStore: fetchPromise = null
-    deactivate serverStore
-
-    %% ═══════════════════════════════════════════════════════════════════════════
-    Note over UI,API: 📊 COMPUTED GETTERS
-    %% ═══════════════════════════════════════════════════════════════════════════
-
-    Note over serverStore: Getters from props:
-
-    rect rgb(240, 255, 240)
-        Note over serverStore: defaultParams<br/>→ props.default_generation_settings.params<br/>(temperature, top_p, top_k, etc.)
-    end
-
-    rect rgb(240, 255, 240)
-        Note over serverStore: contextSize<br/>→ props.default_generation_settings.n_ctx
-    end
-
-    rect rgb(255, 240, 240)
-        Note over serverStore: isRouterMode<br/>→ role === ServerRole.ROUTER
-    end
-
-    rect rgb(255, 240, 240)
-        Note over serverStore: isModelMode<br/>→ role === ServerRole.MODEL
-    end
-
-    %% ═══════════════════════════════════════════════════════════════════════════
-    Note over UI,API: 🔗 RELATIONSHIPS
-    %% ═══════════════════════════════════════════════════════════════════════════
-
-    Note over serverStore: Used by:
-    Note right of serverStore: - modelsStore: role detection, MODEL mode modalities<br/>- settingsStore: syncWithServerDefaults (defaultParams)<br/>- chatStore: contextSize for processing state<br/>- UI components: isRouterMode for conditional rendering
-
-    %% ═══════════════════════════════════════════════════════════════════════════
-    Note over UI,API: ❌ ERROR HANDLING
-    %% ═══════════════════════════════════════════════════════════════════════════
-
-    Note over serverStore: getErrorMessage(): string | null<br/>Returns formatted error for UI display
-
-    Note over serverStore: clear(): void<br/>Resets all state (props, error, loading, role)
-```

tools/ui/docs/flows/settings-flow.md

@@ -1,156 +0,0 @@
-```mermaid
-sequenceDiagram
-    participant UI as 🧩 ChatSettings
-    participant settingsStore as 🗄️ settingsStore
-    participant serverStore as 🗄️ serverStore
-    participant ParamSvc as ⚙️ ParameterSyncService
-    participant LS as 💾 LocalStorage
-
-    Note over settingsStore: State:<br/>config: SettingsConfigType<br/>theme: string ("auto" | "light" | "dark")<br/>isInitialized: boolean<br/>userOverrides: Set&lt;string&gt;
-
-    %% ═══════════════════════════════════════════════════════════════════════════
-    Note over UI,LS: 🚀 INITIALIZATION
-    %% ═══════════════════════════════════════════════════════════════════════════
-
-    Note over settingsStore: Auto-initialized in constructor (browser only)
-    settingsStore->>settingsStore: initialize()
-    activate settingsStore
-
-    settingsStore->>settingsStore: loadConfig()
-    settingsStore->>LS: get("llama-config")
-    LS-->>settingsStore: StoredConfig | null
-
-    alt config exists
-        settingsStore->>settingsStore: Merge with SETTING_CONFIG_DEFAULT
-        Note right of settingsStore: Fill missing keys with defaults
-    else no config
-        settingsStore->>settingsStore: config = SETTING_CONFIG_DEFAULT
-    end
-
-    settingsStore->>LS: get("llama-userOverrides")
-    LS-->>settingsStore: string[] | null
-    settingsStore->>settingsStore: userOverrides = new Set(data)
-
-    settingsStore->>settingsStore: loadTheme()
-    settingsStore->>LS: get("llama-theme")
-    LS-->>settingsStore: theme | "auto"
-
-    settingsStore->>settingsStore: isInitialized = true
-    deactivate settingsStore
-
-    %% ═══════════════════════════════════════════════════════════════════════════
-    Note over UI,LS: 🔄 SYNC WITH SERVER DEFAULTS
-    %% ═══════════════════════════════════════════════════════════════════════════
-
-    Note over UI: Triggered from +layout.svelte when serverStore.props loaded
-    UI->>settingsStore: syncWithServerDefaults()
-    activate settingsStore
-
-    settingsStore->>serverStore: defaultParams
-    serverStore-->>settingsStore: {temperature, top_p, top_k, ...}
-
-    loop each SYNCABLE_PARAMETER
-        alt key NOT in userOverrides
-            settingsStore->>settingsStore: config[key] = serverDefault[key]
-            Note right of settingsStore: Non-overridden params adopt server default
-        else key in userOverrides
-            Note right of settingsStore: Keep user value, skip server default
-        end
-    end
-
-    alt serverStore.props has uiSettings
-        settingsStore->>settingsStore: Apply uiSettings from server
-        Note right of settingsStore: Server-provided UI settings<br/>(e.g. showRawOutputSwitch)
-    end
-
-    settingsStore->>settingsStore: saveConfig()
-    deactivate settingsStore
-
-    %% ═══════════════════════════════════════════════════════════════════════════
-    Note over UI,LS: ⚙️ UPDATE CONFIG
-    %% ═══════════════════════════════════════════════════════════════════════════
-
-    UI->>settingsStore: updateConfig(key, value)
-    activate settingsStore
-    settingsStore->>settingsStore: config[key] = value
-
-    alt value matches server default for key
-        settingsStore->>settingsStore: userOverrides.delete(key)
-        Note right of settingsStore: Matches server default, remove override
-    else value differs from server default
-        settingsStore->>settingsStore: userOverrides.add(key)
-        Note right of settingsStore: Mark as user-modified (won't be overwritten)
-    end
-
-    settingsStore->>settingsStore: saveConfig()
-    settingsStore->>LS: set(CONFIG_LOCALSTORAGE_KEY, config)
-    settingsStore->>LS: set(USER_OVERRIDES_LOCALSTORAGE_KEY, [...userOverrides])
-    deactivate settingsStore
-
-    UI->>settingsStore: updateMultipleConfig({key1: val1, key2: val2})
-    activate settingsStore
-    Note right of settingsStore: Batch update, single save
-    settingsStore->>settingsStore: For each key: config[key] = value
-    settingsStore->>settingsStore: For each key: userOverrides.add(key)
-    settingsStore->>settingsStore: saveConfig()
-    deactivate settingsStore
-
-    %% ═══════════════════════════════════════════════════════════════════════════
-    Note over UI,LS: 🔄 RESET
-    %% ═══════════════════════════════════════════════════════════════════════════
-
-    UI->>settingsStore: resetConfig()
-    activate settingsStore
-    settingsStore->>settingsStore: config = {...SETTING_CONFIG_DEFAULT}
-    settingsStore->>settingsStore: userOverrides.clear()
-    Note right of settingsStore: All params reset to defaults<br/>Next syncWithServerDefaults will adopt server values
-    settingsStore->>settingsStore: saveConfig()
-    deactivate settingsStore
-
-    UI->>settingsStore: resetParameterToServerDefault(key)
-    activate settingsStore
-    settingsStore->>settingsStore: userOverrides.delete(key)
-    settingsStore->>serverStore: defaultParams[key]
-    settingsStore->>settingsStore: config[key] = serverDefault
-    settingsStore->>settingsStore: saveConfig()
-    deactivate settingsStore
-
-    %% ═══════════════════════════════════════════════════════════════════════════
-    Note over UI,LS: 🎨 THEME
-    %% ═══════════════════════════════════════════════════════════════════════════
-
-    UI->>settingsStore: updateTheme(newTheme)
-    activate settingsStore
-    settingsStore->>settingsStore: theme = newTheme
-    settingsStore->>settingsStore: saveTheme()
-    settingsStore->>LS: set("llama-theme", theme)
-    deactivate settingsStore
-
-    %% ═══════════════════════════════════════════════════════════════════════════
-    Note over UI,LS: 📊 PARAMETER INFO
-    %% ═══════════════════════════════════════════════════════════════════════════
-
-    UI->>settingsStore: getParameterInfo(key)
-    settingsStore->>ParamSvc: getParameterInfo(key, config, serverDefaults, userOverrides)
-    ParamSvc-->>settingsStore: ParameterInfo
-    Note right of ParamSvc: {<br/>  currentValue,<br/>  serverDefault,<br/>  isUserOverride: boolean,<br/>  canSync: boolean,<br/>  isDifferentFromServer: boolean<br/>}
-
-    UI->>settingsStore: getParameterDiff()
-    settingsStore->>ParamSvc: createParameterDiff(config, serverDefaults, userOverrides)
-    ParamSvc-->>settingsStore: ParameterDiff[]
-    Note right of ParamSvc: Array of parameters where user != server
-
-    %% ═══════════════════════════════════════════════════════════════════════════
-    Note over UI,LS: 📋 CONFIG CATEGORIES
-    %% ═══════════════════════════════════════════════════════════════════════════
-
-    Note over settingsStore: Syncable with server (from /props):
-    rect rgb(240, 255, 240)
-        Note over settingsStore: temperature, top_p, top_k, min_p<br/>repeat_penalty, presence_penalty, frequency_penalty<br/>dynatemp_range, dynatemp_exponent<br/>typ_p, xtc_probability, xtc_threshold<br/>dry_multiplier, dry_base, dry_allowed_length, dry_penalty_last_n
-    end
-
-    Note over settingsStore: UI-only (not synced):
-    rect rgb(255, 240, 240)
-        Note over settingsStore: systemMessage, custom (JSON)<br/>showStatistics, enableContinueGeneration<br/>autoMicOnEmpty, disableAutoScroll<br/>apiKey, pdfAsImage, disableReasoningParsing, showRawOutputSwitch
-    end
-```

tools/ui/eslint.config.js

@@ -12,6 +12,49 @@ import { fileURLToPath } from 'node:url';
 import ts from 'typescript-eslint';
 
 const gitignorePath = fileURLToPath(new URL('./.gitignore', import.meta.url));
+// Require a blank line between consecutive class accessors (get/set). The core
+// `padding-line-between-statements` rule only handles statements, not class
+// members, so this is enforced with a small custom rule.
+const blankLineBetweenAccessors = {
+	create(context) {
+		return {
+			MethodDefinition(node) {
+				if (node.kind !== 'get' && node.kind !== 'set') return;
+
+				const body = node.parent;
+
+				if (!body || body.type !== 'ClassBody') return;
+
+				const index = body.body.indexOf(node);
+
+				if (index <= 0) return;
+
+				const prev = body.body[index - 1];
+
+				if (prev.type !== 'MethodDefinition' || (prev.kind !== 'get' && prev.kind !== 'set'))
+					return;
+
+				if (node.loc.start.line - prev.loc.end.line <= 1) {
+					context.report({
+						fix(fixer) {
+							// Insert after the previous accessor's closing brace so the blank
+							// line keeps the current accessor's indentation.
+							return fixer.insertTextAfter(prev, '\n');
+						},
+						message: 'Expected a blank line between class accessors (get/set).',
+						node
+					});
+				}
+			}
+		};
+	},
+	meta: {
+		docs: { description: 'Require a blank line between consecutive class accessors (get/set).' },
+		fixable: 'whitespace',
+		schema: [],
+		type: 'layout'
+	}
+};
 
 export default ts.config(
 	includeIgnoreFile(gitignorePath),
@@ -22,7 +65,11 @@ export default ts.config(
 	...svelte.configs.prettier,
 	{
 		languageOptions: { globals: { ...globals.browser, ...globals.node } },
-		plugins: { perfectionist, 'simple-import-sort': simpleImportSort },
+		plugins: {
+			local: { rules: { 'blank-line-between-accessors': blankLineBetweenAccessors } },
+			perfectionist,
+			'simple-import-sort': simpleImportSort
+		},
 		rules: {
 			// Snippet bodies often ignore one or more of the parent's params
 			// (e.g. `{#snippet children(_meta, ctx)}` when only ctx is read).
@@ -30,8 +77,11 @@ export default ts.config(
 				'error',
 				{ argsIgnorePattern: '^_', varsIgnorePattern: '^_' }
 			],
+
 			// Enforce empty line at end of file
 			'eol-last': 'error',
+			// Enforce a blank line between consecutive get/set accessors
+			'local/blank-line-between-accessors': 'error',
 			// typescript-eslint strongly recommend that you do not use the no-undef lint rule on TypeScript projects.
 			// see: https://typescript-eslint.io/troubleshooting/faqs/eslint/#i-get-errors-from-the-no-undef-rule-about-global-variables-not-being-defined-even-though-there-are-no-typescript-errors
 			'no-undef': 'off',
@@ -61,6 +111,38 @@ export default ts.config(
 				{ blankLine: 'always', next: ['return', 'throw', 'break', 'continue'], prev: '*' }
 			],
 
+			// Class member order: public fields -> private fields -> constructor -> getters
+			// -> setters -> public methods -> private methods, alphabetical within each.
+			// Svelte $derived fields must stay in dependency order (forward references are
+			// rejected), so the two stores that rely on that are exempted below.
+			'perfectionist/sort-classes': [
+				'error',
+				{
+					customGroups: [
+						{ groupName: 'public-field', modifiers: ['public'], selector: 'property' },
+						{ groupName: 'private-field', modifiers: ['private'], selector: 'property' },
+						{ groupName: 'get-method', selector: 'get-method' },
+						{ groupName: 'set-method', selector: 'set-method' },
+						{ groupName: 'public-method', modifiers: ['public'], selector: 'method' },
+						{ groupName: 'private-method', modifiers: ['private'], selector: 'method' }
+					],
+					groups: [
+						'public-field',
+						'private-field',
+						'constructor',
+						'get-method',
+						'set-method',
+						'public-method',
+						'private-method',
+						'unknown'
+					],
+					type: 'natural',
+					// Keep members in dependency order (Svelte rejects forward references in
+					// $derived fields), while still sorting the rest alphabetically.
+					useExperimentalDependencyDetection: true
+				}
+			],
+
 			// Alphabetical order for enum members
 			'perfectionist/sort-enums': ['error', { type: 'natural' }],
 

tools/ui/src/lib/components/app/chat/ChatAttachments/ChatAttachmentsPreview/ChatAttachmentsPreview.svelte

@@ -139,7 +139,7 @@
 	let fileSize = $derived(currentItem?.size ? formatFileSize(currentItem.size) : '');
 
 	let hasVisionModality = $derived(
-		currentItem && activeModelId ? modelsStore.modelSupportsVision(activeModelId) : false
+		currentItem && activeModelId ? modelsStore.props.modelSupportsVision(activeModelId) : false
 	);
 
 	let audioSrc = $derived(

tools/ui/src/lib/components/app/chat/ChatForm/ChatForm.svelte

@@ -28,7 +28,6 @@
 	import {
 		chatStore,
 		conversationsStore,
-		mcpResourceStore,
 		mcpStore,
 		modelsStore,
 		serverStore,
@@ -140,7 +139,9 @@
 	// float above the box.
 	let mentionAnchor: HTMLDivElement | null = $state(null);
 
-	let cwd = $derived(conversationsStore.activeConversation?.cwd ?? conversationsStore.pendingCwd);
+	let cwd = $derived(
+		conversationsStore.activeConversation?.cwd ?? conversationsStore.preferences.pendingCwd
+	);
 
 	const pickers = useChatFormPickers({
 		focusInput: refocusInput,
@@ -151,7 +152,8 @@
 		getShowModelSelector: () => showModelSelector,
 		getValue: () => value,
 		hasCwdTools: () => toolsStore.hasEnabledCwdTools,
-		hasPrompts: () => mcpStore.hasPromptsCapability(conversationsStore.getAllMcpServerOverrides()),
+		hasPrompts: () =>
+			mcpStore.hasPromptsCapability(conversationsStore.preferences.getAllMcpServerOverrides()),
 		openModelSelector: () => chatFormActionsRef?.openModelSelector(),
 		setCaretOffset: (offset) => inputRef?.setCaretOffset(offset),
 		setValue: (v) => {
@@ -170,7 +172,7 @@
 			onValueChange?.('');
 		}
 
-		await conversationsStore.setCwd(newDir);
+		await conversationsStore.preferences.setCwd(newDir);
 
 		if (conversationsStore.activeConversation) {
 			await chatStore.recordCwdChange(newDir?.trim() || null);
@@ -595,7 +597,7 @@
 				{useRichInput}
 			/>
 
-			{#if mcpResourceStore.hasAttachments}
+			{#if mcpStore.resources.hasAttachments}
 				<ChatFormMcpResourcesList
 					class="mb-3"
 					onResourceClick={(uri) => {

tools/ui/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionAdd/ChatFormActionAddMcpServersSubmenu.svelte

@@ -38,11 +38,11 @@
 	}
 
 	function isServerEnabledForChat(serverId: string): boolean {
-		return conversationsStore.isMcpServerEnabledForChat(serverId);
+		return conversationsStore.preferences.isMcpServerEnabledForChat(serverId);
 	}
 
 	async function toggleServerForChat(serverId: string) {
-		await conversationsStore.toggleMcpServerForChat(serverId);
+		await conversationsStore.preferences.toggleMcpServerForChat(serverId);
 	}
 
 	function handleMcpSubMenuOpen(open: boolean) {